libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2019 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "oacc-int.h"
  31 #include "openacc.h"
  32 #include <assert.h>
  33 #include <stdlib.h>
  34 #include <strings.h>
  35 #include <stdbool.h>
  36 #include <string.h>
  37
  38 /* This lock is used to protect access to cached_base_dev, dispatchers and
  39    the (abstract) initialisation state of attached offloading devices.  */
  40
  41 static gomp_mutex_t acc_device_lock;
  42
  43 /* A cached version of the dispatcher for the global "current" accelerator type,
  44    e.g. used as the default when creating new host threads.  This is the
  45    device-type equivalent of goacc_device_num (which specifies which device to
  46    use out of potentially several of the same type).  If there are several
  47    devices of a given type, this points at the first one.  */
  48
  49 static struct gomp_device_descr *cached_base_dev = NULL;
  50
  51 #if defined HAVE_TLS || defined USE_EMUTLS
  52 __thread struct goacc_thread *goacc_tls_data;
  53 #else
  54 pthread_key_t goacc_tls_key;
  55 #endif
  56 static pthread_key_t goacc_cleanup_key;
  57
  58 static struct goacc_thread *goacc_threads;
  59 static gomp_mutex_t goacc_thread_lock;
  60
  61 /* An array of dispatchers for device types, indexed by the type.  This array
  62    only references "base" devices, and other instances of the same type are
  63    found by simply indexing from each such device (which are stored linearly,
  64    grouped by device in target.c:devices).  */
  65 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  66
  67 attribute_hidden void
  68 goacc_register (struct gomp_device_descr *disp)
  69 {
  70   /* Only register the 0th device here.  */
  71   if (disp->target_id != 0)
  72     return;
  73
  74   gomp_mutex_lock (&acc_device_lock);
  75
  76   assert (acc_device_type (disp->type) != acc_device_none
  77           && acc_device_type (disp->type) != acc_device_default
  78           && acc_device_type (disp->type) != acc_device_not_host);
  79   assert (!dispatchers[disp->type]);
  80   dispatchers[disp->type] = disp;
  81
  82   gomp_mutex_unlock (&acc_device_lock);
  83 }
  84
  85 static bool
  86 known_device_type_p (acc_device_t d)
  87 {
  88   return d >= 0 && d < _ACC_device_hwm;
  89 }
  90
  91 static void
  92 unknown_device_type_error (acc_device_t invalid_type)
  93 {
  94   gomp_fatal ("unknown device type %u", invalid_type);
  95 }
  96
  97 /* OpenACC names some things a little differently.  */
  98
  99 static const char *
 100 get_openacc_name (const char *name)
 101 {
 102   if (strcmp (name, "nvptx") == 0)
 103     return "nvidia";
 104   else
 105     return name;
 106 }
 107
 108 static const char *
 109 name_of_acc_device_t (enum acc_device_t type)
 110 {
 111   switch (type)
 112     {
 113     case acc_device_none: return "none";
 114     case acc_device_default: return "default";
 115     case acc_device_host: return "host";
 116     case acc_device_not_host: return "not_host";
 117     case acc_device_nvidia: return "nvidia";
 118     default: unknown_device_type_error (type);
 119     }
 120   __builtin_unreachable ();
 121 }
 122
 123 /* ACC_DEVICE_LOCK must be held before calling this function.  If FAIL_IS_ERROR
 124    is true, this function raises an error if there are no devices of type D,
 125    otherwise it returns NULL in that case.  */
 126
 127 static struct gomp_device_descr *
 128 resolve_device (acc_device_t d, bool fail_is_error)
 129 {
 130   acc_device_t d_arg = d;
 131
 132   switch (d)
 133     {
 134     case acc_device_default:
 135       {
 136         if (goacc_device_type)
 137           {
 138             /* Lookup the named device.  */
 139             while (known_device_type_p (++d))
 140               if (dispatchers[d]
 141                   && !strcasecmp (goacc_device_type,
 142                                   get_openacc_name (dispatchers[d]->name))
 143                   && dispatchers[d]->get_num_devices_func () > 0)
 144                 goto found;
 145
 146             if (fail_is_error)
 147               {
 148                 gomp_mutex_unlock (&acc_device_lock);
 149                 gomp_fatal ("device type %s not supported", goacc_device_type);
 150               }
 151             else
 152               return NULL;
 153           }
 154
 155         /* No default device specified, so start scanning for any non-host
 156            device that is available.  */
 157         d = acc_device_not_host;
 158       }
 159       /* FALLTHROUGH */
 160
 161     case acc_device_not_host:
 162       /* Find the first available device after acc_device_not_host.  */
 163       while (known_device_type_p (++d))
 164         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 165           goto found;
 166       if (d_arg == acc_device_default)
 167         {
 168           d = acc_device_host;
 169           goto found;
 170         }
 171       if (fail_is_error)
 172         {
 173           gomp_mutex_unlock (&acc_device_lock);
 174           gomp_fatal ("no device found");
 175         }
 176       else
 177         return NULL;
 178       break;
 179
 180     case acc_device_host:
 181       break;
 182
 183     default:
 184       if (!known_device_type_p (d))
 185         {
 186           if (fail_is_error)
 187             goto unsupported_device;
 188           else
 189             return NULL;
 190         }
 191       break;
 192     }
 193  found:
 194
 195   assert (d != acc_device_none
 196           && d != acc_device_default
 197           && d != acc_device_not_host);
 198
 199   if (dispatchers[d] == NULL && fail_is_error)
 200     {
 201     unsupported_device:
 202       gomp_mutex_unlock (&acc_device_lock);
 203       gomp_fatal ("device type %s not supported", name_of_acc_device_t (d));
 204     }
 205
 206   return dispatchers[d];
 207 }
 208
 209 /* Emit a suitable error if no device of a particular type is available, or
 210    the given device number is out-of-range.  */
 211 static void
 212 acc_dev_num_out_of_range (acc_device_t d, int ord, int ndevs)
 213 {
 214   if (ndevs == 0)
 215     gomp_fatal ("no devices of type %s available", name_of_acc_device_t (d));
 216   else
 217     gomp_fatal ("device %u out of range", ord);
 218 }
 219
 220 /* This is called when plugins have been initialized, and serves to call
 221    (indirectly) the target's device_init hook.  Calling multiple times without
 222    an intervening acc_shutdown_1 call is an error.  ACC_DEVICE_LOCK must be
 223    held before calling this function.  */
 224
 225 static struct gomp_device_descr *
 226 acc_init_1 (acc_device_t d, acc_construct_t parent_construct, int implicit)
 227 {
 228   bool check_not_nested_p;
 229   if (implicit)
 230     {
 231       /* In the implicit case, there should (TODO: must?) already be something
 232          have been set up for an outer construct.  */
 233       check_not_nested_p = false;
 234     }
 235   else
 236     {
 237       check_not_nested_p = true;
 238       /* TODO: should we set 'thr->prof_info' etc. in this case ('acc_init')?
 239          The problem is, that we don't have 'thr' yet?  (So,
 240          'check_not_nested_p = true' also is pointless actually.)  */
 241     }
 242   bool profiling_p = GOACC_PROFILING_DISPATCH_P (check_not_nested_p);
 243
 244   acc_prof_info prof_info;
 245   if (profiling_p)
 246     {
 247       prof_info.event_type = acc_ev_device_init_start;
 248       prof_info.valid_bytes = _ACC_PROF_INFO_VALID_BYTES;
 249       prof_info.version = _ACC_PROF_INFO_VERSION;
 250       prof_info.device_type = d;
 251       prof_info.device_number = goacc_device_num;
 252       prof_info.thread_id = -1;
 253       prof_info.async = acc_async_sync;
 254       prof_info.async_queue = prof_info.async;
 255       prof_info.src_file = NULL;
 256       prof_info.func_name = NULL;
 257       prof_info.line_no = -1;
 258       prof_info.end_line_no = -1;
 259       prof_info.func_line_no = -1;
 260       prof_info.func_end_line_no = -1;
 261     }
 262   acc_event_info device_init_event_info;
 263   if (profiling_p)
 264     {
 265       device_init_event_info.other_event.event_type = prof_info.event_type;
 266       device_init_event_info.other_event.valid_bytes
 267         = _ACC_OTHER_EVENT_INFO_VALID_BYTES;
 268       device_init_event_info.other_event.parent_construct = parent_construct;
 269       device_init_event_info.other_event.implicit = implicit;
 270       device_init_event_info.other_event.tool_info = NULL;
 271     }
 272   acc_api_info api_info;
 273   if (profiling_p)
 274     {
 275       api_info.device_api = acc_device_api_none;
 276       api_info.valid_bytes = _ACC_API_INFO_VALID_BYTES;
 277       api_info.device_type = prof_info.device_type;
 278       api_info.vendor = -1;
 279       api_info.device_handle = NULL;
 280       api_info.context_handle = NULL;
 281       api_info.async_handle = NULL;
 282     }
 283
 284   if (profiling_p)
 285     goacc_profiling_dispatch (&prof_info, &device_init_event_info, &api_info);
 286
 287   struct gomp_device_descr *base_dev, *acc_dev;
 288   int ndevs;
 289
 290   base_dev = resolve_device (d, true);
 291
 292   ndevs = base_dev->get_num_devices_func ();
 293
 294   if (ndevs <= 0 || goacc_device_num >= ndevs)
 295     acc_dev_num_out_of_range (d, goacc_device_num, ndevs);
 296
 297   acc_dev = &base_dev[goacc_device_num];
 298
 299   gomp_mutex_lock (&acc_dev->lock);
 300   if (acc_dev->state == GOMP_DEVICE_INITIALIZED)
 301     {
 302       gomp_mutex_unlock (&acc_dev->lock);
 303       gomp_fatal ("device already active");
 304     }
 305
 306   gomp_init_device (acc_dev);
 307   gomp_mutex_unlock (&acc_dev->lock);
 308
 309   if (profiling_p)
 310     {
 311       prof_info.event_type = acc_ev_device_init_end;
 312       device_init_event_info.other_event.event_type = prof_info.event_type;
 313       goacc_profiling_dispatch (&prof_info, &device_init_event_info,
 314                                 &api_info);
 315     }
 316
 317   return base_dev;
 318 }
 319
 320 /* ACC_DEVICE_LOCK must be held before calling this function.  */
 321
 322 static void
 323 acc_shutdown_1 (acc_device_t d)
 324 {
 325   struct gomp_device_descr *base_dev;
 326   struct goacc_thread *walk;
 327   int ndevs, i;
 328   bool devices_active = false;
 329
 330   /* Get the base device for this device type.  */
 331   base_dev = resolve_device (d, true);
 332
 333   ndevs = base_dev->get_num_devices_func ();
 334
 335   /* Unload all the devices of this type that have been opened.  */
 336   for (i = 0; i < ndevs; i++)
 337     {
 338       struct gomp_device_descr *acc_dev = &base_dev[i];
 339
 340       gomp_mutex_lock (&acc_dev->lock);
 341       gomp_unload_device (acc_dev);
 342       gomp_mutex_unlock (&acc_dev->lock);
 343     }
 344
 345   gomp_mutex_lock (&goacc_thread_lock);
 346
 347   /* Free target-specific TLS data and close all devices.  */
 348   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 349     {
 350       if (walk->target_tls)
 351         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 352
 353       walk->target_tls = NULL;
 354
 355       /* This would mean the user is shutting down OpenACC in the middle of an
 356          "acc data" pragma.  Likely not intentional.  */
 357       if (walk->mapped_data)
 358         {
 359           gomp_mutex_unlock (&goacc_thread_lock);
 360           gomp_fatal ("shutdown in 'acc data' region");
 361         }
 362
 363       /* Similarly, if this happens then user code has done something weird.  */
 364       if (walk->saved_bound_dev)
 365         {
 366           gomp_mutex_unlock (&goacc_thread_lock);
 367           gomp_fatal ("shutdown during host fallback");
 368         }
 369
 370       if (walk->dev)
 371         {
 372           gomp_mutex_lock (&walk->dev->lock);
 373
 374           while (walk->dev->mem_map.root)
 375             {
 376               splay_tree_key k = &walk->dev->mem_map.root->key;
 377               if (k->aux)
 378                 k->aux->link_key = NULL;
 379               gomp_remove_var (walk->dev, k);
 380             }
 381
 382           gomp_mutex_unlock (&walk->dev->lock);
 383
 384           walk->dev = NULL;
 385           walk->base_dev = NULL;
 386         }
 387     }
 388
 389   gomp_mutex_unlock (&goacc_thread_lock);
 390
 391   /* Close all the devices of this type that have been opened.  */
 392   bool ret = true;
 393   for (i = 0; i < ndevs; i++)
 394     {
 395       struct gomp_device_descr *acc_dev = &base_dev[i];
 396       gomp_mutex_lock (&acc_dev->lock);
 397       if (acc_dev->state == GOMP_DEVICE_INITIALIZED)
 398         {
 399           devices_active = true;
 400           ret &= gomp_fini_device (acc_dev);
 401           acc_dev->state = GOMP_DEVICE_UNINITIALIZED;
 402         }
 403       gomp_mutex_unlock (&acc_dev->lock);
 404     }
 405
 406   if (!ret)
 407     gomp_fatal ("device finalization failed");
 408
 409   if (!devices_active)
 410     gomp_fatal ("no device initialized");
 411 }
 412
 413 static struct goacc_thread *
 414 goacc_new_thread (void)
 415 {
 416   struct goacc_thread *thr = gomp_malloc (sizeof (struct goacc_thread));
 417
 418 #if defined HAVE_TLS || defined USE_EMUTLS
 419   goacc_tls_data = thr;
 420 #else
 421   pthread_setspecific (goacc_tls_key, thr);
 422 #endif
 423
 424   pthread_setspecific (goacc_cleanup_key, thr);
 425
 426   gomp_mutex_lock (&goacc_thread_lock);
 427   thr->next = goacc_threads;
 428   goacc_threads = thr;
 429   gomp_mutex_unlock (&goacc_thread_lock);
 430
 431   return thr;
 432 }
 433
 434 static void
 435 goacc_destroy_thread (void *data)
 436 {
 437   struct goacc_thread *thr = data, *walk, *prev;
 438
 439   gomp_mutex_lock (&goacc_thread_lock);
 440
 441   if (thr)
 442     {
 443       struct gomp_device_descr *acc_dev = thr->dev;
 444
 445       if (acc_dev && thr->target_tls)
 446         {
 447           acc_dev->openacc.destroy_thread_data_func (thr->target_tls);
 448           thr->target_tls = NULL;
 449         }
 450
 451       assert (!thr->mapped_data);
 452
 453       /* Remove from thread list.  */
 454       for (prev = NULL, walk = goacc_threads; walk;
 455            prev = walk, walk = walk->next)
 456         if (walk == thr)
 457           {
 458             if (prev == NULL)
 459               goacc_threads = walk->next;
 460             else
 461               prev->next = walk->next;
 462
 463             free (thr);
 464
 465             break;
 466           }
 467
 468       assert (walk);
 469     }
 470
 471   gomp_mutex_unlock (&goacc_thread_lock);
 472 }
 473
 474 /* Use the ORD'th device instance for the current host thread (or -1 for the
 475    current global default).  The device (and the runtime) must be initialised
 476    before calling this function.  */
 477
 478 void
 479 goacc_attach_host_thread_to_device (int ord)
 480 {
 481   struct goacc_thread *thr = goacc_thread ();
 482   struct gomp_device_descr *acc_dev = NULL, *base_dev = NULL;
 483   int num_devices;
 484
 485   if (thr && thr->dev && (thr->dev->target_id == ord || ord < 0))
 486     return;
 487
 488   if (ord < 0)
 489     ord = goacc_device_num;
 490
 491   /* Decide which type of device to use.  If the current thread has a device
 492      type already (e.g. set by acc_set_device_type), use that, else use the
 493      global default.  */
 494   if (thr && thr->base_dev)
 495     base_dev = thr->base_dev;
 496   else
 497     {
 498       assert (cached_base_dev);
 499       base_dev = cached_base_dev;
 500     }
 501
 502   num_devices = base_dev->get_num_devices_func ();
 503   if (num_devices <= 0 || ord >= num_devices)
 504     acc_dev_num_out_of_range (acc_device_type (base_dev->type), ord,
 505                               num_devices);
 506
 507   if (!thr)
 508     thr = goacc_new_thread ();
 509
 510   thr->base_dev = base_dev;
 511   thr->dev = acc_dev = &base_dev[ord];
 512   thr->saved_bound_dev = NULL;
 513   thr->mapped_data = NULL;
 514   thr->prof_info = NULL;
 515   thr->api_info = NULL;
 516   /* Initially, all callbacks for all events are enabled.  */
 517   thr->prof_callbacks_enabled = true;
 518
 519   thr->target_tls
 520     = acc_dev->openacc.create_thread_data_func (ord);
 521 }
 522
 523 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 524    init/shutdown is per-process or per-thread.  We choose per-process.  */
 525
 526 void
 527 acc_init (acc_device_t d)
 528 {
 529   if (!known_device_type_p (d))
 530     unknown_device_type_error (d);
 531
 532   gomp_init_targets_once ();
 533
 534   gomp_mutex_lock (&acc_device_lock);
 535   cached_base_dev = acc_init_1 (d, acc_construct_runtime_api, 0);
 536   gomp_mutex_unlock (&acc_device_lock);
 537
 538   goacc_attach_host_thread_to_device (-1);
 539 }
 540
 541 ialias (acc_init)
 542
 543 void
 544 acc_shutdown (acc_device_t d)
 545 {
 546   if (!known_device_type_p (d))
 547     unknown_device_type_error (d);
 548
 549   gomp_init_targets_once ();
 550
 551   gomp_mutex_lock (&acc_device_lock);
 552
 553   acc_shutdown_1 (d);
 554
 555   gomp_mutex_unlock (&acc_device_lock);
 556 }
 557
 558 ialias (acc_shutdown)
 559
 560 int
 561 acc_get_num_devices (acc_device_t d)
 562 {
 563   if (!known_device_type_p (d))
 564     unknown_device_type_error (d);
 565
 566   int n = 0;
 567   struct gomp_device_descr *acc_dev;
 568
 569   if (d == acc_device_none)
 570     return 0;
 571
 572   gomp_init_targets_once ();
 573
 574   gomp_mutex_lock (&acc_device_lock);
 575   acc_dev = resolve_device (d, false);
 576   gomp_mutex_unlock (&acc_device_lock);
 577
 578   if (!acc_dev)
 579     return 0;
 580
 581   n = acc_dev->get_num_devices_func ();
 582   if (n < 0)
 583     n = 0;
 584
 585   return n;
 586 }
 587
 588 ialias (acc_get_num_devices)
 589
 590 /* Set the device type for the current thread only (using the current global
 591    default device number), initialising that device if necessary.  Also set the
 592    default device type for new threads to D.  */
 593
 594 void
 595 acc_set_device_type (acc_device_t d)
 596 {
 597   if (!known_device_type_p (d))
 598     unknown_device_type_error (d);
 599
 600   struct gomp_device_descr *base_dev, *acc_dev;
 601   struct goacc_thread *thr = goacc_thread ();
 602
 603   acc_prof_info prof_info;
 604   acc_api_info api_info;
 605   bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);
 606   if (profiling_p)
 607     prof_info.device_type = d;
 608
 609   gomp_init_targets_once ();
 610
 611   gomp_mutex_lock (&acc_device_lock);
 612
 613   cached_base_dev = base_dev = resolve_device (d, true);
 614   acc_dev = &base_dev[goacc_device_num];
 615
 616   gomp_mutex_lock (&acc_dev->lock);
 617   if (acc_dev->state == GOMP_DEVICE_UNINITIALIZED)
 618     gomp_init_device (acc_dev);
 619   gomp_mutex_unlock (&acc_dev->lock);
 620
 621   gomp_mutex_unlock (&acc_device_lock);
 622
 623   /* We're changing device type: invalidate the current thread's dev and
 624      base_dev pointers.  */
 625   if (thr && thr->base_dev != base_dev)
 626     {
 627       thr->base_dev = thr->dev = NULL;
 628       if (thr->mapped_data)
 629         gomp_fatal ("acc_set_device_type in 'acc data' region");
 630     }
 631
 632   goacc_attach_host_thread_to_device (-1);
 633
 634   if (profiling_p)
 635     {
 636       thr->prof_info = NULL;
 637       thr->api_info = NULL;
 638     }
 639 }
 640
 641 ialias (acc_set_device_type)
 642
 643 acc_device_t
 644 acc_get_device_type (void)
 645 {
 646   acc_device_t res = acc_device_none;
 647   struct gomp_device_descr *dev;
 648   struct goacc_thread *thr = goacc_thread ();
 649
 650   if (thr && thr->base_dev)
 651     res = acc_device_type (thr->base_dev->type);
 652   else
 653     {
 654       acc_prof_info prof_info;
 655       acc_api_info api_info;
 656       bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);
 657
 658       gomp_init_targets_once ();
 659
 660       gomp_mutex_lock (&acc_device_lock);
 661       dev = resolve_device (acc_device_default, true);
 662       gomp_mutex_unlock (&acc_device_lock);
 663       res = acc_device_type (dev->type);
 664
 665       if (profiling_p)
 666         {
 667           thr->prof_info = NULL;
 668           thr->api_info = NULL;
 669         }
 670     }
 671
 672   assert (res != acc_device_default
 673           && res != acc_device_not_host);
 674
 675   return res;
 676 }
 677
 678 ialias (acc_get_device_type)
 679
 680 int
 681 acc_get_device_num (acc_device_t d)
 682 {
 683   if (!known_device_type_p (d))
 684     unknown_device_type_error (d);
 685
 686   const struct gomp_device_descr *dev;
 687   struct goacc_thread *thr = goacc_thread ();
 688
 689   acc_prof_info prof_info;
 690   acc_api_info api_info;
 691   bool profiling_p = GOACC_PROFILING_SETUP_P (thr, &prof_info, &api_info);
 692   if (profiling_p)
 693     prof_info.device_type = d;
 694
 695   gomp_init_targets_once ();
 696
 697   gomp_mutex_lock (&acc_device_lock);
 698   dev = resolve_device (d, true);
 699   gomp_mutex_unlock (&acc_device_lock);
 700
 701   if (profiling_p)
 702     {
 703       thr->prof_info = NULL;
 704       thr->api_info = NULL;
 705     }
 706
 707   if (thr && thr->base_dev == dev && thr->dev)
 708     return thr->dev->target_id;
 709
 710   return goacc_device_num;
 711 }
 712
 713 ialias (acc_get_device_num)
 714
 715 void
 716 acc_set_device_num (int ord, acc_device_t d)
 717 {
 718   if (!known_device_type_p (d))
 719     unknown_device_type_error (d);
 720
 721   struct gomp_device_descr *base_dev, *acc_dev;
 722   int num_devices;
 723
 724   gomp_init_targets_once ();
 725
 726   if (ord < 0)
 727     ord = goacc_device_num;
 728
 729   if ((int) d == 0)
 730     /* Set whatever device is being used by the current host thread to use
 731        device instance ORD.  It's unclear if this is supposed to affect other
 732        host threads too (OpenACC 2.0 (3.2.4) acc_set_device_num).  */
 733     goacc_attach_host_thread_to_device (ord);
 734   else
 735     {
 736       gomp_mutex_lock (&acc_device_lock);
 737
 738       cached_base_dev = base_dev = resolve_device (d, true);
 739
 740       num_devices = base_dev->get_num_devices_func ();
 741
 742       if (num_devices <= 0 || ord >= num_devices)
 743         acc_dev_num_out_of_range (d, ord, num_devices);
 744
 745       acc_dev = &base_dev[ord];
 746
 747       gomp_mutex_lock (&acc_dev->lock);
 748       if (acc_dev->state == GOMP_DEVICE_UNINITIALIZED)
 749         gomp_init_device (acc_dev);
 750       gomp_mutex_unlock (&acc_dev->lock);
 751
 752       gomp_mutex_unlock (&acc_device_lock);
 753
 754       goacc_attach_host_thread_to_device (ord);
 755     }
 756
 757   goacc_device_num = ord;
 758 }
 759
 760 ialias (acc_set_device_num)
 761
 762 /* For -O and higher, the compiler always attempts to expand acc_on_device, but
 763    if the user disables the builtin, or calls it via a pointer, we'll need this
 764    version.
 765
 766    Compile this with optimization, so that the compiler expands
 767    this, rather than generating infinitely recursive code.
 768
 769    The function just forwards its argument to __builtin_acc_on_device.  It does
 770    not verify that the argument is a valid acc_device_t enumeration value.  */
 771
 772 int __attribute__ ((__optimize__ ("O2")))
 773 acc_on_device (acc_device_t dev)
 774 {
 775   return __builtin_acc_on_device (dev);
 776 }
 777
 778 ialias (acc_on_device)
 779
 780 attribute_hidden void
 781 goacc_runtime_initialize (void)
 782 {
 783   gomp_mutex_init (&acc_device_lock);
 784
 785 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 786   pthread_key_create (&goacc_tls_key, NULL);
 787 #endif
 788
 789   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 790
 791   cached_base_dev = NULL;
 792
 793   goacc_threads = NULL;
 794   gomp_mutex_init (&goacc_thread_lock);
 795
 796   /* Initialize and register the 'host' device type.  */
 797   goacc_host_init ();
 798 }
 799
 800 /* Compiler helper functions */
 801
 802 attribute_hidden void
 803 goacc_save_and_set_bind (acc_device_t d)
 804 {
 805   struct goacc_thread *thr = goacc_thread ();
 806
 807   assert (!thr->saved_bound_dev);
 808
 809   thr->saved_bound_dev = thr->dev;
 810   thr->dev = dispatchers[d];
 811 }
 812
 813 attribute_hidden void
 814 goacc_restore_bind (void)
 815 {
 816   struct goacc_thread *thr = goacc_thread ();
 817
 818   thr->dev = thr->saved_bound_dev;
 819   thr->saved_bound_dev = NULL;
 820 }
 821
 822 /* This is called from any OpenACC support function that may need to implicitly
 823    initialize the libgomp runtime, either globally or from a new host thread.
 824    On exit "goacc_thread" will return a valid & populated thread block.  */
 825
 826 attribute_hidden void
 827 goacc_lazy_initialize (void)
 828 {
 829   struct goacc_thread *thr = goacc_thread ();
 830
 831   if (thr && thr->dev)
 832     return;
 833
 834   gomp_init_targets_once ();
 835
 836   gomp_mutex_lock (&acc_device_lock);
 837   if (!cached_base_dev)
 838     cached_base_dev = acc_init_1 (acc_device_default,
 839                                   acc_construct_parallel, 1);
 840   gomp_mutex_unlock (&acc_device_lock);
 841
 842   goacc_attach_host_thread_to_device (-1);
 843 }