libgomp/libgomp.h

   1 /* Copyright (C) 2005-2021 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains data types and function declarations that are not
  27    part of the official OpenACC or OpenMP user interfaces.  There are
  28    declarations in here that are part of the GNU Offloading and Multi
  29    Processing ABI, in that the compiler is required to know about them
  30    and use them.
  31
  32    The convention is that the all caps prefix "GOMP" is used group items
  33    that are part of the external ABI, and the lower case prefix "gomp"
  34    is used group items that are completely private to the library.  */
  35
  36 #ifndef LIBGOMP_H
  37 #define LIBGOMP_H 1
  38
  39 #ifndef _LIBGOMP_CHECKING_
  40 /* Define to 1 to perform internal sanity checks.  */
  41 #define _LIBGOMP_CHECKING_ 0
  42 #endif
  43
  44 #include "config.h"
  45 #include <stdint.h>
  46 #include "libgomp-plugin.h"
  47 #include "gomp-constants.h"
  48
  49 #ifdef HAVE_PTHREAD_H
  50 #include <pthread.h>
  51 #endif
  52 #include <stdbool.h>
  53 #include <stdlib.h>
  54 #include <stdarg.h>
  55
  56 /* Needed for memset in priority_queue.c.  */
  57 #if _LIBGOMP_CHECKING_
  58 # ifdef STRING_WITH_STRINGS
  59 #  include <string.h>
  60 #  include <strings.h>
  61 # else
  62 #  ifdef HAVE_STRING_H
  63 #   include <string.h>
  64 #  else
  65 #   ifdef HAVE_STRINGS_H
  66 #    include <strings.h>
  67 #   endif
  68 #  endif
  69 # endif
  70 #endif
  71
  72 #ifdef HAVE_ATTRIBUTE_VISIBILITY
  73 # pragma GCC visibility push(hidden)
  74 #endif
  75
  76 /* If we were a C++ library, we'd get this from <std/atomic>.  */
  77 enum memmodel
  78 {
  79   MEMMODEL_RELAXED = 0,
  80   MEMMODEL_CONSUME = 1,
  81   MEMMODEL_ACQUIRE = 2,
  82   MEMMODEL_RELEASE = 3,
  83   MEMMODEL_ACQ_REL = 4,
  84   MEMMODEL_SEQ_CST = 5
  85 };
  86
  87 /* alloc.c */
  88
  89 #if defined(HAVE_ALIGNED_ALLOC) \
  90     || defined(HAVE__ALIGNED_MALLOC) \
  91     || defined(HAVE_POSIX_MEMALIGN) \
  92     || defined(HAVE_MEMALIGN)
  93 /* Defined if gomp_aligned_alloc doesn't use fallback version
  94    and free can be used instead of gomp_aligned_free.  */
  95 #define GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC 1
  96 #endif
  97
  98 #if defined(GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC) && !defined(__AMDGCN__)
  99 #define GOMP_USE_ALIGNED_WORK_SHARES 1
 100 #endif
 101
 102 extern void *gomp_malloc (size_t) __attribute__((malloc));
 103 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc));
 104 extern void *gomp_realloc (void *, size_t);
 105 extern void *gomp_aligned_alloc (size_t, size_t)
 106   __attribute__((malloc, alloc_size (2)));
 107 extern void gomp_aligned_free (void *);
 108
 109 /* Avoid conflicting prototypes of alloca() in system headers by using
 110    GCC's builtin alloca().  */
 111 #define gomp_alloca(x)  __builtin_alloca(x)
 112
 113 /* Optimized allocators for team-specific data that will die with the team.  */
 114
 115 #ifdef __AMDGCN__
 116 /* The arena is initialized in config/gcn/team.c.  */
 117 #define TEAM_ARENA_SIZE  64*1024  /* Must match the value in plugin-gcn.c.  */
 118 #define TEAM_ARENA_START 16  /* LDS offset of free pointer.  */
 119 #define TEAM_ARENA_FREE  24  /* LDS offset of free pointer.  */
 120 #define TEAM_ARENA_END   32  /* LDS offset of end pointer.  */
 121
 122 static inline void * __attribute__((malloc))
 123 team_malloc (size_t size)
 124 {
 125   /* 4-byte align the size.  */
 126   size = (size + 3) & ~3;
 127
 128   /* Allocate directly from the arena.
 129      The compiler does not support DS atomics, yet. */
 130   void *result;
 131   asm ("ds_add_rtn_u64 %0, %1, %2\n\ts_waitcnt 0"
 132        : "=v"(result) : "v"(TEAM_ARENA_FREE), "v"(size), "e"(1L) : "memory");
 133
 134   /* Handle OOM.  */
 135   if (result + size > *(void * __lds *)TEAM_ARENA_END)
 136     {
 137       /* While this is experimental, let's make sure we know when OOM
 138          happens.  */
 139       const char msg[] = "GCN team arena exhausted\n";
 140       write (2, msg, sizeof(msg)-1);
 141
 142       /* Fall back to using the heap (slowly).  */
 143       result = gomp_malloc (size);
 144     }
 145   return result;
 146 }
 147
 148 static inline void * __attribute__((malloc))
 149 team_malloc_cleared (size_t size)
 150 {
 151   char *result = team_malloc (size);
 152
 153   /* Clear the allocated memory.  */
 154   __builtin_memset (result, 0, size);
 155
 156   return result;
 157 }
 158
 159 static inline void
 160 team_free (void *ptr)
 161 {
 162   /* The whole arena is freed when the kernel exits.
 163      However, if we fell back to using heap then we should free it.
 164      It would be better if this function could be a no-op, but at least
 165      LDS loads are cheap.  */
 166   if (ptr < *(void * __lds *)TEAM_ARENA_START
 167       || ptr >= *(void * __lds *)TEAM_ARENA_END)
 168     free (ptr);
 169 }
 170 #else
 171 #define team_malloc(...) gomp_malloc (__VA_ARGS__)
 172 #define team_malloc_cleared(...) gomp_malloc_cleared (__VA_ARGS__)
 173 #define team_free(...) free (__VA_ARGS__)
 174 #endif
 175
 176 /* error.c */
 177
 178 extern void gomp_vdebug (int, const char *, va_list);
 179 extern void gomp_debug (int, const char *, ...)
 180         __attribute__ ((format (printf, 2, 3)));
 181 #define gomp_vdebug(KIND, FMT, VALIST) \
 182   do { \
 183     if (__builtin_expect (gomp_debug_var, 0)) \
 184       (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
 185   } while (0)
 186 #define gomp_debug(KIND, ...) \
 187   do { \
 188     if (__builtin_expect (gomp_debug_var, 0)) \
 189       (gomp_debug) ((KIND), __VA_ARGS__); \
 190   } while (0)
 191 extern void gomp_verror (const char *, va_list);
 192 extern void gomp_error (const char *, ...)
 193         __attribute__ ((format (printf, 1, 2)));
 194 extern void gomp_vfatal (const char *, va_list)
 195         __attribute__ ((noreturn));
 196 extern void gomp_fatal (const char *, ...)
 197         __attribute__ ((noreturn, format (printf, 1, 2)));
 198
 199 struct gomp_task;
 200 struct gomp_taskgroup;
 201 struct htab;
 202
 203 #include "priority_queue.h"
 204 #include "sem.h"
 205 #include "mutex.h"
 206 #include "bar.h"
 207 #include "simple-bar.h"
 208 #include "ptrlock.h"
 209
 210
 211 /* This structure contains the data to control one work-sharing construct,
 212    either a LOOP (FOR/DO) or a SECTIONS.  */
 213
 214 enum gomp_schedule_type
 215 {
 216   GFS_RUNTIME,
 217   GFS_STATIC,
 218   GFS_DYNAMIC,
 219   GFS_GUIDED,
 220   GFS_AUTO,
 221   GFS_MONOTONIC = 0x80000000U
 222 };
 223
 224 struct gomp_doacross_work_share
 225 {
 226   union {
 227     /* chunk_size copy, as ws->chunk_size is multiplied by incr for
 228        GFS_DYNAMIC.  */
 229     long chunk_size;
 230     /* Likewise, but for ull implementation.  */
 231     unsigned long long chunk_size_ull;
 232     /* For schedule(static,0) this is the number
 233        of iterations assigned to the last thread, i.e. number of
 234        iterations / number of threads.  */
 235     long q;
 236     /* Likewise, but for ull implementation.  */
 237     unsigned long long q_ull;
 238   };
 239   /* Size of each array entry (padded to cache line size).  */
 240   unsigned long elt_sz;
 241   /* Number of dimensions in sink vectors.  */
 242   unsigned int ncounts;
 243   /* True if the iterations can be flattened.  */
 244   bool flattened;
 245   /* Actual array (of elt_sz sized units), aligned to cache line size.
 246      This is indexed by team_id for GFS_STATIC and outermost iteration
 247      / chunk_size for other schedules.  */
 248   unsigned char *array;
 249   /* These two are only used for schedule(static,0).  */
 250   /* This one is number of iterations % number of threads.  */
 251   long t;
 252   union {
 253     /* And this one is cached t * (q + 1).  */
 254     long boundary;
 255     /* Likewise, but for the ull implementation.  */
 256     unsigned long long boundary_ull;
 257   };
 258   /* Pointer to extra memory if needed for lastprivate(conditional).  */
 259   void *extra;
 260   /* Array of shift counts for each dimension if they can be flattened.  */
 261   unsigned int shift_counts[];
 262 };
 263
 264 /* Like struct gomp_work_share, but only the 1st cacheline of it plus
 265    flexible array at the end.
 266    Keep in sync with struct gomp_work_share.  */
 267 struct gomp_work_share_1st_cacheline
 268 {
 269   enum gomp_schedule_type sched;
 270   int mode;
 271   union {
 272     struct {
 273       long chunk_size, end, incr;
 274     };
 275     struct {
 276       unsigned long long chunk_size_ull, end_ull, incr_ull;
 277     };
 278   };
 279   union {
 280     unsigned *ordered_team_ids;
 281     struct gomp_doacross_work_share *doacross;
 282   };
 283   unsigned ordered_num_used, ordered_owner, ordered_cur;
 284   struct gomp_work_share *next_alloc;
 285   char pad[];
 286 };
 287
 288 struct gomp_work_share
 289 {
 290   /* This member records the SCHEDULE clause to be used for this construct.
 291      The user specification of "runtime" will already have been resolved.
 292      If this is a SECTIONS construct, this value will always be DYNAMIC.  */
 293   enum gomp_schedule_type sched;
 294
 295   int mode;
 296
 297   union {
 298     struct {
 299       /* This is the chunk_size argument to the SCHEDULE clause.  */
 300       long chunk_size;
 301
 302       /* This is the iteration end point.  If this is a SECTIONS construct,
 303          this is the number of contained sections.  */
 304       long end;
 305
 306       /* This is the iteration step.  If this is a SECTIONS construct, this
 307          is always 1.  */
 308       long incr;
 309     };
 310
 311     struct {
 312       /* The same as above, but for the unsigned long long loop variants.  */
 313       unsigned long long chunk_size_ull;
 314       unsigned long long end_ull;
 315       unsigned long long incr_ull;
 316     };
 317   };
 318
 319   union {
 320     /* This is a circular queue that details which threads will be allowed
 321        into the ordered region and in which order.  When a thread allocates
 322        iterations on which it is going to work, it also registers itself at
 323        the end of the array.  When a thread reaches the ordered region, it
 324        checks to see if it is the one at the head of the queue.  If not, it
 325        blocks on its RELEASE semaphore.  */
 326     unsigned *ordered_team_ids;
 327
 328     /* This is a pointer to DOACROSS work share data.  */
 329     struct gomp_doacross_work_share *doacross;
 330   };
 331
 332   /* This is the number of threads that have registered themselves in
 333      the circular queue ordered_team_ids.  */
 334   unsigned ordered_num_used;
 335
 336   /* This is the team_id of the currently acknowledged owner of the ordered
 337      section, or -1u if the ordered section has not been acknowledged by
 338      any thread.  This is distinguished from the thread that is *allowed*
 339      to take the section next.  */
 340   unsigned ordered_owner;
 341
 342   /* This is the index into the circular queue ordered_team_ids of the
 343      current thread that's allowed into the ordered reason.  */
 344   unsigned ordered_cur;
 345
 346   /* This is a chain of allocated gomp_work_share blocks, valid only
 347      in the first gomp_work_share struct in the block.  */
 348   struct gomp_work_share *next_alloc;
 349
 350   /* The above fields are written once during workshare initialization,
 351      or related to ordered worksharing.  Make sure the following fields
 352      are in a different cache line.  */
 353
 354   /* This lock protects the update of the following members.  */
 355 #ifdef GOMP_USE_ALIGNED_WORK_SHARES
 356   gomp_mutex_t lock __attribute__((aligned (64)));
 357 #else
 358   char pad[64 - offsetof (struct gomp_work_share_1st_cacheline, pad)];
 359   gomp_mutex_t lock;
 360 #endif
 361
 362   /* This is the count of the number of threads that have exited the work
 363      share construct.  If the construct was marked nowait, they have moved on
 364      to other work; otherwise they're blocked on a barrier.  The last member
 365      of the team to exit the work share construct must deallocate it.  */
 366   unsigned threads_completed;
 367
 368   union {
 369     /* This is the next iteration value to be allocated.  In the case of
 370        GFS_STATIC loops, this the iteration start point and never changes.  */
 371     long next;
 372
 373     /* The same, but with unsigned long long type.  */
 374     unsigned long long next_ull;
 375
 376     /* This is the returned data structure for SINGLE COPYPRIVATE.  */
 377     void *copyprivate;
 378   };
 379
 380   union {
 381     /* Link to gomp_work_share struct for next work sharing construct
 382        encountered after this one.  */
 383     gomp_ptrlock_t next_ws;
 384
 385     /* gomp_work_share structs are chained in the free work share cache
 386        through this.  */
 387     struct gomp_work_share *next_free;
 388   };
 389
 390   /* Task reductions for this work-sharing construct.  */
 391   uintptr_t *task_reductions;
 392
 393   /* If only few threads are in the team, ordered_team_ids can point
 394      to this array which fills the padding at the end of this struct.  */
 395   unsigned inline_ordered_team_ids[0];
 396 };
 397
 398 extern char gomp_workshare_struct_check1
 399   [offsetof (struct gomp_work_share_1st_cacheline, next_alloc)
 400    == offsetof (struct gomp_work_share, next_alloc) ? 1 : -1];
 401 extern char gomp_workshare_struct_check2
 402   [offsetof (struct gomp_work_share, lock) == 64 ? 1 : -1];
 403
 404 /* This structure contains all of the thread-local data associated with
 405    a thread team.  This is the data that must be saved when a thread
 406    encounters a nested PARALLEL construct.  */
 407
 408 struct gomp_team_state
 409 {
 410   /* This is the team of which the thread is currently a member.  */
 411   struct gomp_team *team;
 412
 413   /* This is the work share construct which this thread is currently
 414      processing.  Recall that with NOWAIT, not all threads may be
 415      processing the same construct.  */
 416   struct gomp_work_share *work_share;
 417
 418   /* This is the previous work share construct or NULL if there wasn't any.
 419      When all threads are done with the current work sharing construct,
 420      the previous one can be freed.  The current one can't, as its
 421      next_ws field is used.  */
 422   struct gomp_work_share *last_work_share;
 423
 424   /* This is the ID of this thread within the team.  This value is
 425      guaranteed to be between 0 and N-1, where N is the number of
 426      threads in the team.  */
 427   unsigned team_id;
 428
 429   /* Nesting level.  */
 430   unsigned level;
 431
 432   /* Active nesting level.  Only active parallel regions are counted.  */
 433   unsigned active_level;
 434
 435   /* Place-partition-var, offset and length into gomp_places_list array.  */
 436   unsigned place_partition_off;
 437   unsigned place_partition_len;
 438
 439   /* Def-allocator-var ICV.  */
 440   uintptr_t def_allocator;
 441
 442 #ifdef HAVE_SYNC_BUILTINS
 443   /* Number of single stmts encountered.  */
 444   unsigned long single_count;
 445 #endif
 446
 447   /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the
 448      trip number through the loop.  So first time a particular loop
 449      is encountered this number is 0, the second time through the loop
 450      is 1, etc.  This is unused when the compiler knows in advance that
 451      the loop is statically scheduled.  */
 452   unsigned long static_trip;
 453 };
 454
 455 struct target_mem_desc;
 456
 457 /* These are the OpenMP 4.0 Internal Control Variables described in
 458    section 2.3.1.  Those described as having one copy per task are
 459    stored within the structure; those described as having one copy
 460    for the whole program are (naturally) global variables.  */
 461
 462 struct gomp_task_icv
 463 {
 464   unsigned long nthreads_var;
 465   enum gomp_schedule_type run_sched_var;
 466   int run_sched_chunk_size;
 467   int default_device_var;
 468   unsigned int thread_limit_var;
 469   bool dyn_var;
 470   unsigned char max_active_levels_var;
 471   char bind_var;
 472   /* Internal ICV.  */
 473   struct target_mem_desc *target_data;
 474 };
 475
 476 enum gomp_target_offload_t
 477 {
 478   GOMP_TARGET_OFFLOAD_DEFAULT,
 479   GOMP_TARGET_OFFLOAD_MANDATORY,
 480   GOMP_TARGET_OFFLOAD_DISABLED
 481 };
 482
 483 #define gomp_supported_active_levels UCHAR_MAX
 484
 485 extern struct gomp_task_icv gomp_global_icv;
 486 #ifndef HAVE_SYNC_BUILTINS
 487 extern gomp_mutex_t gomp_managed_threads_lock;
 488 #endif
 489 extern bool gomp_cancel_var;
 490 extern enum gomp_target_offload_t gomp_target_offload_var;
 491 extern int gomp_max_task_priority_var;
 492 extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var;
 493 extern unsigned long gomp_available_cpus, gomp_managed_threads;
 494 extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len;
 495 extern char *gomp_bind_var_list;
 496 extern unsigned long gomp_bind_var_list_len;
 497 extern void **gomp_places_list;
 498 extern unsigned long gomp_places_list_len;
 499 extern unsigned int gomp_num_teams_var;
 500 extern int gomp_nteams_var;
 501 extern int gomp_teams_thread_limit_var;
 502 extern int gomp_debug_var;
 503 extern bool gomp_display_affinity_var;
 504 extern char *gomp_affinity_format_var;
 505 extern size_t gomp_affinity_format_len;
 506 extern uintptr_t gomp_def_allocator;
 507 extern int goacc_device_num;
 508 extern char *goacc_device_type;
 509 extern int goacc_default_dims[GOMP_DIM_MAX];
 510
 511 enum gomp_task_kind
 512 {
 513   /* Implicit task.  */
 514   GOMP_TASK_IMPLICIT,
 515   /* Undeferred task.  */
 516   GOMP_TASK_UNDEFERRED,
 517   /* Task created by GOMP_task and waiting to be run.  */
 518   GOMP_TASK_WAITING,
 519   /* Task currently executing or scheduled and about to execute.  */
 520   GOMP_TASK_TIED,
 521   /* Used for target tasks that have vars mapped and async run started,
 522      but not yet completed.  Once that completes, they will be readded
 523      into the queues as GOMP_TASK_WAITING in order to perform the var
 524      unmapping.  */
 525   GOMP_TASK_ASYNC_RUNNING,
 526   /* Task that has finished executing but is waiting for its
 527      completion event to be fulfilled.  */
 528   GOMP_TASK_DETACHED
 529 };
 530
 531 struct gomp_task_depend_entry
 532 {
 533   /* Address of dependency.  */
 534   void *addr;
 535   struct gomp_task_depend_entry *next;
 536   struct gomp_task_depend_entry *prev;
 537   /* Task that provides the dependency in ADDR.  */
 538   struct gomp_task *task;
 539   /* Depend entry is of type "IN".  */
 540   bool is_in;
 541   bool redundant;
 542   bool redundant_out;
 543 };
 544
 545 struct gomp_dependers_vec
 546 {
 547   size_t n_elem;
 548   size_t allocated;
 549   struct gomp_task *elem[];
 550 };
 551
 552 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies.  */
 553
 554 struct gomp_taskwait
 555 {
 556   bool in_taskwait;
 557   bool in_depend_wait;
 558   /* Number of tasks we are waiting for.  */
 559   size_t n_depend;
 560   gomp_sem_t taskwait_sem;
 561 };
 562
 563 /* This structure describes a "task" to be run by a thread.  */
 564
 565 struct gomp_task
 566 {
 567   /* Parent of this task.  */
 568   struct gomp_task *parent;
 569   /* Children of this task.  */
 570   struct priority_queue children_queue;
 571   /* Taskgroup this task belongs in.  */
 572   struct gomp_taskgroup *taskgroup;
 573   /* Tasks that depend on this task.  */
 574   struct gomp_dependers_vec *dependers;
 575   struct htab *depend_hash;
 576   struct gomp_taskwait *taskwait;
 577   /* Number of items in DEPEND.  */
 578   size_t depend_count;
 579   /* Number of tasks this task depends on.  Once this counter reaches
 580      0, we have no unsatisfied dependencies, and this task can be put
 581      into the various queues to be scheduled.  */
 582   size_t num_dependees;
 583
 584   union {
 585       /* Valid only if deferred_p is false.  */
 586       gomp_sem_t *completion_sem;
 587       /* Valid only if deferred_p is true.  Set to the team that executes the
 588          task if the task is detached and the completion event has yet to be
 589          fulfilled.  */
 590       struct gomp_team *detach_team;
 591     };
 592   bool deferred_p;
 593
 594   /* Priority of this task.  */
 595   int priority;
 596   /* The priority node for this task in each of the different queues.
 597      We put this here to avoid allocating space for each priority
 598      node.  Then we play offsetof() games to convert between pnode[]
 599      entries and the gomp_task in which they reside.  */
 600   struct priority_node pnode[3];
 601
 602   struct gomp_task_icv icv;
 603   void (*fn) (void *);
 604   void *fn_data;
 605   enum gomp_task_kind kind;
 606   bool in_tied_task;
 607   bool final_task;
 608   bool copy_ctors_done;
 609   /* Set for undeferred tasks with unsatisfied dependencies which
 610      block further execution of their parent until the dependencies
 611      are satisfied.  */
 612   bool parent_depends_on;
 613   /* Dependencies provided and/or needed for this task.  DEPEND_COUNT
 614      is the number of items available.  */
 615   struct gomp_task_depend_entry depend[];
 616 };
 617
 618 /* This structure describes a single #pragma omp taskgroup.  */
 619
 620 struct gomp_taskgroup
 621 {
 622   struct gomp_taskgroup *prev;
 623   /* Queue of tasks that belong in this taskgroup.  */
 624   struct priority_queue taskgroup_queue;
 625   uintptr_t *reductions;
 626   bool in_taskgroup_wait;
 627   bool cancelled;
 628   bool workshare;
 629   gomp_sem_t taskgroup_sem;
 630   size_t num_children;
 631 };
 632
 633 /* Various state of OpenMP async offloading tasks.  */
 634 enum gomp_target_task_state
 635 {
 636   GOMP_TARGET_TASK_DATA,
 637   GOMP_TARGET_TASK_BEFORE_MAP,
 638   GOMP_TARGET_TASK_FALLBACK,
 639   GOMP_TARGET_TASK_READY_TO_RUN,
 640   GOMP_TARGET_TASK_RUNNING,
 641   GOMP_TARGET_TASK_FINISHED
 642 };
 643
 644 /* This structure describes a target task.  */
 645
 646 struct gomp_target_task
 647 {
 648   struct gomp_device_descr *devicep;
 649   void (*fn) (void *);
 650   size_t mapnum;
 651   size_t *sizes;
 652   unsigned short *kinds;
 653   unsigned int flags;
 654   enum gomp_target_task_state state;
 655   struct target_mem_desc *tgt;
 656   struct gomp_task *task;
 657   struct gomp_team *team;
 658   /* Device-specific target arguments.  */
 659   void **args;
 660   void *hostaddrs[];
 661 };
 662
 663 /* This structure describes a "team" of threads.  These are the threads
 664    that are spawned by a PARALLEL constructs, as well as the work sharing
 665    constructs that the team encounters.  */
 666
 667 struct gomp_team
 668 {
 669   /* This is the number of threads in the current team.  */
 670   unsigned nthreads;
 671
 672   /* This is number of gomp_work_share structs that have been allocated
 673      as a block last time.  */
 674   unsigned work_share_chunk;
 675
 676   /* This is the saved team state that applied to a master thread before
 677      the current thread was created.  */
 678   struct gomp_team_state prev_ts;
 679
 680   /* This semaphore should be used by the master thread instead of its
 681      "native" semaphore in the thread structure.  Required for nested
 682      parallels, as the master is a member of two teams.  */
 683   gomp_sem_t master_release;
 684
 685   /* This points to an array with pointers to the release semaphore
 686      of the threads in the team.  */
 687   gomp_sem_t **ordered_release;
 688
 689   /* List of work shares on which gomp_fini_work_share hasn't been
 690      called yet.  If the team hasn't been cancelled, this should be
 691      equal to each thr->ts.work_share, but otherwise it can be a possibly
 692      long list of workshares.  */
 693   struct gomp_work_share *work_shares_to_free;
 694
 695   /* List of gomp_work_share structs chained through next_free fields.
 696      This is populated and taken off only by the first thread in the
 697      team encountering a new work sharing construct, in a critical
 698      section.  */
 699   struct gomp_work_share *work_share_list_alloc;
 700
 701   /* List of gomp_work_share structs freed by free_work_share.  New
 702      entries are atomically added to the start of the list, and
 703      alloc_work_share can safely only move all but the first entry
 704      to work_share_list alloc, as free_work_share can happen concurrently
 705      with alloc_work_share.  */
 706   struct gomp_work_share *work_share_list_free;
 707
 708 #ifdef HAVE_SYNC_BUILTINS
 709   /* Number of simple single regions encountered by threads in this
 710      team.  */
 711   unsigned long single_count;
 712 #else
 713   /* Mutex protecting addition of workshares to work_share_list_free.  */
 714   gomp_mutex_t work_share_list_free_lock;
 715 #endif
 716
 717   /* This barrier is used for most synchronization of the team.  */
 718   gomp_barrier_t barrier;
 719
 720   /* Initial work shares, to avoid allocating any gomp_work_share
 721      structs in the common case.  */
 722   struct gomp_work_share work_shares[8];
 723
 724   gomp_mutex_t task_lock;
 725   /* Scheduled tasks.  */
 726   struct priority_queue task_queue;
 727   /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team.  */
 728   unsigned int task_count;
 729   /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled.  */
 730   unsigned int task_queued_count;
 731   /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
 732      directly in gomp_barrier_handle_tasks; tasks spawned
 733      from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
 734      that is called from a task run from gomp_barrier_handle_tasks.
 735      task_running_count should be always <= team->nthreads,
 736      and if current task isn't in_tied_task, then it will be
 737      even < team->nthreads.  */
 738   unsigned int task_running_count;
 739   int work_share_cancelled;
 740   int team_cancelled;
 741
 742   /* Number of tasks waiting for their completion event to be fulfilled.  */
 743   unsigned int task_detach_count;
 744
 745   /* This array contains structures for implicit tasks.  */
 746   struct gomp_task implicit_task[];
 747 };
 748
 749 /* This structure contains all data that is private to libgomp and is
 750    allocated per thread.  */
 751
 752 struct gomp_thread
 753 {
 754   /* This is the function that the thread should run upon launch.  */
 755   void (*fn) (void *data);
 756   void *data;
 757
 758   /* This is the current team state for this thread.  The ts.team member
 759      is NULL only if the thread is idle.  */
 760   struct gomp_team_state ts;
 761
 762   /* This is the task that the thread is currently executing.  */
 763   struct gomp_task *task;
 764
 765   /* This semaphore is used for ordered loops.  */
 766   gomp_sem_t release;
 767
 768   /* Place this thread is bound to plus one, or zero if not bound
 769      to any place.  */
 770   unsigned int place;
 771
 772   /* User pthread thread pool */
 773   struct gomp_thread_pool *thread_pool;
 774
 775 #ifdef LIBGOMP_USE_PTHREADS
 776   /* omp_get_num_teams () - 1.  */
 777   unsigned int num_teams;
 778
 779   /* omp_get_team_num ().  */
 780   unsigned int team_num;
 781 #endif
 782
 783 #if defined(LIBGOMP_USE_PTHREADS) \
 784     && (!defined(HAVE_TLS) \
 785         || !defined(__GLIBC__) \
 786         || !defined(USING_INITIAL_EXEC_TLS))
 787   /* pthread_t of the thread containing this gomp_thread.
 788      On Linux when using initial-exec TLS,
 789      (typeof (pthread_t)) gomp_thread () - pthread_self ()
 790      is constant in all threads, so we can optimize and not
 791      store it.  */
 792 #define GOMP_NEEDS_THREAD_HANDLE 1
 793   pthread_t handle;
 794 #endif
 795 };
 796
 797
 798 struct gomp_thread_pool
 799 {
 800   /* This array manages threads spawned from the top level, which will
 801      return to the idle loop once the current PARALLEL construct ends.  */
 802   struct gomp_thread **threads;
 803   unsigned threads_size;
 804   unsigned threads_used;
 805   /* The last team is used for non-nested teams to delay their destruction to
 806      make sure all the threads in the team move on to the pool's barrier before
 807      the team's barrier is destroyed.  */
 808   struct gomp_team *last_team;
 809   /* Number of threads running in this contention group.  */
 810   unsigned long threads_busy;
 811
 812   /* This barrier holds and releases threads waiting in thread pools.  */
 813   gomp_simple_barrier_t threads_dock;
 814 };
 815
 816 enum gomp_cancel_kind
 817 {
 818   GOMP_CANCEL_PARALLEL = 1,
 819   GOMP_CANCEL_LOOP = 2,
 820   GOMP_CANCEL_FOR = GOMP_CANCEL_LOOP,
 821   GOMP_CANCEL_DO = GOMP_CANCEL_LOOP,
 822   GOMP_CANCEL_SECTIONS = 4,
 823   GOMP_CANCEL_TASKGROUP = 8
 824 };
 825
 826 /* ... and here is that TLS data.  */
 827
 828 #if defined __nvptx__
 829 extern struct gomp_thread *nvptx_thrs __attribute__((shared));
 830 static inline struct gomp_thread *gomp_thread (void)
 831 {
 832   int tid;
 833   asm ("mov.u32 %0, %%tid.y;" : "=r" (tid));
 834   return nvptx_thrs + tid;
 835 }
 836 #elif defined __AMDGCN__
 837 static inline struct gomp_thread *gcn_thrs (void)
 838 {
 839   /* The value is at the bottom of LDS.  */
 840   struct gomp_thread * __lds *thrs = (struct gomp_thread * __lds *)4;
 841   return *thrs;
 842 }
 843 static inline void set_gcn_thrs (struct gomp_thread *val)
 844 {
 845   /* The value is at the bottom of LDS.  */
 846   struct gomp_thread * __lds *thrs = (struct gomp_thread * __lds *)4;
 847   *thrs = val;
 848 }
 849 static inline struct gomp_thread *gomp_thread (void)
 850 {
 851   int tid = __builtin_gcn_dim_pos(1);
 852   return gcn_thrs () + tid;
 853 }
 854 #elif defined HAVE_TLS || defined USE_EMUTLS
 855 extern __thread struct gomp_thread gomp_tls_data;
 856 static inline struct gomp_thread *gomp_thread (void)
 857 {
 858   return &gomp_tls_data;
 859 }
 860 #else
 861 extern pthread_key_t gomp_tls_key;
 862 static inline struct gomp_thread *gomp_thread (void)
 863 {
 864   return pthread_getspecific (gomp_tls_key);
 865 }
 866 #endif
 867
 868 extern struct gomp_task_icv *gomp_new_icv (void);
 869
 870 /* Here's how to access the current copy of the ICVs.  */
 871
 872 static inline struct gomp_task_icv *gomp_icv (bool write)
 873 {
 874   struct gomp_task *task = gomp_thread ()->task;
 875   if (task)
 876     return &task->icv;
 877   else if (write)
 878     return gomp_new_icv ();
 879   else
 880     return &gomp_global_icv;
 881 }
 882
 883 #ifdef LIBGOMP_USE_PTHREADS
 884 /* The attributes to be used during thread creation.  */
 885 extern pthread_attr_t gomp_thread_attr;
 886
 887 extern pthread_key_t gomp_thread_destructor;
 888 #endif
 889
 890 /* Function prototypes.  */
 891
 892 /* affinity.c */
 893
 894 extern void gomp_init_affinity (void);
 895 #ifdef LIBGOMP_USE_PTHREADS
 896 extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int);
 897 #endif
 898 extern void **gomp_affinity_alloc (unsigned long, bool);
 899 extern void gomp_affinity_init_place (void *);
 900 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
 901                                     long, bool);
 902 extern bool gomp_affinity_remove_cpu (void *, unsigned long);
 903 extern bool gomp_affinity_copy_place (void *, void *, long);
 904 extern bool gomp_affinity_same_place (void *, void *);
 905 extern bool gomp_affinity_finalize_place_list (bool);
 906 extern bool gomp_affinity_init_level (int, unsigned long, bool);
 907 extern void gomp_affinity_print_place (void *);
 908 extern void gomp_get_place_proc_ids_8 (int, int64_t *);
 909 extern void gomp_display_affinity_place (char *, size_t, size_t *, int);
 910
 911 /* affinity-fmt.c */
 912
 913 extern bool gomp_print_string (const char *str, size_t len);
 914 extern void gomp_set_affinity_format (const char *, size_t);
 915 extern void gomp_display_string (char *, size_t, size_t *, const char *,
 916                                  size_t);
 917 #ifdef LIBGOMP_USE_PTHREADS
 918 typedef pthread_t gomp_thread_handle;
 919 #else
 920 typedef struct {} gomp_thread_handle;
 921 #endif
 922 extern size_t gomp_display_affinity (char *, size_t, const char *,
 923                                      gomp_thread_handle,
 924                                      struct gomp_team_state *, unsigned int);
 925 extern void gomp_display_affinity_thread (gomp_thread_handle,
 926                                           struct gomp_team_state *,
 927                                           unsigned int) __attribute__((cold));
 928
 929 /* iter.c */
 930
 931 extern int gomp_iter_static_next (long *, long *);
 932 extern bool gomp_iter_dynamic_next_locked (long *, long *);
 933 extern bool gomp_iter_guided_next_locked (long *, long *);
 934
 935 #ifdef HAVE_SYNC_BUILTINS
 936 extern bool gomp_iter_dynamic_next (long *, long *);
 937 extern bool gomp_iter_guided_next (long *, long *);
 938 #endif
 939
 940 /* iter_ull.c */
 941
 942 extern int gomp_iter_ull_static_next (unsigned long long *,
 943                                       unsigned long long *);
 944 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *,
 945                                                unsigned long long *);
 946 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *,
 947                                               unsigned long long *);
 948
 949 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
 950 extern bool gomp_iter_ull_dynamic_next (unsigned long long *,
 951                                         unsigned long long *);
 952 extern bool gomp_iter_ull_guided_next (unsigned long long *,
 953                                        unsigned long long *);
 954 #endif
 955
 956 /* ordered.c */
 957
 958 extern void gomp_ordered_first (void);
 959 extern void gomp_ordered_last (void);
 960 extern void gomp_ordered_next (void);
 961 extern void gomp_ordered_static_init (void);
 962 extern void gomp_ordered_static_next (void);
 963 extern void gomp_ordered_sync (void);
 964 extern void gomp_doacross_init (unsigned, long *, long, size_t);
 965 extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
 966                                     unsigned long long, size_t);
 967
 968 /* parallel.c */
 969
 970 extern unsigned gomp_resolve_num_threads (unsigned, unsigned);
 971
 972 /* proc.c (in config/) */
 973
 974 extern void gomp_init_num_threads (void);
 975 extern unsigned gomp_dynamic_max_threads (void);
 976
 977 /* task.c */
 978
 979 extern void gomp_init_task (struct gomp_task *, struct gomp_task *,
 980                             struct gomp_task_icv *);
 981 extern void gomp_end_task (void);
 982 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t);
 983 extern void gomp_task_maybe_wait_for_dependencies (void **);
 984 extern bool gomp_create_target_task (struct gomp_device_descr *,
 985                                      void (*) (void *), size_t, void **,
 986                                      size_t *, unsigned short *, unsigned int,
 987                                      void **, void **,
 988                                      enum gomp_target_task_state);
 989 extern struct gomp_taskgroup *gomp_parallel_reduction_register (uintptr_t *,
 990                                                                 unsigned);
 991 extern void gomp_workshare_taskgroup_start (void);
 992 extern void gomp_workshare_task_reduction_register (uintptr_t *, uintptr_t *);
 993
 994 static void inline
 995 gomp_finish_task (struct gomp_task *task)
 996 {
 997   if (__builtin_expect (task->depend_hash != NULL, 0))
 998     free (task->depend_hash);
 999 }
1000
1001 /* team.c */
1002
1003 extern struct gomp_team *gomp_new_team (unsigned);
1004 extern void gomp_team_start (void (*) (void *), void *, unsigned,
1005                              unsigned, struct gomp_team *,
1006                              struct gomp_taskgroup *);
1007 extern void gomp_team_end (void);
1008 extern void gomp_free_thread (void *);
1009 extern int gomp_pause_host (void);
1010
1011 /* target.c */
1012
1013 extern void gomp_init_targets_once (void);
1014 extern int gomp_get_num_devices (void);
1015 extern bool gomp_target_task_fn (void *);
1016
1017 /* Splay tree definitions.  */
1018 typedef struct splay_tree_node_s *splay_tree_node;
1019 typedef struct splay_tree_s *splay_tree;
1020 typedef struct splay_tree_key_s *splay_tree_key;
1021
1022 struct target_var_desc {
1023   /* Splay key.  */
1024   splay_tree_key key;
1025   /* True if data should be copied from device to host at the end.  */
1026   bool copy_from;
1027   /* True if data always should be copied from device to host at the end.  */
1028   bool always_copy_from;
1029   /* True if this is for OpenACC 'attach'.  */
1030   bool is_attach;
1031   /* If GOMP_MAP_TO_PSET had a NULL pointer; used for Fortran descriptors,
1032      which were initially unallocated.  */
1033   bool has_null_ptr_assoc;
1034   /* Relative offset against key host_start.  */
1035   uintptr_t offset;
1036   /* Actual length.  */
1037   uintptr_t length;
1038 };
1039
1040 struct target_mem_desc {
1041   /* Reference count.  */
1042   uintptr_t refcount;
1043   /* All the splay nodes allocated together.  */
1044   splay_tree_node array;
1045   /* Start of the target region.  */
1046   uintptr_t tgt_start;
1047   /* End of the targer region.  */
1048   uintptr_t tgt_end;
1049   /* Handle to free.  */
1050   void *to_free;
1051   /* Previous target_mem_desc.  */
1052   struct target_mem_desc *prev;
1053   /* Number of items in following list.  */
1054   size_t list_count;
1055
1056   /* Corresponding target device descriptor.  */
1057   struct gomp_device_descr *device_descr;
1058
1059   /* List of target items to remove (or decrease refcount)
1060      at the end of region.  */
1061   struct target_var_desc list[];
1062 };
1063
1064 /* Special value for refcount - mask to indicate existence of special
1065    values. Right now we allocate 3 bits.  */
1066 #define REFCOUNT_SPECIAL (~(uintptr_t) 0x7)
1067
1068 /* Special value for refcount - infinity.  */
1069 #define REFCOUNT_INFINITY (REFCOUNT_SPECIAL | 0)
1070 /* Special value for refcount - tgt_offset contains target address of the
1071    artificial pointer to "omp declare target link" object.  */
1072 #define REFCOUNT_LINK     (REFCOUNT_SPECIAL | 1)
1073
1074 /* Special value for refcount - structure element sibling list items.
1075    All such key refounts have REFCOUNT_STRUCTELEM bits set, with _FLAG_FIRST
1076    and _FLAG_LAST indicating first and last in the created sibling sequence.  */
1077 #define REFCOUNT_STRUCTELEM (REFCOUNT_SPECIAL | 4)
1078 #define REFCOUNT_STRUCTELEM_P(V)                        \
1079   (((V) & REFCOUNT_STRUCTELEM) == REFCOUNT_STRUCTELEM)
1080 /* The first leading key with _FLAG_FIRST set houses the actual reference count
1081    in the structelem_refcount field. Other siblings point to this counter value
1082    through its structelem_refcount_ptr field.  */
1083 #define REFCOUNT_STRUCTELEM_FLAG_FIRST (1)
1084 /* The last key in the sibling sequence has this set. This is required to
1085    indicate the sequence boundary, when we remove the structure sibling list
1086    from the map.  */
1087 #define REFCOUNT_STRUCTELEM_FLAG_LAST  (2)
1088
1089 #define REFCOUNT_STRUCTELEM_FIRST_P(V)                                  \
1090   (REFCOUNT_STRUCTELEM_P (V) && ((V) & REFCOUNT_STRUCTELEM_FLAG_FIRST))
1091 #define REFCOUNT_STRUCTELEM_LAST_P(V)                                   \
1092   (REFCOUNT_STRUCTELEM_P (V) && ((V) & REFCOUNT_STRUCTELEM_FLAG_LAST))
1093
1094 /* Special offset values.  */
1095 #define OFFSET_INLINED (~(uintptr_t) 0)
1096 #define OFFSET_POINTER (~(uintptr_t) 1)
1097 #define OFFSET_STRUCT (~(uintptr_t) 2)
1098
1099 /* Auxiliary structure for infrequently-used or API-specific data.  */
1100
1101 struct splay_tree_aux {
1102   /* Pointer to the original mapping of "omp declare target link" object.  */
1103   splay_tree_key link_key;
1104   /* For a block with attached pointers, the attachment counters for each.
1105      Only used for OpenACC.  */
1106   uintptr_t *attach_count;
1107 };
1108
1109 struct splay_tree_key_s {
1110   /* Address of the host object.  */
1111   uintptr_t host_start;
1112   /* Address immediately after the host object.  */
1113   uintptr_t host_end;
1114   /* Descriptor of the target memory.  */
1115   struct target_mem_desc *tgt;
1116   /* Offset from tgt->tgt_start to the start of the target object.  */
1117   uintptr_t tgt_offset;
1118   /* Reference count.  */
1119   uintptr_t refcount;
1120   union {
1121     /* Dynamic reference count.  */
1122     uintptr_t dynamic_refcount;
1123
1124     /* Unified reference count for structure element siblings, this is used
1125        when REFCOUNT_STRUCTELEM_FIRST_P(k->refcount) == true, the first sibling
1126        in a structure element sibling list item sequence.  */
1127     uintptr_t structelem_refcount;
1128
1129     /* When REFCOUNT_STRUCTELEM_P (k->refcount) == true, this field points
1130        into the (above) structelem_refcount field of the _FIRST splay_tree_key,
1131        the first key in the created sequence. All structure element siblings
1132        share a single refcount in this manner. Since these two fields won't be
1133        used at the same time, they are stashed in a union.  */
1134     uintptr_t *structelem_refcount_ptr;
1135   };
1136   struct splay_tree_aux *aux;
1137 };
1138
1139 /* The comparison function.  */
1140
1141 static inline int
1142 splay_compare (splay_tree_key x, splay_tree_key y)
1143 {
1144   if (x->host_start == x->host_end
1145       && y->host_start == y->host_end)
1146     return 0;
1147   if (x->host_end <= y->host_start)
1148     return -1;
1149   if (x->host_start >= y->host_end)
1150     return 1;
1151   return 0;
1152 }
1153
1154 #include "splay-tree.h"
1155
1156 typedef struct acc_dispatch_t
1157 {
1158   /* Execute.  */
1159   __typeof (GOMP_OFFLOAD_openacc_exec) *exec_func;
1160
1161   /* Create/destroy TLS data.  */
1162   __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func;
1163   __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data)
1164     *destroy_thread_data_func;
1165
1166   struct {
1167     /* Once created and put into the "active" list, asyncqueues are then never
1168        destructed and removed from the "active" list, other than if the TODO
1169        device is shut down.  */
1170     gomp_mutex_t lock;
1171     int nasyncqueue;
1172     struct goacc_asyncqueue **asyncqueue;
1173     struct goacc_asyncqueue_list *active;
1174
1175     __typeof (GOMP_OFFLOAD_openacc_async_construct) *construct_func;
1176     __typeof (GOMP_OFFLOAD_openacc_async_destruct) *destruct_func;
1177     __typeof (GOMP_OFFLOAD_openacc_async_test) *test_func;
1178     __typeof (GOMP_OFFLOAD_openacc_async_synchronize) *synchronize_func;
1179     __typeof (GOMP_OFFLOAD_openacc_async_serialize) *serialize_func;
1180     __typeof (GOMP_OFFLOAD_openacc_async_queue_callback) *queue_callback_func;
1181
1182     __typeof (GOMP_OFFLOAD_openacc_async_exec) *exec_func;
1183     __typeof (GOMP_OFFLOAD_openacc_async_dev2host) *dev2host_func;
1184     __typeof (GOMP_OFFLOAD_openacc_async_host2dev) *host2dev_func;
1185   } async;
1186
1187   __typeof (GOMP_OFFLOAD_openacc_get_property) *get_property_func;
1188
1189   /* NVIDIA target specific routines.  */
1190   struct {
1191     __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device)
1192       *get_current_device_func;
1193     __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context)
1194       *get_current_context_func;
1195     __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream) *get_stream_func;
1196     __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream) *set_stream_func;
1197   } cuda;
1198 } acc_dispatch_t;
1199
1200 /* Various state of the accelerator device.  */
1201 enum gomp_device_state
1202 {
1203   GOMP_DEVICE_UNINITIALIZED,
1204   GOMP_DEVICE_INITIALIZED,
1205   GOMP_DEVICE_FINALIZED
1206 };
1207
1208 /* This structure describes accelerator device.
1209    It contains name of the corresponding libgomp plugin, function handlers for
1210    interaction with the device, ID-number of the device, and information about
1211    mapped memory.  */
1212 struct gomp_device_descr
1213 {
1214   /* Immutable data, which is only set during initialization, and which is not
1215      guarded by the lock.  */
1216
1217   /* The name of the device.  */
1218   const char *name;
1219
1220   /* Capabilities of device (supports OpenACC, OpenMP).  */
1221   unsigned int capabilities;
1222
1223   /* This is the ID number of device among devices of the same type.  */
1224   int target_id;
1225
1226   /* This is the TYPE of device.  */
1227   enum offload_target_type type;
1228
1229   /* Function handlers.  */
1230   __typeof (GOMP_OFFLOAD_get_name) *get_name_func;
1231   __typeof (GOMP_OFFLOAD_get_caps) *get_caps_func;
1232   __typeof (GOMP_OFFLOAD_get_type) *get_type_func;
1233   __typeof (GOMP_OFFLOAD_get_num_devices) *get_num_devices_func;
1234   __typeof (GOMP_OFFLOAD_init_device) *init_device_func;
1235   __typeof (GOMP_OFFLOAD_fini_device) *fini_device_func;
1236   __typeof (GOMP_OFFLOAD_version) *version_func;
1237   __typeof (GOMP_OFFLOAD_load_image) *load_image_func;
1238   __typeof (GOMP_OFFLOAD_unload_image) *unload_image_func;
1239   __typeof (GOMP_OFFLOAD_alloc) *alloc_func;
1240   __typeof (GOMP_OFFLOAD_free) *free_func;
1241   __typeof (GOMP_OFFLOAD_dev2host) *dev2host_func;
1242   __typeof (GOMP_OFFLOAD_host2dev) *host2dev_func;
1243   __typeof (GOMP_OFFLOAD_dev2dev) *dev2dev_func;
1244   __typeof (GOMP_OFFLOAD_can_run) *can_run_func;
1245   __typeof (GOMP_OFFLOAD_run) *run_func;
1246   __typeof (GOMP_OFFLOAD_async_run) *async_run_func;
1247
1248   /* Splay tree containing information about mapped memory regions.  */
1249   struct splay_tree_s mem_map;
1250
1251   /* Mutex for the mutable data.  */
1252   gomp_mutex_t lock;
1253
1254   /* Current state of the device.  OpenACC allows to move from INITIALIZED state
1255      back to UNINITIALIZED state.  OpenMP allows only to move from INITIALIZED
1256      to FINALIZED state (at program shutdown).  */
1257   enum gomp_device_state state;
1258
1259   /* OpenACC-specific data and functions.  */
1260   /* This is mutable because of its mutable target_data member.  */
1261   acc_dispatch_t openacc;
1262 };
1263
1264 /* Kind of the pragma, for which gomp_map_vars () is called.  */
1265 enum gomp_map_vars_kind
1266 {
1267   GOMP_MAP_VARS_OPENACC    = 1,
1268   GOMP_MAP_VARS_TARGET     = 2,
1269   GOMP_MAP_VARS_DATA       = 4,
1270   GOMP_MAP_VARS_ENTER_DATA = 8
1271 };
1272
1273 extern void gomp_acc_declare_allocate (bool, size_t, void **, size_t *,
1274                                        unsigned short *);
1275 struct gomp_coalesce_buf;
1276 extern void gomp_copy_host2dev (struct gomp_device_descr *,
1277                                 struct goacc_asyncqueue *, void *, const void *,
1278                                 size_t, bool, struct gomp_coalesce_buf *);
1279 extern void gomp_copy_dev2host (struct gomp_device_descr *,
1280                                 struct goacc_asyncqueue *, void *, const void *,
1281                                 size_t);
1282 extern uintptr_t gomp_map_val (struct target_mem_desc *, void **, size_t);
1283 extern void gomp_attach_pointer (struct gomp_device_descr *,
1284                                  struct goacc_asyncqueue *, splay_tree,
1285                                  splay_tree_key, uintptr_t, size_t,
1286                                  struct gomp_coalesce_buf *, bool);
1287 extern void gomp_detach_pointer (struct gomp_device_descr *,
1288                                  struct goacc_asyncqueue *, splay_tree_key,
1289                                  uintptr_t, bool, struct gomp_coalesce_buf *);
1290 extern struct target_mem_desc *goacc_map_vars (struct gomp_device_descr *,
1291                                                struct goacc_asyncqueue *,
1292                                                size_t, void **, void **,
1293                                                size_t *, void *, bool,
1294                                                enum gomp_map_vars_kind);
1295 extern void goacc_unmap_vars (struct target_mem_desc *, bool,
1296                               struct goacc_asyncqueue *);
1297 extern void gomp_init_device (struct gomp_device_descr *);
1298 extern bool gomp_fini_device (struct gomp_device_descr *);
1299 extern void gomp_unload_device (struct gomp_device_descr *);
1300 extern bool gomp_remove_var (struct gomp_device_descr *, splay_tree_key);
1301 extern void gomp_remove_var_async (struct gomp_device_descr *, splay_tree_key,
1302                                    struct goacc_asyncqueue *);
1303
1304 /* work.c */
1305
1306 extern void gomp_init_work_share (struct gomp_work_share *, size_t, unsigned);
1307 extern void gomp_fini_work_share (struct gomp_work_share *);
1308 extern bool gomp_work_share_start (size_t);
1309 extern void gomp_work_share_end (void);
1310 extern bool gomp_work_share_end_cancel (void);
1311 extern void gomp_work_share_end_nowait (void);
1312
1313 static inline void
1314 gomp_work_share_init_done (void)
1315 {
1316   struct gomp_thread *thr = gomp_thread ();
1317   if (__builtin_expect (thr->ts.last_work_share != NULL, 1))
1318     gomp_ptrlock_set (&thr->ts.last_work_share->next_ws, thr->ts.work_share);
1319 }
1320
1321 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1322 # pragma GCC visibility pop
1323 #endif
1324
1325 /* Now that we're back to default visibility, include the globals.  */
1326 #include "libgomp_g.h"
1327
1328 /* Include omp.h by parts.  */
1329 #include "omp-lock.h"
1330 #define _LIBGOMP_OMP_LOCK_DEFINED 1
1331 #include "omp.h.in"
1332
1333 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \
1334     || !defined (HAVE_ATTRIBUTE_ALIAS) \
1335     || !defined (HAVE_AS_SYMVER_DIRECTIVE) \
1336     || !defined (PIC) \
1337     || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT)
1338 # undef LIBGOMP_GNU_SYMBOL_VERSIONING
1339 #endif
1340
1341 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING
1342 extern void gomp_init_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1343 extern void gomp_destroy_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1344 extern void gomp_set_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1345 extern void gomp_unset_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1346 extern int gomp_test_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
1347 extern void gomp_init_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1348 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1349 extern void gomp_set_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1350 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1351 extern int gomp_test_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
1352
1353 extern void gomp_init_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1354 extern void gomp_destroy_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1355 extern void gomp_set_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1356 extern void gomp_unset_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1357 extern int gomp_test_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
1358 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1359 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1360 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1361 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1362 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
1363
1364 # define omp_lock_symver(fn) \
1365   __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
1366   __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
1367 #else
1368 # define gomp_init_lock_30 omp_init_lock
1369 # define gomp_destroy_lock_30 omp_destroy_lock
1370 # define gomp_set_lock_30 omp_set_lock
1371 # define gomp_unset_lock_30 omp_unset_lock
1372 # define gomp_test_lock_30 omp_test_lock
1373 # define gomp_init_nest_lock_30 omp_init_nest_lock
1374 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock
1375 # define gomp_set_nest_lock_30 omp_set_nest_lock
1376 # define gomp_unset_nest_lock_30 omp_unset_nest_lock
1377 # define gomp_test_nest_lock_30 omp_test_nest_lock
1378 #endif
1379
1380 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1381 # define attribute_hidden __attribute__ ((visibility ("hidden")))
1382 #else
1383 # define attribute_hidden
1384 #endif
1385
1386 #if __GNUC__ >= 9
1387 #  define HAVE_ATTRIBUTE_COPY
1388 #endif
1389
1390 #ifdef HAVE_ATTRIBUTE_COPY
1391 # define attribute_copy(arg) __attribute__ ((copy (arg)))
1392 #else
1393 # define attribute_copy(arg)
1394 #endif
1395
1396 #ifdef HAVE_ATTRIBUTE_ALIAS
1397 # define strong_alias(fn, al) \
1398   extern __typeof (fn) al __attribute__ ((alias (#fn))) attribute_copy (fn);
1399
1400 # define ialias_ulp     ialias_str1(__USER_LABEL_PREFIX__)
1401 # define ialias_str1(x) ialias_str2(x)
1402 # define ialias_str2(x) #x
1403 # define ialias(fn) \
1404   extern __typeof (fn) gomp_ialias_##fn \
1405     __attribute__ ((alias (#fn))) attribute_hidden attribute_copy (fn);
1406 # define ialias_redirect(fn) \
1407   extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
1408 # define ialias_call(fn) gomp_ialias_ ## fn
1409 #else
1410 # define ialias(fn)
1411 # define ialias_redirect(fn)
1412 # define ialias_call(fn) fn
1413 #endif
1414
1415 /* Helper function for priority_node_to_task() and
1416    task_to_priority_node().
1417
1418    Return the offset from a task to its priority_node entry.  The
1419    priority_node entry is has a type of TYPE.  */
1420
1421 static inline size_t
1422 priority_queue_offset (enum priority_queue_type type)
1423 {
1424   return offsetof (struct gomp_task, pnode[(int) type]);
1425 }
1426
1427 /* Return the task associated with a priority NODE of type TYPE.  */
1428
1429 static inline struct gomp_task *
1430 priority_node_to_task (enum priority_queue_type type,
1431                        struct priority_node *node)
1432 {
1433   return (struct gomp_task *) ((char *) node - priority_queue_offset (type));
1434 }
1435
1436 /* Return the priority node of type TYPE for a given TASK.  */
1437
1438 static inline struct priority_node *
1439 task_to_priority_node (enum priority_queue_type type,
1440                        struct gomp_task *task)
1441 {
1442   return (struct priority_node *) ((char *) task
1443                                    + priority_queue_offset (type));
1444 }
1445
1446 #ifdef LIBGOMP_USE_PTHREADS
1447 static inline gomp_thread_handle
1448 gomp_thread_self (void)
1449 {
1450   return pthread_self ();
1451 }
1452
1453 static inline gomp_thread_handle
1454 gomp_thread_to_pthread_t (struct gomp_thread *thr)
1455 {
1456   struct gomp_thread *this_thr = gomp_thread ();
1457   if (thr == this_thr)
1458     return pthread_self ();
1459 #ifdef GOMP_NEEDS_THREAD_HANDLE
1460   return thr->handle;
1461 #else
1462   /* On Linux with initial-exec TLS, the pthread_t of the thread containing
1463      thr can be computed from thr, this_thr and pthread_self (),
1464      as the distance between this_thr and pthread_self () is constant.  */
1465   return pthread_self () + ((uintptr_t) thr - (uintptr_t) this_thr);
1466 #endif
1467 }
1468 #else
1469 static inline gomp_thread_handle
1470 gomp_thread_self (void)
1471 {
1472   return (gomp_thread_handle) {};
1473 }
1474
1475 static inline gomp_thread_handle
1476 gomp_thread_to_pthread_t (struct gomp_thread *thr)
1477 {
1478   (void) thr;
1479   return gomp_thread_self ();
1480 }
1481 #endif
1482
1483 #endif /* LIBGOMP_H */