libgomp/libgomp.h

   1 /* Copyright (C) 2005-2015 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains data types and function declarations that are not
  27    part of the official OpenACC or OpenMP user interfaces.  There are
  28    declarations in here that are part of the GNU Offloading and Multi
  29    Processing ABI, in that the compiler is required to know about them
  30    and use them.
  31
  32    The convention is that the all caps prefix "GOMP" is used group items
  33    that are part of the external ABI, and the lower case prefix "gomp"
  34    is used group items that are completely private to the library.  */
  35
  36 #ifndef LIBGOMP_H
  37 #define LIBGOMP_H 1
  38
  39 #ifndef _LIBGOMP_CHECKING_
  40 /* Define to 1 to perform internal sanity checks.  */
  41 #define _LIBGOMP_CHECKING_ 0
  42 #endif
  43
  44 #include "config.h"
  45 #include "gstdint.h"
  46 #include "libgomp-plugin.h"
  47
  48 #include <pthread.h>
  49 #include <stdbool.h>
  50 #include <stdlib.h>
  51 #include <stdarg.h>
  52
  53 #ifdef HAVE_ATTRIBUTE_VISIBILITY
  54 # pragma GCC visibility push(hidden)
  55 #endif
  56
  57 /* If we were a C++ library, we'd get this from <std/atomic>.  */
  58 enum memmodel
  59 {
  60   MEMMODEL_RELAXED = 0,
  61   MEMMODEL_CONSUME = 1,
  62   MEMMODEL_ACQUIRE = 2,
  63   MEMMODEL_RELEASE = 3,
  64   MEMMODEL_ACQ_REL = 4,
  65   MEMMODEL_SEQ_CST = 5
  66 };
  67
  68 #include "sem.h"
  69 #include "mutex.h"
  70 #include "bar.h"
  71 #include "ptrlock.h"
  72
  73
  74 /* This structure contains the data to control one work-sharing construct,
  75    either a LOOP (FOR/DO) or a SECTIONS.  */
  76
  77 enum gomp_schedule_type
  78 {
  79   GFS_RUNTIME,
  80   GFS_STATIC,
  81   GFS_DYNAMIC,
  82   GFS_GUIDED,
  83   GFS_AUTO
  84 };
  85
  86 struct gomp_doacross_work_share
  87 {
  88   union {
  89     /* chunk_size copy, as ws->chunk_size is multiplied by incr for
  90        GFS_DYNAMIC.  */
  91     long chunk_size;
  92     /* Likewise, but for ull implementation.  */
  93     unsigned long long chunk_size_ull;
  94     /* For schedule(static,0) this is the number
  95        of iterations assigned to the last thread, i.e. number of
  96        iterations / number of threads.  */
  97     long q;
  98     /* Likewise, but for ull implementation.  */
  99     unsigned long long q_ull;
 100   };
 101   /* Size of each array entry (padded to cache line size).  */
 102   unsigned long elt_sz;
 103   /* Number of dimensions in sink vectors.  */
 104   unsigned int ncounts;
 105   /* True if the iterations can be flattened.  */
 106   bool flattened;
 107   /* Actual array (of elt_sz sized units), aligned to cache line size.
 108      This is indexed by team_id for GFS_STATIC and outermost iteration
 109      / chunk_size for other schedules.  */
 110   unsigned char *array;
 111   /* These two are only used for schedule(static,0).  */
 112   /* This one is number of iterations % number of threads.  */
 113   long t;
 114   union {
 115     /* And this one is cached t * (q + 1).  */
 116     long boundary;
 117     /* Likewise, but for the ull implementation.  */
 118     unsigned long long boundary_ull;
 119   };
 120   /* Array of shift counts for each dimension if they can be flattened.  */
 121   unsigned int shift_counts[];
 122 };
 123
 124 struct gomp_work_share
 125 {
 126   /* This member records the SCHEDULE clause to be used for this construct.
 127      The user specification of "runtime" will already have been resolved.
 128      If this is a SECTIONS construct, this value will always be DYNAMIC.  */
 129   enum gomp_schedule_type sched;
 130
 131   int mode;
 132
 133   union {
 134     struct {
 135       /* This is the chunk_size argument to the SCHEDULE clause.  */
 136       long chunk_size;
 137
 138       /* This is the iteration end point.  If this is a SECTIONS construct,
 139          this is the number of contained sections.  */
 140       long end;
 141
 142       /* This is the iteration step.  If this is a SECTIONS construct, this
 143          is always 1.  */
 144       long incr;
 145     };
 146
 147     struct {
 148       /* The same as above, but for the unsigned long long loop variants.  */
 149       unsigned long long chunk_size_ull;
 150       unsigned long long end_ull;
 151       unsigned long long incr_ull;
 152     };
 153   };
 154
 155   union {
 156     /* This is a circular queue that details which threads will be allowed
 157        into the ordered region and in which order.  When a thread allocates
 158        iterations on which it is going to work, it also registers itself at
 159        the end of the array.  When a thread reaches the ordered region, it
 160        checks to see if it is the one at the head of the queue.  If not, it
 161        blocks on its RELEASE semaphore.  */
 162     unsigned *ordered_team_ids;
 163
 164     /* This is a pointer to DOACROSS work share data.  */
 165     struct gomp_doacross_work_share *doacross;
 166   };
 167
 168   /* This is the number of threads that have registered themselves in
 169      the circular queue ordered_team_ids.  */
 170   unsigned ordered_num_used;
 171
 172   /* This is the team_id of the currently acknowledged owner of the ordered
 173      section, or -1u if the ordered section has not been acknowledged by
 174      any thread.  This is distinguished from the thread that is *allowed*
 175      to take the section next.  */
 176   unsigned ordered_owner;
 177
 178   /* This is the index into the circular queue ordered_team_ids of the
 179      current thread that's allowed into the ordered reason.  */
 180   unsigned ordered_cur;
 181
 182   /* This is a chain of allocated gomp_work_share blocks, valid only
 183      in the first gomp_work_share struct in the block.  */
 184   struct gomp_work_share *next_alloc;
 185
 186   /* The above fields are written once during workshare initialization,
 187      or related to ordered worksharing.  Make sure the following fields
 188      are in a different cache line.  */
 189
 190   /* This lock protects the update of the following members.  */
 191   gomp_mutex_t lock __attribute__((aligned (64)));
 192
 193   /* This is the count of the number of threads that have exited the work
 194      share construct.  If the construct was marked nowait, they have moved on
 195      to other work; otherwise they're blocked on a barrier.  The last member
 196      of the team to exit the work share construct must deallocate it.  */
 197   unsigned threads_completed;
 198
 199   union {
 200     /* This is the next iteration value to be allocated.  In the case of
 201        GFS_STATIC loops, this the iteration start point and never changes.  */
 202     long next;
 203
 204     /* The same, but with unsigned long long type.  */
 205     unsigned long long next_ull;
 206
 207     /* This is the returned data structure for SINGLE COPYPRIVATE.  */
 208     void *copyprivate;
 209   };
 210
 211   union {
 212     /* Link to gomp_work_share struct for next work sharing construct
 213        encountered after this one.  */
 214     gomp_ptrlock_t next_ws;
 215
 216     /* gomp_work_share structs are chained in the free work share cache
 217        through this.  */
 218     struct gomp_work_share *next_free;
 219   };
 220
 221   /* If only few threads are in the team, ordered_team_ids can point
 222      to this array which fills the padding at the end of this struct.  */
 223   unsigned inline_ordered_team_ids[0];
 224 };
 225
 226 /* This structure contains all of the thread-local data associated with
 227    a thread team.  This is the data that must be saved when a thread
 228    encounters a nested PARALLEL construct.  */
 229
 230 struct gomp_team_state
 231 {
 232   /* This is the team of which the thread is currently a member.  */
 233   struct gomp_team *team;
 234
 235   /* This is the work share construct which this thread is currently
 236      processing.  Recall that with NOWAIT, not all threads may be
 237      processing the same construct.  */
 238   struct gomp_work_share *work_share;
 239
 240   /* This is the previous work share construct or NULL if there wasn't any.
 241      When all threads are done with the current work sharing construct,
 242      the previous one can be freed.  The current one can't, as its
 243      next_ws field is used.  */
 244   struct gomp_work_share *last_work_share;
 245
 246   /* This is the ID of this thread within the team.  This value is
 247      guaranteed to be between 0 and N-1, where N is the number of
 248      threads in the team.  */
 249   unsigned team_id;
 250
 251   /* Nesting level.  */
 252   unsigned level;
 253
 254   /* Active nesting level.  Only active parallel regions are counted.  */
 255   unsigned active_level;
 256
 257   /* Place-partition-var, offset and length into gomp_places_list array.  */
 258   unsigned place_partition_off;
 259   unsigned place_partition_len;
 260
 261 #ifdef HAVE_SYNC_BUILTINS
 262   /* Number of single stmts encountered.  */
 263   unsigned long single_count;
 264 #endif
 265
 266   /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the
 267      trip number through the loop.  So first time a particular loop
 268      is encountered this number is 0, the second time through the loop
 269      is 1, etc.  This is unused when the compiler knows in advance that
 270      the loop is statically scheduled.  */
 271   unsigned long static_trip;
 272 };
 273
 274 struct target_mem_desc;
 275
 276 /* These are the OpenMP 4.0 Internal Control Variables described in
 277    section 2.3.1.  Those described as having one copy per task are
 278    stored within the structure; those described as having one copy
 279    for the whole program are (naturally) global variables.  */
 280
 281 struct gomp_task_icv
 282 {
 283   unsigned long nthreads_var;
 284   enum gomp_schedule_type run_sched_var;
 285   int run_sched_chunk_size;
 286   int default_device_var;
 287   unsigned int thread_limit_var;
 288   bool dyn_var;
 289   bool nest_var;
 290   char bind_var;
 291   /* Internal ICV.  */
 292   struct target_mem_desc *target_data;
 293 };
 294
 295 extern struct gomp_task_icv gomp_global_icv;
 296 #ifndef HAVE_SYNC_BUILTINS
 297 extern gomp_mutex_t gomp_managed_threads_lock;
 298 #endif
 299 extern unsigned long gomp_max_active_levels_var;
 300 extern bool gomp_cancel_var;
 301 extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var;
 302 extern unsigned long gomp_available_cpus, gomp_managed_threads;
 303 extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len;
 304 extern char *gomp_bind_var_list;
 305 extern unsigned long gomp_bind_var_list_len;
 306 extern void **gomp_places_list;
 307 extern unsigned long gomp_places_list_len;
 308 extern int gomp_debug_var;
 309 extern int goacc_device_num;
 310 extern char *goacc_device_type;
 311
 312 enum gomp_task_kind
 313 {
 314   /* Implicit task.  */
 315   GOMP_TASK_IMPLICIT,
 316   /* Undeferred task.  */
 317   GOMP_TASK_UNDEFERRED,
 318   /* Task created by GOMP_task and waiting to be run.  */
 319   GOMP_TASK_WAITING,
 320   /* Task currently executing or scheduled and about to execute.  */
 321   GOMP_TASK_TIED
 322 };
 323
 324 struct gomp_task;
 325 struct gomp_taskgroup;
 326 struct htab;
 327
 328 struct gomp_task_depend_entry
 329 {
 330   /* Address of dependency.  */
 331   void *addr;
 332   struct gomp_task_depend_entry *next;
 333   struct gomp_task_depend_entry *prev;
 334   /* Task that provides the dependency in ADDR.  */
 335   struct gomp_task *task;
 336   /* Depend entry is of type "IN".  */
 337   bool is_in;
 338   bool redundant;
 339   bool redundant_out;
 340 };
 341
 342 struct gomp_dependers_vec
 343 {
 344   size_t n_elem;
 345   size_t allocated;
 346   struct gomp_task *elem[];
 347 };
 348
 349 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies.  */
 350
 351 struct gomp_taskwait
 352 {
 353   bool in_taskwait;
 354   bool in_depend_wait;
 355   size_t n_depend;
 356   struct gomp_task *last_parent_depends_on;
 357   gomp_sem_t taskwait_sem;
 358 };
 359
 360 /* This structure describes a "task" to be run by a thread.  */
 361
 362 struct gomp_task
 363 {
 364   /* Parent circular list.  See children description below.  */
 365   struct gomp_task *parent;
 366   /* Circular list representing the children of this task.
 367
 368      In this list we first have parent_depends_on ready to run tasks,
 369      then !parent_depends_on ready to run tasks, and finally already
 370      running tasks.  */
 371   struct gomp_task *children;
 372   struct gomp_task *next_child;
 373   struct gomp_task *prev_child;
 374   /* Circular task_queue in `struct gomp_team'.
 375
 376      GOMP_TASK_WAITING tasks come before GOMP_TASK_TIED tasks.  */
 377   struct gomp_task *next_queue;
 378   struct gomp_task *prev_queue;
 379   /* Circular queue in gomp_taskgroup->children.
 380
 381      GOMP_TASK_WAITING tasks come before GOMP_TASK_TIED tasks.  */
 382   struct gomp_task *next_taskgroup;
 383   struct gomp_task *prev_taskgroup;
 384   /* Taskgroup this task belongs in.  */
 385   struct gomp_taskgroup *taskgroup;
 386   /* Tasks that depend on this task.  */
 387   struct gomp_dependers_vec *dependers;
 388   struct htab *depend_hash;
 389   struct gomp_taskwait *taskwait;
 390   /* Number of items in DEPEND.  */
 391   size_t depend_count;
 392   /* Number of tasks in the DEPENDERS field above.  */
 393   size_t num_dependees;
 394   struct gomp_task_icv icv;
 395   void (*fn) (void *);
 396   void *fn_data;
 397   enum gomp_task_kind kind;
 398   bool in_tied_task;
 399   bool final_task;
 400   bool copy_ctors_done;
 401   /* Set for undeferred tasks with unsatisfied dependencies which
 402      block further execution of their parent until the dependencies
 403      are satisfied.  */
 404   bool parent_depends_on;
 405   /* Dependencies provided and/or needed for this task.  DEPEND_COUNT
 406      is the number of items available.  */
 407   struct gomp_task_depend_entry depend[];
 408 };
 409
 410 struct gomp_taskgroup
 411 {
 412   struct gomp_taskgroup *prev;
 413   /* Circular list of tasks that belong in this taskgroup.
 414
 415      Tasks are chained by next/prev_taskgroup within gomp_task, and
 416      are sorted by GOMP_TASK_WAITING tasks, and then GOMP_TASK_TIED
 417      tasks.  */
 418   struct gomp_task *children;
 419   bool in_taskgroup_wait;
 420   bool cancelled;
 421   gomp_sem_t taskgroup_sem;
 422   size_t num_children;
 423 };
 424
 425 struct gomp_target_task
 426 {
 427   struct gomp_device_descr *devicep;
 428   void (*fn) (void *);
 429   size_t mapnum;
 430   size_t *sizes;
 431   unsigned short *kinds;
 432   unsigned int flags;
 433   void *hostaddrs[];
 434 };
 435
 436 /* This structure describes a "team" of threads.  These are the threads
 437    that are spawned by a PARALLEL constructs, as well as the work sharing
 438    constructs that the team encounters.  */
 439
 440 struct gomp_team
 441 {
 442   /* This is the number of threads in the current team.  */
 443   unsigned nthreads;
 444
 445   /* This is number of gomp_work_share structs that have been allocated
 446      as a block last time.  */
 447   unsigned work_share_chunk;
 448
 449   /* This is the saved team state that applied to a master thread before
 450      the current thread was created.  */
 451   struct gomp_team_state prev_ts;
 452
 453   /* This semaphore should be used by the master thread instead of its
 454      "native" semaphore in the thread structure.  Required for nested
 455      parallels, as the master is a member of two teams.  */
 456   gomp_sem_t master_release;
 457
 458   /* This points to an array with pointers to the release semaphore
 459      of the threads in the team.  */
 460   gomp_sem_t **ordered_release;
 461
 462   /* List of work shares on which gomp_fini_work_share hasn't been
 463      called yet.  If the team hasn't been cancelled, this should be
 464      equal to each thr->ts.work_share, but otherwise it can be a possibly
 465      long list of workshares.  */
 466   struct gomp_work_share *work_shares_to_free;
 467
 468   /* List of gomp_work_share structs chained through next_free fields.
 469      This is populated and taken off only by the first thread in the
 470      team encountering a new work sharing construct, in a critical
 471      section.  */
 472   struct gomp_work_share *work_share_list_alloc;
 473
 474   /* List of gomp_work_share structs freed by free_work_share.  New
 475      entries are atomically added to the start of the list, and
 476      alloc_work_share can safely only move all but the first entry
 477      to work_share_list alloc, as free_work_share can happen concurrently
 478      with alloc_work_share.  */
 479   struct gomp_work_share *work_share_list_free;
 480
 481 #ifdef HAVE_SYNC_BUILTINS
 482   /* Number of simple single regions encountered by threads in this
 483      team.  */
 484   unsigned long single_count;
 485 #else
 486   /* Mutex protecting addition of workshares to work_share_list_free.  */
 487   gomp_mutex_t work_share_list_free_lock;
 488 #endif
 489
 490   /* This barrier is used for most synchronization of the team.  */
 491   gomp_barrier_t barrier;
 492
 493   /* Initial work shares, to avoid allocating any gomp_work_share
 494      structs in the common case.  */
 495   struct gomp_work_share work_shares[8];
 496
 497   gomp_mutex_t task_lock;
 498   /* Scheduled tasks.  Chain fields are next/prev_queue within a
 499      gomp_task.  */
 500   struct gomp_task *task_queue;
 501   /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team.  */
 502   unsigned int task_count;
 503   /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled.  */
 504   unsigned int task_queued_count;
 505   /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
 506      directly in gomp_barrier_handle_tasks; tasks spawned
 507      from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
 508      that is called from a task run from gomp_barrier_handle_tasks.
 509      task_running_count should be always <= team->nthreads,
 510      and if current task isn't in_tied_task, then it will be
 511      even < team->nthreads.  */
 512   unsigned int task_running_count;
 513   int work_share_cancelled;
 514   int team_cancelled;
 515
 516   /* This array contains structures for implicit tasks.  */
 517   struct gomp_task implicit_task[];
 518 };
 519
 520 /* This structure contains all data that is private to libgomp and is
 521    allocated per thread.  */
 522
 523 struct gomp_thread
 524 {
 525   /* This is the function that the thread should run upon launch.  */
 526   void (*fn) (void *data);
 527   void *data;
 528
 529   /* This is the current team state for this thread.  The ts.team member
 530      is NULL only if the thread is idle.  */
 531   struct gomp_team_state ts;
 532
 533   /* This is the task that the thread is currently executing.  */
 534   struct gomp_task *task;
 535
 536   /* This semaphore is used for ordered loops.  */
 537   gomp_sem_t release;
 538
 539   /* Place this thread is bound to plus one, or zero if not bound
 540      to any place.  */
 541   unsigned int place;
 542
 543   /* User pthread thread pool */
 544   struct gomp_thread_pool *thread_pool;
 545 };
 546
 547
 548 struct gomp_thread_pool
 549 {
 550   /* This array manages threads spawned from the top level, which will
 551      return to the idle loop once the current PARALLEL construct ends.  */
 552   struct gomp_thread **threads;
 553   unsigned threads_size;
 554   unsigned threads_used;
 555   /* The last team is used for non-nested teams to delay their destruction to
 556      make sure all the threads in the team move on to the pool's barrier before
 557      the team's barrier is destroyed.  */
 558   struct gomp_team *last_team;
 559   /* Number of threads running in this contention group.  */
 560   unsigned long threads_busy;
 561
 562   /* This barrier holds and releases threads waiting in threads.  */
 563   gomp_barrier_t threads_dock;
 564 };
 565
 566 enum gomp_cancel_kind
 567 {
 568   GOMP_CANCEL_PARALLEL = 1,
 569   GOMP_CANCEL_LOOP = 2,
 570   GOMP_CANCEL_FOR = GOMP_CANCEL_LOOP,
 571   GOMP_CANCEL_DO = GOMP_CANCEL_LOOP,
 572   GOMP_CANCEL_SECTIONS = 4,
 573   GOMP_CANCEL_TASKGROUP = 8
 574 };
 575
 576 /* ... and here is that TLS data.  */
 577
 578 #if defined HAVE_TLS || defined USE_EMUTLS
 579 extern __thread struct gomp_thread gomp_tls_data;
 580 static inline struct gomp_thread *gomp_thread (void)
 581 {
 582   return &gomp_tls_data;
 583 }
 584 #else
 585 extern pthread_key_t gomp_tls_key;
 586 static inline struct gomp_thread *gomp_thread (void)
 587 {
 588   return pthread_getspecific (gomp_tls_key);
 589 }
 590 #endif
 591
 592 extern struct gomp_task_icv *gomp_new_icv (void);
 593
 594 /* Here's how to access the current copy of the ICVs.  */
 595
 596 static inline struct gomp_task_icv *gomp_icv (bool write)
 597 {
 598   struct gomp_task *task = gomp_thread ()->task;
 599   if (task)
 600     return &task->icv;
 601   else if (write)
 602     return gomp_new_icv ();
 603   else
 604     return &gomp_global_icv;
 605 }
 606
 607 /* The attributes to be used during thread creation.  */
 608 extern pthread_attr_t gomp_thread_attr;
 609
 610 extern pthread_key_t gomp_thread_destructor;
 611
 612 /* Function prototypes.  */
 613
 614 /* affinity.c */
 615
 616 extern void gomp_init_affinity (void);
 617 extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int);
 618 extern void **gomp_affinity_alloc (unsigned long, bool);
 619 extern void gomp_affinity_init_place (void *);
 620 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
 621                                     long, bool);
 622 extern bool gomp_affinity_remove_cpu (void *, unsigned long);
 623 extern bool gomp_affinity_copy_place (void *, void *, long);
 624 extern bool gomp_affinity_same_place (void *, void *);
 625 extern bool gomp_affinity_finalize_place_list (bool);
 626 extern bool gomp_affinity_init_level (int, unsigned long, bool);
 627 extern void gomp_affinity_print_place (void *);
 628 extern void gomp_get_place_proc_ids_8 (int, int64_t *);
 629
 630 /* alloc.c */
 631
 632 extern void *gomp_malloc (size_t) __attribute__((malloc));
 633 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc));
 634 extern void *gomp_realloc (void *, size_t);
 635
 636 /* Avoid conflicting prototypes of alloca() in system headers by using
 637    GCC's builtin alloca().  */
 638 #define gomp_alloca(x)  __builtin_alloca(x)
 639
 640 /* error.c */
 641
 642 extern void gomp_vdebug (int, const char *, va_list);
 643 extern void gomp_debug (int, const char *, ...)
 644         __attribute__ ((format (printf, 2, 3)));
 645 #define gomp_vdebug(KIND, FMT, VALIST) \
 646   do { \
 647     if (__builtin_expect (gomp_debug_var, 0)) \
 648       (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
 649   } while (0)
 650 #define gomp_debug(KIND, ...) \
 651   do { \
 652     if (__builtin_expect (gomp_debug_var, 0)) \
 653       (gomp_debug) ((KIND), __VA_ARGS__); \
 654   } while (0)
 655 extern void gomp_verror (const char *, va_list);
 656 extern void gomp_error (const char *, ...)
 657         __attribute__ ((format (printf, 1, 2)));
 658 extern void gomp_vfatal (const char *, va_list)
 659         __attribute__ ((noreturn));
 660 extern void gomp_fatal (const char *, ...)
 661         __attribute__ ((noreturn, format (printf, 1, 2)));
 662
 663 /* iter.c */
 664
 665 extern int gomp_iter_static_next (long *, long *);
 666 extern bool gomp_iter_dynamic_next_locked (long *, long *);
 667 extern bool gomp_iter_guided_next_locked (long *, long *);
 668
 669 #ifdef HAVE_SYNC_BUILTINS
 670 extern bool gomp_iter_dynamic_next (long *, long *);
 671 extern bool gomp_iter_guided_next (long *, long *);
 672 #endif
 673
 674 /* iter_ull.c */
 675
 676 extern int gomp_iter_ull_static_next (unsigned long long *,
 677                                       unsigned long long *);
 678 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *,
 679                                                unsigned long long *);
 680 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *,
 681                                               unsigned long long *);
 682
 683 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
 684 extern bool gomp_iter_ull_dynamic_next (unsigned long long *,
 685                                         unsigned long long *);
 686 extern bool gomp_iter_ull_guided_next (unsigned long long *,
 687                                        unsigned long long *);
 688 #endif
 689
 690 /* ordered.c */
 691
 692 extern void gomp_ordered_first (void);
 693 extern void gomp_ordered_last (void);
 694 extern void gomp_ordered_next (void);
 695 extern void gomp_ordered_static_init (void);
 696 extern void gomp_ordered_static_next (void);
 697 extern void gomp_ordered_sync (void);
 698 extern void gomp_doacross_init (unsigned, long *, long);
 699 extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
 700                                     unsigned long long);
 701
 702 /* parallel.c */
 703
 704 extern unsigned gomp_resolve_num_threads (unsigned, unsigned);
 705
 706 /* proc.c (in config/) */
 707
 708 extern void gomp_init_num_threads (void);
 709 extern unsigned gomp_dynamic_max_threads (void);
 710
 711 /* task.c */
 712
 713 extern void gomp_init_task (struct gomp_task *, struct gomp_task *,
 714                             struct gomp_task_icv *);
 715 extern void gomp_end_task (void);
 716 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t);
 717 extern void gomp_task_maybe_wait_for_dependencies (void **);
 718 extern void gomp_create_target_task (struct gomp_device_descr *,
 719                                      void (*) (void *), size_t, void **,
 720                                      size_t *, unsigned short *, unsigned int,
 721                                      void **);
 722
 723 static void inline
 724 gomp_finish_task (struct gomp_task *task)
 725 {
 726   if (__builtin_expect (task->depend_hash != NULL, 0))
 727     free (task->depend_hash);
 728 }
 729
 730 /* team.c */
 731
 732 extern struct gomp_team *gomp_new_team (unsigned);
 733 extern void gomp_team_start (void (*) (void *), void *, unsigned,
 734                              unsigned, struct gomp_team *);
 735 extern void gomp_team_end (void);
 736 extern void gomp_free_thread (void *);
 737
 738 /* target.c */
 739
 740 extern void gomp_init_targets_once (void);
 741 extern int gomp_get_num_devices (void);
 742 extern void gomp_target_task_fn (void *);
 743
 744 typedef struct splay_tree_node_s *splay_tree_node;
 745 typedef struct splay_tree_s *splay_tree;
 746 typedef struct splay_tree_key_s *splay_tree_key;
 747
 748 struct target_var_desc {
 749   /* Splay key.  */
 750   splay_tree_key key;
 751   /* True if data should be copied from device to host at the end.  */
 752   bool copy_from;
 753   /* True if data always should be copied from device to host at the end.  */
 754   bool always_copy_from;
 755   /* Relative offset against key host_start.  */
 756   uintptr_t offset;
 757   /* Actual length.  */
 758   uintptr_t length;
 759 };
 760
 761 struct target_mem_desc {
 762   /* Reference count.  */
 763   uintptr_t refcount;
 764   /* All the splay nodes allocated together.  */
 765   splay_tree_node array;
 766   /* Start of the target region.  */
 767   uintptr_t tgt_start;
 768   /* End of the targer region.  */
 769   uintptr_t tgt_end;
 770   /* Handle to free.  */
 771   void *to_free;
 772   /* Previous target_mem_desc.  */
 773   struct target_mem_desc *prev;
 774   /* Number of items in following list.  */
 775   size_t list_count;
 776
 777   /* Corresponding target device descriptor.  */
 778   struct gomp_device_descr *device_descr;
 779
 780   /* List of target items to remove (or decrease refcount)
 781      at the end of region.  */
 782   struct target_var_desc list[];
 783 };
 784
 785 /* Special value for refcount - infinity.  */
 786 #define REFCOUNT_INFINITY (~(uintptr_t) 0)
 787
 788 struct splay_tree_key_s {
 789   /* Address of the host object.  */
 790   uintptr_t host_start;
 791   /* Address immediately after the host object.  */
 792   uintptr_t host_end;
 793   /* Descriptor of the target memory.  */
 794   struct target_mem_desc *tgt;
 795   /* Offset from tgt->tgt_start to the start of the target object.  */
 796   uintptr_t tgt_offset;
 797   /* Reference count.  */
 798   uintptr_t refcount;
 799   /* Asynchronous reference count.  */
 800   uintptr_t async_refcount;
 801 };
 802
 803 #include "splay-tree.h"
 804
 805 typedef struct acc_dispatch_t
 806 {
 807   /* This is a linked list of data mapped using the
 808      acc_map_data/acc_unmap_data or "acc enter data"/"acc exit data" pragmas.
 809      Unlike mapped_data in the goacc_thread struct, unmapping can
 810      happen out-of-order with respect to mapping.  */
 811   /* This is guarded by the lock in the "outer" struct gomp_device_descr.  */
 812   struct target_mem_desc *data_environ;
 813
 814   /* Execute.  */
 815   void (*exec_func) (void (*) (void *), size_t, void **, void **, size_t *,
 816                      unsigned short *, int, unsigned *, void *);
 817
 818   /* Async cleanup callback registration.  */
 819   void (*register_async_cleanup_func) (void *);
 820
 821   /* Asynchronous routines.  */
 822   int (*async_test_func) (int);
 823   int (*async_test_all_func) (void);
 824   void (*async_wait_func) (int);
 825   void (*async_wait_async_func) (int, int);
 826   void (*async_wait_all_func) (void);
 827   void (*async_wait_all_async_func) (int);
 828   void (*async_set_async_func) (int);
 829
 830   /* Create/destroy TLS data.  */
 831   void *(*create_thread_data_func) (int);
 832   void (*destroy_thread_data_func) (void *);
 833
 834   /* NVIDIA target specific routines.  */
 835   struct {
 836     void *(*get_current_device_func) (void);
 837     void *(*get_current_context_func) (void);
 838     void *(*get_stream_func) (int);
 839     int (*set_stream_func) (int, void *);
 840   } cuda;
 841 } acc_dispatch_t;
 842
 843 /* This structure describes accelerator device.
 844    It contains name of the corresponding libgomp plugin, function handlers for
 845    interaction with the device, ID-number of the device, and information about
 846    mapped memory.  */
 847 struct gomp_device_descr
 848 {
 849   /* Immutable data, which is only set during initialization, and which is not
 850      guarded by the lock.  */
 851
 852   /* The name of the device.  */
 853   const char *name;
 854
 855   /* Capabilities of device (supports OpenACC, OpenMP).  */
 856   unsigned int capabilities;
 857
 858   /* This is the ID number of device among devices of the same type.  */
 859   int target_id;
 860
 861   /* This is the TYPE of device.  */
 862   enum offload_target_type type;
 863
 864   /* Function handlers.  */
 865   const char *(*get_name_func) (void);
 866   unsigned int (*get_caps_func) (void);
 867   int (*get_type_func) (void);
 868   int (*get_num_devices_func) (void);
 869   void (*init_device_func) (int);
 870   void (*fini_device_func) (int);
 871   unsigned (*version_func) (void);
 872   int (*load_image_func) (int, unsigned, const void *, struct addr_pair **);
 873   void (*unload_image_func) (int, unsigned, const void *);
 874   void *(*alloc_func) (int, size_t);
 875   void (*free_func) (int, void *);
 876   void *(*dev2host_func) (int, void *, const void *, size_t);
 877   void *(*host2dev_func) (int, void *, const void *, size_t);
 878   void *(*dev2dev_func) (int, void *, const void *, size_t);
 879   void (*run_func) (int, void *, void *);
 880
 881   /* Splay tree containing information about mapped memory regions.  */
 882   struct splay_tree_s mem_map;
 883
 884   /* Mutex for the mutable data.  */
 885   gomp_mutex_t lock;
 886
 887   /* Set to true when device is initialized.  */
 888   bool is_initialized;
 889
 890   /* OpenACC-specific data and functions.  */
 891   /* This is mutable because of its mutable data_environ and target_data
 892      members.  */
 893   acc_dispatch_t openacc;
 894 };
 895
 896 /* Kind of the pragma, for which gomp_map_vars () is called.  */
 897 enum gomp_map_vars_kind
 898 {
 899   GOMP_MAP_VARS_OPENACC,
 900   GOMP_MAP_VARS_TARGET,
 901   GOMP_MAP_VARS_DATA,
 902   GOMP_MAP_VARS_ENTER_DATA
 903 };
 904
 905 extern void gomp_acc_insert_pointer (size_t, void **, size_t *, void *);
 906 extern void gomp_acc_remove_pointer (void *, bool, int, int);
 907
 908 extern struct target_mem_desc *gomp_map_vars (struct gomp_device_descr *,
 909                                               size_t, void **, void **,
 910                                               size_t *, void *, bool,
 911                                               enum gomp_map_vars_kind);
 912 extern void gomp_copy_from_async (struct target_mem_desc *);
 913 extern void gomp_unmap_vars (struct target_mem_desc *, bool);
 914 extern void gomp_init_device (struct gomp_device_descr *);
 915 extern void gomp_free_memmap (struct splay_tree_s *);
 916 extern void gomp_fini_device (struct gomp_device_descr *);
 917 extern void gomp_unload_device (struct gomp_device_descr *);
 918
 919 /* work.c */
 920
 921 extern void gomp_init_work_share (struct gomp_work_share *, bool, unsigned);
 922 extern void gomp_fini_work_share (struct gomp_work_share *);
 923 extern bool gomp_work_share_start (bool);
 924 extern void gomp_work_share_end (void);
 925 extern bool gomp_work_share_end_cancel (void);
 926 extern void gomp_work_share_end_nowait (void);
 927
 928 static inline void
 929 gomp_work_share_init_done (void)
 930 {
 931   struct gomp_thread *thr = gomp_thread ();
 932   if (__builtin_expect (thr->ts.last_work_share != NULL, 1))
 933     gomp_ptrlock_set (&thr->ts.last_work_share->next_ws, thr->ts.work_share);
 934 }
 935
 936 #ifdef HAVE_ATTRIBUTE_VISIBILITY
 937 # pragma GCC visibility pop
 938 #endif
 939
 940 /* Now that we're back to default visibility, include the globals.  */
 941 #include "libgomp_g.h"
 942
 943 /* Include omp.h by parts.  */
 944 #include "omp-lock.h"
 945 #define _LIBGOMP_OMP_LOCK_DEFINED 1
 946 #include "omp.h.in"
 947
 948 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \
 949     || !defined (HAVE_ATTRIBUTE_ALIAS) \
 950     || !defined (HAVE_AS_SYMVER_DIRECTIVE) \
 951     || !defined (PIC) \
 952     || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT)
 953 # undef LIBGOMP_GNU_SYMBOL_VERSIONING
 954 #endif
 955
 956 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING
 957 extern void gomp_init_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 958 extern void gomp_destroy_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 959 extern void gomp_set_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 960 extern void gomp_unset_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 961 extern int gomp_test_lock_30 (omp_lock_t *) __GOMP_NOTHROW;
 962 extern void gomp_init_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 963 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 964 extern void gomp_set_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 965 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 966 extern int gomp_test_nest_lock_30 (omp_nest_lock_t *) __GOMP_NOTHROW;
 967
 968 extern void gomp_init_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 969 extern void gomp_destroy_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 970 extern void gomp_set_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 971 extern void gomp_unset_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 972 extern int gomp_test_lock_25 (omp_lock_25_t *) __GOMP_NOTHROW;
 973 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 974 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 975 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 976 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 977 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 978
 979 # define strong_alias(fn, al) \
 980   extern __typeof (fn) al __attribute__ ((alias (#fn)));
 981 # define omp_lock_symver(fn) \
 982   __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
 983   __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
 984 #else
 985 # define gomp_init_lock_30 omp_init_lock
 986 # define gomp_destroy_lock_30 omp_destroy_lock
 987 # define gomp_set_lock_30 omp_set_lock
 988 # define gomp_unset_lock_30 omp_unset_lock
 989 # define gomp_test_lock_30 omp_test_lock
 990 # define gomp_init_nest_lock_30 omp_init_nest_lock
 991 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock
 992 # define gomp_set_nest_lock_30 omp_set_nest_lock
 993 # define gomp_unset_nest_lock_30 omp_unset_nest_lock
 994 # define gomp_test_nest_lock_30 omp_test_nest_lock
 995 #endif
 996
 997 #ifdef HAVE_ATTRIBUTE_VISIBILITY
 998 # define attribute_hidden __attribute__ ((visibility ("hidden")))
 999 #else
1000 # define attribute_hidden
1001 #endif
1002
1003 #ifdef HAVE_ATTRIBUTE_ALIAS
1004 # define ialias_ulp     ialias_str1(__USER_LABEL_PREFIX__)
1005 # define ialias_str1(x) ialias_str2(x)
1006 # define ialias_str2(x) #x
1007 # define ialias(fn) \
1008   extern __typeof (fn) gomp_ialias_##fn \
1009     __attribute__ ((alias (#fn))) attribute_hidden;
1010 # define ialias_redirect(fn) \
1011   extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
1012 # define ialias_call(fn) gomp_ialias_ ## fn
1013 #else
1014 # define ialias(fn)
1015 # define ialias_redirect(fn)
1016 # define ialias_call(fn) fn
1017 #endif
1018
1019 #endif /* LIBGOMP_H */