-/* Copyright (C) 2005-2015 Free Software Foundation, Inc.
+/* Copyright (C) 2005-2024 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Offloading and Multi Processing Library
#endif
#include "config.h"
-#include "gstdint.h"
+#include <stdint.h>
#include "libgomp-plugin.h"
+#include "gomp-constants.h"
+#ifdef HAVE_PTHREAD_H
#include <pthread.h>
+#endif
#include <stdbool.h>
#include <stdlib.h>
#include <stdarg.h>
+/* Needed for memset in priority_queue.c. */
+#if _LIBGOMP_CHECKING_
+# ifdef STRING_WITH_STRINGS
+# include <string.h>
+# include <strings.h>
+# else
+# ifdef HAVE_STRING_H
+# include <string.h>
+# else
+# ifdef HAVE_STRINGS_H
+# include <strings.h>
+# endif
+# endif
+# endif
+#endif
+
#ifdef HAVE_ATTRIBUTE_VISIBILITY
# pragma GCC visibility push(hidden)
#endif
MEMMODEL_SEQ_CST = 5
};
+/* alloc.c */
+
+#if defined(HAVE_ALIGNED_ALLOC) \
+ || defined(HAVE_POSIX_MEMALIGN) \
+ || defined(HAVE_MEMALIGN)
+/* Defined if gomp_aligned_alloc doesn't use fallback version
+ and free can be used instead of gomp_aligned_free. */
+#define GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC 1
+#endif
+
+#if defined(GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC) && !defined(__AMDGCN__)
+#define GOMP_USE_ALIGNED_WORK_SHARES 1
+#endif
+
+extern void *gomp_malloc (size_t) __attribute__((malloc));
+extern void *gomp_malloc_cleared (size_t) __attribute__((malloc));
+extern void *gomp_realloc (void *, size_t);
+extern void *gomp_aligned_alloc (size_t, size_t)
+ __attribute__((malloc, alloc_size (2)));
+extern void gomp_aligned_free (void *);
+
+/* Avoid conflicting prototypes of alloca() in system headers by using
+ GCC's builtin alloca(). */
+#define gomp_alloca(x) __builtin_alloca(x)
+
+/* Optimized allocators for team-specific data that will die with the team. */
+
+#ifdef __AMDGCN__
+#include "libgomp-gcn.h"
+/* The arena is initialized in config/gcn/team.c. */
+
+static inline void * __attribute__((malloc))
+team_malloc (size_t size)
+{
+ /* 4-byte align the size. */
+ size = (size + 3) & ~3;
+
+ /* Allocate directly from the arena.
+ The compiler does not support DS atomics, yet. */
+ void *result;
+ asm ("ds_add_rtn_u64 %0, %1, %2\n\ts_waitcnt 0"
+ : "=v"(result) : "v"(TEAM_ARENA_FREE), "v"(size), "e"(1L) : "memory");
+
+ /* Handle OOM. */
+ if (result + size > *(void * __lds *)TEAM_ARENA_END)
+ {
+ /* While this is experimental, let's make sure we know when OOM
+ happens. */
+ const char msg[] = "GCN team arena exhausted;"
+ " configure with GCN_TEAM_ARENA_SIZE=bytes\n";
+ write (2, msg, sizeof(msg)-1);
+
+ /* Fall back to using the heap (slowly). */
+ result = gomp_malloc (size);
+ }
+ return result;
+}
+
+static inline void * __attribute__((malloc))
+team_malloc_cleared (size_t size)
+{
+ char *result = team_malloc (size);
+
+ /* Clear the allocated memory. */
+ __builtin_memset (result, 0, size);
+
+ return result;
+}
+
+static inline void
+team_free (void *ptr)
+{
+ /* The whole arena is freed when the kernel exits.
+ However, if we fell back to using heap then we should free it.
+ It would be better if this function could be a no-op, but at least
+ LDS loads are cheap. */
+ if (ptr < *(void * __lds *)TEAM_ARENA_START
+ || ptr >= *(void * __lds *)TEAM_ARENA_END)
+ free (ptr);
+}
+#else
+#define team_malloc(...) gomp_malloc (__VA_ARGS__)
+#define team_malloc_cleared(...) gomp_malloc_cleared (__VA_ARGS__)
+#define team_free(...) free (__VA_ARGS__)
+#endif
+
+/* error.c */
+
+extern void gomp_vdebug (int, const char *, va_list);
+extern void gomp_debug (int, const char *, ...)
+ __attribute__ ((format (printf, 2, 3)));
+#define gomp_vdebug(KIND, FMT, VALIST) \
+ do { \
+ if (__builtin_expect (gomp_debug_var, 0)) \
+ (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
+ } while (0)
+#define gomp_debug(KIND, ...) \
+ do { \
+ if (__builtin_expect (gomp_debug_var, 0)) \
+ (gomp_debug) ((KIND), __VA_ARGS__); \
+ } while (0)
+extern void gomp_verror (const char *, va_list);
+extern void gomp_error (const char *, ...)
+ __attribute__ ((format (printf, 1, 2)));
+extern void gomp_vfatal (const char *, va_list)
+ __attribute__ ((noreturn));
+extern void gomp_fatal (const char *, ...)
+ __attribute__ ((noreturn, format (printf, 1, 2)));
+
+struct gomp_task;
+struct gomp_taskgroup;
+struct htab;
+
+#include "priority_queue.h"
#include "sem.h"
#include "mutex.h"
#include "bar.h"
+#include "simple-bar.h"
#include "ptrlock.h"
GFS_STATIC,
GFS_DYNAMIC,
GFS_GUIDED,
- GFS_AUTO
+ GFS_AUTO,
+ GFS_MONOTONIC = 0x80000000U
};
struct gomp_doacross_work_share
/* Likewise, but for the ull implementation. */
unsigned long long boundary_ull;
};
+ /* Pointer to extra memory if needed for lastprivate(conditional). */
+ void *extra;
/* Array of shift counts for each dimension if they can be flattened. */
unsigned int shift_counts[];
};
+/* Like struct gomp_work_share, but only the 1st cacheline of it plus
+ flexible array at the end.
+ Keep in sync with struct gomp_work_share. */
+struct gomp_work_share_1st_cacheline
+{
+ enum gomp_schedule_type sched;
+ int mode;
+ union {
+ struct {
+ long chunk_size, end, incr;
+ };
+ struct {
+ unsigned long long chunk_size_ull, end_ull, incr_ull;
+ };
+ };
+ union {
+ unsigned *ordered_team_ids;
+ struct gomp_doacross_work_share *doacross;
+ };
+ unsigned ordered_num_used, ordered_owner, ordered_cur;
+ struct gomp_work_share *next_alloc;
+ char pad[];
+};
+
struct gomp_work_share
{
/* This member records the SCHEDULE clause to be used for this construct.
are in a different cache line. */
/* This lock protects the update of the following members. */
+#ifdef GOMP_USE_ALIGNED_WORK_SHARES
gomp_mutex_t lock __attribute__((aligned (64)));
+#else
+ char pad[64 - offsetof (struct gomp_work_share_1st_cacheline, pad)];
+ gomp_mutex_t lock;
+#endif
/* This is the count of the number of threads that have exited the work
share construct. If the construct was marked nowait, they have moved on
struct gomp_work_share *next_free;
};
+ /* Task reductions for this work-sharing construct. */
+ uintptr_t *task_reductions;
+
/* If only few threads are in the team, ordered_team_ids can point
to this array which fills the padding at the end of this struct. */
unsigned inline_ordered_team_ids[0];
};
+extern char gomp_workshare_struct_check1
+ [offsetof (struct gomp_work_share_1st_cacheline, next_alloc)
+ == offsetof (struct gomp_work_share, next_alloc) ? 1 : -1];
+extern char gomp_workshare_struct_check2
+ [offsetof (struct gomp_work_share, lock) == 64 ? 1 : -1];
+
/* This structure contains all of the thread-local data associated with
a thread team. This is the data that must be saved when a thread
encounters a nested PARALLEL construct. */
unsigned place_partition_off;
unsigned place_partition_len;
+ /* Def-allocator-var ICV. */
+ uintptr_t def_allocator;
+
#ifdef HAVE_SYNC_BUILTINS
/* Number of single stmts encountered. */
unsigned long single_count;
struct target_mem_desc;
+enum gomp_icvs
+{
+ GOMP_ICV_NTEAMS = 1,
+ GOMP_ICV_SCHEDULE = 2,
+ GOMP_ICV_SCHEDULE_CHUNK_SIZE = 3,
+ GOMP_ICV_DYNAMIC = 4,
+ GOMP_ICV_TEAMS_THREAD_LIMIT = 5,
+ GOMP_ICV_THREAD_LIMIT = 6,
+ GOMP_ICV_NTHREADS = 7,
+ GOMP_ICV_NTHREADS_LIST = 8,
+ GOMP_ICV_NTHREADS_LIST_LEN = 9,
+ GOMP_ICV_BIND = 10,
+ GOMP_ICV_BIND_LIST = 11,
+ GOMP_ICV_BIND_LIST_LEN = 12,
+ GOMP_ICV_MAX_ACTIVE_LEVELS = 13,
+ GOMP_ICV_WAIT_POLICY = 14,
+ GOMP_ICV_STACKSIZE = 15,
+ GOMP_ICV_DEFAULT_DEVICE = 16,
+ GOMP_ICV_CANCELLATION = 17,
+ GOMP_ICV_DISPLAY_AFFINITY = 18,
+ GOMP_ICV_TARGET_OFFLOAD = 19,
+ GOMP_ICV_MAX_TASK_PRIORITY = 20,
+ GOMP_ICV_ALLOCATOR = 21
+};
+
+enum gomp_device_num
+{
+ GOMP_DEVICE_NUM_FOR_DEV = -1,
+ GOMP_DEVICE_NUM_FOR_ALL = -2,
+ GOMP_DEVICE_NUM_FOR_NO_SUFFIX = -3
+};
+
/* These are the OpenMP 4.0 Internal Control Variables described in
section 2.3.1. Those described as having one copy per task are
stored within the structure; those described as having one copy
int default_device_var;
unsigned int thread_limit_var;
bool dyn_var;
- bool nest_var;
+ unsigned char max_active_levels_var;
char bind_var;
/* Internal ICV. */
struct target_mem_desc *target_data;
};
+enum gomp_env_suffix
+{
+ GOMP_ENV_SUFFIX_UNKNOWN = 0,
+ GOMP_ENV_SUFFIX_NONE = 1,
+ GOMP_ENV_SUFFIX_DEV = 2,
+ GOMP_ENV_SUFFIX_ALL = 4,
+ GOMP_ENV_SUFFIX_DEV_X = 8
+};
+
+/* Struct that contains all ICVs for which we need to store initial values.
+ Keeping the initial values is needed for omp_display_env. Moreover initial
+ _DEV and _ALL variants of environment variables are also used to determine
+ actually used values for devices and for the host. */
+struct gomp_initial_icvs
+{
+ unsigned long *nthreads_var_list;
+ char *bind_var_list;
+ unsigned long nthreads_var;
+ unsigned long nthreads_var_list_len;
+ unsigned long bind_var_list_len;
+ unsigned long stacksize;
+ int run_sched_chunk_size;
+ int default_device_var;
+ int nteams_var;
+ int teams_thread_limit_var;
+ int wait_policy;
+ unsigned int thread_limit_var;
+ enum gomp_schedule_type run_sched_var;
+ bool dyn_var;
+ unsigned char max_active_levels_var;
+ char bind_var;
+};
+
+struct gomp_default_icv
+{
+ unsigned long nthreads_var;
+ enum gomp_schedule_type run_sched_var;
+ int run_sched_chunk_size;
+ int default_device_var;
+ unsigned int thread_limit_var;
+ int nteams_var;
+ int teams_thread_limit_var;
+ bool dyn_var;
+ unsigned char max_active_levels_var;
+ char bind_var;
+};
+
+/* DEVICE_NUM "-1" is reserved for "_DEV" icvs.
+ DEVICE_NUM "-2" is reserved for "_ALL" icvs.
+ DEVICE_NUM "-3" is reserved for ICVs without suffix.
+ Non-negative DEVICE_NUM is for "_DEV_X" icvs. */
+struct gomp_icv_list
+{
+ int device_num;
+ uint32_t flags;
+ struct gomp_initial_icvs icvs;
+ struct gomp_icv_list *next;
+};
+
+struct gomp_offload_icvs
+{
+ int device_num;
+ int default_device;
+ int nteams;
+ int teams_thread_limit;
+};
+
+struct gomp_offload_icv_list
+{
+ int device_num;
+ struct gomp_offload_icvs icvs;
+ struct gomp_offload_icv_list *next;
+};
+
+enum gomp_target_offload_t
+{
+ GOMP_TARGET_OFFLOAD_DEFAULT,
+ GOMP_TARGET_OFFLOAD_MANDATORY,
+ GOMP_TARGET_OFFLOAD_DISABLED
+};
+
+#define gomp_supported_active_levels UCHAR_MAX
+
extern struct gomp_task_icv gomp_global_icv;
#ifndef HAVE_SYNC_BUILTINS
extern gomp_mutex_t gomp_managed_threads_lock;
#endif
-extern unsigned long gomp_max_active_levels_var;
extern bool gomp_cancel_var;
+extern enum gomp_target_offload_t gomp_target_offload_var;
+extern int gomp_max_task_priority_var;
extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var;
extern unsigned long gomp_available_cpus, gomp_managed_threads;
extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len;
extern unsigned long gomp_bind_var_list_len;
extern void **gomp_places_list;
extern unsigned long gomp_places_list_len;
+extern unsigned int gomp_num_teams_var;
+extern int gomp_nteams_var;
+extern int gomp_teams_thread_limit_var;
extern int gomp_debug_var;
+extern bool gomp_display_affinity_var;
+extern char *gomp_affinity_format_var;
+extern size_t gomp_affinity_format_len;
+extern uintptr_t gomp_def_allocator;
+extern const struct gomp_default_icv gomp_default_icv_values;
+extern struct gomp_icv_list *gomp_initial_icv_list;
+extern struct gomp_offload_icv_list *gomp_offload_icv_list;
extern int goacc_device_num;
extern char *goacc_device_type;
+extern int goacc_default_dims[GOMP_DIM_MAX];
enum gomp_task_kind
{
/* Task created by GOMP_task and waiting to be run. */
GOMP_TASK_WAITING,
/* Task currently executing or scheduled and about to execute. */
- GOMP_TASK_TIED
+ GOMP_TASK_TIED,
+ /* Used for target tasks that have vars mapped and async run started,
+ but not yet completed. Once that completes, they will be readded
+ into the queues as GOMP_TASK_WAITING in order to perform the var
+ unmapping. */
+ GOMP_TASK_ASYNC_RUNNING,
+ /* Task that has finished executing but is waiting for its
+ completion event to be fulfilled. */
+ GOMP_TASK_DETACHED
};
-struct gomp_task;
-struct gomp_taskgroup;
-struct htab;
-
struct gomp_task_depend_entry
{
/* Address of dependency. */
struct gomp_task_depend_entry *prev;
/* Task that provides the dependency in ADDR. */
struct gomp_task *task;
- /* Depend entry is of type "IN". */
- bool is_in;
+ /* Depend entry is of type "IN" (1) or "INOUTSET" (2). */
+ unsigned char is_in;
bool redundant;
bool redundant_out;
};
{
bool in_taskwait;
bool in_depend_wait;
+ /* Number of tasks we are waiting for. */
size_t n_depend;
- struct gomp_task *last_parent_depends_on;
gomp_sem_t taskwait_sem;
};
struct gomp_task
{
- /* Parent circular list. See children description below. */
+ /* Parent of this task. */
struct gomp_task *parent;
- /* Circular list representing the children of this task.
-
- In this list we first have parent_depends_on ready to run tasks,
- then !parent_depends_on ready to run tasks, and finally already
- running tasks. */
- struct gomp_task *children;
- struct gomp_task *next_child;
- struct gomp_task *prev_child;
- /* Circular task_queue in `struct gomp_team'.
-
- GOMP_TASK_WAITING tasks come before GOMP_TASK_TIED tasks. */
- struct gomp_task *next_queue;
- struct gomp_task *prev_queue;
- /* Circular queue in gomp_taskgroup->children.
-
- GOMP_TASK_WAITING tasks come before GOMP_TASK_TIED tasks. */
- struct gomp_task *next_taskgroup;
- struct gomp_task *prev_taskgroup;
+ /* Children of this task. */
+ struct priority_queue children_queue;
/* Taskgroup this task belongs in. */
struct gomp_taskgroup *taskgroup;
/* Tasks that depend on this task. */
struct gomp_dependers_vec *dependers;
struct htab *depend_hash;
struct gomp_taskwait *taskwait;
+ /* Last depend({,in}out:omp_all_memory) child if any. */
+ struct gomp_task *depend_all_memory;
/* Number of items in DEPEND. */
size_t depend_count;
- /* Number of tasks in the DEPENDERS field above. */
+ /* Number of tasks this task depends on. Once this counter reaches
+ 0, we have no unsatisfied dependencies, and this task can be put
+ into the various queues to be scheduled. */
size_t num_dependees;
+
+ union {
+ /* Valid only if deferred_p is false. */
+ gomp_sem_t *completion_sem;
+ /* Valid only if deferred_p is true. Set to the team that executes the
+ task if the task is detached and the completion event has yet to be
+ fulfilled. */
+ struct gomp_team *detach_team;
+ };
+ bool deferred_p;
+
+ /* Priority of this task. */
+ int priority;
+ /* The priority node for this task in each of the different queues.
+ We put this here to avoid allocating space for each priority
+ node. Then we play offsetof() games to convert between pnode[]
+ entries and the gomp_task in which they reside. */
+ struct priority_node pnode[3];
+
struct gomp_task_icv icv;
void (*fn) (void *);
void *fn_data;
struct gomp_task_depend_entry depend[];
};
+/* This structure describes a single #pragma omp taskgroup. */
+
struct gomp_taskgroup
{
struct gomp_taskgroup *prev;
- /* Circular list of tasks that belong in this taskgroup.
-
- Tasks are chained by next/prev_taskgroup within gomp_task, and
- are sorted by GOMP_TASK_WAITING tasks, and then GOMP_TASK_TIED
- tasks. */
- struct gomp_task *children;
+ /* Queue of tasks that belong in this taskgroup. */
+ struct priority_queue taskgroup_queue;
+ uintptr_t *reductions;
bool in_taskgroup_wait;
bool cancelled;
+ bool workshare;
gomp_sem_t taskgroup_sem;
size_t num_children;
};
+/* Various state of OpenMP async offloading tasks. */
+enum gomp_target_task_state
+{
+ GOMP_TARGET_TASK_DATA,
+ GOMP_TARGET_TASK_BEFORE_MAP,
+ GOMP_TARGET_TASK_FALLBACK,
+ GOMP_TARGET_TASK_READY_TO_RUN,
+ GOMP_TARGET_TASK_RUNNING,
+ GOMP_TARGET_TASK_FINISHED
+};
+
+/* This structure describes a target task. */
+
struct gomp_target_task
{
struct gomp_device_descr *devicep;
size_t *sizes;
unsigned short *kinds;
unsigned int flags;
+ enum gomp_target_task_state state;
+ struct target_mem_desc *tgt;
+ struct gomp_task *task;
+ struct gomp_team *team;
+ /* Device-specific target arguments. */
+ void **args;
void *hostaddrs[];
};
struct gomp_work_share work_shares[8];
gomp_mutex_t task_lock;
- /* Scheduled tasks. Chain fields are next/prev_queue within a
- gomp_task. */
- struct gomp_task *task_queue;
+ /* Scheduled tasks. */
+ struct priority_queue task_queue;
/* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */
unsigned int task_count;
/* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */
int work_share_cancelled;
int team_cancelled;
+ /* Number of tasks waiting for their completion event to be fulfilled. */
+ unsigned int task_detach_count;
+
/* This array contains structures for implicit tasks. */
struct gomp_task implicit_task[];
};
/* User pthread thread pool */
struct gomp_thread_pool *thread_pool;
+
+#ifdef LIBGOMP_USE_PTHREADS
+ /* omp_get_num_teams () - 1. */
+ unsigned int num_teams;
+
+ /* omp_get_team_num (). */
+ unsigned int team_num;
+#endif
+
+#if defined(LIBGOMP_USE_PTHREADS) \
+ && (!defined(HAVE_TLS) \
+ || !defined(__GLIBC__) \
+ || !defined(USING_INITIAL_EXEC_TLS))
+ /* pthread_t of the thread containing this gomp_thread.
+ On Linux when using initial-exec TLS,
+ (typeof (pthread_t)) gomp_thread () - pthread_self ()
+ is constant in all threads, so we can optimize and not
+ store it. */
+#define GOMP_NEEDS_THREAD_HANDLE 1
+ pthread_t handle;
+#endif
};
/* Number of threads running in this contention group. */
unsigned long threads_busy;
- /* This barrier holds and releases threads waiting in threads. */
- gomp_barrier_t threads_dock;
+ /* This barrier holds and releases threads waiting in thread pools. */
+ gomp_simple_barrier_t threads_dock;
};
enum gomp_cancel_kind
/* ... and here is that TLS data. */
-#if defined HAVE_TLS || defined USE_EMUTLS
+#if defined __nvptx__
+extern struct gomp_thread *nvptx_thrs __attribute__((shared));
+static inline struct gomp_thread *gomp_thread (void)
+{
+ int tid;
+ asm ("mov.u32 %0, %%tid.y;" : "=r" (tid));
+ return nvptx_thrs + tid;
+}
+#elif defined __AMDGCN__
+static inline struct gomp_thread *gcn_thrs (void)
+{
+ /* The value is at the bottom of LDS. */
+ struct gomp_thread * __lds *thrs = (struct gomp_thread * __lds *)4;
+ return *thrs;
+}
+static inline void set_gcn_thrs (struct gomp_thread *val)
+{
+ /* The value is at the bottom of LDS. */
+ struct gomp_thread * __lds *thrs = (struct gomp_thread * __lds *)4;
+ *thrs = val;
+}
+static inline struct gomp_thread *gomp_thread (void)
+{
+ int tid = __builtin_gcn_dim_pos(1);
+ return gcn_thrs () + tid;
+}
+#elif defined HAVE_TLS || defined USE_EMUTLS
extern __thread struct gomp_thread gomp_tls_data;
static inline struct gomp_thread *gomp_thread (void)
{
return &gomp_global_icv;
}
+#ifdef LIBGOMP_USE_PTHREADS
/* The attributes to be used during thread creation. */
extern pthread_attr_t gomp_thread_attr;
extern pthread_key_t gomp_thread_destructor;
+#endif
/* Function prototypes. */
/* affinity.c */
extern void gomp_init_affinity (void);
+#ifdef LIBGOMP_USE_PTHREADS
extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int);
+#endif
extern void **gomp_affinity_alloc (unsigned long, bool);
extern void gomp_affinity_init_place (void *);
extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
extern bool gomp_affinity_init_level (int, unsigned long, bool);
extern void gomp_affinity_print_place (void *);
extern void gomp_get_place_proc_ids_8 (int, int64_t *);
+extern void gomp_display_affinity_place (char *, size_t, size_t *, int);
-/* alloc.c */
+/* affinity-fmt.c */
-extern void *gomp_malloc (size_t) __attribute__((malloc));
-extern void *gomp_malloc_cleared (size_t) __attribute__((malloc));
-extern void *gomp_realloc (void *, size_t);
-
-/* Avoid conflicting prototypes of alloca() in system headers by using
- GCC's builtin alloca(). */
-#define gomp_alloca(x) __builtin_alloca(x)
+extern bool gomp_print_string (const char *str, size_t len);
+extern void gomp_set_affinity_format (const char *, size_t);
+extern void gomp_display_string (char *, size_t, size_t *, const char *,
+ size_t);
+#ifdef LIBGOMP_USE_PTHREADS
+typedef pthread_t gomp_thread_handle;
+#else
+typedef struct {} gomp_thread_handle;
+#endif
+extern size_t gomp_display_affinity (char *, size_t, const char *,
+ gomp_thread_handle,
+ struct gomp_team_state *, unsigned int);
+extern void gomp_display_affinity_thread (gomp_thread_handle,
+ struct gomp_team_state *,
+ unsigned int) __attribute__((cold));
-/* error.c */
+/* env.c */
-extern void gomp_vdebug (int, const char *, va_list);
-extern void gomp_debug (int, const char *, ...)
- __attribute__ ((format (printf, 2, 3)));
-#define gomp_vdebug(KIND, FMT, VALIST) \
- do { \
- if (__builtin_expect (gomp_debug_var, 0)) \
- (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
- } while (0)
-#define gomp_debug(KIND, ...) \
- do { \
- if (__builtin_expect (gomp_debug_var, 0)) \
- (gomp_debug) ((KIND), __VA_ARGS__); \
- } while (0)
-extern void gomp_verror (const char *, va_list);
-extern void gomp_error (const char *, ...)
- __attribute__ ((format (printf, 1, 2)));
-extern void gomp_vfatal (const char *, va_list)
- __attribute__ ((noreturn));
-extern void gomp_fatal (const char *, ...)
- __attribute__ ((noreturn, format (printf, 1, 2)));
+extern struct gomp_icv_list *gomp_get_initial_icv_item (int dev_num);
+extern bool gomp_get_icv_flag (uint32_t value, enum gomp_icvs icv);
/* iter.c */
extern void gomp_ordered_static_init (void);
extern void gomp_ordered_static_next (void);
extern void gomp_ordered_sync (void);
-extern void gomp_doacross_init (unsigned, long *, long);
+extern void gomp_doacross_init (unsigned, long *, long, size_t);
extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
- unsigned long long);
+ unsigned long long, size_t);
/* parallel.c */
extern void gomp_end_task (void);
extern void gomp_barrier_handle_tasks (gomp_barrier_state_t);
extern void gomp_task_maybe_wait_for_dependencies (void **);
-extern void gomp_create_target_task (struct gomp_device_descr *,
+extern bool gomp_create_target_task (struct gomp_device_descr *,
void (*) (void *), size_t, void **,
size_t *, unsigned short *, unsigned int,
- void **);
+ void **, void **,
+ enum gomp_target_task_state);
+extern struct gomp_taskgroup *gomp_parallel_reduction_register (uintptr_t *,
+ unsigned);
+extern void gomp_workshare_taskgroup_start (void);
+extern void gomp_workshare_task_reduction_register (uintptr_t *, uintptr_t *);
static void inline
gomp_finish_task (struct gomp_task *task)
extern struct gomp_team *gomp_new_team (unsigned);
extern void gomp_team_start (void (*) (void *), void *, unsigned,
- unsigned, struct gomp_team *);
+ unsigned, struct gomp_team *,
+ struct gomp_taskgroup *);
extern void gomp_team_end (void);
extern void gomp_free_thread (void *);
+extern int gomp_pause_host (void);
/* target.c */
extern void gomp_init_targets_once (void);
extern int gomp_get_num_devices (void);
-extern void gomp_target_task_fn (void *);
+extern bool gomp_target_task_fn (void *);
+extern void gomp_target_rev (uint64_t, uint64_t, uint64_t, uint64_t, uint64_t,
+ int, struct goacc_asyncqueue *);
+/* Splay tree definitions. */
typedef struct splay_tree_node_s *splay_tree_node;
typedef struct splay_tree_s *splay_tree;
typedef struct splay_tree_key_s *splay_tree_key;
bool copy_from;
/* True if data always should be copied from device to host at the end. */
bool always_copy_from;
+ /* True if this is for OpenACC 'attach'. */
+ bool is_attach;
+ /* If GOMP_MAP_TO_PSET had a NULL pointer; used for Fortran descriptors,
+ which were initially unallocated. */
+ bool has_null_ptr_assoc;
/* Relative offset against key host_start. */
uintptr_t offset;
/* Actual length. */
uintptr_t length;
};
+struct target_mem_desc;
+
+/* Special value for refcount - mask to indicate existence of special
+ values. Right now we allocate 3 bits. */
+#define REFCOUNT_SPECIAL (~(uintptr_t) 0x7)
+
+/* Special value for refcount - infinity. */
+#define REFCOUNT_INFINITY (REFCOUNT_SPECIAL | 0)
+/* Special value for refcount - tgt_offset contains target address of the
+ artificial pointer to "omp declare target link" object. */
+#define REFCOUNT_LINK (REFCOUNT_SPECIAL | 1)
+
+/* Special value for refcount - structure element sibling list items.
+ All such key refounts have REFCOUNT_STRUCTELEM bits set, with _FLAG_FIRST
+ and _FLAG_LAST indicating first and last in the created sibling sequence. */
+#define REFCOUNT_STRUCTELEM (REFCOUNT_SPECIAL | 4)
+#define REFCOUNT_STRUCTELEM_P(V) \
+ (((V) & REFCOUNT_STRUCTELEM) == REFCOUNT_STRUCTELEM)
+/* The first leading key with _FLAG_FIRST set houses the actual reference count
+ in the structelem_refcount field. Other siblings point to this counter value
+ through its structelem_refcount_ptr field. */
+#define REFCOUNT_STRUCTELEM_FLAG_FIRST (1)
+/* The last key in the sibling sequence has this set. This is required to
+ indicate the sequence boundary, when we remove the structure sibling list
+ from the map. */
+#define REFCOUNT_STRUCTELEM_FLAG_LAST (2)
+
+#define REFCOUNT_STRUCTELEM_FIRST_P(V) \
+ (REFCOUNT_STRUCTELEM_P (V) && ((V) & REFCOUNT_STRUCTELEM_FLAG_FIRST))
+#define REFCOUNT_STRUCTELEM_LAST_P(V) \
+ (REFCOUNT_STRUCTELEM_P (V) && ((V) & REFCOUNT_STRUCTELEM_FLAG_LAST))
+
+/* Special offset values. */
+#define OFFSET_INLINED (~(uintptr_t) 0)
+#define OFFSET_POINTER (~(uintptr_t) 1)
+#define OFFSET_STRUCT (~(uintptr_t) 2)
+
+/* Auxiliary structure for infrequently-used or API-specific data. */
+
+struct splay_tree_aux {
+ /* Pointer to the original mapping of "omp declare target link" object. */
+ splay_tree_key link_key;
+ /* For a block with attached pointers, the attachment counters for each.
+ Only used for OpenACC. */
+ uintptr_t *attach_count;
+};
+
+struct splay_tree_key_s {
+ /* Address of the host object. */
+ uintptr_t host_start;
+ /* Address immediately after the host object. */
+ uintptr_t host_end;
+ /* Descriptor of the target memory. */
+ struct target_mem_desc *tgt;
+ /* Offset from tgt->tgt_start to the start of the target object. */
+ uintptr_t tgt_offset;
+ /* Reference count. */
+ uintptr_t refcount;
+ union {
+ /* Dynamic reference count. */
+ uintptr_t dynamic_refcount;
+
+ /* Unified reference count for structure element siblings, this is used
+ when REFCOUNT_STRUCTELEM_FIRST_P(k->refcount) == true, the first sibling
+ in a structure element sibling list item sequence. */
+ uintptr_t structelem_refcount;
+
+ /* When REFCOUNT_STRUCTELEM_P (k->refcount) == true, this field points
+ into the (above) structelem_refcount field of the _FIRST splay_tree_key,
+ the first key in the created sequence. All structure element siblings
+ share a single refcount in this manner. Since these two fields won't be
+ used at the same time, they are stashed in a union. */
+ uintptr_t *structelem_refcount_ptr;
+ };
+ struct splay_tree_aux *aux;
+};
+
+/* The comparison function. */
+
+static inline int
+splay_compare (splay_tree_key x, splay_tree_key y)
+{
+ if (x->host_start == x->host_end
+ && y->host_start == y->host_end)
+ return 0;
+ if (x->host_end <= y->host_start)
+ return -1;
+ if (x->host_start >= y->host_end)
+ return 1;
+ return 0;
+}
+
+#include "splay-tree.h"
+
+/* Reverse offload splay-tree handling (functions only). */
+
+struct reverse_splay_tree_key_s {
+ /* Address of the device object. */
+ uint64_t dev;
+ splay_tree_key k;
+};
+
+typedef struct reverse_splay_tree_node_s *reverse_splay_tree_node;
+typedef struct reverse_splay_tree_s *reverse_splay_tree;
+typedef struct reverse_splay_tree_key_s *reverse_splay_tree_key;
+
+static inline int
+reverse_splay_compare (reverse_splay_tree_key x, reverse_splay_tree_key y)
+{
+ if (x->dev < y->dev)
+ return -1;
+ if (x->dev > y->dev)
+ return 1;
+ return 0;
+}
+
+#define splay_tree_prefix reverse
+#define splay_tree_static
+#include "splay-tree.h"
+
+/* Indirect target function splay-tree handling. */
+
+struct indirect_splay_tree_key_s {
+ uint64_t host_addr, target_addr;
+};
+
+typedef struct indirect_splay_tree_node_s *indirect_splay_tree_node;
+typedef struct indirect_splay_tree_s *indirect_splay_tree;
+typedef struct indirect_splay_tree_key_s *indirect_splay_tree_key;
+
+static inline int
+indirect_splay_compare (indirect_splay_tree_key x, indirect_splay_tree_key y)
+{
+ if (x->host_addr < y->host_addr)
+ return -1;
+ if (x->host_addr > y->host_addr)
+ return 1;
+ return 0;
+}
+
+#define splay_tree_prefix indirect
+#include "splay-tree.h"
+
struct target_mem_desc {
/* Reference count. */
uintptr_t refcount;
/* All the splay nodes allocated together. */
splay_tree_node array;
+ /* Likewise for the reverse lookup device->host for reverse offload. */
+ reverse_splay_tree_node rev_array;
/* Start of the target region. */
uintptr_t tgt_start;
/* End of the targer region. */
struct target_var_desc list[];
};
-/* Special value for refcount - infinity. */
-#define REFCOUNT_INFINITY (~(uintptr_t) 0)
-
-struct splay_tree_key_s {
- /* Address of the host object. */
- uintptr_t host_start;
- /* Address immediately after the host object. */
- uintptr_t host_end;
- /* Descriptor of the target memory. */
- struct target_mem_desc *tgt;
- /* Offset from tgt->tgt_start to the start of the target object. */
- uintptr_t tgt_offset;
- /* Reference count. */
- uintptr_t refcount;
- /* Asynchronous reference count. */
- uintptr_t async_refcount;
-};
-
-#include "splay-tree.h"
typedef struct acc_dispatch_t
{
- /* This is a linked list of data mapped using the
- acc_map_data/acc_unmap_data or "acc enter data"/"acc exit data" pragmas.
- Unlike mapped_data in the goacc_thread struct, unmapping can
- happen out-of-order with respect to mapping. */
- /* This is guarded by the lock in the "outer" struct gomp_device_descr. */
- struct target_mem_desc *data_environ;
-
/* Execute. */
- void (*exec_func) (void (*) (void *), size_t, void **, void **, size_t *,
- unsigned short *, int, unsigned *, void *);
-
- /* Async cleanup callback registration. */
- void (*register_async_cleanup_func) (void *);
-
- /* Asynchronous routines. */
- int (*async_test_func) (int);
- int (*async_test_all_func) (void);
- void (*async_wait_func) (int);
- void (*async_wait_async_func) (int, int);
- void (*async_wait_all_func) (void);
- void (*async_wait_all_async_func) (int);
- void (*async_set_async_func) (int);
+ __typeof (GOMP_OFFLOAD_openacc_exec) *exec_func;
/* Create/destroy TLS data. */
- void *(*create_thread_data_func) (int);
- void (*destroy_thread_data_func) (void *);
+ __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func;
+ __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data)
+ *destroy_thread_data_func;
+
+ struct {
+ /* Once created and put into the "active" list, asyncqueues are then never
+ destructed and removed from the "active" list, other than if the TODO
+ device is shut down. */
+ gomp_mutex_t lock;
+ int nasyncqueue;
+ struct goacc_asyncqueue **asyncqueue;
+ struct goacc_asyncqueue_list *active;
+
+ __typeof (GOMP_OFFLOAD_openacc_async_construct) *construct_func;
+ __typeof (GOMP_OFFLOAD_openacc_async_destruct) *destruct_func;
+ __typeof (GOMP_OFFLOAD_openacc_async_test) *test_func;
+ __typeof (GOMP_OFFLOAD_openacc_async_synchronize) *synchronize_func;
+ __typeof (GOMP_OFFLOAD_openacc_async_serialize) *serialize_func;
+ __typeof (GOMP_OFFLOAD_openacc_async_queue_callback) *queue_callback_func;
+
+ __typeof (GOMP_OFFLOAD_openacc_async_exec) *exec_func;
+ __typeof (GOMP_OFFLOAD_openacc_async_dev2host) *dev2host_func;
+ __typeof (GOMP_OFFLOAD_openacc_async_host2dev) *host2dev_func;
+ } async;
+
+ __typeof (GOMP_OFFLOAD_openacc_get_property) *get_property_func;
/* NVIDIA target specific routines. */
struct {
- void *(*get_current_device_func) (void);
- void *(*get_current_context_func) (void);
- void *(*get_stream_func) (int);
- int (*set_stream_func) (int, void *);
+ __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device)
+ *get_current_device_func;
+ __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context)
+ *get_current_context_func;
+ __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream) *get_stream_func;
+ __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream) *set_stream_func;
} cuda;
} acc_dispatch_t;
+/* Various state of the accelerator device. */
+enum gomp_device_state
+{
+ GOMP_DEVICE_UNINITIALIZED,
+ GOMP_DEVICE_INITIALIZED,
+ GOMP_DEVICE_FINALIZED
+};
+
/* This structure describes accelerator device.
It contains name of the corresponding libgomp plugin, function handlers for
interaction with the device, ID-number of the device, and information about
enum offload_target_type type;
/* Function handlers. */
- const char *(*get_name_func) (void);
- unsigned int (*get_caps_func) (void);
- int (*get_type_func) (void);
- int (*get_num_devices_func) (void);
- void (*init_device_func) (int);
- void (*fini_device_func) (int);
- unsigned (*version_func) (void);
- int (*load_image_func) (int, unsigned, const void *, struct addr_pair **);
- void (*unload_image_func) (int, unsigned, const void *);
- void *(*alloc_func) (int, size_t);
- void (*free_func) (int, void *);
- void *(*dev2host_func) (int, void *, const void *, size_t);
- void *(*host2dev_func) (int, void *, const void *, size_t);
- void *(*dev2dev_func) (int, void *, const void *, size_t);
- void (*run_func) (int, void *, void *);
+ __typeof (GOMP_OFFLOAD_get_name) *get_name_func;
+ __typeof (GOMP_OFFLOAD_get_caps) *get_caps_func;
+ __typeof (GOMP_OFFLOAD_get_type) *get_type_func;
+ __typeof (GOMP_OFFLOAD_get_num_devices) *get_num_devices_func;
+ __typeof (GOMP_OFFLOAD_init_device) *init_device_func;
+ __typeof (GOMP_OFFLOAD_fini_device) *fini_device_func;
+ __typeof (GOMP_OFFLOAD_version) *version_func;
+ __typeof (GOMP_OFFLOAD_load_image) *load_image_func;
+ __typeof (GOMP_OFFLOAD_unload_image) *unload_image_func;
+ __typeof (GOMP_OFFLOAD_alloc) *alloc_func;
+ __typeof (GOMP_OFFLOAD_free) *free_func;
+ __typeof (GOMP_OFFLOAD_dev2host) *dev2host_func;
+ __typeof (GOMP_OFFLOAD_host2dev) *host2dev_func;
+ __typeof (GOMP_OFFLOAD_memcpy2d) *memcpy2d_func;
+ __typeof (GOMP_OFFLOAD_memcpy3d) *memcpy3d_func;
+ __typeof (GOMP_OFFLOAD_dev2dev) *dev2dev_func;
+ __typeof (GOMP_OFFLOAD_can_run) *can_run_func;
+ __typeof (GOMP_OFFLOAD_run) *run_func;
+ __typeof (GOMP_OFFLOAD_async_run) *async_run_func;
/* Splay tree containing information about mapped memory regions. */
struct splay_tree_s mem_map;
+ struct reverse_splay_tree_s mem_map_rev;
/* Mutex for the mutable data. */
gomp_mutex_t lock;
- /* Set to true when device is initialized. */
- bool is_initialized;
+ /* Current state of the device. OpenACC allows to move from INITIALIZED state
+ back to UNINITIALIZED state. OpenMP allows only to move from INITIALIZED
+ to FINALIZED state (at program shutdown). */
+ enum gomp_device_state state;
/* OpenACC-specific data and functions. */
- /* This is mutable because of its mutable data_environ and target_data
- members. */
+ /* This is mutable because of its mutable target_data member. */
acc_dispatch_t openacc;
};
/* Kind of the pragma, for which gomp_map_vars () is called. */
enum gomp_map_vars_kind
{
- GOMP_MAP_VARS_OPENACC,
- GOMP_MAP_VARS_TARGET,
- GOMP_MAP_VARS_DATA,
- GOMP_MAP_VARS_ENTER_DATA
+ GOMP_MAP_VARS_OPENACC = 1,
+ GOMP_MAP_VARS_TARGET = 2,
+ GOMP_MAP_VARS_DATA = 4,
+ GOMP_MAP_VARS_ENTER_DATA = 8
};
-extern void gomp_acc_insert_pointer (size_t, void **, size_t *, void *);
-extern void gomp_acc_remove_pointer (void *, bool, int, int);
-
-extern struct target_mem_desc *gomp_map_vars (struct gomp_device_descr *,
- size_t, void **, void **,
- size_t *, void *, bool,
- enum gomp_map_vars_kind);
-extern void gomp_copy_from_async (struct target_mem_desc *);
-extern void gomp_unmap_vars (struct target_mem_desc *, bool);
+extern void gomp_acc_declare_allocate (bool, size_t, void **, size_t *,
+ unsigned short *);
+struct gomp_coalesce_buf;
+extern void gomp_copy_host2dev (struct gomp_device_descr *,
+ struct goacc_asyncqueue *, void *, const void *,
+ size_t, bool, struct gomp_coalesce_buf *);
+extern void gomp_copy_dev2host (struct gomp_device_descr *,
+ struct goacc_asyncqueue *, void *, const void *,
+ size_t);
+extern uintptr_t gomp_map_val (struct target_mem_desc *, void **, size_t);
+extern void gomp_attach_pointer (struct gomp_device_descr *,
+ struct goacc_asyncqueue *, splay_tree,
+ splay_tree_key, uintptr_t, size_t,
+ struct gomp_coalesce_buf *, bool);
+extern void gomp_detach_pointer (struct gomp_device_descr *,
+ struct goacc_asyncqueue *, splay_tree_key,
+ uintptr_t, bool, struct gomp_coalesce_buf *);
+extern struct target_mem_desc *goacc_map_vars (struct gomp_device_descr *,
+ struct goacc_asyncqueue *,
+ size_t, void **, void **,
+ size_t *, void *, bool,
+ enum gomp_map_vars_kind);
+extern void goacc_unmap_vars (struct target_mem_desc *, bool,
+ struct goacc_asyncqueue *);
extern void gomp_init_device (struct gomp_device_descr *);
-extern void gomp_free_memmap (struct splay_tree_s *);
-extern void gomp_fini_device (struct gomp_device_descr *);
+extern bool gomp_fini_device (struct gomp_device_descr *);
extern void gomp_unload_device (struct gomp_device_descr *);
+extern bool gomp_remove_var (struct gomp_device_descr *, splay_tree_key);
+extern void gomp_remove_var_async (struct gomp_device_descr *, splay_tree_key,
+ struct goacc_asyncqueue *);
/* work.c */
-extern void gomp_init_work_share (struct gomp_work_share *, bool, unsigned);
+extern void gomp_init_work_share (struct gomp_work_share *, size_t, unsigned);
extern void gomp_fini_work_share (struct gomp_work_share *);
-extern bool gomp_work_share_start (bool);
+extern bool gomp_work_share_start (size_t);
extern void gomp_work_share_end (void);
extern bool gomp_work_share_end_cancel (void);
extern void gomp_work_share_end_nowait (void);
extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
-# define strong_alias(fn, al) \
- extern __typeof (fn) al __attribute__ ((alias (#fn)));
# define omp_lock_symver(fn) \
__asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
__asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
# define attribute_hidden
#endif
+#if __GNUC__ >= 9
+# define HAVE_ATTRIBUTE_COPY
+#endif
+
+#ifdef HAVE_ATTRIBUTE_COPY
+# define attribute_copy(arg) __attribute__ ((copy (arg)))
+#else
+# define attribute_copy(arg)
+#endif
+
#ifdef HAVE_ATTRIBUTE_ALIAS
+# define strong_alias(fn, al) \
+ extern __typeof (fn) al __attribute__ ((alias (#fn))) attribute_copy (fn);
+
# define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__)
# define ialias_str1(x) ialias_str2(x)
# define ialias_str2(x) #x
# define ialias(fn) \
extern __typeof (fn) gomp_ialias_##fn \
- __attribute__ ((alias (#fn))) attribute_hidden;
+ __attribute__ ((alias (#fn))) attribute_hidden attribute_copy (fn);
# define ialias_redirect(fn) \
extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
# define ialias_call(fn) gomp_ialias_ ## fn
# define ialias_call(fn) fn
#endif
+/* Helper function for priority_node_to_task() and
+ task_to_priority_node().
+
+ Return the offset from a task to its priority_node entry. The
+ priority_node entry is has a type of TYPE. */
+
+static inline size_t
+priority_queue_offset (enum priority_queue_type type)
+{
+ return offsetof (struct gomp_task, pnode[(int) type]);
+}
+
+/* Return the task associated with a priority NODE of type TYPE. */
+
+static inline struct gomp_task *
+priority_node_to_task (enum priority_queue_type type,
+ struct priority_node *node)
+{
+ return (struct gomp_task *) ((char *) node - priority_queue_offset (type));
+}
+
+/* Return the priority node of type TYPE for a given TASK. */
+
+static inline struct priority_node *
+task_to_priority_node (enum priority_queue_type type,
+ struct gomp_task *task)
+{
+ return (struct priority_node *) ((char *) task
+ + priority_queue_offset (type));
+}
+
+#ifdef LIBGOMP_USE_PTHREADS
+static inline gomp_thread_handle
+gomp_thread_self (void)
+{
+ return pthread_self ();
+}
+
+static inline gomp_thread_handle
+gomp_thread_to_pthread_t (struct gomp_thread *thr)
+{
+ struct gomp_thread *this_thr = gomp_thread ();
+ if (thr == this_thr)
+ return pthread_self ();
+#ifdef GOMP_NEEDS_THREAD_HANDLE
+ return thr->handle;
+#else
+ /* On Linux with initial-exec TLS, the pthread_t of the thread containing
+ thr can be computed from thr, this_thr and pthread_self (),
+ as the distance between this_thr and pthread_self () is constant. */
+ return pthread_self () + ((uintptr_t) thr - (uintptr_t) this_thr);
+#endif
+}
+#else
+static inline gomp_thread_handle
+gomp_thread_self (void)
+{
+ return (gomp_thread_handle) {};
+}
+
+static inline gomp_thread_handle
+gomp_thread_to_pthread_t (struct gomp_thread *thr)
+{
+ (void) thr;
+ return gomp_thread_self ();
+}
+#endif
+
#endif /* LIBGOMP_H */
projects / thirdparty / gcc.git / blobdiff