-/* Copyright (C) 2013-2023 Free Software Foundation, Inc.
+/* Copyright (C) 2013-2024 Free Software Foundation, Inc.
Contributed by Jakub Jelinek <jakub@redhat.com>.
This file is part of the GNU Offloading and Multi Processing Library
/* Define another splay tree instantiation - for reverse offload. */
#define splay_tree_prefix reverse
+#define splay_tree_static
#define splay_tree_c
#include "splay-tree.h"
static struct gomp_device_descr *
resolve_device (int device_id, bool remapped)
{
+ /* Get number of devices and thus ensure that 'gomp_init_targets_once' was
+ called, which must be done before using default_device_var. */
+ int num_devices = gomp_get_num_devices ();
+
if (remapped && device_id == GOMP_DEVICE_ICV)
{
struct gomp_task_icv *icv = gomp_icv (false);
if (device_id == (remapped ? GOMP_DEVICE_HOST_FALLBACK
: omp_initial_device))
return NULL;
- if (device_id == omp_invalid_device)
+ if (gomp_target_offload_var == GOMP_TARGET_OFFLOAD_MANDATORY
+ && num_devices == 0)
+ gomp_fatal ("OMP_TARGET_OFFLOAD is set to MANDATORY, "
+ "but only the host device is available");
+ else if (device_id == omp_invalid_device)
gomp_fatal ("omp_invalid_device encountered");
else if (gomp_target_offload_var == GOMP_TARGET_OFFLOAD_MANDATORY)
gomp_fatal ("OMP_TARGET_OFFLOAD is set to MANDATORY, "
return NULL;
}
- else if (device_id >= gomp_get_num_devices ())
+ else if (device_id >= num_devices)
{
if (gomp_target_offload_var == GOMP_TARGET_OFFLOAD_MANDATORY
- && device_id != num_devices_openmp)
+ && device_id != num_devices)
gomp_fatal ("OMP_TARGET_OFFLOAD is set to MANDATORY, "
"but device not found");
case GOMP_MAP_FORCE_ALLOC:
case GOMP_MAP_FORCE_FROM:
case GOMP_MAP_ALWAYS_FROM:
+ case GOMP_MAP_ALWAYS_PRESENT_FROM:
+ case GOMP_MAP_FORCE_PRESENT:
return false;
default:
return true;
else
tgt_var->length = newn->host_end - newn->host_start;
- if ((kind & GOMP_MAP_FLAG_FORCE)
+ if (GOMP_MAP_FORCE_P (kind)
/* For implicit maps, old contained in new is valid. */
|| !(implicit_subset
/* Otherwise, new contained inside old is considered valid. */
if (n == NULL)
{
if (allow_zero_length_array_sections)
- cur_node.tgt_offset = 0;
+ cur_node.tgt_offset = cur_node.host_start;
else
{
gomp_mutex_unlock (&devicep->lock);
cur_node.host_start = (uintptr_t) hostaddrs[i];
cur_node.host_end = cur_node.host_start + sizes[i];
- splay_tree_key n2 = splay_tree_lookup (mem_map, &cur_node);
+ splay_tree_key n2 = gomp_map_0len_lookup (mem_map, &cur_node);
kind = get_kind (short_mapkind, kinds, i);
implicit = get_implicit (short_mapkind, kinds, i);
if (n2
if ((void *) target == NULL)
{
- gomp_mutex_unlock (&devicep->lock);
- gomp_fatal ("attempt to attach null pointer");
+ /* As a special case, allow attaching NULL host pointers. This
+ allows e.g. unassociated Fortran pointers to be mapped
+ properly. */
+ data = 0;
+
+ gomp_debug (1,
+ "%s: attaching NULL host pointer, target %p "
+ "(struct base %p)\n", __FUNCTION__, (void *) devptr,
+ (void *) (n->tgt->tgt_start + n->tgt_offset));
+
+ gomp_copy_host2dev (devicep, aq, (void *) devptr, (void *) &data,
+ sizeof (void *), true, cbufp);
+
+ return;
}
s.host_start = target + bias;
{
if (allow_zero_length_array_sections)
/* When allowing attachment to zero-length array sections, we
- allow attaching to NULL pointers when the target region is not
- mapped. */
- data = 0;
+ copy the host pointer when the target region is not mapped. */
+ data = target;
else
{
gomp_mutex_unlock (&devicep->lock);
tgt->list[i].offset = 0;
continue;
}
- else if ((kind & typemask) == GOMP_MAP_STRUCT)
+ else if ((kind & typemask) == GOMP_MAP_STRUCT
+ || (kind & typemask) == GOMP_MAP_STRUCT_UNORD)
{
size_t first = i + 1;
size_t last = i + sizes[i];
tgt->list[i].key = NULL;
if (!aq
&& gomp_to_device_kind_p (get_kind (short_mapkind, kinds, i)
- & typemask))
+ & typemask)
+ && sizes[i] != 0)
gomp_coalesce_buf_add (&cbuf,
tgt_size - cur_node.host_end
+ (uintptr_t) hostaddrs[i],
if (!n)
{
tgt->list[i].key = NULL;
- tgt->list[i].offset = OFFSET_POINTER;
+ tgt->list[i].offset = OFFSET_INLINED;
continue;
}
}
{
uintptr_t target = (uintptr_t) hostaddrs[i];
void *devptr = *(void**) hostaddrs[i+1] + sizes[i+1];
+ /* Per
+ <https://inbox.sourceware.org/gcc-patches/87o7pe12ke.fsf@euler.schwinge.homeip.net>
+ "OpenMP: Handle descriptors in target's firstprivate [PR104949]"
+ this probably needs revision for 'aq' usage. */
+ assert (!aq);
gomp_copy_host2dev (devicep, aq, devptr, &target,
sizeof (void *), false, cbufp);
++i;
tgt->list[i].offset = OFFSET_INLINED;
}
continue;
+ case GOMP_MAP_STRUCT_UNORD:
+ if (sizes[i] > 1)
+ {
+ void *first = hostaddrs[i + 1];
+ for (size_t j = i + 1; j < i + sizes[i]; j++)
+ if (hostaddrs[j + 1] != first)
+ {
+ gomp_mutex_unlock (&devicep->lock);
+ gomp_fatal ("Mapped array elements must be the "
+ "same (%p vs %p)", first,
+ hostaddrs[j + 1]);
+ }
+ }
+ /* Fallthrough. */
case GOMP_MAP_STRUCT:
first = i + 1;
last = i + sizes[i];
+ sizes[last];
if (tgt->list[first].key != NULL)
continue;
+ if (sizes[last] == 0)
+ cur_node.host_end++;
n = splay_tree_lookup (mem_map, &cur_node);
+ if (sizes[last] == 0)
+ cur_node.host_end--;
+ if (n == NULL && cur_node.host_start == cur_node.host_end)
+ {
+ gomp_mutex_unlock (&devicep->lock);
+ gomp_fatal ("Struct pointer member not mapped (%p)",
+ (void*) hostaddrs[first]);
+ }
if (n == NULL)
{
size_t align = (size_t) 1 << (kind >> rshift);
k->host_end = k->host_start + sizeof (void *);
splay_tree_key n = splay_tree_lookup (mem_map, k);
if (n && n->refcount != REFCOUNT_LINK)
- gomp_map_vars_existing (devicep, aq, n, k, &tgt->list[i],
- kind & typemask, false, implicit, cbufp,
- refcount_set);
+ {
+ if (field_tgt_clear != FIELD_TGT_EMPTY)
+ {
+ /* For this condition to be true, there must be a
+ duplicate struct element mapping. This can happen with
+ GOMP_MAP_STRUCT_UNORD mappings, for example. */
+ tgt->list[i].key = n;
+ if (openmp_p)
+ {
+ assert ((n->refcount & REFCOUNT_STRUCTELEM) != 0);
+ assert (field_tgt_structelem_first != NULL);
+
+ if (i == field_tgt_clear)
+ {
+ n->refcount |= REFCOUNT_STRUCTELEM_FLAG_LAST;
+ field_tgt_structelem_first = NULL;
+ }
+ }
+ if (i == field_tgt_clear)
+ field_tgt_clear = FIELD_TGT_EMPTY;
+ gomp_increment_refcount (n, refcount_set);
+ tgt->list[i].copy_from
+ = GOMP_MAP_COPY_FROM_P (kind & typemask);
+ tgt->list[i].always_copy_from
+ = GOMP_MAP_ALWAYS_FROM_P (kind & typemask);
+ tgt->list[i].is_attach = false;
+ tgt->list[i].offset = 0;
+ tgt->list[i].length = k->host_end - k->host_start;
+ }
+ else
+ gomp_map_vars_existing (devicep, aq, n, k, &tgt->list[i],
+ kind & typemask, false, implicit,
+ cbufp, refcount_set);
+ }
else
{
k->aux = NULL;
i = j - 1;
break;
case GOMP_MAP_FORCE_PRESENT:
+ case GOMP_MAP_ALWAYS_PRESENT_TO:
+ case GOMP_MAP_ALWAYS_PRESENT_FROM:
+ case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
{
/* We already looked up the memory region above and it
was missing. */
size_t size = k->host_end - k->host_start;
gomp_mutex_unlock (&devicep->lock);
#ifdef HAVE_INTTYPES_H
- gomp_fatal ("present clause: !acc_is_present (%p, "
- "%"PRIu64" (0x%"PRIx64"))",
- (void *) k->host_start,
- (uint64_t) size, (uint64_t) size);
+ gomp_fatal ("present clause: not present on the device "
+ "(addr: %p, size: %"PRIu64" (0x%"PRIx64"), "
+ "dev: %d)", (void *) k->host_start,
+ (uint64_t) size, (uint64_t) size,
+ devicep->target_id);
#else
- gomp_fatal ("present clause: !acc_is_present (%p, "
- "%lu (0x%lx))", (void *) k->host_start,
- (unsigned long) size, (unsigned long) size);
+ gomp_fatal ("present clause: not present on the device "
+ "(addr: %p, size: %lu (0x%lx), dev: %d)",
+ (void *) k->host_start,
+ (unsigned long) size, (unsigned long) size,
+ devicep->target_id);
#endif
}
break;
gomp_copy_dev2host (devicep, NULL, hostaddr, devaddr, size);
}
}
+ else
+ {
+ int kind = get_kind (short_mapkind, kinds, i);
+
+ if (GOMP_MAP_PRESENT_P (kind))
+ {
+ /* We already looked up the memory region above and it
+ was missing. */
+ gomp_mutex_unlock (&devicep->lock);
+#ifdef HAVE_INTTYPES_H
+ gomp_fatal ("present clause: not present on the device "
+ "(addr: %p, size: %"PRIu64" (0x%"PRIx64"), "
+ "dev: %d)", (void *) hostaddrs[i],
+ (uint64_t) sizes[i], (uint64_t) sizes[i],
+ devicep->target_id);
+#else
+ gomp_fatal ("present clause: not present on the device "
+ "(addr: %p, size: %lu (0x%lx), dev: %d)",
+ (void *) hostaddrs[i], (unsigned long) sizes[i],
+ (unsigned long) sizes[i], devicep->target_id);
+#endif
+ }
+ }
}
gomp_mutex_unlock (&devicep->lock);
}
void **host_funcs_end = ((void ***) host_table)[1];
void **host_var_table = ((void ***) host_table)[2];
void **host_vars_end = ((void ***) host_table)[3];
+ void **host_ind_func_table = NULL;
+ void **host_ind_funcs_end = NULL;
- /* The func table contains only addresses, the var table contains addresses
- and corresponding sizes. */
+ if (GOMP_VERSION_SUPPORTS_INDIRECT_FUNCS (version))
+ {
+ host_ind_func_table = ((void ***) host_table)[4];
+ host_ind_funcs_end = ((void ***) host_table)[5];
+ }
+
+ /* The func and ind_func tables contain only addresses, the var table
+ contains addresses and corresponding sizes. */
int num_funcs = host_funcs_end - host_func_table;
int num_vars = (host_vars_end - host_var_table) / 2;
+ int num_ind_funcs = (host_ind_funcs_end - host_ind_func_table);
/* Load image to device and get target addresses for the image. */
struct addr_pair *target_table = NULL;
num_target_entries
= devicep->load_image_func (devicep->target_id, version,
target_data, &target_table,
- rev_lookup ? &rev_target_fn_table : NULL);
+ rev_lookup ? &rev_target_fn_table : NULL,
+ num_ind_funcs
+ ? (uint64_t *) host_ind_func_table : NULL);
if (num_target_entries != num_funcs + num_vars
/* "+1" due to the additional ICV struct. */
if (devicep->state == GOMP_DEVICE_INITIALIZED)
{
unsigned i;
-
+
/* Unload from device all images registered at the moment. */
for (i = 0; i < num_offload_images; i++)
{
void
gomp_target_rev (uint64_t fn_ptr, uint64_t mapnum, uint64_t devaddrs_ptr,
uint64_t sizes_ptr, uint64_t kinds_ptr, int dev_num,
- void (*dev_to_host_cpy) (void *, const void *, size_t, void*),
- void (*host_to_dev_cpy) (void *, const void *, size_t, void*),
- void *token)
+ struct goacc_asyncqueue *aq)
{
/* Return early if there is no offload code. */
if (sizeof (OFFLOAD_PLUGINS) == sizeof (""))
devaddrs = (uint64_t *) gomp_malloc (mapnum * sizeof (uint64_t));
sizes = (uint64_t *) gomp_malloc (mapnum * sizeof (uint64_t));
kinds = (unsigned short *) gomp_malloc (mapnum * sizeof (unsigned short));
- if (dev_to_host_cpy)
- {
- dev_to_host_cpy (devaddrs, (const void *) (uintptr_t) devaddrs_ptr,
- mapnum * sizeof (uint64_t), token);
- dev_to_host_cpy (sizes, (const void *) (uintptr_t) sizes_ptr,
- mapnum * sizeof (uint64_t), token);
- dev_to_host_cpy (kinds, (const void *) (uintptr_t) kinds_ptr,
- mapnum * sizeof (unsigned short), token);
- }
- else
- {
- gomp_copy_dev2host (devicep, NULL, devaddrs,
- (const void *) (uintptr_t) devaddrs_ptr,
- mapnum * sizeof (uint64_t));
- gomp_copy_dev2host (devicep, NULL, sizes,
- (const void *) (uintptr_t) sizes_ptr,
- mapnum * sizeof (uint64_t));
- gomp_copy_dev2host (devicep, NULL, kinds, (const void *) (uintptr_t) kinds_ptr,
- mapnum * sizeof (unsigned short));
- }
+ gomp_copy_dev2host (devicep, aq, devaddrs,
+ (const void *) (uintptr_t) devaddrs_ptr,
+ mapnum * sizeof (uint64_t));
+ gomp_copy_dev2host (devicep, aq, sizes,
+ (const void *) (uintptr_t) sizes_ptr,
+ mapnum * sizeof (uint64_t));
+ gomp_copy_dev2host (devicep, aq, kinds,
+ (const void *) (uintptr_t) kinds_ptr,
+ mapnum * sizeof (unsigned short));
+ if (aq && !devicep->openacc.async.synchronize_func (aq))
+ exit (EXIT_FAILURE);
}
size_t tgt_align = 0, tgt_size = 0;
if (devicep->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
memcpy (tgt + tgt_size, (void *) (uintptr_t) devaddrs[i],
(size_t) sizes[i]);
- else if (dev_to_host_cpy)
- dev_to_host_cpy (tgt + tgt_size, (void *) (uintptr_t) devaddrs[i],
- (size_t) sizes[i], token);
else
- gomp_copy_dev2host (devicep, NULL, tgt + tgt_size,
- (void *) (uintptr_t) devaddrs[i],
- (size_t) sizes[i]);
+ {
+ gomp_copy_dev2host (devicep, aq, tgt + tgt_size,
+ (void *) (uintptr_t) devaddrs[i],
+ (size_t) sizes[i]);
+ if (aq && !devicep->openacc.async.synchronize_func (aq))
+ exit (EXIT_FAILURE);
+ }
devaddrs[i] = (uint64_t) (uintptr_t) tgt + tgt_size;
tgt_size = tgt_size + sizes[i];
if ((devicep->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
case GOMP_MAP_DELETE:
case GOMP_MAP_RELEASE:
case GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION:
- /* Assume it is present; look it up - but ignore otherwise. */
+ /* Assume it is present; look it up - but ignore unless the
+ present clause is there. */
case GOMP_MAP_ALLOC:
case GOMP_MAP_FROM:
case GOMP_MAP_FORCE_ALLOC:
case GOMP_MAP_FORCE_TOFROM:
case GOMP_MAP_ALWAYS_TO:
case GOMP_MAP_ALWAYS_TOFROM:
+ case GOMP_MAP_FORCE_PRESENT:
+ case GOMP_MAP_ALWAYS_PRESENT_FROM:
+ case GOMP_MAP_ALWAYS_PRESENT_TO:
+ case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
case GOMP_MAP_ZERO_LEN_ARRAY_SECTION:
cdata[i].devaddr = devaddrs[i];
bool zero_len = (kind == GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
devaddrs[i] + sizes[i], zero_len);
cdata[i].present = n2 != NULL;
}
- if (!cdata[i].present
+ if (!cdata[i].present && GOMP_MAP_PRESENT_P (kind))
+ {
+ gomp_mutex_unlock (&devicep->lock);
+#ifdef HAVE_INTTYPES_H
+ gomp_fatal ("present clause: no corresponding data on "
+ "parent device at %p with size %"PRIu64,
+ (void *) (uintptr_t) devaddrs[i],
+ (uint64_t) sizes[i]);
+#else
+ gomp_fatal ("present clause: no corresponding data on "
+ "parent device at %p with size %lu",
+ (void *) (uintptr_t) devaddrs[i],
+ (unsigned long) sizes[i]);
+#endif
+ break;
+ }
+ else if (!cdata[i].present
&& kind != GOMP_MAP_DELETE
&& kind != GOMP_MAP_RELEASE
&& kind != GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION)
&& (kind == GOMP_MAP_TO || kind == GOMP_MAP_TOFROM))
|| kind == GOMP_MAP_FORCE_TO
|| kind == GOMP_MAP_FORCE_TOFROM
- || kind == GOMP_MAP_ALWAYS_TO
- || kind == GOMP_MAP_ALWAYS_TOFROM)
+ || GOMP_MAP_ALWAYS_TO_P (kind))
{
- if (dev_to_host_cpy)
- dev_to_host_cpy ((void *) (uintptr_t) devaddrs[i],
- (void *) (uintptr_t) cdata[i].devaddr,
- sizes[i], token);
- else
- gomp_copy_dev2host (devicep, NULL,
- (void *) (uintptr_t) devaddrs[i],
- (void *) (uintptr_t) cdata[i].devaddr,
- sizes[i]);
+ gomp_copy_dev2host (devicep, aq,
+ (void *) (uintptr_t) devaddrs[i],
+ (void *) (uintptr_t) cdata[i].devaddr,
+ sizes[i]);
+ if (aq && !devicep->openacc.async.synchronize_func (aq))
+ {
+ gomp_mutex_unlock (&devicep->lock);
+ exit (EXIT_FAILURE);
+ }
}
if (struct_cpy)
struct_cpy--;
devaddrs[i]
= (uint64_t) (uintptr_t) gomp_aligned_alloc (align,
sizes[i]);
- if (dev_to_host_cpy)
- dev_to_host_cpy ((void *) (uintptr_t) devaddrs[i],
- (void *) (uintptr_t) cdata[i].devaddr,
- sizes[i], token);
- else
- gomp_copy_dev2host (devicep, NULL,
- (void *) (uintptr_t) devaddrs[i],
- (void *) (uintptr_t) cdata[i].devaddr,
- sizes[i]);
+ gomp_copy_dev2host (devicep, aq,
+ (void *) (uintptr_t) devaddrs[i],
+ (void *) (uintptr_t) cdata[i].devaddr,
+ sizes[i]);
+ if (aq && !devicep->openacc.async.synchronize_func (aq))
+ {
+ gomp_mutex_unlock (&devicep->lock);
+ exit (EXIT_FAILURE);
+ }
}
for (j = i + 1; j < mapnum; j++)
{
case GOMP_MAP_FORCE_TOFROM:
case GOMP_MAP_ALWAYS_FROM:
case GOMP_MAP_ALWAYS_TOFROM:
+ case GOMP_MAP_ALWAYS_PRESENT_FROM:
+ case GOMP_MAP_ALWAYS_PRESENT_TOFROM:
copy = true;
/* FALLTHRU */
case GOMP_MAP_FROM:
case GOMP_MAP_TOFROM:
- if (copy && host_to_dev_cpy)
- host_to_dev_cpy ((void *) (uintptr_t) cdata[i].devaddr,
- (void *) (uintptr_t) devaddrs[i],
- sizes[i], token);
- else if (copy)
- gomp_copy_host2dev (devicep, NULL,
- (void *) (uintptr_t) cdata[i].devaddr,
- (void *) (uintptr_t) devaddrs[i],
- sizes[i], false, NULL);
+ if (copy)
+ {
+ gomp_copy_host2dev (devicep, aq,
+ (void *) (uintptr_t) cdata[i].devaddr,
+ (void *) (uintptr_t) devaddrs[i],
+ sizes[i], false, NULL);
+ if (aq && !devicep->openacc.async.synchronize_func (aq))
+ exit (EXIT_FAILURE);
+ }
default:
break;
}
size_t i, j;
if ((flags & GOMP_TARGET_FLAG_EXIT_DATA) == 0)
for (i = 0; i < mapnum; i++)
- if ((kinds[i] & 0xff) == GOMP_MAP_STRUCT)
+ if ((kinds[i] & 0xff) == GOMP_MAP_STRUCT
+ || (kinds[i] & 0xff) == GOMP_MAP_STRUCT_UNORD)
{
gomp_map_vars (devicep, sizes[i] + 1, &hostaddrs[i], NULL, &sizes[i],
&kinds[i], true, &refcount_set,
htab_t refcount_set = htab_create (ttask->mapnum);
if ((ttask->flags & GOMP_TARGET_FLAG_EXIT_DATA) == 0)
for (i = 0; i < ttask->mapnum; i++)
- if ((ttask->kinds[i] & 0xff) == GOMP_MAP_STRUCT)
+ if ((ttask->kinds[i] & 0xff) == GOMP_MAP_STRUCT
+ || (ttask->kinds[i] & 0xff) == GOMP_MAP_STRUCT_UNORD)
{
gomp_map_vars (devicep, ttask->sizes[i] + 1, &ttask->hostaddrs[i],
NULL, &ttask->sizes[i], &ttask->kinds[i], true,
const size_t *dst_dimensions,
const size_t *src_dimensions,
struct gomp_device_descr *dst_devicep,
- struct gomp_device_descr *src_devicep)
+ struct gomp_device_descr *src_devicep,
+ size_t *tmp_size, void **tmp)
{
size_t dst_slice = element_size;
size_t src_slice = element_size;
(const char *) src + src_off,
length);
else
- ret = 0;
+ {
+ if (*tmp_size == 0)
+ {
+ *tmp_size = length;
+ *tmp = malloc (length);
+ if (*tmp == NULL)
+ return ENOMEM;
+ }
+ else if (*tmp_size < length)
+ {
+ *tmp_size = length;
+ free (*tmp);
+ *tmp = malloc (length);
+ if (*tmp == NULL)
+ return ENOMEM;
+ }
+ ret = src_devicep->dev2host_func (src_devicep->target_id, *tmp,
+ (const char *) src + src_off,
+ length);
+ if (ret == 1)
+ ret = dst_devicep->host2dev_func (dst_devicep->target_id,
+ (char *) dst + dst_off, *tmp,
+ length);
+ }
return ret ? 0 : EINVAL;
}
- /* FIXME: it would be nice to have some plugin function to handle
- num_dims == 2 and num_dims == 3 more efficiently. Larger ones can
- be handled in the generic recursion below, and for host-host it
- should be used even for any num_dims >= 2. */
+ /* host->device, device->host and intra device. */
+ if (num_dims == 2
+ && ((src_devicep
+ && src_devicep == dst_devicep
+ && src_devicep->memcpy2d_func)
+ || (!src_devicep != !dst_devicep
+ && ((src_devicep && src_devicep->memcpy2d_func)
+ || (dst_devicep && dst_devicep->memcpy2d_func)))))
+ {
+ size_t vol_sz1, dst_sz1, src_sz1, dst_off_sz1, src_off_sz1;
+ int dst_id = dst_devicep ? dst_devicep->target_id : -1;
+ int src_id = src_devicep ? src_devicep->target_id : -1;
+ struct gomp_device_descr *devp = dst_devicep ? dst_devicep : src_devicep;
+
+ if (__builtin_mul_overflow (volume[1], element_size, &vol_sz1)
+ || __builtin_mul_overflow (dst_dimensions[1], element_size, &dst_sz1)
+ || __builtin_mul_overflow (src_dimensions[1], element_size, &src_sz1)
+ || __builtin_mul_overflow (dst_offsets[1], element_size, &dst_off_sz1)
+ || __builtin_mul_overflow (src_offsets[1], element_size,
+ &src_off_sz1))
+ return EINVAL;
+ ret = devp->memcpy2d_func (dst_id, src_id, vol_sz1, volume[0],
+ dst, dst_off_sz1, dst_offsets[0], dst_sz1,
+ src, src_off_sz1, src_offsets[0], src_sz1);
+ if (ret != -1)
+ return ret ? 0 : EINVAL;
+ }
+ else if (num_dims == 3
+ && ((src_devicep
+ && src_devicep == dst_devicep
+ && src_devicep->memcpy3d_func)
+ || (!src_devicep != !dst_devicep
+ && ((src_devicep && src_devicep->memcpy3d_func)
+ || (dst_devicep && dst_devicep->memcpy3d_func)))))
+ {
+ size_t vol_sz2, dst_sz2, src_sz2, dst_off_sz2, src_off_sz2;
+ int dst_id = dst_devicep ? dst_devicep->target_id : -1;
+ int src_id = src_devicep ? src_devicep->target_id : -1;
+ struct gomp_device_descr *devp = dst_devicep ? dst_devicep : src_devicep;
+
+ if (__builtin_mul_overflow (volume[2], element_size, &vol_sz2)
+ || __builtin_mul_overflow (dst_dimensions[2], element_size, &dst_sz2)
+ || __builtin_mul_overflow (src_dimensions[2], element_size, &src_sz2)
+ || __builtin_mul_overflow (dst_offsets[2], element_size, &dst_off_sz2)
+ || __builtin_mul_overflow (src_offsets[2], element_size,
+ &src_off_sz2))
+ return EINVAL;
+ ret = devp->memcpy3d_func (dst_id, src_id, vol_sz2, volume[1], volume[0],
+ dst, dst_off_sz2, dst_offsets[1],
+ dst_offsets[0], dst_sz2, dst_dimensions[1],
+ src, src_off_sz2, src_offsets[1],
+ src_offsets[0], src_sz2, src_dimensions[1]);
+ if (ret != -1)
+ return ret ? 0 : EINVAL;
+ }
for (i = 1; i < num_dims; i++)
if (__builtin_mul_overflow (dst_slice, dst_dimensions[i], &dst_slice)
volume + 1, dst_offsets + 1,
src_offsets + 1, dst_dimensions + 1,
src_dimensions + 1, dst_devicep,
- src_devicep);
+ src_devicep, tmp_size, tmp);
if (ret)
return ret;
dst_off += dst_slice;
if (ret)
return ret;
- if (*src_devicep != NULL && *dst_devicep != NULL && *src_devicep != *dst_devicep)
- return EINVAL;
-
return 0;
}
struct gomp_device_descr *dst_devicep,
struct gomp_device_descr *src_devicep)
{
- if (src_devicep)
+ size_t tmp_size = 0;
+ void *tmp = NULL;
+ bool lock_src;
+ bool lock_dst;
+
+ lock_src = src_devicep != NULL;
+ lock_dst = dst_devicep != NULL && src_devicep != dst_devicep;
+ if (lock_src)
gomp_mutex_lock (&src_devicep->lock);
- else if (dst_devicep)
+ if (lock_dst)
gomp_mutex_lock (&dst_devicep->lock);
int ret = omp_target_memcpy_rect_worker (dst, src, element_size, num_dims,
volume, dst_offsets, src_offsets,
dst_dimensions, src_dimensions,
- dst_devicep, src_devicep);
- if (src_devicep)
+ dst_devicep, src_devicep,
+ &tmp_size, &tmp);
+ if (lock_src)
gomp_mutex_unlock (&src_devicep->lock);
- else if (dst_devicep)
+ if (lock_dst)
gomp_mutex_unlock (&dst_devicep->lock);
+ if (tmp)
+ free (tmp);
return ret;
}
DLSYM (free);
DLSYM (dev2host);
DLSYM (host2dev);
+ DLSYM_OPT (memcpy2d, memcpy2d);
+ DLSYM_OPT (memcpy3d, memcpy3d);
device->capabilities = device->get_caps_func ();
if (device->capabilities & GOMP_OFFLOAD_CAP_OPENMP_400)
{
if (devs[i].capabilities & GOMP_OFFLOAD_CAP_OPENACC_200)
goacc_register (&devs[i]);
}
+ if (gomp_global_icv.default_device_var == INT_MIN)
+ {
+ /* This implies OMP_TARGET_OFFLOAD=mandatory. */
+ struct gomp_icv_list *none;
+ none = gomp_get_initial_icv_item (GOMP_DEVICE_NUM_FOR_NO_SUFFIX);
+ gomp_global_icv.default_device_var = (num_devs_openmp
+ ? 0 : omp_invalid_device);
+ none->icvs.default_device_var = gomp_global_icv.default_device_var;
+ }
num_devices = num_devs;
num_devices_openmp = num_devs_openmp;
projects / thirdparty / gcc.git / blobdiff