[thirdparty/gcc.git] / libgomp / oacc-parallel.c

/* Copyright (C) 2013-2016 Free Software Foundation, Inc.

   Contributed by Mentor Embedded.

   This file is part of the GNU Offloading and Multi Processing Library
   (libgomp).

   Libgomp is free software; you can redistribute it and/or modify it
   under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3, or (at your option)
   any later version.

   Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
   FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
   more details.

   Under Section 7 of GPL version 3, you are granted additional
   permissions described in the GCC Runtime Library Exception, version
   3.1, as published by the Free Software Foundation.

   You should have received a copy of the GNU General Public License and
   a copy of the GCC Runtime Library Exception along with this program;
   see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
   <http://www.gnu.org/licenses/>.  */

/* This file handles OpenACC constructs.  */

#include "openacc.h"
#include "libgomp.h"
#include "libgomp_g.h"
#include "gomp-constants.h"
#include "oacc-int.h"
#ifdef HAVE_INTTYPES_H
# include <inttypes.h>  /* For PRIu64.  */
#endif
#include <string.h>
#include <stdarg.h>
#include <assert.h>

static int
find_pset (int pos, size_t mapnum, unsigned short *kinds)
{
  if (pos + 1 >= mapnum)
    return 0;

  unsigned char kind = kinds[pos+1] & 0xff;

  return kind == GOMP_MAP_TO_PSET;
}

static void goacc_wait (int async, int num_waits, va_list *ap);


/* Launch a possibly offloaded function on DEVICE.  FN is the host fn
   address.  MAPNUM, HOSTADDRS, SIZES & KINDS  describe the memory
   blocks to be copied to/from the device.  Varadic arguments are
   keyed optional parameters terminated with a zero.  */

void
GOACC_parallel_keyed (int device, void (*fn) (void *),
		      size_t mapnum, void **hostaddrs, size_t *sizes,
		      unsigned short *kinds, ...)
{
  bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
  va_list ap;
  struct goacc_thread *thr;
  struct gomp_device_descr *acc_dev;
  struct target_mem_desc *tgt;
  void **devaddrs;
  unsigned int i;
  struct splay_tree_key_s k;
  splay_tree_key tgt_fn_key;
  void (*tgt_fn);
  int async = GOMP_ASYNC_SYNC;
  unsigned dims[GOMP_DIM_MAX];
  unsigned tag;

#ifdef HAVE_INTTYPES_H
  gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n",
	      __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds);
#else
  gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n",
	      __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds);
#endif
  goacc_lazy_initialize ();

  thr = goacc_thread ();
  acc_dev = thr->dev;

  /* Host fallback if "if" clause is false or if the current device is set to
     the host.  */
  if (host_fallback)
    {
      goacc_save_and_set_bind (acc_device_host);
      fn (hostaddrs);
      goacc_restore_bind ();
      return;
    }
  else if (acc_device_type (acc_dev->type) == acc_device_host)
    {
      fn (hostaddrs);
      return;
    }

  va_start (ap, kinds);
  /* TODO: This will need amending when device_type is implemented.  */
  while ((tag = va_arg (ap, unsigned)) != 0)
    {
      if (GOMP_LAUNCH_DEVICE (tag))
	gomp_fatal ("device_type '%d' offload parameters, libgomp is too old",
		    GOMP_LAUNCH_DEVICE (tag));

      switch (GOMP_LAUNCH_CODE (tag))
	{
	case GOMP_LAUNCH_DIM:
	  {
	    unsigned mask = GOMP_LAUNCH_OP (tag);

	    for (i = 0; i != GOMP_DIM_MAX; i++)
	      if (mask & GOMP_DIM_MASK (i))
		dims[i] = va_arg (ap, unsigned);
	  }
	  break;

	case GOMP_LAUNCH_ASYNC:
	  {
	    /* Small constant values are encoded in the operand.  */
	    async = GOMP_LAUNCH_OP (tag);

	    if (async == GOMP_LAUNCH_OP_MAX)
	      async = va_arg (ap, unsigned);
	    break;
	  }

	case GOMP_LAUNCH_WAIT:
	  {
	    unsigned num_waits = GOMP_LAUNCH_OP (tag);

	    if (num_waits)
	      goacc_wait (async, num_waits, &ap);
	    break;
	  }

	default:
	  gomp_fatal ("unrecognized offload code '%d',"
		      " libgomp is too old", GOMP_LAUNCH_CODE (tag));
	}
    }
  va_end (ap);
  
  acc_dev->openacc.async_set_async_func (async);

  if (!(acc_dev->capabilities & GOMP_OFFLOAD_CAP_NATIVE_EXEC))
    {
      k.host_start = (uintptr_t) fn;
      k.host_end = k.host_start + 1;
      gomp_mutex_lock (&acc_dev->lock);
      tgt_fn_key = splay_tree_lookup (&acc_dev->mem_map, &k);
      gomp_mutex_unlock (&acc_dev->lock);

      if (tgt_fn_key == NULL)
	gomp_fatal ("target function wasn't mapped");

      tgt_fn = (void (*)) tgt_fn_key->tgt_offset;
    }
  else
    tgt_fn = (void (*)) fn;

  tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true,
		       GOMP_MAP_VARS_OPENACC);

  devaddrs = gomp_alloca (sizeof (void *) * mapnum);
  for (i = 0; i < mapnum; i++)
    devaddrs[i] = (void *) (tgt->list[i].key->tgt->tgt_start
			    + tgt->list[i].key->tgt_offset);

  acc_dev->openacc.exec_func (tgt_fn, mapnum, hostaddrs, devaddrs,
			      async, dims, tgt);

  /* If running synchronously, unmap immediately.  */
  if (async < acc_async_noval)
    gomp_unmap_vars (tgt, true);
  else
    {
      gomp_copy_from_async (tgt);
      acc_dev->openacc.register_async_cleanup_func (tgt);
    }

  acc_dev->openacc.async_set_async_func (acc_async_sync);
}

/* Legacy entry point, only provide host execution.  */

void
GOACC_parallel (int device, void (*fn) (void *),
		size_t mapnum, void **hostaddrs, size_t *sizes,
		unsigned short *kinds,
		int num_gangs, int num_workers, int vector_length,
		int async, int num_waits, ...)
{
  goacc_save_and_set_bind (acc_device_host);
  fn (hostaddrs);
  goacc_restore_bind ();
}

void
GOACC_data_start (int device, size_t mapnum,
		  void **hostaddrs, size_t *sizes, unsigned short *kinds)
{
  bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
  struct target_mem_desc *tgt;

#ifdef HAVE_INTTYPES_H
  gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n",
	      __FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds);
#else
  gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n",
	      __FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds);
#endif

  goacc_lazy_initialize ();

  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  /* Host fallback or 'do nothing'.  */
  if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
      || host_fallback)
    {
      tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true,
			   GOMP_MAP_VARS_OPENACC);
      tgt->prev = thr->mapped_data;
      thr->mapped_data = tgt;

      return;
    }

  gomp_debug (0, "  %s: prepare mappings\n", __FUNCTION__);
  tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true,
		       GOMP_MAP_VARS_OPENACC);
  gomp_debug (0, "  %s: mappings prepared\n", __FUNCTION__);
  tgt->prev = thr->mapped_data;
  thr->mapped_data = tgt;
}

void
GOACC_data_end (void)
{
  struct goacc_thread *thr = goacc_thread ();
  struct target_mem_desc *tgt = thr->mapped_data;

  gomp_debug (0, "  %s: restore mappings\n", __FUNCTION__);
  thr->mapped_data = tgt->prev;
  gomp_unmap_vars (tgt, true);
  gomp_debug (0, "  %s: mappings restored\n", __FUNCTION__);
}

void
GOACC_enter_exit_data (int device, size_t mapnum,
		       void **hostaddrs, size_t *sizes, unsigned short *kinds,
		       int async, int num_waits, ...)
{
  struct goacc_thread *thr;
  struct gomp_device_descr *acc_dev;
  bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
  bool data_enter = false;
  size_t i;

  goacc_lazy_initialize ();

  thr = goacc_thread ();
  acc_dev = thr->dev;

  if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
      || host_fallback)
    return;

  if (num_waits)
    {
      va_list ap;

      va_start (ap, num_waits);
      goacc_wait (async, num_waits, &ap);
      va_end (ap);
    }

  acc_dev->openacc.async_set_async_func (async);

  /* Determine if this is an "acc enter data".  */
  for (i = 0; i < mapnum; ++i)
    {
      unsigned char kind = kinds[i] & 0xff;

      if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET)
	continue;

      if (kind == GOMP_MAP_FORCE_ALLOC
	  || kind == GOMP_MAP_FORCE_PRESENT
	  || kind == GOMP_MAP_FORCE_TO)
	{
	  data_enter = true;
	  break;
	}

      if (kind == GOMP_MAP_FORCE_DEALLOC
	  || kind == GOMP_MAP_FORCE_FROM)
	break;

      gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
		      kind);
    }

  if (data_enter)
    {
      for (i = 0; i < mapnum; i++)
	{
	  unsigned char kind = kinds[i] & 0xff;

	  /* Scan for PSETs.  */
	  int psets = find_pset (i, mapnum, kinds);

	  if (!psets)
	    {
	      switch (kind)
		{
		case GOMP_MAP_POINTER:
		  gomp_acc_insert_pointer (1, &hostaddrs[i], &sizes[i],
					&kinds[i]);
		  break;
		case GOMP_MAP_FORCE_ALLOC:
		  acc_create (hostaddrs[i], sizes[i]);
		  break;
		case GOMP_MAP_FORCE_PRESENT:
		  acc_present_or_copyin (hostaddrs[i], sizes[i]);
		  break;
		case GOMP_MAP_FORCE_TO:
		  acc_present_or_copyin (hostaddrs[i], sizes[i]);
		  break;
		default:
		  gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
			      kind);
		  break;
		}
	    }
	  else
	    {
	      gomp_acc_insert_pointer (3, &hostaddrs[i], &sizes[i], &kinds[i]);
	      /* Increment 'i' by two because OpenACC requires fortran
		 arrays to be contiguous, so each PSET is associated with
		 one of MAP_FORCE_ALLOC/MAP_FORCE_PRESET/MAP_FORCE_TO, and
		 one MAP_POINTER.  */
	      i += 2;
	    }
	}
    }
  else
    for (i = 0; i < mapnum; ++i)
      {
	unsigned char kind = kinds[i] & 0xff;

	int psets = find_pset (i, mapnum, kinds);

	if (!psets)
	  {
	    switch (kind)
	      {
	      case GOMP_MAP_POINTER:
		gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff)
					 == GOMP_MAP_FORCE_FROM,
					 async, 1);
		break;
	      case GOMP_MAP_FORCE_DEALLOC:
		acc_delete (hostaddrs[i], sizes[i]);
		break;
	      case GOMP_MAP_FORCE_FROM:
		acc_copyout (hostaddrs[i], sizes[i]);
		break;
	      default:
		gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
			    kind);
		break;
	      }
	  }
	else
	  {
	    gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff)
				     == GOMP_MAP_FORCE_FROM, async, 3);
	    /* See the above comment.  */
	    i += 2;
	  }
      }

  acc_dev->openacc.async_set_async_func (acc_async_sync);
}

static void
goacc_wait (int async, int num_waits, va_list *ap)
{
  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  while (num_waits--)
    {
      int qid = va_arg (*ap, int);
      
      if (acc_async_test (qid))
	continue;

      if (async == acc_async_sync)
	acc_wait (qid);
      else if (qid == async)
	;/* If we're waiting on the same asynchronous queue as we're
	    launching on, the queue itself will order work as
	    required, so there's no need to wait explicitly.  */
      else
	acc_dev->openacc.async_wait_async_func (qid, async);
    }
}

void
GOACC_update (int device, size_t mapnum,
	      void **hostaddrs, size_t *sizes, unsigned short *kinds,
	      int async, int num_waits, ...)
{
  bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
  size_t i;

  goacc_lazy_initialize ();

  struct goacc_thread *thr = goacc_thread ();
  struct gomp_device_descr *acc_dev = thr->dev;

  if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
      || host_fallback)
    return;

  if (num_waits)
    {
      va_list ap;

      va_start (ap, num_waits);
      goacc_wait (async, num_waits, &ap);
      va_end (ap);
    }

  acc_dev->openacc.async_set_async_func (async);

  for (i = 0; i < mapnum; ++i)
    {
      unsigned char kind = kinds[i] & 0xff;

      switch (kind)
	{
	case GOMP_MAP_POINTER:
	case GOMP_MAP_TO_PSET:
	  break;

	case GOMP_MAP_FORCE_TO:
	  acc_update_device (hostaddrs[i], sizes[i]);
	  break;

	case GOMP_MAP_FORCE_FROM:
	  acc_update_self (hostaddrs[i], sizes[i]);
	  break;

	default:
	  gomp_fatal (">>>> GOACC_update UNHANDLED kind 0x%.2x", kind);
	  break;
	}
    }

  acc_dev->openacc.async_set_async_func (acc_async_sync);
}

void
GOACC_wait (int async, int num_waits, ...)
{
  if (num_waits)
    {
      va_list ap;

      va_start (ap, num_waits);
      goacc_wait (async, num_waits, &ap);
      va_end (ap);
    }
  else if (async == acc_async_sync)
    acc_wait_all ();
  else if (async == acc_async_noval)
    goacc_thread ()->dev->openacc.async_wait_all_async_func (acc_async_noval);
}

int
GOACC_get_num_threads (void)
{
  return 1;
}

int
GOACC_get_thread_num (void)
{
  return 0;
}

void
GOACC_declare (int device, size_t mapnum,
	       void **hostaddrs, size_t *sizes, unsigned short *kinds)
{
  int i;

  for (i = 0; i < mapnum; i++)
    {
      unsigned char kind = kinds[i] & 0xff;

      if (kind == GOMP_MAP_POINTER || kind == GOMP_MAP_TO_PSET)
	continue;

      switch (kind)
	{
	  case GOMP_MAP_FORCE_ALLOC:
	  case GOMP_MAP_FORCE_DEALLOC:
	  case GOMP_MAP_FORCE_FROM:
	  case GOMP_MAP_FORCE_TO:
	  case GOMP_MAP_POINTER:
	    GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
				   &kinds[i], 0, 0);
	    break;

	  case GOMP_MAP_FORCE_DEVICEPTR:
	    break;

	  case GOMP_MAP_ALLOC:
	    if (!acc_is_present (hostaddrs[i], sizes[i]))
	      GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
				     &kinds[i], 0, 0);
	    break;

	  case GOMP_MAP_TO:
	    GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
				   &kinds[i], 0, 0);

	    break;

	  case GOMP_MAP_FROM:
	    kinds[i] = GOMP_MAP_FORCE_FROM;
	    GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
				   &kinds[i], 0, 0);
	    break;

	  case GOMP_MAP_FORCE_PRESENT:
	    if (!acc_is_present (hostaddrs[i], sizes[i]))
	      gomp_fatal ("[%p,%ld] is not mapped", hostaddrs[i],
			  (unsigned long) sizes[i]);
	    break;

	  default:
	    assert (0);
	    break;
	}
    }
}
Commit	Line	Data
818ab71a	1	/* Copyright (C) 2013-2016 Free Software Foundation, Inc.
41dbbb37 TS	2
	3	Contributed by Mentor Embedded.
	4
	5	This file is part of the GNU Offloading and Multi Processing Library
	6	(libgomp).
	7
	8	Libgomp is free software; you can redistribute it and/or modify it
	9	under the terms of the GNU General Public License as published by
	10	the Free Software Foundation; either version 3, or (at your option)
	11	any later version.
	12
	13	Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
	14	WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
	15	FOR A PARTICULAR PURPOSE. See the GNU General Public License for
	16	more details.
	17
	18	Under Section 7 of GPL version 3, you are granted additional
	19	permissions described in the GCC Runtime Library Exception, version
	20	3.1, as published by the Free Software Foundation.
	21
	22	You should have received a copy of the GNU General Public License and
	23	a copy of the GCC Runtime Library Exception along with this program;
	24	see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
	25	<http://www.gnu.org/licenses/>. */
	26
	27	/* This file handles OpenACC constructs. */
	28
	29	#include "openacc.h"
	30	#include "libgomp.h"
	31	#include "libgomp_g.h"
	32	#include "gomp-constants.h"
	33	#include "oacc-int.h"
01c0b3b0 KT	34	#ifdef HAVE_INTTYPES_H
	35	# include <inttypes.h> /* For PRIu64. */
	36	#endif
41dbbb37 TS	37	#include <string.h>
	38	#include <stdarg.h>
	39	#include <assert.h>
41dbbb37 TS	40
	41	static int
	42	find_pset (int pos, size_t mapnum, unsigned short *kinds)
	43	{
	44	if (pos + 1 >= mapnum)
	45	return 0;
	46
	47	unsigned char kind = kinds[pos+1] & 0xff;
	48
	49	return kind == GOMP_MAP_TO_PSET;
	50	}
	51
3e32ee19 NS	52	static void goacc_wait (int async, int num_waits, va_list *ap);
	53
	54
	55	/* Launch a possibly offloaded function on DEVICE. FN is the host fn
	56	address. MAPNUM, HOSTADDRS, SIZES & KINDS describe the memory
	57	blocks to be copied to/from the device. Varadic arguments are
	58	keyed optional parameters terminated with a zero. */
41dbbb37 TS	59
41dbbb37 TS	60	void
3e32ee19 NS	61	GOACC_parallel_keyed (int device, void (fn) (void ),
	62	size_t mapnum, void *hostaddrs, size_t sizes,
	63	unsigned short *kinds, ...)
41dbbb37 TS	64	{
	65	bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
	66	va_list ap;
	67	struct goacc_thread *thr;
	68	struct gomp_device_descr *acc_dev;
	69	struct target_mem_desc *tgt;
	70	void **devaddrs;
	71	unsigned int i;
	72	struct splay_tree_key_s k;
	73	splay_tree_key tgt_fn_key;
	74	void (*tgt_fn);
3e32ee19 NS	75	int async = GOMP_ASYNC_SYNC;
	76	unsigned dims[GOMP_DIM_MAX];
	77	unsigned tag;
41dbbb37	78
01c0b3b0	79	#ifdef HAVE_INTTYPES_H
3e32ee19 NS	80	gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n",
3e32ee19 NS	81	__FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds);
01c0b3b0	82	#else
3e32ee19 NS	83	gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n",
3e32ee19 NS	84	__FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds);
01c0b3b0	85	#endif
d93bdab5	86	goacc_lazy_initialize ();
41dbbb37 TS	87
	88	thr = goacc_thread ();
	89	acc_dev = thr->dev;
	90
	91	/* Host fallback if "if" clause is false or if the current device is set to
	92	the host. */
	93	if (host_fallback)
	94	{
	95	goacc_save_and_set_bind (acc_device_host);
	96	fn (hostaddrs);
	97	goacc_restore_bind ();
	98	return;
	99	}
	100	else if (acc_device_type (acc_dev->type) == acc_device_host)
	101	{
	102	fn (hostaddrs);
	103	return;
	104	}
	105
3e32ee19 NS	106	va_start (ap, kinds);
	107	/* TODO: This will need amending when device_type is implemented. */
	108	while ((tag = va_arg (ap, unsigned)) != 0)
a091118d	109	{
3e32ee19 NS	110	if (GOMP_LAUNCH_DEVICE (tag))
	111	gomp_fatal ("device_type '%d' offload parameters, libgomp is too old",
	112	GOMP_LAUNCH_DEVICE (tag));
	113
	114	switch (GOMP_LAUNCH_CODE (tag))
	115	{
	116	case GOMP_LAUNCH_DIM:
	117	{
	118	unsigned mask = GOMP_LAUNCH_OP (tag);
	119
	120	for (i = 0; i != GOMP_DIM_MAX; i++)
	121	if (mask & GOMP_DIM_MASK (i))
	122	dims[i] = va_arg (ap, unsigned);
	123	}
	124	break;
	125
	126	case GOMP_LAUNCH_ASYNC:
	127	{
	128	/* Small constant values are encoded in the operand. */
	129	async = GOMP_LAUNCH_OP (tag);
	130
	131	if (async == GOMP_LAUNCH_OP_MAX)
	132	async = va_arg (ap, unsigned);
	133	break;
	134	}
	135
	136	case GOMP_LAUNCH_WAIT:
	137	{
	138	unsigned num_waits = GOMP_LAUNCH_OP (tag);
	139
	140	if (num_waits)
	141	goacc_wait (async, num_waits, &ap);
	142	break;
	143	}
	144
	145	default:
	146	gomp_fatal ("unrecognized offload code '%d',"
	147	" libgomp is too old", GOMP_LAUNCH_CODE (tag));
	148	}
a091118d	149	}
3e32ee19	150	va_end (ap);
41dbbb37	151
41dbbb37 TS	152	acc_dev->openacc.async_set_async_func (async);
	153
	154	if (!(acc_dev->capabilities & GOMP_OFFLOAD_CAP_NATIVE_EXEC))
	155	{
	156	k.host_start = (uintptr_t) fn;
	157	k.host_end = k.host_start + 1;
a51df54e IV	158	gomp_mutex_lock (&acc_dev->lock);
	159	tgt_fn_key = splay_tree_lookup (&acc_dev->mem_map, &k);
	160	gomp_mutex_unlock (&acc_dev->lock);
41dbbb37 TS	161
	162	if (tgt_fn_key == NULL)
	163	gomp_fatal ("target function wasn't mapped");
	164
d93bdab5	165	tgt_fn = (void (*)) tgt_fn_key->tgt_offset;
41dbbb37 TS	166	}
	167	else
	168	tgt_fn = (void (*)) fn;
	169
	170	tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true,
d9a6bd32	171	GOMP_MAP_VARS_OPENACC);
41dbbb37	172
6e36114c	173	devaddrs = gomp_alloca (sizeof (void ) mapnum);
41dbbb37	174	for (i = 0; i < mapnum; i++)
d9a6bd32 JJ	175	devaddrs[i] = (void *) (tgt->list[i].key->tgt->tgt_start
d9a6bd32 JJ	176	+ tgt->list[i].key->tgt_offset);
41dbbb37	177
5c06742f NS	178	acc_dev->openacc.exec_func (tgt_fn, mapnum, hostaddrs, devaddrs,
5c06742f NS	179	async, dims, tgt);
41dbbb37 TS	180
	181	/* If running synchronously, unmap immediately. */
	182	if (async < acc_async_noval)
	183	gomp_unmap_vars (tgt, true);
	184	else
	185	{
	186	gomp_copy_from_async (tgt);
	187	acc_dev->openacc.register_async_cleanup_func (tgt);
	188	}
	189
	190	acc_dev->openacc.async_set_async_func (acc_async_sync);
	191	}
	192
3e32ee19 NS	193	/* Legacy entry point, only provide host execution. */
	194
	195	void
	196	GOACC_parallel (int device, void (fn) (void ),
	197	size_t mapnum, void *hostaddrs, size_t sizes,
	198	unsigned short *kinds,
	199	int num_gangs, int num_workers, int vector_length,
	200	int async, int num_waits, ...)
	201	{
	202	goacc_save_and_set_bind (acc_device_host);
	203	fn (hostaddrs);
	204	goacc_restore_bind ();
	205	}
	206
41dbbb37	207	void
128b26dc	208	GOACC_data_start (int device, size_t mapnum,
41dbbb37 TS	209	void *hostaddrs, size_t sizes, unsigned short *kinds)
	210	{
	211	bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
	212	struct target_mem_desc *tgt;
	213
01c0b3b0 KT	214	#ifdef HAVE_INTTYPES_H
	215	gomp_debug (0, "%s: mapnum=%"PRIu64", hostaddrs=%p, size=%p, kinds=%p\n",
	216	__FUNCTION__, (uint64_t) mapnum, hostaddrs, sizes, kinds);
	217	#else
	218	gomp_debug (0, "%s: mapnum=%lu, hostaddrs=%p, sizes=%p, kinds=%p\n",
	219	__FUNCTION__, (unsigned long) mapnum, hostaddrs, sizes, kinds);
	220	#endif
41dbbb37	221
d93bdab5	222	goacc_lazy_initialize ();
41dbbb37 TS	223
	224	struct goacc_thread *thr = goacc_thread ();
	225	struct gomp_device_descr *acc_dev = thr->dev;
	226
	227	/* Host fallback or 'do nothing'. */
	228	if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
	229	\|\| host_fallback)
	230	{
d9a6bd32 JJ	231	tgt = gomp_map_vars (NULL, 0, NULL, NULL, NULL, NULL, true,
d9a6bd32 JJ	232	GOMP_MAP_VARS_OPENACC);
41dbbb37 TS	233	tgt->prev = thr->mapped_data;
	234	thr->mapped_data = tgt;
	235
	236	return;
	237	}
	238
	239	gomp_debug (0, " %s: prepare mappings\n", __FUNCTION__);
	240	tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs, NULL, sizes, kinds, true,
d9a6bd32	241	GOMP_MAP_VARS_OPENACC);
41dbbb37 TS	242	gomp_debug (0, " %s: mappings prepared\n", __FUNCTION__);
	243	tgt->prev = thr->mapped_data;
	244	thr->mapped_data = tgt;
	245	}
	246
	247	void
	248	GOACC_data_end (void)
	249	{
	250	struct goacc_thread *thr = goacc_thread ();
	251	struct target_mem_desc *tgt = thr->mapped_data;
	252
	253	gomp_debug (0, " %s: restore mappings\n", __FUNCTION__);
	254	thr->mapped_data = tgt->prev;
	255	gomp_unmap_vars (tgt, true);
	256	gomp_debug (0, " %s: mappings restored\n", __FUNCTION__);
	257	}
	258
	259	void
128b26dc	260	GOACC_enter_exit_data (int device, size_t mapnum,
41dbbb37 TS	261	void *hostaddrs, size_t sizes, unsigned short *kinds,
	262	int async, int num_waits, ...)
	263	{
	264	struct goacc_thread *thr;
	265	struct gomp_device_descr *acc_dev;
	266	bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
	267	bool data_enter = false;
	268	size_t i;
	269
d93bdab5	270	goacc_lazy_initialize ();
41dbbb37 TS	271
	272	thr = goacc_thread ();
	273	acc_dev = thr->dev;
	274
	275	if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
	276	\|\| host_fallback)
	277	return;
	278
a091118d	279	if (num_waits)
41dbbb37 TS	280	{
	281	va_list ap;
	282
	283	va_start (ap, num_waits);
3e32ee19	284	goacc_wait (async, num_waits, &ap);
41dbbb37 TS	285	va_end (ap);
	286	}
	287
	288	acc_dev->openacc.async_set_async_func (async);
	289
	290	/* Determine if this is an "acc enter data". */
	291	for (i = 0; i < mapnum; ++i)
	292	{
	293	unsigned char kind = kinds[i] & 0xff;
	294
	295	if (kind == GOMP_MAP_POINTER \|\| kind == GOMP_MAP_TO_PSET)
	296	continue;
	297
	298	if (kind == GOMP_MAP_FORCE_ALLOC
	299	\|\| kind == GOMP_MAP_FORCE_PRESENT
	300	\|\| kind == GOMP_MAP_FORCE_TO)
	301	{
	302	data_enter = true;
	303	break;
	304	}
	305
	306	if (kind == GOMP_MAP_FORCE_DEALLOC
	307	\|\| kind == GOMP_MAP_FORCE_FROM)
	308	break;
	309
	310	gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
	311	kind);
	312	}
	313
	314	if (data_enter)
	315	{
	316	for (i = 0; i < mapnum; i++)
	317	{
	318	unsigned char kind = kinds[i] & 0xff;
	319
	320	/* Scan for PSETs. */
	321	int psets = find_pset (i, mapnum, kinds);
	322
	323	if (!psets)
	324	{
	325	switch (kind)
	326	{
	327	case GOMP_MAP_POINTER:
	328	gomp_acc_insert_pointer (1, &hostaddrs[i], &sizes[i],
	329	&kinds[i]);
	330	break;
	331	case GOMP_MAP_FORCE_ALLOC:
	332	acc_create (hostaddrs[i], sizes[i]);
	333	break;
	334	case GOMP_MAP_FORCE_PRESENT:
	335	acc_present_or_copyin (hostaddrs[i], sizes[i]);
	336	break;
	337	case GOMP_MAP_FORCE_TO:
	338	acc_present_or_copyin (hostaddrs[i], sizes[i]);
	339	break;
	340	default:
	341	gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
	342	kind);
	343	break;
	344	}
	345	}
	346	else
	347	{
	348	gomp_acc_insert_pointer (3, &hostaddrs[i], &sizes[i], &kinds[i]);
349	/* Increment 'i' by two because OpenACC requires fortran
350	arrays to be contiguous, so each PSET is associated with
351	one of MAP_FORCE_ALLOC/MAP_FORCE_PRESET/MAP_FORCE_TO, and
352	one MAP_POINTER. */
353	i += 2;
354	}
355	}
356	}
357	else
358	for (i = 0; i < mapnum; ++i)
359	{
360	unsigned char kind = kinds[i] & 0xff;
361
362	int psets = find_pset (i, mapnum, kinds);
363
364	if (!psets)
365	{
366	switch (kind)
367	{
368	case GOMP_MAP_POINTER:
369	gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff)
370	== GOMP_MAP_FORCE_FROM,
371	async, 1);
372	break;
373	case GOMP_MAP_FORCE_DEALLOC:
374	acc_delete (hostaddrs[i], sizes[i]);
375	break;
376	case GOMP_MAP_FORCE_FROM:
377	acc_copyout (hostaddrs[i], sizes[i]);
378	break;
379	default:
380	gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
381	kind);
382	break;
383	}
384	}
385	else
386	{
387	gomp_acc_remove_pointer (hostaddrs[i], (kinds[i] & 0xff)
388	== GOMP_MAP_FORCE_FROM, async, 3);
389	/* See the above comment. */
390	i += 2;
391	}
392	}
393
394	acc_dev->openacc.async_set_async_func (acc_async_sync);
395	}
396
397	static void
3e32ee19	398	goacc_wait (int async, int num_waits, va_list *ap)
41dbbb37 TS	399	{
	400	struct goacc_thread *thr = goacc_thread ();
	401	struct gomp_device_descr *acc_dev = thr->dev;
41dbbb37	402
a091118d	403	while (num_waits--)
41dbbb37	404	{
3e32ee19 NS	405	int qid = va_arg (*ap, int);
3e32ee19 NS	406
41dbbb37 TS	407	if (acc_async_test (qid))
	408	continue;
	409
a091118d NS	410	if (async == acc_async_sync)
	411	acc_wait (qid);
	412	else if (qid == async)
	413	;/* If we're waiting on the same asynchronous queue as we're
	414	launching on, the queue itself will order work as
	415	required, so there's no need to wait explicitly. */
	416	else
41dbbb37 TS	417	acc_dev->openacc.async_wait_async_func (qid, async);
	418	}
	419	}
	420
	421	void
128b26dc	422	GOACC_update (int device, size_t mapnum,
41dbbb37 TS	423	void *hostaddrs, size_t sizes, unsigned short *kinds,
	424	int async, int num_waits, ...)
	425	{
	426	bool host_fallback = device == GOMP_DEVICE_HOST_FALLBACK;
	427	size_t i;
	428
d93bdab5	429	goacc_lazy_initialize ();
41dbbb37 TS	430
	431	struct goacc_thread *thr = goacc_thread ();
	432	struct gomp_device_descr *acc_dev = thr->dev;
	433
	434	if ((acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
	435	\|\| host_fallback)
	436	return;
	437
a091118d	438	if (num_waits)
41dbbb37 TS	439	{
	440	va_list ap;
	441
	442	va_start (ap, num_waits);
3e32ee19	443	goacc_wait (async, num_waits, &ap);
41dbbb37 TS	444	va_end (ap);
	445	}
	446
	447	acc_dev->openacc.async_set_async_func (async);
	448
	449	for (i = 0; i < mapnum; ++i)
	450	{
	451	unsigned char kind = kinds[i] & 0xff;
	452
	453	switch (kind)
	454	{
	455	case GOMP_MAP_POINTER:
	456	case GOMP_MAP_TO_PSET:
	457	break;
	458
	459	case GOMP_MAP_FORCE_TO:
	460	acc_update_device (hostaddrs[i], sizes[i]);
	461	break;
	462
	463	case GOMP_MAP_FORCE_FROM:
	464	acc_update_self (hostaddrs[i], sizes[i]);
	465	break;
	466
	467	default:
	468	gomp_fatal (">>>> GOACC_update UNHANDLED kind 0x%.2x", kind);
	469	break;
	470	}
	471	}
	472
	473	acc_dev->openacc.async_set_async_func (acc_async_sync);
	474	}
	475
	476	void
	477	GOACC_wait (int async, int num_waits, ...)
	478	{
a091118d NS	479	if (num_waits)
	480	{
	481	va_list ap;
41dbbb37	482
a091118d	483	va_start (ap, num_waits);
3e32ee19	484	goacc_wait (async, num_waits, &ap);
a091118d NS	485	va_end (ap);
	486	}
	487	else if (async == acc_async_sync)
	488	acc_wait_all ();
	489	else if (async == acc_async_noval)
a051317b	490	goacc_thread ()->dev->openacc.async_wait_all_async_func (acc_async_noval);
41dbbb37 TS	491	}
	492
	493	int
	494	GOACC_get_num_threads (void)
	495	{
	496	return 1;
	497	}
	498
	499	int
	500	GOACC_get_thread_num (void)
	501	{
	502	return 0;
	503	}
6e232ba4 JN	504
	505	void
	506	GOACC_declare (int device, size_t mapnum,
	507	void *hostaddrs, size_t sizes, unsigned short *kinds)
	508	{
	509	int i;
	510
	511	for (i = 0; i < mapnum; i++)
	512	{
	513	unsigned char kind = kinds[i] & 0xff;
	514
	515	if (kind == GOMP_MAP_POINTER \|\| kind == GOMP_MAP_TO_PSET)
	516	continue;
	517
	518	switch (kind)
	519	{
	520	case GOMP_MAP_FORCE_ALLOC:
	521	case GOMP_MAP_FORCE_DEALLOC:
	522	case GOMP_MAP_FORCE_FROM:
	523	case GOMP_MAP_FORCE_TO:
	524	case GOMP_MAP_POINTER:
	525	GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
	526	&kinds[i], 0, 0);
	527	break;
	528
	529	case GOMP_MAP_FORCE_DEVICEPTR:
	530	break;
	531
	532	case GOMP_MAP_ALLOC:
	533	if (!acc_is_present (hostaddrs[i], sizes[i]))
	534	GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
	535	&kinds[i], 0, 0);
	536	break;
	537
	538	case GOMP_MAP_TO:
	539	GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
	540	&kinds[i], 0, 0);
	541
	542	break;
	543
	544	case GOMP_MAP_FROM:
	545	kinds[i] = GOMP_MAP_FORCE_FROM;
	546	GOACC_enter_exit_data (device, 1, &hostaddrs[i], &sizes[i],
	547	&kinds[i], 0, 0);
	548	break;
	549
	550	case GOMP_MAP_FORCE_PRESENT:
	551	if (!acc_is_present (hostaddrs[i], sizes[i]))
	552	gomp_fatal ("[%p,%ld] is not mapped", hostaddrs[i],
	553	(unsigned long) sizes[i]);
	554	break;
	555
	556	default:
	557	assert (0);
	558	break;
	559	}
	560	}
	561	}