libgomp/oacc-mem.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2018 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "openacc.h"
  30 #include "config.h"
  31 #include "libgomp.h"
  32 #include "gomp-constants.h"
  33 #include "oacc-int.h"
  34 #include <stdint.h>
  35 #include <string.h>
  36 #include <assert.h>
  37
  38 /* Return block containing [H->S), or NULL if not contained.  The device lock
  39    for DEV must be locked on entry, and remains locked on exit.  */
  40
  41 static splay_tree_key
  42 lookup_host (struct gomp_device_descr *dev, void *h, size_t s)
  43 {
  44   struct splay_tree_key_s node;
  45   splay_tree_key key;
  46
  47   node.host_start = (uintptr_t) h;
  48   node.host_end = (uintptr_t) h + s;
  49
  50   key = splay_tree_lookup (&dev->mem_map, &node);
  51
  52   return key;
  53 }
  54
  55 /* Return block containing [D->S), or NULL if not contained.
  56    The list isn't ordered by device address, so we have to iterate
  57    over the whole array.  This is not expected to be a common
  58    operation.  The device lock associated with TGT must be locked on entry, and
  59    remains locked on exit.  */
  60
  61 static splay_tree_key
  62 lookup_dev (struct target_mem_desc *tgt, void *d, size_t s)
  63 {
  64   int i;
  65   struct target_mem_desc *t;
  66
  67   if (!tgt)
  68     return NULL;
  69
  70   for (t = tgt; t != NULL; t = t->prev)
  71     {
  72       if (t->tgt_start <= (uintptr_t) d && t->tgt_end >= (uintptr_t) d + s)
  73         break;
  74     }
  75
  76   if (!t)
  77     return NULL;
  78
  79   for (i = 0; i < t->list_count; i++)
  80     {
  81       void * offset;
  82
  83       splay_tree_key k = &t->array[i].key;
  84       offset = d - t->tgt_start + k->tgt_offset;
  85
  86       if (k->host_start + offset <= (void *) k->host_end)
  87         return k;
  88     }
  89
  90   return NULL;
  91 }
  92
  93 /* OpenACC is silent on how memory exhaustion is indicated.  We return
  94    NULL.  */
  95
  96 void *
  97 acc_malloc (size_t s)
  98 {
  99   if (!s)
 100     return NULL;
 101
 102   goacc_lazy_initialize ();
 103
 104   struct goacc_thread *thr = goacc_thread ();
 105
 106   assert (thr->dev);
 107
 108   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 109     return malloc (s);
 110
 111   return thr->dev->alloc_func (thr->dev->target_id, s);
 112 }
 113
 114 /* OpenACC 2.0a (3.2.16) doesn't specify what to do in the event
 115    the device address is mapped. We choose to check if it mapped,
 116    and if it is, to unmap it. */
 117 void
 118 acc_free (void *d)
 119 {
 120   splay_tree_key k;
 121
 122   if (!d)
 123     return;
 124
 125   struct goacc_thread *thr = goacc_thread ();
 126
 127   assert (thr && thr->dev);
 128
 129   struct gomp_device_descr *acc_dev = thr->dev;
 130
 131   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 132     return free (d);
 133
 134   gomp_mutex_lock (&acc_dev->lock);
 135
 136   /* We don't have to call lazy open here, as the ptr value must have
 137      been returned by acc_malloc.  It's not permitted to pass NULL in
 138      (unless you got that null from acc_malloc).  */
 139   if ((k = lookup_dev (acc_dev->openacc.data_environ, d, 1)))
 140     {
 141       void *offset;
 142
 143       offset = d - k->tgt->tgt_start + k->tgt_offset;
 144
 145       gomp_mutex_unlock (&acc_dev->lock);
 146
 147       acc_unmap_data ((void *)(k->host_start + offset));
 148     }
 149   else
 150     gomp_mutex_unlock (&acc_dev->lock);
 151
 152   if (!acc_dev->free_func (acc_dev->target_id, d))
 153     gomp_fatal ("error in freeing device memory in %s", __FUNCTION__);
 154 }
 155
 156 static void
 157 memcpy_tofrom_device (bool from, void *d, void *h, size_t s, int async,
 158                       const char *libfnname)
 159 {
 160   /* No need to call lazy open here, as the device pointer must have
 161      been obtained from a routine that did that.  */
 162   struct goacc_thread *thr = goacc_thread ();
 163
 164   assert (thr && thr->dev);
 165
 166   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 167     {
 168       if (from)
 169         memmove (h, d, s);
 170       else
 171         memmove (d, h, s);
 172       return;
 173     }
 174
 175   if (async > acc_async_sync)
 176     thr->dev->openacc.async_set_async_func (async);
 177
 178   bool ret = (from
 179               ? thr->dev->dev2host_func (thr->dev->target_id, h, d, s)
 180               : thr->dev->host2dev_func (thr->dev->target_id, d, h, s));
 181
 182   if (async > acc_async_sync)
 183     thr->dev->openacc.async_set_async_func (acc_async_sync);
 184
 185   if (!ret)
 186     gomp_fatal ("error in %s", libfnname);
 187 }
 188
 189 void
 190 acc_memcpy_to_device (void *d, void *h, size_t s)
 191 {
 192   memcpy_tofrom_device (false, d, h, s, acc_async_sync, __FUNCTION__);
 193 }
 194
 195 void
 196 acc_memcpy_to_device_async (void *d, void *h, size_t s, int async)
 197 {
 198   memcpy_tofrom_device (false, d, h, s, async, __FUNCTION__);
 199 }
 200
 201 void
 202 acc_memcpy_from_device (void *h, void *d, size_t s)
 203 {
 204   memcpy_tofrom_device (true, d, h, s, acc_async_sync, __FUNCTION__);
 205 }
 206
 207 void
 208 acc_memcpy_from_device_async (void *h, void *d, size_t s, int async)
 209 {
 210   memcpy_tofrom_device (true, d, h, s, async, __FUNCTION__);
 211 }
 212
 213 /* Return the device pointer that corresponds to host data H.  Or NULL
 214    if no mapping.  */
 215
 216 void *
 217 acc_deviceptr (void *h)
 218 {
 219   splay_tree_key n;
 220   void *d;
 221   void *offset;
 222
 223   goacc_lazy_initialize ();
 224
 225   struct goacc_thread *thr = goacc_thread ();
 226   struct gomp_device_descr *dev = thr->dev;
 227
 228   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 229     return h;
 230
 231   gomp_mutex_lock (&dev->lock);
 232
 233   n = lookup_host (dev, h, 1);
 234
 235   if (!n)
 236     {
 237       gomp_mutex_unlock (&dev->lock);
 238       return NULL;
 239     }
 240
 241   offset = h - n->host_start;
 242
 243   d = n->tgt->tgt_start + n->tgt_offset + offset;
 244
 245   gomp_mutex_unlock (&dev->lock);
 246
 247   return d;
 248 }
 249
 250 /* Return the host pointer that corresponds to device data D.  Or NULL
 251    if no mapping.  */
 252
 253 void *
 254 acc_hostptr (void *d)
 255 {
 256   splay_tree_key n;
 257   void *h;
 258   void *offset;
 259
 260   goacc_lazy_initialize ();
 261
 262   struct goacc_thread *thr = goacc_thread ();
 263   struct gomp_device_descr *acc_dev = thr->dev;
 264
 265   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 266     return d;
 267
 268   gomp_mutex_lock (&acc_dev->lock);
 269
 270   n = lookup_dev (acc_dev->openacc.data_environ, d, 1);
 271
 272   if (!n)
 273     {
 274       gomp_mutex_unlock (&acc_dev->lock);
 275       return NULL;
 276     }
 277
 278   offset = d - n->tgt->tgt_start + n->tgt_offset;
 279
 280   h = n->host_start + offset;
 281
 282   gomp_mutex_unlock (&acc_dev->lock);
 283
 284   return h;
 285 }
 286
 287 /* Return 1 if host data [H,+S] is present on the device.  */
 288
 289 int
 290 acc_is_present (void *h, size_t s)
 291 {
 292   splay_tree_key n;
 293
 294   if (!s || !h)
 295     return 0;
 296
 297   goacc_lazy_initialize ();
 298
 299   struct goacc_thread *thr = goacc_thread ();
 300   struct gomp_device_descr *acc_dev = thr->dev;
 301
 302   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 303     return h != NULL;
 304
 305   gomp_mutex_lock (&acc_dev->lock);
 306
 307   n = lookup_host (acc_dev, h, s);
 308
 309   if (n && ((uintptr_t)h < n->host_start
 310             || (uintptr_t)h + s > n->host_end
 311             || s > n->host_end - n->host_start))
 312     n = NULL;
 313
 314   gomp_mutex_unlock (&acc_dev->lock);
 315
 316   return n != NULL;
 317 }
 318
 319 /* Create a mapping for host [H,+S] -> device [D,+S] */
 320
 321 void
 322 acc_map_data (void *h, void *d, size_t s)
 323 {
 324   struct target_mem_desc *tgt = NULL;
 325   size_t mapnum = 1;
 326   void *hostaddrs = h;
 327   void *devaddrs = d;
 328   size_t sizes = s;
 329   unsigned short kinds = GOMP_MAP_ALLOC;
 330
 331   goacc_lazy_initialize ();
 332
 333   struct goacc_thread *thr = goacc_thread ();
 334   struct gomp_device_descr *acc_dev = thr->dev;
 335
 336   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 337     {
 338       if (d != h)
 339         gomp_fatal ("cannot map data on shared-memory system");
 340     }
 341   else
 342     {
 343       struct goacc_thread *thr = goacc_thread ();
 344
 345       if (!d || !h || !s)
 346         gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
 347                     (void *)h, (int)s, (void *)d, (int)s);
 348
 349       gomp_mutex_lock (&acc_dev->lock);
 350
 351       if (lookup_host (acc_dev, h, s))
 352         {
 353           gomp_mutex_unlock (&acc_dev->lock);
 354           gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h,
 355                       (int)s);
 356         }
 357
 358       if (lookup_dev (thr->dev->openacc.data_environ, d, s))
 359         {
 360           gomp_mutex_unlock (&acc_dev->lock);
 361           gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d,
 362                       (int)s);
 363         }
 364
 365       gomp_mutex_unlock (&acc_dev->lock);
 366
 367       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, &devaddrs, &sizes,
 368                            &kinds, true, GOMP_MAP_VARS_OPENACC);
 369       tgt->list[0].key->refcount = REFCOUNT_INFINITY;
 370     }
 371
 372   gomp_mutex_lock (&acc_dev->lock);
 373   tgt->prev = acc_dev->openacc.data_environ;
 374   acc_dev->openacc.data_environ = tgt;
 375   gomp_mutex_unlock (&acc_dev->lock);
 376 }
 377
 378 void
 379 acc_unmap_data (void *h)
 380 {
 381   struct goacc_thread *thr = goacc_thread ();
 382   struct gomp_device_descr *acc_dev = thr->dev;
 383
 384   /* No need to call lazy open, as the address must have been mapped.  */
 385
 386   /* This is a no-op on shared-memory targets.  */
 387   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 388     return;
 389
 390   size_t host_size;
 391
 392   gomp_mutex_lock (&acc_dev->lock);
 393
 394   splay_tree_key n = lookup_host (acc_dev, h, 1);
 395   struct target_mem_desc *t;
 396
 397   if (!n)
 398     {
 399       gomp_mutex_unlock (&acc_dev->lock);
 400       gomp_fatal ("%p is not a mapped block", (void *)h);
 401     }
 402
 403   host_size = n->host_end - n->host_start;
 404
 405   if (n->host_start != (uintptr_t) h)
 406     {
 407       gomp_mutex_unlock (&acc_dev->lock);
 408       gomp_fatal ("[%p,%d] surrounds %p",
 409                   (void *) n->host_start, (int) host_size, (void *) h);
 410     }
 411
 412   /* Mark for removal.  */
 413   n->refcount = 1;
 414
 415   t = n->tgt;
 416
 417   if (t->refcount == 2)
 418     {
 419       struct target_mem_desc *tp;
 420
 421       /* This is the last reference, so pull the descriptor off the
 422          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 423          freeing the device memory. */
 424       t->tgt_end = 0;
 425       t->to_free = 0;
 426
 427       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 428            tp = t, t = t->prev)
 429         if (n->tgt == t)
 430           {
 431             if (tp)
 432               tp->prev = t->prev;
 433             else
 434               acc_dev->openacc.data_environ = t->prev;
 435
 436             break;
 437           }
 438     }
 439
 440   gomp_mutex_unlock (&acc_dev->lock);
 441
 442   gomp_unmap_vars (t, true);
 443 }
 444
 445 #define FLAG_PRESENT (1 << 0)
 446 #define FLAG_CREATE (1 << 1)
 447 #define FLAG_COPY (1 << 2)
 448
 449 static void *
 450 present_create_copy (unsigned f, void *h, size_t s, int async)
 451 {
 452   void *d;
 453   splay_tree_key n;
 454
 455   if (!h || !s)
 456     gomp_fatal ("[%p,+%d] is a bad range", (void *)h, (int)s);
 457
 458   goacc_lazy_initialize ();
 459
 460   struct goacc_thread *thr = goacc_thread ();
 461   struct gomp_device_descr *acc_dev = thr->dev;
 462
 463   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 464     return h;
 465
 466   gomp_mutex_lock (&acc_dev->lock);
 467
 468   n = lookup_host (acc_dev, h, s);
 469   if (n)
 470     {
 471       /* Present. */
 472       d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 473
 474       if (!(f & FLAG_PRESENT))
 475         {
 476           gomp_mutex_unlock (&acc_dev->lock);
 477           gomp_fatal ("[%p,+%d] already mapped to [%p,+%d]",
 478                       (void *)h, (int)s, (void *)d, (int)s);
 479         }
 480       if ((h + s) > (void *)n->host_end)
 481         {
 482           gomp_mutex_unlock (&acc_dev->lock);
 483           gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 484         }
 485
 486       if (n->refcount != REFCOUNT_INFINITY)
 487         {
 488           n->refcount++;
 489           n->dynamic_refcount++;
 490         }
 491       gomp_mutex_unlock (&acc_dev->lock);
 492     }
 493   else if (!(f & FLAG_CREATE))
 494     {
 495       gomp_mutex_unlock (&acc_dev->lock);
 496       gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 497     }
 498   else
 499     {
 500       struct target_mem_desc *tgt;
 501       size_t mapnum = 1;
 502       unsigned short kinds;
 503       void *hostaddrs = h;
 504
 505       if (f & FLAG_COPY)
 506         kinds = GOMP_MAP_TO;
 507       else
 508         kinds = GOMP_MAP_ALLOC;
 509
 510       gomp_mutex_unlock (&acc_dev->lock);
 511
 512       if (async > acc_async_sync)
 513         acc_dev->openacc.async_set_async_func (async);
 514
 515       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, NULL, &s, &kinds, true,
 516                            GOMP_MAP_VARS_OPENACC);
 517       /* Initialize dynamic refcount.  */
 518       tgt->list[0].key->dynamic_refcount = 1;
 519
 520       if (async > acc_async_sync)
 521         acc_dev->openacc.async_set_async_func (acc_async_sync);
 522
 523       gomp_mutex_lock (&acc_dev->lock);
 524
 525       d = tgt->to_free;
 526       tgt->prev = acc_dev->openacc.data_environ;
 527       acc_dev->openacc.data_environ = tgt;
 528
 529       gomp_mutex_unlock (&acc_dev->lock);
 530     }
 531
 532   return d;
 533 }
 534
 535 void *
 536 acc_create (void *h, size_t s)
 537 {
 538   return present_create_copy (FLAG_PRESENT | FLAG_CREATE, h, s, acc_async_sync);
 539 }
 540
 541 void
 542 acc_create_async (void *h, size_t s, int async)
 543 {
 544   present_create_copy (FLAG_PRESENT | FLAG_CREATE, h, s, async);
 545 }
 546
 547 void *
 548 acc_copyin (void *h, size_t s)
 549 {
 550   return present_create_copy (FLAG_PRESENT | FLAG_CREATE | FLAG_COPY, h, s,
 551                               acc_async_sync);
 552 }
 553
 554 void
 555 acc_copyin_async (void *h, size_t s, int async)
 556 {
 557   present_create_copy (FLAG_PRESENT | FLAG_CREATE | FLAG_COPY, h, s, async);
 558 }
 559
 560 void *
 561 acc_present_or_create (void *h, size_t s)
 562 {
 563   return present_create_copy (FLAG_PRESENT | FLAG_CREATE, h, s, acc_async_sync);
 564 }
 565
 566 /* acc_pcreate is acc_present_or_create by a different name.  */
 567 #ifdef HAVE_ATTRIBUTE_ALIAS
 568 strong_alias (acc_present_or_create, acc_pcreate)
 569 #else
 570 void *
 571 acc_pcreate (void *h, size_t s)
 572 {
 573   return acc_present_or_create (h, s);
 574 }
 575 #endif
 576
 577 void *
 578 acc_present_or_copyin (void *h, size_t s)
 579 {
 580   return present_create_copy (FLAG_PRESENT | FLAG_CREATE | FLAG_COPY, h, s,
 581                               acc_async_sync);
 582 }
 583
 584 /* acc_pcopyin is acc_present_or_copyin by a different name.  */
 585 #ifdef HAVE_ATTRIBUTE_ALIAS
 586 strong_alias (acc_present_or_copyin, acc_pcopyin)
 587 #else
 588 void *
 589 acc_pcopyin (void *h, size_t s)
 590 {
 591   return acc_present_or_copyin (h, s);
 592 }
 593 #endif
 594
 595 #define FLAG_COPYOUT  (1 << 0)
 596 #define FLAG_FINALIZE (1 << 1)
 597
 598 static void
 599 delete_copyout (unsigned f, void *h, size_t s, int async, const char *libfnname)
 600 {
 601   size_t host_size;
 602   splay_tree_key n;
 603   void *d;
 604   struct goacc_thread *thr = goacc_thread ();
 605   struct gomp_device_descr *acc_dev = thr->dev;
 606
 607   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 608     return;
 609
 610   gomp_mutex_lock (&acc_dev->lock);
 611
 612   n = lookup_host (acc_dev, h, s);
 613
 614   /* No need to call lazy open, as the data must already have been
 615      mapped.  */
 616
 617   if (!n)
 618     {
 619       gomp_mutex_unlock (&acc_dev->lock);
 620       gomp_fatal ("[%p,%d] is not mapped", (void *)h, (int)s);
 621     }
 622
 623   d = (void *) (n->tgt->tgt_start + n->tgt_offset
 624                 + (uintptr_t) h - n->host_start);
 625
 626   host_size = n->host_end - n->host_start;
 627
 628   if (n->host_start != (uintptr_t) h || host_size != s)
 629     {
 630       gomp_mutex_unlock (&acc_dev->lock);
 631       gomp_fatal ("[%p,%d] surrounds2 [%p,+%d]",
 632                   (void *) n->host_start, (int) host_size, (void *) h, (int) s);
 633     }
 634
 635   if (n->refcount == REFCOUNT_INFINITY)
 636     {
 637       n->refcount = 0;
 638       n->dynamic_refcount = 0;
 639     }
 640   if (n->refcount < n->dynamic_refcount)
 641     {
 642       gomp_mutex_unlock (&acc_dev->lock);
 643       gomp_fatal ("Dynamic reference counting assert fail\n");
 644     }
 645
 646   if (f & FLAG_FINALIZE)
 647     {
 648       n->refcount -= n->dynamic_refcount;
 649       n->dynamic_refcount = 0;
 650     }
 651   else if (n->dynamic_refcount)
 652     {
 653       n->dynamic_refcount--;
 654       n->refcount--;
 655     }
 656
 657   if (n->refcount == 0)
 658     {
 659       if (n->tgt->refcount == 2)
 660         {
 661           struct target_mem_desc *tp, *t;
 662           for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 663                tp = t, t = t->prev)
 664             if (n->tgt == t)
 665               {
 666                 if (tp)
 667                   tp->prev = t->prev;
 668                 else
 669                   acc_dev->openacc.data_environ = t->prev;
 670                 break;
 671               }
 672         }
 673
 674       if (f & FLAG_COPYOUT)
 675         {
 676           if (async > acc_async_sync)
 677             acc_dev->openacc.async_set_async_func (async);
 678           acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 679           if (async > acc_async_sync)
 680             acc_dev->openacc.async_set_async_func (acc_async_sync);
 681         }
 682
 683       gomp_remove_var (acc_dev, n);
 684     }
 685
 686   gomp_mutex_unlock (&acc_dev->lock);
 687 }
 688
 689 void
 690 acc_delete (void *h , size_t s)
 691 {
 692   delete_copyout (0, h, s, acc_async_sync, __FUNCTION__);
 693 }
 694
 695 void
 696 acc_delete_async (void *h , size_t s, int async)
 697 {
 698   delete_copyout (0, h, s, async, __FUNCTION__);
 699 }
 700
 701 void
 702 acc_delete_finalize (void *h , size_t s)
 703 {
 704   delete_copyout (FLAG_FINALIZE, h, s, acc_async_sync, __FUNCTION__);
 705 }
 706
 707 void
 708 acc_delete_finalize_async (void *h , size_t s, int async)
 709 {
 710   delete_copyout (FLAG_FINALIZE, h, s, async, __FUNCTION__);
 711 }
 712
 713 void
 714 acc_copyout (void *h, size_t s)
 715 {
 716   delete_copyout (FLAG_COPYOUT, h, s, acc_async_sync, __FUNCTION__);
 717 }
 718
 719 void
 720 acc_copyout_async (void *h, size_t s, int async)
 721 {
 722   delete_copyout (FLAG_COPYOUT, h, s, async, __FUNCTION__);
 723 }
 724
 725 void
 726 acc_copyout_finalize (void *h, size_t s)
 727 {
 728   delete_copyout (FLAG_COPYOUT | FLAG_FINALIZE, h, s, acc_async_sync,
 729                   __FUNCTION__);
 730 }
 731
 732 void
 733 acc_copyout_finalize_async (void *h, size_t s, int async)
 734 {
 735   delete_copyout (FLAG_COPYOUT | FLAG_FINALIZE, h, s, async, __FUNCTION__);
 736 }
 737
 738 static void
 739 update_dev_host (int is_dev, void *h, size_t s, int async)
 740 {
 741   splay_tree_key n;
 742   void *d;
 743
 744   goacc_lazy_initialize ();
 745
 746   struct goacc_thread *thr = goacc_thread ();
 747   struct gomp_device_descr *acc_dev = thr->dev;
 748
 749   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 750     return;
 751
 752   gomp_mutex_lock (&acc_dev->lock);
 753
 754   n = lookup_host (acc_dev, h, s);
 755
 756   if (!n)
 757     {
 758       gomp_mutex_unlock (&acc_dev->lock);
 759       gomp_fatal ("[%p,%d] is not mapped", h, (int)s);
 760     }
 761
 762   d = (void *) (n->tgt->tgt_start + n->tgt_offset
 763                 + (uintptr_t) h - n->host_start);
 764
 765   if (async > acc_async_sync)
 766     acc_dev->openacc.async_set_async_func (async);
 767
 768   if (is_dev)
 769     acc_dev->host2dev_func (acc_dev->target_id, d, h, s);
 770   else
 771     acc_dev->dev2host_func (acc_dev->target_id, h, d, s);
 772
 773   if (async > acc_async_sync)
 774     acc_dev->openacc.async_set_async_func (acc_async_sync);
 775
 776   gomp_mutex_unlock (&acc_dev->lock);
 777 }
 778
 779 void
 780 acc_update_device (void *h, size_t s)
 781 {
 782   update_dev_host (1, h, s, acc_async_sync);
 783 }
 784
 785 void
 786 acc_update_device_async (void *h, size_t s, int async)
 787 {
 788   update_dev_host (1, h, s, async);
 789 }
 790
 791 void
 792 acc_update_self (void *h, size_t s)
 793 {
 794   update_dev_host (0, h, s, acc_async_sync);
 795 }
 796
 797 void
 798 acc_update_self_async (void *h, size_t s, int async)
 799 {
 800   update_dev_host (0, h, s, async);
 801 }
 802
 803 void
 804 gomp_acc_insert_pointer (size_t mapnum, void **hostaddrs, size_t *sizes,
 805                          void *kinds)
 806 {
 807   struct target_mem_desc *tgt;
 808   struct goacc_thread *thr = goacc_thread ();
 809   struct gomp_device_descr *acc_dev = thr->dev;
 810
 811   if (acc_is_present (*hostaddrs, *sizes))
 812     {
 813       splay_tree_key n;
 814       gomp_mutex_lock (&acc_dev->lock);
 815       n = lookup_host (acc_dev, *hostaddrs, *sizes);
 816       gomp_mutex_unlock (&acc_dev->lock);
 817
 818       tgt = n->tgt;
 819       for (size_t i = 0; i < tgt->list_count; i++)
 820         if (tgt->list[i].key == n)
 821           {
 822             for (size_t j = 0; j < mapnum; j++)
 823               if (i + j < tgt->list_count && tgt->list[i + j].key)
 824                 {
 825                   tgt->list[i + j].key->refcount++;
 826                   tgt->list[i + j].key->dynamic_refcount++;
 827                 }
 828             return;
 829           }
 830       /* Should not reach here.  */
 831       gomp_fatal ("Dynamic refcount incrementing failed for pointer/pset");
 832     }
 833
 834   gomp_debug (0, "  %s: prepare mappings\n", __FUNCTION__);
 835   tgt = gomp_map_vars (acc_dev, mapnum, hostaddrs,
 836                        NULL, sizes, kinds, true, GOMP_MAP_VARS_OPENACC);
 837   gomp_debug (0, "  %s: mappings prepared\n", __FUNCTION__);
 838
 839   /* Initialize dynamic refcount.  */
 840   tgt->list[0].key->dynamic_refcount = 1;
 841
 842   gomp_mutex_lock (&acc_dev->lock);
 843   tgt->prev = acc_dev->openacc.data_environ;
 844   acc_dev->openacc.data_environ = tgt;
 845   gomp_mutex_unlock (&acc_dev->lock);
 846 }
 847
 848 void
 849 gomp_acc_remove_pointer (void *h, size_t s, bool force_copyfrom, int async,
 850                          int finalize, int mapnum)
 851 {
 852   struct goacc_thread *thr = goacc_thread ();
 853   struct gomp_device_descr *acc_dev = thr->dev;
 854   splay_tree_key n;
 855   struct target_mem_desc *t;
 856   int minrefs = (mapnum == 1) ? 2 : 3;
 857
 858   if (!acc_is_present (h, s))
 859     return;
 860
 861   gomp_mutex_lock (&acc_dev->lock);
 862
 863   n = lookup_host (acc_dev, h, 1);
 864
 865   if (!n)
 866     {
 867       gomp_mutex_unlock (&acc_dev->lock);
 868       gomp_fatal ("%p is not a mapped block", (void *)h);
 869     }
 870
 871   gomp_debug (0, "  %s: restore mappings\n", __FUNCTION__);
 872
 873   t = n->tgt;
 874
 875   if (n->refcount < n->dynamic_refcount)
 876     {
 877       gomp_mutex_unlock (&acc_dev->lock);
 878       gomp_fatal ("Dynamic reference counting assert fail\n");
 879     }
 880
 881   if (finalize)
 882     {
 883       n->refcount -= n->dynamic_refcount;
 884       n->dynamic_refcount = 0;
 885     }
 886   else if (n->dynamic_refcount)
 887     {
 888       n->dynamic_refcount--;
 889       n->refcount--;
 890     }
 891
 892   gomp_mutex_unlock (&acc_dev->lock);
 893
 894   if (n->refcount == 0)
 895     {
 896       if (t->refcount == minrefs)
 897         {
 898           /* This is the last reference, so pull the descriptor off the
 899              chain. This prevents gomp_unmap_vars via gomp_unmap_tgt from
 900              freeing the device memory. */
 901           struct target_mem_desc *tp;
 902           for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 903                tp = t, t = t->prev)
 904             {
 905               if (n->tgt == t)
 906                 {
 907                   if (tp)
 908                     tp->prev = t->prev;
 909                   else
 910                     acc_dev->openacc.data_environ = t->prev;
 911                   break;
 912                 }
 913             }
 914         }
 915
 916       /* Set refcount to 1 to allow gomp_unmap_vars to unmap it.  */
 917       n->refcount = 1;
 918       t->refcount = minrefs;
 919       for (size_t i = 0; i < t->list_count; i++)
 920         if (t->list[i].key == n)
 921           {
 922             t->list[i].copy_from = force_copyfrom ? 1 : 0;
 923             break;
 924           }
 925
 926       /* If running synchronously, unmap immediately.  */
 927       if (async < acc_async_noval)
 928         gomp_unmap_vars (t, true);
 929       else
 930         t->device_descr->openacc.register_async_cleanup_func (t, async);
 931     }
 932
 933   gomp_mutex_unlock (&acc_dev->lock);
 934
 935   gomp_debug (0, "  %s: mappings restored\n", __FUNCTION__);
 936 }