libgomp/oacc-mem.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2019 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "openacc.h"
  30 #include "config.h"
  31 #include "libgomp.h"
  32 #include "gomp-constants.h"
  33 #include "oacc-int.h"
  34 #include <stdint.h>
  35 #include <string.h>
  36 #include <assert.h>
  37
  38 /* Return block containing [H->S), or NULL if not contained.  The device lock
  39    for DEV must be locked on entry, and remains locked on exit.  */
  40
  41 static splay_tree_key
  42 lookup_host (struct gomp_device_descr *dev, void *h, size_t s)
  43 {
  44   struct splay_tree_key_s node;
  45   splay_tree_key key;
  46
  47   node.host_start = (uintptr_t) h;
  48   node.host_end = (uintptr_t) h + s;
  49
  50   key = splay_tree_lookup (&dev->mem_map, &node);
  51
  52   return key;
  53 }
  54
  55 /* Return block containing [D->S), or NULL if not contained.
  56    The list isn't ordered by device address, so we have to iterate
  57    over the whole array.  This is not expected to be a common
  58    operation.  The device lock associated with TGT must be locked on entry, and
  59    remains locked on exit.  */
  60
  61 static splay_tree_key
  62 lookup_dev (struct target_mem_desc *tgt, void *d, size_t s)
  63 {
  64   int i;
  65   struct target_mem_desc *t;
  66
  67   if (!tgt)
  68     return NULL;
  69
  70   for (t = tgt; t != NULL; t = t->prev)
  71     {
  72       if (t->tgt_start <= (uintptr_t) d && t->tgt_end >= (uintptr_t) d + s)
  73         break;
  74     }
  75
  76   if (!t)
  77     return NULL;
  78
  79   for (i = 0; i < t->list_count; i++)
  80     {
  81       void * offset;
  82
  83       splay_tree_key k = &t->array[i].key;
  84       offset = d - t->tgt_start + k->tgt_offset;
  85
  86       if (k->host_start + offset <= (void *) k->host_end)
  87         return k;
  88     }
  89
  90   return NULL;
  91 }
  92
  93 /* OpenACC is silent on how memory exhaustion is indicated.  We return
  94    NULL.  */
  95
  96 void *
  97 acc_malloc (size_t s)
  98 {
  99   if (!s)
 100     return NULL;
 101
 102   goacc_lazy_initialize ();
 103
 104   struct goacc_thread *thr = goacc_thread ();
 105
 106   assert (thr->dev);
 107
 108   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 109     return malloc (s);
 110
 111   return thr->dev->alloc_func (thr->dev->target_id, s);
 112 }
 113
 114 /* OpenACC 2.0a (3.2.16) doesn't specify what to do in the event
 115    the device address is mapped. We choose to check if it mapped,
 116    and if it is, to unmap it. */
 117 void
 118 acc_free (void *d)
 119 {
 120   splay_tree_key k;
 121
 122   if (!d)
 123     return;
 124
 125   struct goacc_thread *thr = goacc_thread ();
 126
 127   assert (thr && thr->dev);
 128
 129   struct gomp_device_descr *acc_dev = thr->dev;
 130
 131   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 132     return free (d);
 133
 134   gomp_mutex_lock (&acc_dev->lock);
 135
 136   /* We don't have to call lazy open here, as the ptr value must have
 137      been returned by acc_malloc.  It's not permitted to pass NULL in
 138      (unless you got that null from acc_malloc).  */
 139   if ((k = lookup_dev (acc_dev->openacc.data_environ, d, 1)))
 140     {
 141       void *offset;
 142
 143       offset = d - k->tgt->tgt_start + k->tgt_offset;
 144
 145       gomp_mutex_unlock (&acc_dev->lock);
 146
 147       acc_unmap_data ((void *)(k->host_start + offset));
 148     }
 149   else
 150     gomp_mutex_unlock (&acc_dev->lock);
 151
 152   if (!acc_dev->free_func (acc_dev->target_id, d))
 153     gomp_fatal ("error in freeing device memory in %s", __FUNCTION__);
 154 }
 155
 156 static void
 157 memcpy_tofrom_device (bool from, void *d, void *h, size_t s, int async,
 158                       const char *libfnname)
 159 {
 160   /* No need to call lazy open here, as the device pointer must have
 161      been obtained from a routine that did that.  */
 162   struct goacc_thread *thr = goacc_thread ();
 163
 164   assert (thr && thr->dev);
 165
 166   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 167     {
 168       if (from)
 169         memmove (h, d, s);
 170       else
 171         memmove (d, h, s);
 172       return;
 173     }
 174
 175   goacc_aq aq = get_goacc_asyncqueue (async);
 176   if (from)
 177     gomp_copy_dev2host (thr->dev, aq, h, d, s);
 178   else
 179     gomp_copy_host2dev (thr->dev, aq, d, h, s, /* TODO: cbuf? */ NULL);
 180 }
 181
 182 void
 183 acc_memcpy_to_device (void *d, void *h, size_t s)
 184 {
 185   memcpy_tofrom_device (false, d, h, s, acc_async_sync, __FUNCTION__);
 186 }
 187
 188 void
 189 acc_memcpy_to_device_async (void *d, void *h, size_t s, int async)
 190 {
 191   memcpy_tofrom_device (false, d, h, s, async, __FUNCTION__);
 192 }
 193
 194 void
 195 acc_memcpy_from_device (void *h, void *d, size_t s)
 196 {
 197   memcpy_tofrom_device (true, d, h, s, acc_async_sync, __FUNCTION__);
 198 }
 199
 200 void
 201 acc_memcpy_from_device_async (void *h, void *d, size_t s, int async)
 202 {
 203   memcpy_tofrom_device (true, d, h, s, async, __FUNCTION__);
 204 }
 205
 206 /* Return the device pointer that corresponds to host data H.  Or NULL
 207    if no mapping.  */
 208
 209 void *
 210 acc_deviceptr (void *h)
 211 {
 212   splay_tree_key n;
 213   void *d;
 214   void *offset;
 215
 216   goacc_lazy_initialize ();
 217
 218   struct goacc_thread *thr = goacc_thread ();
 219   struct gomp_device_descr *dev = thr->dev;
 220
 221   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 222     return h;
 223
 224   gomp_mutex_lock (&dev->lock);
 225
 226   n = lookup_host (dev, h, 1);
 227
 228   if (!n)
 229     {
 230       gomp_mutex_unlock (&dev->lock);
 231       return NULL;
 232     }
 233
 234   offset = h - n->host_start;
 235
 236   d = n->tgt->tgt_start + n->tgt_offset + offset;
 237
 238   gomp_mutex_unlock (&dev->lock);
 239
 240   return d;
 241 }
 242
 243 /* Return the host pointer that corresponds to device data D.  Or NULL
 244    if no mapping.  */
 245
 246 void *
 247 acc_hostptr (void *d)
 248 {
 249   splay_tree_key n;
 250   void *h;
 251   void *offset;
 252
 253   goacc_lazy_initialize ();
 254
 255   struct goacc_thread *thr = goacc_thread ();
 256   struct gomp_device_descr *acc_dev = thr->dev;
 257
 258   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 259     return d;
 260
 261   gomp_mutex_lock (&acc_dev->lock);
 262
 263   n = lookup_dev (acc_dev->openacc.data_environ, d, 1);
 264
 265   if (!n)
 266     {
 267       gomp_mutex_unlock (&acc_dev->lock);
 268       return NULL;
 269     }
 270
 271   offset = d - n->tgt->tgt_start + n->tgt_offset;
 272
 273   h = n->host_start + offset;
 274
 275   gomp_mutex_unlock (&acc_dev->lock);
 276
 277   return h;
 278 }
 279
 280 /* Return 1 if host data [H,+S] is present on the device.  */
 281
 282 int
 283 acc_is_present (void *h, size_t s)
 284 {
 285   splay_tree_key n;
 286
 287   if (!s || !h)
 288     return 0;
 289
 290   goacc_lazy_initialize ();
 291
 292   struct goacc_thread *thr = goacc_thread ();
 293   struct gomp_device_descr *acc_dev = thr->dev;
 294
 295   if (thr->dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 296     return h != NULL;
 297
 298   gomp_mutex_lock (&acc_dev->lock);
 299
 300   n = lookup_host (acc_dev, h, s);
 301
 302   if (n && ((uintptr_t)h < n->host_start
 303             || (uintptr_t)h + s > n->host_end
 304             || s > n->host_end - n->host_start))
 305     n = NULL;
 306
 307   gomp_mutex_unlock (&acc_dev->lock);
 308
 309   return n != NULL;
 310 }
 311
 312 /* Create a mapping for host [H,+S] -> device [D,+S] */
 313
 314 void
 315 acc_map_data (void *h, void *d, size_t s)
 316 {
 317   struct target_mem_desc *tgt = NULL;
 318   size_t mapnum = 1;
 319   void *hostaddrs = h;
 320   void *devaddrs = d;
 321   size_t sizes = s;
 322   unsigned short kinds = GOMP_MAP_ALLOC;
 323
 324   goacc_lazy_initialize ();
 325
 326   struct goacc_thread *thr = goacc_thread ();
 327   struct gomp_device_descr *acc_dev = thr->dev;
 328
 329   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 330     {
 331       if (d != h)
 332         gomp_fatal ("cannot map data on shared-memory system");
 333     }
 334   else
 335     {
 336       struct goacc_thread *thr = goacc_thread ();
 337
 338       if (!d || !h || !s)
 339         gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
 340                     (void *)h, (int)s, (void *)d, (int)s);
 341
 342       gomp_mutex_lock (&acc_dev->lock);
 343
 344       if (lookup_host (acc_dev, h, s))
 345         {
 346           gomp_mutex_unlock (&acc_dev->lock);
 347           gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h,
 348                       (int)s);
 349         }
 350
 351       if (lookup_dev (thr->dev->openacc.data_environ, d, s))
 352         {
 353           gomp_mutex_unlock (&acc_dev->lock);
 354           gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d,
 355                       (int)s);
 356         }
 357
 358       gomp_mutex_unlock (&acc_dev->lock);
 359
 360       tgt = gomp_map_vars (acc_dev, mapnum, &hostaddrs, &devaddrs, &sizes,
 361                            &kinds, true, GOMP_MAP_VARS_OPENACC);
 362       tgt->list[0].key->refcount = REFCOUNT_INFINITY;
 363     }
 364
 365   gomp_mutex_lock (&acc_dev->lock);
 366   tgt->prev = acc_dev->openacc.data_environ;
 367   acc_dev->openacc.data_environ = tgt;
 368   gomp_mutex_unlock (&acc_dev->lock);
 369 }
 370
 371 void
 372 acc_unmap_data (void *h)
 373 {
 374   struct goacc_thread *thr = goacc_thread ();
 375   struct gomp_device_descr *acc_dev = thr->dev;
 376
 377   /* No need to call lazy open, as the address must have been mapped.  */
 378
 379   /* This is a no-op on shared-memory targets.  */
 380   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 381     return;
 382
 383   size_t host_size;
 384
 385   gomp_mutex_lock (&acc_dev->lock);
 386
 387   splay_tree_key n = lookup_host (acc_dev, h, 1);
 388   struct target_mem_desc *t;
 389
 390   if (!n)
 391     {
 392       gomp_mutex_unlock (&acc_dev->lock);
 393       gomp_fatal ("%p is not a mapped block", (void *)h);
 394     }
 395
 396   host_size = n->host_end - n->host_start;
 397
 398   if (n->host_start != (uintptr_t) h)
 399     {
 400       gomp_mutex_unlock (&acc_dev->lock);
 401       gomp_fatal ("[%p,%d] surrounds %p",
 402                   (void *) n->host_start, (int) host_size, (void *) h);
 403     }
 404
 405   /* Mark for removal.  */
 406   n->refcount = 1;
 407
 408   t = n->tgt;
 409
 410   if (t->refcount == 2)
 411     {
 412       struct target_mem_desc *tp;
 413
 414       /* This is the last reference, so pull the descriptor off the
 415          chain. This avoids gomp_unmap_vars via gomp_unmap_tgt from
 416          freeing the device memory. */
 417       t->tgt_end = 0;
 418       t->to_free = 0;
 419
 420       for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 421            tp = t, t = t->prev)
 422         if (n->tgt == t)
 423           {
 424             if (tp)
 425               tp->prev = t->prev;
 426             else
 427               acc_dev->openacc.data_environ = t->prev;
 428
 429             break;
 430           }
 431     }
 432
 433   gomp_mutex_unlock (&acc_dev->lock);
 434
 435   gomp_unmap_vars (t, true);
 436 }
 437
 438 #define FLAG_PRESENT (1 << 0)
 439 #define FLAG_CREATE (1 << 1)
 440 #define FLAG_COPY (1 << 2)
 441
 442 static void *
 443 present_create_copy (unsigned f, void *h, size_t s, int async)
 444 {
 445   void *d;
 446   splay_tree_key n;
 447
 448   if (!h || !s)
 449     gomp_fatal ("[%p,+%d] is a bad range", (void *)h, (int)s);
 450
 451   goacc_lazy_initialize ();
 452
 453   struct goacc_thread *thr = goacc_thread ();
 454   struct gomp_device_descr *acc_dev = thr->dev;
 455
 456   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 457     return h;
 458
 459   gomp_mutex_lock (&acc_dev->lock);
 460
 461   n = lookup_host (acc_dev, h, s);
 462   if (n)
 463     {
 464       /* Present. */
 465       d = (void *) (n->tgt->tgt_start + n->tgt_offset);
 466
 467       if (!(f & FLAG_PRESENT))
 468         {
 469           gomp_mutex_unlock (&acc_dev->lock);
 470           gomp_fatal ("[%p,+%d] already mapped to [%p,+%d]",
 471                       (void *)h, (int)s, (void *)d, (int)s);
 472         }
 473       if ((h + s) > (void *)n->host_end)
 474         {
 475           gomp_mutex_unlock (&acc_dev->lock);
 476           gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 477         }
 478
 479       if (n->refcount != REFCOUNT_INFINITY)
 480         {
 481           n->refcount++;
 482           n->dynamic_refcount++;
 483         }
 484       gomp_mutex_unlock (&acc_dev->lock);
 485     }
 486   else if (!(f & FLAG_CREATE))
 487     {
 488       gomp_mutex_unlock (&acc_dev->lock);
 489       gomp_fatal ("[%p,+%d] not mapped", (void *)h, (int)s);
 490     }
 491   else
 492     {
 493       struct target_mem_desc *tgt;
 494       size_t mapnum = 1;
 495       unsigned short kinds;
 496       void *hostaddrs = h;
 497
 498       if (f & FLAG_COPY)
 499         kinds = GOMP_MAP_TO;
 500       else
 501         kinds = GOMP_MAP_ALLOC;
 502
 503       gomp_mutex_unlock (&acc_dev->lock);
 504
 505       goacc_aq aq = get_goacc_asyncqueue (async);
 506
 507       tgt = gomp_map_vars_async (acc_dev, aq, mapnum, &hostaddrs, NULL, &s,
 508                                  &kinds, true, GOMP_MAP_VARS_OPENACC);
 509       /* Initialize dynamic refcount.  */
 510       tgt->list[0].key->dynamic_refcount = 1;
 511
 512       gomp_mutex_lock (&acc_dev->lock);
 513
 514       d = tgt->to_free;
 515       tgt->prev = acc_dev->openacc.data_environ;
 516       acc_dev->openacc.data_environ = tgt;
 517
 518       gomp_mutex_unlock (&acc_dev->lock);
 519     }
 520
 521   return d;
 522 }
 523
 524 void *
 525 acc_create (void *h, size_t s)
 526 {
 527   return present_create_copy (FLAG_PRESENT | FLAG_CREATE, h, s, acc_async_sync);
 528 }
 529
 530 void
 531 acc_create_async (void *h, size_t s, int async)
 532 {
 533   present_create_copy (FLAG_PRESENT | FLAG_CREATE, h, s, async);
 534 }
 535
 536 /* acc_present_or_create used to be what acc_create is now.  */
 537 /* acc_pcreate is acc_present_or_create by a different name.  */
 538 #ifdef HAVE_ATTRIBUTE_ALIAS
 539 strong_alias (acc_create, acc_present_or_create)
 540 strong_alias (acc_create, acc_pcreate)
 541 #else
 542 void *
 543 acc_present_or_create (void *h, size_t s)
 544 {
 545   return acc_create (h, s);
 546 }
 547
 548 void *
 549 acc_pcreate (void *h, size_t s)
 550 {
 551   return acc_create (h, s);
 552 }
 553 #endif
 554
 555 void *
 556 acc_copyin (void *h, size_t s)
 557 {
 558   return present_create_copy (FLAG_PRESENT | FLAG_CREATE | FLAG_COPY, h, s,
 559                               acc_async_sync);
 560 }
 561
 562 void
 563 acc_copyin_async (void *h, size_t s, int async)
 564 {
 565   present_create_copy (FLAG_PRESENT | FLAG_CREATE | FLAG_COPY, h, s, async);
 566 }
 567
 568 /* acc_present_or_copyin used to be what acc_copyin is now.  */
 569 /* acc_pcopyin is acc_present_or_copyin by a different name.  */
 570 #ifdef HAVE_ATTRIBUTE_ALIAS
 571 strong_alias (acc_copyin, acc_present_or_copyin)
 572 strong_alias (acc_copyin, acc_pcopyin)
 573 #else
 574 void *
 575 acc_present_or_copyin (void *h, size_t s)
 576 {
 577   return acc_copyin (h, s);
 578 }
 579
 580 void *
 581 acc_pcopyin (void *h, size_t s)
 582 {
 583   return acc_copyin (h, s);
 584 }
 585 #endif
 586
 587 #define FLAG_COPYOUT  (1 << 0)
 588 #define FLAG_FINALIZE (1 << 1)
 589
 590 static void
 591 delete_copyout (unsigned f, void *h, size_t s, int async, const char *libfnname)
 592 {
 593   size_t host_size;
 594   splay_tree_key n;
 595   void *d;
 596   struct goacc_thread *thr = goacc_thread ();
 597   struct gomp_device_descr *acc_dev = thr->dev;
 598
 599   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 600     return;
 601
 602   gomp_mutex_lock (&acc_dev->lock);
 603
 604   n = lookup_host (acc_dev, h, s);
 605
 606   /* No need to call lazy open, as the data must already have been
 607      mapped.  */
 608
 609   if (!n)
 610     {
 611       gomp_mutex_unlock (&acc_dev->lock);
 612       gomp_fatal ("[%p,%d] is not mapped", (void *)h, (int)s);
 613     }
 614
 615   d = (void *) (n->tgt->tgt_start + n->tgt_offset
 616                 + (uintptr_t) h - n->host_start);
 617
 618   host_size = n->host_end - n->host_start;
 619
 620   if (n->host_start != (uintptr_t) h || host_size != s)
 621     {
 622       gomp_mutex_unlock (&acc_dev->lock);
 623       gomp_fatal ("[%p,%d] surrounds2 [%p,+%d]",
 624                   (void *) n->host_start, (int) host_size, (void *) h, (int) s);
 625     }
 626
 627   if (n->refcount == REFCOUNT_INFINITY)
 628     {
 629       n->refcount = 0;
 630       n->dynamic_refcount = 0;
 631     }
 632   if (n->refcount < n->dynamic_refcount)
 633     {
 634       gomp_mutex_unlock (&acc_dev->lock);
 635       gomp_fatal ("Dynamic reference counting assert fail\n");
 636     }
 637
 638   if (f & FLAG_FINALIZE)
 639     {
 640       n->refcount -= n->dynamic_refcount;
 641       n->dynamic_refcount = 0;
 642     }
 643   else if (n->dynamic_refcount)
 644     {
 645       n->dynamic_refcount--;
 646       n->refcount--;
 647     }
 648
 649   if (n->refcount == 0)
 650     {
 651       if (n->tgt->refcount == 2)
 652         {
 653           struct target_mem_desc *tp, *t;
 654           for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 655                tp = t, t = t->prev)
 656             if (n->tgt == t)
 657               {
 658                 if (tp)
 659                   tp->prev = t->prev;
 660                 else
 661                   acc_dev->openacc.data_environ = t->prev;
 662                 break;
 663               }
 664         }
 665
 666       if (f & FLAG_COPYOUT)
 667         {
 668           goacc_aq aq = get_goacc_asyncqueue (async);
 669           gomp_copy_dev2host (acc_dev, aq, h, d, s);
 670         }
 671       gomp_remove_var (acc_dev, n);
 672     }
 673
 674   gomp_mutex_unlock (&acc_dev->lock);
 675 }
 676
 677 void
 678 acc_delete (void *h , size_t s)
 679 {
 680   delete_copyout (0, h, s, acc_async_sync, __FUNCTION__);
 681 }
 682
 683 void
 684 acc_delete_async (void *h , size_t s, int async)
 685 {
 686   delete_copyout (0, h, s, async, __FUNCTION__);
 687 }
 688
 689 void
 690 acc_delete_finalize (void *h , size_t s)
 691 {
 692   delete_copyout (FLAG_FINALIZE, h, s, acc_async_sync, __FUNCTION__);
 693 }
 694
 695 void
 696 acc_delete_finalize_async (void *h , size_t s, int async)
 697 {
 698   delete_copyout (FLAG_FINALIZE, h, s, async, __FUNCTION__);
 699 }
 700
 701 void
 702 acc_copyout (void *h, size_t s)
 703 {
 704   delete_copyout (FLAG_COPYOUT, h, s, acc_async_sync, __FUNCTION__);
 705 }
 706
 707 void
 708 acc_copyout_async (void *h, size_t s, int async)
 709 {
 710   delete_copyout (FLAG_COPYOUT, h, s, async, __FUNCTION__);
 711 }
 712
 713 void
 714 acc_copyout_finalize (void *h, size_t s)
 715 {
 716   delete_copyout (FLAG_COPYOUT | FLAG_FINALIZE, h, s, acc_async_sync,
 717                   __FUNCTION__);
 718 }
 719
 720 void
 721 acc_copyout_finalize_async (void *h, size_t s, int async)
 722 {
 723   delete_copyout (FLAG_COPYOUT | FLAG_FINALIZE, h, s, async, __FUNCTION__);
 724 }
 725
 726 static void
 727 update_dev_host (int is_dev, void *h, size_t s, int async)
 728 {
 729   splay_tree_key n;
 730   void *d;
 731
 732   goacc_lazy_initialize ();
 733
 734   struct goacc_thread *thr = goacc_thread ();
 735   struct gomp_device_descr *acc_dev = thr->dev;
 736
 737   if (acc_dev->capabilities & GOMP_OFFLOAD_CAP_SHARED_MEM)
 738     return;
 739
 740   gomp_mutex_lock (&acc_dev->lock);
 741
 742   n = lookup_host (acc_dev, h, s);
 743
 744   if (!n)
 745     {
 746       gomp_mutex_unlock (&acc_dev->lock);
 747       gomp_fatal ("[%p,%d] is not mapped", h, (int)s);
 748     }
 749
 750   d = (void *) (n->tgt->tgt_start + n->tgt_offset
 751                 + (uintptr_t) h - n->host_start);
 752
 753   goacc_aq aq = get_goacc_asyncqueue (async);
 754
 755   if (is_dev)
 756     gomp_copy_host2dev (acc_dev, aq, d, h, s, /* TODO: cbuf? */ NULL);
 757   else
 758     gomp_copy_dev2host (acc_dev, aq, h, d, s);
 759
 760   gomp_mutex_unlock (&acc_dev->lock);
 761 }
 762
 763 void
 764 acc_update_device (void *h, size_t s)
 765 {
 766   update_dev_host (1, h, s, acc_async_sync);
 767 }
 768
 769 void
 770 acc_update_device_async (void *h, size_t s, int async)
 771 {
 772   update_dev_host (1, h, s, async);
 773 }
 774
 775 void
 776 acc_update_self (void *h, size_t s)
 777 {
 778   update_dev_host (0, h, s, acc_async_sync);
 779 }
 780
 781 void
 782 acc_update_self_async (void *h, size_t s, int async)
 783 {
 784   update_dev_host (0, h, s, async);
 785 }
 786
 787 void
 788 gomp_acc_insert_pointer (size_t mapnum, void **hostaddrs, size_t *sizes,
 789                          void *kinds, int async)
 790 {
 791   struct target_mem_desc *tgt;
 792   struct goacc_thread *thr = goacc_thread ();
 793   struct gomp_device_descr *acc_dev = thr->dev;
 794
 795   if (acc_is_present (*hostaddrs, *sizes))
 796     {
 797       splay_tree_key n;
 798       gomp_mutex_lock (&acc_dev->lock);
 799       n = lookup_host (acc_dev, *hostaddrs, *sizes);
 800       gomp_mutex_unlock (&acc_dev->lock);
 801
 802       tgt = n->tgt;
 803       for (size_t i = 0; i < tgt->list_count; i++)
 804         if (tgt->list[i].key == n)
 805           {
 806             for (size_t j = 0; j < mapnum; j++)
 807               if (i + j < tgt->list_count && tgt->list[i + j].key)
 808                 {
 809                   tgt->list[i + j].key->refcount++;
 810                   tgt->list[i + j].key->dynamic_refcount++;
 811                 }
 812             return;
 813           }
 814       /* Should not reach here.  */
 815       gomp_fatal ("Dynamic refcount incrementing failed for pointer/pset");
 816     }
 817
 818   gomp_debug (0, "  %s: prepare mappings\n", __FUNCTION__);
 819   goacc_aq aq = get_goacc_asyncqueue (async);
 820   tgt = gomp_map_vars_async (acc_dev, aq, mapnum, hostaddrs,
 821                              NULL, sizes, kinds, true, GOMP_MAP_VARS_OPENACC);
 822   gomp_debug (0, "  %s: mappings prepared\n", __FUNCTION__);
 823
 824   /* Initialize dynamic refcount.  */
 825   tgt->list[0].key->dynamic_refcount = 1;
 826
 827   gomp_mutex_lock (&acc_dev->lock);
 828   tgt->prev = acc_dev->openacc.data_environ;
 829   acc_dev->openacc.data_environ = tgt;
 830   gomp_mutex_unlock (&acc_dev->lock);
 831 }
 832
 833 void
 834 gomp_acc_remove_pointer (void *h, size_t s, bool force_copyfrom, int async,
 835                          int finalize, int mapnum)
 836 {
 837   struct goacc_thread *thr = goacc_thread ();
 838   struct gomp_device_descr *acc_dev = thr->dev;
 839   splay_tree_key n;
 840   struct target_mem_desc *t;
 841   int minrefs = (mapnum == 1) ? 2 : 3;
 842
 843   if (!acc_is_present (h, s))
 844     return;
 845
 846   gomp_mutex_lock (&acc_dev->lock);
 847
 848   n = lookup_host (acc_dev, h, 1);
 849
 850   if (!n)
 851     {
 852       gomp_mutex_unlock (&acc_dev->lock);
 853       gomp_fatal ("%p is not a mapped block", (void *)h);
 854     }
 855
 856   gomp_debug (0, "  %s: restore mappings\n", __FUNCTION__);
 857
 858   t = n->tgt;
 859
 860   if (n->refcount < n->dynamic_refcount)
 861     {
 862       gomp_mutex_unlock (&acc_dev->lock);
 863       gomp_fatal ("Dynamic reference counting assert fail\n");
 864     }
 865
 866   if (finalize)
 867     {
 868       n->refcount -= n->dynamic_refcount;
 869       n->dynamic_refcount = 0;
 870     }
 871   else if (n->dynamic_refcount)
 872     {
 873       n->dynamic_refcount--;
 874       n->refcount--;
 875     }
 876
 877   gomp_mutex_unlock (&acc_dev->lock);
 878
 879   if (n->refcount == 0)
 880     {
 881       if (t->refcount == minrefs)
 882         {
 883           /* This is the last reference, so pull the descriptor off the
 884              chain. This prevents gomp_unmap_vars via gomp_unmap_tgt from
 885              freeing the device memory. */
 886           struct target_mem_desc *tp;
 887           for (tp = NULL, t = acc_dev->openacc.data_environ; t != NULL;
 888                tp = t, t = t->prev)
 889             {
 890               if (n->tgt == t)
 891                 {
 892                   if (tp)
 893                     tp->prev = t->prev;
 894                   else
 895                     acc_dev->openacc.data_environ = t->prev;
 896                   break;
 897                 }
 898             }
 899         }
 900
 901       /* Set refcount to 1 to allow gomp_unmap_vars to unmap it.  */
 902       n->refcount = 1;
 903       t->refcount = minrefs;
 904       for (size_t i = 0; i < t->list_count; i++)
 905         if (t->list[i].key == n)
 906           {
 907             t->list[i].copy_from = force_copyfrom ? 1 : 0;
 908             break;
 909           }
 910
 911       /* If running synchronously, unmap immediately.  */
 912       if (async < acc_async_noval)
 913         gomp_unmap_vars (t, true);
 914       else
 915         {
 916           goacc_aq aq = get_goacc_asyncqueue (async);
 917           gomp_unmap_vars_async (t, true, aq);
 918         }
 919     }
 920
 921   gomp_mutex_unlock (&acc_dev->lock);
 922
 923   gomp_debug (0, "  %s: mappings restored\n", __FUNCTION__);
 924 }