'-foffload-memory=pinned' using offloading device interfaces for non-contiguous array...

Changes related to og12 commit 15d0f61a7fecdc8fd12857c40879ea3730f6d99f
"Merge non-contiguous array support patches".

	libgomp/
	* target.c (gomp_map_vars_internal)
	<non-contiguous array support>: Handle 'always_pinned_mode'.

diff --git a/libgomp/ChangeLog.omp b/libgomp/ChangeLog.omp
index 1b02c057562feac688b4fb0068b00132201f0830..09cf9c6f3c14c8b2bc1ff0848cbc867c865e14b4 100644 (file)
--- a/libgomp/ChangeLog.omp
+++ b/libgomp/ChangeLog.omp
@@ -1,5 +1,8 @@
 2023-04-03  Thomas Schwinge  <thomas@codesourcery.com>
 
+       * target.c (gomp_map_vars_internal)
+       <non-contiguous array support>: Handle 'always_pinned_mode'.
+
        * libgomp-plugin.h (GOMP_OFFLOAD_page_locked_host_free): Add
        'struct goacc_asyncqueue *' formal parameter.
        (GOMP_OFFLOAD_page_locked_host_register)

diff --git a/libgomp/target.c b/libgomp/target.c
index ed2fc09cf44ea1c1c31b67b935c4ad1ae7536ef6..38eb5d1aa5b766d3979aae62ebc83613311cf4f2 100644 (file)
--- a/libgomp/target.c
+++ b/libgomp/target.c
@@ -2087,6 +2087,23 @@ gomp_map_vars_internal (struct gomp_device_descr *devicep,
                      k->dynamic_refcount = 0;
                      k->aux = NULL;
                      k->tgt_offset = tgt_size;
+                     k->page_locked_host_p = false;
+                     if (always_pinned_mode)
+                       {
+                         void *ptr = (void *) k->host_start;
+                         size_t size = k->host_end - k->host_start;
+                         int page_locked_host_p = 0;
+                         if (size != 0)
+                           page_locked_host_p = gomp_page_locked_host_register_dev
+                             (devicep, ptr, size, kind & typemask);
+                         if (page_locked_host_p < 0)
+                           {
+                             gomp_mutex_unlock (&devicep->lock);
+                             exit (EXIT_FAILURE);
+                           }
+                         if (page_locked_host_p)
+                           k->page_locked_host_p = true;
+                       }
 
                      tgt_size += nca->data_row_size;
 
@@ -2120,16 +2137,44 @@ gomp_map_vars_internal (struct gomp_device_descr *devicep,
                 accelerator side ptrblock and copy it in.  */
              if (nca->ptrblock_size)
                {
-                 void *ptrblock = gomp_malloc (nca->ptrblock_size);
+                 void *ptrblock;
+                 if (always_pinned_mode)
+                   {
+                     ptrblock
+                       = gomp_page_locked_host_alloc_dev (devicep,
+                                                          nca->ptrblock_size,
+                                                          false);
+                     if (!ptrblock)
+                       {
+                         gomp_mutex_unlock (&devicep->lock);
+                         exit (EXIT_FAILURE);
+                       }
+                   }
+                 else
+                   ptrblock = gomp_malloc (nca->ptrblock_size);
                  goacc_noncontig_array_create_ptrblock
                    (nca, ptrblock, target_ptrblock);
                  gomp_copy_host2dev (devicep, aq, target_ptrblock, ptrblock,
                                      nca->ptrblock_size, false, cbufp);
-                 if (aq)
-                   /* Free once the transfer has completed.  */
-                   devicep->openacc.async.queue_callback_func (aq, free, ptrblock);
+                 if (always_pinned_mode)
+                   {
+                     if (!gomp_page_locked_host_free_dev (devicep,
+                                                          ptrblock,
+                                                          aq))
+                       {
+                         gomp_mutex_unlock (&devicep->lock);
+                         exit (EXIT_FAILURE);
+                       }
+                   }
                  else
-                   free (ptrblock);
+                   {
+                     if (aq)
+                       /* Free once the transfer has completed.  */
+                       devicep->openacc.async.queue_callback_func
+                         (aq, free, ptrblock);
+                     else
+                       free (ptrblock);
+                   }
                }
            }
        }