openmp, nvptx: low-lat memory access traits

The NVPTX low latency memory is not accessible outside the team that allocates
it, and therefore should be unavailable for allocators with the access trait
"all".  This change means that the omp_low_lat_mem_alloc predefined
allocator now implicitly implies the "pteam" trait.

	Backport of a patch posted at
	https://gcc.gnu.org/pipermail/gcc-patches/2022-January/589355.html

libgomp/ChangeLog:

	* allocator.c (MEMSPACE_VALIDATE): New macro.
	(omp_aligned_alloc): Use MEMSPACE_VALIDATE.
	(omp_aligned_calloc): Likewise.
	(omp_realloc): Likewise.
	* config/nvptx/allocator.c (nvptx_memspace_validate): New function.
	(MEMSPACE_VALIDATE): New macro.
	* testsuite/libgomp.c/allocators-4.c (main): Add access trait.
	* testsuite/libgomp.c/allocators-6.c (main): Add access trait.
	* testsuite/libgomp.c/allocators-7.c: New test.

diff --git a/libgomp/ChangeLog.omp b/libgomp/ChangeLog.omp
index 50773dc4f672e9297eb0a62830308c0461d6f120..c0626439b0df07d8ec1dc6e34728ec9cfe1cc837 100644 (file)
--- a/libgomp/ChangeLog.omp
+++ b/libgomp/ChangeLog.omp
@@ -1,3 +1,18 @@
+2022-03-11  Andrew Stubbs <ams@codesourcery.com>
+
+       Backport of a patch posted at
+       https://gcc.gnu.org/pipermail/gcc-patches/2022-January/589355.html
+
+       * allocator.c (MEMSPACE_VALIDATE): New macro.
+       (omp_aligned_alloc): Use MEMSPACE_VALIDATE.
+       (omp_aligned_calloc): Likewise.
+       (omp_realloc): Likewise.
+       * config/nvptx/allocator.c (nvptx_memspace_validate): New function.
+       (MEMSPACE_VALIDATE): New macro.
+       * testsuite/libgomp.c/allocators-4.c (main): Add access trait.
+       * testsuite/libgomp.c/allocators-6.c (main): Add access trait.
+       * testsuite/libgomp.c/allocators-7.c: New test.
+
 2022-03-11  Andrew Stubbs <ams@codesourcery.com>
 
        Backport of a patch posted at

diff --git a/libgomp/allocator.c b/libgomp/allocator.c
index 724ecf3703c4459a9ef7fc3f398f23c31cb65d79..7052052ad96af9baea2c47557b5ea99009977240 100644 (file)
--- a/libgomp/allocator.c
+++ b/libgomp/allocator.c
@@ -51,6 +51,9 @@
 #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
   (PIN ? NULL : free (ADDR))
 #endif
+#ifndef MEMSPACE_VALIDATE
+#define MEMSPACE_VALIDATE(MEMSPACE, ACCESS) 1
+#endif
 
 /* Map the predefined allocators to the correct memory space.
    The index to this table is the omp_allocator_handle_t enum value.  */
@@ -279,6 +282,10 @@ retry:
   if (__builtin_add_overflow (size, new_size, &new_size))
     goto fail;
 
+  if (allocator_data
+      && !MEMSPACE_VALIDATE (allocator_data->memspace, allocator_data->access))
+    goto fail;
+
   if (__builtin_expect (allocator_data
                        && allocator_data->pool_size < ~(uintptr_t) 0, 0))
     {
@@ -505,6 +512,10 @@ retry:
   if (__builtin_add_overflow (size_temp, new_size, &new_size))
     goto fail;
 
+  if (allocator_data
+      && !MEMSPACE_VALIDATE (allocator_data->memspace, allocator_data->access))
+    goto fail;
+
   if (__builtin_expect (allocator_data
                        && allocator_data->pool_size < ~(uintptr_t) 0, 0))
     {
@@ -675,6 +686,10 @@ retry:
     goto fail;
   old_size = data->size;
 
+  if (allocator_data
+      && !MEMSPACE_VALIDATE (allocator_data->memspace, allocator_data->access))
+    goto fail;
+
   if (__builtin_expect (allocator_data
                        && allocator_data->pool_size < ~(uintptr_t) 0, 0))
     {

diff --git a/libgomp/config/nvptx/allocator.c b/libgomp/config/nvptx/allocator.c
index f740b97f6acaf7a8a259f15b911ec0f668dd8f30..0102680b7170825b7d00b8c909b7e7547c9dab12 100644 (file)
--- a/libgomp/config/nvptx/allocator.c
+++ b/libgomp/config/nvptx/allocator.c
@@ -358,6 +358,15 @@ nvptx_memspace_realloc (omp_memspace_handle_t memspace, void *addr,
     return realloc (addr, size);
 }
 
+static inline int
+nvptx_memspace_validate (omp_memspace_handle_t memspace, unsigned access)
+{
+  /* Disallow use of low-latency memory when it must be accessible by
+     all threads.  */
+  return (memspace != omp_low_lat_mem_space
+         || access != omp_atv_all);
+}
+
 #define MEMSPACE_ALLOC(MEMSPACE, SIZE, PIN) \
   nvptx_memspace_alloc (MEMSPACE, SIZE)
 #define MEMSPACE_CALLOC(MEMSPACE, SIZE, PIN) \
@@ -366,5 +375,7 @@ nvptx_memspace_realloc (omp_memspace_handle_t memspace, void *addr,
   nvptx_memspace_realloc (MEMSPACE, ADDR, OLDSIZE, SIZE)
 #define MEMSPACE_FREE(MEMSPACE, ADDR, SIZE, PIN) \
   nvptx_memspace_free (MEMSPACE, ADDR, SIZE)
+#define MEMSPACE_VALIDATE(MEMSPACE, ACCESS) \
+  nvptx_memspace_validate (MEMSPACE, ACCESS)
 
 #include "../../allocator.c"

diff --git a/libgomp/testsuite/libgomp.c/allocators-4.c b/libgomp/testsuite/libgomp.c/allocators-4.c
index 9fa6aa1624f334366401d79e4ffba65484a160a5..83a60933c2144843a8fc2b85e5d23df2a7ee2c75 100644 (file)
--- a/libgomp/testsuite/libgomp.c/allocators-4.c
+++ b/libgomp/testsuite/libgomp.c/allocators-4.c
@@ -23,10 +23,11 @@ main ()
   #pragma omp target
   {
     /* Ensure that the memory we get *is* low-latency with a null-fallback.  */
-    omp_alloctrait_t traits[1]
-      = { { omp_atk_fallback, omp_atv_null_fb } };
+    omp_alloctrait_t traits[2]
+      = { { omp_atk_fallback, omp_atv_null_fb },
+         { omp_atk_access, omp_atv_pteam } };
     omp_allocator_handle_t lowlat = omp_init_allocator (omp_low_lat_mem_space,
-                                                       1, traits);
+                                                       2, traits);
 
     int size = 4;
 

diff --git a/libgomp/testsuite/libgomp.c/allocators-6.c b/libgomp/testsuite/libgomp.c/allocators-6.c
index 90bf73095ef5450a5a11119f7797cd6d23144552..9c22466fbd66751906eb90ee29ee3a6596cb6ac5 100644 (file)
--- a/libgomp/testsuite/libgomp.c/allocators-6.c
+++ b/libgomp/testsuite/libgomp.c/allocators-6.c
@@ -23,10 +23,11 @@ main ()
   #pragma omp target
   {
     /* Ensure that the memory we get *is* low-latency with a null-fallback.  */
-    omp_alloctrait_t traits[1]
-      = { { omp_atk_fallback, omp_atv_null_fb } };
+    omp_alloctrait_t traits[2]
+      = { { omp_atk_fallback, omp_atv_null_fb },
+         { omp_atk_access, omp_atv_pteam } };
     omp_allocator_handle_t lowlat = omp_init_allocator (omp_low_lat_mem_space,
-                                                       1, traits);
+                                                       2, traits);
 
     int size = 16;
 

diff --git a/libgomp/testsuite/libgomp.c/allocators-7.c b/libgomp/testsuite/libgomp.c/allocators-7.c
new file mode 100644 (file)
index 0000000..a0a738b
--- /dev/null
+++ b/libgomp/testsuite/libgomp.c/allocators-7.c
@@ -0,0 +1,68 @@
+/* { dg-do run } */
+
+/* { dg-require-effective-target offload_device } */
+/* { dg-xfail-if "not implemented" { ! offload_target_nvptx } } */
+
+/* Test that GPU low-latency allocation is limited to team access.  */
+
+#include <stddef.h>
+#include <omp.h>
+
+#pragma omp requires dynamic_allocators
+
+int
+main ()
+{
+  #pragma omp target
+  {
+    /* Ensure that the memory we get *is* low-latency with a null-fallback.  */
+    omp_alloctrait_t traits[2]
+      = { { omp_atk_fallback, omp_atv_null_fb },
+         { omp_atk_access, omp_atv_pteam } };
+    omp_allocator_handle_t lowlat = omp_init_allocator (omp_low_lat_mem_space,
+                                                       2, traits);
+
+    omp_alloctrait_t traits_all[2]
+      = { { omp_atk_fallback, omp_atv_null_fb },
+         { omp_atk_access, omp_atv_all } };
+    omp_allocator_handle_t lowlat_all
+      = omp_init_allocator (omp_low_lat_mem_space, 2, traits_all);
+
+    omp_alloctrait_t traits_default[1]
+      = { { omp_atk_fallback, omp_atv_null_fb } };
+    omp_allocator_handle_t lowlat_default
+      = omp_init_allocator (omp_low_lat_mem_space, 1, traits_default);
+
+    void *a = omp_alloc(1, lowlat);        // good
+    void *b = omp_alloc(1, lowlat_all);     // bad
+    void *c = omp_alloc(1, lowlat_default); // bad
+
+    if (!a || b || c)
+      __builtin_abort ();
+
+    omp_free (a, lowlat);
+
+
+    a = omp_calloc(1, 1, lowlat);        // good
+    b = omp_calloc(1, 1, lowlat_all);     // bad
+    c = omp_calloc(1, 1, lowlat_default); // bad
+
+    if (!a || b || c)
+      __builtin_abort ();
+
+    omp_free (a, lowlat);
+
+
+    a = omp_realloc(NULL, 1, lowlat, lowlat);                // good
+    b = omp_realloc(NULL, 1, lowlat_all, lowlat_all);        // bad
+    c = omp_realloc(NULL, 1, lowlat_default, lowlat_default); // bad
+
+    if (!a || b || c)
+      __builtin_abort ();
+
+    omp_free (a, lowlat);
+  }
+
+return 0;
+}
+