Daily bump.

[thirdparty/gcc.git] / libgomp / ordered.c

diff --git a/libgomp/ordered.c b/libgomp/ordered.c
index fdac3ee8f5854d677f5481ba7bc7fd6e94485736..8272b8fe21b70d075cf80691555272a954dc5969 100644 (file)
--- a/libgomp/ordered.c
+++ b/libgomp/ordered.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2005-2015 Free Software Foundation, Inc.
+/* Copyright (C) 2005-2024 Free Software Foundation, Inc.
    Contributed by Richard Henderson <rth@redhat.com>.
 
    This file is part of the GNU Offloading and Multi Processing Library
@@ -214,7 +214,7 @@ gomp_ordered_sync (void)
      Either way we get correct results.
      However, there is an implicit flush on entry to an ordered region,
      so we do need to have a barrier here.  If we were taking a lock
-     this could be MEMMODEL_RELEASE since the acquire would be coverd
+     this could be MEMMODEL_RELEASE since the acquire would be covered
      by the lock.  */
 
   __atomic_thread_fence (MEMMODEL_ACQ_REL);
@@ -246,7 +246,7 @@ GOMP_ordered_start (void)
 
    However, the current implementation has a flaw in that it does not allow
    the next thread into the ORDERED section immediately after the current
-   thread exits the ORDERED section in its last iteration.  The existance
+   thread exits the ORDERED section in its last iteration.  The existence
    of this function allows the implementation to change.  */
 
 void
@@ -259,7 +259,8 @@ GOMP_ordered_end (void)
 #define MAX_COLLAPSED_BITS (__SIZEOF_LONG__ * __CHAR_BIT__)
 
 void
-gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size)
+gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size,
+                   size_t extra)
 {
   struct gomp_thread *thr = gomp_thread ();
   struct gomp_team *team = thr->ts.team;
@@ -269,13 +270,24 @@ gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size)
   struct gomp_doacross_work_share *doacross;
 
   if (team == NULL || team->nthreads == 1)
-    return;
+    {
+    empty:
+      if (!extra)
+       ws->doacross = NULL;
+      else
+       {
+         doacross = gomp_malloc_cleared (sizeof (*doacross) + extra);
+         doacross->extra = (void *) (doacross + 1);
+         ws->doacross = doacross;
+       }
+      return;
+    }
 
   for (i = 0; i < ncounts; i++)
     {
       /* If any count is 0, GOMP_doacross_{post,wait} can't be called.  */
       if (counts[i] == 0)
-       return;
+       goto empty;
 
       if (num_bits <= MAX_COLLAPSED_BITS)
        {
@@ -297,6 +309,8 @@ gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size)
 
   if (ws->sched == GFS_STATIC)
     num_ents = team->nthreads;
+  else if (ws->sched == GFS_GUIDED)
+    num_ents = counts[0];
   else
     num_ents = (counts[0] - 1) / chunk_size + 1;
   if (num_bits <= MAX_COLLAPSED_BITS)
@@ -312,7 +326,7 @@ gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size)
   elt_sz = (elt_sz + 63) & ~63UL;
 
   doacross = gomp_malloc (sizeof (*doacross) + 63 + num_ents * elt_sz
-                         + shift_sz);
+                         + shift_sz + extra);
   doacross->chunk_size = chunk_size;
   doacross->elt_sz = elt_sz;
   doacross->ncounts = ncounts;
@@ -320,6 +334,13 @@ gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size)
   doacross->array = (unsigned char *)
                    ((((uintptr_t) (doacross + 1)) + 63 + shift_sz)
                     & ~(uintptr_t) 63);
+  if (extra)
+    {
+      doacross->extra = doacross->array + num_ents * elt_sz;
+      memset (doacross->extra, '\0', extra);
+    }
+  else
+    doacross->extra = NULL;
   if (num_bits <= MAX_COLLAPSED_BITS)
     {
       unsigned int shift_count = 0;
@@ -358,7 +379,8 @@ GOMP_doacross_post (long *counts)
   unsigned long ent;
   unsigned int i;
 
-  if (__builtin_expect (doacross == NULL, 0))
+  if (__builtin_expect (doacross == NULL, 0)
+      || __builtin_expect (doacross->array == NULL, 0))
     {
       __sync_synchronize ();
       return;
@@ -366,6 +388,8 @@ GOMP_doacross_post (long *counts)
 
   if (__builtin_expect (ws->sched == GFS_STATIC, 1))
     ent = thr->ts.team_id;
+  else if (ws->sched == GFS_GUIDED)
+    ent = counts[0];
   else
     ent = counts[0] / doacross->chunk_size;
   unsigned long *array = (unsigned long *) (doacross->array
@@ -407,7 +431,8 @@ GOMP_doacross_wait (long first, ...)
   unsigned long ent;
   unsigned int i;
 
-  if (__builtin_expect (doacross == NULL, 0))
+  if (__builtin_expect (doacross == NULL, 0)
+      || __builtin_expect (doacross->array == NULL, 0))
     {
       __sync_synchronize ();
       return;
@@ -426,6 +451,8 @@ GOMP_doacross_wait (long first, ...)
       else
        ent = first / ws->chunk_size % thr->ts.team->nthreads;
     }
+  else if (ws->sched == GFS_GUIDED)
+    ent = first;
   else
     ent = first / doacross->chunk_size;
   unsigned long *array = (unsigned long *) (doacross->array
@@ -482,7 +509,8 @@ GOMP_doacross_wait (long first, ...)
 typedef unsigned long long gomp_ull;
 
 void
-gomp_doacross_ull_init (unsigned ncounts, gomp_ull *counts, gomp_ull chunk_size)
+gomp_doacross_ull_init (unsigned ncounts, gomp_ull *counts,
+                       gomp_ull chunk_size, size_t extra)
 {
   struct gomp_thread *thr = gomp_thread ();
   struct gomp_team *team = thr->ts.team;
@@ -492,13 +520,24 @@ gomp_doacross_ull_init (unsigned ncounts, gomp_ull *counts, gomp_ull chunk_size)
   struct gomp_doacross_work_share *doacross;
 
   if (team == NULL || team->nthreads == 1)
-    return;
+    {
+    empty:
+      if (!extra)
+       ws->doacross = NULL;
+      else
+       {
+         doacross = gomp_malloc_cleared (sizeof (*doacross) + extra);
+         doacross->extra = (void *) (doacross + 1);
+         ws->doacross = doacross;
+       }
+      return;
+    }
 
   for (i = 0; i < ncounts; i++)
     {
       /* If any count is 0, GOMP_doacross_{post,wait} can't be called.  */
       if (counts[i] == 0)
-       return;
+       goto empty;
 
       if (num_bits <= MAX_COLLAPSED_BITS)
        {
@@ -520,6 +559,8 @@ gomp_doacross_ull_init (unsigned ncounts, gomp_ull *counts, gomp_ull chunk_size)
 
   if (ws->sched == GFS_STATIC)
     num_ents = team->nthreads;
+  else if (ws->sched == GFS_GUIDED)
+    num_ents = counts[0];
   else
     num_ents = (counts[0] - 1) / chunk_size + 1;
   if (num_bits <= MAX_COLLAPSED_BITS)
@@ -549,6 +590,13 @@ gomp_doacross_ull_init (unsigned ncounts, gomp_ull *counts, gomp_ull chunk_size)
   doacross->array = (unsigned char *)
                    ((((uintptr_t) (doacross + 1)) + 63 + shift_sz)
                     & ~(uintptr_t) 63);
+  if (extra)
+    {
+      doacross->extra = doacross->array + num_ents * elt_sz;
+      memset (doacross->extra, '\0', extra);
+    }
+  else
+    doacross->extra = NULL;
   if (num_bits <= MAX_COLLAPSED_BITS)
     {
       unsigned int shift_count = 0;
@@ -587,7 +635,8 @@ GOMP_doacross_ull_post (gomp_ull *counts)
   unsigned long ent;
   unsigned int i;
 
-  if (__builtin_expect (doacross == NULL, 0))
+  if (__builtin_expect (doacross == NULL, 0)
+      || __builtin_expect (doacross->array == NULL, 0))
     {
       __sync_synchronize ();
       return;
@@ -595,6 +644,8 @@ GOMP_doacross_ull_post (gomp_ull *counts)
 
   if (__builtin_expect (ws->sched == GFS_STATIC, 1))
     ent = thr->ts.team_id;
+  else if (ws->sched == GFS_GUIDED)
+    ent = counts[0];
   else
     ent = counts[0] / doacross->chunk_size_ull;
 
@@ -657,7 +708,8 @@ GOMP_doacross_ull_wait (gomp_ull first, ...)
   unsigned long ent;
   unsigned int i;
 
-  if (__builtin_expect (doacross == NULL, 0))
+  if (__builtin_expect (doacross == NULL, 0)
+      || __builtin_expect (doacross->array == NULL, 0))
     {
       __sync_synchronize ();
       return;
@@ -676,6 +728,8 @@ GOMP_doacross_ull_wait (gomp_ull first, ...)
       else
        ent = first / ws->chunk_size_ull % thr->ts.team->nthreads;
     }
+  else if (ws->sched == GFS_GUIDED)
+    ent = first;
   else
     ent = first / doacross->chunk_size_ull;