]>
Commit | Line | Data |
---|---|---|
83ffe9cd | 1 | /* Copyright (C) 2005-2023 Free Software Foundation, Inc. |
953ff289 DN |
2 | Contributed by Richard Henderson <rth@redhat.com>. |
3 | ||
f1f3453e TS |
4 | This file is part of the GNU Offloading and Multi Processing Library |
5 | (libgomp). | |
953ff289 DN |
6 | |
7 | Libgomp is free software; you can redistribute it and/or modify it | |
748086b7 JJ |
8 | under the terms of the GNU General Public License as published by |
9 | the Free Software Foundation; either version 3, or (at your option) | |
10 | any later version. | |
953ff289 DN |
11 | |
12 | Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY | |
13 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS | |
748086b7 | 14 | FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
953ff289 DN |
15 | more details. |
16 | ||
748086b7 JJ |
17 | Under Section 7 of GPL version 3, you are granted additional |
18 | permissions described in the GCC Runtime Library Exception, version | |
19 | 3.1, as published by the Free Software Foundation. | |
20 | ||
21 | You should have received a copy of the GNU General Public License and | |
22 | a copy of the GCC Runtime Library Exception along with this program; | |
23 | see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
24 | <http://www.gnu.org/licenses/>. */ | |
953ff289 DN |
25 | |
26 | /* This file contains routines to manage the work-share queue for a team | |
27 | of threads. */ | |
28 | ||
29 | #include "libgomp.h" | |
a68ab351 | 30 | #include <stddef.h> |
953ff289 DN |
31 | #include <stdlib.h> |
32 | #include <string.h> | |
33 | ||
34 | ||
a68ab351 JJ |
35 | /* Allocate a new work share structure, preferably from current team's |
36 | free gomp_work_share cache. */ | |
953ff289 | 37 | |
a68ab351 JJ |
38 | static struct gomp_work_share * |
39 | alloc_work_share (struct gomp_team *team) | |
953ff289 DN |
40 | { |
41 | struct gomp_work_share *ws; | |
a68ab351 | 42 | unsigned int i; |
953ff289 | 43 | |
a68ab351 JJ |
44 | /* This is called in a critical section. */ |
45 | if (team->work_share_list_alloc != NULL) | |
46 | { | |
47 | ws = team->work_share_list_alloc; | |
48 | team->work_share_list_alloc = ws->next_free; | |
49 | return ws; | |
50 | } | |
953ff289 | 51 | |
a68ab351 JJ |
52 | #ifdef HAVE_SYNC_BUILTINS |
53 | ws = team->work_share_list_free; | |
54 | /* We need atomic read from work_share_list_free, | |
55 | as free_work_share can be called concurrently. */ | |
56 | __asm ("" : "+r" (ws)); | |
57 | ||
58 | if (ws && ws->next_free) | |
59 | { | |
60 | struct gomp_work_share *next = ws->next_free; | |
61 | ws->next_free = NULL; | |
62 | team->work_share_list_alloc = next->next_free; | |
63 | return next; | |
64 | } | |
65 | #else | |
66 | gomp_mutex_lock (&team->work_share_list_free_lock); | |
67 | ws = team->work_share_list_free; | |
68 | if (ws) | |
69 | { | |
70 | team->work_share_list_alloc = ws->next_free; | |
71 | team->work_share_list_free = NULL; | |
72 | gomp_mutex_unlock (&team->work_share_list_free_lock); | |
73 | return ws; | |
74 | } | |
75 | gomp_mutex_unlock (&team->work_share_list_free_lock); | |
76 | #endif | |
953ff289 | 77 | |
a68ab351 | 78 | team->work_share_chunk *= 2; |
28567c40 JJ |
79 | /* Allocating gomp_work_share structures aligned is just an |
80 | optimization, don't do it when using the fallback method. */ | |
17da2c74 | 81 | #ifdef GOMP_USE_ALIGNED_WORK_SHARES |
28567c40 JJ |
82 | ws = gomp_aligned_alloc (__alignof (struct gomp_work_share), |
83 | team->work_share_chunk | |
84 | * sizeof (struct gomp_work_share)); | |
85 | #else | |
a68ab351 | 86 | ws = gomp_malloc (team->work_share_chunk * sizeof (struct gomp_work_share)); |
28567c40 | 87 | #endif |
a68ab351 JJ |
88 | ws->next_alloc = team->work_shares[0].next_alloc; |
89 | team->work_shares[0].next_alloc = ws; | |
90 | team->work_share_list_alloc = &ws[1]; | |
91 | for (i = 1; i < team->work_share_chunk - 1; i++) | |
92 | ws[i].next_free = &ws[i + 1]; | |
93 | ws[i].next_free = NULL; | |
953ff289 DN |
94 | return ws; |
95 | } | |
96 | ||
a68ab351 JJ |
97 | /* Initialize an already allocated struct gomp_work_share. |
98 | This shouldn't touch the next_alloc field. */ | |
99 | ||
100 | void | |
28567c40 | 101 | gomp_init_work_share (struct gomp_work_share *ws, size_t ordered, |
a68ab351 JJ |
102 | unsigned nthreads) |
103 | { | |
104 | gomp_mutex_init (&ws->lock); | |
105 | if (__builtin_expect (ordered, 0)) | |
106 | { | |
28567c40 JJ |
107 | #define INLINE_ORDERED_TEAM_IDS_SIZE \ |
108 | (sizeof (struct gomp_work_share) \ | |
109 | - offsetof (struct gomp_work_share, inline_ordered_team_ids)) | |
110 | ||
111 | if (__builtin_expect (ordered != 1, 0)) | |
112 | { | |
fcfb8032 JJ |
113 | size_t o = nthreads * sizeof (*ws->ordered_team_ids); |
114 | o += __alignof__ (long long) - 1; | |
115 | if ((offsetof (struct gomp_work_share, inline_ordered_team_ids) | |
57a957cb JJ |
116 | & (__alignof__ (long long) - 1)) == 0 |
117 | && __alignof__ (struct gomp_work_share) | |
118 | >= __alignof__ (long long)) | |
fcfb8032 JJ |
119 | o &= ~(__alignof__ (long long) - 1); |
120 | ordered += o - 1; | |
28567c40 JJ |
121 | } |
122 | else | |
123 | ordered = nthreads * sizeof (*ws->ordered_team_ids); | |
124 | if (ordered > INLINE_ORDERED_TEAM_IDS_SIZE) | |
cee16451 | 125 | ws->ordered_team_ids = team_malloc (ordered); |
a68ab351 JJ |
126 | else |
127 | ws->ordered_team_ids = ws->inline_ordered_team_ids; | |
28567c40 | 128 | memset (ws->ordered_team_ids, '\0', ordered); |
a68ab351 JJ |
129 | ws->ordered_num_used = 0; |
130 | ws->ordered_owner = -1; | |
131 | ws->ordered_cur = 0; | |
132 | } | |
133 | else | |
28567c40 | 134 | ws->ordered_team_ids = ws->inline_ordered_team_ids; |
a68ab351 JJ |
135 | gomp_ptrlock_init (&ws->next_ws, NULL); |
136 | ws->threads_completed = 0; | |
137 | } | |
953ff289 | 138 | |
a68ab351 JJ |
139 | /* Do any needed destruction of gomp_work_share fields before it |
140 | is put back into free gomp_work_share cache or freed. */ | |
953ff289 | 141 | |
a68ab351 JJ |
142 | void |
143 | gomp_fini_work_share (struct gomp_work_share *ws) | |
953ff289 DN |
144 | { |
145 | gomp_mutex_destroy (&ws->lock); | |
a68ab351 | 146 | if (ws->ordered_team_ids != ws->inline_ordered_team_ids) |
cee16451 | 147 | team_free (ws->ordered_team_ids); |
a68ab351 | 148 | gomp_ptrlock_destroy (&ws->next_ws); |
953ff289 DN |
149 | } |
150 | ||
a68ab351 JJ |
151 | /* Free a work share struct, if not orphaned, put it into current |
152 | team's free gomp_work_share cache. */ | |
153 | ||
154 | static inline void | |
155 | free_work_share (struct gomp_team *team, struct gomp_work_share *ws) | |
156 | { | |
157 | gomp_fini_work_share (ws); | |
158 | if (__builtin_expect (team == NULL, 0)) | |
159 | free (ws); | |
160 | else | |
161 | { | |
162 | struct gomp_work_share *next_ws; | |
163 | #ifdef HAVE_SYNC_BUILTINS | |
164 | do | |
165 | { | |
166 | next_ws = team->work_share_list_free; | |
167 | ws->next_free = next_ws; | |
168 | } | |
169 | while (!__sync_bool_compare_and_swap (&team->work_share_list_free, | |
170 | next_ws, ws)); | |
171 | #else | |
172 | gomp_mutex_lock (&team->work_share_list_free_lock); | |
173 | next_ws = team->work_share_list_free; | |
174 | ws->next_free = next_ws; | |
175 | team->work_share_list_free = ws; | |
176 | gomp_mutex_unlock (&team->work_share_list_free_lock); | |
177 | #endif | |
178 | } | |
179 | } | |
953ff289 DN |
180 | |
181 | /* The current thread is ready to begin the next work sharing construct. | |
182 | In all cases, thr->ts.work_share is updated to point to the new | |
183 | structure. In all cases the work_share lock is locked. Return true | |
184 | if this was the first thread to reach this point. */ | |
185 | ||
186 | bool | |
28567c40 | 187 | gomp_work_share_start (size_t ordered) |
953ff289 DN |
188 | { |
189 | struct gomp_thread *thr = gomp_thread (); | |
190 | struct gomp_team *team = thr->ts.team; | |
191 | struct gomp_work_share *ws; | |
953ff289 DN |
192 | |
193 | /* Work sharing constructs can be orphaned. */ | |
194 | if (team == NULL) | |
195 | { | |
17da2c74 | 196 | #ifdef GOMP_USE_ALIGNED_WORK_SHARES |
c7abdf46 JJ |
197 | ws = gomp_aligned_alloc (__alignof (struct gomp_work_share), |
198 | sizeof (*ws)); | |
199 | #else | |
a68ab351 | 200 | ws = gomp_malloc (sizeof (*ws)); |
c7abdf46 | 201 | #endif |
a68ab351 | 202 | gomp_init_work_share (ws, ordered, 1); |
953ff289 | 203 | thr->ts.work_share = ws; |
28567c40 | 204 | return true; |
953ff289 DN |
205 | } |
206 | ||
a68ab351 JJ |
207 | ws = thr->ts.work_share; |
208 | thr->ts.last_work_share = ws; | |
209 | ws = gomp_ptrlock_get (&ws->next_ws); | |
210 | if (ws == NULL) | |
953ff289 | 211 | { |
a68ab351 JJ |
212 | /* This thread encountered a new ws first. */ |
213 | struct gomp_work_share *ws = alloc_work_share (team); | |
214 | gomp_init_work_share (ws, ordered, team->nthreads); | |
953ff289 | 215 | thr->ts.work_share = ws; |
a68ab351 | 216 | return true; |
953ff289 | 217 | } |
a68ab351 | 218 | else |
953ff289 | 219 | { |
a68ab351 JJ |
220 | thr->ts.work_share = ws; |
221 | return false; | |
953ff289 | 222 | } |
953ff289 DN |
223 | } |
224 | ||
953ff289 DN |
225 | /* The current thread is done with its current work sharing construct. |
226 | This version does imply a barrier at the end of the work-share. */ | |
227 | ||
228 | void | |
229 | gomp_work_share_end (void) | |
230 | { | |
231 | struct gomp_thread *thr = gomp_thread (); | |
232 | struct gomp_team *team = thr->ts.team; | |
a68ab351 | 233 | gomp_barrier_state_t bstate; |
953ff289 DN |
234 | |
235 | /* Work sharing constructs can be orphaned. */ | |
236 | if (team == NULL) | |
237 | { | |
a68ab351 JJ |
238 | free_work_share (NULL, thr->ts.work_share); |
239 | thr->ts.work_share = NULL; | |
953ff289 DN |
240 | return; |
241 | } | |
242 | ||
a68ab351 | 243 | bstate = gomp_barrier_wait_start (&team->barrier); |
953ff289 | 244 | |
a68ab351 | 245 | if (gomp_barrier_last_thread (bstate)) |
953ff289 | 246 | { |
a68ab351 | 247 | if (__builtin_expect (thr->ts.last_work_share != NULL, 1)) |
acf0174b JJ |
248 | { |
249 | team->work_shares_to_free = thr->ts.work_share; | |
250 | free_work_share (team, thr->ts.last_work_share); | |
251 | } | |
953ff289 DN |
252 | } |
253 | ||
a68ab351 JJ |
254 | gomp_team_barrier_wait_end (&team->barrier, bstate); |
255 | thr->ts.last_work_share = NULL; | |
953ff289 DN |
256 | } |
257 | ||
acf0174b JJ |
258 | /* The current thread is done with its current work sharing construct. |
259 | This version implies a cancellable barrier at the end of the work-share. */ | |
260 | ||
261 | bool | |
262 | gomp_work_share_end_cancel (void) | |
263 | { | |
264 | struct gomp_thread *thr = gomp_thread (); | |
265 | struct gomp_team *team = thr->ts.team; | |
266 | gomp_barrier_state_t bstate; | |
267 | ||
268 | /* Cancellable work sharing constructs cannot be orphaned. */ | |
269 | bstate = gomp_barrier_wait_cancel_start (&team->barrier); | |
270 | ||
271 | if (gomp_barrier_last_thread (bstate)) | |
272 | { | |
273 | if (__builtin_expect (thr->ts.last_work_share != NULL, 1)) | |
274 | { | |
275 | team->work_shares_to_free = thr->ts.work_share; | |
276 | free_work_share (team, thr->ts.last_work_share); | |
277 | } | |
278 | } | |
279 | thr->ts.last_work_share = NULL; | |
280 | ||
281 | return gomp_team_barrier_wait_cancel_end (&team->barrier, bstate); | |
282 | } | |
283 | ||
953ff289 DN |
284 | /* The current thread is done with its current work sharing construct. |
285 | This version does NOT imply a barrier at the end of the work-share. */ | |
286 | ||
287 | void | |
288 | gomp_work_share_end_nowait (void) | |
289 | { | |
290 | struct gomp_thread *thr = gomp_thread (); | |
291 | struct gomp_team *team = thr->ts.team; | |
292 | struct gomp_work_share *ws = thr->ts.work_share; | |
293 | unsigned completed; | |
294 | ||
953ff289 DN |
295 | /* Work sharing constructs can be orphaned. */ |
296 | if (team == NULL) | |
297 | { | |
a68ab351 JJ |
298 | free_work_share (NULL, ws); |
299 | thr->ts.work_share = NULL; | |
953ff289 DN |
300 | return; |
301 | } | |
302 | ||
a68ab351 JJ |
303 | if (__builtin_expect (thr->ts.last_work_share == NULL, 0)) |
304 | return; | |
305 | ||
953ff289 DN |
306 | #ifdef HAVE_SYNC_BUILTINS |
307 | completed = __sync_add_and_fetch (&ws->threads_completed, 1); | |
308 | #else | |
309 | gomp_mutex_lock (&ws->lock); | |
310 | completed = ++ws->threads_completed; | |
311 | gomp_mutex_unlock (&ws->lock); | |
312 | #endif | |
313 | ||
314 | if (completed == team->nthreads) | |
acf0174b JJ |
315 | { |
316 | team->work_shares_to_free = thr->ts.work_share; | |
317 | free_work_share (team, thr->ts.last_work_share); | |
318 | } | |
a68ab351 | 319 | thr->ts.last_work_share = NULL; |
953ff289 | 320 | } |