1 // Copyright (C) 2002-2020 Free Software Foundation, Inc.
3 // This file is part of GCC.
5 // GCC is free software; you can redistribute it and/or modify
6 // it under the terms of the GNU General Public License as published by
7 // the Free Software Foundation; either version 3, or (at your option)
10 // GCC is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 // GNU General Public License for more details.
15 // Under Section 7 of GPL version 3, you are granted additional
16 // permissions described in the GCC Runtime Library Exception, version
17 // 3.1, as published by the Free Software Foundation.
19 // You should have received a copy of the GNU General Public License and
20 // a copy of the GCC Runtime Library Exception along with this program;
21 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
22 // <http://www.gnu.org/licenses/>.
24 // Written by Mark Mitchell, CodeSourcery LLC, <mark@codesourcery.com>
25 // Thread support written by Jason Merrill, Red Hat Inc. <jason@redhat.com>
27 #include <bits/c++config.h>
31 #include <ext/atomicity.h>
32 #include <ext/concurrence.h>
33 #include <bits/atomic_lockfree_defines.h>
34 #if defined(__GTHREADS) && defined(__GTHREAD_HAS_COND) \
35 && (ATOMIC_INT_LOCK_FREE > 1) && defined(_GLIBCXX_HAVE_LINUX_FUTEX)
39 # define _GLIBCXX_USE_FUTEX
40 # define _GLIBCXX_FUTEX_WAIT 0
41 # define _GLIBCXX_FUTEX_WAKE 1
44 // The IA64/generic ABI uses the first byte of the guard variable.
45 // The ARM EABI uses the least significant bit.
47 // Thread-safe static local initialization support.
49 # ifndef _GLIBCXX_USE_FUTEX
52 // A single mutex controlling all static initializations.
53 static __gnu_cxx::__recursive_mutex
* static_mutex
;
55 typedef char fake_recursive_mutex
[sizeof(__gnu_cxx::__recursive_mutex
)]
56 __attribute__ ((aligned(__alignof__(__gnu_cxx::__recursive_mutex
))));
57 fake_recursive_mutex fake_mutex
;
60 { static_mutex
= new (&fake_mutex
) __gnu_cxx::__recursive_mutex(); }
62 __gnu_cxx::__recursive_mutex
&
65 static __gthread_once_t once
= __GTHREAD_ONCE_INIT
;
66 __gthread_once(&once
, init
);
70 // Simple wrapper for exception safety.
74 mutex_wrapper() : unlock(true)
75 { get_static_mutex().lock(); }
80 static_mutex
->unlock();
86 # if defined(__GTHREAD_HAS_COND) && !defined(_GLIBCXX_USE_FUTEX)
89 // A single condition variable controlling all static initializations.
90 static __gnu_cxx::__cond
* static_cond
;
92 // using a fake type to avoid initializing a static class.
93 typedef char fake_cond_t
[sizeof(__gnu_cxx::__cond
)]
94 __attribute__ ((aligned(__alignof__(__gnu_cxx::__cond
))));
95 fake_cond_t fake_cond
;
97 static void init_static_cond()
98 { static_cond
= new (&fake_cond
) __gnu_cxx::__cond(); }
103 static __gthread_once_t once
= __GTHREAD_ONCE_INIT
;
104 __gthread_once(&once
, init_static_cond
);
110 # ifndef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
112 // Test the guard variable with a memory load with
113 // acquire semantics.
116 __test_and_acquire (__cxxabiv1::__guard
*g
)
119 unsigned char *__p
= reinterpret_cast<unsigned char *>(g
);
120 __atomic_load (__p
, &__c
, __ATOMIC_ACQUIRE
);
122 return _GLIBCXX_GUARD_TEST(&__c
);
124 # define _GLIBCXX_GUARD_TEST_AND_ACQUIRE(G) __test_and_acquire (G)
127 # ifndef _GLIBCXX_GUARD_SET_AND_RELEASE
129 // Set the guard variable to 1 with memory order release semantics.
132 __set_and_release (__cxxabiv1::__guard
*g
)
134 unsigned char *__p
= reinterpret_cast<unsigned char *>(g
);
135 unsigned char val
= 1;
136 __atomic_store (__p
, &val
, __ATOMIC_RELEASE
);
139 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) __set_and_release (G)
142 #else /* !__GTHREADS */
144 # undef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
145 # undef _GLIBCXX_GUARD_SET_AND_RELEASE
146 # define _GLIBCXX_GUARD_SET_AND_RELEASE(G) _GLIBCXX_GUARD_SET (G)
148 #endif /* __GTHREADS */
151 // Here are C++ run-time routines for guarded initialization of static
152 // variables. There are 4 scenarios under which these routines are called:
154 // 1. Threads not supported (__GTHREADS not defined)
155 // 2. Threads are supported but not enabled at run-time.
156 // 3. Threads enabled at run-time but __gthreads_* are not fully POSIX.
157 // 4. Threads enabled at run-time and __gthreads_* support all POSIX threads
158 // primitives we need here.
160 // The old code supported scenarios 1-3 but was broken since it used a global
161 // mutex for all threads and had the mutex locked during the whole duration of
162 // initialization of a guarded static variable. The following created a
163 // dead-lock with the old code.
165 // Thread 1 acquires the global mutex.
166 // Thread 1 starts initializing static variable.
167 // Thread 1 creates thread 2 during initialization.
168 // Thread 2 attempts to acquire mutex to initialize another variable.
169 // Thread 2 blocks since thread 1 is locking the mutex.
170 // Thread 1 waits for result from thread 2 and also blocks. A deadlock.
172 // The new code here can handle this situation and thus is more robust. However,
173 // we need to use the POSIX thread condition variable, which is not supported
174 // in all platforms, notably older versions of Microsoft Windows. The gthr*.h
175 // headers define a symbol __GTHREAD_HAS_COND for platforms that support POSIX
176 // like condition variables. For platforms that do not support condition
177 // variables, we need to fall back to the old code.
179 // If _GLIBCXX_USE_FUTEX, no global mutex or condition variable is used,
180 // only atomic operations are used together with futex syscall.
181 // Valid values of the first integer in guard are:
182 // 0 No thread encountered the guarded init
183 // yet or it has been aborted.
184 // _GLIBCXX_GUARD_BIT The guarded static var has been successfully
186 // _GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
187 // and no other thread is waiting for its
189 // (_GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
190 // | _GLIBCXX_GUARD_WAITING_BIT) and some other threads are waiting until
191 // it is initialized.
195 #ifdef _GLIBCXX_USE_FUTEX
198 static inline int __guard_test_bit (const int __byte
, const int __val
)
200 union { int __i
; char __c
[sizeof (int)]; } __u
= { 0 };
201 __u
.__c
[__byte
] = __val
;
208 init_in_progress_flag(__guard
* g
)
209 { return ((char *)g
)[1]; }
212 set_init_in_progress_flag(__guard
* g
, int v
)
213 { ((char *)g
)[1] = v
; }
216 throw_recursive_init_exception()
219 throw __gnu_cxx::recursive_init_error();
221 // Use __builtin_trap so we don't require abort().
226 // acquire() is a helper function used to acquire guard if thread support is
227 // not compiled in or is compiled in but not enabled at run-time.
231 // Quit if the object is already initialized.
232 if (_GLIBCXX_GUARD_TEST(g
))
235 if (init_in_progress_flag(g
))
236 throw_recursive_init_exception();
238 set_init_in_progress_flag(g
, 1);
243 int __cxa_guard_acquire (__guard
*g
)
246 // If the target can reorder loads, we need to insert a read memory
247 // barrier so that accesses to the guarded variable happen after the
249 if (_GLIBCXX_GUARD_TEST_AND_ACQUIRE (g
))
252 # ifdef _GLIBCXX_USE_FUTEX
253 // If __atomic_* and futex syscall are supported, don't use any global
256 // Use the same bits in the guard variable whether single-threaded or not,
257 // so that __cxa_guard_release and __cxa_guard_abort match the logic here
258 // even if __libc_single_threaded becomes false between now and then.
260 if (__gnu_cxx::__is_single_threaded())
262 // No need to use atomics, and no need to wait for other threads.
263 int *gi
= (int *) (void *) g
;
266 *gi
= _GLIBCXX_GUARD_PENDING_BIT
;
270 throw_recursive_init_exception();
274 int *gi
= (int *) (void *) g
;
275 const int guard_bit
= _GLIBCXX_GUARD_BIT
;
276 const int pending_bit
= _GLIBCXX_GUARD_PENDING_BIT
;
277 const int waiting_bit
= _GLIBCXX_GUARD_WAITING_BIT
;
282 if (__atomic_compare_exchange_n(gi
, &expected
, pending_bit
, false,
286 // This thread should do the initialization.
290 if (expected
== guard_bit
)
292 // Already initialized.
296 if (expected
== pending_bit
)
299 int newv
= expected
| waiting_bit
;
300 if (!__atomic_compare_exchange_n(gi
, &expected
, newv
, false,
304 if (expected
== guard_bit
)
306 // Make a thread that failed to set the
307 // waiting bit exit the function earlier,
308 // if it detects that another thread has
309 // successfully finished initialising.
319 syscall (SYS_futex
, gi
, _GLIBCXX_FUTEX_WAIT
, expected
, 0);
322 # else // ! _GLIBCXX_USE_FUTEX
323 if (__gthread_active_p ())
327 while (1) // When this loop is executing, mutex is locked.
329 # ifdef __GTHREAD_HAS_COND
330 // The static is already initialized.
331 if (_GLIBCXX_GUARD_TEST(g
))
332 return 0; // The mutex will be unlocked via wrapper
334 if (init_in_progress_flag(g
))
336 // The guarded static is currently being initialized by
337 // another thread, so we release mutex and wait for the
338 // condition variable. We will lock the mutex again after
340 get_static_cond().wait_recursive(&get_static_mutex());
344 set_init_in_progress_flag(g
, 1);
345 return 1; // The mutex will be unlocked via wrapper.
348 // This provides compatibility with older systems not supporting
349 // POSIX like condition variables.
353 return 1; // The mutex still locked.
355 return 0; // The mutex will be unlocked via wrapper.
360 #endif // ! __GTHREADS
366 void __cxa_guard_abort (__guard
*g
) noexcept
368 #ifdef _GLIBCXX_USE_FUTEX
369 // If __atomic_* and futex syscall are supported, don't use any global
372 if (__gnu_cxx::__is_single_threaded())
374 // No need to use atomics, and no other threads to wake.
375 int *gi
= (int *) (void *) g
;
381 int *gi
= (int *) (void *) g
;
382 const int waiting_bit
= _GLIBCXX_GUARD_WAITING_BIT
;
383 int old
= __atomic_exchange_n (gi
, 0, __ATOMIC_ACQ_REL
);
385 if ((old
& waiting_bit
) != 0)
386 syscall (SYS_futex
, gi
, _GLIBCXX_FUTEX_WAKE
, INT_MAX
);
389 #elif defined(__GTHREAD_HAS_COND)
390 if (__gthread_active_p())
394 set_init_in_progress_flag(g
, 0);
396 // If we abort, we still need to wake up all other threads waiting for
397 // the condition variable.
398 get_static_cond().broadcast();
403 set_init_in_progress_flag(g
, 0);
404 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
405 // This provides compatibility with older systems not supporting POSIX like
406 // condition variables.
407 if (__gthread_active_p ())
408 static_mutex
->unlock();
413 void __cxa_guard_release (__guard
*g
) noexcept
415 #ifdef _GLIBCXX_USE_FUTEX
416 // If __atomic_* and futex syscall are supported, don't use any global
419 if (__gnu_cxx::__is_single_threaded())
421 int *gi
= (int *) (void *) g
;
422 *gi
= _GLIBCXX_GUARD_BIT
;
427 int *gi
= (int *) (void *) g
;
428 const int guard_bit
= _GLIBCXX_GUARD_BIT
;
429 const int waiting_bit
= _GLIBCXX_GUARD_WAITING_BIT
;
430 int old
= __atomic_exchange_n (gi
, guard_bit
, __ATOMIC_ACQ_REL
);
432 if ((old
& waiting_bit
) != 0)
433 syscall (SYS_futex
, gi
, _GLIBCXX_FUTEX_WAKE
, INT_MAX
);
437 #elif defined(__GTHREAD_HAS_COND)
438 if (__gthread_active_p())
442 set_init_in_progress_flag(g
, 0);
443 _GLIBCXX_GUARD_SET_AND_RELEASE(g
);
445 get_static_cond().broadcast();
450 set_init_in_progress_flag(g
, 0);
451 _GLIBCXX_GUARD_SET_AND_RELEASE (g
);
453 #if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
454 // This provides compatibility with older systems not supporting POSIX like
455 // condition variables.
456 if (__gthread_active_p())
457 static_mutex
->unlock();