-// Copyright (C) 2002, 2004, 2006 Free Software Foundation, Inc.
+// Copyright (C) 2002-2024 Free Software Foundation, Inc.
//
// This file is part of GCC.
//
// GCC is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation; either version 2, or (at your option)
+// the Free Software Foundation; either version 3, or (at your option)
// any later version.
// GCC is distributed in the hope that it will be useful,
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
-// You should have received a copy of the GNU General Public License
-// along with GCC; see the file COPYING. If not, write to
-// the Free Software Foundation, 51 Franklin Street, Fifth Floor,
-// Boston, MA 02110-1301, USA.
+// Under Section 7 of GPL version 3, you are granted additional
+// permissions described in the GCC Runtime Library Exception, version
+// 3.1, as published by the Free Software Foundation.
-// As a special exception, you may use this file as part of a free software
-// library without restriction. Specifically, if other files instantiate
-// templates or use macros or inline functions from this file, or you compile
-// this file and link it with other files to produce an executable, this
-// file does not by itself cause the resulting executable to be covered by
-// the GNU General Public License. This exception does not however
-// invalidate any other reasons why the executable file might be covered by
-// the GNU General Public License.
+// You should have received a copy of the GNU General Public License and
+// a copy of the GCC Runtime Library Exception along with this program;
+// see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
+// <http://www.gnu.org/licenses/>.
// Written by Mark Mitchell, CodeSourcery LLC, <mark@codesourcery.com>
// Thread support written by Jason Merrill, Red Hat Inc. <jason@redhat.com>
#include <bits/c++config.h>
#include <cxxabi.h>
#include <exception>
+#include <new>
+
+#ifdef __USING_MCFGTHREAD__
+
+#include <mcfgthread/cxa.h>
+
+namespace __cxxabiv1 {
+
+extern "C" int
+__cxa_guard_acquire (__guard* g) _GLIBCXX_NOTHROW
+ {
+ return __MCF_cxa_guard_acquire(g);
+ }
+
+extern "C" void
+__cxa_guard_release (__guard* g) _GLIBCXX_NOTHROW
+ {
+ __MCF_cxa_guard_release(g);
+ }
+
+extern "C" void
+__cxa_guard_abort (__guard* g) _GLIBCXX_NOTHROW
+ {
+ __MCF_cxa_guard_abort(g);
+ }
+
+} // namespace __cxxabiv1
+
+#else // __USING_MCFGTHREAD__
+
#include <ext/atomicity.h>
#include <ext/concurrence.h>
+#include <bits/atomic_lockfree_defines.h>
+#if defined(__GTHREADS) && defined(__GTHREAD_HAS_COND) \
+ && (ATOMIC_INT_LOCK_FREE > 1) && defined(_GLIBCXX_HAVE_LINUX_FUTEX)
+# include <climits>
+# include <syscall.h>
+# include <unistd.h>
+# define _GLIBCXX_USE_FUTEX
+# define _GLIBCXX_FUTEX_WAIT 0
+# define _GLIBCXX_FUTEX_WAKE 1
+#endif
// The IA64/generic ABI uses the first byte of the guard variable.
// The ARM EABI uses the least significant bit.
// Thread-safe static local initialization support.
#ifdef __GTHREADS
+# ifndef _GLIBCXX_USE_FUTEX
namespace
{
// A single mutex controlling all static initializations.
- __gnu_cxx::__recursive_mutex static_mutex;
+ static __gnu_cxx::__recursive_mutex* static_mutex;
+
+ typedef char fake_recursive_mutex[sizeof(__gnu_cxx::__recursive_mutex)]
+ __attribute__ ((aligned(__alignof__(__gnu_cxx::__recursive_mutex))));
+ fake_recursive_mutex fake_mutex;
+
+ static void init()
+ { static_mutex = new (&fake_mutex) __gnu_cxx::__recursive_mutex(); }
+
+ __gnu_cxx::__recursive_mutex&
+ get_static_mutex()
+ {
+ static __gthread_once_t once = __GTHREAD_ONCE_INIT;
+ __gthread_once(&once, init);
+ return *static_mutex;
+ }
+
+ // Simple wrapper for exception safety.
+ struct mutex_wrapper
+ {
+ bool unlock;
+ mutex_wrapper() : unlock(true)
+ { get_static_mutex().lock(); }
+
+ ~mutex_wrapper()
+ {
+ if (unlock)
+ static_mutex->unlock();
+ }
+ };
}
+# endif
+
+# if defined(__GTHREAD_HAS_COND) && !defined(_GLIBCXX_USE_FUTEX)
+namespace
+{
+ // A single condition variable controlling all static initializations.
+ static __gnu_cxx::__cond* static_cond;
+
+ // using a fake type to avoid initializing a static class.
+ typedef char fake_cond_t[sizeof(__gnu_cxx::__cond)]
+ __attribute__ ((aligned(__alignof__(__gnu_cxx::__cond))));
+ fake_cond_t fake_cond;
+
+ static void init_static_cond()
+ { static_cond = new (&fake_cond) __gnu_cxx::__cond(); }
+
+ __gnu_cxx::__cond&
+ get_static_cond()
+ {
+ static __gthread_once_t once = __GTHREAD_ONCE_INIT;
+ __gthread_once(&once, init_static_cond);
+ return *static_cond;
+ }
+}
+# endif
+
+# ifndef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
+
+// Test the guard variable with a memory load with
+// acquire semantics.
-#ifndef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
inline bool
__test_and_acquire (__cxxabiv1::__guard *g)
{
- bool b = _GLIBCXX_GUARD_TEST (g);
- _GLIBCXX_READ_MEM_BARRIER;
- return b;
+ unsigned char __c;
+ unsigned char *__p = reinterpret_cast<unsigned char *>(g);
+ __atomic_load (__p, &__c, __ATOMIC_ACQUIRE);
+ (void) __p;
+ return _GLIBCXX_GUARD_TEST(&__c);
}
-#define _GLIBCXX_GUARD_TEST_AND_ACQUIRE(G) __test_and_acquire (G)
-#endif
+# define _GLIBCXX_GUARD_TEST_AND_ACQUIRE(G) __test_and_acquire (G)
+# endif
+
+# ifndef _GLIBCXX_GUARD_SET_AND_RELEASE
+
+// Set the guard variable to 1 with memory order release semantics.
-#ifndef _GLIBCXX_GUARD_SET_AND_RELEASE
inline void
__set_and_release (__cxxabiv1::__guard *g)
{
- _GLIBCXX_WRITE_MEM_BARRIER;
- _GLIBCXX_GUARD_SET (g);
+ unsigned char *__p = reinterpret_cast<unsigned char *>(g);
+ unsigned char val = 1;
+ __atomic_store (__p, &val, __ATOMIC_RELEASE);
+ (void) __p;
}
-#define _GLIBCXX_GUARD_SET_AND_RELEASE(G) __set_and_release (G)
-#endif
+# define _GLIBCXX_GUARD_SET_AND_RELEASE(G) __set_and_release (G)
+# endif
#else /* !__GTHREADS */
-#undef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
-#undef _GLIBCXX_GUARD_SET_AND_RELEASE
-#define _GLIBCXX_GUARD_SET_AND_RELEASE(G) _GLIBCXX_GUARD_SET (G)
+# undef _GLIBCXX_GUARD_TEST_AND_ACQUIRE
+# undef _GLIBCXX_GUARD_SET_AND_RELEASE
+# define _GLIBCXX_GUARD_SET_AND_RELEASE(G) _GLIBCXX_GUARD_SET (G)
#endif /* __GTHREADS */
-namespace __gnu_cxx
-{
- // 6.7[stmt.dcl]/4: If control re-enters the declaration (recursively)
- // while the object is being initialized, the behavior is undefined.
-
- // Since we already have a library function to handle locking, we might
- // as well check for this situation and throw an exception.
- // We use the second byte of the guard variable to remember that we're
- // in the middle of an initialization.
- class recursive_init_error: public std::exception
- {
- public:
- recursive_init_error() throw() { }
- virtual ~recursive_init_error() throw ();
- };
+//
+// Here are C++ run-time routines for guarded initialization of static
+// variables. There are 4 scenarios under which these routines are called:
+//
+// 1. Threads not supported (__GTHREADS not defined)
+// 2. Threads are supported but not enabled at run-time.
+// 3. Threads enabled at run-time but __gthreads_* are not fully POSIX.
+// 4. Threads enabled at run-time and __gthreads_* support all POSIX threads
+// primitives we need here.
+//
+// The old code supported scenarios 1-3 but was broken since it used a global
+// mutex for all threads and had the mutex locked during the whole duration of
+// initialization of a guarded static variable. The following created a
+// dead-lock with the old code.
+//
+// Thread 1 acquires the global mutex.
+// Thread 1 starts initializing static variable.
+// Thread 1 creates thread 2 during initialization.
+// Thread 2 attempts to acquire mutex to initialize another variable.
+// Thread 2 blocks since thread 1 is locking the mutex.
+// Thread 1 waits for result from thread 2 and also blocks. A deadlock.
+//
+// The new code here can handle this situation and thus is more robust. However,
+// we need to use the POSIX thread condition variable, which is not supported
+// in all platforms, notably older versions of Microsoft Windows. The gthr*.h
+// headers define a symbol __GTHREAD_HAS_COND for platforms that support POSIX
+// like condition variables. For platforms that do not support condition
+// variables, we need to fall back to the old code.
- recursive_init_error::~recursive_init_error() throw() { }
-}
+// If _GLIBCXX_USE_FUTEX, no global mutex or condition variable is used,
+// only atomic operations are used together with futex syscall.
+// Valid values of the first integer in guard are:
+// 0 No thread encountered the guarded init
+// yet or it has been aborted.
+// _GLIBCXX_GUARD_BIT The guarded static var has been successfully
+// initialized.
+// _GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
+// and no other thread is waiting for its
+// initialization.
+// (_GLIBCXX_GUARD_PENDING_BIT The guarded static var is being initialized
+// | _GLIBCXX_GUARD_WAITING_BIT) and some other threads are waiting until
+// it is initialized.
namespace __cxxabiv1
{
- static inline int
- recursion_push (__guard* g)
+#ifdef _GLIBCXX_USE_FUTEX
+ namespace
{
- return ((char *)g)[1]++;
+ static inline int __guard_test_bit (const int __byte, const int __val)
+ {
+ union { int __i; char __c[sizeof (int)]; } __u = { 0 };
+ __u.__c[__byte] = __val;
+ return __u.__i;
+ }
}
+#endif
+
+ static inline int
+ init_in_progress_flag(__guard* g)
+ { return ((char *)g)[1]; }
+
+ static inline void
+ set_init_in_progress_flag(__guard* g, int v)
+ { ((char *)g)[1] = v; }
static inline void
- recursion_pop (__guard* g)
+ throw_recursive_init_exception()
{
- --((char *)g)[1];
+#if __cpp_exceptions
+ throw __gnu_cxx::recursive_init_error();
+#else
+ // Use __builtin_trap so we don't require abort().
+ __builtin_trap();
+#endif
}
+ // acquire() is a helper function used to acquire guard if thread support is
+ // not compiled in or is compiled in but not enabled at run-time.
static int
- acquire_1 (__guard *g)
+ acquire(__guard *g)
{
- if (_GLIBCXX_GUARD_TEST (g))
+ // Quit if the object is already initialized.
+ if (_GLIBCXX_GUARD_TEST(g))
return 0;
- if (recursion_push (g))
- {
-#ifdef __EXCEPTIONS
- throw __gnu_cxx::recursive_init_error();
-#else
- // Use __builtin_trap so we don't require abort().
- __builtin_trap ();
-#endif
- }
+ if (init_in_progress_flag(g))
+ throw_recursive_init_exception();
+
+ set_init_in_progress_flag(g, 1);
return 1;
}
if (_GLIBCXX_GUARD_TEST_AND_ACQUIRE (g))
return 0;
- if (__gthread_active_p ())
+# ifdef _GLIBCXX_USE_FUTEX
+ // If __atomic_* and futex syscall are supported, don't use any global
+ // mutex.
+
+ // Use the same bits in the guard variable whether single-threaded or not,
+ // so that __cxa_guard_release and __cxa_guard_abort match the logic here
+ // even if __libc_single_threaded becomes false between now and then.
+
+ if (__gnu_cxx::__is_single_threaded())
{
- // Simple wrapper for exception safety.
- struct mutex_wrapper
- {
- bool unlock;
- mutex_wrapper (): unlock(true)
+ // No need to use atomics, and no need to wait for other threads.
+ int *gi = (int *) (void *) g;
+ if (*gi == 0)
{
- static_mutex.lock();
+ *gi = _GLIBCXX_GUARD_PENDING_BIT;
+ return 1;
}
- ~mutex_wrapper ()
+ else
+ throw_recursive_init_exception();
+ }
+ else
+ {
+ int *gi = (int *) (void *) g;
+ const int guard_bit = _GLIBCXX_GUARD_BIT;
+ const int pending_bit = _GLIBCXX_GUARD_PENDING_BIT;
+ const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
+
+ while (1)
{
- if (unlock)
- static_mutex.unlock();
+ int expected(0);
+ if (__atomic_compare_exchange_n(gi, &expected, pending_bit, false,
+ __ATOMIC_ACQ_REL,
+ __ATOMIC_ACQUIRE))
+ {
+ // This thread should do the initialization.
+ return 1;
+ }
+
+ if (expected == guard_bit)
+ {
+ // Already initialized.
+ return 0;
+ }
+
+ if (expected == pending_bit)
+ {
+ // Use acquire here.
+ int newv = expected | waiting_bit;
+ if (!__atomic_compare_exchange_n(gi, &expected, newv, false,
+ __ATOMIC_ACQ_REL,
+ __ATOMIC_ACQUIRE))
+ {
+ if (expected == guard_bit)
+ {
+ // Make a thread that failed to set the
+ // waiting bit exit the function earlier,
+ // if it detects that another thread has
+ // successfully finished initialising.
+ return 0;
+ }
+ if (expected == 0)
+ continue;
+ }
+
+ expected = newv;
+ }
+
+ syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAIT, expected, 0);
}
- } mw;
+ }
+# else // ! _GLIBCXX_USE_FUTEX
+ if (__gthread_active_p ())
+ {
+ mutex_wrapper mw;
- if (acquire_1 (g))
+ while (1) // When this loop is executing, mutex is locked.
{
- mw.unlock = false;
- return 1;
- }
+# ifdef __GTHREAD_HAS_COND
+ // The static is already initialized.
+ if (_GLIBCXX_GUARD_TEST(g))
+ return 0; // The mutex will be unlocked via wrapper
- return 0;
+ if (init_in_progress_flag(g))
+ {
+ // The guarded static is currently being initialized by
+ // another thread, so we release mutex and wait for the
+ // condition variable. We will lock the mutex again after
+ // this.
+ get_static_cond().wait_recursive(&get_static_mutex());
+ }
+ else
+ {
+ set_init_in_progress_flag(g, 1);
+ return 1; // The mutex will be unlocked via wrapper.
+ }
+# else
+ // This provides compatibility with older systems not supporting
+ // POSIX like condition variables.
+ if (acquire(g))
+ {
+ mw.unlock = false;
+ return 1; // The mutex still locked.
+ }
+ return 0; // The mutex will be unlocked via wrapper.
+# endif
+ }
}
-#endif
+# endif
+#endif // ! __GTHREADS
- return acquire_1 (g);
+ return acquire (g);
}
extern "C"
- void __cxa_guard_abort (__guard *g)
+ void __cxa_guard_abort (__guard *g) noexcept
{
- recursion_pop (g);
-#ifdef __GTHREADS
+#ifdef _GLIBCXX_USE_FUTEX
+ // If __atomic_* and futex syscall are supported, don't use any global
+ // mutex.
+
+ if (__gnu_cxx::__is_single_threaded())
+ {
+ // No need to use atomics, and no other threads to wake.
+ int *gi = (int *) (void *) g;
+ *gi = 0;
+ return;
+ }
+ else
+ {
+ int *gi = (int *) (void *) g;
+ const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
+ int old = __atomic_exchange_n (gi, 0, __ATOMIC_ACQ_REL);
+
+ if ((old & waiting_bit) != 0)
+ syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
+ return;
+ }
+#elif defined(__GTHREAD_HAS_COND)
+ if (__gthread_active_p())
+ {
+ mutex_wrapper mw;
+
+ set_init_in_progress_flag(g, 0);
+
+ // If we abort, we still need to wake up all other threads waiting for
+ // the condition variable.
+ get_static_cond().broadcast();
+ return;
+ }
+#endif
+
+ set_init_in_progress_flag(g, 0);
+#if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
+ // This provides compatibility with older systems not supporting POSIX like
+ // condition variables.
if (__gthread_active_p ())
- static_mutex.unlock();
+ static_mutex->unlock();
#endif
}
extern "C"
- void __cxa_guard_release (__guard *g)
+ void __cxa_guard_release (__guard *g) noexcept
{
- recursion_pop (g);
+#ifdef _GLIBCXX_USE_FUTEX
+ // If __atomic_* and futex syscall are supported, don't use any global
+ // mutex.
+
+ if (__gnu_cxx::__is_single_threaded())
+ {
+ int *gi = (int *) (void *) g;
+ *gi = _GLIBCXX_GUARD_BIT;
+ return;
+ }
+ else
+ {
+ int *gi = (int *) (void *) g;
+ const int guard_bit = _GLIBCXX_GUARD_BIT;
+ const int waiting_bit = _GLIBCXX_GUARD_WAITING_BIT;
+ int old = __atomic_exchange_n (gi, guard_bit, __ATOMIC_ACQ_REL);
+
+ if ((old & waiting_bit) != 0)
+ syscall (SYS_futex, gi, _GLIBCXX_FUTEX_WAKE, INT_MAX);
+ return;
+ }
+
+#elif defined(__GTHREAD_HAS_COND)
+ if (__gthread_active_p())
+ {
+ mutex_wrapper mw;
+
+ set_init_in_progress_flag(g, 0);
+ _GLIBCXX_GUARD_SET_AND_RELEASE(g);
+
+ get_static_cond().broadcast();
+ return;
+ }
+#endif
+
+ set_init_in_progress_flag(g, 0);
_GLIBCXX_GUARD_SET_AND_RELEASE (g);
-#ifdef __GTHREADS
- if (__gthread_active_p ())
- static_mutex.unlock();
+
+#if defined(__GTHREADS) && !defined(__GTHREAD_HAS_COND)
+ // This provides compatibility with older systems not supporting POSIX like
+ // condition variables.
+ if (__gthread_active_p())
+ static_mutex->unlock();
#endif
}
}
+
+#endif // __USING_MCFGTHREAD__