3 // Copyright (C) 2020-2021 Free Software Foundation, Inc.
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
25 /** @file bits/atomic_wait.h
26 * This is an internal header file, included by other library headers.
27 * Do not attempt to use it directly. @headername{atomic}
30 #ifndef _GLIBCXX_ATOMIC_WAIT_H
31 #define _GLIBCXX_ATOMIC_WAIT_H 1
33 #pragma GCC system_header
35 #include <bits/c++config.h>
36 #if defined _GLIBCXX_HAS_GTHREADS || defined _GLIBCXX_HAVE_LINUX_FUTEX
37 #include <bits/functional_hash.h>
38 #include <bits/gthr.h>
39 #include <ext/numeric_traits.h>
41 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
46 # include <bits/functexcept.h>
47 // TODO get this from Autoconf
48 # define _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE 1
50 # include <bits/std_mutex.h> // std::mutex, std::__condvar
53 #define __cpp_lib_atomic_wait 201907L
55 namespace std
_GLIBCXX_VISIBILITY(default)
57 _GLIBCXX_BEGIN_NAMESPACE_VERSION
60 using __platform_wait_t
= int;
62 constexpr auto __atomic_spin_count_1
= 16;
63 constexpr auto __atomic_spin_count_2
= 12;
65 template<typename _Tp
>
66 inline constexpr bool __platform_wait_uses_type
67 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
68 = is_same_v
<remove_cv_t
<_Tp
>, __platform_wait_t
>;
73 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
74 enum class __futex_wait_flags
: int
76 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX_PRIVATE
85 __wait_private
= __wait
| __private_flag
,
86 __wake_private
= __wake
| __private_flag
,
87 __wait_bitset_private
= __wait_bitset
| __private_flag
,
88 __wake_bitset_private
= __wake_bitset
| __private_flag
,
89 __bitset_match_any
= -1
92 template<typename _Tp
>
94 __platform_wait(const _Tp
* __addr
, __platform_wait_t __val
) noexcept
98 auto __e
= syscall (SYS_futex
, static_cast<const void*>(__addr
),
99 static_cast<int>(__futex_wait_flags::__wait_private
),
101 if (!__e
|| errno
== EAGAIN
)
103 else if (errno
!= EINTR
)
104 __throw_system_error(__e
);
108 template<typename _Tp
>
110 __platform_notify(const _Tp
* __addr
, bool __all
) noexcept
112 syscall (SYS_futex
, static_cast<const void*>(__addr
),
113 static_cast<int>(__futex_wait_flags::__wake_private
),
114 __all
? INT_MAX
: 1);
120 alignas(64) __platform_wait_t _M_ver
= 0;
121 alignas(64) __platform_wait_t _M_wait
= 0;
123 #ifndef _GLIBCXX_HAVE_LINUX_FUTEX
124 using __lock_t
= lock_guard
<mutex
>;
128 __waiters() noexcept
= default;
132 _M_enter_wait() noexcept
134 __platform_wait_t __res
;
135 __atomic_load(&_M_ver
, &__res
, __ATOMIC_ACQUIRE
);
136 __atomic_fetch_add(&_M_wait
, 1, __ATOMIC_ACQ_REL
);
141 _M_leave_wait() noexcept
143 __atomic_fetch_sub(&_M_wait
, 1, __ATOMIC_ACQ_REL
);
147 _M_do_wait(__platform_wait_t __version
) noexcept
149 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
150 __platform_wait(&_M_ver
, __version
);
152 __platform_wait_t __cur
= 0;
153 while (__cur
<= __version
)
155 __waiters::__lock_t
__l(_M_mtx
);
157 __platform_wait_t __last
= __cur
;
158 __atomic_load(&_M_ver
, &__cur
, __ATOMIC_ACQUIRE
);
160 break; // break the loop if version overflows
166 _M_waiting() const noexcept
168 __platform_wait_t __res
;
169 __atomic_load(&_M_wait
, &__res
, __ATOMIC_ACQUIRE
);
174 _M_notify(bool __all
) noexcept
176 __atomic_fetch_add(&_M_ver
, 1, __ATOMIC_ACQ_REL
);
177 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
178 __platform_notify(&_M_ver
, __all
);
188 _S_for(const void* __t
)
190 const unsigned char __mask
= 0xf;
191 static __waiters __w
[__mask
+ 1];
193 auto __key
= _Hash_impl::hash(__t
) & __mask
;
201 __platform_wait_t _M_version
;
203 template<typename _Tp
>
204 __waiter(const _Tp
* __addr
) noexcept
205 : _M_w(__waiters::_S_for(static_cast<const void*>(__addr
)))
206 , _M_version(_M_w
._M_enter_wait())
210 { _M_w
._M_leave_wait(); }
212 void _M_do_wait() noexcept
213 { _M_w
._M_do_wait(_M_version
); }
217 __thread_yield() noexcept
219 #if defined _GLIBCXX_HAS_GTHREADS && defined _GLIBCXX_USE_SCHED_YIELD
225 __thread_relax() noexcept
227 #if defined __i386__ || defined __x86_64__
228 __builtin_ia32_pause();
233 } // namespace __detail
235 template<typename _Pred
>
237 __atomic_spin(_Pred
& __pred
) noexcept
239 for (auto __i
= 0; __i
< __detail::__atomic_spin_count_1
; ++__i
)
244 if (__i
< __detail::__atomic_spin_count_2
)
245 __detail::__thread_relax();
247 __detail::__thread_yield();
252 template<typename _Tp
, typename _Pred
>
254 __atomic_wait(const _Tp
* __addr
, _Tp __old
, _Pred __pred
) noexcept
256 using namespace __detail
;
257 if (std::__atomic_spin(__pred
))
260 __waiter
__w(__addr
);
263 if constexpr (__platform_wait_uses_type
<_Tp
>)
265 __platform_wait(__addr
, __old
);
269 // TODO support timed backoff when this can be moved into the lib
275 template<typename _Tp
>
277 __atomic_notify(const _Tp
* __addr
, bool __all
) noexcept
279 using namespace __detail
;
280 auto& __w
= __waiters::_S_for((void*)__addr
);
281 if (!__w
._M_waiting())
284 #ifdef _GLIBCXX_HAVE_LINUX_FUTEX
285 if constexpr (__platform_wait_uses_type
<_Tp
>)
287 __platform_notify((__platform_wait_t
*)(void*) __addr
, __all
);
292 __w
._M_notify(__all
);
295 _GLIBCXX_END_NAMESPACE_VERSION
297 #endif // GTHREADS || LINUX_FUTEX
298 #endif // _GLIBCXX_ATOMIC_WAIT_H