]>
Commit | Line | Data |
---|---|---|
eae801ba TR |
1 | // futex -*- C++ -*- |
2 | ||
8d9254fc | 3 | // Copyright (C) 2015-2020 Free Software Foundation, Inc. |
eae801ba TR |
4 | // |
5 | // This file is part of the GNU ISO C++ Library. This library is free | |
6 | // software; you can redistribute it and/or modify it under the | |
7 | // terms of the GNU General Public License as published by the | |
8 | // Free Software Foundation; either version 3, or (at your option) | |
9 | // any later version. | |
10 | ||
11 | // This library is distributed in the hope that it will be useful, | |
12 | // but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | // GNU General Public License for more details. | |
15 | ||
16 | // Under Section 7 of GPL version 3, you are granted additional | |
17 | // permissions described in the GCC Runtime Library Exception, version | |
18 | // 3.1, as published by the Free Software Foundation. | |
19 | ||
20 | // You should have received a copy of the GNU General Public License and | |
21 | // a copy of the GCC Runtime Library Exception along with this program; | |
22 | // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see | |
23 | // <http://www.gnu.org/licenses/>. | |
24 | ||
eae801ba | 25 | #include <bits/atomic_futex.h> |
8ba7f29e | 26 | #ifdef _GLIBCXX_HAS_GTHREADS |
1e3919ac | 27 | #if defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1 |
eae801ba TR |
28 | #include <chrono> |
29 | #include <climits> | |
30 | #include <syscall.h> | |
31 | #include <unistd.h> | |
32 | #include <sys/time.h> | |
33 | #include <errno.h> | |
e7e0eeeb | 34 | #include <ext/numeric_traits.h> |
eae801ba TR |
35 | #include <debug/debug.h> |
36 | ||
01d412ef MC |
37 | #ifdef _GLIBCXX_USE_CLOCK_GETTIME_SYSCALL |
38 | #include <unistd.h> | |
39 | #include <sys/syscall.h> | |
40 | #endif | |
41 | ||
eae801ba TR |
42 | // Constants for the wait/wake futex syscall operations |
43 | const unsigned futex_wait_op = 0; | |
5bad23ce | 44 | const unsigned futex_wait_bitset_op = 9; |
01d412ef | 45 | const unsigned futex_clock_monotonic_flag = 0; |
5bad23ce MC |
46 | const unsigned futex_clock_realtime_flag = 256; |
47 | const unsigned futex_bitset_match_any = ~0; | |
eae801ba TR |
48 | const unsigned futex_wake_op = 1; |
49 | ||
e7e0eeeb JW |
50 | namespace std _GLIBCXX_VISIBILITY(default) |
51 | { | |
52 | _GLIBCXX_BEGIN_NAMESPACE_VERSION | |
53 | ||
91004436 JW |
54 | using __gnu_cxx::__int_traits; |
55 | ||
5bad23ce MC |
56 | namespace |
57 | { | |
58 | std::atomic<bool> futex_clock_realtime_unavailable; | |
01d412ef | 59 | std::atomic<bool> futex_clock_monotonic_unavailable; |
5bad23ce | 60 | |
e7e0eeeb JW |
61 | // Return the relative duration from (now_s + now_ns) to (abs_s + abs_ns) |
62 | // as a timespec. | |
63 | struct timespec | |
64 | relative_timespec(chrono::seconds abs_s, chrono::nanoseconds abs_ns, | |
65 | time_t now_s, long now_ns) | |
66 | { | |
67 | struct timespec rt; | |
68 | ||
69 | // Did we already time out? | |
70 | if (now_s > abs_s.count()) | |
71 | { | |
72 | rt.tv_sec = -1; | |
73 | return rt; | |
74 | } | |
75 | ||
b8d36dcc | 76 | const auto rel_s = abs_s.count() - now_s; |
e7e0eeeb | 77 | |
b8d36dcc | 78 | // Convert the absolute timeout to a relative timeout, without overflow. |
91004436 | 79 | if (rel_s > __int_traits<time_t>::__max) [[unlikely]] |
e7e0eeeb | 80 | { |
b8d36dcc JW |
81 | rt.tv_sec = __int_traits<time_t>::__max; |
82 | rt.tv_nsec = 999999999; | |
83 | } | |
84 | else | |
85 | { | |
86 | rt.tv_sec = rel_s; | |
87 | rt.tv_nsec = abs_ns.count() - now_ns; | |
88 | if (rt.tv_nsec < 0) | |
89 | { | |
90 | rt.tv_nsec += 1000000000; | |
91 | --rt.tv_sec; | |
92 | } | |
e7e0eeeb JW |
93 | } |
94 | ||
95 | return rt; | |
96 | } | |
97 | } // namespace | |
eae801ba TR |
98 | |
99 | bool | |
e7e0eeeb JW |
100 | __atomic_futex_unsigned_base:: |
101 | _M_futex_wait_until(unsigned *__addr, unsigned __val, bool __has_timeout, | |
102 | chrono::seconds __s, chrono::nanoseconds __ns) | |
eae801ba TR |
103 | { |
104 | if (!__has_timeout) | |
105 | { | |
106 | // Ignore whether we actually succeeded to block because at worst, | |
107 | // we will fall back to spin-waiting. The only thing we could do | |
108 | // here on errors is abort. | |
109 | int ret __attribute__((unused)); | |
55089c2b | 110 | ret = syscall (SYS_futex, __addr, futex_wait_op, __val, nullptr); |
9fbd2e55 | 111 | __glibcxx_assert(ret == 0 || errno == EINTR || errno == EAGAIN); |
eae801ba TR |
112 | return true; |
113 | } | |
114 | else | |
115 | { | |
5bad23ce MC |
116 | if (!futex_clock_realtime_unavailable.load(std::memory_order_relaxed)) |
117 | { | |
93fc4774 | 118 | // futex sets errno=EINVAL for absolute timeouts before the epoch. |
91004436 | 119 | if (__s.count() < 0) |
93fc4774 JW |
120 | return false; |
121 | ||
91004436 JW |
122 | struct timespec rt; |
123 | if (__s.count() > __int_traits<time_t>::__max) [[unlikely]] | |
124 | rt.tv_sec = __int_traits<time_t>::__max; | |
125 | else | |
126 | rt.tv_sec = __s.count(); | |
127 | rt.tv_nsec = __ns.count(); | |
128 | ||
5bad23ce MC |
129 | if (syscall (SYS_futex, __addr, |
130 | futex_wait_bitset_op | futex_clock_realtime_flag, | |
131 | __val, &rt, nullptr, futex_bitset_match_any) == -1) | |
132 | { | |
133 | __glibcxx_assert(errno == EINTR || errno == EAGAIN | |
134 | || errno == ETIMEDOUT || errno == ENOSYS); | |
135 | if (errno == ETIMEDOUT) | |
136 | return false; | |
137 | if (errno == ENOSYS) | |
138 | { | |
139 | futex_clock_realtime_unavailable.store(true, | |
140 | std::memory_order_relaxed); | |
141 | // Fall through to legacy implementation if the system | |
142 | // call is unavailable. | |
143 | } | |
144 | else | |
145 | return true; | |
146 | } | |
147 | else | |
148 | return true; | |
149 | } | |
150 | ||
151 | // We only get to here if futex_clock_realtime_unavailable was | |
152 | // true or has just been set to true. | |
eae801ba TR |
153 | struct timeval tv; |
154 | gettimeofday (&tv, NULL); | |
e7e0eeeb | 155 | |
eae801ba | 156 | // Convert the absolute timeout value to a relative timeout |
e7e0eeeb JW |
157 | auto rt = relative_timespec(__s, __ns, tv.tv_sec, tv.tv_usec * 1000); |
158 | ||
eae801ba TR |
159 | // Did we already time out? |
160 | if (rt.tv_sec < 0) | |
161 | return false; | |
162 | ||
163 | if (syscall (SYS_futex, __addr, futex_wait_op, __val, &rt) == -1) | |
164 | { | |
9fbd2e55 JW |
165 | __glibcxx_assert(errno == EINTR || errno == EAGAIN |
166 | || errno == ETIMEDOUT); | |
eae801ba TR |
167 | if (errno == ETIMEDOUT) |
168 | return false; | |
169 | } | |
170 | return true; | |
171 | } | |
172 | } | |
173 | ||
01d412ef | 174 | bool |
e7e0eeeb JW |
175 | __atomic_futex_unsigned_base:: |
176 | _M_futex_wait_until_steady(unsigned *__addr, unsigned __val, | |
177 | bool __has_timeout, | |
178 | chrono::seconds __s, chrono::nanoseconds __ns) | |
01d412ef MC |
179 | { |
180 | if (!__has_timeout) | |
181 | { | |
182 | // Ignore whether we actually succeeded to block because at worst, | |
183 | // we will fall back to spin-waiting. The only thing we could do | |
184 | // here on errors is abort. | |
185 | int ret __attribute__((unused)); | |
186 | ret = syscall (SYS_futex, __addr, futex_wait_op, __val, nullptr); | |
187 | __glibcxx_assert(ret == 0 || errno == EINTR || errno == EAGAIN); | |
188 | return true; | |
189 | } | |
190 | else | |
191 | { | |
192 | if (!futex_clock_monotonic_unavailable.load(std::memory_order_relaxed)) | |
193 | { | |
93fc4774 | 194 | // futex sets errno=EINVAL for absolute timeouts before the epoch. |
91004436 | 195 | if (__s.count() < 0) [[unlikely]] |
93fc4774 JW |
196 | return false; |
197 | ||
91004436 JW |
198 | struct timespec rt; |
199 | if (__s.count() > __int_traits<time_t>::__max) [[unlikely]] | |
200 | rt.tv_sec = __int_traits<time_t>::__max; | |
201 | else | |
202 | rt.tv_sec = __s.count(); | |
203 | rt.tv_nsec = __ns.count(); | |
204 | ||
01d412ef MC |
205 | if (syscall (SYS_futex, __addr, |
206 | futex_wait_bitset_op | futex_clock_monotonic_flag, | |
207 | __val, &rt, nullptr, futex_bitset_match_any) == -1) | |
208 | { | |
209 | __glibcxx_assert(errno == EINTR || errno == EAGAIN | |
210 | || errno == ETIMEDOUT || errno == ENOSYS); | |
211 | if (errno == ETIMEDOUT) | |
212 | return false; | |
213 | else if (errno == ENOSYS) | |
214 | { | |
215 | futex_clock_monotonic_unavailable.store(true, | |
216 | std::memory_order_relaxed); | |
217 | // Fall through to legacy implementation if the system | |
218 | // call is unavailable. | |
219 | } | |
220 | else | |
221 | return true; | |
222 | } | |
223 | } | |
224 | ||
225 | // We only get to here if futex_clock_monotonic_unavailable was | |
226 | // true or has just been set to true. | |
227 | struct timespec ts; | |
228 | #ifdef _GLIBCXX_USE_CLOCK_GETTIME_SYSCALL | |
229 | syscall(SYS_clock_gettime, CLOCK_MONOTONIC, &ts); | |
230 | #else | |
231 | clock_gettime(CLOCK_MONOTONIC, &ts); | |
232 | #endif | |
e7e0eeeb | 233 | |
01d412ef | 234 | // Convert the absolute timeout value to a relative timeout |
e7e0eeeb JW |
235 | auto rt = relative_timespec(__s, __ns, ts.tv_sec, ts.tv_nsec); |
236 | ||
01d412ef MC |
237 | // Did we already time out? |
238 | if (rt.tv_sec < 0) | |
239 | return false; | |
240 | ||
241 | if (syscall (SYS_futex, __addr, futex_wait_op, __val, &rt) == -1) | |
242 | { | |
243 | __glibcxx_assert(errno == EINTR || errno == EAGAIN | |
244 | || errno == ETIMEDOUT); | |
245 | if (errno == ETIMEDOUT) | |
246 | return false; | |
247 | } | |
248 | return true; | |
249 | } | |
250 | } | |
251 | ||
eae801ba TR |
252 | void |
253 | __atomic_futex_unsigned_base::_M_futex_notify_all(unsigned* __addr) | |
254 | { | |
255 | // This syscall can fail for various reasons, including in situations | |
256 | // in which there is no real error. Thus, we don't bother checking | |
257 | // the error codes. See the futex documentation and glibc for background. | |
258 | syscall (SYS_futex, __addr, futex_wake_op, INT_MAX); | |
259 | } | |
260 | ||
8dcf3d3c | 261 | _GLIBCXX_END_NAMESPACE_VERSION |
eae801ba | 262 | } |
8ba7f29e JW |
263 | #endif // defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1 |
264 | #endif // _GLIBCXX_HAS_GTHREADS |