#if defined(_GLIBCXX_HAVE_LINUX_FUTEX) && ATOMIC_INT_LOCK_FREE > 1
struct __atomic_futex_unsigned_base
{
- // Returns false iff a timeout occurred.
+ // __s and __ns are measured against CLOCK_REALTIME. Returns false
+ // iff a timeout occurred.
bool
_M_futex_wait_until(unsigned *__addr, unsigned __val, bool __has_timeout,
chrono::seconds __s, chrono::nanoseconds __ns);
+ // __s and __ns are measured against CLOCK_MONOTONIC. Returns
+ // false iff a timeout occurred.
+ bool
+ _M_futex_wait_until_steady(unsigned *__addr, unsigned __val,
+ bool __has_timeout, chrono::seconds __s, chrono::nanoseconds __ns);
+
// This can be executed after the object has been destroyed.
static void _M_futex_notify_all(unsigned* __addr);
};
// value if equal is false.
// The assumed value is the caller's assumption about the current value
// when making the call.
+ // __s and __ns are measured against CLOCK_REALTIME.
unsigned
_M_load_and_test_until(unsigned __assumed, unsigned __operand,
bool __equal, memory_order __mo, bool __has_timeout,
}
}
+ // If a timeout occurs, returns a current value after the timeout;
+ // otherwise, returns the operand's value if equal is true or a different
+ // value if equal is false.
+ // The assumed value is the caller's assumption about the current value
+ // when making the call.
+ // __s and __ns are measured against CLOCK_MONOTONIC.
+ unsigned
+ _M_load_and_test_until_steady(unsigned __assumed, unsigned __operand,
+ bool __equal, memory_order __mo, bool __has_timeout,
+ chrono::seconds __s, chrono::nanoseconds __ns)
+ {
+ for (;;)
+ {
+ // Don't bother checking the value again because we expect the caller
+ // to have done it recently.
+ // memory_order_relaxed is sufficient because we can rely on just the
+ // modification order (store_notify uses an atomic RMW operation too),
+ // and the futex syscalls synchronize between themselves.
+ _M_data.fetch_or(_Waiter_bit, memory_order_relaxed);
+ bool __ret = _M_futex_wait_until_steady((unsigned*)(void*)&_M_data,
+ __assumed | _Waiter_bit,
+ __has_timeout, __s, __ns);
+ // Fetch the current value after waiting (clears _Waiter_bit).
+ __assumed = _M_load(__mo);
+ if (!__ret || ((__operand == __assumed) == __equal))
+ return __assumed;
+ // TODO adapt wait time
+ }
+ }
+
// Returns the operand's value if equal is true or a different value if
// equal is false.
// The assumed value is the caller's assumption about the current value
true, __s.time_since_epoch(), __ns);
}
+ template<typename _Dur>
+ unsigned
+ _M_load_and_test_until_impl(unsigned __assumed, unsigned __operand,
+ bool __equal, memory_order __mo,
+ const chrono::time_point<std::chrono::steady_clock, _Dur>& __atime)
+ {
+ auto __s = chrono::time_point_cast<chrono::seconds>(__atime);
+ auto __ns = chrono::duration_cast<chrono::nanoseconds>(__atime - __s);
+ // XXX correct?
+ return _M_load_and_test_until_steady(__assumed, __operand, __equal, __mo,
+ true, __s.time_since_epoch(), __ns);
+ }
+
public:
_GLIBCXX_ALWAYS_INLINE unsigned
return (__i & ~_Waiter_bit) == __val;
}
+ // Returns false iff a timeout occurred.
+ template<typename _Duration>
+ _GLIBCXX_ALWAYS_INLINE bool
+ _M_load_when_equal_until(unsigned __val, memory_order __mo,
+ const chrono::time_point<std::chrono::steady_clock, _Duration>& __atime)
+ {
+ unsigned __i = _M_load(__mo);
+ if ((__i & ~_Waiter_bit) == __val)
+ return true;
+ // TODO Spin-wait first. Ignore effect on timeout.
+ __i = _M_load_and_test_until_impl(__i, __val, true, __mo, __atime);
+ return (__i & ~_Waiter_bit) == __val;
+ }
+
_GLIBCXX_ALWAYS_INLINE void
_M_store_notify_all(unsigned __val, memory_order __mo)
{
#include <errno.h>
#include <debug/debug.h>
+#ifdef _GLIBCXX_USE_CLOCK_GETTIME_SYSCALL
+#include <unistd.h>
+#include <sys/syscall.h>
+#endif
+
// Constants for the wait/wake futex syscall operations
const unsigned futex_wait_op = 0;
const unsigned futex_wait_bitset_op = 9;
+const unsigned futex_clock_monotonic_flag = 0;
const unsigned futex_clock_realtime_flag = 256;
const unsigned futex_bitset_match_any = ~0;
const unsigned futex_wake_op = 1;
namespace
{
std::atomic<bool> futex_clock_realtime_unavailable;
+ std::atomic<bool> futex_clock_monotonic_unavailable;
}
namespace std _GLIBCXX_VISIBILITY(default)
}
}
+ bool
+ __atomic_futex_unsigned_base::_M_futex_wait_until_steady(unsigned *__addr,
+ unsigned __val,
+ bool __has_timeout, chrono::seconds __s, chrono::nanoseconds __ns)
+ {
+ if (!__has_timeout)
+ {
+ // Ignore whether we actually succeeded to block because at worst,
+ // we will fall back to spin-waiting. The only thing we could do
+ // here on errors is abort.
+ int ret __attribute__((unused));
+ ret = syscall (SYS_futex, __addr, futex_wait_op, __val, nullptr);
+ __glibcxx_assert(ret == 0 || errno == EINTR || errno == EAGAIN);
+ return true;
+ }
+ else
+ {
+ if (!futex_clock_monotonic_unavailable.load(std::memory_order_relaxed))
+ {
+ struct timespec rt;
+ rt.tv_sec = __s.count();
+ rt.tv_nsec = __ns.count();
+
+ if (syscall (SYS_futex, __addr,
+ futex_wait_bitset_op | futex_clock_monotonic_flag,
+ __val, &rt, nullptr, futex_bitset_match_any) == -1)
+ {
+ __glibcxx_assert(errno == EINTR || errno == EAGAIN
+ || errno == ETIMEDOUT || errno == ENOSYS);
+ if (errno == ETIMEDOUT)
+ return false;
+ else if (errno == ENOSYS)
+ {
+ futex_clock_monotonic_unavailable.store(true,
+ std::memory_order_relaxed);
+ // Fall through to legacy implementation if the system
+ // call is unavailable.
+ }
+ else
+ return true;
+ }
+ }
+
+ // We only get to here if futex_clock_monotonic_unavailable was
+ // true or has just been set to true.
+ struct timespec ts;
+#ifdef _GLIBCXX_USE_CLOCK_GETTIME_SYSCALL
+ syscall(SYS_clock_gettime, CLOCK_MONOTONIC, &ts);
+#else
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+#endif
+ // Convert the absolute timeout value to a relative timeout
+ struct timespec rt;
+ rt.tv_sec = __s.count() - ts.tv_sec;
+ rt.tv_nsec = __ns.count() - ts.tv_nsec;
+ if (rt.tv_nsec < 0)
+ {
+ rt.tv_nsec += 1000000000;
+ --rt.tv_sec;
+ }
+ // Did we already time out?
+ if (rt.tv_sec < 0)
+ return false;
+
+ if (syscall (SYS_futex, __addr, futex_wait_op, __val, &rt) == -1)
+ {
+ __glibcxx_assert(errno == EINTR || errno == EAGAIN
+ || errno == ETIMEDOUT);
+ if (errno == ETIMEDOUT)
+ return false;
+ }
+ return true;
+ }
+ }
+
void
__atomic_futex_unsigned_base::_M_futex_notify_all(unsigned* __addr)
{
projects / thirdparty / gcc.git / commitdiff
