-/* Copyright (C) 2008, 2009, 2011, 2012 Free Software Foundation, Inc.
+/* Copyright (C) 2008-2018 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Transactional Memory Library (libitm).
extern __thread gtm_thread_tls _gtm_thr_tls;
#endif
-gtm_rwlock GTM::gtm_thread::serial_lock;
+// Put this at the start of a cacheline so that serial_lock's writers and
+// htm_fastpath fields are on the same cacheline, so that HW transactions
+// only have to pay one cacheline capacity to monitor both.
+gtm_rwlock GTM::gtm_thread::serial_lock
+ __attribute__((aligned(HW_CACHELINE_SIZE)));
gtm_thread *GTM::gtm_thread::list_of_threads = 0;
unsigned GTM::gtm_thread::number_of_threads = 0;
-gtm_stmlock GTM::gtm_stmlock_array[LOCK_ARRAY_SIZE];
-atomic<gtm_version> GTM::gtm_clock;
-
/* ??? Move elsewhere when we figure out library initialization. */
uint64_t GTM::gtm_spin_count_var = 1000;
static pthread_key_t thr_release_key;
static pthread_once_t thr_release_once = PTHREAD_ONCE_INIT;
-
/* Allocate a transaction structure. */
void *
GTM::gtm_thread::operator new (size_t s)
number_of_threads_changed(number_of_threads - 1, number_of_threads);
serial_lock.write_unlock ();
+ init_cpp_exceptions ();
+
if (pthread_once(&thr_release_once, thread_exit_init))
GTM_fatal("Initializing thread release TLS key failed.");
// Any non-null value is sufficient to trigger destruction of this
return a_runInstrumentedCode;
}
+#ifdef TARGET_BEGIN_TRANSACTION_ATTRIBUTE
+/* This macro can be used to define target specific attributes for this
+ function. For example, S/390 requires floating point to be disabled in
+ begin_transaction. */
+TARGET_BEGIN_TRANSACTION_ATTRIBUTE
+#endif
uint32_t
GTM::gtm_thread::begin_transaction (uint32_t prop, const gtm_jmpbuf *jb)
{
if (unlikely(prop & pr_undoLogCode))
GTM_fatal("pr_undoLogCode not supported");
+#ifdef USE_HTM_FASTPATH
+ // HTM fastpath. Only chosen in the absence of transaction_cancel to allow
+ // using an uninstrumented code path.
+ // The fastpath is enabled only by dispatch_htm's method group, which uses
+ // serial-mode methods as fallback. Serial-mode transactions cannot execute
+ // concurrently with HW transactions because the latter monitor the serial
+ // lock's writer flag and thus abort if another thread is or becomes a
+ // serial transaction. Therefore, if the fastpath is enabled, then a
+ // transaction is not executing as a HW transaction iff the serial lock is
+ // write-locked. Also, HW transactions monitor the fastpath control
+ // variable, so that they will only execute if dispatch_htm is still the
+ // current method group. This allows us to use htm_fastpath and the serial
+ // lock's writers flag to reliable determine whether the current thread runs
+ // a HW transaction, and thus we do not need to maintain this information in
+ // per-thread state.
+ // If an uninstrumented code path is not available, we can still run
+ // instrumented code from a HW transaction because the HTM fastpath kicks
+ // in early in both begin and commit, and the transaction is not canceled.
+ // HW transactions might get requests to switch to serial-irrevocable mode,
+ // but these can be ignored because the HTM provides all necessary
+ // correctness guarantees. Transactions cannot detect whether they are
+ // indeed in serial mode, and HW transactions should never need serial mode
+ // for any internal changes (e.g., they never abort visibly to the STM code
+ // and thus do not trigger the standard retry handling).
+#ifndef HTM_CUSTOM_FASTPATH
+ if (likely(serial_lock.get_htm_fastpath() && (prop & pr_hasNoAbort)))
+ {
+ // Note that the snapshot of htm_fastpath that we take here could be
+ // outdated, and a different method group than dispatch_htm may have
+ // been chosen in the meantime. Therefore, take care not not touch
+ // anything besides the serial lock, which is independent of method
+ // groups.
+ for (uint32_t t = serial_lock.get_htm_fastpath(); t; t--)
+ {
+ uint32_t ret = htm_begin();
+ if (htm_begin_success(ret))
+ {
+ // We are executing a transaction now.
+ // Monitor the writer flag in the serial-mode lock, and abort
+ // if there is an active or waiting serial-mode transaction.
+ // Also checks that htm_fastpath is still nonzero and thus
+ // HW transactions are allowed to run.
+ // Note that this can also happen due to an enclosing
+ // serial-mode transaction; we handle this case below.
+ if (unlikely(serial_lock.htm_fastpath_disabled()))
+ htm_abort();
+ else
+ // We do not need to set a_saveLiveVariables because of HTM.
+ return (prop & pr_uninstrumentedCode) ?
+ a_runUninstrumentedCode : a_runInstrumentedCode;
+ }
+ // The transaction has aborted. Don't retry if it's unlikely that
+ // retrying the transaction will be successful.
+ if (!htm_abort_should_retry(ret))
+ break;
+ // Check whether the HTM fastpath has been disabled.
+ if (!serial_lock.get_htm_fastpath())
+ break;
+ // Wait until any concurrent serial-mode transactions have finished.
+ // This is an empty critical section, but won't be elided.
+ if (serial_lock.htm_fastpath_disabled())
+ {
+ tx = gtm_thr();
+ if (unlikely(tx == NULL))
+ {
+ // See below.
+ tx = new gtm_thread();
+ set_gtm_thr(tx);
+ }
+ // Check whether there is an enclosing serial-mode transaction;
+ // if so, we just continue as a nested transaction and don't
+ // try to use the HTM fastpath. This case can happen when an
+ // outermost relaxed transaction calls unsafe code that starts
+ // a transaction.
+ if (tx->nesting > 0)
+ break;
+ // Another thread is running a serial-mode transaction. Wait.
+ serial_lock.read_lock(tx);
+ serial_lock.read_unlock(tx);
+ // TODO We should probably reset the retry count t here, unless
+ // we have retried so often that we should go serial to avoid
+ // starvation.
+ }
+ }
+ }
+#else
+ // If we have a custom HTM fastpath in ITM_beginTransaction, we implement
+ // just the retry policy here. We communicate with the custom fastpath
+ // through additional property bits and return codes, and either transfer
+ // control back to the custom fastpath or run the fallback mechanism. The
+ // fastpath synchronization algorithm itself is the same.
+ // pr_HTMRetryableAbort states that a HW transaction started by the custom
+ // HTM fastpath aborted, and that we thus have to decide whether to retry
+ // the fastpath (returning a_tryHTMFastPath) or just proceed with the
+ // fallback method.
+ if (likely(serial_lock.get_htm_fastpath() && (prop & pr_HTMRetryableAbort)))
+ {
+ tx = gtm_thr();
+ if (unlikely(tx == NULL))
+ {
+ // See below.
+ tx = new gtm_thread();
+ set_gtm_thr(tx);
+ }
+ // If this is the first abort, reset the retry count. We abuse
+ // restart_total for the retry count, which is fine because our only
+ // other fallback will use serial transactions, which don't use
+ // restart_total but will reset it when committing.
+ if (!(prop & pr_HTMRetriedAfterAbort))
+ tx->restart_total = gtm_thread::serial_lock.get_htm_fastpath();
+
+ if (--tx->restart_total > 0)
+ {
+ // Wait until any concurrent serial-mode transactions have finished.
+ // Essentially the same code as above.
+ if (!serial_lock.get_htm_fastpath())
+ goto stop_custom_htm_fastpath;
+ if (serial_lock.htm_fastpath_disabled())
+ {
+ if (tx->nesting > 0)
+ goto stop_custom_htm_fastpath;
+ serial_lock.read_lock(tx);
+ serial_lock.read_unlock(tx);
+ }
+ // Let ITM_beginTransaction retry the custom HTM fastpath.
+ return a_tryHTMFastPath;
+ }
+ }
+ stop_custom_htm_fastpath:
+#endif
+#endif
+
tx = gtm_thr();
if (unlikely(tx == NULL))
{
{
// Outermost transaction
disp = tx->decide_begin_dispatch (prop);
- if (disp == dispatch_serialirr() || disp == dispatch_serial())
- {
- tx->state = STATE_SERIAL;
- if (disp == dispatch_serialirr())
- tx->state |= STATE_IRREVOCABLE;
- serial_lock.write_lock ();
- }
- else
- serial_lock.read_lock (tx);
-
set_abi_disp (disp);
}
#endif
}
+ // Log the number of uncaught exceptions if we might have to roll back this
+ // state.
+ if (tx->cxa_uncaught_count_ptr != 0)
+ tx->cxa_uncaught_count = *tx->cxa_uncaught_count_ptr;
+
// Run dispatch-specific restart code. Retry until we succeed.
GTM::gtm_restart_reason rr;
while ((rr = disp->begin_or_restart()) != NO_RESTART)
// Save everything that we might have to restore on restarts or aborts.
jb = tx->jb;
undolog_size = tx->undolog.size();
- memcpy(&alloc_actions, &tx->alloc_actions, sizeof(alloc_actions));
+
+ /* FIXME! Assignment of an aatree like alloc_actions is unsafe; if either
+ *this or *tx is destroyed, the other ends up pointing to a freed node. */
+#pragma GCC diagnostic warning "-Wdeprecated-copy"
+ alloc_actions = tx->alloc_actions;
+
user_actions_size = tx->user_actions.size();
id = tx->id;
prop = tx->prop;
cxa_catch_count = tx->cxa_catch_count;
- cxa_unthrown = tx->cxa_unthrown;
+ cxa_uncaught_count = tx->cxa_uncaught_count;
disp = abi_disp();
nesting = tx->nesting;
}
// commits of nested transactions. Allocation actions must be committed
// before committing the snapshot.
tx->jb = jb;
- memcpy(&tx->alloc_actions, &alloc_actions, sizeof(alloc_actions));
+ tx->alloc_actions = alloc_actions;
tx->id = id;
tx->prop = prop;
}
prop = cp->prop;
if (cp->disp != abi_disp())
set_abi_disp(cp->disp);
- memcpy(&alloc_actions, &cp->alloc_actions, sizeof(alloc_actions));
+ alloc_actions = cp->alloc_actions;
nesting = cp->nesting;
}
else
gtm_word priv_time = 0;
if (abi_disp()->trycommit (priv_time))
{
- // The transaction is now inactive. Everything that we still have to do
- // will not synchronize with other transactions anymore.
+ // The transaction is now finished but we will still access some shared
+ // data if we have to ensure privatization safety.
+ bool do_read_unlock = false;
if (state & gtm_thread::STATE_SERIAL)
{
gtm_thread::serial_lock.write_unlock ();
priv_time = 0;
}
else
- gtm_thread::serial_lock.read_unlock (this);
+ {
+ // If we have to ensure privatization safety, we must not yet
+ // release the read lock and become inactive because (1) we still
+ // have to go through the list of all transactions, which can be
+ // modified by serial mode threads, and (2) we interpret each
+ // transactions' shared_state in the context of what we believe to
+ // be the current method group (and serial mode transactions can
+ // change the method group). Therefore, if we have to ensure
+ // privatization safety, delay becoming inactive but set a maximum
+ // snapshot time (we have committed and thus have an empty snapshot,
+ // so it will always be most recent). Use release MO so that this
+ // synchronizes with other threads observing our snapshot time.
+ if (priv_time)
+ {
+ do_read_unlock = true;
+ shared_state.store((~(typeof gtm_thread::shared_state)0) - 1,
+ memory_order_release);
+ }
+ else
+ gtm_thread::serial_lock.read_unlock (this);
+ }
state = 0;
// We can commit the undo log after dispatch-specific commit and after
undolog.commit ();
// Reset further transaction state.
cxa_catch_count = 0;
- cxa_unthrown = NULL;
restart_total = 0;
// Ensure privatization safety, if necessary.
// acquisitions). This ensures that if we read prior to other
// reader transactions setting their shared_state to 0, then those
// readers will observe our updates. We can reuse the seq_cst fence
- // in serial_lock.read_unlock() however, so we don't need another
- // one here.
+ // in serial_lock.read_unlock() if we performed that; if not, we
+ // issue the fence.
+ if (do_read_unlock)
+ atomic_thread_fence (memory_order_seq_cst);
// TODO Don't just spin but also block using cond vars / futexes
// here. Should probably be integrated with the serial lock code.
for (gtm_thread *it = gtm_thread::list_of_threads; it != 0;
}
}
- // After ensuring privatization safety, we execute potentially
- // privatizing actions (e.g., calling free()). User actions are first.
+ // After ensuring privatization safety, we are now truly inactive and
+ // thus can release the read lock. We will also execute potentially
+ // privatizing actions (e.g., calling free()). User actions are first.
+ if (do_read_unlock)
+ gtm_thread::serial_lock.read_unlock (this);
commit_user_actions ();
commit_allocations (false, 0);
void ITM_REGPARM
_ITM_commitTransaction(void)
{
+#if defined(USE_HTM_FASTPATH)
+ // HTM fastpath. If we are not executing a HW transaction, then we will be
+ // a serial-mode transaction. If we are, then there will be no other
+ // concurrent serial-mode transaction.
+ // See gtm_thread::begin_transaction.
+ if (likely(!gtm_thread::serial_lock.htm_fastpath_disabled()))
+ {
+ htm_commit();
+ return;
+ }
+#endif
gtm_thread *tx = gtm_thr();
if (!tx->trycommit ())
tx->restart (RESTART_VALIDATE_COMMIT);
void ITM_REGPARM
_ITM_commitTransactionEH(void *exc_ptr)
{
+#if defined(USE_HTM_FASTPATH)
+ // See _ITM_commitTransaction.
+ if (likely(!gtm_thread::serial_lock.htm_fastpath_disabled()))
+ {
+ htm_commit();
+ return;
+ }
+#endif
gtm_thread *tx = gtm_thr();
if (!tx->trycommit ())
{
projects / thirdparty / gcc.git / blobdiff