libitm/libitm_i.h

   1 /* Copyright (C) 2008-2016 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Transactional Memory Library (libitm).
   5
   6    Libitm is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    Libitm is distributed in the hope that it will be useful, but WITHOUT ANY
  12    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  13    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14    more details.
  15
  16    Under Section 7 of GPL version 3, you are granted additional
  17    permissions described in the GCC Runtime Library Exception, version
  18    3.1, as published by the Free Software Foundation.
  19
  20    You should have received a copy of the GNU General Public License and
  21    a copy of the GCC Runtime Library Exception along with this program;
  22    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  23    <http://www.gnu.org/licenses/>.  */
  24
  25 /* The following are internal implementation functions and definitions.
  26    To distinguish them from those defined by the Intel ABI, they all
  27    begin with GTM/gtm.  */
  28
  29 #ifndef LIBITM_I_H
  30 #define LIBITM_I_H 1
  31
  32 #include "libitm.h"
  33 #include "config.h"
  34
  35 #include <assert.h>
  36 #include <stdlib.h>
  37 #include <string.h>
  38 #include <unwind.h>
  39 #include "local_type_traits"
  40 #include "local_atomic"
  41
  42 /* Don't require libgcc_s.so for exceptions.  */
  43 extern void _Unwind_DeleteException (_Unwind_Exception*) __attribute__((weak));
  44
  45
  46 #include "common.h"
  47
  48 namespace GTM HIDDEN {
  49
  50 using namespace std;
  51
  52 // A helper template for accessing an unsigned integral of SIZE bytes.
  53 template<size_t SIZE> struct sized_integral { };
  54 template<> struct sized_integral<1> { typedef uint8_t type; };
  55 template<> struct sized_integral<2> { typedef uint16_t type; };
  56 template<> struct sized_integral<4> { typedef uint32_t type; };
  57 template<> struct sized_integral<8> { typedef uint64_t type; };
  58
  59 typedef unsigned int gtm_word __attribute__((mode (word)));
  60
  61 // These values are given to GTM_restart_transaction and indicate the
  62 // reason for the restart.  The reason is used to decide what STM
  63 // implementation should be used during the next iteration.
  64 enum gtm_restart_reason
  65 {
  66   RESTART_REALLOCATE,
  67   RESTART_LOCKED_READ,
  68   RESTART_LOCKED_WRITE,
  69   RESTART_VALIDATE_READ,
  70   RESTART_VALIDATE_WRITE,
  71   RESTART_VALIDATE_COMMIT,
  72   RESTART_SERIAL_IRR,
  73   RESTART_NOT_READONLY,
  74   RESTART_CLOSED_NESTING,
  75   RESTART_INIT_METHOD_GROUP,
  76   NUM_RESTARTS,
  77   NO_RESTART = NUM_RESTARTS
  78 };
  79
  80 } // namespace GTM
  81
  82 #include "target.h"
  83 #include "rwlock.h"
  84 #include "aatree.h"
  85 #include "dispatch.h"
  86 #include "containers.h"
  87
  88 #ifdef __USER_LABEL_PREFIX__
  89 # define UPFX UPFX1(__USER_LABEL_PREFIX__)
  90 # define UPFX1(t) UPFX2(t)
  91 # define UPFX2(t) #t
  92 #else
  93 # define UPFX
  94 #endif
  95
  96 namespace GTM HIDDEN {
  97
  98 // A log of (de)allocation actions.  We defer handling of some actions until
  99 // a commit of the outermost transaction.  We also rely on potentially having
 100 // both an allocation and a deallocation for the same piece of memory in the
 101 // log; the order in which such entries are processed does not matter because
 102 // the actions are not in conflict (see below).
 103 // This type is private to alloc.c, but needs to be defined so that
 104 // the template used inside gtm_thread can instantiate.
 105 struct gtm_alloc_action
 106 {
 107   // Iff free_fn_sz is nonzero, it must be used instead of free_fn, and vice
 108   // versa.
 109   void (*free_fn)(void *);
 110   void (*free_fn_sz)(void *, size_t);
 111   size_t sz;
 112   // If true, this is an allocation; we discard the log entry on outermost
 113   // commit, and deallocate on abort.  If false, this is a deallocation and
 114   // we deallocate on outermost commit and discard the log entry on abort.
 115   bool allocated;
 116 };
 117
 118 struct gtm_thread;
 119
 120 // A transaction checkpoint: data that has to saved and restored when doing
 121 // closed nesting.
 122 struct gtm_transaction_cp
 123 {
 124   gtm_jmpbuf jb;
 125   size_t undolog_size;
 126   aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
 127   size_t user_actions_size;
 128   _ITM_transactionId_t id;
 129   uint32_t prop;
 130   uint32_t cxa_catch_count;
 131   unsigned int cxa_uncaught_count;
 132   // We might want to use a different but compatible dispatch method for
 133   // a nested transaction.
 134   abi_dispatch *disp;
 135   // Nesting level of this checkpoint (1 means that this is a checkpoint of
 136   // the outermost transaction).
 137   uint32_t nesting;
 138
 139   void save(gtm_thread* tx);
 140   void commit(gtm_thread* tx);
 141 };
 142
 143 // An undo log for writes.
 144 struct gtm_undolog
 145 {
 146   vector<gtm_word> undolog;
 147
 148   // Log the previous value at a certain address.
 149   // The easiest way to inline this is to just define this here.
 150   void log(const void *ptr, size_t len)
 151   {
 152     size_t words = (len + sizeof(gtm_word) - 1) / sizeof(gtm_word);
 153     gtm_word *undo = undolog.push(words + 2);
 154     memcpy(undo, ptr, len);
 155     undo[words] = len;
 156     undo[words + 1] = (gtm_word) ptr;
 157   }
 158
 159   void commit () { undolog.clear(); }
 160   size_t size() const { return undolog.size(); }
 161
 162   // In local.cc
 163   void rollback (gtm_thread* tx, size_t until_size = 0);
 164 };
 165
 166 // An entry of a read or write log.  Used by multi-lock TM methods.
 167 struct gtm_rwlog_entry
 168 {
 169   atomic<gtm_word> *orec;
 170   gtm_word value;
 171 };
 172
 173 // Contains all thread-specific data required by the entire library.
 174 // This includes all data relevant to a single transaction. Because most
 175 // thread-specific data is about the current transaction, we also refer to
 176 // the transaction-specific parts of gtm_thread as "the transaction" (the
 177 // same applies to names of variables and arguments).
 178 // All but the shared part of this data structure are thread-local data.
 179 // gtm_thread could be split into transaction-specific structures and other
 180 // per-thread data (with those parts then nested in gtm_thread), but this
 181 // would make it harder to later rearrange individual members to optimize data
 182 // accesses. Thus, for now we keep one flat object, and will only split it if
 183 // the code gets too messy.
 184 struct gtm_thread
 185 {
 186
 187   struct user_action
 188   {
 189     _ITM_userCommitFunction fn;
 190     void *arg;
 191     bool on_commit;
 192     _ITM_transactionId_t resuming_id;
 193   };
 194
 195   // The jump buffer by which GTM_longjmp restarts the transaction.
 196   // This field *must* be at the beginning of the transaction.
 197   gtm_jmpbuf jb;
 198
 199   // Data used by local.c for the undo log for both local and shared memory.
 200   gtm_undolog undolog;
 201
 202   // Read and write logs.  Used by multi-lock TM methods.
 203   vector<gtm_rwlog_entry> readlog;
 204   vector<gtm_rwlog_entry> writelog;
 205
 206   // Data used by alloc.c for the malloc/free undo log.
 207   aa_tree<uintptr_t, gtm_alloc_action> alloc_actions;
 208
 209   // Data used by useraction.c for the user-defined commit/abort handlers.
 210   vector<user_action> user_actions;
 211
 212   // A numerical identifier for this transaction.
 213   _ITM_transactionId_t id;
 214
 215   // The _ITM_codeProperties of this transaction as given by the compiler.
 216   uint32_t prop;
 217
 218   // The nesting depth for subsequently started transactions. This variable
 219   // will be set to 1 when starting an outermost transaction.
 220   uint32_t nesting;
 221
 222   // Set if this transaction owns the serial write lock.
 223   // Can be reset only when restarting the outermost transaction.
 224   static const uint32_t STATE_SERIAL            = 0x0001;
 225   // Set if the serial-irrevocable dispatch table is installed.
 226   // Implies that no logging is being done, and abort is not possible.
 227   // Can be reset only when restarting the outermost transaction.
 228   static const uint32_t STATE_IRREVOCABLE       = 0x0002;
 229
 230   // A bitmask of the above.
 231   uint32_t state;
 232
 233   // In order to reduce cacheline contention on global_tid during
 234   // beginTransaction, we allocate a block of 2**N ids to the thread
 235   // all at once.  This number is the next value to be allocated from
 236   // the block, or 0 % 2**N if no such block is allocated.
 237   _ITM_transactionId_t local_tid;
 238
 239   // Data used by eh_cpp.c for managing exceptions within the transaction.
 240   uint32_t cxa_catch_count;
 241   // If cxa_uncaught_count_ptr is 0, we don't need to roll back exceptions.
 242   unsigned int *cxa_uncaught_count_ptr;
 243   unsigned int cxa_uncaught_count;
 244   void *eh_in_flight;
 245
 246   // Checkpoints for closed nesting.
 247   vector<gtm_transaction_cp> parent_txns;
 248
 249   // Data used by retry.c for deciding what STM implementation should
 250   // be used for the next iteration of the transaction.
 251   // Only restart_total is reset to zero when the transaction commits, the
 252   // other counters are total values for all previously executed transactions.
 253   // restart_total is also used by the HTM fastpath in a different way.
 254   uint32_t restart_reason[NUM_RESTARTS];
 255   uint32_t restart_total;
 256
 257   // *** The shared part of gtm_thread starts here. ***
 258   // Shared state is on separate cachelines to avoid false sharing with
 259   // thread-local parts of gtm_thread.
 260
 261   // Points to the next thread in the list of all threads.
 262   gtm_thread *next_thread __attribute__((__aligned__(HW_CACHELINE_SIZE)));
 263
 264   // If this transaction is inactive, shared_state is ~0. Otherwise, this is
 265   // an active or serial transaction.
 266   atomic<gtm_word> shared_state;
 267
 268   // The lock that provides access to serial mode.  Non-serialized
 269   // transactions acquire read locks; a serialized transaction aquires
 270   // a write lock.
 271   // Accessed from assembly language, thus the "asm" specifier on
 272   // the name, avoiding complex name mangling.
 273   static gtm_rwlock serial_lock __asm__(UPFX "gtm_serial_lock");
 274
 275   // The head of the list of all threads' transactions.
 276   static gtm_thread *list_of_threads;
 277   // The number of all registered threads.
 278   static unsigned number_of_threads;
 279
 280   // In alloc.cc
 281   void commit_allocations (bool, aa_tree<uintptr_t, gtm_alloc_action>*);
 282   void record_allocation (void *, void (*)(void *));
 283   void forget_allocation (void *, void (*)(void *));
 284   void forget_allocation (void *, size_t, void (*)(void *, size_t));
 285   void discard_allocation (const void *ptr)
 286   {
 287     alloc_actions.erase((uintptr_t) ptr);
 288   }
 289
 290   // In beginend.cc
 291   void rollback (gtm_transaction_cp *cp = 0, bool aborting = false);
 292   bool trycommit ();
 293   void restart (gtm_restart_reason, bool finish_serial_upgrade = false)
 294         ITM_NORETURN;
 295
 296   gtm_thread();
 297   ~gtm_thread();
 298
 299   static void *operator new(size_t);
 300   static void operator delete(void *);
 301
 302   // Invoked from assembly language, thus the "asm" specifier on
 303   // the name, avoiding complex name mangling.
 304   static uint32_t begin_transaction(uint32_t, const gtm_jmpbuf *)
 305         __asm__(UPFX "GTM_begin_transaction") ITM_REGPARM;
 306   // In eh_cpp.cc
 307   void init_cpp_exceptions ();
 308   void revert_cpp_exceptions (gtm_transaction_cp *cp = 0);
 309
 310   // In retry.cc
 311   // Must be called outside of transactions (i.e., after rollback).
 312   void decide_retry_strategy (gtm_restart_reason);
 313   abi_dispatch* decide_begin_dispatch (uint32_t prop);
 314   void number_of_threads_changed(unsigned previous, unsigned now);
 315   // Must be called from serial mode. Does not call set_abi_disp().
 316   void set_default_dispatch(abi_dispatch* disp);
 317
 318   // In method-serial.cc
 319   void serialirr_mode ();
 320
 321   // In useraction.cc
 322   void rollback_user_actions (size_t until_size = 0);
 323   void commit_user_actions ();
 324 };
 325
 326 } // namespace GTM
 327
 328 #include "tls.h"
 329
 330 namespace GTM HIDDEN {
 331
 332 // An unscaled count of the number of times we should spin attempting to
 333 // acquire locks before we block the current thread and defer to the OS.
 334 // This variable isn't used when the standard POSIX lock implementations
 335 // are used.
 336 extern uint64_t gtm_spin_count_var;
 337
 338 extern "C" uint32_t GTM_longjmp (uint32_t, const gtm_jmpbuf *, uint32_t)
 339         ITM_NORETURN ITM_REGPARM;
 340
 341 extern "C" void GTM_LB (const void *, size_t) ITM_REGPARM;
 342
 343 extern void GTM_error (const char *fmt, ...)
 344         __attribute__((format (printf, 1, 2)));
 345 extern void GTM_fatal (const char *fmt, ...)
 346         __attribute__((noreturn, format (printf, 1, 2)));
 347
 348 extern abi_dispatch *dispatch_serial();
 349 extern abi_dispatch *dispatch_serialirr();
 350 extern abi_dispatch *dispatch_serialirr_onwrite();
 351 extern abi_dispatch *dispatch_gl_wt();
 352 extern abi_dispatch *dispatch_ml_wt();
 353 extern abi_dispatch *dispatch_htm();
 354
 355 // Control variable for the HTM fastpath that uses serial mode as fallback.
 356 // Non-zero if the HTM fastpath is enabled. See gtm_thread::begin_transaction.
 357 // Accessed from assembly language, thus the "asm" specifier on
 358 // the name, avoiding complex name mangling.
 359 extern uint32_t htm_fastpath __asm__(UPFX "gtm_htm_fastpath");
 360
 361 } // namespace GTM
 362
 363 #endif // LIBITM_I_H