1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // Main internal TSan header file.
13 // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
14 // function-scope locals)
15 // - All functions/classes/etc reside in namespace __tsan, except for those
16 // declared in tsan_interface.h.
17 // - Platform-specific files should be used instead of ifdefs (*).
18 // - No system headers included in header files (*).
19 // - Platform specific headres included only into platform-specific files (*).
21 // (*) Except when inlining is critical for performance.
22 //===----------------------------------------------------------------------===//
27 #include "sanitizer_common/sanitizer_common.h"
28 #include "sanitizer_common/sanitizer_allocator64.h"
29 #include "tsan_clock.h"
30 #include "tsan_defs.h"
31 #include "tsan_flags.h"
32 #include "tsan_sync.h"
33 #include "tsan_trace.h"
34 #include "tsan_vector.h"
35 #include "tsan_report.h"
39 // Descriptor of user's memory block.
49 #if defined(TSAN_COMPAT_SHADOW) && TSAN_COMPAT_SHADOW
50 const uptr kAllocatorSpace
= 0x7d0000000000ULL
;
52 const uptr kAllocatorSpace
= 0x7d0000000000ULL
;
54 const uptr kAllocatorSize
= 0x10000000000ULL
; // 1T.
56 typedef SizeClassAllocator64
<kAllocatorSpace
, kAllocatorSize
, sizeof(MBlock
),
57 DefaultSizeClassMap
> PrimaryAllocator
;
58 typedef SizeClassAllocatorLocalCache
<PrimaryAllocator::kNumClasses
,
59 PrimaryAllocator
> AllocatorCache
;
60 typedef LargeMmapAllocator SecondaryAllocator
;
61 typedef CombinedAllocator
<PrimaryAllocator
, AllocatorCache
,
62 SecondaryAllocator
> Allocator
;
63 Allocator
*allocator();
66 void TsanCheckFailed(const char *file
, int line
, const char *cond
,
69 // FastState (from most significant bit):
77 FastState(u64 tid
, u64 epoch
) {
78 x_
= tid
<< kTidShift
;
79 x_
|= epoch
<< kClkShift
;
80 DCHECK(tid
== this->tid());
81 DCHECK(epoch
== this->epoch());
84 explicit FastState(u64 x
)
93 u64 res
= x_
>> kTidShift
;
98 u64 res
= (x_
<< (kTidBits
+ 1)) >> (64 - kClkBits
);
102 void IncrementEpoch() {
103 u64 old_epoch
= epoch();
104 x_
+= 1 << kClkShift
;
105 DCHECK_EQ(old_epoch
+ 1, epoch());
109 void SetIgnoreBit() { x_
|= kIgnoreBit
; }
110 void ClearIgnoreBit() { x_
&= ~kIgnoreBit
; }
111 bool GetIgnoreBit() const { return x_
& kIgnoreBit
; }
115 static const int kTidShift
= 64 - kTidBits
- 1;
116 static const int kClkShift
= kTidShift
- kClkBits
;
117 static const u64 kIgnoreBit
= 1ull;
118 static const u64 kFreedBit
= 1ull << 63;
122 // Shadow (from most significant bit):
129 class Shadow
: public FastState
{
131 explicit Shadow(u64 x
) : FastState(x
) { }
133 explicit Shadow(const FastState
&s
) : FastState(s
.x_
) { }
135 void SetAddr0AndSizeLog(u64 addr0
, unsigned kAccessSizeLog
) {
136 DCHECK_EQ(x_
& 31, 0);
138 DCHECK_LE(kAccessSizeLog
, 3);
139 x_
|= (kAccessSizeLog
<< 3) | addr0
;
140 DCHECK_EQ(kAccessSizeLog
, size_log());
141 DCHECK_EQ(addr0
, this->addr0());
144 void SetWrite(unsigned kAccessIsWrite
) {
145 DCHECK_EQ(x_
& 32, 0);
148 DCHECK_EQ(kAccessIsWrite
, is_write());
151 bool IsZero() const { return x_
== 0; }
153 static inline bool TidsAreEqual(const Shadow s1
, const Shadow s2
) {
154 u64 shifted_xor
= (s1
.x_
^ s2
.x_
) >> kTidShift
;
155 DCHECK_EQ(shifted_xor
== 0, s1
.tid() == s2
.tid());
156 return shifted_xor
== 0;
159 static inline bool Addr0AndSizeAreEqual(const Shadow s1
, const Shadow s2
) {
160 u64 masked_xor
= (s1
.x_
^ s2
.x_
) & 31;
161 return masked_xor
== 0;
164 static inline bool TwoRangesIntersect(Shadow s1
, Shadow s2
,
165 unsigned kS2AccessSize
) {
167 u64 diff
= s1
.addr0() - s2
.addr0();
168 if ((s64
)diff
< 0) { // s1.addr0 < s2.addr0 // NOLINT
169 // if (s1.addr0() + size1) > s2.addr0()) return true;
170 if (s1
.size() > -diff
) res
= true;
172 // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
173 if (kS2AccessSize
> diff
) res
= true;
175 DCHECK_EQ(res
, TwoRangesIntersectSLOW(s1
, s2
));
176 DCHECK_EQ(res
, TwoRangesIntersectSLOW(s2
, s1
));
180 // The idea behind the offset is as follows.
181 // Consider that we have 8 bool's contained within a single 8-byte block
182 // (mapped to a single shadow "cell"). Now consider that we write to the bools
183 // from a single thread (which we consider the common case).
184 // W/o offsetting each access will have to scan 4 shadow values at average
185 // to find the corresponding shadow value for the bool.
186 // With offsetting we start scanning shadow with the offset so that
187 // each access hits necessary shadow straight off (at least in an expected
189 // This logic works seamlessly for any layout of user data. For example,
190 // if user data is {int, short, char, char}, then accesses to the int are
191 // offsetted to 0, short - 4, 1st char - 6, 2nd char - 7. Hopefully, accesses
192 // from a single thread won't need to scan all 8 shadow values.
193 unsigned ComputeSearchOffset() {
196 u64
addr0() const { return x_
& 7; }
197 u64
size() const { return 1ull << size_log(); }
198 bool is_write() const { return x_
& 32; }
200 // The idea behind the freed bit is as follows.
201 // When the memory is freed (or otherwise unaccessible) we write to the shadow
202 // values with tid/epoch related to the free and the freed bit set.
203 // During memory accesses processing the freed bit is considered
204 // as msb of tid. So any access races with shadow with freed bit set
205 // (it is as if write from a thread with which we never synchronized before).
206 // This allows us to detect accesses to freed memory w/o additional
207 // overheads in memory access processing and at the same time restore
208 // tid/epoch of free.
213 bool GetFreedAndReset() {
214 bool res
= x_
& kFreedBit
;
220 u64
size_log() const { return (x_
>> 3) & 3; }
222 static bool TwoRangesIntersectSLOW(const Shadow s1
, const Shadow s2
) {
223 if (s1
.addr0() == s2
.addr0()) return true;
224 if (s1
.addr0() < s2
.addr0() && s1
.addr0() + s1
.size() > s2
.addr0())
226 if (s2
.addr0() < s1
.addr0() && s2
.addr0() + s2
.size() > s1
.addr0())
232 struct SignalContext
;
234 // This struct is stored in TLS.
236 FastState fast_state
;
237 // Synch epoch represents the threads's epoch before the last synchronization
238 // action. It allows to reduce number of shadow state updates.
239 // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
240 // if we are processing write to X from the same thread at epoch=200,
241 // we do nothing, because both writes happen in the same 'synch epoch'.
242 // That is, if another memory access does not race with the former write,
243 // it does not race with the latter as well.
244 // QUESTION: can we can squeeze this into ThreadState::Fast?
245 // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
246 // taken by epoch between synchs.
247 // This way we can save one load from tls.
248 u64 fast_synch_epoch
;
249 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
250 // We do not distinguish beteween ignoring reads and writes
251 // for better performance.
252 int ignore_reads_and_writes
;
253 uptr
*shadow_stack_pos
;
254 u64
*racy_shadow_addr
;
258 // C/C++ uses embed shadow stack of fixed size.
259 uptr shadow_stack
[kShadowStackSize
];
261 // Go uses satellite shadow stack with dynamic size.
263 uptr
*shadow_stack_end
;
267 AllocatorCache alloc_cache
;
279 DeadlockDetector deadlock_detector
;
281 bool in_signal_handler
;
282 SignalContext
*signal_ctx
;
285 u32 last_sleep_stack_id
;
286 ThreadClock last_sleep_clock
;
289 // Set in regions of runtime that must be signal-safe and fork-safe.
290 // If set, malloc must not be called.
293 explicit ThreadState(Context
*ctx
, int tid
, int unique_id
, u64 epoch
,
294 uptr stk_addr
, uptr stk_size
,
295 uptr tls_addr
, uptr tls_size
);
301 extern THREADLOCAL
char cur_thread_placeholder
[];
302 INLINE ThreadState
*cur_thread() {
303 return reinterpret_cast<ThreadState
*>(&cur_thread_placeholder
);
308 ThreadStatusInvalid
, // Non-existent thread, data is invalid.
309 ThreadStatusCreated
, // Created but not yet running.
310 ThreadStatusRunning
, // The thread is currently running.
311 ThreadStatusFinished
, // Joinable thread is finished but not yet joined.
312 ThreadStatusDead
// Joined, but some info (trace) is still alive.
315 // An info about a thread that is hold for some time after its termination.
316 struct ThreadDeadInfo
{
320 struct ThreadContext
{
322 int unique_id
; // Non-rolling thread id.
324 uptr user_id
; // Some opaque user thread id (e.g. pthread_t).
330 // Epoch at which the thread had started.
331 // If we see an event from the thread stamped by an older epoch,
332 // the event is from a dead thread that shared tid with this thread.
335 StackTrace creation_stack
;
336 ThreadDeadInfo
*dead_info
;
337 ThreadContext
*dead_next
; // In dead thread list.
339 explicit ThreadContext(int tid
);
344 bool operator==(const RacyStacks
&other
) const {
345 if (hash
[0] == other
.hash
[0] && hash
[1] == other
.hash
[1])
347 if (hash
[0] == other
.hash
[1] && hash
[1] == other
.hash
[0])
358 struct FiredSuppression
{
372 int nmissed_expected
;
376 unsigned unique_thread_seq
;
378 int max_alive_threads
;
379 ThreadContext
*threads
[kMaxTid
];
381 ThreadContext
* dead_list_head
;
382 ThreadContext
* dead_list_tail
;
384 Vector
<RacyStacks
> racy_stacks
;
385 Vector
<RacyAddress
> racy_addresses
;
386 Vector
<FiredSuppression
> fired_suppressions
;
391 u64 int_alloc_cnt
[MBlockTypeCount
];
392 u64 int_alloc_siz
[MBlockTypeCount
];
407 explicit ScopedReport(ReportType typ
);
410 void AddStack(const StackTrace
*stack
);
411 void AddMemoryAccess(uptr addr
, Shadow s
, const StackTrace
*stack
);
412 void AddThread(const ThreadContext
*tctx
);
413 void AddMutex(const SyncVar
*s
);
414 void AddLocation(uptr addr
, uptr size
);
415 void AddSleep(u32 stack_id
);
417 const ReportDesc
*GetReport() const;
423 ScopedReport(const ScopedReport
&);
424 void operator = (const ScopedReport
&);
427 void RestoreStack(int tid
, const u64 epoch
, StackTrace
*stk
);
429 void StatAggregate(u64
*dst
, u64
*src
);
430 void StatOutput(u64
*stat
);
431 void ALWAYS_INLINE INLINE
StatInc(ThreadState
*thr
, StatType typ
, u64 n
= 1) {
436 void MapShadow(uptr addr
, uptr size
);
437 void InitializeShadowMemory();
438 void InitializeInterceptors();
439 void InitializeDynamicAnnotations();
441 void ReportRace(ThreadState
*thr
);
442 bool OutputReport(Context
*ctx
,
443 const ScopedReport
&srep
,
444 const ReportStack
*suppress_stack
= 0);
445 bool IsFiredSuppression(Context
*ctx
,
446 const ScopedReport
&srep
,
447 const StackTrace
&trace
);
448 bool IsExpectedReport(uptr addr
, uptr size
);
450 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
451 # define DPrintf Printf
453 # define DPrintf(...)
456 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
457 # define DPrintf2 Printf
459 # define DPrintf2(...)
462 u32
CurrentStackId(ThreadState
*thr
, uptr pc
);
463 void PrintCurrentStack(ThreadState
*thr
, uptr pc
);
465 void Initialize(ThreadState
*thr
);
466 int Finalize(ThreadState
*thr
);
468 void MemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
469 int kAccessSizeLog
, bool kAccessIsWrite
);
470 void MemoryAccessImpl(ThreadState
*thr
, uptr addr
,
471 int kAccessSizeLog
, bool kAccessIsWrite
,
472 u64
*shadow_mem
, Shadow cur
);
473 void MemoryRead1Byte(ThreadState
*thr
, uptr pc
, uptr addr
);
474 void MemoryWrite1Byte(ThreadState
*thr
, uptr pc
, uptr addr
);
475 void MemoryRead8Byte(ThreadState
*thr
, uptr pc
, uptr addr
);
476 void MemoryWrite8Byte(ThreadState
*thr
, uptr pc
, uptr addr
);
477 void MemoryAccessRange(ThreadState
*thr
, uptr pc
, uptr addr
,
478 uptr size
, bool is_write
);
479 void MemoryResetRange(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
480 void MemoryRangeFreed(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
481 void MemoryRangeImitateWrite(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
482 void IgnoreCtl(ThreadState
*thr
, bool write
, bool begin
);
484 void FuncEntry(ThreadState
*thr
, uptr pc
);
485 void FuncExit(ThreadState
*thr
);
487 int ThreadCreate(ThreadState
*thr
, uptr pc
, uptr uid
, bool detached
);
488 void ThreadStart(ThreadState
*thr
, int tid
, uptr os_id
);
489 void ThreadFinish(ThreadState
*thr
);
490 int ThreadTid(ThreadState
*thr
, uptr pc
, uptr uid
);
491 void ThreadJoin(ThreadState
*thr
, uptr pc
, int tid
);
492 void ThreadDetach(ThreadState
*thr
, uptr pc
, int tid
);
493 void ThreadFinalize(ThreadState
*thr
);
494 int ThreadCount(ThreadState
*thr
);
495 void ProcessPendingSignals(ThreadState
*thr
);
497 void MutexCreate(ThreadState
*thr
, uptr pc
, uptr addr
,
498 bool rw
, bool recursive
, bool linker_init
);
499 void MutexDestroy(ThreadState
*thr
, uptr pc
, uptr addr
);
500 void MutexLock(ThreadState
*thr
, uptr pc
, uptr addr
);
501 void MutexUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
502 void MutexReadLock(ThreadState
*thr
, uptr pc
, uptr addr
);
503 void MutexReadUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
504 void MutexReadOrWriteUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
506 void Acquire(ThreadState
*thr
, uptr pc
, uptr addr
);
507 void AcquireGlobal(ThreadState
*thr
, uptr pc
);
508 void Release(ThreadState
*thr
, uptr pc
, uptr addr
);
509 void ReleaseStore(ThreadState
*thr
, uptr pc
, uptr addr
);
510 void AfterSleep(ThreadState
*thr
, uptr pc
);
512 // The hacky call uses custom calling convention and an assembly thunk.
513 // It is considerably faster that a normal call for the caller
514 // if it is not executed (it is intended for slow paths from hot functions).
515 // The trick is that the call preserves all registers and the compiler
516 // does not treat it as a call.
517 // If it does not work for you, use normal call.
518 #if 0 && TSAN_DEBUG == 0
519 // The caller may not create the stack frame for itself at all,
520 // so we create a reserve stack frame for it (1024b must be enough).
521 #define HACKY_CALL(f) \
522 __asm__ __volatile__("sub $1024, %%rsp;" \
523 "/*.cfi_adjust_cfa_offset 1024;*/" \
524 ".hidden " #f "_thunk;" \
525 "call " #f "_thunk;" \
526 "add $1024, %%rsp;" \
527 "/*.cfi_adjust_cfa_offset -1024;*/" \
530 #define HACKY_CALL(f) f()
533 void TraceSwitch(ThreadState
*thr
);
535 extern "C" void __tsan_trace_switch();
536 void ALWAYS_INLINE INLINE
TraceAddEvent(ThreadState
*thr
, u64 epoch
,
537 EventType typ
, uptr addr
) {
538 StatInc(thr
, StatEvents
);
539 if (UNLIKELY((epoch
% kTracePartSize
) == 0)) {
541 HACKY_CALL(__tsan_trace_switch
);
546 Event
*evp
= &thr
->trace
.events
[epoch
% kTraceSize
];
547 Event ev
= (u64
)addr
| ((u64
)typ
<< 61);
551 } // namespace __tsan