1 //===-- sanitizer_allocator.cpp -------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is shared between AddressSanitizer and ThreadSanitizer
10 // run-time libraries.
11 // This allocator is used inside run-times.
12 //===----------------------------------------------------------------------===//
14 #include "sanitizer_allocator.h"
16 #include "sanitizer_allocator_checks.h"
17 #include "sanitizer_allocator_internal.h"
18 #include "sanitizer_atomic.h"
19 #include "sanitizer_common.h"
21 namespace __sanitizer
{
23 // Default allocator names.
24 const char *PrimaryAllocatorName
= "SizeClassAllocator";
25 const char *SecondaryAllocatorName
= "LargeMmapAllocator";
27 // ThreadSanitizer for Go uses libc malloc/free.
28 #if defined(SANITIZER_USE_MALLOC)
29 # if SANITIZER_LINUX && !SANITIZER_ANDROID
30 extern "C" void *__libc_malloc(uptr size
);
32 extern "C" void *__libc_memalign(uptr alignment
, uptr size
);
34 extern "C" void *__libc_realloc(void *ptr
, uptr size
);
35 extern "C" void __libc_free(void *ptr
);
38 # define __libc_malloc malloc
40 static void *__libc_memalign(uptr alignment
, uptr size
) {
42 uptr error
= posix_memalign(&p
, alignment
, size
);
43 if (error
) return nullptr;
47 # define __libc_realloc realloc
48 # define __libc_free free
51 static void *RawInternalAlloc(uptr size
, InternalAllocatorCache
*cache
,
56 return __libc_malloc(size
);
58 return __libc_memalign(alignment
, size
);
60 // Windows does not provide __libc_memalign/posix_memalign. It provides
61 // __aligned_malloc, but the allocated blocks can't be passed to free,
62 // they need to be passed to __aligned_free. InternalAlloc interface does
63 // not account for such requirement. Alignemnt does not seem to be used
64 // anywhere in runtime, so just call __libc_malloc for now.
65 DCHECK_EQ(alignment
, 0);
66 return __libc_malloc(size
);
70 static void *RawInternalRealloc(void *ptr
, uptr size
,
71 InternalAllocatorCache
*cache
) {
73 return __libc_realloc(ptr
, size
);
76 static void RawInternalFree(void *ptr
, InternalAllocatorCache
*cache
) {
81 InternalAllocator
*internal_allocator() {
85 #else // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
87 static ALIGNED(64) char internal_alloc_placeholder
[sizeof(InternalAllocator
)];
88 static atomic_uint8_t internal_allocator_initialized
;
89 static StaticSpinMutex internal_alloc_init_mu
;
91 static InternalAllocatorCache internal_allocator_cache
;
92 static StaticSpinMutex internal_allocator_cache_mu
;
94 InternalAllocator
*internal_allocator() {
95 InternalAllocator
*internal_allocator_instance
=
96 reinterpret_cast<InternalAllocator
*>(&internal_alloc_placeholder
);
97 if (atomic_load(&internal_allocator_initialized
, memory_order_acquire
) == 0) {
98 SpinMutexLock
l(&internal_alloc_init_mu
);
99 if (atomic_load(&internal_allocator_initialized
, memory_order_relaxed
) ==
101 internal_allocator_instance
->Init(kReleaseToOSIntervalNever
);
102 atomic_store(&internal_allocator_initialized
, 1, memory_order_release
);
105 return internal_allocator_instance
;
108 static void *RawInternalAlloc(uptr size
, InternalAllocatorCache
*cache
,
110 if (alignment
== 0) alignment
= 8;
112 SpinMutexLock
l(&internal_allocator_cache_mu
);
113 return internal_allocator()->Allocate(&internal_allocator_cache
, size
,
116 return internal_allocator()->Allocate(cache
, size
, alignment
);
119 static void *RawInternalRealloc(void *ptr
, uptr size
,
120 InternalAllocatorCache
*cache
) {
123 SpinMutexLock
l(&internal_allocator_cache_mu
);
124 return internal_allocator()->Reallocate(&internal_allocator_cache
, ptr
,
127 return internal_allocator()->Reallocate(cache
, ptr
, size
, alignment
);
130 static void RawInternalFree(void *ptr
, InternalAllocatorCache
*cache
) {
132 SpinMutexLock
l(&internal_allocator_cache_mu
);
133 return internal_allocator()->Deallocate(&internal_allocator_cache
, ptr
);
135 internal_allocator()->Deallocate(cache
, ptr
);
138 #endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
140 const u64 kBlockMagic
= 0x6A6CB03ABCEBC041ull
;
142 static void NORETURN
ReportInternalAllocatorOutOfMemory(uptr requested_size
) {
143 SetAllocatorOutOfMemory();
144 Report("FATAL: %s: internal allocator is out of memory trying to allocate "
145 "0x%zx bytes\n", SanitizerToolName
, requested_size
);
149 void *InternalAlloc(uptr size
, InternalAllocatorCache
*cache
, uptr alignment
) {
150 if (size
+ sizeof(u64
) < size
)
152 void *p
= RawInternalAlloc(size
+ sizeof(u64
), cache
, alignment
);
154 ReportInternalAllocatorOutOfMemory(size
+ sizeof(u64
));
155 ((u64
*)p
)[0] = kBlockMagic
;
156 return (char*)p
+ sizeof(u64
);
159 void *InternalRealloc(void *addr
, uptr size
, InternalAllocatorCache
*cache
) {
161 return InternalAlloc(size
, cache
);
162 if (size
+ sizeof(u64
) < size
)
164 addr
= (char*)addr
- sizeof(u64
);
165 size
= size
+ sizeof(u64
);
166 CHECK_EQ(kBlockMagic
, ((u64
*)addr
)[0]);
167 void *p
= RawInternalRealloc(addr
, size
, cache
);
169 ReportInternalAllocatorOutOfMemory(size
);
170 return (char*)p
+ sizeof(u64
);
173 void *InternalReallocArray(void *addr
, uptr count
, uptr size
,
174 InternalAllocatorCache
*cache
) {
175 if (UNLIKELY(CheckForCallocOverflow(count
, size
))) {
177 "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) "
178 "cannot be represented in type size_t\n",
179 SanitizerToolName
, count
, size
);
182 return InternalRealloc(addr
, count
* size
, cache
);
185 void *InternalCalloc(uptr count
, uptr size
, InternalAllocatorCache
*cache
) {
186 if (UNLIKELY(CheckForCallocOverflow(count
, size
))) {
187 Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) "
188 "cannot be represented in type size_t\n", SanitizerToolName
, count
,
192 void *p
= InternalAlloc(count
* size
, cache
);
194 internal_memset(p
, 0, count
* size
);
198 void InternalFree(void *addr
, InternalAllocatorCache
*cache
) {
201 addr
= (char*)addr
- sizeof(u64
);
202 CHECK_EQ(kBlockMagic
, ((u64
*)addr
)[0]);
204 RawInternalFree(addr
, cache
);
208 constexpr uptr kLowLevelAllocatorDefaultAlignment
= 8;
209 static uptr low_level_alloc_min_alignment
= kLowLevelAllocatorDefaultAlignment
;
210 static LowLevelAllocateCallback low_level_alloc_callback
;
212 void *LowLevelAllocator::Allocate(uptr size
) {
213 // Align allocation size.
214 size
= RoundUpTo(size
, low_level_alloc_min_alignment
);
215 if (allocated_end_
- allocated_current_
< (sptr
)size
) {
216 uptr size_to_allocate
= RoundUpTo(size
, GetPageSizeCached());
218 (char*)MmapOrDie(size_to_allocate
, __func__
);
219 allocated_end_
= allocated_current_
+ size_to_allocate
;
220 if (low_level_alloc_callback
) {
221 low_level_alloc_callback((uptr
)allocated_current_
,
225 CHECK(allocated_end_
- allocated_current_
>= (sptr
)size
);
226 void *res
= allocated_current_
;
227 allocated_current_
+= size
;
231 void SetLowLevelAllocateMinAlignment(uptr alignment
) {
232 CHECK(IsPowerOfTwo(alignment
));
233 low_level_alloc_min_alignment
= Max(alignment
, low_level_alloc_min_alignment
);
236 void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback
) {
237 low_level_alloc_callback
= callback
;
240 // Allocator's OOM and other errors handling support.
242 static atomic_uint8_t allocator_out_of_memory
= {0};
243 static atomic_uint8_t allocator_may_return_null
= {0};
245 bool IsAllocatorOutOfMemory() {
246 return atomic_load_relaxed(&allocator_out_of_memory
);
249 void SetAllocatorOutOfMemory() {
250 atomic_store_relaxed(&allocator_out_of_memory
, 1);
253 bool AllocatorMayReturnNull() {
254 return atomic_load(&allocator_may_return_null
, memory_order_relaxed
);
257 void SetAllocatorMayReturnNull(bool may_return_null
) {
258 atomic_store(&allocator_may_return_null
, may_return_null
,
259 memory_order_relaxed
);
262 void PrintHintAllocatorCannotReturnNull() {
263 Report("HINT: if you don't care about these errors you may set "
264 "allocator_may_return_null=1\n");
267 } // namespace __sanitizer