]>
Commit | Line | Data |
---|---|---|
b667dd70 | 1 | //===-- sanitizer_allocator.cpp -------------------------------------------===// |
f35db108 | 2 | // |
b667dd70 ML |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. | |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
f35db108 WM |
6 | // |
7 | //===----------------------------------------------------------------------===// | |
8 | // | |
9 | // This file is shared between AddressSanitizer and ThreadSanitizer | |
10 | // run-time libraries. | |
ef1b3fda | 11 | // This allocator is used inside run-times. |
f35db108 | 12 | //===----------------------------------------------------------------------===// |
696d846a | 13 | |
ef1b3fda | 14 | #include "sanitizer_allocator.h" |
10189819 | 15 | |
5d3805fc | 16 | #include "sanitizer_allocator_checks.h" |
ef1b3fda | 17 | #include "sanitizer_allocator_internal.h" |
10189819 | 18 | #include "sanitizer_atomic.h" |
f35db108 WM |
19 | #include "sanitizer_common.h" |
20 | ||
ef1b3fda KS |
21 | namespace __sanitizer { |
22 | ||
eac97531 ML |
23 | // Default allocator names. |
24 | const char *PrimaryAllocatorName = "SizeClassAllocator"; | |
25 | const char *SecondaryAllocatorName = "LargeMmapAllocator"; | |
26 | ||
ef1b3fda | 27 | // ThreadSanitizer for Go uses libc malloc/free. |
3c6331c2 | 28 | #if defined(SANITIZER_USE_MALLOC) |
ef1b3fda KS |
29 | # if SANITIZER_LINUX && !SANITIZER_ANDROID |
30 | extern "C" void *__libc_malloc(uptr size); | |
10189819 MO |
31 | # if !SANITIZER_GO |
32 | extern "C" void *__libc_memalign(uptr alignment, uptr size); | |
33 | # endif | |
34 | extern "C" void *__libc_realloc(void *ptr, uptr size); | |
f35db108 | 35 | extern "C" void __libc_free(void *ptr); |
ef1b3fda KS |
36 | # else |
37 | # include <stdlib.h> | |
10189819 MO |
38 | # define __libc_malloc malloc |
39 | # if !SANITIZER_GO | |
40 | static void *__libc_memalign(uptr alignment, uptr size) { | |
41 | void *p; | |
42 | uptr error = posix_memalign(&p, alignment, size); | |
43 | if (error) return nullptr; | |
44 | return p; | |
45 | } | |
46 | # endif | |
47 | # define __libc_realloc realloc | |
48 | # define __libc_free free | |
ef1b3fda | 49 | # endif |
f35db108 | 50 | |
10189819 MO |
51 | static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, |
52 | uptr alignment) { | |
53 | (void)cache; | |
54 | #if !SANITIZER_GO | |
55 | if (alignment == 0) | |
56 | return __libc_malloc(size); | |
57 | else | |
58 | return __libc_memalign(alignment, size); | |
59 | #else | |
60 | // Windows does not provide __libc_memalign/posix_memalign. It provides | |
61 | // __aligned_malloc, but the allocated blocks can't be passed to free, | |
62 | // they need to be passed to __aligned_free. InternalAlloc interface does | |
63 | // not account for such requirement. Alignemnt does not seem to be used | |
64 | // anywhere in runtime, so just call __libc_malloc for now. | |
65 | DCHECK_EQ(alignment, 0); | |
66 | return __libc_malloc(size); | |
67 | #endif | |
68 | } | |
69 | ||
70 | static void *RawInternalRealloc(void *ptr, uptr size, | |
71 | InternalAllocatorCache *cache) { | |
ef1b3fda | 72 | (void)cache; |
10189819 | 73 | return __libc_realloc(ptr, size); |
ef1b3fda KS |
74 | } |
75 | ||
76 | static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { | |
77 | (void)cache; | |
10189819 | 78 | __libc_free(ptr); |
ef1b3fda KS |
79 | } |
80 | ||
81 | InternalAllocator *internal_allocator() { | |
82 | return 0; | |
83 | } | |
84 | ||
10189819 | 85 | #else // SANITIZER_GO || defined(SANITIZER_USE_MALLOC) |
ef1b3fda KS |
86 | |
87 | static ALIGNED(64) char internal_alloc_placeholder[sizeof(InternalAllocator)]; | |
88 | static atomic_uint8_t internal_allocator_initialized; | |
89 | static StaticSpinMutex internal_alloc_init_mu; | |
90 | ||
91 | static InternalAllocatorCache internal_allocator_cache; | |
92 | static StaticSpinMutex internal_allocator_cache_mu; | |
93 | ||
94 | InternalAllocator *internal_allocator() { | |
95 | InternalAllocator *internal_allocator_instance = | |
96 | reinterpret_cast<InternalAllocator *>(&internal_alloc_placeholder); | |
97 | if (atomic_load(&internal_allocator_initialized, memory_order_acquire) == 0) { | |
98 | SpinMutexLock l(&internal_alloc_init_mu); | |
99 | if (atomic_load(&internal_allocator_initialized, memory_order_relaxed) == | |
100 | 0) { | |
5d3805fc | 101 | internal_allocator_instance->Init(kReleaseToOSIntervalNever); |
ef1b3fda KS |
102 | atomic_store(&internal_allocator_initialized, 1, memory_order_release); |
103 | } | |
104 | } | |
105 | return internal_allocator_instance; | |
106 | } | |
107 | ||
10189819 MO |
108 | static void *RawInternalAlloc(uptr size, InternalAllocatorCache *cache, |
109 | uptr alignment) { | |
110 | if (alignment == 0) alignment = 8; | |
111 | if (cache == 0) { | |
112 | SpinMutexLock l(&internal_allocator_cache_mu); | |
113 | return internal_allocator()->Allocate(&internal_allocator_cache, size, | |
5d3805fc | 114 | alignment); |
10189819 | 115 | } |
5d3805fc | 116 | return internal_allocator()->Allocate(cache, size, alignment); |
10189819 MO |
117 | } |
118 | ||
119 | static void *RawInternalRealloc(void *ptr, uptr size, | |
120 | InternalAllocatorCache *cache) { | |
121 | uptr alignment = 8; | |
ef1b3fda KS |
122 | if (cache == 0) { |
123 | SpinMutexLock l(&internal_allocator_cache_mu); | |
10189819 MO |
124 | return internal_allocator()->Reallocate(&internal_allocator_cache, ptr, |
125 | size, alignment); | |
ef1b3fda | 126 | } |
10189819 | 127 | return internal_allocator()->Reallocate(cache, ptr, size, alignment); |
ef1b3fda KS |
128 | } |
129 | ||
130 | static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { | |
696d846a | 131 | if (!cache) { |
ef1b3fda KS |
132 | SpinMutexLock l(&internal_allocator_cache_mu); |
133 | return internal_allocator()->Deallocate(&internal_allocator_cache, ptr); | |
134 | } | |
135 | internal_allocator()->Deallocate(cache, ptr); | |
136 | } | |
137 | ||
10189819 | 138 | #endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC) |
f35db108 WM |
139 | |
140 | const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull; | |
141 | ||
eac97531 ML |
142 | static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) { |
143 | SetAllocatorOutOfMemory(); | |
144 | Report("FATAL: %s: internal allocator is out of memory trying to allocate " | |
145 | "0x%zx bytes\n", SanitizerToolName, requested_size); | |
146 | Die(); | |
147 | } | |
148 | ||
10189819 | 149 | void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) { |
f35db108 | 150 | if (size + sizeof(u64) < size) |
696d846a | 151 | return nullptr; |
10189819 | 152 | void *p = RawInternalAlloc(size + sizeof(u64), cache, alignment); |
eac97531 ML |
153 | if (UNLIKELY(!p)) |
154 | ReportInternalAllocatorOutOfMemory(size + sizeof(u64)); | |
f35db108 WM |
155 | ((u64*)p)[0] = kBlockMagic; |
156 | return (char*)p + sizeof(u64); | |
157 | } | |
158 | ||
10189819 MO |
159 | void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { |
160 | if (!addr) | |
161 | return InternalAlloc(size, cache); | |
162 | if (size + sizeof(u64) < size) | |
163 | return nullptr; | |
164 | addr = (char*)addr - sizeof(u64); | |
165 | size = size + sizeof(u64); | |
166 | CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); | |
167 | void *p = RawInternalRealloc(addr, size, cache); | |
eac97531 ML |
168 | if (UNLIKELY(!p)) |
169 | ReportInternalAllocatorOutOfMemory(size); | |
10189819 MO |
170 | return (char*)p + sizeof(u64); |
171 | } | |
172 | ||
b667dd70 ML |
173 | void *InternalReallocArray(void *addr, uptr count, uptr size, |
174 | InternalAllocatorCache *cache) { | |
175 | if (UNLIKELY(CheckForCallocOverflow(count, size))) { | |
176 | Report( | |
177 | "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) " | |
178 | "cannot be represented in type size_t\n", | |
179 | SanitizerToolName, count, size); | |
180 | Die(); | |
181 | } | |
182 | return InternalRealloc(addr, count * size, cache); | |
183 | } | |
184 | ||
10189819 | 185 | void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { |
eac97531 ML |
186 | if (UNLIKELY(CheckForCallocOverflow(count, size))) { |
187 | Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) " | |
188 | "cannot be represented in type size_t\n", SanitizerToolName, count, | |
189 | size); | |
190 | Die(); | |
191 | } | |
10189819 | 192 | void *p = InternalAlloc(count * size, cache); |
eac97531 ML |
193 | if (LIKELY(p)) |
194 | internal_memset(p, 0, count * size); | |
10189819 MO |
195 | return p; |
196 | } | |
197 | ||
ef1b3fda | 198 | void InternalFree(void *addr, InternalAllocatorCache *cache) { |
696d846a | 199 | if (!addr) |
f35db108 WM |
200 | return; |
201 | addr = (char*)addr - sizeof(u64); | |
ef1b3fda | 202 | CHECK_EQ(kBlockMagic, ((u64*)addr)[0]); |
f35db108 | 203 | ((u64*)addr)[0] = 0; |
ef1b3fda | 204 | RawInternalFree(addr, cache); |
f35db108 WM |
205 | } |
206 | ||
f35db108 | 207 | // LowLevelAllocator |
eac97531 ML |
208 | constexpr uptr kLowLevelAllocatorDefaultAlignment = 8; |
209 | static uptr low_level_alloc_min_alignment = kLowLevelAllocatorDefaultAlignment; | |
f35db108 WM |
210 | static LowLevelAllocateCallback low_level_alloc_callback; |
211 | ||
212 | void *LowLevelAllocator::Allocate(uptr size) { | |
213 | // Align allocation size. | |
eac97531 | 214 | size = RoundUpTo(size, low_level_alloc_min_alignment); |
f35db108 | 215 | if (allocated_end_ - allocated_current_ < (sptr)size) { |
3c6331c2 | 216 | uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached()); |
f35db108 | 217 | allocated_current_ = |
dee5ea7a | 218 | (char*)MmapOrDie(size_to_allocate, __func__); |
f35db108 WM |
219 | allocated_end_ = allocated_current_ + size_to_allocate; |
220 | if (low_level_alloc_callback) { | |
221 | low_level_alloc_callback((uptr)allocated_current_, | |
222 | size_to_allocate); | |
223 | } | |
224 | } | |
225 | CHECK(allocated_end_ - allocated_current_ >= (sptr)size); | |
226 | void *res = allocated_current_; | |
227 | allocated_current_ += size; | |
228 | return res; | |
229 | } | |
230 | ||
eac97531 ML |
231 | void SetLowLevelAllocateMinAlignment(uptr alignment) { |
232 | CHECK(IsPowerOfTwo(alignment)); | |
233 | low_level_alloc_min_alignment = Max(alignment, low_level_alloc_min_alignment); | |
234 | } | |
235 | ||
f35db108 WM |
236 | void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback) { |
237 | low_level_alloc_callback = callback; | |
238 | } | |
239 | ||
eac97531 ML |
240 | // Allocator's OOM and other errors handling support. |
241 | ||
5d3805fc JJ |
242 | static atomic_uint8_t allocator_out_of_memory = {0}; |
243 | static atomic_uint8_t allocator_may_return_null = {0}; | |
10189819 | 244 | |
5d3805fc JJ |
245 | bool IsAllocatorOutOfMemory() { |
246 | return atomic_load_relaxed(&allocator_out_of_memory); | |
247 | } | |
10189819 | 248 | |
eac97531 ML |
249 | void SetAllocatorOutOfMemory() { |
250 | atomic_store_relaxed(&allocator_out_of_memory, 1); | |
ef1b3fda KS |
251 | } |
252 | ||
5d3805fc JJ |
253 | bool AllocatorMayReturnNull() { |
254 | return atomic_load(&allocator_may_return_null, memory_order_relaxed); | |
255 | } | |
256 | ||
257 | void SetAllocatorMayReturnNull(bool may_return_null) { | |
258 | atomic_store(&allocator_may_return_null, may_return_null, | |
259 | memory_order_relaxed); | |
260 | } | |
261 | ||
eac97531 ML |
262 | void PrintHintAllocatorCannotReturnNull() { |
263 | Report("HINT: if you don't care about these errors you may set " | |
264 | "allocator_may_return_null=1\n"); | |
5d3805fc JJ |
265 | } |
266 | ||
696d846a | 267 | } // namespace __sanitizer |