]>
Commit | Line | Data |
---|---|---|
f5e7e959 | 1 | /* Repeating a memory blob, with alias mapping optimization. |
2b778ceb | 2 | Copyright (C) 2018-2021 Free Software Foundation, Inc. |
f5e7e959 FW |
3 | This file is part of the GNU C Library. |
4 | ||
5 | The GNU C Library is free software; you can redistribute it and/or | |
6 | modify it under the terms of the GNU Lesser General Public | |
7 | License as published by the Free Software Foundation; either | |
8 | version 2.1 of the License, or (at your option) any later version. | |
9 | ||
10 | The GNU C Library is distributed in the hope that it will be useful, | |
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of | |
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
13 | Lesser General Public License for more details. | |
14 | ||
15 | You should have received a copy of the GNU Lesser General Public | |
16 | License along with the GNU C Library; if not, see | |
5a82c748 | 17 | <https://www.gnu.org/licenses/>. */ |
f5e7e959 FW |
18 | |
19 | #include <errno.h> | |
20 | #include <fcntl.h> | |
21 | #include <stdbool.h> | |
22 | #include <stdlib.h> | |
23 | #include <string.h> | |
24 | #include <support/blob_repeat.h> | |
25 | #include <support/check.h> | |
a91e9301 | 26 | #include <support/test-driver.h> |
f5e7e959 | 27 | #include <support/support.h> |
f5e7e959 FW |
28 | #include <support/xunistd.h> |
29 | #include <sys/mman.h> | |
30 | #include <unistd.h> | |
31 | #include <wchar.h> | |
32 | ||
33 | /* Small allocations should use malloc directly instead of the mmap | |
34 | optimization because mappings carry a lot of overhead. */ | |
35 | static const size_t maximum_small_size = 4 * 1024 * 1024; | |
36 | ||
37 | /* Internal helper for fill. */ | |
38 | static void | |
39 | fill0 (char *target, const char *element, size_t element_size, | |
40 | size_t count) | |
41 | { | |
42 | while (count > 0) | |
43 | { | |
44 | memcpy (target, element, element_size); | |
45 | target += element_size; | |
46 | --count; | |
47 | } | |
48 | } | |
49 | ||
50 | /* Fill the buffer at TARGET with COUNT copies of the ELEMENT_SIZE | |
51 | bytes starting at ELEMENT. */ | |
52 | static void | |
53 | fill (char *target, const char *element, size_t element_size, | |
54 | size_t count) | |
55 | { | |
56 | if (element_size == 0 || count == 0) | |
57 | return; | |
58 | else if (element_size == 1) | |
59 | memset (target, element[0], count); | |
60 | else if (element_size == sizeof (wchar_t)) | |
61 | { | |
62 | wchar_t wc; | |
63 | memcpy (&wc, element, sizeof (wc)); | |
64 | wmemset ((wchar_t *) target, wc, count); | |
65 | } | |
66 | else if (element_size < 1024 && count > 4096) | |
67 | { | |
68 | /* Use larger copies for really small element sizes. */ | |
69 | char buffer[8192]; | |
70 | size_t buffer_count = sizeof (buffer) / element_size; | |
71 | fill0 (buffer, element, element_size, buffer_count); | |
72 | while (count > 0) | |
73 | { | |
74 | size_t copy_count = buffer_count; | |
75 | if (copy_count > count) | |
76 | copy_count = count; | |
77 | size_t copy_bytes = copy_count * element_size; | |
78 | memcpy (target, buffer, copy_bytes); | |
79 | target += copy_bytes; | |
80 | count -= copy_count; | |
81 | } | |
82 | } | |
83 | else | |
84 | fill0 (target, element, element_size, count); | |
85 | } | |
86 | ||
87 | /* Use malloc instead of mmap for small allocations and unusual size | |
88 | combinations. */ | |
89 | static struct support_blob_repeat | |
90 | allocate_malloc (size_t total_size, const void *element, size_t element_size, | |
91 | size_t count) | |
92 | { | |
93 | void *buffer = malloc (total_size); | |
94 | if (buffer == NULL) | |
95 | return (struct support_blob_repeat) { 0 }; | |
96 | fill (buffer, element, element_size, count); | |
97 | return (struct support_blob_repeat) | |
98 | { | |
99 | .start = buffer, | |
100 | .size = total_size, | |
101 | .use_malloc = true | |
102 | }; | |
103 | } | |
104 | ||
105 | /* Return the least common multiple of PAGE_SIZE and ELEMENT_SIZE, | |
106 | avoiding overflow. This assumes that PAGE_SIZE is a power of | |
107 | two. */ | |
108 | static size_t | |
109 | minimum_stride_size (size_t page_size, size_t element_size) | |
110 | { | |
111 | TEST_VERIFY_EXIT (page_size > 0); | |
112 | TEST_VERIFY_EXIT (element_size > 0); | |
113 | ||
114 | /* Compute the number of trailing zeros common to both sizes. */ | |
115 | unsigned int common_zeros = __builtin_ctzll (page_size | element_size); | |
116 | ||
117 | /* In the product, this power of two appears twice, but in the least | |
118 | common multiple, it appears only once. Therefore, shift one | |
119 | factor. */ | |
120 | size_t multiple; | |
0253580a AZ |
121 | if (__builtin_mul_overflow (page_size >> common_zeros, element_size, |
122 | &multiple)) | |
f5e7e959 FW |
123 | return 0; |
124 | return multiple; | |
125 | } | |
126 | ||
127 | /* Allocations larger than maximum_small_size potentially use mmap | |
0e28cfff FW |
128 | with alias mappings. If SHARED, the alias mappings are created |
129 | using MAP_SHARED instead of MAP_PRIVATE. */ | |
f5e7e959 FW |
130 | static struct support_blob_repeat |
131 | allocate_big (size_t total_size, const void *element, size_t element_size, | |
0e28cfff | 132 | size_t count, bool shared) |
f5e7e959 FW |
133 | { |
134 | unsigned long page_size = xsysconf (_SC_PAGESIZE); | |
135 | size_t stride_size = minimum_stride_size (page_size, element_size); | |
136 | if (stride_size == 0) | |
137 | { | |
138 | errno = EOVERFLOW; | |
139 | return (struct support_blob_repeat) { 0 }; | |
140 | } | |
141 | ||
142 | /* Ensure that the stride size is at least maximum_small_size. This | |
143 | is necessary to reduce the number of distinct mappings. */ | |
144 | if (stride_size < maximum_small_size) | |
145 | stride_size | |
146 | = ((maximum_small_size + stride_size - 1) / stride_size) * stride_size; | |
147 | ||
148 | if (stride_size > total_size) | |
149 | /* The mmap optimization would not save anything. */ | |
150 | return allocate_malloc (total_size, element, element_size, count); | |
151 | ||
152 | /* Reserve the memory region. If we cannot create the mapping, | |
153 | there is no reason to set up the backing file. */ | |
154 | void *target = mmap (NULL, total_size, PROT_NONE, | |
155 | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); | |
156 | if (target == MAP_FAILED) | |
157 | return (struct support_blob_repeat) { 0 }; | |
158 | ||
a91e9301 FW |
159 | /* Create the backing file for the repeated mapping. Call mkstemp |
160 | directly to remove the resources backing the temporary file | |
161 | immediately, once support_blob_repeat_free is called. Using | |
162 | create_temp_file would result in a warning during post-test | |
163 | cleanup. */ | |
f5e7e959 FW |
164 | int fd; |
165 | { | |
a91e9301 FW |
166 | char *temppath = xasprintf ("%s/support_blob_repeat-XXXXXX", test_dir); |
167 | fd = mkstemp (temppath); | |
f5e7e959 | 168 | if (fd < 0) |
a91e9301 | 169 | FAIL_EXIT1 ("mkstemp (\"%s\"): %m", temppath); |
f5e7e959 FW |
170 | xunlink (temppath); |
171 | free (temppath); | |
172 | } | |
173 | ||
174 | /* Make sure that there is backing storage, so that the fill | |
175 | operation will not fault. */ | |
176 | if (posix_fallocate (fd, 0, stride_size) != 0) | |
177 | FAIL_EXIT1 ("posix_fallocate (%zu): %m", stride_size); | |
178 | ||
179 | /* The stride size must still be a multiple of the page size and | |
180 | element size. */ | |
181 | TEST_VERIFY_EXIT ((stride_size % page_size) == 0); | |
182 | TEST_VERIFY_EXIT ((stride_size % element_size) == 0); | |
183 | ||
184 | /* Fill the backing store. */ | |
185 | { | |
186 | void *ptr = mmap (target, stride_size, PROT_READ | PROT_WRITE, | |
187 | MAP_FIXED | MAP_FILE | MAP_SHARED, fd, 0); | |
188 | if (ptr == MAP_FAILED) | |
189 | { | |
190 | int saved_errno = errno; | |
191 | xmunmap (target, total_size); | |
192 | xclose (fd); | |
193 | errno = saved_errno; | |
194 | return (struct support_blob_repeat) { 0 }; | |
195 | } | |
196 | if (ptr != target) | |
197 | FAIL_EXIT1 ("mapping of %zu bytes moved from %p to %p", | |
198 | stride_size, target, ptr); | |
199 | ||
200 | /* Write the repeating data. */ | |
201 | fill (target, element, element_size, stride_size / element_size); | |
202 | ||
203 | /* Return to a PROT_NONE mapping, just to be on the safe side. */ | |
204 | ptr = mmap (target, stride_size, PROT_NONE, | |
205 | MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); | |
206 | if (ptr == MAP_FAILED) | |
207 | FAIL_EXIT1 ("Failed to reinstate PROT_NONE mapping: %m"); | |
208 | if (ptr != target) | |
209 | FAIL_EXIT1 ("PROT_NONE mapping of %zu bytes moved from %p to %p", | |
210 | stride_size, target, ptr); | |
211 | } | |
212 | ||
213 | /* Create the alias mappings. */ | |
214 | { | |
215 | size_t remaining_size = total_size; | |
216 | char *current = target; | |
0e28cfff FW |
217 | int flags = MAP_FIXED | MAP_FILE; |
218 | if (shared) | |
219 | flags |= MAP_SHARED; | |
220 | else | |
221 | flags |= MAP_PRIVATE; | |
f5e7e959 FW |
222 | #ifdef MAP_NORESERVE |
223 | flags |= MAP_NORESERVE; | |
224 | #endif | |
225 | while (remaining_size > 0) | |
226 | { | |
227 | size_t to_map = stride_size; | |
228 | if (to_map > remaining_size) | |
229 | to_map = remaining_size; | |
230 | void *ptr = mmap (current, to_map, PROT_READ | PROT_WRITE, | |
231 | flags, fd, 0); | |
232 | if (ptr == MAP_FAILED) | |
233 | { | |
234 | int saved_errno = errno; | |
235 | xmunmap (target, total_size); | |
236 | xclose (fd); | |
237 | errno = saved_errno; | |
238 | return (struct support_blob_repeat) { 0 }; | |
239 | } | |
240 | if (ptr != current) | |
241 | FAIL_EXIT1 ("MAP_PRIVATE mapping of %zu bytes moved from %p to %p", | |
242 | to_map, target, ptr); | |
243 | remaining_size -= to_map; | |
244 | current += to_map; | |
245 | } | |
246 | } | |
247 | ||
248 | xclose (fd); | |
249 | ||
250 | return (struct support_blob_repeat) | |
251 | { | |
252 | .start = target, | |
253 | .size = total_size, | |
254 | .use_malloc = false | |
255 | }; | |
256 | } | |
257 | ||
258 | struct support_blob_repeat | |
0e28cfff FW |
259 | repeat_allocate (const void *element, size_t element_size, |
260 | size_t count, bool shared) | |
f5e7e959 FW |
261 | { |
262 | size_t total_size; | |
0253580a | 263 | if (__builtin_mul_overflow (element_size, count, &total_size)) |
f5e7e959 FW |
264 | { |
265 | errno = EOVERFLOW; | |
266 | return (struct support_blob_repeat) { 0 }; | |
267 | } | |
268 | if (total_size <= maximum_small_size) | |
269 | return allocate_malloc (total_size, element, element_size, count); | |
270 | else | |
0e28cfff FW |
271 | return allocate_big (total_size, element, element_size, count, shared); |
272 | } | |
273 | ||
274 | struct support_blob_repeat | |
275 | support_blob_repeat_allocate (const void *element, size_t element_size, | |
276 | size_t count) | |
277 | { | |
278 | return repeat_allocate (element, element_size, count, false); | |
279 | } | |
280 | ||
281 | struct support_blob_repeat | |
282 | support_blob_repeat_allocate_shared (const void *element, size_t element_size, | |
283 | size_t count) | |
284 | { | |
285 | return repeat_allocate (element, element_size, count, true); | |
f5e7e959 FW |
286 | } |
287 | ||
288 | void | |
289 | support_blob_repeat_free (struct support_blob_repeat *blob) | |
290 | { | |
291 | if (blob->size > 0) | |
292 | { | |
293 | int saved_errno = errno; | |
294 | if (blob->use_malloc) | |
295 | free (blob->start); | |
296 | else | |
297 | xmunmap (blob->start, blob->size); | |
298 | errno = saved_errno; | |
299 | } | |
300 | *blob = (struct support_blob_repeat) { 0 }; | |
301 | } |