]>
Commit | Line | Data |
---|---|---|
16e9f408 LP |
1 | /*** |
2 | This file is part of systemd. | |
3 | ||
4 | Copyright 2012 Lennart Poettering | |
5 | ||
6 | systemd is free software; you can redistribute it and/or modify it | |
7 | under the terms of the GNU Lesser General Public License as published by | |
8 | the Free Software Foundation; either version 2.1 of the License, or | |
9 | (at your option) any later version. | |
10 | ||
11 | systemd is distributed in the hope that it will be useful, but | |
12 | WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
14 | Lesser General Public License for more details. | |
15 | ||
16 | You should have received a copy of the GNU Lesser General Public License | |
17 | along with systemd; If not, see <http://www.gnu.org/licenses/>. | |
18 | ***/ | |
19 | ||
16e9f408 LP |
20 | #include <errno.h> |
21 | #include <stdlib.h> | |
f8019684 | 22 | #include <sys/mman.h> |
16e9f408 | 23 | |
b5efdb8a | 24 | #include "alloc-util.h" |
23e096cc | 25 | #include "fd-util.h" |
f8019684 LP |
26 | #include "hashmap.h" |
27 | #include "list.h" | |
28 | #include "log.h" | |
f8019684 | 29 | #include "macro.h" |
16e9f408 | 30 | #include "mmap-cache.h" |
cf0fbc49 TA |
31 | #include "sigbus.h" |
32 | #include "util.h" | |
16e9f408 | 33 | |
f8019684 LP |
34 | typedef struct Window Window; |
35 | typedef struct Context Context; | |
36 | typedef struct FileDescriptor FileDescriptor; | |
84168d80 | 37 | |
f8019684 LP |
38 | struct Window { |
39 | MMapCache *cache; | |
40 | ||
739731cd LP |
41 | bool invalidated:1; |
42 | bool keep_always:1; | |
43 | bool in_unused:1; | |
16e9f408 | 44 | |
68667801 | 45 | int prot; |
16e9f408 LP |
46 | void *ptr; |
47 | uint64_t offset; | |
f8019684 LP |
48 | size_t size; |
49 | ||
50 | FileDescriptor *fd; | |
16e9f408 | 51 | |
f8019684 LP |
52 | LIST_FIELDS(Window, by_fd); |
53 | LIST_FIELDS(Window, unused); | |
54 | ||
55 | LIST_HEAD(Context, contexts); | |
56 | }; | |
16e9f408 | 57 | |
f8019684 LP |
58 | struct Context { |
59 | MMapCache *cache; | |
60 | unsigned id; | |
61 | Window *window; | |
16e9f408 | 62 | |
f8019684 LP |
63 | LIST_FIELDS(Context, by_window); |
64 | }; | |
65 | ||
66 | struct FileDescriptor { | |
67 | MMapCache *cache; | |
16e9f408 | 68 | int fd; |
fa6ac760 | 69 | bool sigbus; |
f8019684 LP |
70 | LIST_HEAD(Window, windows); |
71 | }; | |
16e9f408 LP |
72 | |
73 | struct MMapCache { | |
f8019684 | 74 | int n_ref; |
68667801 | 75 | unsigned n_windows; |
16e9f408 | 76 | |
bf807d4d LP |
77 | unsigned n_hit, n_missed; |
78 | ||
f8019684 | 79 | Hashmap *fds; |
69adae51 | 80 | Context *contexts[MMAP_CACHE_MAX_CONTEXTS]; |
16e9f408 | 81 | |
f8019684 LP |
82 | LIST_HEAD(Window, unused); |
83 | Window *last_unused; | |
16e9f408 LP |
84 | }; |
85 | ||
f8019684 | 86 | #define WINDOWS_MIN 64 |
fad5a6c6 MS |
87 | |
88 | #ifdef ENABLE_DEBUG_MMAP_CACHE | |
89 | /* Tiny windows increase mmap activity and the chance of exposing unsafe use. */ | |
90 | # define WINDOW_SIZE (page_size()) | |
91 | #else | |
92 | # define WINDOW_SIZE (8ULL*1024ULL*1024ULL) | |
93 | #endif | |
16e9f408 | 94 | |
f8019684 LP |
95 | MMapCache* mmap_cache_new(void) { |
96 | MMapCache *m; | |
16e9f408 | 97 | |
f8019684 LP |
98 | m = new0(MMapCache, 1); |
99 | if (!m) | |
100 | return NULL; | |
16e9f408 | 101 | |
f8019684 LP |
102 | m->n_ref = 1; |
103 | return m; | |
16e9f408 LP |
104 | } |
105 | ||
f8019684 | 106 | MMapCache* mmap_cache_ref(MMapCache *m) { |
16e9f408 | 107 | assert(m); |
f8019684 | 108 | assert(m->n_ref > 0); |
16e9f408 | 109 | |
313cefa1 | 110 | m->n_ref++; |
f8019684 LP |
111 | return m; |
112 | } | |
f65425cb | 113 | |
f8019684 LP |
114 | static void window_unlink(Window *w) { |
115 | Context *c; | |
f65425cb | 116 | |
f8019684 | 117 | assert(w); |
16e9f408 | 118 | |
f8019684 LP |
119 | if (w->ptr) |
120 | munmap(w->ptr, w->size); | |
16e9f408 | 121 | |
f8019684 | 122 | if (w->fd) |
71fda00f | 123 | LIST_REMOVE(by_fd, w->fd->windows, w); |
16e9f408 | 124 | |
f8019684 LP |
125 | if (w->in_unused) { |
126 | if (w->cache->last_unused == w) | |
127 | w->cache->last_unused = w->unused_prev; | |
16e9f408 | 128 | |
71fda00f | 129 | LIST_REMOVE(unused, w->cache->unused, w); |
f65425cb | 130 | } |
16e9f408 | 131 | |
f8019684 LP |
132 | LIST_FOREACH(by_window, c, w->contexts) { |
133 | assert(c->window == w); | |
134 | c->window = NULL; | |
f65425cb | 135 | } |
16e9f408 LP |
136 | } |
137 | ||
fa6ac760 LP |
138 | static void window_invalidate(Window *w) { |
139 | assert(w); | |
140 | ||
141 | if (w->invalidated) | |
142 | return; | |
143 | ||
144 | /* Replace the window with anonymous pages. This is useful | |
145 | * when we hit a SIGBUS and want to make sure the file cannot | |
146 | * trigger any further SIGBUS, possibly overrunning the sigbus | |
147 | * queue. */ | |
148 | ||
149 | assert_se(mmap(w->ptr, w->size, w->prot, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0) == w->ptr); | |
150 | w->invalidated = true; | |
151 | } | |
152 | ||
f8019684 LP |
153 | static void window_free(Window *w) { |
154 | assert(w); | |
f65425cb | 155 | |
f8019684 | 156 | window_unlink(w); |
89de6947 | 157 | w->cache->n_windows--; |
f8019684 LP |
158 | free(w); |
159 | } | |
f65425cb | 160 | |
44a6b1b6 | 161 | _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) { |
f8019684 LP |
162 | assert(w); |
163 | assert(fd >= 0); | |
164 | assert(size > 0); | |
16e9f408 | 165 | |
f8019684 LP |
166 | return |
167 | w->fd && | |
168 | fd == w->fd->fd && | |
169 | prot == w->prot && | |
170 | offset >= w->offset && | |
171 | offset + size <= w->offset + w->size; | |
16e9f408 LP |
172 | } |
173 | ||
6a491490 | 174 | static Window *window_add(MMapCache *m, FileDescriptor *fd, int prot, bool keep_always, uint64_t offset, size_t size, void *ptr) { |
f8019684 | 175 | Window *w; |
16e9f408 LP |
176 | |
177 | assert(m); | |
6a491490 | 178 | assert(fd); |
16e9f408 | 179 | |
f8019684 | 180 | if (!m->last_unused || m->n_windows <= WINDOWS_MIN) { |
f65425cb | 181 | |
f8019684 LP |
182 | /* Allocate a new window */ |
183 | w = new0(Window, 1); | |
184 | if (!w) | |
185 | return NULL; | |
89de6947 | 186 | m->n_windows++; |
f65425cb | 187 | } else { |
16e9f408 | 188 | |
f8019684 LP |
189 | /* Reuse an existing one */ |
190 | w = m->last_unused; | |
191 | window_unlink(w); | |
192 | zero(*w); | |
f65425cb | 193 | } |
f8019684 LP |
194 | |
195 | w->cache = m; | |
6a491490 VC |
196 | w->fd = fd; |
197 | w->prot = prot; | |
198 | w->keep_always = keep_always; | |
199 | w->offset = offset; | |
200 | w->size = size; | |
201 | w->ptr = ptr; | |
202 | ||
203 | LIST_PREPEND(by_fd, fd->windows, w); | |
204 | ||
f8019684 | 205 | return w; |
16e9f408 LP |
206 | } |
207 | ||
f8019684 LP |
208 | static void context_detach_window(Context *c) { |
209 | Window *w; | |
16e9f408 | 210 | |
f8019684 | 211 | assert(c); |
16e9f408 | 212 | |
f8019684 | 213 | if (!c->window) |
16e9f408 LP |
214 | return; |
215 | ||
f8019684 LP |
216 | w = c->window; |
217 | c->window = NULL; | |
71fda00f | 218 | LIST_REMOVE(by_window, w->contexts, c); |
16e9f408 | 219 | |
1b8951e5 | 220 | if (!w->contexts && !w->keep_always) { |
f8019684 | 221 | /* Not used anymore? */ |
fad5a6c6 MS |
222 | #ifdef ENABLE_DEBUG_MMAP_CACHE |
223 | /* Unmap unused windows immediately to expose use-after-unmap | |
224 | * by SIGSEGV. */ | |
225 | window_free(w); | |
226 | #else | |
71fda00f | 227 | LIST_PREPEND(unused, c->cache->unused, w); |
f8019684 LP |
228 | if (!c->cache->last_unused) |
229 | c->cache->last_unused = w; | |
16e9f408 | 230 | |
f8019684 | 231 | w->in_unused = true; |
fad5a6c6 | 232 | #endif |
f8019684 | 233 | } |
16e9f408 LP |
234 | } |
235 | ||
f8019684 LP |
236 | static void context_attach_window(Context *c, Window *w) { |
237 | assert(c); | |
238 | assert(w); | |
16e9f408 | 239 | |
f8019684 | 240 | if (c->window == w) |
16e9f408 LP |
241 | return; |
242 | ||
f8019684 | 243 | context_detach_window(c); |
16e9f408 | 244 | |
e18021f7 | 245 | if (w->in_unused) { |
f8019684 | 246 | /* Used again? */ |
71fda00f | 247 | LIST_REMOVE(unused, c->cache->unused, w); |
a2ab7ee6 CG |
248 | if (c->cache->last_unused == w) |
249 | c->cache->last_unused = w->unused_prev; | |
16e9f408 | 250 | |
f8019684 LP |
251 | w->in_unused = false; |
252 | } | |
f65425cb | 253 | |
f8019684 | 254 | c->window = w; |
71fda00f | 255 | LIST_PREPEND(by_window, w->contexts, c); |
16e9f408 LP |
256 | } |
257 | ||
f8019684 LP |
258 | static Context *context_add(MMapCache *m, unsigned id) { |
259 | Context *c; | |
16e9f408 LP |
260 | |
261 | assert(m); | |
262 | ||
69adae51 | 263 | c = m->contexts[id]; |
f8019684 LP |
264 | if (c) |
265 | return c; | |
266 | ||
f8019684 LP |
267 | c = new0(Context, 1); |
268 | if (!c) | |
269 | return NULL; | |
16e9f408 | 270 | |
f8019684 LP |
271 | c->cache = m; |
272 | c->id = id; | |
16e9f408 | 273 | |
69adae51 MS |
274 | assert(!m->contexts[id]); |
275 | m->contexts[id] = c; | |
16e9f408 | 276 | |
f8019684 | 277 | return c; |
16e9f408 LP |
278 | } |
279 | ||
f8019684 LP |
280 | static void context_free(Context *c) { |
281 | assert(c); | |
16e9f408 | 282 | |
f8019684 | 283 | context_detach_window(c); |
16e9f408 | 284 | |
69adae51 MS |
285 | if (c->cache) { |
286 | assert(c->cache->contexts[c->id] == c); | |
287 | c->cache->contexts[c->id] = NULL; | |
288 | } | |
16e9f408 | 289 | |
f8019684 LP |
290 | free(c); |
291 | } | |
292 | ||
293 | static void fd_free(FileDescriptor *f) { | |
294 | assert(f); | |
295 | ||
296 | while (f->windows) | |
297 | window_free(f->windows); | |
298 | ||
299 | if (f->cache) | |
23e096cc | 300 | assert_se(hashmap_remove(f->cache->fds, FD_TO_PTR(f->fd))); |
f8019684 LP |
301 | |
302 | free(f); | |
303 | } | |
304 | ||
305 | static FileDescriptor* fd_add(MMapCache *m, int fd) { | |
306 | FileDescriptor *f; | |
307 | int r; | |
308 | ||
309 | assert(m); | |
310 | assert(fd >= 0); | |
311 | ||
23e096cc | 312 | f = hashmap_get(m->fds, FD_TO_PTR(fd)); |
f8019684 LP |
313 | if (f) |
314 | return f; | |
315 | ||
d5099efc | 316 | r = hashmap_ensure_allocated(&m->fds, NULL); |
f8019684 | 317 | if (r < 0) |
16e9f408 | 318 | return NULL; |
16e9f408 | 319 | |
f8019684 LP |
320 | f = new0(FileDescriptor, 1); |
321 | if (!f) | |
16e9f408 | 322 | return NULL; |
16e9f408 | 323 | |
f8019684 LP |
324 | f->cache = m; |
325 | f->fd = fd; | |
326 | ||
23e096cc | 327 | r = hashmap_put(m->fds, FD_TO_PTR(fd), f); |
6b430fdb ZJS |
328 | if (r < 0) |
329 | return mfree(f); | |
16e9f408 | 330 | |
f8019684 | 331 | return f; |
16e9f408 LP |
332 | } |
333 | ||
f8019684 | 334 | static void mmap_cache_free(MMapCache *m) { |
f8019684 | 335 | FileDescriptor *f; |
69adae51 | 336 | int i; |
f8019684 | 337 | |
16e9f408 | 338 | assert(m); |
16e9f408 | 339 | |
69adae51 MS |
340 | for (i = 0; i < MMAP_CACHE_MAX_CONTEXTS; i++) |
341 | if (m->contexts[i]) | |
342 | context_free(m->contexts[i]); | |
8e6d9397 | 343 | |
f8019684 LP |
344 | while ((f = hashmap_first(m->fds))) |
345 | fd_free(f); | |
346 | ||
8e6d9397 GM |
347 | hashmap_free(m->fds); |
348 | ||
f8019684 LP |
349 | while (m->unused) |
350 | window_free(m->unused); | |
351 | ||
352 | free(m); | |
16e9f408 LP |
353 | } |
354 | ||
355 | MMapCache* mmap_cache_unref(MMapCache *m) { | |
f649045c LP |
356 | |
357 | if (!m) | |
358 | return NULL; | |
359 | ||
16e9f408 LP |
360 | assert(m->n_ref > 0); |
361 | ||
313cefa1 | 362 | m->n_ref--; |
f8019684 | 363 | if (m->n_ref == 0) |
16e9f408 | 364 | mmap_cache_free(m); |
16e9f408 LP |
365 | |
366 | return NULL; | |
367 | } | |
368 | ||
f8019684 LP |
369 | static int make_room(MMapCache *m) { |
370 | assert(m); | |
371 | ||
372 | if (!m->last_unused) | |
373 | return 0; | |
374 | ||
375 | window_free(m->last_unused); | |
376 | return 1; | |
377 | } | |
378 | ||
379 | static int try_context( | |
380 | MMapCache *m, | |
381 | int fd, | |
382 | int prot, | |
383 | unsigned context, | |
384 | bool keep_always, | |
385 | uint64_t offset, | |
386 | size_t size, | |
1b8951e5 | 387 | void **ret) { |
f8019684 LP |
388 | |
389 | Context *c; | |
f65425cb | 390 | |
16e9f408 | 391 | assert(m); |
f8019684 LP |
392 | assert(m->n_ref > 0); |
393 | assert(fd >= 0); | |
394 | assert(size > 0); | |
1b8951e5 | 395 | assert(ret); |
16e9f408 | 396 | |
69adae51 | 397 | c = m->contexts[context]; |
f8019684 | 398 | if (!c) |
16e9f408 | 399 | return 0; |
16e9f408 | 400 | |
f8019684 | 401 | assert(c->id == context); |
16e9f408 | 402 | |
f8019684 LP |
403 | if (!c->window) |
404 | return 0; | |
f65425cb | 405 | |
f8019684 | 406 | if (!window_matches(c->window, fd, prot, offset, size)) { |
f65425cb | 407 | |
f8019684 LP |
408 | /* Drop the reference to the window, since it's unnecessary now */ |
409 | context_detach_window(c); | |
410 | return 0; | |
f65425cb LP |
411 | } |
412 | ||
fa6ac760 LP |
413 | if (c->window->fd->sigbus) |
414 | return -EIO; | |
415 | ||
739731cd | 416 | c->window->keep_always = c->window->keep_always || keep_always; |
16e9f408 | 417 | |
1b8951e5 | 418 | *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset); |
f8019684 | 419 | return 1; |
16e9f408 LP |
420 | } |
421 | ||
f8019684 LP |
422 | static int find_mmap( |
423 | MMapCache *m, | |
424 | int fd, | |
425 | int prot, | |
426 | unsigned context, | |
427 | bool keep_always, | |
428 | uint64_t offset, | |
429 | size_t size, | |
1b8951e5 | 430 | void **ret) { |
f8019684 LP |
431 | |
432 | FileDescriptor *f; | |
433 | Window *w; | |
434 | Context *c; | |
16e9f408 LP |
435 | |
436 | assert(m); | |
f8019684 LP |
437 | assert(m->n_ref > 0); |
438 | assert(fd >= 0); | |
439 | assert(size > 0); | |
16e9f408 | 440 | |
23e096cc | 441 | f = hashmap_get(m->fds, FD_TO_PTR(fd)); |
f8019684 LP |
442 | if (!f) |
443 | return 0; | |
16e9f408 | 444 | |
f8019684 | 445 | assert(f->fd == fd); |
16e9f408 | 446 | |
fa6ac760 LP |
447 | if (f->sigbus) |
448 | return -EIO; | |
449 | ||
f8019684 LP |
450 | LIST_FOREACH(by_fd, w, f->windows) |
451 | if (window_matches(w, fd, prot, offset, size)) | |
452 | break; | |
16e9f408 | 453 | |
f8019684 LP |
454 | if (!w) |
455 | return 0; | |
456 | ||
457 | c = context_add(m, context); | |
458 | if (!c) | |
459 | return -ENOMEM; | |
460 | ||
461 | context_attach_window(c, w); | |
739731cd | 462 | w->keep_always = w->keep_always || keep_always; |
16e9f408 | 463 | |
1b8951e5 | 464 | *ret = (uint8_t*) w->ptr + (offset - w->offset); |
f8019684 | 465 | return 1; |
16e9f408 LP |
466 | } |
467 | ||
db87967e VC |
468 | static int mmap_try_harder(MMapCache *m, void *addr, int fd, int prot, int flags, uint64_t offset, size_t size, void **res) { |
469 | void *ptr; | |
470 | ||
471 | assert(m); | |
472 | assert(fd >= 0); | |
473 | assert(res); | |
474 | ||
475 | for (;;) { | |
476 | int r; | |
477 | ||
478 | ptr = mmap(addr, size, prot, flags, fd, offset); | |
479 | if (ptr != MAP_FAILED) | |
480 | break; | |
481 | if (errno != ENOMEM) | |
3f0083a2 | 482 | return negative_errno(); |
db87967e VC |
483 | |
484 | r = make_room(m); | |
485 | if (r < 0) | |
486 | return r; | |
487 | if (r == 0) | |
488 | return -ENOMEM; | |
489 | } | |
490 | ||
491 | *res = ptr; | |
492 | return 0; | |
493 | } | |
494 | ||
f8019684 | 495 | static int add_mmap( |
16e9f408 LP |
496 | MMapCache *m, |
497 | int fd, | |
16e9f408 LP |
498 | int prot, |
499 | unsigned context, | |
fcde2389 | 500 | bool keep_always, |
16e9f408 | 501 | uint64_t offset, |
f8019684 | 502 | size_t size, |
fcde2389 | 503 | struct stat *st, |
1b8951e5 | 504 | void **ret) { |
16e9f408 | 505 | |
16e9f408 | 506 | uint64_t woffset, wsize; |
f8019684 LP |
507 | Context *c; |
508 | FileDescriptor *f; | |
509 | Window *w; | |
510 | void *d; | |
16e9f408 LP |
511 | int r; |
512 | ||
513 | assert(m); | |
f8019684 | 514 | assert(m->n_ref > 0); |
16e9f408 | 515 | assert(fd >= 0); |
16e9f408 | 516 | assert(size > 0); |
1b8951e5 | 517 | assert(ret); |
16e9f408 LP |
518 | |
519 | woffset = offset & ~((uint64_t) page_size() - 1ULL); | |
520 | wsize = size + (offset - woffset); | |
521 | wsize = PAGE_ALIGN(wsize); | |
522 | ||
523 | if (wsize < WINDOW_SIZE) { | |
524 | uint64_t delta; | |
525 | ||
beec0085 | 526 | delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2); |
16e9f408 LP |
527 | |
528 | if (delta > offset) | |
529 | woffset = 0; | |
530 | else | |
531 | woffset -= delta; | |
532 | ||
533 | wsize = WINDOW_SIZE; | |
534 | } | |
535 | ||
fcde2389 LP |
536 | if (st) { |
537 | /* Memory maps that are larger then the files | |
c5315881 | 538 | underneath have undefined behavior. Hence, clamp |
fcde2389 LP |
539 | things to the file size if we know it */ |
540 | ||
541 | if (woffset >= (uint64_t) st->st_size) | |
542 | return -EADDRNOTAVAIL; | |
543 | ||
544 | if (woffset + wsize > (uint64_t) st->st_size) | |
545 | wsize = PAGE_ALIGN(st->st_size - woffset); | |
546 | } | |
547 | ||
db87967e VC |
548 | r = mmap_try_harder(m, NULL, fd, prot, MAP_SHARED, woffset, wsize, &d); |
549 | if (r < 0) | |
550 | return r; | |
16e9f408 | 551 | |
f8019684 LP |
552 | c = context_add(m, context); |
553 | if (!c) | |
b67ddc7b | 554 | goto outofmem; |
16e9f408 | 555 | |
f8019684 LP |
556 | f = fd_add(m, fd); |
557 | if (!f) | |
b67ddc7b | 558 | goto outofmem; |
16e9f408 | 559 | |
6a491490 | 560 | w = window_add(m, f, prot, keep_always, woffset, wsize, d); |
f8019684 | 561 | if (!w) |
b67ddc7b | 562 | goto outofmem; |
16e9f408 | 563 | |
f8019684 LP |
564 | context_detach_window(c); |
565 | c->window = w; | |
71fda00f | 566 | LIST_PREPEND(by_window, w->contexts, c); |
16e9f408 | 567 | |
1b8951e5 | 568 | *ret = (uint8_t*) w->ptr + (offset - w->offset); |
16e9f408 | 569 | return 1; |
b67ddc7b PDS |
570 | |
571 | outofmem: | |
3f0083a2 | 572 | (void) munmap(d, wsize); |
b67ddc7b | 573 | return -ENOMEM; |
16e9f408 LP |
574 | } |
575 | ||
576 | int mmap_cache_get( | |
577 | MMapCache *m, | |
578 | int fd, | |
579 | int prot, | |
580 | unsigned context, | |
fcde2389 | 581 | bool keep_always, |
16e9f408 | 582 | uint64_t offset, |
f8019684 | 583 | size_t size, |
fcde2389 | 584 | struct stat *st, |
1b8951e5 | 585 | void **ret) { |
16e9f408 | 586 | |
16e9f408 LP |
587 | int r; |
588 | ||
589 | assert(m); | |
f8019684 | 590 | assert(m->n_ref > 0); |
16e9f408 | 591 | assert(fd >= 0); |
16e9f408 | 592 | assert(size > 0); |
1b8951e5 | 593 | assert(ret); |
69adae51 | 594 | assert(context < MMAP_CACHE_MAX_CONTEXTS); |
16e9f408 | 595 | |
f8019684 | 596 | /* Check whether the current context is the right one already */ |
1b8951e5 | 597 | r = try_context(m, fd, prot, context, keep_always, offset, size, ret); |
bf807d4d | 598 | if (r != 0) { |
313cefa1 | 599 | m->n_hit++; |
16e9f408 | 600 | return r; |
bf807d4d | 601 | } |
16e9f408 | 602 | |
f8019684 | 603 | /* Search for a matching mmap */ |
1b8951e5 | 604 | r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret); |
bf807d4d | 605 | if (r != 0) { |
313cefa1 | 606 | m->n_hit++; |
16e9f408 | 607 | return r; |
bf807d4d LP |
608 | } |
609 | ||
610 | m->n_missed++; | |
16e9f408 | 611 | |
f8019684 | 612 | /* Create a new mmap */ |
1b8951e5 | 613 | return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret); |
ae97089d ZJS |
614 | } |
615 | ||
fa6ac760 LP |
616 | unsigned mmap_cache_get_hit(MMapCache *m) { |
617 | assert(m); | |
618 | ||
619 | return m->n_hit; | |
620 | } | |
621 | ||
622 | unsigned mmap_cache_get_missed(MMapCache *m) { | |
623 | assert(m); | |
624 | ||
625 | return m->n_missed; | |
626 | } | |
627 | ||
628 | static void mmap_cache_process_sigbus(MMapCache *m) { | |
629 | bool found = false; | |
f8019684 | 630 | FileDescriptor *f; |
fa6ac760 LP |
631 | Iterator i; |
632 | int r; | |
16e9f408 LP |
633 | |
634 | assert(m); | |
16e9f408 | 635 | |
fa6ac760 LP |
636 | /* Iterate through all triggered pages and mark their files as |
637 | * invalidated */ | |
638 | for (;;) { | |
639 | bool ours; | |
640 | void *addr; | |
641 | ||
642 | r = sigbus_pop(&addr); | |
643 | if (_likely_(r == 0)) | |
644 | break; | |
645 | if (r < 0) { | |
646 | log_error_errno(r, "SIGBUS handling failed: %m"); | |
647 | abort(); | |
648 | } | |
649 | ||
650 | ours = false; | |
651 | HASHMAP_FOREACH(f, m->fds, i) { | |
652 | Window *w; | |
653 | ||
654 | LIST_FOREACH(by_fd, w, f->windows) { | |
655 | if ((uint8_t*) addr >= (uint8_t*) w->ptr && | |
656 | (uint8_t*) addr < (uint8_t*) w->ptr + w->size) { | |
657 | found = ours = f->sigbus = true; | |
658 | break; | |
659 | } | |
660 | } | |
661 | ||
662 | if (ours) | |
663 | break; | |
664 | } | |
665 | ||
666 | /* Didn't find a matching window, give up */ | |
667 | if (!ours) { | |
668 | log_error("Unknown SIGBUS page, aborting."); | |
669 | abort(); | |
670 | } | |
671 | } | |
672 | ||
673 | /* The list of triggered pages is now empty. Now, let's remap | |
674 | * all windows of the triggered file to anonymous maps, so | |
675 | * that no page of the file in question is triggered again, so | |
676 | * that we can be sure not to hit the queue size limit. */ | |
677 | if (_likely_(!found)) | |
16e9f408 | 678 | return; |
16e9f408 | 679 | |
fa6ac760 LP |
680 | HASHMAP_FOREACH(f, m->fds, i) { |
681 | Window *w; | |
682 | ||
683 | if (!f->sigbus) | |
684 | continue; | |
685 | ||
686 | LIST_FOREACH(by_fd, w, f->windows) | |
687 | window_invalidate(w); | |
688 | } | |
f8019684 | 689 | } |
16e9f408 | 690 | |
fa6ac760 LP |
691 | bool mmap_cache_got_sigbus(MMapCache *m, int fd) { |
692 | FileDescriptor *f; | |
693 | ||
bf807d4d | 694 | assert(m); |
fa6ac760 | 695 | assert(fd >= 0); |
bf807d4d | 696 | |
fa6ac760 LP |
697 | mmap_cache_process_sigbus(m); |
698 | ||
23e096cc | 699 | f = hashmap_get(m->fds, FD_TO_PTR(fd)); |
fa6ac760 LP |
700 | if (!f) |
701 | return false; | |
702 | ||
703 | return f->sigbus; | |
bf807d4d LP |
704 | } |
705 | ||
fa6ac760 LP |
706 | void mmap_cache_close_fd(MMapCache *m, int fd) { |
707 | FileDescriptor *f; | |
708 | ||
bf807d4d | 709 | assert(m); |
fa6ac760 | 710 | assert(fd >= 0); |
bf807d4d | 711 | |
fa6ac760 LP |
712 | /* Make sure that any queued SIGBUS are first dispatched, so |
713 | * that we don't end up with a SIGBUS entry we cannot relate | |
714 | * to any existing memory map */ | |
715 | ||
716 | mmap_cache_process_sigbus(m); | |
717 | ||
23e096cc | 718 | f = hashmap_get(m->fds, FD_TO_PTR(fd)); |
fa6ac760 LP |
719 | if (!f) |
720 | return; | |
721 | ||
722 | fd_free(f); | |
bf807d4d | 723 | } |