1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
4 This file is part of systemd.
6 Copyright 2012 Lennart Poettering
8 systemd is free software; you can redistribute it and/or modify it
9 under the terms of the GNU Lesser General Public License as published by
10 the Free Software Foundation; either version 2.1 of the License, or
11 (at your option) any later version.
13 systemd is distributed in the hope that it will be useful, but
14 WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 Lesser General Public License for more details.
18 You should have received a copy of the GNU Lesser General Public License
19 along with systemd; If not, see <http://www.gnu.org/licenses/>.
26 #include "alloc-util.h"
32 #include "mmap-cache.h"
36 typedef struct Window Window
;
37 typedef struct Context Context
;
38 typedef struct FileDescriptor FileDescriptor
;
54 LIST_FIELDS(Window
, by_fd
);
55 LIST_FIELDS(Window
, unused
);
57 LIST_HEAD(Context
, contexts
);
65 LIST_FIELDS(Context
, by_window
);
68 struct FileDescriptor
{
72 LIST_HEAD(Window
, windows
);
79 unsigned n_hit
, n_missed
;
83 Context
*contexts
[MMAP_CACHE_MAX_CONTEXTS
];
85 LIST_HEAD(Window
, unused
);
89 #define WINDOWS_MIN 64
91 #ifdef ENABLE_DEBUG_MMAP_CACHE
92 /* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
93 # define WINDOW_SIZE (page_size())
95 # define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
98 MMapCache
* mmap_cache_new(void) {
101 m
= new0(MMapCache
, 1);
109 MMapCache
* mmap_cache_ref(MMapCache
*m
) {
111 assert(m
->n_ref
> 0);
117 static void window_unlink(Window
*w
) {
123 munmap(w
->ptr
, w
->size
);
126 LIST_REMOVE(by_fd
, w
->fd
->windows
, w
);
129 if (w
->cache
->last_unused
== w
)
130 w
->cache
->last_unused
= w
->unused_prev
;
132 LIST_REMOVE(unused
, w
->cache
->unused
, w
);
135 LIST_FOREACH(by_window
, c
, w
->contexts
) {
136 assert(c
->window
== w
);
141 static void window_invalidate(Window
*w
) {
147 /* Replace the window with anonymous pages. This is useful
148 * when we hit a SIGBUS and want to make sure the file cannot
149 * trigger any further SIGBUS, possibly overrunning the sigbus
152 assert_se(mmap(w
->ptr
, w
->size
, w
->prot
, MAP_PRIVATE
|MAP_ANONYMOUS
|MAP_FIXED
, -1, 0) == w
->ptr
);
153 w
->invalidated
= true;
156 static void window_free(Window
*w
) {
160 w
->cache
->n_windows
--;
164 _pure_
static bool window_matches(Window
*w
, int fd
, int prot
, uint64_t offset
, size_t size
) {
173 offset
>= w
->offset
&&
174 offset
+ size
<= w
->offset
+ w
->size
;
177 static Window
*window_add(MMapCache
*m
) {
182 if (!m
->last_unused
|| m
->n_windows
<= WINDOWS_MIN
) {
184 /* Allocate a new window */
191 /* Reuse an existing one */
201 static void context_detach_window(Context
*c
) {
211 LIST_REMOVE(by_window
, w
->contexts
, c
);
213 if (!w
->contexts
&& !w
->keep_always
) {
214 /* Not used anymore? */
215 #ifdef ENABLE_DEBUG_MMAP_CACHE
216 /* Unmap unused windows immediately to expose use-after-unmap
220 LIST_PREPEND(unused
, c
->cache
->unused
, w
);
221 if (!c
->cache
->last_unused
)
222 c
->cache
->last_unused
= w
;
229 static void context_attach_window(Context
*c
, Window
*w
) {
236 context_detach_window(c
);
240 LIST_REMOVE(unused
, c
->cache
->unused
, w
);
241 if (c
->cache
->last_unused
== w
)
242 c
->cache
->last_unused
= w
->unused_prev
;
244 w
->in_unused
= false;
248 LIST_PREPEND(by_window
, w
->contexts
, c
);
251 static Context
*context_add(MMapCache
*m
, unsigned id
) {
260 c
= new0(Context
, 1);
267 assert(!m
->contexts
[id
]);
273 static void context_free(Context
*c
) {
276 context_detach_window(c
);
279 assert(c
->cache
->contexts
[c
->id
] == c
);
280 c
->cache
->contexts
[c
->id
] = NULL
;
286 static void fd_free(FileDescriptor
*f
) {
290 window_free(f
->windows
);
293 assert_se(hashmap_remove(f
->cache
->fds
, FD_TO_PTR(f
->fd
)));
298 static FileDescriptor
* fd_add(MMapCache
*m
, int fd
) {
305 f
= hashmap_get(m
->fds
, FD_TO_PTR(fd
));
309 r
= hashmap_ensure_allocated(&m
->fds
, NULL
);
313 f
= new0(FileDescriptor
, 1);
320 r
= hashmap_put(m
->fds
, FD_TO_PTR(fd
), f
);
329 static void mmap_cache_free(MMapCache
*m
) {
335 for (i
= 0; i
< MMAP_CACHE_MAX_CONTEXTS
; i
++)
337 context_free(m
->contexts
[i
]);
339 while ((f
= hashmap_first(m
->fds
)))
342 hashmap_free(m
->fds
);
345 window_free(m
->unused
);
350 MMapCache
* mmap_cache_unref(MMapCache
*m
) {
355 assert(m
->n_ref
> 0);
364 static int make_room(MMapCache
*m
) {
370 window_free(m
->last_unused
);
374 static int try_context(
387 assert(m
->n_ref
> 0);
392 c
= m
->contexts
[context
];
396 assert(c
->id
== context
);
401 if (!window_matches(c
->window
, fd
, prot
, offset
, size
)) {
403 /* Drop the reference to the window, since it's unnecessary now */
404 context_detach_window(c
);
408 if (c
->window
->fd
->sigbus
)
411 c
->window
->keep_always
|= keep_always
;
413 *ret
= (uint8_t*) c
->window
->ptr
+ (offset
- c
->window
->offset
);
417 static int find_mmap(
432 assert(m
->n_ref
> 0);
436 f
= hashmap_get(m
->fds
, FD_TO_PTR(fd
));
445 LIST_FOREACH(by_fd
, w
, f
->windows
)
446 if (window_matches(w
, fd
, prot
, offset
, size
))
452 c
= context_add(m
, context
);
456 context_attach_window(c
, w
);
457 w
->keep_always
+= keep_always
;
459 *ret
= (uint8_t*) w
->ptr
+ (offset
- w
->offset
);
474 uint64_t woffset
, wsize
;
482 assert(m
->n_ref
> 0);
487 woffset
= offset
& ~((uint64_t) page_size() - 1ULL);
488 wsize
= size
+ (offset
- woffset
);
489 wsize
= PAGE_ALIGN(wsize
);
491 if (wsize
< WINDOW_SIZE
) {
494 delta
= PAGE_ALIGN((WINDOW_SIZE
- wsize
) / 2);
505 /* Memory maps that are larger then the files
506 underneath have undefined behavior. Hence, clamp
507 things to the file size if we know it */
509 if (woffset
>= (uint64_t) st
->st_size
)
510 return -EADDRNOTAVAIL
;
512 if (woffset
+ wsize
> (uint64_t) st
->st_size
)
513 wsize
= PAGE_ALIGN(st
->st_size
- woffset
);
517 d
= mmap(NULL
, wsize
, prot
, MAP_SHARED
, fd
, woffset
);
530 c
= context_add(m
, context
);
542 w
->keep_always
= keep_always
;
549 LIST_PREPEND(by_fd
, f
->windows
, w
);
551 context_detach_window(c
);
553 LIST_PREPEND(by_window
, w
->contexts
, c
);
555 *ret
= (uint8_t*) w
->ptr
+ (offset
- w
->offset
);
577 assert(m
->n_ref
> 0);
581 assert(context
< MMAP_CACHE_MAX_CONTEXTS
);
583 /* Check whether the current context is the right one already */
584 r
= try_context(m
, fd
, prot
, context
, keep_always
, offset
, size
, ret
);
590 /* Search for a matching mmap */
591 r
= find_mmap(m
, fd
, prot
, context
, keep_always
, offset
, size
, ret
);
599 /* Create a new mmap */
600 return add_mmap(m
, fd
, prot
, context
, keep_always
, offset
, size
, st
, ret
);
603 unsigned mmap_cache_get_hit(MMapCache
*m
) {
609 unsigned mmap_cache_get_missed(MMapCache
*m
) {
615 static void mmap_cache_process_sigbus(MMapCache
*m
) {
623 /* Iterate through all triggered pages and mark their files as
629 r
= sigbus_pop(&addr
);
630 if (_likely_(r
== 0))
633 log_error_errno(r
, "SIGBUS handling failed: %m");
638 HASHMAP_FOREACH(f
, m
->fds
, i
) {
641 LIST_FOREACH(by_fd
, w
, f
->windows
) {
642 if ((uint8_t*) addr
>= (uint8_t*) w
->ptr
&&
643 (uint8_t*) addr
< (uint8_t*) w
->ptr
+ w
->size
) {
644 found
= ours
= f
->sigbus
= true;
653 /* Didn't find a matching window, give up */
655 log_error("Unknown SIGBUS page, aborting.");
660 /* The list of triggered pages is now empty. Now, let's remap
661 * all windows of the triggered file to anonymous maps, so
662 * that no page of the file in question is triggered again, so
663 * that we can be sure not to hit the queue size limit. */
664 if (_likely_(!found
))
667 HASHMAP_FOREACH(f
, m
->fds
, i
) {
673 LIST_FOREACH(by_fd
, w
, f
->windows
)
674 window_invalidate(w
);
678 bool mmap_cache_got_sigbus(MMapCache
*m
, int fd
) {
684 mmap_cache_process_sigbus(m
);
686 f
= hashmap_get(m
->fds
, FD_TO_PTR(fd
));
693 void mmap_cache_close_fd(MMapCache
*m
, int fd
) {
699 /* Make sure that any queued SIGBUS are first dispatched, so
700 * that we don't end up with a SIGBUS entry we cannot relate
701 * to any existing memory map */
703 mmap_cache_process_sigbus(m
);
705 f
= hashmap_get(m
->fds
, FD_TO_PTR(fd
));