-/*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
-
/***
This file is part of systemd.
#include <errno.h>
#include <stdlib.h>
#include <sys/mman.h>
-#include <string.h>
+#include "alloc-util.h"
+#include "fd-util.h"
#include "hashmap.h"
#include "list.h"
#include "log.h"
-#include "util.h"
#include "macro.h"
#include "mmap-cache.h"
+#include "sigbus.h"
+#include "util.h"
typedef struct Window Window;
typedef struct Context Context;
struct Window {
MMapCache *cache;
- bool keep_always;
- bool in_unused;
+ bool invalidated:1;
+ bool keep_always:1;
+ bool in_unused:1;
+ int prot;
void *ptr;
uint64_t offset;
- int prot;
size_t size;
FileDescriptor *fd;
struct FileDescriptor {
MMapCache *cache;
int fd;
+ bool sigbus;
LIST_HEAD(Window, windows);
};
struct MMapCache {
int n_ref;
+ unsigned n_windows;
- Hashmap *fds;
- Hashmap *contexts;
+ unsigned n_hit, n_missed;
- unsigned n_windows;
+ Hashmap *fds;
+ Context *contexts[MMAP_CACHE_MAX_CONTEXTS];
LIST_HEAD(Window, unused);
Window *last_unused;
};
#define WINDOWS_MIN 64
-#define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
+
+#ifdef ENABLE_DEBUG_MMAP_CACHE
+/* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
+# define WINDOW_SIZE (page_size())
+#else
+# define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
+#endif
MMapCache* mmap_cache_new(void) {
MMapCache *m;
assert(m);
assert(m->n_ref > 0);
- m->n_ref ++;
+ m->n_ref++;
return m;
}
munmap(w->ptr, w->size);
if (w->fd)
- LIST_REMOVE(Window, by_fd, w->fd->windows, w);
+ LIST_REMOVE(by_fd, w->fd->windows, w);
if (w->in_unused) {
if (w->cache->last_unused == w)
w->cache->last_unused = w->unused_prev;
- LIST_REMOVE(Window, unused, w->cache->unused, w);
+ LIST_REMOVE(unused, w->cache->unused, w);
}
LIST_FOREACH(by_window, c, w->contexts) {
}
}
+static void window_invalidate(Window *w) {
+ assert(w);
+
+ if (w->invalidated)
+ return;
+
+ /* Replace the window with anonymous pages. This is useful
+ * when we hit a SIGBUS and want to make sure the file cannot
+ * trigger any further SIGBUS, possibly overrunning the sigbus
+ * queue. */
+
+ assert_se(mmap(w->ptr, w->size, w->prot, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0) == w->ptr);
+ w->invalidated = true;
+}
+
static void window_free(Window *w) {
assert(w);
window_unlink(w);
+ w->cache->n_windows--;
free(w);
}
-static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
+_pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
assert(w);
assert(fd >= 0);
assert(size > 0);
offset + size <= w->offset + w->size;
}
-static Window *window_add(MMapCache *m) {
+static Window *window_add(MMapCache *m, FileDescriptor *fd, int prot, bool keep_always, uint64_t offset, size_t size, void *ptr) {
Window *w;
assert(m);
+ assert(fd);
if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
w = new0(Window, 1);
if (!w)
return NULL;
+ m->n_windows++;
} else {
/* Reuse an existing one */
}
w->cache = m;
+ w->fd = fd;
+ w->prot = prot;
+ w->keep_always = keep_always;
+ w->offset = offset;
+ w->size = size;
+ w->ptr = ptr;
+
+ LIST_PREPEND(by_fd, fd->windows, w);
+
return w;
}
w = c->window;
c->window = NULL;
- LIST_REMOVE(Context, by_window, w->contexts, c);
+ LIST_REMOVE(by_window, w->contexts, c);
- if (!w->contexts) {
+ if (!w->contexts && !w->keep_always) {
/* Not used anymore? */
- LIST_PREPEND(Window, unused, c->cache->unused, w);
+#ifdef ENABLE_DEBUG_MMAP_CACHE
+ /* Unmap unused windows immediately to expose use-after-unmap
+ * by SIGSEGV. */
+ window_free(w);
+#else
+ LIST_PREPEND(unused, c->cache->unused, w);
if (!c->cache->last_unused)
c->cache->last_unused = w;
w->in_unused = true;
+#endif
}
}
context_detach_window(c);
- if (!w->contexts) {
+ if (w->in_unused) {
/* Used again? */
- LIST_REMOVE(Window, unused, c->cache->unused, w);
- if (!c->cache->last_unused)
- c->cache->last_unused = w;
+ LIST_REMOVE(unused, c->cache->unused, w);
+ if (c->cache->last_unused == w)
+ c->cache->last_unused = w->unused_prev;
w->in_unused = false;
}
c->window = w;
- LIST_PREPEND(Context, by_window, w->contexts, c);
+ LIST_PREPEND(by_window, w->contexts, c);
}
static Context *context_add(MMapCache *m, unsigned id) {
Context *c;
- int r;
assert(m);
- c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1));
+ c = m->contexts[id];
if (c)
return c;
- r = hashmap_ensure_allocated(&m->contexts, trivial_hash_func, trivial_compare_func);
- if (r < 0)
- return NULL;
-
c = new0(Context, 1);
if (!c)
return NULL;
c->cache = m;
c->id = id;
- r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c);
- if (r < 0) {
- free(c);
- return NULL;
- }
+ assert(!m->contexts[id]);
+ m->contexts[id] = c;
return c;
}
context_detach_window(c);
- if (c->cache)
- assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1)));
+ if (c->cache) {
+ assert(c->cache->contexts[c->id] == c);
+ c->cache->contexts[c->id] = NULL;
+ }
free(c);
}
window_free(f->windows);
if (f->cache)
- assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
+ assert_se(hashmap_remove(f->cache->fds, FD_TO_PTR(f->fd)));
free(f);
}
assert(m);
assert(fd >= 0);
- f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
+ f = hashmap_get(m->fds, FD_TO_PTR(fd));
if (f)
return f;
- r = hashmap_ensure_allocated(&m->fds, trivial_hash_func, trivial_compare_func);
+ r = hashmap_ensure_allocated(&m->fds, NULL);
if (r < 0)
return NULL;
f->cache = m;
f->fd = fd;
- r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
- if (r < 0) {
- free(f);
- return NULL;
- }
+ r = hashmap_put(m->fds, FD_TO_PTR(fd), f);
+ if (r < 0)
+ return mfree(f);
return f;
}
static void mmap_cache_free(MMapCache *m) {
- Context *c;
FileDescriptor *f;
+ int i;
assert(m);
- while ((c = hashmap_first(m->contexts)))
- context_free(c);
+ for (i = 0; i < MMAP_CACHE_MAX_CONTEXTS; i++)
+ if (m->contexts[i])
+ context_free(m->contexts[i]);
while ((f = hashmap_first(m->fds)))
fd_free(f);
+ hashmap_free(m->fds);
+
while (m->unused)
window_free(m->unused);
}
MMapCache* mmap_cache_unref(MMapCache *m) {
- assert(m);
+
+ if (!m)
+ return NULL;
+
assert(m->n_ref > 0);
- m->n_ref --;
+ m->n_ref--;
if (m->n_ref == 0)
mmap_cache_free(m);
assert(size > 0);
assert(ret);
- c = hashmap_get(m->contexts, UINT_TO_PTR(context+1));
+ c = m->contexts[context];
if (!c)
return 0;
return 0;
}
+ if (c->window->fd->sigbus)
+ return -EIO;
+
c->window->keep_always = c->window->keep_always || keep_always;
*ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
assert(m->n_ref > 0);
assert(fd >= 0);
assert(size > 0);
- assert(ret);
- f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
+ f = hashmap_get(m->fds, FD_TO_PTR(fd));
if (!f)
return 0;
assert(f->fd == fd);
+ if (f->sigbus)
+ return -EIO;
+
LIST_FOREACH(by_fd, w, f->windows)
if (window_matches(w, fd, prot, offset, size))
break;
return 1;
}
+static int mmap_try_harder(MMapCache *m, void *addr, int fd, int prot, int flags, uint64_t offset, size_t size, void **res) {
+ void *ptr;
+
+ assert(m);
+ assert(fd >= 0);
+ assert(res);
+
+ for (;;) {
+ int r;
+
+ ptr = mmap(addr, size, prot, flags, fd, offset);
+ if (ptr != MAP_FAILED)
+ break;
+ if (errno != ENOMEM)
+ return negative_errno();
+
+ r = make_room(m);
+ if (r < 0)
+ return r;
+ if (r == 0)
+ return -ENOMEM;
+ }
+
+ *res = ptr;
+ return 0;
+}
+
static int add_mmap(
MMapCache *m,
int fd,
wsize = PAGE_ALIGN(st->st_size - woffset);
}
- for (;;) {
- d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
- if (d != MAP_FAILED)
- break;
- if (errno != ENOMEM)
- return -errno;
-
- r = make_room(m);
- if (r < 0)
- return r;
- if (r == 0)
- return -ENOMEM;
- }
+ r = mmap_try_harder(m, NULL, fd, prot, MAP_SHARED, woffset, wsize, &d);
+ if (r < 0)
+ return r;
c = context_add(m, context);
if (!c)
- return -ENOMEM;
+ goto outofmem;
f = fd_add(m, fd);
if (!f)
- return -ENOMEM;
+ goto outofmem;
- w = window_add(m);
+ w = window_add(m, f, prot, keep_always, woffset, wsize, d);
if (!w)
- return -ENOMEM;
-
- w->keep_always = keep_always;
- w->ptr = d;
- w->offset = woffset;
- w->prot = prot;
- w->size = wsize;
- w->fd = f;
-
- LIST_PREPEND(Window, by_fd, f->windows, w);
+ goto outofmem;
context_detach_window(c);
c->window = w;
- LIST_PREPEND(Context, by_window, w->contexts, c);
+ LIST_PREPEND(by_window, w->contexts, c);
*ret = (uint8_t*) w->ptr + (offset - w->offset);
return 1;
+
+outofmem:
+ (void) munmap(d, wsize);
+ return -ENOMEM;
}
int mmap_cache_get(
assert(fd >= 0);
assert(size > 0);
assert(ret);
+ assert(context < MMAP_CACHE_MAX_CONTEXTS);
/* Check whether the current context is the right one already */
r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
- if (r != 0)
+ if (r != 0) {
+ m->n_hit++;
return r;
+ }
/* Search for a matching mmap */
r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
- if (r != 0)
+ if (r != 0) {
+ m->n_hit++;
return r;
+ }
+
+ m->n_missed++;
/* Create a new mmap */
return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
}
-void mmap_cache_close_fd(MMapCache *m, int fd) {
+unsigned mmap_cache_get_hit(MMapCache *m) {
+ assert(m);
+
+ return m->n_hit;
+}
+
+unsigned mmap_cache_get_missed(MMapCache *m) {
+ assert(m);
+
+ return m->n_missed;
+}
+
+static void mmap_cache_process_sigbus(MMapCache *m) {
+ bool found = false;
+ FileDescriptor *f;
+ Iterator i;
+ int r;
+
+ assert(m);
+
+ /* Iterate through all triggered pages and mark their files as
+ * invalidated */
+ for (;;) {
+ bool ours;
+ void *addr;
+
+ r = sigbus_pop(&addr);
+ if (_likely_(r == 0))
+ break;
+ if (r < 0) {
+ log_error_errno(r, "SIGBUS handling failed: %m");
+ abort();
+ }
+
+ ours = false;
+ HASHMAP_FOREACH(f, m->fds, i) {
+ Window *w;
+
+ LIST_FOREACH(by_fd, w, f->windows) {
+ if ((uint8_t*) addr >= (uint8_t*) w->ptr &&
+ (uint8_t*) addr < (uint8_t*) w->ptr + w->size) {
+ found = ours = f->sigbus = true;
+ break;
+ }
+ }
+
+ if (ours)
+ break;
+ }
+
+ /* Didn't find a matching window, give up */
+ if (!ours) {
+ log_error("Unknown SIGBUS page, aborting.");
+ abort();
+ }
+ }
+
+ /* The list of triggered pages is now empty. Now, let's remap
+ * all windows of the triggered file to anonymous maps, so
+ * that no page of the file in question is triggered again, so
+ * that we can be sure not to hit the queue size limit. */
+ if (_likely_(!found))
+ return;
+
+ HASHMAP_FOREACH(f, m->fds, i) {
+ Window *w;
+
+ if (!f->sigbus)
+ continue;
+
+ LIST_FOREACH(by_fd, w, f->windows)
+ window_invalidate(w);
+ }
+}
+
+bool mmap_cache_got_sigbus(MMapCache *m, int fd) {
FileDescriptor *f;
assert(m);
assert(fd >= 0);
- f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
+ mmap_cache_process_sigbus(m);
+
+ f = hashmap_get(m->fds, FD_TO_PTR(fd));
if (!f)
- return;
+ return false;
- fd_free(f);
+ return f->sigbus;
}
-void mmap_cache_close_context(MMapCache *m, unsigned context) {
- Context *c;
+void mmap_cache_close_fd(MMapCache *m, int fd) {
+ FileDescriptor *f;
assert(m);
+ assert(fd >= 0);
- c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1));
- if (!c)
+ /* Make sure that any queued SIGBUS are first dispatched, so
+ * that we don't end up with a SIGBUS entry we cannot relate
+ * to any existing memory map */
+
+ mmap_cache_process_sigbus(m);
+
+ f = hashmap_get(m->fds, FD_TO_PTR(fd));
+ if (!f)
return;
- context_free(c);
+ fd_free(f);
}