]>
Commit | Line | Data |
---|---|---|
16e9f408 LP |
1 | /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/ |
2 | ||
3 | /*** | |
4 | This file is part of systemd. | |
5 | ||
6 | Copyright 2012 Lennart Poettering | |
7 | ||
8 | systemd is free software; you can redistribute it and/or modify it | |
9 | under the terms of the GNU Lesser General Public License as published by | |
10 | the Free Software Foundation; either version 2.1 of the License, or | |
11 | (at your option) any later version. | |
12 | ||
13 | systemd is distributed in the hope that it will be useful, but | |
14 | WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | Lesser General Public License for more details. | |
17 | ||
18 | You should have received a copy of the GNU Lesser General Public License | |
19 | along with systemd; If not, see <http://www.gnu.org/licenses/>. | |
20 | ***/ | |
21 | ||
16e9f408 LP |
22 | #include <errno.h> |
23 | #include <stdlib.h> | |
f8019684 | 24 | #include <sys/mman.h> |
16e9f408 LP |
25 | #include <string.h> |
26 | ||
f8019684 LP |
27 | #include "hashmap.h" |
28 | #include "list.h" | |
29 | #include "log.h" | |
16e9f408 | 30 | #include "util.h" |
f8019684 | 31 | #include "macro.h" |
16e9f408 LP |
32 | #include "mmap-cache.h" |
33 | ||
f8019684 LP |
34 | typedef struct Window Window; |
35 | typedef struct Context Context; | |
36 | typedef struct FileDescriptor FileDescriptor; | |
84168d80 | 37 | |
f8019684 LP |
38 | struct Window { |
39 | MMapCache *cache; | |
40 | ||
41 | bool keep_always; | |
42 | bool in_unused; | |
16e9f408 | 43 | |
68667801 | 44 | int prot; |
16e9f408 LP |
45 | void *ptr; |
46 | uint64_t offset; | |
f8019684 LP |
47 | size_t size; |
48 | ||
49 | FileDescriptor *fd; | |
16e9f408 | 50 | |
f8019684 LP |
51 | LIST_FIELDS(Window, by_fd); |
52 | LIST_FIELDS(Window, unused); | |
53 | ||
54 | LIST_HEAD(Context, contexts); | |
55 | }; | |
16e9f408 | 56 | |
f8019684 LP |
57 | struct Context { |
58 | MMapCache *cache; | |
59 | unsigned id; | |
60 | Window *window; | |
16e9f408 | 61 | |
f8019684 LP |
62 | LIST_FIELDS(Context, by_window); |
63 | }; | |
64 | ||
65 | struct FileDescriptor { | |
66 | MMapCache *cache; | |
16e9f408 | 67 | int fd; |
f8019684 LP |
68 | LIST_HEAD(Window, windows); |
69 | }; | |
16e9f408 LP |
70 | |
71 | struct MMapCache { | |
f8019684 | 72 | int n_ref; |
68667801 | 73 | unsigned n_windows; |
16e9f408 | 74 | |
f8019684 LP |
75 | Hashmap *fds; |
76 | Hashmap *contexts; | |
16e9f408 | 77 | |
f8019684 LP |
78 | LIST_HEAD(Window, unused); |
79 | Window *last_unused; | |
16e9f408 LP |
80 | }; |
81 | ||
f8019684 LP |
82 | #define WINDOWS_MIN 64 |
83 | #define WINDOW_SIZE (8ULL*1024ULL*1024ULL) | |
16e9f408 | 84 | |
f8019684 LP |
85 | MMapCache* mmap_cache_new(void) { |
86 | MMapCache *m; | |
16e9f408 | 87 | |
f8019684 LP |
88 | m = new0(MMapCache, 1); |
89 | if (!m) | |
90 | return NULL; | |
16e9f408 | 91 | |
f8019684 LP |
92 | m->n_ref = 1; |
93 | return m; | |
16e9f408 LP |
94 | } |
95 | ||
f8019684 | 96 | MMapCache* mmap_cache_ref(MMapCache *m) { |
16e9f408 | 97 | assert(m); |
f8019684 | 98 | assert(m->n_ref > 0); |
16e9f408 | 99 | |
f8019684 LP |
100 | m->n_ref ++; |
101 | return m; | |
102 | } | |
f65425cb | 103 | |
f8019684 LP |
104 | static void window_unlink(Window *w) { |
105 | Context *c; | |
f65425cb | 106 | |
f8019684 | 107 | assert(w); |
16e9f408 | 108 | |
f8019684 LP |
109 | if (w->ptr) |
110 | munmap(w->ptr, w->size); | |
16e9f408 | 111 | |
f8019684 LP |
112 | if (w->fd) |
113 | LIST_REMOVE(Window, by_fd, w->fd->windows, w); | |
16e9f408 | 114 | |
f8019684 LP |
115 | if (w->in_unused) { |
116 | if (w->cache->last_unused == w) | |
117 | w->cache->last_unused = w->unused_prev; | |
16e9f408 | 118 | |
f8019684 | 119 | LIST_REMOVE(Window, unused, w->cache->unused, w); |
f65425cb | 120 | } |
16e9f408 | 121 | |
f8019684 LP |
122 | LIST_FOREACH(by_window, c, w->contexts) { |
123 | assert(c->window == w); | |
124 | c->window = NULL; | |
f65425cb | 125 | } |
16e9f408 LP |
126 | } |
127 | ||
f8019684 LP |
128 | static void window_free(Window *w) { |
129 | assert(w); | |
f65425cb | 130 | |
f8019684 | 131 | window_unlink(w); |
89de6947 | 132 | w->cache->n_windows--; |
f8019684 LP |
133 | free(w); |
134 | } | |
f65425cb | 135 | |
44a6b1b6 | 136 | _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) { |
f8019684 LP |
137 | assert(w); |
138 | assert(fd >= 0); | |
139 | assert(size > 0); | |
16e9f408 | 140 | |
f8019684 LP |
141 | return |
142 | w->fd && | |
143 | fd == w->fd->fd && | |
144 | prot == w->prot && | |
145 | offset >= w->offset && | |
146 | offset + size <= w->offset + w->size; | |
16e9f408 LP |
147 | } |
148 | ||
f8019684 LP |
149 | static Window *window_add(MMapCache *m) { |
150 | Window *w; | |
16e9f408 LP |
151 | |
152 | assert(m); | |
16e9f408 | 153 | |
f8019684 | 154 | if (!m->last_unused || m->n_windows <= WINDOWS_MIN) { |
f65425cb | 155 | |
f8019684 LP |
156 | /* Allocate a new window */ |
157 | w = new0(Window, 1); | |
158 | if (!w) | |
159 | return NULL; | |
89de6947 | 160 | m->n_windows++; |
f65425cb | 161 | } else { |
16e9f408 | 162 | |
f8019684 LP |
163 | /* Reuse an existing one */ |
164 | w = m->last_unused; | |
165 | window_unlink(w); | |
166 | zero(*w); | |
f65425cb | 167 | } |
f8019684 LP |
168 | |
169 | w->cache = m; | |
170 | return w; | |
16e9f408 LP |
171 | } |
172 | ||
f8019684 LP |
173 | static void context_detach_window(Context *c) { |
174 | Window *w; | |
16e9f408 | 175 | |
f8019684 | 176 | assert(c); |
16e9f408 | 177 | |
f8019684 | 178 | if (!c->window) |
16e9f408 LP |
179 | return; |
180 | ||
f8019684 LP |
181 | w = c->window; |
182 | c->window = NULL; | |
183 | LIST_REMOVE(Context, by_window, w->contexts, c); | |
16e9f408 | 184 | |
e18021f7 | 185 | if (!w->contexts && !w->keep_always) { |
f8019684 LP |
186 | /* Not used anymore? */ |
187 | LIST_PREPEND(Window, unused, c->cache->unused, w); | |
188 | if (!c->cache->last_unused) | |
189 | c->cache->last_unused = w; | |
16e9f408 | 190 | |
f8019684 LP |
191 | w->in_unused = true; |
192 | } | |
16e9f408 LP |
193 | } |
194 | ||
f8019684 LP |
195 | static void context_attach_window(Context *c, Window *w) { |
196 | assert(c); | |
197 | assert(w); | |
16e9f408 | 198 | |
f8019684 | 199 | if (c->window == w) |
16e9f408 LP |
200 | return; |
201 | ||
f8019684 | 202 | context_detach_window(c); |
16e9f408 | 203 | |
e18021f7 | 204 | if (w->in_unused) { |
f8019684 LP |
205 | /* Used again? */ |
206 | LIST_REMOVE(Window, unused, c->cache->unused, w); | |
a2ab7ee6 CG |
207 | if (c->cache->last_unused == w) |
208 | c->cache->last_unused = w->unused_prev; | |
16e9f408 | 209 | |
f8019684 LP |
210 | w->in_unused = false; |
211 | } | |
f65425cb | 212 | |
f8019684 LP |
213 | c->window = w; |
214 | LIST_PREPEND(Context, by_window, w->contexts, c); | |
16e9f408 LP |
215 | } |
216 | ||
f8019684 LP |
217 | static Context *context_add(MMapCache *m, unsigned id) { |
218 | Context *c; | |
219 | int r; | |
16e9f408 LP |
220 | |
221 | assert(m); | |
222 | ||
f8019684 LP |
223 | c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1)); |
224 | if (c) | |
225 | return c; | |
226 | ||
227 | r = hashmap_ensure_allocated(&m->contexts, trivial_hash_func, trivial_compare_func); | |
228 | if (r < 0) | |
229 | return NULL; | |
230 | ||
231 | c = new0(Context, 1); | |
232 | if (!c) | |
233 | return NULL; | |
16e9f408 | 234 | |
f8019684 LP |
235 | c->cache = m; |
236 | c->id = id; | |
16e9f408 | 237 | |
f8019684 LP |
238 | r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c); |
239 | if (r < 0) { | |
240 | free(c); | |
241 | return NULL; | |
16e9f408 LP |
242 | } |
243 | ||
f8019684 | 244 | return c; |
16e9f408 LP |
245 | } |
246 | ||
f8019684 LP |
247 | static void context_free(Context *c) { |
248 | assert(c); | |
16e9f408 | 249 | |
f8019684 | 250 | context_detach_window(c); |
16e9f408 | 251 | |
f8019684 LP |
252 | if (c->cache) |
253 | assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1))); | |
16e9f408 | 254 | |
f8019684 LP |
255 | free(c); |
256 | } | |
257 | ||
258 | static void fd_free(FileDescriptor *f) { | |
259 | assert(f); | |
260 | ||
261 | while (f->windows) | |
262 | window_free(f->windows); | |
263 | ||
264 | if (f->cache) | |
265 | assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1))); | |
266 | ||
267 | free(f); | |
268 | } | |
269 | ||
270 | static FileDescriptor* fd_add(MMapCache *m, int fd) { | |
271 | FileDescriptor *f; | |
272 | int r; | |
273 | ||
274 | assert(m); | |
275 | assert(fd >= 0); | |
276 | ||
277 | f = hashmap_get(m->fds, INT_TO_PTR(fd + 1)); | |
278 | if (f) | |
279 | return f; | |
280 | ||
281 | r = hashmap_ensure_allocated(&m->fds, trivial_hash_func, trivial_compare_func); | |
282 | if (r < 0) | |
16e9f408 | 283 | return NULL; |
16e9f408 | 284 | |
f8019684 LP |
285 | f = new0(FileDescriptor, 1); |
286 | if (!f) | |
16e9f408 | 287 | return NULL; |
16e9f408 | 288 | |
f8019684 LP |
289 | f->cache = m; |
290 | f->fd = fd; | |
291 | ||
292 | r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f); | |
293 | if (r < 0) { | |
294 | free(f); | |
16e9f408 LP |
295 | return NULL; |
296 | } | |
297 | ||
f8019684 | 298 | return f; |
16e9f408 LP |
299 | } |
300 | ||
f8019684 LP |
301 | static void mmap_cache_free(MMapCache *m) { |
302 | Context *c; | |
303 | FileDescriptor *f; | |
304 | ||
16e9f408 | 305 | assert(m); |
16e9f408 | 306 | |
f8019684 LP |
307 | while ((c = hashmap_first(m->contexts))) |
308 | context_free(c); | |
309 | ||
8e6d9397 GM |
310 | hashmap_free(m->contexts); |
311 | ||
f8019684 LP |
312 | while ((f = hashmap_first(m->fds))) |
313 | fd_free(f); | |
314 | ||
8e6d9397 GM |
315 | hashmap_free(m->fds); |
316 | ||
f8019684 LP |
317 | while (m->unused) |
318 | window_free(m->unused); | |
319 | ||
320 | free(m); | |
16e9f408 LP |
321 | } |
322 | ||
323 | MMapCache* mmap_cache_unref(MMapCache *m) { | |
324 | assert(m); | |
325 | assert(m->n_ref > 0); | |
326 | ||
f8019684 LP |
327 | m->n_ref --; |
328 | if (m->n_ref == 0) | |
16e9f408 | 329 | mmap_cache_free(m); |
16e9f408 LP |
330 | |
331 | return NULL; | |
332 | } | |
333 | ||
f8019684 LP |
334 | static int make_room(MMapCache *m) { |
335 | assert(m); | |
336 | ||
337 | if (!m->last_unused) | |
338 | return 0; | |
339 | ||
340 | window_free(m->last_unused); | |
341 | return 1; | |
342 | } | |
343 | ||
344 | static int try_context( | |
345 | MMapCache *m, | |
346 | int fd, | |
347 | int prot, | |
348 | unsigned context, | |
349 | bool keep_always, | |
350 | uint64_t offset, | |
351 | size_t size, | |
352 | void **ret) { | |
353 | ||
354 | Context *c; | |
f65425cb | 355 | |
16e9f408 | 356 | assert(m); |
f8019684 LP |
357 | assert(m->n_ref > 0); |
358 | assert(fd >= 0); | |
359 | assert(size > 0); | |
360 | assert(ret); | |
16e9f408 | 361 | |
f8019684 LP |
362 | c = hashmap_get(m->contexts, UINT_TO_PTR(context+1)); |
363 | if (!c) | |
16e9f408 | 364 | return 0; |
16e9f408 | 365 | |
f8019684 | 366 | assert(c->id == context); |
16e9f408 | 367 | |
f8019684 LP |
368 | if (!c->window) |
369 | return 0; | |
f65425cb | 370 | |
f8019684 | 371 | if (!window_matches(c->window, fd, prot, offset, size)) { |
f65425cb | 372 | |
f8019684 LP |
373 | /* Drop the reference to the window, since it's unnecessary now */ |
374 | context_detach_window(c); | |
375 | return 0; | |
f65425cb LP |
376 | } |
377 | ||
f8019684 | 378 | c->window->keep_always = c->window->keep_always || keep_always; |
16e9f408 | 379 | |
f8019684 LP |
380 | *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset); |
381 | return 1; | |
16e9f408 LP |
382 | } |
383 | ||
f8019684 LP |
384 | static int find_mmap( |
385 | MMapCache *m, | |
386 | int fd, | |
387 | int prot, | |
388 | unsigned context, | |
389 | bool keep_always, | |
390 | uint64_t offset, | |
391 | size_t size, | |
392 | void **ret) { | |
393 | ||
394 | FileDescriptor *f; | |
395 | Window *w; | |
396 | Context *c; | |
16e9f408 LP |
397 | |
398 | assert(m); | |
f8019684 LP |
399 | assert(m->n_ref > 0); |
400 | assert(fd >= 0); | |
401 | assert(size > 0); | |
402 | assert(ret); | |
16e9f408 | 403 | |
f8019684 LP |
404 | f = hashmap_get(m->fds, INT_TO_PTR(fd + 1)); |
405 | if (!f) | |
406 | return 0; | |
16e9f408 | 407 | |
f8019684 | 408 | assert(f->fd == fd); |
16e9f408 | 409 | |
f8019684 LP |
410 | LIST_FOREACH(by_fd, w, f->windows) |
411 | if (window_matches(w, fd, prot, offset, size)) | |
412 | break; | |
16e9f408 | 413 | |
f8019684 LP |
414 | if (!w) |
415 | return 0; | |
416 | ||
417 | c = context_add(m, context); | |
418 | if (!c) | |
419 | return -ENOMEM; | |
420 | ||
421 | context_attach_window(c, w); | |
422 | w->keep_always = w->keep_always || keep_always; | |
16e9f408 | 423 | |
f8019684 LP |
424 | *ret = (uint8_t*) w->ptr + (offset - w->offset); |
425 | return 1; | |
16e9f408 LP |
426 | } |
427 | ||
f8019684 | 428 | static int add_mmap( |
16e9f408 LP |
429 | MMapCache *m, |
430 | int fd, | |
16e9f408 LP |
431 | int prot, |
432 | unsigned context, | |
fcde2389 | 433 | bool keep_always, |
16e9f408 | 434 | uint64_t offset, |
f8019684 | 435 | size_t size, |
fcde2389 | 436 | struct stat *st, |
16e9f408 LP |
437 | void **ret) { |
438 | ||
16e9f408 | 439 | uint64_t woffset, wsize; |
f8019684 LP |
440 | Context *c; |
441 | FileDescriptor *f; | |
442 | Window *w; | |
443 | void *d; | |
16e9f408 LP |
444 | int r; |
445 | ||
446 | assert(m); | |
f8019684 | 447 | assert(m->n_ref > 0); |
16e9f408 | 448 | assert(fd >= 0); |
16e9f408 LP |
449 | assert(size > 0); |
450 | assert(ret); | |
451 | ||
452 | woffset = offset & ~((uint64_t) page_size() - 1ULL); | |
453 | wsize = size + (offset - woffset); | |
454 | wsize = PAGE_ALIGN(wsize); | |
455 | ||
456 | if (wsize < WINDOW_SIZE) { | |
457 | uint64_t delta; | |
458 | ||
beec0085 | 459 | delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2); |
16e9f408 LP |
460 | |
461 | if (delta > offset) | |
462 | woffset = 0; | |
463 | else | |
464 | woffset -= delta; | |
465 | ||
466 | wsize = WINDOW_SIZE; | |
467 | } | |
468 | ||
fcde2389 LP |
469 | if (st) { |
470 | /* Memory maps that are larger then the files | |
c5315881 | 471 | underneath have undefined behavior. Hence, clamp |
fcde2389 LP |
472 | things to the file size if we know it */ |
473 | ||
474 | if (woffset >= (uint64_t) st->st_size) | |
475 | return -EADDRNOTAVAIL; | |
476 | ||
477 | if (woffset + wsize > (uint64_t) st->st_size) | |
478 | wsize = PAGE_ALIGN(st->st_size - woffset); | |
479 | } | |
480 | ||
16e9f408 LP |
481 | for (;;) { |
482 | d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset); | |
483 | if (d != MAP_FAILED) | |
484 | break; | |
485 | if (errno != ENOMEM) | |
486 | return -errno; | |
487 | ||
f8019684 | 488 | r = make_room(m); |
16e9f408 LP |
489 | if (r < 0) |
490 | return r; | |
491 | if (r == 0) | |
492 | return -ENOMEM; | |
493 | } | |
494 | ||
f8019684 LP |
495 | c = context_add(m, context); |
496 | if (!c) | |
497 | return -ENOMEM; | |
16e9f408 | 498 | |
f8019684 LP |
499 | f = fd_add(m, fd); |
500 | if (!f) | |
501 | return -ENOMEM; | |
16e9f408 | 502 | |
f8019684 LP |
503 | w = window_add(m); |
504 | if (!w) | |
505 | return -ENOMEM; | |
16e9f408 | 506 | |
f8019684 LP |
507 | w->keep_always = keep_always; |
508 | w->ptr = d; | |
509 | w->offset = woffset; | |
510 | w->prot = prot; | |
511 | w->size = wsize; | |
512 | w->fd = f; | |
16e9f408 | 513 | |
f8019684 | 514 | LIST_PREPEND(Window, by_fd, f->windows, w); |
16e9f408 | 515 | |
f8019684 LP |
516 | context_detach_window(c); |
517 | c->window = w; | |
518 | LIST_PREPEND(Context, by_window, w->contexts, c); | |
16e9f408 | 519 | |
f8019684 | 520 | *ret = (uint8_t*) w->ptr + (offset - w->offset); |
16e9f408 LP |
521 | return 1; |
522 | } | |
523 | ||
524 | int mmap_cache_get( | |
525 | MMapCache *m, | |
526 | int fd, | |
527 | int prot, | |
528 | unsigned context, | |
fcde2389 | 529 | bool keep_always, |
16e9f408 | 530 | uint64_t offset, |
f8019684 | 531 | size_t size, |
fcde2389 | 532 | struct stat *st, |
16e9f408 LP |
533 | void **ret) { |
534 | ||
16e9f408 LP |
535 | int r; |
536 | ||
537 | assert(m); | |
f8019684 | 538 | assert(m->n_ref > 0); |
16e9f408 | 539 | assert(fd >= 0); |
16e9f408 LP |
540 | assert(size > 0); |
541 | assert(ret); | |
542 | ||
f8019684 LP |
543 | /* Check whether the current context is the right one already */ |
544 | r = try_context(m, fd, prot, context, keep_always, offset, size, ret); | |
16e9f408 LP |
545 | if (r != 0) |
546 | return r; | |
547 | ||
f8019684 LP |
548 | /* Search for a matching mmap */ |
549 | r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret); | |
16e9f408 LP |
550 | if (r != 0) |
551 | return r; | |
552 | ||
f8019684 LP |
553 | /* Create a new mmap */ |
554 | return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret); | |
16e9f408 LP |
555 | } |
556 | ||
557 | void mmap_cache_close_fd(MMapCache *m, int fd) { | |
f8019684 | 558 | FileDescriptor *f; |
16e9f408 LP |
559 | |
560 | assert(m); | |
f8019684 | 561 | assert(fd >= 0); |
16e9f408 | 562 | |
f8019684 LP |
563 | f = hashmap_get(m->fds, INT_TO_PTR(fd + 1)); |
564 | if (!f) | |
16e9f408 | 565 | return; |
16e9f408 | 566 | |
f8019684 LP |
567 | fd_free(f); |
568 | } | |
16e9f408 | 569 | |
f8019684 LP |
570 | void mmap_cache_close_context(MMapCache *m, unsigned context) { |
571 | Context *c; | |
16e9f408 | 572 | |
f8019684 | 573 | assert(m); |
16e9f408 | 574 | |
f8019684 LP |
575 | c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1)); |
576 | if (!c) | |
577 | return; | |
16e9f408 | 578 | |
f8019684 | 579 | context_free(c); |
16e9f408 | 580 | } |