src/journal/mmap-cache.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <stdlib.h>
  24 #include <sys/mman.h>
  25 #include <string.h>
  26
  27 #include "hashmap.h"
  28 #include "list.h"
  29 #include "log.h"
  30 #include "util.h"
  31 #include "macro.h"
  32 #include "mmap-cache.h"
  33
  34 typedef struct Window Window;
  35 typedef struct Context Context;
  36 typedef struct FileDescriptor FileDescriptor;
  37
  38 struct Window {
  39         MMapCache *cache;
  40
  41         bool keep_always;
  42         bool in_unused;
  43
  44         int prot;
  45         void *ptr;
  46         uint64_t offset;
  47         size_t size;
  48
  49         FileDescriptor *fd;
  50
  51         LIST_FIELDS(Window, by_fd);
  52         LIST_FIELDS(Window, unused);
  53
  54         LIST_HEAD(Context, contexts);
  55 };
  56
  57 struct Context {
  58         MMapCache *cache;
  59         unsigned id;
  60         Window *window;
  61
  62         LIST_FIELDS(Context, by_window);
  63 };
  64
  65 struct FileDescriptor {
  66         MMapCache *cache;
  67         int fd;
  68         LIST_HEAD(Window, windows);
  69 };
  70
  71 struct MMapCache {
  72         int n_ref;
  73         unsigned n_windows;
  74
  75         unsigned n_hit, n_missed;
  76
  77
  78         Hashmap *fds;
  79         Hashmap *contexts;
  80
  81         LIST_HEAD(Window, unused);
  82         Window *last_unused;
  83 };
  84
  85 #define WINDOWS_MIN 64
  86
  87 #ifdef ENABLE_DEBUG_MMAP_CACHE
  88 /* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
  89 # define WINDOW_SIZE (page_size())
  90 #else
  91 # define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
  92 #endif
  93
  94 MMapCache* mmap_cache_new(void) {
  95         MMapCache *m;
  96
  97         m = new0(MMapCache, 1);
  98         if (!m)
  99                 return NULL;
 100
 101         m->n_ref = 1;
 102         return m;
 103 }
 104
 105 MMapCache* mmap_cache_ref(MMapCache *m) {
 106         assert(m);
 107         assert(m->n_ref > 0);
 108
 109         m->n_ref ++;
 110         return m;
 111 }
 112
 113 static void window_unlink(Window *w) {
 114         Context *c;
 115
 116         assert(w);
 117
 118         if (w->ptr)
 119                 munmap(w->ptr, w->size);
 120
 121         if (w->fd)
 122                 LIST_REMOVE(by_fd, w->fd->windows, w);
 123
 124         if (w->in_unused) {
 125                 if (w->cache->last_unused == w)
 126                         w->cache->last_unused = w->unused_prev;
 127
 128                 LIST_REMOVE(unused, w->cache->unused, w);
 129         }
 130
 131         LIST_FOREACH(by_window, c, w->contexts) {
 132                 assert(c->window == w);
 133                 c->window = NULL;
 134         }
 135 }
 136
 137 static void window_free(Window *w) {
 138         assert(w);
 139
 140         window_unlink(w);
 141         w->cache->n_windows--;
 142         free(w);
 143 }
 144
 145 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
 146         assert(w);
 147         assert(fd >= 0);
 148         assert(size > 0);
 149
 150         return
 151                 w->fd &&
 152                 fd == w->fd->fd &&
 153                 prot == w->prot &&
 154                 offset >= w->offset &&
 155                 offset + size <= w->offset + w->size;
 156 }
 157
 158 static Window *window_add(MMapCache *m) {
 159         Window *w;
 160
 161         assert(m);
 162
 163         if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
 164
 165                 /* Allocate a new window */
 166                 w = new0(Window, 1);
 167                 if (!w)
 168                         return NULL;
 169                 m->n_windows++;
 170         } else {
 171
 172                 /* Reuse an existing one */
 173                 w = m->last_unused;
 174                 window_unlink(w);
 175                 zero(*w);
 176         }
 177
 178         w->cache = m;
 179         return w;
 180 }
 181
 182 static void context_detach_window(Context *c) {
 183         Window *w;
 184
 185         assert(c);
 186
 187         if (!c->window)
 188                 return;
 189
 190         w = c->window;
 191         c->window = NULL;
 192         LIST_REMOVE(by_window, w->contexts, c);
 193
 194         if (!w->contexts && !w->keep_always) {
 195                 /* Not used anymore? */
 196 #ifdef ENABLE_DEBUG_MMAP_CACHE
 197                 /* Unmap unused windows immediately to expose use-after-unmap
 198                  * by SIGSEGV. */
 199                 window_free(w);
 200 #else
 201                 LIST_PREPEND(unused, c->cache->unused, w);
 202                 if (!c->cache->last_unused)
 203                         c->cache->last_unused = w;
 204
 205                 w->in_unused = true;
 206 #endif
 207         }
 208 }
 209
 210 static void context_attach_window(Context *c, Window *w) {
 211         assert(c);
 212         assert(w);
 213
 214         if (c->window == w)
 215                 return;
 216
 217         context_detach_window(c);
 218
 219         if (w->in_unused) {
 220                 /* Used again? */
 221                 LIST_REMOVE(unused, c->cache->unused, w);
 222                 if (c->cache->last_unused == w)
 223                         c->cache->last_unused = w->unused_prev;
 224
 225                 w->in_unused = false;
 226         }
 227
 228         c->window = w;
 229         LIST_PREPEND(by_window, w->contexts, c);
 230 }
 231
 232 static Context *context_add(MMapCache *m, unsigned id) {
 233         Context *c;
 234         int r;
 235
 236         assert(m);
 237
 238         c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1));
 239         if (c)
 240                 return c;
 241
 242         r = hashmap_ensure_allocated(&m->contexts, NULL);
 243         if (r < 0)
 244                 return NULL;
 245
 246         c = new0(Context, 1);
 247         if (!c)
 248                 return NULL;
 249
 250         c->cache = m;
 251         c->id = id;
 252
 253         r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c);
 254         if (r < 0) {
 255                 free(c);
 256                 return NULL;
 257         }
 258
 259         return c;
 260 }
 261
 262 static void context_free(Context *c) {
 263         assert(c);
 264
 265         context_detach_window(c);
 266
 267         if (c->cache)
 268                 assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1)));
 269
 270         free(c);
 271 }
 272
 273 static void fd_free(FileDescriptor *f) {
 274         assert(f);
 275
 276         while (f->windows)
 277                 window_free(f->windows);
 278
 279         if (f->cache)
 280                 assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
 281
 282         free(f);
 283 }
 284
 285 static FileDescriptor* fd_add(MMapCache *m, int fd) {
 286         FileDescriptor *f;
 287         int r;
 288
 289         assert(m);
 290         assert(fd >= 0);
 291
 292         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 293         if (f)
 294                 return f;
 295
 296         r = hashmap_ensure_allocated(&m->fds, NULL);
 297         if (r < 0)
 298                 return NULL;
 299
 300         f = new0(FileDescriptor, 1);
 301         if (!f)
 302                 return NULL;
 303
 304         f->cache = m;
 305         f->fd = fd;
 306
 307         r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
 308         if (r < 0) {
 309                 free(f);
 310                 return NULL;
 311         }
 312
 313         return f;
 314 }
 315
 316 static void mmap_cache_free(MMapCache *m) {
 317         Context *c;
 318         FileDescriptor *f;
 319
 320         assert(m);
 321
 322         while ((c = hashmap_first(m->contexts)))
 323                 context_free(c);
 324
 325         hashmap_free(m->contexts);
 326
 327         while ((f = hashmap_first(m->fds)))
 328                 fd_free(f);
 329
 330         hashmap_free(m->fds);
 331
 332         while (m->unused)
 333                 window_free(m->unused);
 334
 335         free(m);
 336 }
 337
 338 MMapCache* mmap_cache_unref(MMapCache *m) {
 339         assert(m);
 340         assert(m->n_ref > 0);
 341
 342         m->n_ref --;
 343         if (m->n_ref == 0)
 344                 mmap_cache_free(m);
 345
 346         return NULL;
 347 }
 348
 349 static int make_room(MMapCache *m) {
 350         assert(m);
 351
 352         if (!m->last_unused)
 353                 return 0;
 354
 355         window_free(m->last_unused);
 356         return 1;
 357 }
 358
 359 static int try_context(
 360                 MMapCache *m,
 361                 int fd,
 362                 int prot,
 363                 unsigned context,
 364                 bool keep_always,
 365                 uint64_t offset,
 366                 size_t size,
 367                 void **ret) {
 368
 369         Context *c;
 370
 371         assert(m);
 372         assert(m->n_ref > 0);
 373         assert(fd >= 0);
 374         assert(size > 0);
 375         assert(ret);
 376
 377         c = hashmap_get(m->contexts, UINT_TO_PTR(context+1));
 378         if (!c)
 379                 return 0;
 380
 381         assert(c->id == context);
 382
 383         if (!c->window)
 384                 return 0;
 385
 386         if (!window_matches(c->window, fd, prot, offset, size)) {
 387
 388                 /* Drop the reference to the window, since it's unnecessary now */
 389                 context_detach_window(c);
 390                 return 0;
 391         }
 392
 393         c->window->keep_always |= keep_always;
 394
 395         *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
 396         return 1;
 397 }
 398
 399 static int find_mmap(
 400                 MMapCache *m,
 401                 int fd,
 402                 int prot,
 403                 unsigned context,
 404                 bool keep_always,
 405                 uint64_t offset,
 406                 size_t size,
 407                 void **ret) {
 408
 409         FileDescriptor *f;
 410         Window *w;
 411         Context *c;
 412
 413         assert(m);
 414         assert(m->n_ref > 0);
 415         assert(fd >= 0);
 416         assert(size > 0);
 417
 418         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 419         if (!f)
 420                 return 0;
 421
 422         assert(f->fd == fd);
 423
 424         LIST_FOREACH(by_fd, w, f->windows)
 425                 if (window_matches(w, fd, prot, offset, size))
 426                         break;
 427
 428         if (!w)
 429                 return 0;
 430
 431         c = context_add(m, context);
 432         if (!c)
 433                 return -ENOMEM;
 434
 435         context_attach_window(c, w);
 436         w->keep_always += keep_always;
 437
 438         *ret = (uint8_t*) w->ptr + (offset - w->offset);
 439         return 1;
 440 }
 441
 442 static int add_mmap(
 443                 MMapCache *m,
 444                 int fd,
 445                 int prot,
 446                 unsigned context,
 447                 bool keep_always,
 448                 uint64_t offset,
 449                 size_t size,
 450                 struct stat *st,
 451                 void **ret) {
 452
 453         uint64_t woffset, wsize;
 454         Context *c;
 455         FileDescriptor *f;
 456         Window *w;
 457         void *d;
 458         int r;
 459
 460         assert(m);
 461         assert(m->n_ref > 0);
 462         assert(fd >= 0);
 463         assert(size > 0);
 464         assert(ret);
 465
 466         woffset = offset & ~((uint64_t) page_size() - 1ULL);
 467         wsize = size + (offset - woffset);
 468         wsize = PAGE_ALIGN(wsize);
 469
 470         if (wsize < WINDOW_SIZE) {
 471                 uint64_t delta;
 472
 473                 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
 474
 475                 if (delta > offset)
 476                         woffset = 0;
 477                 else
 478                         woffset -= delta;
 479
 480                 wsize = WINDOW_SIZE;
 481         }
 482
 483         if (st) {
 484                 /* Memory maps that are larger then the files
 485                    underneath have undefined behavior. Hence, clamp
 486                    things to the file size if we know it */
 487
 488                 if (woffset >= (uint64_t) st->st_size)
 489                         return -EADDRNOTAVAIL;
 490
 491                 if (woffset + wsize > (uint64_t) st->st_size)
 492                         wsize = PAGE_ALIGN(st->st_size - woffset);
 493         }
 494
 495         for (;;) {
 496                 d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
 497                 if (d != MAP_FAILED)
 498                         break;
 499                 if (errno != ENOMEM)
 500                         return -errno;
 501
 502                 r = make_room(m);
 503                 if (r < 0)
 504                         return r;
 505                 if (r == 0)
 506                         return -ENOMEM;
 507         }
 508
 509         c = context_add(m, context);
 510         if (!c)
 511                 goto outofmem;
 512
 513         f = fd_add(m, fd);
 514         if (!f)
 515                 goto outofmem;
 516
 517         w = window_add(m);
 518         if (!w)
 519                 goto outofmem;
 520
 521         w->keep_always = keep_always;
 522         w->ptr = d;
 523         w->offset = woffset;
 524         w->prot = prot;
 525         w->size = wsize;
 526         w->fd = f;
 527
 528         LIST_PREPEND(by_fd, f->windows, w);
 529
 530         context_detach_window(c);
 531         c->window = w;
 532         LIST_PREPEND(by_window, w->contexts, c);
 533
 534         *ret = (uint8_t*) w->ptr + (offset - w->offset);
 535         return 1;
 536
 537 outofmem:
 538         munmap(d, wsize);
 539         return -ENOMEM;
 540 }
 541
 542 int mmap_cache_get(
 543                 MMapCache *m,
 544                 int fd,
 545                 int prot,
 546                 unsigned context,
 547                 bool keep_always,
 548                 uint64_t offset,
 549                 size_t size,
 550                 struct stat *st,
 551                 void **ret) {
 552
 553         int r;
 554
 555         assert(m);
 556         assert(m->n_ref > 0);
 557         assert(fd >= 0);
 558         assert(size > 0);
 559         assert(ret);
 560
 561         /* Check whether the current context is the right one already */
 562         r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
 563         if (r != 0) {
 564                 m->n_hit ++;
 565                 return r;
 566         }
 567
 568         /* Search for a matching mmap */
 569         r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
 570         if (r != 0) {
 571                 m->n_hit ++;
 572                 return r;
 573         }
 574
 575         m->n_missed++;
 576
 577         /* Create a new mmap */
 578         return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
 579 }
 580
 581 void mmap_cache_close_fd(MMapCache *m, int fd) {
 582         FileDescriptor *f;
 583
 584         assert(m);
 585         assert(fd >= 0);
 586
 587         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 588         if (!f)
 589                 return;
 590
 591         fd_free(f);
 592 }
 593
 594 void mmap_cache_close_context(MMapCache *m, unsigned context) {
 595         Context *c;
 596
 597         assert(m);
 598
 599         c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1));
 600         if (!c)
 601                 return;
 602
 603         context_free(c);
 604 }
 605
 606 unsigned mmap_cache_get_hit(MMapCache *m) {
 607         assert(m);
 608
 609         return m->n_hit;
 610 }
 611
 612 unsigned mmap_cache_get_missed(MMapCache *m) {
 613         assert(m);
 614
 615         return m->n_missed;
 616 }