src/journal/mmap-cache.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <stdlib.h>
  24 #include <sys/mman.h>
  25 #include <string.h>
  26
  27 #include "hashmap.h"
  28 #include "list.h"
  29 #include "log.h"
  30 #include "util.h"
  31 #include "macro.h"
  32 #include "mmap-cache.h"
  33
  34 typedef struct Window Window;
  35 typedef struct Context Context;
  36 typedef struct FileDescriptor FileDescriptor;
  37
  38 struct Window {
  39         MMapCache *cache;
  40
  41         unsigned keep_always;
  42         bool in_unused;
  43
  44         int prot;
  45         void *ptr;
  46         uint64_t offset;
  47         size_t size;
  48
  49         FileDescriptor *fd;
  50
  51         LIST_FIELDS(Window, by_fd);
  52         LIST_FIELDS(Window, unused);
  53
  54         LIST_HEAD(Context, contexts);
  55 };
  56
  57 struct Context {
  58         MMapCache *cache;
  59         unsigned id;
  60         Window *window;
  61
  62         LIST_FIELDS(Context, by_window);
  63 };
  64
  65 struct FileDescriptor {
  66         MMapCache *cache;
  67         int fd;
  68         LIST_HEAD(Window, windows);
  69 };
  70
  71 struct MMapCache {
  72         int n_ref;
  73         unsigned n_windows;
  74
  75         unsigned n_hit, n_missed;
  76
  77
  78         Hashmap *fds;
  79         Hashmap *contexts;
  80
  81         LIST_HEAD(Window, unused);
  82         Window *last_unused;
  83 };
  84
  85 #define WINDOWS_MIN 64
  86
  87 #ifdef ENABLE_DEBUG_MMAP_CACHE
  88 /* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
  89 # define WINDOW_SIZE (page_size())
  90 #else
  91 # define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
  92 #endif
  93
  94 MMapCache* mmap_cache_new(void) {
  95         MMapCache *m;
  96
  97         m = new0(MMapCache, 1);
  98         if (!m)
  99                 return NULL;
 100
 101         m->n_ref = 1;
 102         return m;
 103 }
 104
 105 MMapCache* mmap_cache_ref(MMapCache *m) {
 106         assert(m);
 107         assert(m->n_ref > 0);
 108
 109         m->n_ref ++;
 110         return m;
 111 }
 112
 113 static void window_unlink(Window *w) {
 114         Context *c;
 115
 116         assert(w);
 117
 118         if (w->ptr)
 119                 munmap(w->ptr, w->size);
 120
 121         if (w->fd)
 122                 LIST_REMOVE(by_fd, w->fd->windows, w);
 123
 124         if (w->in_unused) {
 125                 if (w->cache->last_unused == w)
 126                         w->cache->last_unused = w->unused_prev;
 127
 128                 LIST_REMOVE(unused, w->cache->unused, w);
 129         }
 130
 131         LIST_FOREACH(by_window, c, w->contexts) {
 132                 assert(c->window == w);
 133                 c->window = NULL;
 134         }
 135 }
 136
 137 static void window_free(Window *w) {
 138         assert(w);
 139
 140         window_unlink(w);
 141         w->cache->n_windows--;
 142         free(w);
 143 }
 144
 145 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
 146         assert(w);
 147         assert(fd >= 0);
 148         assert(size > 0);
 149
 150         return
 151                 w->fd &&
 152                 fd == w->fd->fd &&
 153                 prot == w->prot &&
 154                 offset >= w->offset &&
 155                 offset + size <= w->offset + w->size;
 156 }
 157
 158 static Window *window_add(MMapCache *m) {
 159         Window *w;
 160
 161         assert(m);
 162
 163         if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
 164
 165                 /* Allocate a new window */
 166                 w = new0(Window, 1);
 167                 if (!w)
 168                         return NULL;
 169                 m->n_windows++;
 170         } else {
 171
 172                 /* Reuse an existing one */
 173                 w = m->last_unused;
 174                 window_unlink(w);
 175                 zero(*w);
 176         }
 177
 178         w->cache = m;
 179         return w;
 180 }
 181
 182 static void context_detach_window(Context *c) {
 183         Window *w;
 184
 185         assert(c);
 186
 187         if (!c->window)
 188                 return;
 189
 190         w = c->window;
 191         c->window = NULL;
 192         LIST_REMOVE(by_window, w->contexts, c);
 193
 194         if (!w->contexts && w->keep_always == 0) {
 195                 /* Not used anymore? */
 196 #ifdef ENABLE_DEBUG_MMAP_CACHE
 197                 /* Unmap unused windows immediately to expose use-after-unmap
 198                  * by SIGSEGV. */
 199                 window_free(w);
 200 #else
 201                 LIST_PREPEND(unused, c->cache->unused, w);
 202                 if (!c->cache->last_unused)
 203                         c->cache->last_unused = w;
 204
 205                 w->in_unused = true;
 206 #endif
 207         }
 208 }
 209
 210 static void context_attach_window(Context *c, Window *w) {
 211         assert(c);
 212         assert(w);
 213
 214         if (c->window == w)
 215                 return;
 216
 217         context_detach_window(c);
 218
 219         if (w->in_unused) {
 220                 /* Used again? */
 221                 LIST_REMOVE(unused, c->cache->unused, w);
 222                 if (c->cache->last_unused == w)
 223                         c->cache->last_unused = w->unused_prev;
 224
 225                 w->in_unused = false;
 226         }
 227
 228         c->window = w;
 229         LIST_PREPEND(by_window, w->contexts, c);
 230 }
 231
 232 static Context *context_add(MMapCache *m, unsigned id) {
 233         Context *c;
 234         int r;
 235
 236         assert(m);
 237
 238         c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1));
 239         if (c)
 240                 return c;
 241
 242         r = hashmap_ensure_allocated(&m->contexts, NULL);
 243         if (r < 0)
 244                 return NULL;
 245
 246         c = new0(Context, 1);
 247         if (!c)
 248                 return NULL;
 249
 250         c->cache = m;
 251         c->id = id;
 252
 253         r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c);
 254         if (r < 0) {
 255                 free(c);
 256                 return NULL;
 257         }
 258
 259         return c;
 260 }
 261
 262 static void context_free(Context *c) {
 263         assert(c);
 264
 265         context_detach_window(c);
 266
 267         if (c->cache)
 268                 assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1)));
 269
 270         free(c);
 271 }
 272
 273 static void fd_free(FileDescriptor *f) {
 274         assert(f);
 275
 276         while (f->windows)
 277                 window_free(f->windows);
 278
 279         if (f->cache)
 280                 assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
 281
 282         free(f);
 283 }
 284
 285 static FileDescriptor* fd_add(MMapCache *m, int fd) {
 286         FileDescriptor *f;
 287         int r;
 288
 289         assert(m);
 290         assert(fd >= 0);
 291
 292         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 293         if (f)
 294                 return f;
 295
 296         r = hashmap_ensure_allocated(&m->fds, NULL);
 297         if (r < 0)
 298                 return NULL;
 299
 300         f = new0(FileDescriptor, 1);
 301         if (!f)
 302                 return NULL;
 303
 304         f->cache = m;
 305         f->fd = fd;
 306
 307         r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
 308         if (r < 0) {
 309                 free(f);
 310                 return NULL;
 311         }
 312
 313         return f;
 314 }
 315
 316 static void mmap_cache_free(MMapCache *m) {
 317         Context *c;
 318         FileDescriptor *f;
 319
 320         assert(m);
 321
 322         while ((c = hashmap_first(m->contexts)))
 323                 context_free(c);
 324
 325         hashmap_free(m->contexts);
 326
 327         while ((f = hashmap_first(m->fds)))
 328                 fd_free(f);
 329
 330         hashmap_free(m->fds);
 331
 332         while (m->unused)
 333                 window_free(m->unused);
 334
 335         free(m);
 336 }
 337
 338 MMapCache* mmap_cache_unref(MMapCache *m) {
 339         assert(m);
 340         assert(m->n_ref > 0);
 341
 342         m->n_ref --;
 343         if (m->n_ref == 0)
 344                 mmap_cache_free(m);
 345
 346         return NULL;
 347 }
 348
 349 static int make_room(MMapCache *m) {
 350         assert(m);
 351
 352         if (!m->last_unused)
 353                 return 0;
 354
 355         window_free(m->last_unused);
 356         return 1;
 357 }
 358
 359 static int try_context(
 360                 MMapCache *m,
 361                 int fd,
 362                 int prot,
 363                 unsigned context,
 364                 bool keep_always,
 365                 uint64_t offset,
 366                 size_t size,
 367                 void **ret,
 368                 void **release_cookie) {
 369
 370         Context *c;
 371
 372         assert(m);
 373         assert(m->n_ref > 0);
 374         assert(fd >= 0);
 375         assert(size > 0);
 376
 377         c = hashmap_get(m->contexts, UINT_TO_PTR(context+1));
 378         if (!c)
 379                 return 0;
 380
 381         assert(c->id == context);
 382
 383         if (!c->window)
 384                 return 0;
 385
 386         if (!window_matches(c->window, fd, prot, offset, size)) {
 387
 388                 /* Drop the reference to the window, since it's unnecessary now */
 389                 context_detach_window(c);
 390                 return 0;
 391         }
 392
 393         c->window->keep_always += keep_always;
 394
 395         if (ret)
 396                 *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
 397         if (keep_always && release_cookie)
 398                 *release_cookie = c->window;
 399         return 1;
 400 }
 401
 402 static int find_mmap(
 403                 MMapCache *m,
 404                 int fd,
 405                 int prot,
 406                 unsigned context,
 407                 bool keep_always,
 408                 uint64_t offset,
 409                 size_t size,
 410                 void **ret,
 411                 void **release_cookie) {
 412
 413         FileDescriptor *f;
 414         Window *w;
 415         Context *c;
 416
 417         assert(m);
 418         assert(m->n_ref > 0);
 419         assert(fd >= 0);
 420         assert(size > 0);
 421
 422         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 423         if (!f)
 424                 return 0;
 425
 426         assert(f->fd == fd);
 427
 428         LIST_FOREACH(by_fd, w, f->windows)
 429                 if (window_matches(w, fd, prot, offset, size))
 430                         break;
 431
 432         if (!w)
 433                 return 0;
 434
 435         c = context_add(m, context);
 436         if (!c)
 437                 return -ENOMEM;
 438
 439         context_attach_window(c, w);
 440         w->keep_always += keep_always;
 441
 442         if (ret)
 443                 *ret = (uint8_t*) w->ptr + (offset - w->offset);
 444         if (keep_always && release_cookie)
 445                 *release_cookie = c->window;
 446         return 1;
 447 }
 448
 449 static int add_mmap(
 450                 MMapCache *m,
 451                 int fd,
 452                 int prot,
 453                 unsigned context,
 454                 bool keep_always,
 455                 uint64_t offset,
 456                 size_t size,
 457                 struct stat *st,
 458                 void **ret,
 459                 void **release_cookie) {
 460
 461         uint64_t woffset, wsize;
 462         Context *c;
 463         FileDescriptor *f;
 464         Window *w;
 465         void *d;
 466         int r;
 467
 468         assert(m);
 469         assert(m->n_ref > 0);
 470         assert(fd >= 0);
 471         assert(size > 0);
 472
 473         woffset = offset & ~((uint64_t) page_size() - 1ULL);
 474         wsize = size + (offset - woffset);
 475         wsize = PAGE_ALIGN(wsize);
 476
 477         if (wsize < WINDOW_SIZE) {
 478                 uint64_t delta;
 479
 480                 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
 481
 482                 if (delta > offset)
 483                         woffset = 0;
 484                 else
 485                         woffset -= delta;
 486
 487                 wsize = WINDOW_SIZE;
 488         }
 489
 490         if (st) {
 491                 /* Memory maps that are larger then the files
 492                    underneath have undefined behavior. Hence, clamp
 493                    things to the file size if we know it */
 494
 495                 if (woffset >= (uint64_t) st->st_size)
 496                         return -EADDRNOTAVAIL;
 497
 498                 if (woffset + wsize > (uint64_t) st->st_size)
 499                         wsize = PAGE_ALIGN(st->st_size - woffset);
 500         }
 501
 502         for (;;) {
 503                 d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
 504                 if (d != MAP_FAILED)
 505                         break;
 506                 if (errno != ENOMEM)
 507                         return -errno;
 508
 509                 r = make_room(m);
 510                 if (r < 0)
 511                         return r;
 512                 if (r == 0)
 513                         return -ENOMEM;
 514         }
 515
 516         c = context_add(m, context);
 517         if (!c)
 518                 goto outofmem;
 519
 520         f = fd_add(m, fd);
 521         if (!f)
 522                 goto outofmem;
 523
 524         w = window_add(m);
 525         if (!w)
 526                 goto outofmem;
 527
 528         w->keep_always = keep_always;
 529         w->ptr = d;
 530         w->offset = woffset;
 531         w->prot = prot;
 532         w->size = wsize;
 533         w->fd = f;
 534
 535         LIST_PREPEND(by_fd, f->windows, w);
 536
 537         context_detach_window(c);
 538         c->window = w;
 539         LIST_PREPEND(by_window, w->contexts, c);
 540
 541         if (ret)
 542                 *ret = (uint8_t*) w->ptr + (offset - w->offset);
 543         if (keep_always && release_cookie)
 544                 *release_cookie = c->window;
 545         return 1;
 546
 547 outofmem:
 548         munmap(d, wsize);
 549         return -ENOMEM;
 550 }
 551
 552 int mmap_cache_get(
 553                 MMapCache *m,
 554                 int fd,
 555                 int prot,
 556                 unsigned context,
 557                 bool keep_always,
 558                 uint64_t offset,
 559                 size_t size,
 560                 struct stat *st,
 561                 void **ret,
 562                 void **release_cookie) {
 563
 564         int r;
 565
 566         assert(m);
 567         assert(m->n_ref > 0);
 568         assert(fd >= 0);
 569         assert(size > 0);
 570
 571         /* Check whether the current context is the right one already */
 572         r = try_context(m, fd, prot, context, keep_always, offset, size, ret, release_cookie);
 573         if (r != 0) {
 574                 m->n_hit ++;
 575                 return r;
 576         }
 577
 578         /* Search for a matching mmap */
 579         r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret, release_cookie);
 580         if (r != 0) {
 581                 m->n_hit ++;
 582                 return r;
 583         }
 584
 585         m->n_missed++;
 586
 587         /* Create a new mmap */
 588         return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret, release_cookie);
 589 }
 590
 591 int mmap_cache_release(
 592                 MMapCache *m,
 593                 int fd,
 594                 void *release_cookie) {
 595
 596         FileDescriptor *f;
 597         Window *w;
 598
 599         assert(m);
 600         assert(m->n_ref > 0);
 601         assert(fd >= 0);
 602
 603         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 604         if (!f)
 605                 return -EBADF;
 606
 607         assert(f->fd == fd);
 608
 609         LIST_FOREACH(by_fd, w, f->windows)
 610                 if (w == release_cookie)
 611                         break;
 612
 613         if (!w)
 614                 return -ENOENT;
 615
 616         if (w->keep_always == 0)
 617                 return -ENOLCK;
 618
 619         w->keep_always -= 1;
 620         return 0;
 621 }
 622
 623 void mmap_cache_close_fd(MMapCache *m, int fd) {
 624         FileDescriptor *f;
 625
 626         assert(m);
 627         assert(fd >= 0);
 628
 629         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 630         if (!f)
 631                 return;
 632
 633         fd_free(f);
 634 }
 635
 636 void mmap_cache_close_context(MMapCache *m, unsigned context) {
 637         Context *c;
 638
 639         assert(m);
 640
 641         c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1));
 642         if (!c)
 643                 return;
 644
 645         context_free(c);
 646 }
 647
 648 unsigned mmap_cache_get_hit(MMapCache *m) {
 649         assert(m);
 650
 651         return m->n_hit;
 652 }
 653
 654 unsigned mmap_cache_get_missed(MMapCache *m) {
 655         assert(m);
 656
 657         return m->n_missed;
 658 }