src/journal/mmap-cache.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <stdlib.h>
  24 #include <sys/mman.h>
  25 #include <string.h>
  26
  27 #include "hashmap.h"
  28 #include "list.h"
  29 #include "log.h"
  30 #include "util.h"
  31 #include "macro.h"
  32 #include "mmap-cache.h"
  33
  34 typedef struct Window Window;
  35 typedef struct Context Context;
  36 typedef struct FileDescriptor FileDescriptor;
  37
  38 struct Window {
  39         MMapCache *cache;
  40
  41         bool keep_always;
  42         bool in_unused;
  43
  44         int prot;
  45         void *ptr;
  46         uint64_t offset;
  47         size_t size;
  48
  49         FileDescriptor *fd;
  50
  51         LIST_FIELDS(Window, by_fd);
  52         LIST_FIELDS(Window, unused);
  53
  54         LIST_HEAD(Context, contexts);
  55 };
  56
  57 struct Context {
  58         MMapCache *cache;
  59         unsigned id;
  60         Window *window;
  61
  62         LIST_FIELDS(Context, by_window);
  63 };
  64
  65 struct FileDescriptor {
  66         MMapCache *cache;
  67         int fd;
  68         LIST_HEAD(Window, windows);
  69 };
  70
  71 struct MMapCache {
  72         int n_ref;
  73         unsigned n_windows;
  74
  75         unsigned n_hit, n_missed;
  76
  77
  78         Hashmap *fds;
  79         Hashmap *contexts;
  80
  81         LIST_HEAD(Window, unused);
  82         Window *last_unused;
  83 };
  84
  85 #define WINDOWS_MIN 64
  86 #define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
  87
  88 MMapCache* mmap_cache_new(void) {
  89         MMapCache *m;
  90
  91         m = new0(MMapCache, 1);
  92         if (!m)
  93                 return NULL;
  94
  95         m->n_ref = 1;
  96         return m;
  97 }
  98
  99 MMapCache* mmap_cache_ref(MMapCache *m) {
 100         assert(m);
 101         assert(m->n_ref > 0);
 102
 103         m->n_ref ++;
 104         return m;
 105 }
 106
 107 static void window_unlink(Window *w) {
 108         Context *c;
 109
 110         assert(w);
 111
 112         if (w->ptr)
 113                 munmap(w->ptr, w->size);
 114
 115         if (w->fd)
 116                 LIST_REMOVE(by_fd, w->fd->windows, w);
 117
 118         if (w->in_unused) {
 119                 if (w->cache->last_unused == w)
 120                         w->cache->last_unused = w->unused_prev;
 121
 122                 LIST_REMOVE(unused, w->cache->unused, w);
 123         }
 124
 125         LIST_FOREACH(by_window, c, w->contexts) {
 126                 assert(c->window == w);
 127                 c->window = NULL;
 128         }
 129 }
 130
 131 static void window_free(Window *w) {
 132         assert(w);
 133
 134         window_unlink(w);
 135         w->cache->n_windows--;
 136         free(w);
 137 }
 138
 139 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
 140         assert(w);
 141         assert(fd >= 0);
 142         assert(size > 0);
 143
 144         return
 145                 w->fd &&
 146                 fd == w->fd->fd &&
 147                 prot == w->prot &&
 148                 offset >= w->offset &&
 149                 offset + size <= w->offset + w->size;
 150 }
 151
 152 static Window *window_add(MMapCache *m) {
 153         Window *w;
 154
 155         assert(m);
 156
 157         if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
 158
 159                 /* Allocate a new window */
 160                 w = new0(Window, 1);
 161                 if (!w)
 162                         return NULL;
 163                 m->n_windows++;
 164         } else {
 165
 166                 /* Reuse an existing one */
 167                 w = m->last_unused;
 168                 window_unlink(w);
 169                 zero(*w);
 170         }
 171
 172         w->cache = m;
 173         return w;
 174 }
 175
 176 static void context_detach_window(Context *c) {
 177         Window *w;
 178
 179         assert(c);
 180
 181         if (!c->window)
 182                 return;
 183
 184         w = c->window;
 185         c->window = NULL;
 186         LIST_REMOVE(by_window, w->contexts, c);
 187
 188         if (!w->contexts && !w->keep_always) {
 189                 /* Not used anymore? */
 190                 LIST_PREPEND(unused, c->cache->unused, w);
 191                 if (!c->cache->last_unused)
 192                         c->cache->last_unused = w;
 193
 194                 w->in_unused = true;
 195         }
 196 }
 197
 198 static void context_attach_window(Context *c, Window *w) {
 199         assert(c);
 200         assert(w);
 201
 202         if (c->window == w)
 203                 return;
 204
 205         context_detach_window(c);
 206
 207         if (w->in_unused) {
 208                 /* Used again? */
 209                 LIST_REMOVE(unused, c->cache->unused, w);
 210                 if (c->cache->last_unused == w)
 211                         c->cache->last_unused = w->unused_prev;
 212
 213                 w->in_unused = false;
 214         }
 215
 216         c->window = w;
 217         LIST_PREPEND(by_window, w->contexts, c);
 218 }
 219
 220 static Context *context_add(MMapCache *m, unsigned id) {
 221         Context *c;
 222         int r;
 223
 224         assert(m);
 225
 226         c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1));
 227         if (c)
 228                 return c;
 229
 230         r = hashmap_ensure_allocated(&m->contexts, trivial_hash_func, trivial_compare_func);
 231         if (r < 0)
 232                 return NULL;
 233
 234         c = new0(Context, 1);
 235         if (!c)
 236                 return NULL;
 237
 238         c->cache = m;
 239         c->id = id;
 240
 241         r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c);
 242         if (r < 0) {
 243                 free(c);
 244                 return NULL;
 245         }
 246
 247         return c;
 248 }
 249
 250 static void context_free(Context *c) {
 251         assert(c);
 252
 253         context_detach_window(c);
 254
 255         if (c->cache)
 256                 assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1)));
 257
 258         free(c);
 259 }
 260
 261 static void fd_free(FileDescriptor *f) {
 262         assert(f);
 263
 264         while (f->windows)
 265                 window_free(f->windows);
 266
 267         if (f->cache)
 268                 assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
 269
 270         free(f);
 271 }
 272
 273 static FileDescriptor* fd_add(MMapCache *m, int fd) {
 274         FileDescriptor *f;
 275         int r;
 276
 277         assert(m);
 278         assert(fd >= 0);
 279
 280         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 281         if (f)
 282                 return f;
 283
 284         r = hashmap_ensure_allocated(&m->fds, trivial_hash_func, trivial_compare_func);
 285         if (r < 0)
 286                 return NULL;
 287
 288         f = new0(FileDescriptor, 1);
 289         if (!f)
 290                 return NULL;
 291
 292         f->cache = m;
 293         f->fd = fd;
 294
 295         r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
 296         if (r < 0) {
 297                 free(f);
 298                 return NULL;
 299         }
 300
 301         return f;
 302 }
 303
 304 static void mmap_cache_free(MMapCache *m) {
 305         Context *c;
 306         FileDescriptor *f;
 307
 308         assert(m);
 309
 310         while ((c = hashmap_first(m->contexts)))
 311                 context_free(c);
 312
 313         hashmap_free(m->contexts);
 314
 315         while ((f = hashmap_first(m->fds)))
 316                 fd_free(f);
 317
 318         hashmap_free(m->fds);
 319
 320         while (m->unused)
 321                 window_free(m->unused);
 322
 323         free(m);
 324 }
 325
 326 MMapCache* mmap_cache_unref(MMapCache *m) {
 327         assert(m);
 328         assert(m->n_ref > 0);
 329
 330         m->n_ref --;
 331         if (m->n_ref == 0)
 332                 mmap_cache_free(m);
 333
 334         return NULL;
 335 }
 336
 337 static int make_room(MMapCache *m) {
 338         assert(m);
 339
 340         if (!m->last_unused)
 341                 return 0;
 342
 343         window_free(m->last_unused);
 344         return 1;
 345 }
 346
 347 static int try_context(
 348                 MMapCache *m,
 349                 int fd,
 350                 int prot,
 351                 unsigned context,
 352                 bool keep_always,
 353                 uint64_t offset,
 354                 size_t size,
 355                 void **ret) {
 356
 357         Context *c;
 358
 359         assert(m);
 360         assert(m->n_ref > 0);
 361         assert(fd >= 0);
 362         assert(size > 0);
 363         assert(ret);
 364
 365         c = hashmap_get(m->contexts, UINT_TO_PTR(context+1));
 366         if (!c)
 367                 return 0;
 368
 369         assert(c->id == context);
 370
 371         if (!c->window)
 372                 return 0;
 373
 374         if (!window_matches(c->window, fd, prot, offset, size)) {
 375
 376                 /* Drop the reference to the window, since it's unnecessary now */
 377                 context_detach_window(c);
 378                 return 0;
 379         }
 380
 381         c->window->keep_always = c->window->keep_always || keep_always;
 382
 383         *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
 384         return 1;
 385 }
 386
 387 static int find_mmap(
 388                 MMapCache *m,
 389                 int fd,
 390                 int prot,
 391                 unsigned context,
 392                 bool keep_always,
 393                 uint64_t offset,
 394                 size_t size,
 395                 void **ret) {
 396
 397         FileDescriptor *f;
 398         Window *w;
 399         Context *c;
 400
 401         assert(m);
 402         assert(m->n_ref > 0);
 403         assert(fd >= 0);
 404         assert(size > 0);
 405         assert(ret);
 406
 407         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 408         if (!f)
 409                 return 0;
 410
 411         assert(f->fd == fd);
 412
 413         LIST_FOREACH(by_fd, w, f->windows)
 414                 if (window_matches(w, fd, prot, offset, size))
 415                         break;
 416
 417         if (!w)
 418                 return 0;
 419
 420         c = context_add(m, context);
 421         if (!c)
 422                 return -ENOMEM;
 423
 424         context_attach_window(c, w);
 425         w->keep_always = w->keep_always || keep_always;
 426
 427         *ret = (uint8_t*) w->ptr + (offset - w->offset);
 428         return 1;
 429 }
 430
 431 static int add_mmap(
 432                 MMapCache *m,
 433                 int fd,
 434                 int prot,
 435                 unsigned context,
 436                 bool keep_always,
 437                 uint64_t offset,
 438                 size_t size,
 439                 struct stat *st,
 440                 void **ret) {
 441
 442         uint64_t woffset, wsize;
 443         Context *c;
 444         FileDescriptor *f;
 445         Window *w;
 446         void *d;
 447         int r;
 448
 449         assert(m);
 450         assert(m->n_ref > 0);
 451         assert(fd >= 0);
 452         assert(size > 0);
 453         assert(ret);
 454
 455         woffset = offset & ~((uint64_t) page_size() - 1ULL);
 456         wsize = size + (offset - woffset);
 457         wsize = PAGE_ALIGN(wsize);
 458
 459         if (wsize < WINDOW_SIZE) {
 460                 uint64_t delta;
 461
 462                 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
 463
 464                 if (delta > offset)
 465                         woffset = 0;
 466                 else
 467                         woffset -= delta;
 468
 469                 wsize = WINDOW_SIZE;
 470         }
 471
 472         if (st) {
 473                 /* Memory maps that are larger then the files
 474                    underneath have undefined behavior. Hence, clamp
 475                    things to the file size if we know it */
 476
 477                 if (woffset >= (uint64_t) st->st_size)
 478                         return -EADDRNOTAVAIL;
 479
 480                 if (woffset + wsize > (uint64_t) st->st_size)
 481                         wsize = PAGE_ALIGN(st->st_size - woffset);
 482         }
 483
 484         for (;;) {
 485                 d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
 486                 if (d != MAP_FAILED)
 487                         break;
 488                 if (errno != ENOMEM)
 489                         return -errno;
 490
 491                 r = make_room(m);
 492                 if (r < 0)
 493                         return r;
 494                 if (r == 0)
 495                         return -ENOMEM;
 496         }
 497
 498         c = context_add(m, context);
 499         if (!c)
 500                 return -ENOMEM;
 501
 502         f = fd_add(m, fd);
 503         if (!f)
 504                 return -ENOMEM;
 505
 506         w = window_add(m);
 507         if (!w)
 508                 return -ENOMEM;
 509
 510         w->keep_always = keep_always;
 511         w->ptr = d;
 512         w->offset = woffset;
 513         w->prot = prot;
 514         w->size = wsize;
 515         w->fd = f;
 516
 517         LIST_PREPEND(by_fd, f->windows, w);
 518
 519         context_detach_window(c);
 520         c->window = w;
 521         LIST_PREPEND(by_window, w->contexts, c);
 522
 523         *ret = (uint8_t*) w->ptr + (offset - w->offset);
 524         return 1;
 525 }
 526
 527 int mmap_cache_get(
 528                 MMapCache *m,
 529                 int fd,
 530                 int prot,
 531                 unsigned context,
 532                 bool keep_always,
 533                 uint64_t offset,
 534                 size_t size,
 535                 struct stat *st,
 536                 void **ret) {
 537
 538         int r;
 539
 540         assert(m);
 541         assert(m->n_ref > 0);
 542         assert(fd >= 0);
 543         assert(size > 0);
 544         assert(ret);
 545
 546         /* Check whether the current context is the right one already */
 547         r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
 548         if (r != 0) {
 549                 m->n_hit ++;
 550                 return r;
 551         }
 552
 553         /* Search for a matching mmap */
 554         r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
 555         if (r != 0) {
 556                 m->n_hit ++;
 557                 return r;
 558         }
 559
 560         m->n_missed++;
 561
 562         /* Create a new mmap */
 563         return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
 564 }
 565
 566 void mmap_cache_close_fd(MMapCache *m, int fd) {
 567         FileDescriptor *f;
 568
 569         assert(m);
 570         assert(fd >= 0);
 571
 572         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 573         if (!f)
 574                 return;
 575
 576         fd_free(f);
 577 }
 578
 579 void mmap_cache_close_context(MMapCache *m, unsigned context) {
 580         Context *c;
 581
 582         assert(m);
 583
 584         c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1));
 585         if (!c)
 586                 return;
 587
 588         context_free(c);
 589 }
 590
 591 unsigned mmap_cache_get_hit(MMapCache *m) {
 592         assert(m);
 593
 594         return m->n_hit;
 595 }
 596
 597 unsigned mmap_cache_get_missed(MMapCache *m) {
 598         assert(m);
 599
 600         return m->n_missed;
 601 }