src/journal/mmap-cache.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <stdlib.h>
  24 #include <sys/mman.h>
  25 #include <string.h>
  26
  27 #include "hashmap.h"
  28 #include "list.h"
  29 #include "log.h"
  30 #include "util.h"
  31 #include "macro.h"
  32 #include "mmap-cache.h"
  33
  34 typedef struct Window Window;
  35 typedef struct Context Context;
  36 typedef struct FileDescriptor FileDescriptor;
  37
  38 struct Window {
  39         MMapCache *cache;
  40
  41         unsigned keep_always;
  42         bool in_unused;
  43
  44         int prot;
  45         void *ptr;
  46         uint64_t offset;
  47         size_t size;
  48
  49         FileDescriptor *fd;
  50
  51         LIST_FIELDS(Window, by_fd);
  52         LIST_FIELDS(Window, unused);
  53
  54         LIST_HEAD(Context, contexts);
  55 };
  56
  57 struct Context {
  58         MMapCache *cache;
  59         unsigned id;
  60         Window *window;
  61
  62         LIST_FIELDS(Context, by_window);
  63 };
  64
  65 struct FileDescriptor {
  66         MMapCache *cache;
  67         int fd;
  68         LIST_HEAD(Window, windows);
  69 };
  70
  71 struct MMapCache {
  72         int n_ref;
  73         unsigned n_windows;
  74
  75         unsigned n_hit, n_missed;
  76
  77
  78         Hashmap *fds;
  79         Hashmap *contexts;
  80
  81         LIST_HEAD(Window, unused);
  82         Window *last_unused;
  83 };
  84
  85 #define WINDOWS_MIN 64
  86 #define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
  87
  88 MMapCache* mmap_cache_new(void) {
  89         MMapCache *m;
  90
  91         m = new0(MMapCache, 1);
  92         if (!m)
  93                 return NULL;
  94
  95         m->n_ref = 1;
  96         return m;
  97 }
  98
  99 MMapCache* mmap_cache_ref(MMapCache *m) {
 100         assert(m);
 101         assert(m->n_ref > 0);
 102
 103         m->n_ref ++;
 104         return m;
 105 }
 106
 107 static void window_unlink(Window *w) {
 108         Context *c;
 109
 110         assert(w);
 111
 112         if (w->ptr)
 113                 munmap(w->ptr, w->size);
 114
 115         if (w->fd)
 116                 LIST_REMOVE(by_fd, w->fd->windows, w);
 117
 118         if (w->in_unused) {
 119                 if (w->cache->last_unused == w)
 120                         w->cache->last_unused = w->unused_prev;
 121
 122                 LIST_REMOVE(unused, w->cache->unused, w);
 123         }
 124
 125         LIST_FOREACH(by_window, c, w->contexts) {
 126                 assert(c->window == w);
 127                 c->window = NULL;
 128         }
 129 }
 130
 131 static void window_free(Window *w) {
 132         assert(w);
 133
 134         window_unlink(w);
 135         w->cache->n_windows--;
 136         free(w);
 137 }
 138
 139 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
 140         assert(w);
 141         assert(fd >= 0);
 142         assert(size > 0);
 143
 144         return
 145                 w->fd &&
 146                 fd == w->fd->fd &&
 147                 prot == w->prot &&
 148                 offset >= w->offset &&
 149                 offset + size <= w->offset + w->size;
 150 }
 151
 152 static Window *window_add(MMapCache *m) {
 153         Window *w;
 154
 155         assert(m);
 156
 157         if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
 158
 159                 /* Allocate a new window */
 160                 w = new0(Window, 1);
 161                 if (!w)
 162                         return NULL;
 163                 m->n_windows++;
 164         } else {
 165
 166                 /* Reuse an existing one */
 167                 w = m->last_unused;
 168                 window_unlink(w);
 169                 zero(*w);
 170         }
 171
 172         w->cache = m;
 173         return w;
 174 }
 175
 176 static void context_detach_window(Context *c) {
 177         Window *w;
 178
 179         assert(c);
 180
 181         if (!c->window)
 182                 return;
 183
 184         w = c->window;
 185         c->window = NULL;
 186         LIST_REMOVE(by_window, w->contexts, c);
 187
 188         if (!w->contexts && w->keep_always == 0) {
 189                 /* Not used anymore? */
 190                 LIST_PREPEND(unused, c->cache->unused, w);
 191                 if (!c->cache->last_unused)
 192                         c->cache->last_unused = w;
 193
 194                 w->in_unused = true;
 195         }
 196 }
 197
 198 static void context_attach_window(Context *c, Window *w) {
 199         assert(c);
 200         assert(w);
 201
 202         if (c->window == w)
 203                 return;
 204
 205         context_detach_window(c);
 206
 207         if (w->in_unused) {
 208                 /* Used again? */
 209                 LIST_REMOVE(unused, c->cache->unused, w);
 210                 if (c->cache->last_unused == w)
 211                         c->cache->last_unused = w->unused_prev;
 212
 213                 w->in_unused = false;
 214         }
 215
 216         c->window = w;
 217         LIST_PREPEND(by_window, w->contexts, c);
 218 }
 219
 220 static Context *context_add(MMapCache *m, unsigned id) {
 221         Context *c;
 222         int r;
 223
 224         assert(m);
 225
 226         c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1));
 227         if (c)
 228                 return c;
 229
 230         r = hashmap_ensure_allocated(&m->contexts, trivial_hash_func, trivial_compare_func);
 231         if (r < 0)
 232                 return NULL;
 233
 234         c = new0(Context, 1);
 235         if (!c)
 236                 return NULL;
 237
 238         c->cache = m;
 239         c->id = id;
 240
 241         r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c);
 242         if (r < 0) {
 243                 free(c);
 244                 return NULL;
 245         }
 246
 247         return c;
 248 }
 249
 250 static void context_free(Context *c) {
 251         assert(c);
 252
 253         context_detach_window(c);
 254
 255         if (c->cache)
 256                 assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1)));
 257
 258         free(c);
 259 }
 260
 261 static void fd_free(FileDescriptor *f) {
 262         assert(f);
 263
 264         while (f->windows)
 265                 window_free(f->windows);
 266
 267         if (f->cache)
 268                 assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
 269
 270         free(f);
 271 }
 272
 273 static FileDescriptor* fd_add(MMapCache *m, int fd) {
 274         FileDescriptor *f;
 275         int r;
 276
 277         assert(m);
 278         assert(fd >= 0);
 279
 280         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 281         if (f)
 282                 return f;
 283
 284         r = hashmap_ensure_allocated(&m->fds, trivial_hash_func, trivial_compare_func);
 285         if (r < 0)
 286                 return NULL;
 287
 288         f = new0(FileDescriptor, 1);
 289         if (!f)
 290                 return NULL;
 291
 292         f->cache = m;
 293         f->fd = fd;
 294
 295         r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
 296         if (r < 0) {
 297                 free(f);
 298                 return NULL;
 299         }
 300
 301         return f;
 302 }
 303
 304 static void mmap_cache_free(MMapCache *m) {
 305         Context *c;
 306         FileDescriptor *f;
 307
 308         assert(m);
 309
 310         while ((c = hashmap_first(m->contexts)))
 311                 context_free(c);
 312
 313         hashmap_free(m->contexts);
 314
 315         while ((f = hashmap_first(m->fds)))
 316                 fd_free(f);
 317
 318         hashmap_free(m->fds);
 319
 320         while (m->unused)
 321                 window_free(m->unused);
 322
 323         free(m);
 324 }
 325
 326 MMapCache* mmap_cache_unref(MMapCache *m) {
 327         assert(m);
 328         assert(m->n_ref > 0);
 329
 330         m->n_ref --;
 331         if (m->n_ref == 0)
 332                 mmap_cache_free(m);
 333
 334         return NULL;
 335 }
 336
 337 static int make_room(MMapCache *m) {
 338         assert(m);
 339
 340         if (!m->last_unused)
 341                 return 0;
 342
 343         window_free(m->last_unused);
 344         return 1;
 345 }
 346
 347 static int try_context(
 348                 MMapCache *m,
 349                 int fd,
 350                 int prot,
 351                 unsigned context,
 352                 bool keep_always,
 353                 uint64_t offset,
 354                 size_t size,
 355                 void **ret) {
 356
 357         Context *c;
 358
 359         assert(m);
 360         assert(m->n_ref > 0);
 361         assert(fd >= 0);
 362         assert(size > 0);
 363
 364         c = hashmap_get(m->contexts, UINT_TO_PTR(context+1));
 365         if (!c)
 366                 return 0;
 367
 368         assert(c->id == context);
 369
 370         if (!c->window)
 371                 return 0;
 372
 373         if (!window_matches(c->window, fd, prot, offset, size)) {
 374
 375                 /* Drop the reference to the window, since it's unnecessary now */
 376                 context_detach_window(c);
 377                 return 0;
 378         }
 379
 380         c->window->keep_always += keep_always;
 381
 382         if (ret)
 383                 *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
 384         return 1;
 385 }
 386
 387 static int find_mmap(
 388                 MMapCache *m,
 389                 int fd,
 390                 int prot,
 391                 unsigned context,
 392                 bool keep_always,
 393                 uint64_t offset,
 394                 size_t size,
 395                 void **ret) {
 396
 397         FileDescriptor *f;
 398         Window *w;
 399         Context *c;
 400
 401         assert(m);
 402         assert(m->n_ref > 0);
 403         assert(fd >= 0);
 404         assert(size > 0);
 405
 406         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 407         if (!f)
 408                 return 0;
 409
 410         assert(f->fd == fd);
 411
 412         LIST_FOREACH(by_fd, w, f->windows)
 413                 if (window_matches(w, fd, prot, offset, size))
 414                         break;
 415
 416         if (!w)
 417                 return 0;
 418
 419         c = context_add(m, context);
 420         if (!c)
 421                 return -ENOMEM;
 422
 423         context_attach_window(c, w);
 424         w->keep_always += keep_always;
 425
 426         if (ret)
 427                 *ret = (uint8_t*) w->ptr + (offset - w->offset);
 428         return 1;
 429 }
 430
 431 static int add_mmap(
 432                 MMapCache *m,
 433                 int fd,
 434                 int prot,
 435                 unsigned context,
 436                 bool keep_always,
 437                 uint64_t offset,
 438                 size_t size,
 439                 struct stat *st,
 440                 void **ret) {
 441
 442         uint64_t woffset, wsize;
 443         Context *c;
 444         FileDescriptor *f;
 445         Window *w;
 446         void *d;
 447         int r;
 448
 449         assert(m);
 450         assert(m->n_ref > 0);
 451         assert(fd >= 0);
 452         assert(size > 0);
 453
 454         woffset = offset & ~((uint64_t) page_size() - 1ULL);
 455         wsize = size + (offset - woffset);
 456         wsize = PAGE_ALIGN(wsize);
 457
 458         if (wsize < WINDOW_SIZE) {
 459                 uint64_t delta;
 460
 461                 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
 462
 463                 if (delta > offset)
 464                         woffset = 0;
 465                 else
 466                         woffset -= delta;
 467
 468                 wsize = WINDOW_SIZE;
 469         }
 470
 471         if (st) {
 472                 /* Memory maps that are larger then the files
 473                    underneath have undefined behavior. Hence, clamp
 474                    things to the file size if we know it */
 475
 476                 if (woffset >= (uint64_t) st->st_size)
 477                         return -EADDRNOTAVAIL;
 478
 479                 if (woffset + wsize > (uint64_t) st->st_size)
 480                         wsize = PAGE_ALIGN(st->st_size - woffset);
 481         }
 482
 483         for (;;) {
 484                 d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
 485                 if (d != MAP_FAILED)
 486                         break;
 487                 if (errno != ENOMEM)
 488                         return -errno;
 489
 490                 r = make_room(m);
 491                 if (r < 0)
 492                         return r;
 493                 if (r == 0)
 494                         return -ENOMEM;
 495         }
 496
 497         c = context_add(m, context);
 498         if (!c)
 499                 return -ENOMEM;
 500
 501         f = fd_add(m, fd);
 502         if (!f)
 503                 return -ENOMEM;
 504
 505         w = window_add(m);
 506         if (!w)
 507                 return -ENOMEM;
 508
 509         w->keep_always = keep_always;
 510         w->ptr = d;
 511         w->offset = woffset;
 512         w->prot = prot;
 513         w->size = wsize;
 514         w->fd = f;
 515
 516         LIST_PREPEND(by_fd, f->windows, w);
 517
 518         context_detach_window(c);
 519         c->window = w;
 520         LIST_PREPEND(by_window, w->contexts, c);
 521
 522         if (ret)
 523                 *ret = (uint8_t*) w->ptr + (offset - w->offset);
 524         return 1;
 525 }
 526
 527 int mmap_cache_get(
 528                 MMapCache *m,
 529                 int fd,
 530                 int prot,
 531                 unsigned context,
 532                 bool keep_always,
 533                 uint64_t offset,
 534                 size_t size,
 535                 struct stat *st,
 536                 void **ret) {
 537
 538         int r;
 539
 540         assert(m);
 541         assert(m->n_ref > 0);
 542         assert(fd >= 0);
 543         assert(size > 0);
 544
 545         /* Check whether the current context is the right one already */
 546         r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
 547         if (r != 0) {
 548                 m->n_hit ++;
 549                 return r;
 550         }
 551
 552         /* Search for a matching mmap */
 553         r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
 554         if (r != 0) {
 555                 m->n_hit ++;
 556                 return r;
 557         }
 558
 559         m->n_missed++;
 560
 561         /* Create a new mmap */
 562         return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
 563 }
 564
 565 int mmap_cache_release(
 566                 MMapCache *m,
 567                 int fd,
 568                 int prot,
 569                 unsigned context,
 570                 uint64_t offset,
 571                 size_t size) {
 572
 573         FileDescriptor *f;
 574         Window *w;
 575
 576         assert(m);
 577         assert(m->n_ref > 0);
 578         assert(fd >= 0);
 579         assert(size > 0);
 580
 581         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 582         if (!f)
 583                 return -EBADF;
 584
 585         assert(f->fd == fd);
 586
 587         LIST_FOREACH(by_fd, w, f->windows)
 588                 if (window_matches(w, fd, prot, offset, size))
 589                         break;
 590
 591         if (!w)
 592                 return -ENOENT;
 593
 594         if (w->keep_always == 0)
 595                 return -ENOLCK;
 596
 597         w->keep_always -= 1;
 598         return 0;
 599 }
 600
 601 void mmap_cache_close_fd(MMapCache *m, int fd) {
 602         FileDescriptor *f;
 603
 604         assert(m);
 605         assert(fd >= 0);
 606
 607         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 608         if (!f)
 609                 return;
 610
 611         fd_free(f);
 612 }
 613
 614 void mmap_cache_close_context(MMapCache *m, unsigned context) {
 615         Context *c;
 616
 617         assert(m);
 618
 619         c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1));
 620         if (!c)
 621                 return;
 622
 623         context_free(c);
 624 }
 625
 626 unsigned mmap_cache_get_hit(MMapCache *m) {
 627         assert(m);
 628
 629         return m->n_hit;
 630 }
 631
 632 unsigned mmap_cache_get_missed(MMapCache *m) {
 633         assert(m);
 634
 635         return m->n_missed;
 636 }