src/journal/mmap-cache.c

   1 /***
   2   This file is part of systemd.
   3
   4   Copyright 2012 Lennart Poettering
   5
   6   systemd is free software; you can redistribute it and/or modify it
   7   under the terms of the GNU Lesser General Public License as published by
   8   the Free Software Foundation; either version 2.1 of the License, or
   9   (at your option) any later version.
  10
  11   systemd is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14   Lesser General Public License for more details.
  15
  16   You should have received a copy of the GNU Lesser General Public License
  17   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  18 ***/
  19
  20 #include <errno.h>
  21 #include <stdlib.h>
  22 #include <sys/mman.h>
  23
  24 #include "alloc-util.h"
  25 #include "fd-util.h"
  26 #include "hashmap.h"
  27 #include "list.h"
  28 #include "log.h"
  29 #include "macro.h"
  30 #include "mmap-cache.h"
  31 #include "sigbus.h"
  32 #include "util.h"
  33
  34 typedef struct Window Window;
  35 typedef struct Context Context;
  36 typedef struct FileDescriptor FileDescriptor;
  37
  38 struct Window {
  39         MMapCache *cache;
  40
  41         bool invalidated:1;
  42         bool keep_always:1;
  43         bool in_unused:1;
  44
  45         int prot;
  46         void *ptr;
  47         uint64_t offset;
  48         size_t size;
  49
  50         FileDescriptor *fd;
  51
  52         LIST_FIELDS(Window, by_fd);
  53         LIST_FIELDS(Window, unused);
  54
  55         LIST_HEAD(Context, contexts);
  56 };
  57
  58 struct Context {
  59         MMapCache *cache;
  60         unsigned id;
  61         Window *window;
  62
  63         LIST_FIELDS(Context, by_window);
  64 };
  65
  66 struct FileDescriptor {
  67         MMapCache *cache;
  68         int fd;
  69         bool sigbus;
  70         LIST_HEAD(Window, windows);
  71 };
  72
  73 struct MMapCache {
  74         int n_ref;
  75         unsigned n_windows;
  76
  77         unsigned n_hit, n_missed;
  78
  79         Hashmap *fds;
  80         Context *contexts[MMAP_CACHE_MAX_CONTEXTS];
  81
  82         LIST_HEAD(Window, unused);
  83         Window *last_unused;
  84 };
  85
  86 #define WINDOWS_MIN 64
  87
  88 #ifdef ENABLE_DEBUG_MMAP_CACHE
  89 /* Tiny windows increase mmap activity and the chance of exposing unsafe use. */
  90 # define WINDOW_SIZE (page_size())
  91 #else
  92 # define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
  93 #endif
  94
  95 MMapCache* mmap_cache_new(void) {
  96         MMapCache *m;
  97
  98         m = new0(MMapCache, 1);
  99         if (!m)
 100                 return NULL;
 101
 102         m->n_ref = 1;
 103         return m;
 104 }
 105
 106 MMapCache* mmap_cache_ref(MMapCache *m) {
 107         assert(m);
 108         assert(m->n_ref > 0);
 109
 110         m->n_ref++;
 111         return m;
 112 }
 113
 114 static void window_unlink(Window *w) {
 115         Context *c;
 116
 117         assert(w);
 118
 119         if (w->ptr)
 120                 munmap(w->ptr, w->size);
 121
 122         if (w->fd)
 123                 LIST_REMOVE(by_fd, w->fd->windows, w);
 124
 125         if (w->in_unused) {
 126                 if (w->cache->last_unused == w)
 127                         w->cache->last_unused = w->unused_prev;
 128
 129                 LIST_REMOVE(unused, w->cache->unused, w);
 130         }
 131
 132         LIST_FOREACH(by_window, c, w->contexts) {
 133                 assert(c->window == w);
 134                 c->window = NULL;
 135         }
 136 }
 137
 138 static void window_invalidate(Window *w) {
 139         assert(w);
 140
 141         if (w->invalidated)
 142                 return;
 143
 144         /* Replace the window with anonymous pages. This is useful
 145          * when we hit a SIGBUS and want to make sure the file cannot
 146          * trigger any further SIGBUS, possibly overrunning the sigbus
 147          * queue. */
 148
 149         assert_se(mmap(w->ptr, w->size, w->prot, MAP_PRIVATE|MAP_ANONYMOUS|MAP_FIXED, -1, 0) == w->ptr);
 150         w->invalidated = true;
 151 }
 152
 153 static void window_free(Window *w) {
 154         assert(w);
 155
 156         window_unlink(w);
 157         w->cache->n_windows--;
 158         free(w);
 159 }
 160
 161 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
 162         assert(w);
 163         assert(fd >= 0);
 164         assert(size > 0);
 165
 166         return
 167                 w->fd &&
 168                 fd == w->fd->fd &&
 169                 prot == w->prot &&
 170                 offset >= w->offset &&
 171                 offset + size <= w->offset + w->size;
 172 }
 173
 174 static Window *window_add(MMapCache *m, FileDescriptor *fd, int prot, bool keep_always, uint64_t offset, size_t size, void *ptr) {
 175         Window *w;
 176
 177         assert(m);
 178         assert(fd);
 179
 180         if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
 181
 182                 /* Allocate a new window */
 183                 w = new0(Window, 1);
 184                 if (!w)
 185                         return NULL;
 186                 m->n_windows++;
 187         } else {
 188
 189                 /* Reuse an existing one */
 190                 w = m->last_unused;
 191                 window_unlink(w);
 192                 zero(*w);
 193         }
 194
 195         w->cache = m;
 196         w->fd = fd;
 197         w->prot = prot;
 198         w->keep_always = keep_always;
 199         w->offset = offset;
 200         w->size = size;
 201         w->ptr = ptr;
 202
 203         LIST_PREPEND(by_fd, fd->windows, w);
 204
 205         return w;
 206 }
 207
 208 static void context_detach_window(Context *c) {
 209         Window *w;
 210
 211         assert(c);
 212
 213         if (!c->window)
 214                 return;
 215
 216         w = c->window;
 217         c->window = NULL;
 218         LIST_REMOVE(by_window, w->contexts, c);
 219
 220         if (!w->contexts && !w->keep_always) {
 221                 /* Not used anymore? */
 222 #ifdef ENABLE_DEBUG_MMAP_CACHE
 223                 /* Unmap unused windows immediately to expose use-after-unmap
 224                  * by SIGSEGV. */
 225                 window_free(w);
 226 #else
 227                 LIST_PREPEND(unused, c->cache->unused, w);
 228                 if (!c->cache->last_unused)
 229                         c->cache->last_unused = w;
 230
 231                 w->in_unused = true;
 232 #endif
 233         }
 234 }
 235
 236 static void context_attach_window(Context *c, Window *w) {
 237         assert(c);
 238         assert(w);
 239
 240         if (c->window == w)
 241                 return;
 242
 243         context_detach_window(c);
 244
 245         if (w->in_unused) {
 246                 /* Used again? */
 247                 LIST_REMOVE(unused, c->cache->unused, w);
 248                 if (c->cache->last_unused == w)
 249                         c->cache->last_unused = w->unused_prev;
 250
 251                 w->in_unused = false;
 252         }
 253
 254         c->window = w;
 255         LIST_PREPEND(by_window, w->contexts, c);
 256 }
 257
 258 static Context *context_add(MMapCache *m, unsigned id) {
 259         Context *c;
 260
 261         assert(m);
 262
 263         c = m->contexts[id];
 264         if (c)
 265                 return c;
 266
 267         c = new0(Context, 1);
 268         if (!c)
 269                 return NULL;
 270
 271         c->cache = m;
 272         c->id = id;
 273
 274         assert(!m->contexts[id]);
 275         m->contexts[id] = c;
 276
 277         return c;
 278 }
 279
 280 static void context_free(Context *c) {
 281         assert(c);
 282
 283         context_detach_window(c);
 284
 285         if (c->cache) {
 286                 assert(c->cache->contexts[c->id] == c);
 287                 c->cache->contexts[c->id] = NULL;
 288         }
 289
 290         free(c);
 291 }
 292
 293 static void fd_free(FileDescriptor *f) {
 294         assert(f);
 295
 296         while (f->windows)
 297                 window_free(f->windows);
 298
 299         if (f->cache)
 300                 assert_se(hashmap_remove(f->cache->fds, FD_TO_PTR(f->fd)));
 301
 302         free(f);
 303 }
 304
 305 static FileDescriptor* fd_add(MMapCache *m, int fd) {
 306         FileDescriptor *f;
 307         int r;
 308
 309         assert(m);
 310         assert(fd >= 0);
 311
 312         f = hashmap_get(m->fds, FD_TO_PTR(fd));
 313         if (f)
 314                 return f;
 315
 316         r = hashmap_ensure_allocated(&m->fds, NULL);
 317         if (r < 0)
 318                 return NULL;
 319
 320         f = new0(FileDescriptor, 1);
 321         if (!f)
 322                 return NULL;
 323
 324         f->cache = m;
 325         f->fd = fd;
 326
 327         r = hashmap_put(m->fds, FD_TO_PTR(fd), f);
 328         if (r < 0)
 329                 return mfree(f);
 330
 331         return f;
 332 }
 333
 334 static void mmap_cache_free(MMapCache *m) {
 335         FileDescriptor *f;
 336         int i;
 337
 338         assert(m);
 339
 340         for (i = 0; i < MMAP_CACHE_MAX_CONTEXTS; i++)
 341                 if (m->contexts[i])
 342                         context_free(m->contexts[i]);
 343
 344         while ((f = hashmap_first(m->fds)))
 345                 fd_free(f);
 346
 347         hashmap_free(m->fds);
 348
 349         while (m->unused)
 350                 window_free(m->unused);
 351
 352         free(m);
 353 }
 354
 355 MMapCache* mmap_cache_unref(MMapCache *m) {
 356
 357         if (!m)
 358                 return NULL;
 359
 360         assert(m->n_ref > 0);
 361
 362         m->n_ref--;
 363         if (m->n_ref == 0)
 364                 mmap_cache_free(m);
 365
 366         return NULL;
 367 }
 368
 369 static int make_room(MMapCache *m) {
 370         assert(m);
 371
 372         if (!m->last_unused)
 373                 return 0;
 374
 375         window_free(m->last_unused);
 376         return 1;
 377 }
 378
 379 static int try_context(
 380                 MMapCache *m,
 381                 int fd,
 382                 int prot,
 383                 unsigned context,
 384                 bool keep_always,
 385                 uint64_t offset,
 386                 size_t size,
 387                 void **ret) {
 388
 389         Context *c;
 390
 391         assert(m);
 392         assert(m->n_ref > 0);
 393         assert(fd >= 0);
 394         assert(size > 0);
 395         assert(ret);
 396
 397         c = m->contexts[context];
 398         if (!c)
 399                 return 0;
 400
 401         assert(c->id == context);
 402
 403         if (!c->window)
 404                 return 0;
 405
 406         if (!window_matches(c->window, fd, prot, offset, size)) {
 407
 408                 /* Drop the reference to the window, since it's unnecessary now */
 409                 context_detach_window(c);
 410                 return 0;
 411         }
 412
 413         if (c->window->fd->sigbus)
 414                 return -EIO;
 415
 416         c->window->keep_always = c->window->keep_always || keep_always;
 417
 418         *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
 419         return 1;
 420 }
 421
 422 static int find_mmap(
 423                 MMapCache *m,
 424                 int fd,
 425                 int prot,
 426                 unsigned context,
 427                 bool keep_always,
 428                 uint64_t offset,
 429                 size_t size,
 430                 void **ret) {
 431
 432         FileDescriptor *f;
 433         Window *w;
 434         Context *c;
 435
 436         assert(m);
 437         assert(m->n_ref > 0);
 438         assert(fd >= 0);
 439         assert(size > 0);
 440
 441         f = hashmap_get(m->fds, FD_TO_PTR(fd));
 442         if (!f)
 443                 return 0;
 444
 445         assert(f->fd == fd);
 446
 447         if (f->sigbus)
 448                 return -EIO;
 449
 450         LIST_FOREACH(by_fd, w, f->windows)
 451                 if (window_matches(w, fd, prot, offset, size))
 452                         break;
 453
 454         if (!w)
 455                 return 0;
 456
 457         c = context_add(m, context);
 458         if (!c)
 459                 return -ENOMEM;
 460
 461         context_attach_window(c, w);
 462         w->keep_always = w->keep_always || keep_always;
 463
 464         *ret = (uint8_t*) w->ptr + (offset - w->offset);
 465         return 1;
 466 }
 467
 468 static int mmap_try_harder(MMapCache *m, void *addr, int fd, int prot, int flags, uint64_t offset, size_t size, void **res) {
 469         void *ptr;
 470
 471         assert(m);
 472         assert(fd >= 0);
 473         assert(res);
 474
 475         for (;;) {
 476                 int r;
 477
 478                 ptr = mmap(addr, size, prot, flags, fd, offset);
 479                 if (ptr != MAP_FAILED)
 480                         break;
 481                 if (errno != ENOMEM)
 482                         return negative_errno();
 483
 484                 r = make_room(m);
 485                 if (r < 0)
 486                         return r;
 487                 if (r == 0)
 488                         return -ENOMEM;
 489         }
 490
 491         *res = ptr;
 492         return 0;
 493 }
 494
 495 static int add_mmap(
 496                 MMapCache *m,
 497                 int fd,
 498                 int prot,
 499                 unsigned context,
 500                 bool keep_always,
 501                 uint64_t offset,
 502                 size_t size,
 503                 struct stat *st,
 504                 void **ret) {
 505
 506         uint64_t woffset, wsize;
 507         Context *c;
 508         FileDescriptor *f;
 509         Window *w;
 510         void *d;
 511         int r;
 512
 513         assert(m);
 514         assert(m->n_ref > 0);
 515         assert(fd >= 0);
 516         assert(size > 0);
 517         assert(ret);
 518
 519         woffset = offset & ~((uint64_t) page_size() - 1ULL);
 520         wsize = size + (offset - woffset);
 521         wsize = PAGE_ALIGN(wsize);
 522
 523         if (wsize < WINDOW_SIZE) {
 524                 uint64_t delta;
 525
 526                 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
 527
 528                 if (delta > offset)
 529                         woffset = 0;
 530                 else
 531                         woffset -= delta;
 532
 533                 wsize = WINDOW_SIZE;
 534         }
 535
 536         if (st) {
 537                 /* Memory maps that are larger then the files
 538                    underneath have undefined behavior. Hence, clamp
 539                    things to the file size if we know it */
 540
 541                 if (woffset >= (uint64_t) st->st_size)
 542                         return -EADDRNOTAVAIL;
 543
 544                 if (woffset + wsize > (uint64_t) st->st_size)
 545                         wsize = PAGE_ALIGN(st->st_size - woffset);
 546         }
 547
 548         r = mmap_try_harder(m, NULL, fd, prot, MAP_SHARED, woffset, wsize, &d);
 549         if (r < 0)
 550                 return r;
 551
 552         c = context_add(m, context);
 553         if (!c)
 554                 goto outofmem;
 555
 556         f = fd_add(m, fd);
 557         if (!f)
 558                 goto outofmem;
 559
 560         w = window_add(m, f, prot, keep_always, woffset, wsize, d);
 561         if (!w)
 562                 goto outofmem;
 563
 564         context_detach_window(c);
 565         c->window = w;
 566         LIST_PREPEND(by_window, w->contexts, c);
 567
 568         *ret = (uint8_t*) w->ptr + (offset - w->offset);
 569         return 1;
 570
 571 outofmem:
 572         (void) munmap(d, wsize);
 573         return -ENOMEM;
 574 }
 575
 576 int mmap_cache_get(
 577                 MMapCache *m,
 578                 int fd,
 579                 int prot,
 580                 unsigned context,
 581                 bool keep_always,
 582                 uint64_t offset,
 583                 size_t size,
 584                 struct stat *st,
 585                 void **ret) {
 586
 587         int r;
 588
 589         assert(m);
 590         assert(m->n_ref > 0);
 591         assert(fd >= 0);
 592         assert(size > 0);
 593         assert(ret);
 594         assert(context < MMAP_CACHE_MAX_CONTEXTS);
 595
 596         /* Check whether the current context is the right one already */
 597         r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
 598         if (r != 0) {
 599                 m->n_hit++;
 600                 return r;
 601         }
 602
 603         /* Search for a matching mmap */
 604         r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
 605         if (r != 0) {
 606                 m->n_hit++;
 607                 return r;
 608         }
 609
 610         m->n_missed++;
 611
 612         /* Create a new mmap */
 613         return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
 614 }
 615
 616 unsigned mmap_cache_get_hit(MMapCache *m) {
 617         assert(m);
 618
 619         return m->n_hit;
 620 }
 621
 622 unsigned mmap_cache_get_missed(MMapCache *m) {
 623         assert(m);
 624
 625         return m->n_missed;
 626 }
 627
 628 static void mmap_cache_process_sigbus(MMapCache *m) {
 629         bool found = false;
 630         FileDescriptor *f;
 631         Iterator i;
 632         int r;
 633
 634         assert(m);
 635
 636         /* Iterate through all triggered pages and mark their files as
 637          * invalidated */
 638         for (;;) {
 639                 bool ours;
 640                 void *addr;
 641
 642                 r = sigbus_pop(&addr);
 643                 if (_likely_(r == 0))
 644                         break;
 645                 if (r < 0) {
 646                         log_error_errno(r, "SIGBUS handling failed: %m");
 647                         abort();
 648                 }
 649
 650                 ours = false;
 651                 HASHMAP_FOREACH(f, m->fds, i) {
 652                         Window *w;
 653
 654                         LIST_FOREACH(by_fd, w, f->windows) {
 655                                 if ((uint8_t*) addr >= (uint8_t*) w->ptr &&
 656                                     (uint8_t*) addr < (uint8_t*) w->ptr + w->size) {
 657                                         found = ours = f->sigbus = true;
 658                                         break;
 659                                 }
 660                         }
 661
 662                         if (ours)
 663                                 break;
 664                 }
 665
 666                 /* Didn't find a matching window, give up */
 667                 if (!ours) {
 668                         log_error("Unknown SIGBUS page, aborting.");
 669                         abort();
 670                 }
 671         }
 672
 673         /* The list of triggered pages is now empty. Now, let's remap
 674          * all windows of the triggered file to anonymous maps, so
 675          * that no page of the file in question is triggered again, so
 676          * that we can be sure not to hit the queue size limit. */
 677         if (_likely_(!found))
 678                 return;
 679
 680         HASHMAP_FOREACH(f, m->fds, i) {
 681                 Window *w;
 682
 683                 if (!f->sigbus)
 684                         continue;
 685
 686                 LIST_FOREACH(by_fd, w, f->windows)
 687                         window_invalidate(w);
 688         }
 689 }
 690
 691 bool mmap_cache_got_sigbus(MMapCache *m, int fd) {
 692         FileDescriptor *f;
 693
 694         assert(m);
 695         assert(fd >= 0);
 696
 697         mmap_cache_process_sigbus(m);
 698
 699         f = hashmap_get(m->fds, FD_TO_PTR(fd));
 700         if (!f)
 701                 return false;
 702
 703         return f->sigbus;
 704 }
 705
 706 void mmap_cache_close_fd(MMapCache *m, int fd) {
 707         FileDescriptor *f;
 708
 709         assert(m);
 710         assert(fd >= 0);
 711
 712         /* Make sure that any queued SIGBUS are first dispatched, so
 713          * that we don't end up with a SIGBUS entry we cannot relate
 714          * to any existing memory map */
 715
 716         mmap_cache_process_sigbus(m);
 717
 718         f = hashmap_get(m->fds, FD_TO_PTR(fd));
 719         if (!f)
 720                 return;
 721
 722         fd_free(f);
 723 }