src/journal/mmap-cache.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <stdlib.h>
  24 #include <sys/mman.h>
  25 #include <string.h>
  26
  27 #include "hashmap.h"
  28 #include "list.h"
  29 #include "log.h"
  30 #include "util.h"
  31 #include "macro.h"
  32 #include "mmap-cache.h"
  33
  34 typedef struct Window Window;
  35 typedef struct Context Context;
  36 typedef struct FileDescriptor FileDescriptor;
  37
  38 struct Window {
  39         MMapCache *cache;
  40
  41         bool keep_always;
  42         bool in_unused;
  43
  44         int prot;
  45         void *ptr;
  46         uint64_t offset;
  47         size_t size;
  48
  49         FileDescriptor *fd;
  50
  51         LIST_FIELDS(Window, by_fd);
  52         LIST_FIELDS(Window, unused);
  53
  54         LIST_HEAD(Context, contexts);
  55 };
  56
  57 struct Context {
  58         MMapCache *cache;
  59         unsigned id;
  60         Window *window;
  61
  62         LIST_FIELDS(Context, by_window);
  63 };
  64
  65 struct FileDescriptor {
  66         MMapCache *cache;
  67         int fd;
  68         LIST_HEAD(Window, windows);
  69 };
  70
  71 struct MMapCache {
  72         int n_ref;
  73         unsigned n_windows;
  74
  75         Hashmap *fds;
  76         Hashmap *contexts;
  77
  78         LIST_HEAD(Window, unused);
  79         Window *last_unused;
  80 };
  81
  82 #define WINDOWS_MIN 64
  83 #define WINDOW_SIZE (8ULL*1024ULL*1024ULL)
  84
  85 MMapCache* mmap_cache_new(void) {
  86         MMapCache *m;
  87
  88         m = new0(MMapCache, 1);
  89         if (!m)
  90                 return NULL;
  91
  92         m->n_ref = 1;
  93         return m;
  94 }
  95
  96 MMapCache* mmap_cache_ref(MMapCache *m) {
  97         assert(m);
  98         assert(m->n_ref > 0);
  99
 100         m->n_ref ++;
 101         return m;
 102 }
 103
 104 static void window_unlink(Window *w) {
 105         Context *c;
 106
 107         assert(w);
 108
 109         if (w->ptr)
 110                 munmap(w->ptr, w->size);
 111
 112         if (w->fd)
 113                 LIST_REMOVE(Window, by_fd, w->fd->windows, w);
 114
 115         if (w->in_unused) {
 116                 if (w->cache->last_unused == w)
 117                         w->cache->last_unused = w->unused_prev;
 118
 119                 LIST_REMOVE(Window, unused, w->cache->unused, w);
 120         }
 121
 122         LIST_FOREACH(by_window, c, w->contexts) {
 123                 assert(c->window == w);
 124                 c->window = NULL;
 125         }
 126 }
 127
 128 static void window_free(Window *w) {
 129         assert(w);
 130
 131         window_unlink(w);
 132         w->cache->n_windows--;
 133         free(w);
 134 }
 135
 136 _pure_ static bool window_matches(Window *w, int fd, int prot, uint64_t offset, size_t size) {
 137         assert(w);
 138         assert(fd >= 0);
 139         assert(size > 0);
 140
 141         return
 142                 w->fd &&
 143                 fd == w->fd->fd &&
 144                 prot == w->prot &&
 145                 offset >= w->offset &&
 146                 offset + size <= w->offset + w->size;
 147 }
 148
 149 static Window *window_add(MMapCache *m) {
 150         Window *w;
 151
 152         assert(m);
 153
 154         if (!m->last_unused || m->n_windows <= WINDOWS_MIN) {
 155
 156                 /* Allocate a new window */
 157                 w = new0(Window, 1);
 158                 if (!w)
 159                         return NULL;
 160                 m->n_windows++;
 161         } else {
 162
 163                 /* Reuse an existing one */
 164                 w = m->last_unused;
 165                 window_unlink(w);
 166                 zero(*w);
 167         }
 168
 169         w->cache = m;
 170         return w;
 171 }
 172
 173 static void context_detach_window(Context *c) {
 174         Window *w;
 175
 176         assert(c);
 177
 178         if (!c->window)
 179                 return;
 180
 181         w = c->window;
 182         c->window = NULL;
 183         LIST_REMOVE(Context, by_window, w->contexts, c);
 184
 185         if (!w->contexts && !w->keep_always) {
 186                 /* Not used anymore? */
 187                 LIST_PREPEND(Window, unused, c->cache->unused, w);
 188                 if (!c->cache->last_unused)
 189                         c->cache->last_unused = w;
 190
 191                 w->in_unused = true;
 192         }
 193 }
 194
 195 static void context_attach_window(Context *c, Window *w) {
 196         assert(c);
 197         assert(w);
 198
 199         if (c->window == w)
 200                 return;
 201
 202         context_detach_window(c);
 203
 204         if (w->in_unused) {
 205                 /* Used again? */
 206                 LIST_REMOVE(Window, unused, c->cache->unused, w);
 207                 if (c->cache->last_unused == w)
 208                         c->cache->last_unused = w->unused_prev;
 209
 210                 w->in_unused = false;
 211         }
 212
 213         c->window = w;
 214         LIST_PREPEND(Context, by_window, w->contexts, c);
 215 }
 216
 217 static Context *context_add(MMapCache *m, unsigned id) {
 218         Context *c;
 219         int r;
 220
 221         assert(m);
 222
 223         c = hashmap_get(m->contexts, UINT_TO_PTR(id + 1));
 224         if (c)
 225                 return c;
 226
 227         r = hashmap_ensure_allocated(&m->contexts, trivial_hash_func, trivial_compare_func);
 228         if (r < 0)
 229                 return NULL;
 230
 231         c = new0(Context, 1);
 232         if (!c)
 233                 return NULL;
 234
 235         c->cache = m;
 236         c->id = id;
 237
 238         r = hashmap_put(m->contexts, UINT_TO_PTR(id + 1), c);
 239         if (r < 0) {
 240                 free(c);
 241                 return NULL;
 242         }
 243
 244         return c;
 245 }
 246
 247 static void context_free(Context *c) {
 248         assert(c);
 249
 250         context_detach_window(c);
 251
 252         if (c->cache)
 253                 assert_se(hashmap_remove(c->cache->contexts, UINT_TO_PTR(c->id + 1)));
 254
 255         free(c);
 256 }
 257
 258 static void fd_free(FileDescriptor *f) {
 259         assert(f);
 260
 261         while (f->windows)
 262                 window_free(f->windows);
 263
 264         if (f->cache)
 265                 assert_se(hashmap_remove(f->cache->fds, INT_TO_PTR(f->fd + 1)));
 266
 267         free(f);
 268 }
 269
 270 static FileDescriptor* fd_add(MMapCache *m, int fd) {
 271         FileDescriptor *f;
 272         int r;
 273
 274         assert(m);
 275         assert(fd >= 0);
 276
 277         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 278         if (f)
 279                 return f;
 280
 281         r = hashmap_ensure_allocated(&m->fds, trivial_hash_func, trivial_compare_func);
 282         if (r < 0)
 283                 return NULL;
 284
 285         f = new0(FileDescriptor, 1);
 286         if (!f)
 287                 return NULL;
 288
 289         f->cache = m;
 290         f->fd = fd;
 291
 292         r = hashmap_put(m->fds, UINT_TO_PTR(fd + 1), f);
 293         if (r < 0) {
 294                 free(f);
 295                 return NULL;
 296         }
 297
 298         return f;
 299 }
 300
 301 static void mmap_cache_free(MMapCache *m) {
 302         Context *c;
 303         FileDescriptor *f;
 304
 305         assert(m);
 306
 307         while ((c = hashmap_first(m->contexts)))
 308                 context_free(c);
 309
 310         while ((f = hashmap_first(m->fds)))
 311                 fd_free(f);
 312
 313         while (m->unused)
 314                 window_free(m->unused);
 315
 316         free(m);
 317 }
 318
 319 MMapCache* mmap_cache_unref(MMapCache *m) {
 320         assert(m);
 321         assert(m->n_ref > 0);
 322
 323         m->n_ref --;
 324         if (m->n_ref == 0)
 325                 mmap_cache_free(m);
 326
 327         return NULL;
 328 }
 329
 330 static int make_room(MMapCache *m) {
 331         assert(m);
 332
 333         if (!m->last_unused)
 334                 return 0;
 335
 336         window_free(m->last_unused);
 337         return 1;
 338 }
 339
 340 static int try_context(
 341                 MMapCache *m,
 342                 int fd,
 343                 int prot,
 344                 unsigned context,
 345                 bool keep_always,
 346                 uint64_t offset,
 347                 size_t size,
 348                 void **ret) {
 349
 350         Context *c;
 351
 352         assert(m);
 353         assert(m->n_ref > 0);
 354         assert(fd >= 0);
 355         assert(size > 0);
 356         assert(ret);
 357
 358         c = hashmap_get(m->contexts, UINT_TO_PTR(context+1));
 359         if (!c)
 360                 return 0;
 361
 362         assert(c->id == context);
 363
 364         if (!c->window)
 365                 return 0;
 366
 367         if (!window_matches(c->window, fd, prot, offset, size)) {
 368
 369                 /* Drop the reference to the window, since it's unnecessary now */
 370                 context_detach_window(c);
 371                 return 0;
 372         }
 373
 374         c->window->keep_always = c->window->keep_always || keep_always;
 375
 376         *ret = (uint8_t*) c->window->ptr + (offset - c->window->offset);
 377         return 1;
 378 }
 379
 380 static int find_mmap(
 381                 MMapCache *m,
 382                 int fd,
 383                 int prot,
 384                 unsigned context,
 385                 bool keep_always,
 386                 uint64_t offset,
 387                 size_t size,
 388                 void **ret) {
 389
 390         FileDescriptor *f;
 391         Window *w;
 392         Context *c;
 393
 394         assert(m);
 395         assert(m->n_ref > 0);
 396         assert(fd >= 0);
 397         assert(size > 0);
 398         assert(ret);
 399
 400         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 401         if (!f)
 402                 return 0;
 403
 404         assert(f->fd == fd);
 405
 406         LIST_FOREACH(by_fd, w, f->windows)
 407                 if (window_matches(w, fd, prot, offset, size))
 408                         break;
 409
 410         if (!w)
 411                 return 0;
 412
 413         c = context_add(m, context);
 414         if (!c)
 415                 return -ENOMEM;
 416
 417         context_attach_window(c, w);
 418         w->keep_always = w->keep_always || keep_always;
 419
 420         *ret = (uint8_t*) w->ptr + (offset - w->offset);
 421         return 1;
 422 }
 423
 424 static int add_mmap(
 425                 MMapCache *m,
 426                 int fd,
 427                 int prot,
 428                 unsigned context,
 429                 bool keep_always,
 430                 uint64_t offset,
 431                 size_t size,
 432                 struct stat *st,
 433                 void **ret) {
 434
 435         uint64_t woffset, wsize;
 436         Context *c;
 437         FileDescriptor *f;
 438         Window *w;
 439         void *d;
 440         int r;
 441
 442         assert(m);
 443         assert(m->n_ref > 0);
 444         assert(fd >= 0);
 445         assert(size > 0);
 446         assert(ret);
 447
 448         woffset = offset & ~((uint64_t) page_size() - 1ULL);
 449         wsize = size + (offset - woffset);
 450         wsize = PAGE_ALIGN(wsize);
 451
 452         if (wsize < WINDOW_SIZE) {
 453                 uint64_t delta;
 454
 455                 delta = PAGE_ALIGN((WINDOW_SIZE - wsize) / 2);
 456
 457                 if (delta > offset)
 458                         woffset = 0;
 459                 else
 460                         woffset -= delta;
 461
 462                 wsize = WINDOW_SIZE;
 463         }
 464
 465         if (st) {
 466                 /* Memory maps that are larger then the files
 467                    underneath have undefined behavior. Hence, clamp
 468                    things to the file size if we know it */
 469
 470                 if (woffset >= (uint64_t) st->st_size)
 471                         return -EADDRNOTAVAIL;
 472
 473                 if (woffset + wsize > (uint64_t) st->st_size)
 474                         wsize = PAGE_ALIGN(st->st_size - woffset);
 475         }
 476
 477         for (;;) {
 478                 d = mmap(NULL, wsize, prot, MAP_SHARED, fd, woffset);
 479                 if (d != MAP_FAILED)
 480                         break;
 481                 if (errno != ENOMEM)
 482                         return -errno;
 483
 484                 r = make_room(m);
 485                 if (r < 0)
 486                         return r;
 487                 if (r == 0)
 488                         return -ENOMEM;
 489         }
 490
 491         c = context_add(m, context);
 492         if (!c)
 493                 return -ENOMEM;
 494
 495         f = fd_add(m, fd);
 496         if (!f)
 497                 return -ENOMEM;
 498
 499         w = window_add(m);
 500         if (!w)
 501                 return -ENOMEM;
 502
 503         w->keep_always = keep_always;
 504         w->ptr = d;
 505         w->offset = woffset;
 506         w->prot = prot;
 507         w->size = wsize;
 508         w->fd = f;
 509
 510         LIST_PREPEND(Window, by_fd, f->windows, w);
 511
 512         context_detach_window(c);
 513         c->window = w;
 514         LIST_PREPEND(Context, by_window, w->contexts, c);
 515
 516         *ret = (uint8_t*) w->ptr + (offset - w->offset);
 517         return 1;
 518 }
 519
 520 int mmap_cache_get(
 521                 MMapCache *m,
 522                 int fd,
 523                 int prot,
 524                 unsigned context,
 525                 bool keep_always,
 526                 uint64_t offset,
 527                 size_t size,
 528                 struct stat *st,
 529                 void **ret) {
 530
 531         int r;
 532
 533         assert(m);
 534         assert(m->n_ref > 0);
 535         assert(fd >= 0);
 536         assert(size > 0);
 537         assert(ret);
 538
 539         /* Check whether the current context is the right one already */
 540         r = try_context(m, fd, prot, context, keep_always, offset, size, ret);
 541         if (r != 0)
 542                 return r;
 543
 544         /* Search for a matching mmap */
 545         r = find_mmap(m, fd, prot, context, keep_always, offset, size, ret);
 546         if (r != 0)
 547                 return r;
 548
 549         /* Create a new mmap */
 550         return add_mmap(m, fd, prot, context, keep_always, offset, size, st, ret);
 551 }
 552
 553 void mmap_cache_close_fd(MMapCache *m, int fd) {
 554         FileDescriptor *f;
 555
 556         assert(m);
 557         assert(fd >= 0);
 558
 559         f = hashmap_get(m->fds, INT_TO_PTR(fd + 1));
 560         if (!f)
 561                 return;
 562
 563         fd_free(f);
 564 }
 565
 566 void mmap_cache_close_context(MMapCache *m, unsigned context) {
 567         Context *c;
 568
 569         assert(m);
 570
 571         c = hashmap_get(m->contexts, UINT_TO_PTR(context + 1));
 572         if (!c)
 573                 return;
 574
 575         context_free(c);
 576 }