src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <fcntl.h>
  23 #include <stdio.h>
  24 #include <unistd.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <locale.h>
  29 #include <libgen.h>
  30
  31 #include "util.h"
  32 #include "log.h"
  33 #include "sparse-endian.h"
  34 #include "sd-id128.h"
  35 #include "hashmap.h"
  36 #include "strv.h"
  37 #include "strbuf.h"
  38 #include "strxcpyx.h"
  39 #include "conf-files.h"
  40 #include "mkdir.h"
  41 #include "catalog.h"
  42
  43 const char * const catalog_file_dirs[] = {
  44         "/usr/local/lib/systemd/catalog/",
  45         "/usr/lib/systemd/catalog/",
  46         NULL
  47 };
  48
  49 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  50
  51 typedef struct CatalogHeader {
  52         uint8_t signature[8];  /* "RHHHKSLP" */
  53         le32_t compatible_flags;
  54         le32_t incompatible_flags;
  55         le64_t header_size;
  56         le64_t n_items;
  57         le64_t catalog_item_size;
  58 } CatalogHeader;
  59
  60 typedef struct CatalogItem {
  61         sd_id128_t id;
  62         char language[32];
  63         le64_t offset;
  64 } CatalogItem;
  65
  66 unsigned catalog_hash_func(const void *p) {
  67         const CatalogItem *i = p;
  68
  69         assert_cc(sizeof(unsigned) == sizeof(uint8_t)*4);
  70
  71         return (((unsigned) i->id.bytes[0] << 24) |
  72                 ((unsigned) i->id.bytes[1] << 16) |
  73                 ((unsigned) i->id.bytes[2] << 8) |
  74                 ((unsigned) i->id.bytes[3])) ^
  75                (((unsigned) i->id.bytes[4] << 24) |
  76                 ((unsigned) i->id.bytes[5] << 16) |
  77                 ((unsigned) i->id.bytes[6] << 8) |
  78                 ((unsigned) i->id.bytes[7])) ^
  79                (((unsigned) i->id.bytes[8] << 24) |
  80                 ((unsigned) i->id.bytes[9] << 16) |
  81                 ((unsigned) i->id.bytes[10] << 8) |
  82                 ((unsigned) i->id.bytes[11])) ^
  83                (((unsigned) i->id.bytes[12] << 24) |
  84                 ((unsigned) i->id.bytes[13] << 16) |
  85                 ((unsigned) i->id.bytes[14] << 8) |
  86                 ((unsigned) i->id.bytes[15])) ^
  87                 string_hash_func(i->language);
  88 }
  89
  90 int catalog_compare_func(const void *a, const void *b) {
  91         const CatalogItem *i = a, *j = b;
  92         unsigned k;
  93
  94         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  95                  if (i->id.bytes[k] < j->id.bytes[k])
  96                         return -1;
  97                  if (i->id.bytes[k] > j->id.bytes[k])
  98                         return 1;
  99         }
 100
 101         return strcmp(i->language, j->language);
 102 }
 103
 104 static int finish_item(
 105                 Hashmap *h,
 106                 struct strbuf *sb,
 107                 sd_id128_t id,
 108                 const char *language,
 109                 const char *payload) {
 110
 111         ssize_t offset;
 112         CatalogItem *i;
 113         int r;
 114
 115         assert(h);
 116         assert(sb);
 117         assert(payload);
 118
 119         offset = strbuf_add_string(sb, payload, strlen(payload));
 120         if (offset < 0)
 121                 return log_oom();
 122
 123         i = new0(CatalogItem, 1);
 124         if (!i)
 125                 return log_oom();
 126
 127         i->id = id;
 128         strscpy(i->language, sizeof(i->language), language);
 129         i->offset = htole64((uint64_t) offset);
 130
 131         r = hashmap_put(h, i, i);
 132         if (r == EEXIST) {
 133                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 134                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 135                 free(i);
 136                 return 0;
 137         }
 138
 139         return 0;
 140 }
 141
 142 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 143         _cleanup_fclose_ FILE *f = NULL;
 144         _cleanup_free_ char *payload = NULL;
 145         unsigned n = 0;
 146         sd_id128_t id;
 147         char language[32];
 148         bool got_id = false, empty_line = true;
 149         int r;
 150
 151         assert(h);
 152         assert(sb);
 153         assert(path);
 154
 155         f = fopen(path, "re");
 156         if (!f) {
 157                 log_error("Failed to open file %s: %m", path);
 158                 return -errno;
 159         }
 160
 161         for (;;) {
 162                 char line[LINE_MAX];
 163                 size_t a, b, c;
 164                 char *t;
 165
 166                 if (!fgets(line, sizeof(line), f)) {
 167                         if (feof(f))
 168                                 break;
 169
 170                         log_error("Failed to read file %s: %m", path);
 171                         return -errno;
 172                 }
 173
 174                 n++;
 175
 176                 truncate_nl(line);
 177
 178                 if (line[0] == 0) {
 179                         empty_line = true;
 180                         continue;
 181                 }
 182
 183                 if (strchr(COMMENTS "\n", line[0]))
 184                         continue;
 185
 186                 if (empty_line &&
 187                     strlen(line) >= 2+1+32 &&
 188                     line[0] == '-' &&
 189                     line[1] == '-' &&
 190                     line[2] == ' ' &&
 191                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 192
 193                         bool with_language;
 194                         sd_id128_t jd;
 195
 196                         /* New entry */
 197
 198                         with_language = line[2+1+32] != '\0';
 199                         line[2+1+32] = '\0';
 200
 201                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 202
 203                                 if (got_id) {
 204                                         r = finish_item(h, sb, id, language, payload);
 205                                         if (r < 0)
 206                                                 return r;
 207                                 }
 208
 209                                 if (with_language) {
 210                                         t = strstrip(line + 2 + 1 + 32 + 1);
 211
 212                                         c = strlen(t);
 213                                         if (c <= 0) {
 214                                                 log_error("[%s:%u] Language too short.", path, n);
 215                                                 return -EINVAL;
 216                                         }
 217                                         if (c > sizeof(language) - 1) {
 218                                                 log_error("[%s:%u] language too long.", path, n);
 219                                                 return -EINVAL;
 220                                         }
 221
 222                                         strscpy(language, sizeof(language), t);
 223                                 } else
 224                                         language[0] = '\0';
 225
 226                                 got_id = true;
 227                                 empty_line = false;
 228                                 id = jd;
 229
 230                                 if (payload)
 231                                         payload[0] = '\0';
 232
 233                                 continue;
 234                         }
 235                 }
 236
 237                 /* Payload */
 238                 if (!got_id) {
 239                         log_error("[%s:%u] Got payload before ID.", path, n);
 240                         return -EINVAL;
 241                 }
 242
 243                 a = payload ? strlen(payload) : 0;
 244                 b = strlen(line);
 245
 246                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 247                 t = realloc(payload, c);
 248                 if (!t)
 249                         return log_oom();
 250
 251                 if (empty_line) {
 252                         t[a] = '\n';
 253                         memcpy(t + a + 1, line, b);
 254                         t[a+b+1] = '\n';
 255                         t[a+b+2] = 0;
 256                 } else {
 257                         memcpy(t + a, line, b);
 258                         t[a+b] = '\n';
 259                         t[a+b+1] = 0;
 260                 }
 261
 262                 payload = t;
 263                 empty_line = false;
 264         }
 265
 266         if (got_id) {
 267                 r = finish_item(h, sb, id, language, payload);
 268                 if (r < 0)
 269                         return r;
 270         }
 271
 272         return 0;
 273 }
 274
 275 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 276                           CatalogItem *items, size_t n) {
 277         CatalogHeader header;
 278         _cleanup_fclose_ FILE *w = NULL;
 279         int r;
 280         _cleanup_free_ char *d, *p = NULL;
 281         size_t k;
 282
 283         d = dirname_malloc(database);
 284         if (!d)
 285                 return log_oom();
 286
 287         r = mkdir_p(d, 0775);
 288         if (r < 0) {
 289                 log_error("Recursive mkdir %s: %s", d, strerror(-r));
 290                 return r;
 291         }
 292
 293         r = fopen_temporary(database, &w, &p);
 294         if (r < 0) {
 295                 log_error("Failed to open database for writing: %s: %s",
 296                           database, strerror(-r));
 297                 return r;
 298         }
 299
 300         zero(header);
 301         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 302         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 303         header.catalog_item_size = htole64(sizeof(CatalogItem));
 304         header.n_items = htole64(hashmap_size(h));
 305
 306         r = -EIO;
 307
 308         k = fwrite(&header, 1, sizeof(header), w);
 309         if (k != sizeof(header)) {
 310                 log_error("%s: failed to write header.", p);
 311                 goto error;
 312         }
 313
 314         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 315         if (k != n * sizeof(CatalogItem)) {
 316                 log_error("%s: failed to write database.", p);
 317                 goto error;
 318         }
 319
 320         k = fwrite(sb->buf, 1, sb->len, w);
 321         if (k != sb->len) {
 322                 log_error("%s: failed to write strings.", p);
 323                 goto error;
 324         }
 325
 326         fflush(w);
 327
 328         if (ferror(w)) {
 329                 log_error("%s: failed to write database.", p);
 330                 goto error;
 331         }
 332
 333         fchmod(fileno(w), 0644);
 334
 335         if (rename(p, database) < 0) {
 336                 log_error("rename (%s -> %s) failed: %m", p, database);
 337                 r = -errno;
 338                 goto error;
 339         }
 340
 341         return ftell(w);
 342
 343 error:
 344         unlink(p);
 345         return r;
 346 }
 347
 348 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 349         _cleanup_strv_free_ char **files = NULL;
 350         char **f;
 351         Hashmap *h;
 352         struct strbuf *sb = NULL;
 353         _cleanup_free_ CatalogItem *items = NULL;
 354         CatalogItem *i;
 355         Iterator j;
 356         unsigned n;
 357         long r;
 358
 359         h = hashmap_new(catalog_hash_func, catalog_compare_func);
 360         sb = strbuf_new();
 361
 362         if (!h || !sb) {
 363                 r = log_oom();
 364                 goto finish;
 365         }
 366
 367         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 368         if (r < 0) {
 369                 log_error("Failed to get catalog files: %s", strerror(-r));
 370                 goto finish;
 371         }
 372
 373         STRV_FOREACH(f, files) {
 374                 log_debug("reading file '%s'", *f);
 375                 catalog_import_file(h, sb, *f);
 376         }
 377
 378         if (hashmap_size(h) <= 0) {
 379                 log_info("No items in catalog.");
 380                 r = 0;
 381                 goto finish;
 382         } else
 383                 log_debug("Found %u items in catalog.", hashmap_size(h));
 384
 385         strbuf_complete(sb);
 386
 387         items = new(CatalogItem, hashmap_size(h));
 388         if (!items) {
 389                 r = log_oom();
 390                 goto finish;
 391         }
 392
 393         n = 0;
 394         HASHMAP_FOREACH(i, h, j) {
 395                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 396                           SD_ID128_FORMAT_VAL(i->id),
 397                           isempty(i->language) ? "C" : i->language);
 398                 items[n++] = *i;
 399         }
 400
 401         assert(n == hashmap_size(h));
 402         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 403
 404         r = write_catalog(database, h, sb, items, n);
 405         if (r < 0)
 406                 log_error("Failed to write %s: %s", database, strerror(-r));
 407         else
 408                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 409                           database, n, sb->len, r);
 410
 411         r = 0;
 412
 413 finish:
 414         if (h)
 415                 hashmap_free_free(h);
 416         if (sb)
 417                 strbuf_cleanup(sb);
 418
 419         return r < 0 ? r : 0;
 420 }
 421
 422 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 423         const CatalogHeader *h;
 424         int fd;
 425         void *p;
 426         struct stat st;
 427
 428         assert(_fd);
 429         assert(_st);
 430         assert(_p);
 431
 432         fd = open(database, O_RDONLY|O_CLOEXEC);
 433         if (fd < 0)
 434                 return -errno;
 435
 436         if (fstat(fd, &st) < 0) {
 437                 close_nointr_nofail(fd);
 438                 return -errno;
 439         }
 440
 441         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 442                 close_nointr_nofail(fd);
 443                 return -EINVAL;
 444         }
 445
 446         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 447         if (p == MAP_FAILED) {
 448                 close_nointr_nofail(fd);
 449                 return -errno;
 450         }
 451
 452         h = p;
 453         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 454             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 455             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 456             h->incompatible_flags != 0 ||
 457             le64toh(h->n_items) <= 0 ||
 458             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 459                 close_nointr_nofail(fd);
 460                 munmap(p, st.st_size);
 461                 return -EBADMSG;
 462         }
 463
 464         *_fd = fd;
 465         *_st = st;
 466         *_p = p;
 467
 468         return 0;
 469 }
 470
 471 static const char *find_id(void *p, sd_id128_t id) {
 472         CatalogItem key, *f = NULL;
 473         const CatalogHeader *h = p;
 474         const char *loc;
 475
 476         zero(key);
 477         key.id = id;
 478
 479         loc = setlocale(LC_MESSAGES, NULL);
 480         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 481                 strncpy(key.language, loc, sizeof(key.language));
 482                 key.language[strcspn(key.language, ".@")] = 0;
 483
 484                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 485                 if (!f) {
 486                         char *e;
 487
 488                         e = strchr(key.language, '_');
 489                         if (e) {
 490                                 *e = 0;
 491                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 492                         }
 493                 }
 494         }
 495
 496         if (!f) {
 497                 zero(key.language);
 498                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 499         }
 500
 501         if (!f)
 502                 return NULL;
 503
 504         return (const char*) p +
 505                 le64toh(h->header_size) +
 506                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 507                 le64toh(f->offset);
 508 }
 509
 510 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 511         _cleanup_close_ int fd = -1;
 512         void *p = NULL;
 513         struct stat st;
 514         char *text = NULL;
 515         int r;
 516         const char *s;
 517
 518         assert(_text);
 519
 520         r = open_mmap(database, &fd, &st, &p);
 521         if (r < 0)
 522                 return r;
 523
 524         s = find_id(p, id);
 525         if (!s) {
 526                 r = -ENOENT;
 527                 goto finish;
 528         }
 529
 530         text = strdup(s);
 531         if (!text) {
 532                 r = -ENOMEM;
 533                 goto finish;
 534         }
 535
 536         *_text = text;
 537         r = 0;
 538
 539 finish:
 540         if (p)
 541                 munmap(p, st.st_size);
 542
 543         return r;
 544 }
 545
 546 static char *find_header(const char *s, const char *header) {
 547
 548         for (;;) {
 549                 const char *v, *e;
 550
 551                 v = startswith(s, header);
 552                 if (v) {
 553                         v += strspn(v, WHITESPACE);
 554                         return strndup(v, strcspn(v, NEWLINE));
 555                 }
 556
 557                 /* End of text */
 558                 e = strchr(s, '\n');
 559                 if (!e)
 560                         return NULL;
 561
 562                 /* End of header */
 563                 if (e == s)
 564                         return NULL;
 565
 566                 s = e + 1;
 567         }
 568 }
 569
 570 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 571         if (oneline) {
 572                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 573
 574                 subject = find_header(s, "Subject:");
 575                 defined_by = find_header(s, "Defined-By:");
 576
 577                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 578                         SD_ID128_FORMAT_VAL(id),
 579                         strna(defined_by), strna(subject));
 580         } else
 581                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 582                         SD_ID128_FORMAT_VAL(id), s);
 583 }
 584
 585
 586 int catalog_list(FILE *f, const char *database, bool oneline) {
 587         _cleanup_close_ int fd = -1;
 588         void *p = NULL;
 589         struct stat st;
 590         const CatalogHeader *h;
 591         const CatalogItem *items;
 592         int r;
 593         unsigned n;
 594         sd_id128_t last_id;
 595         bool last_id_set = false;
 596
 597         r = open_mmap(database, &fd, &st, &p);
 598         if (r < 0)
 599                 return r;
 600
 601         h = p;
 602         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 603
 604         for (n = 0; n < le64toh(h->n_items); n++) {
 605                 const char *s;
 606
 607                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 608                         continue;
 609
 610                 assert_se(s = find_id(p, items[n].id));
 611
 612                 dump_catalog_entry(f, items[n].id, s, oneline);
 613
 614                 last_id_set = true;
 615                 last_id = items[n].id;
 616         }
 617
 618         munmap(p, st.st_size);
 619
 620         return 0;
 621 }
 622
 623 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 624         char **item;
 625         int r = 0;
 626
 627         STRV_FOREACH(item, items) {
 628                 sd_id128_t id;
 629                 int k;
 630                 _cleanup_free_ char *msg = NULL;
 631
 632                 k = sd_id128_from_string(*item, &id);
 633                 if (k < 0) {
 634                         log_error("Failed to parse id128 '%s': %s",
 635                                   *item, strerror(-k));
 636                         if (r == 0)
 637                                 r = k;
 638                         continue;
 639                 }
 640
 641                 k = catalog_get(database, id, &msg);
 642                 if (k < 0) {
 643                         log_full(k == -ENOENT ? LOG_NOTICE : LOG_ERR,
 644                                  "Failed to retrieve catalog entry for '%s': %s",
 645                                   *item, strerror(-k));
 646                         if (r == 0)
 647                                 r = k;
 648                         continue;
 649                 }
 650
 651                 dump_catalog_entry(f, id, msg, oneline);
 652         }
 653
 654         return r;
 655 }