src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <fcntl.h>
  23 #include <stdio.h>
  24 #include <unistd.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <locale.h>
  29 #include <libgen.h>
  30
  31 #include "util.h"
  32 #include "log.h"
  33 #include "sparse-endian.h"
  34 #include "sd-id128.h"
  35 #include "hashmap.h"
  36 #include "strv.h"
  37 #include "strbuf.h"
  38 #include "strxcpyx.h"
  39 #include "conf-files.h"
  40 #include "mkdir.h"
  41 #include "catalog.h"
  42 #include "siphash24.h"
  43
  44 const char * const catalog_file_dirs[] = {
  45         "/usr/local/lib/systemd/catalog/",
  46         "/usr/lib/systemd/catalog/",
  47         NULL
  48 };
  49
  50 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  51
  52 typedef struct CatalogHeader {
  53         uint8_t signature[8];  /* "RHHHKSLP" */
  54         le32_t compatible_flags;
  55         le32_t incompatible_flags;
  56         le64_t header_size;
  57         le64_t n_items;
  58         le64_t catalog_item_size;
  59 } CatalogHeader;
  60
  61 typedef struct CatalogItem {
  62         sd_id128_t id;
  63         char language[32];
  64         le64_t offset;
  65 } CatalogItem;
  66
  67 static unsigned long catalog_hash_func(const void *p, const uint8_t hash_key[HASH_KEY_SIZE]) {
  68         const CatalogItem *i = p;
  69         uint64_t u;
  70         size_t l, sz;
  71         void *v;
  72
  73         l = strlen(i->language);
  74         sz = sizeof(i->id) + l;
  75         v = alloca(sz);
  76
  77         memcpy(mempcpy(v, &i->id, sizeof(i->id)), i->language, l);
  78
  79         siphash24((uint8_t*) &u, v, sz, hash_key);
  80
  81         return (unsigned long) u;
  82 }
  83
  84 static int catalog_compare_func(const void *a, const void *b) {
  85         const CatalogItem *i = a, *j = b;
  86         unsigned k;
  87
  88         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  89                  if (i->id.bytes[k] < j->id.bytes[k])
  90                         return -1;
  91                  if (i->id.bytes[k] > j->id.bytes[k])
  92                         return 1;
  93         }
  94
  95         return strcmp(i->language, j->language);
  96 }
  97
  98 const struct hash_ops catalog_hash_ops = {
  99         .hash = catalog_hash_func,
 100         .compare = catalog_compare_func
 101 };
 102
 103 static int finish_item(
 104                 Hashmap *h,
 105                 struct strbuf *sb,
 106                 sd_id128_t id,
 107                 const char *language,
 108                 const char *payload) {
 109
 110         ssize_t offset;
 111         _cleanup_free_ CatalogItem *i = NULL;
 112         int r;
 113
 114         assert(h);
 115         assert(sb);
 116         assert(payload);
 117
 118         offset = strbuf_add_string(sb, payload, strlen(payload));
 119         if (offset < 0)
 120                 return log_oom();
 121
 122         i = new0(CatalogItem, 1);
 123         if (!i)
 124                 return log_oom();
 125
 126         i->id = id;
 127         if (language) {
 128                 assert(strlen(language) > 1 && strlen(language) < 32);
 129                 strcpy(i->language, language);
 130         }
 131         i->offset = htole64((uint64_t) offset);
 132
 133         r = hashmap_put(h, i, i);
 134         if (r == -EEXIST) {
 135                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 136                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 137                 return 0;
 138         } else if (r < 0)
 139                 return r;
 140
 141         i = NULL;
 142         return 0;
 143 }
 144
 145 int catalog_file_lang(const char* filename, char **lang) {
 146         char *beg, *end, *_lang;
 147
 148         end = endswith(filename, ".catalog");
 149         if (!end)
 150                 return 0;
 151
 152         beg = end - 1;
 153         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 154                 beg --;
 155
 156         if (*beg != '.' || end <= beg + 1)
 157                 return 0;
 158
 159         _lang = strndup(beg + 1, end - beg - 1);
 160         if (!_lang)
 161                 return -ENOMEM;
 162
 163         *lang = _lang;
 164         return 1;
 165 }
 166
 167 static int catalog_entry_lang(const char* filename, int line,
 168                               const char* t, const char* deflang, char **lang) {
 169         size_t c;
 170
 171         c = strlen(t);
 172         if (c == 0) {
 173                 log_error("[%s:%u] Language too short.", filename, line);
 174                 return -EINVAL;
 175         }
 176         if (c > 31) {
 177                 log_error("[%s:%u] language too long.", filename, line);
 178                 return -EINVAL;
 179         }
 180
 181         if (deflang) {
 182                 if (streq(t, deflang)) {
 183                         log_warning("[%s:%u] language specified unnecessarily",
 184                                     filename, line);
 185                         return 0;
 186                 } else
 187                         log_warning("[%s:%u] language differs from default for file",
 188                                     filename, line);
 189         }
 190
 191         *lang = strdup(t);
 192         if (!*lang)
 193                         return -ENOMEM;
 194
 195         return 0;
 196 }
 197
 198 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 199         _cleanup_fclose_ FILE *f = NULL;
 200         _cleanup_free_ char *payload = NULL;
 201         unsigned n = 0;
 202         sd_id128_t id;
 203         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 204         bool got_id = false, empty_line = true;
 205         int r;
 206
 207         assert(h);
 208         assert(sb);
 209         assert(path);
 210
 211         f = fopen(path, "re");
 212         if (!f) {
 213                 log_error("Failed to open file %s: %m", path);
 214                 return -errno;
 215         }
 216
 217         r = catalog_file_lang(path, &deflang);
 218         if (r < 0)
 219                 log_error("Failed to determine language for file %s: %m", path);
 220         if (r == 1)
 221                 log_debug("File %s has language %s.", path, deflang);
 222
 223         for (;;) {
 224                 char line[LINE_MAX];
 225                 size_t a, b, c;
 226                 char *t;
 227
 228                 if (!fgets(line, sizeof(line), f)) {
 229                         if (feof(f))
 230                                 break;
 231
 232                         log_error("Failed to read file %s: %m", path);
 233                         return -errno;
 234                 }
 235
 236                 n++;
 237
 238                 truncate_nl(line);
 239
 240                 if (line[0] == 0) {
 241                         empty_line = true;
 242                         continue;
 243                 }
 244
 245                 if (strchr(COMMENTS "\n", line[0]))
 246                         continue;
 247
 248                 if (empty_line &&
 249                     strlen(line) >= 2+1+32 &&
 250                     line[0] == '-' &&
 251                     line[1] == '-' &&
 252                     line[2] == ' ' &&
 253                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 254
 255                         bool with_language;
 256                         sd_id128_t jd;
 257
 258                         /* New entry */
 259
 260                         with_language = line[2+1+32] != '\0';
 261                         line[2+1+32] = '\0';
 262
 263                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 264
 265                                 if (got_id) {
 266                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 267                                         if (r < 0)
 268                                                 return r;
 269
 270                                         free(lang);
 271                                         lang = NULL;
 272                                 }
 273
 274                                 if (with_language) {
 275                                         t = strstrip(line + 2 + 1 + 32 + 1);
 276
 277                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 278                                         if (r < 0)
 279                                                 return r;
 280                                 }
 281
 282                                 got_id = true;
 283                                 empty_line = false;
 284                                 id = jd;
 285
 286                                 if (payload)
 287                                         payload[0] = '\0';
 288
 289                                 continue;
 290                         }
 291                 }
 292
 293                 /* Payload */
 294                 if (!got_id) {
 295                         log_error("[%s:%u] Got payload before ID.", path, n);
 296                         return -EINVAL;
 297                 }
 298
 299                 a = payload ? strlen(payload) : 0;
 300                 b = strlen(line);
 301
 302                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 303                 t = realloc(payload, c);
 304                 if (!t)
 305                         return log_oom();
 306
 307                 if (empty_line) {
 308                         t[a] = '\n';
 309                         memcpy(t + a + 1, line, b);
 310                         t[a+b+1] = '\n';
 311                         t[a+b+2] = 0;
 312                 } else {
 313                         memcpy(t + a, line, b);
 314                         t[a+b] = '\n';
 315                         t[a+b+1] = 0;
 316                 }
 317
 318                 payload = t;
 319                 empty_line = false;
 320         }
 321
 322         if (got_id) {
 323                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 324                 if (r < 0)
 325                         return r;
 326         }
 327
 328         return 0;
 329 }
 330
 331 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 332                           CatalogItem *items, size_t n) {
 333         CatalogHeader header;
 334         _cleanup_fclose_ FILE *w = NULL;
 335         int r;
 336         _cleanup_free_ char *d, *p = NULL;
 337         size_t k;
 338
 339         d = dirname_malloc(database);
 340         if (!d)
 341                 return log_oom();
 342
 343         r = mkdir_p(d, 0775);
 344         if (r < 0) {
 345                 log_error("Recursive mkdir %s: %s", d, strerror(-r));
 346                 return r;
 347         }
 348
 349         r = fopen_temporary(database, &w, &p);
 350         if (r < 0) {
 351                 log_error("Failed to open database for writing: %s: %s",
 352                           database, strerror(-r));
 353                 return r;
 354         }
 355
 356         zero(header);
 357         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 358         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 359         header.catalog_item_size = htole64(sizeof(CatalogItem));
 360         header.n_items = htole64(hashmap_size(h));
 361
 362         r = -EIO;
 363
 364         k = fwrite(&header, 1, sizeof(header), w);
 365         if (k != sizeof(header)) {
 366                 log_error("%s: failed to write header.", p);
 367                 goto error;
 368         }
 369
 370         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 371         if (k != n * sizeof(CatalogItem)) {
 372                 log_error("%s: failed to write database.", p);
 373                 goto error;
 374         }
 375
 376         k = fwrite(sb->buf, 1, sb->len, w);
 377         if (k != sb->len) {
 378                 log_error("%s: failed to write strings.", p);
 379                 goto error;
 380         }
 381
 382         fflush(w);
 383
 384         if (ferror(w)) {
 385                 log_error("%s: failed to write database.", p);
 386                 goto error;
 387         }
 388
 389         fchmod(fileno(w), 0644);
 390
 391         if (rename(p, database) < 0) {
 392                 log_error("rename (%s -> %s) failed: %m", p, database);
 393                 r = -errno;
 394                 goto error;
 395         }
 396
 397         return ftell(w);
 398
 399 error:
 400         unlink(p);
 401         return r;
 402 }
 403
 404 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 405         _cleanup_strv_free_ char **files = NULL;
 406         char **f;
 407         struct strbuf *sb = NULL;
 408         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 409         _cleanup_free_ CatalogItem *items = NULL;
 410         CatalogItem *i;
 411         Iterator j;
 412         unsigned n;
 413         long r;
 414
 415         h = hashmap_new(&catalog_hash_ops);
 416         sb = strbuf_new();
 417
 418         if (!h || !sb) {
 419                 r = log_oom();
 420                 goto finish;
 421         }
 422
 423         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 424         if (r < 0) {
 425                 log_error("Failed to get catalog files: %s", strerror(-r));
 426                 goto finish;
 427         }
 428
 429         STRV_FOREACH(f, files) {
 430                 log_debug("Reading file '%s'", *f);
 431                 r = catalog_import_file(h, sb, *f);
 432                 if (r < 0) {
 433                         log_error("Failed to import file '%s': %s.",
 434                                   *f, strerror(-r));
 435                         goto finish;
 436                 }
 437         }
 438
 439         if (hashmap_size(h) <= 0) {
 440                 log_info("No items in catalog.");
 441                 goto finish;
 442         } else
 443                 log_debug("Found %u items in catalog.", hashmap_size(h));
 444
 445         strbuf_complete(sb);
 446
 447         items = new(CatalogItem, hashmap_size(h));
 448         if (!items) {
 449                 r = log_oom();
 450                 goto finish;
 451         }
 452
 453         n = 0;
 454         HASHMAP_FOREACH(i, h, j) {
 455                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 456                           SD_ID128_FORMAT_VAL(i->id),
 457                           isempty(i->language) ? "C" : i->language);
 458                 items[n++] = *i;
 459         }
 460
 461         assert(n == hashmap_size(h));
 462         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 463
 464         r = write_catalog(database, h, sb, items, n);
 465         if (r < 0)
 466                 log_error("Failed to write %s: %s", database, strerror(-r));
 467         else
 468                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 469                           database, n, sb->len, r);
 470
 471 finish:
 472         if (sb)
 473                 strbuf_cleanup(sb);
 474
 475         return r < 0 ? r : 0;
 476 }
 477
 478 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 479         const CatalogHeader *h;
 480         int fd;
 481         void *p;
 482         struct stat st;
 483
 484         assert(_fd);
 485         assert(_st);
 486         assert(_p);
 487
 488         fd = open(database, O_RDONLY|O_CLOEXEC);
 489         if (fd < 0)
 490                 return -errno;
 491
 492         if (fstat(fd, &st) < 0) {
 493                 safe_close(fd);
 494                 return -errno;
 495         }
 496
 497         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 498                 safe_close(fd);
 499                 return -EINVAL;
 500         }
 501
 502         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 503         if (p == MAP_FAILED) {
 504                 safe_close(fd);
 505                 return -errno;
 506         }
 507
 508         h = p;
 509         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 510             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 511             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 512             h->incompatible_flags != 0 ||
 513             le64toh(h->n_items) <= 0 ||
 514             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 515                 safe_close(fd);
 516                 munmap(p, st.st_size);
 517                 return -EBADMSG;
 518         }
 519
 520         *_fd = fd;
 521         *_st = st;
 522         *_p = p;
 523
 524         return 0;
 525 }
 526
 527 static const char *find_id(void *p, sd_id128_t id) {
 528         CatalogItem key, *f = NULL;
 529         const CatalogHeader *h = p;
 530         const char *loc;
 531
 532         zero(key);
 533         key.id = id;
 534
 535         loc = setlocale(LC_MESSAGES, NULL);
 536         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 537                 strncpy(key.language, loc, sizeof(key.language));
 538                 key.language[strcspn(key.language, ".@")] = 0;
 539
 540                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 541                 if (!f) {
 542                         char *e;
 543
 544                         e = strchr(key.language, '_');
 545                         if (e) {
 546                                 *e = 0;
 547                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 548                         }
 549                 }
 550         }
 551
 552         if (!f) {
 553                 zero(key.language);
 554                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 555         }
 556
 557         if (!f)
 558                 return NULL;
 559
 560         return (const char*) p +
 561                 le64toh(h->header_size) +
 562                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 563                 le64toh(f->offset);
 564 }
 565
 566 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 567         _cleanup_close_ int fd = -1;
 568         void *p = NULL;
 569         struct stat st;
 570         char *text = NULL;
 571         int r;
 572         const char *s;
 573
 574         assert(_text);
 575
 576         r = open_mmap(database, &fd, &st, &p);
 577         if (r < 0)
 578                 return r;
 579
 580         s = find_id(p, id);
 581         if (!s) {
 582                 r = -ENOENT;
 583                 goto finish;
 584         }
 585
 586         text = strdup(s);
 587         if (!text) {
 588                 r = -ENOMEM;
 589                 goto finish;
 590         }
 591
 592         *_text = text;
 593         r = 0;
 594
 595 finish:
 596         if (p)
 597                 munmap(p, st.st_size);
 598
 599         return r;
 600 }
 601
 602 static char *find_header(const char *s, const char *header) {
 603
 604         for (;;) {
 605                 const char *v, *e;
 606
 607                 v = startswith(s, header);
 608                 if (v) {
 609                         v += strspn(v, WHITESPACE);
 610                         return strndup(v, strcspn(v, NEWLINE));
 611                 }
 612
 613                 /* End of text */
 614                 e = strchr(s, '\n');
 615                 if (!e)
 616                         return NULL;
 617
 618                 /* End of header */
 619                 if (e == s)
 620                         return NULL;
 621
 622                 s = e + 1;
 623         }
 624 }
 625
 626 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 627         if (oneline) {
 628                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 629
 630                 subject = find_header(s, "Subject:");
 631                 defined_by = find_header(s, "Defined-By:");
 632
 633                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 634                         SD_ID128_FORMAT_VAL(id),
 635                         strna(defined_by), strna(subject));
 636         } else
 637                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 638                         SD_ID128_FORMAT_VAL(id), s);
 639 }
 640
 641
 642 int catalog_list(FILE *f, const char *database, bool oneline) {
 643         _cleanup_close_ int fd = -1;
 644         void *p = NULL;
 645         struct stat st;
 646         const CatalogHeader *h;
 647         const CatalogItem *items;
 648         int r;
 649         unsigned n;
 650         sd_id128_t last_id;
 651         bool last_id_set = false;
 652
 653         r = open_mmap(database, &fd, &st, &p);
 654         if (r < 0)
 655                 return r;
 656
 657         h = p;
 658         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 659
 660         for (n = 0; n < le64toh(h->n_items); n++) {
 661                 const char *s;
 662
 663                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 664                         continue;
 665
 666                 assert_se(s = find_id(p, items[n].id));
 667
 668                 dump_catalog_entry(f, items[n].id, s, oneline);
 669
 670                 last_id_set = true;
 671                 last_id = items[n].id;
 672         }
 673
 674         munmap(p, st.st_size);
 675
 676         return 0;
 677 }
 678
 679 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 680         char **item;
 681         int r = 0;
 682
 683         STRV_FOREACH(item, items) {
 684                 sd_id128_t id;
 685                 int k;
 686                 _cleanup_free_ char *msg = NULL;
 687
 688                 k = sd_id128_from_string(*item, &id);
 689                 if (k < 0) {
 690                         log_error("Failed to parse id128 '%s': %s",
 691                                   *item, strerror(-k));
 692                         if (r == 0)
 693                                 r = k;
 694                         continue;
 695                 }
 696
 697                 k = catalog_get(database, id, &msg);
 698                 if (k < 0) {
 699                         log_full(k == -ENOENT ? LOG_NOTICE : LOG_ERR,
 700                                  "Failed to retrieve catalog entry for '%s': %s",
 701                                   *item, strerror(-k));
 702                         if (r == 0)
 703                                 r = k;
 704                         continue;
 705                 }
 706
 707                 dump_catalog_entry(f, id, msg, oneline);
 708         }
 709
 710         return r;
 711 }