src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <locale.h>
  25 #include <stdio.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <unistd.h>
  29
  30 #include "sd-id128.h"
  31
  32 #include "alloc-util.h"
  33 #include "catalog.h"
  34 #include "conf-files.h"
  35 #include "fd-util.h"
  36 #include "fileio.h"
  37 #include "hashmap.h"
  38 #include "log.h"
  39 #include "mkdir.h"
  40 #include "path-util.h"
  41 #include "siphash24.h"
  42 #include "sparse-endian.h"
  43 #include "strbuf.h"
  44 #include "string-util.h"
  45 #include "strv.h"
  46 #include "util.h"
  47
  48 const char * const catalog_file_dirs[] = {
  49         "/usr/local/lib/systemd/catalog/",
  50         "/usr/lib/systemd/catalog/",
  51         NULL
  52 };
  53
  54 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  55
  56 typedef struct CatalogHeader {
  57         uint8_t signature[8];  /* "RHHHKSLP" */
  58         le32_t compatible_flags;
  59         le32_t incompatible_flags;
  60         le64_t header_size;
  61         le64_t n_items;
  62         le64_t catalog_item_size;
  63 } CatalogHeader;
  64
  65 typedef struct CatalogItem {
  66         sd_id128_t id;
  67         char language[32];
  68         le64_t offset;
  69 } CatalogItem;
  70
  71 static void catalog_hash_func(const void *p, struct siphash *state) {
  72         const CatalogItem *i = p;
  73
  74         siphash24_compress(&i->id, sizeof(i->id), state);
  75         siphash24_compress(i->language, strlen(i->language), state);
  76 }
  77
  78 static int catalog_compare_func(const void *a, const void *b) {
  79         const CatalogItem *i = a, *j = b;
  80         unsigned k;
  81
  82         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  83                  if (i->id.bytes[k] < j->id.bytes[k])
  84                         return -1;
  85                  if (i->id.bytes[k] > j->id.bytes[k])
  86                         return 1;
  87         }
  88
  89         return strcmp(i->language, j->language);
  90 }
  91
  92 const struct hash_ops catalog_hash_ops = {
  93         .hash = catalog_hash_func,
  94         .compare = catalog_compare_func
  95 };
  96
  97 static int finish_item(
  98                 Hashmap *h,
  99                 struct strbuf *sb,
 100                 sd_id128_t id,
 101                 const char *language,
 102                 const char *payload) {
 103
 104         ssize_t offset;
 105         _cleanup_free_ CatalogItem *i = NULL;
 106         int r;
 107
 108         assert(h);
 109         assert(sb);
 110         assert(payload);
 111
 112         offset = strbuf_add_string(sb, payload, strlen(payload));
 113         if (offset < 0)
 114                 return log_oom();
 115
 116         i = new0(CatalogItem, 1);
 117         if (!i)
 118                 return log_oom();
 119
 120         i->id = id;
 121         if (language) {
 122                 assert(strlen(language) > 1 && strlen(language) < 32);
 123                 strcpy(i->language, language);
 124         }
 125         i->offset = htole64((uint64_t) offset);
 126
 127         r = hashmap_put(h, i, i);
 128         if (r == -EEXIST) {
 129                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 130                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 131                 return 0;
 132         } else if (r < 0)
 133                 return r;
 134
 135         i = NULL;
 136         return 0;
 137 }
 138
 139 int catalog_file_lang(const char* filename, char **lang) {
 140         char *beg, *end, *_lang;
 141
 142         end = endswith(filename, ".catalog");
 143         if (!end)
 144                 return 0;
 145
 146         beg = end - 1;
 147         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 148                 beg --;
 149
 150         if (*beg != '.' || end <= beg + 1)
 151                 return 0;
 152
 153         _lang = strndup(beg + 1, end - beg - 1);
 154         if (!_lang)
 155                 return -ENOMEM;
 156
 157         *lang = _lang;
 158         return 1;
 159 }
 160
 161 static int catalog_entry_lang(const char* filename, int line,
 162                               const char* t, const char* deflang, char **lang) {
 163         size_t c;
 164
 165         c = strlen(t);
 166         if (c == 0) {
 167                 log_error("[%s:%u] Language too short.", filename, line);
 168                 return -EINVAL;
 169         }
 170         if (c > 31) {
 171                 log_error("[%s:%u] language too long.", filename, line);
 172                 return -EINVAL;
 173         }
 174
 175         if (deflang) {
 176                 if (streq(t, deflang)) {
 177                         log_warning("[%s:%u] language specified unnecessarily",
 178                                     filename, line);
 179                         return 0;
 180                 } else
 181                         log_warning("[%s:%u] language differs from default for file",
 182                                     filename, line);
 183         }
 184
 185         *lang = strdup(t);
 186         if (!*lang)
 187                         return -ENOMEM;
 188
 189         return 0;
 190 }
 191
 192 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 193         _cleanup_fclose_ FILE *f = NULL;
 194         _cleanup_free_ char *payload = NULL;
 195         unsigned n = 0;
 196         sd_id128_t id;
 197         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 198         bool got_id = false, empty_line = true;
 199         int r;
 200
 201         assert(h);
 202         assert(sb);
 203         assert(path);
 204
 205         f = fopen(path, "re");
 206         if (!f)
 207                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 208
 209         r = catalog_file_lang(path, &deflang);
 210         if (r < 0)
 211                 log_error_errno(errno, "Failed to determine language for file %s: %m", path);
 212         if (r == 1)
 213                 log_debug("File %s has language %s.", path, deflang);
 214
 215         for (;;) {
 216                 char line[LINE_MAX];
 217                 size_t a, b, c;
 218                 char *t;
 219
 220                 if (!fgets(line, sizeof(line), f)) {
 221                         if (feof(f))
 222                                 break;
 223
 224                         log_error_errno(errno, "Failed to read file %s: %m", path);
 225                         return -errno;
 226                 }
 227
 228                 n++;
 229
 230                 truncate_nl(line);
 231
 232                 if (line[0] == 0) {
 233                         empty_line = true;
 234                         continue;
 235                 }
 236
 237                 if (strchr(COMMENTS "\n", line[0]))
 238                         continue;
 239
 240                 if (empty_line &&
 241                     strlen(line) >= 2+1+32 &&
 242                     line[0] == '-' &&
 243                     line[1] == '-' &&
 244                     line[2] == ' ' &&
 245                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 246
 247                         bool with_language;
 248                         sd_id128_t jd;
 249
 250                         /* New entry */
 251
 252                         with_language = line[2+1+32] != '\0';
 253                         line[2+1+32] = '\0';
 254
 255                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 256
 257                                 if (got_id) {
 258                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 259                                         if (r < 0)
 260                                                 return r;
 261
 262                                         lang = mfree(lang);
 263                                 }
 264
 265                                 if (with_language) {
 266                                         t = strstrip(line + 2 + 1 + 32 + 1);
 267
 268                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 269                                         if (r < 0)
 270                                                 return r;
 271                                 }
 272
 273                                 got_id = true;
 274                                 empty_line = false;
 275                                 id = jd;
 276
 277                                 if (payload)
 278                                         payload[0] = '\0';
 279
 280                                 continue;
 281                         }
 282                 }
 283
 284                 /* Payload */
 285                 if (!got_id) {
 286                         log_error("[%s:%u] Got payload before ID.", path, n);
 287                         return -EINVAL;
 288                 }
 289
 290                 a = payload ? strlen(payload) : 0;
 291                 b = strlen(line);
 292
 293                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 294                 t = realloc(payload, c);
 295                 if (!t)
 296                         return log_oom();
 297
 298                 if (empty_line) {
 299                         t[a] = '\n';
 300                         memcpy(t + a + 1, line, b);
 301                         t[a+b+1] = '\n';
 302                         t[a+b+2] = 0;
 303                 } else {
 304                         memcpy(t + a, line, b);
 305                         t[a+b] = '\n';
 306                         t[a+b+1] = 0;
 307                 }
 308
 309                 payload = t;
 310                 empty_line = false;
 311         }
 312
 313         if (got_id) {
 314                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 315                 if (r < 0)
 316                         return r;
 317         }
 318
 319         return 0;
 320 }
 321
 322 static int64_t write_catalog(const char *database, struct strbuf *sb,
 323                              CatalogItem *items, size_t n) {
 324         CatalogHeader header;
 325         _cleanup_fclose_ FILE *w = NULL;
 326         int r;
 327         _cleanup_free_ char *d, *p = NULL;
 328         size_t k;
 329
 330         d = dirname_malloc(database);
 331         if (!d)
 332                 return log_oom();
 333
 334         r = mkdir_p(d, 0775);
 335         if (r < 0)
 336                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 337
 338         r = fopen_temporary(database, &w, &p);
 339         if (r < 0)
 340                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 341                                        database);
 342
 343         zero(header);
 344         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 345         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 346         header.catalog_item_size = htole64(sizeof(CatalogItem));
 347         header.n_items = htole64(n);
 348
 349         r = -EIO;
 350
 351         k = fwrite(&header, 1, sizeof(header), w);
 352         if (k != sizeof(header)) {
 353                 log_error("%s: failed to write header.", p);
 354                 goto error;
 355         }
 356
 357         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 358         if (k != n * sizeof(CatalogItem)) {
 359                 log_error("%s: failed to write database.", p);
 360                 goto error;
 361         }
 362
 363         k = fwrite(sb->buf, 1, sb->len, w);
 364         if (k != sb->len) {
 365                 log_error("%s: failed to write strings.", p);
 366                 goto error;
 367         }
 368
 369         r = fflush_and_check(w);
 370         if (r < 0) {
 371                 log_error_errno(r, "%s: failed to write database: %m", p);
 372                 goto error;
 373         }
 374
 375         fchmod(fileno(w), 0644);
 376
 377         if (rename(p, database) < 0) {
 378                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 379                 goto error;
 380         }
 381
 382         return ftello(w);
 383
 384 error:
 385         (void) unlink(p);
 386         return r;
 387 }
 388
 389 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 390         _cleanup_strv_free_ char **files = NULL;
 391         char **f;
 392         struct strbuf *sb = NULL;
 393         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 394         _cleanup_free_ CatalogItem *items = NULL;
 395         CatalogItem *i;
 396         Iterator j;
 397         unsigned n;
 398         int r;
 399         int64_t sz;
 400
 401         h = hashmap_new(&catalog_hash_ops);
 402         sb = strbuf_new();
 403
 404         if (!h || !sb) {
 405                 r = log_oom();
 406                 goto finish;
 407         }
 408
 409         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 410         if (r < 0) {
 411                 log_error_errno(r, "Failed to get catalog files: %m");
 412                 goto finish;
 413         }
 414
 415         STRV_FOREACH(f, files) {
 416                 log_debug("Reading file '%s'", *f);
 417                 r = catalog_import_file(h, sb, *f);
 418                 if (r < 0) {
 419                         log_error_errno(r, "Failed to import file '%s': %m", *f);
 420                         goto finish;
 421                 }
 422         }
 423
 424         if (hashmap_size(h) <= 0) {
 425                 log_info("No items in catalog.");
 426                 goto finish;
 427         } else
 428                 log_debug("Found %u items in catalog.", hashmap_size(h));
 429
 430         strbuf_complete(sb);
 431
 432         items = new(CatalogItem, hashmap_size(h));
 433         if (!items) {
 434                 r = log_oom();
 435                 goto finish;
 436         }
 437
 438         n = 0;
 439         HASHMAP_FOREACH(i, h, j) {
 440                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 441                           SD_ID128_FORMAT_VAL(i->id),
 442                           isempty(i->language) ? "C" : i->language);
 443                 items[n++] = *i;
 444         }
 445
 446         assert(n == hashmap_size(h));
 447         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 448
 449         sz = write_catalog(database, sb, items, n);
 450         if (sz < 0)
 451                 r = log_error_errno(sz, "Failed to write %s: %m", database);
 452         else {
 453                 r = 0;
 454                 log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 455                           database, n, sb->len, sz);
 456         }
 457
 458 finish:
 459         strbuf_cleanup(sb);
 460
 461         return r;
 462 }
 463
 464 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 465         const CatalogHeader *h;
 466         int fd;
 467         void *p;
 468         struct stat st;
 469
 470         assert(_fd);
 471         assert(_st);
 472         assert(_p);
 473
 474         fd = open(database, O_RDONLY|O_CLOEXEC);
 475         if (fd < 0)
 476                 return -errno;
 477
 478         if (fstat(fd, &st) < 0) {
 479                 safe_close(fd);
 480                 return -errno;
 481         }
 482
 483         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 484                 safe_close(fd);
 485                 return -EINVAL;
 486         }
 487
 488         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 489         if (p == MAP_FAILED) {
 490                 safe_close(fd);
 491                 return -errno;
 492         }
 493
 494         h = p;
 495         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 496             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 497             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 498             h->incompatible_flags != 0 ||
 499             le64toh(h->n_items) <= 0 ||
 500             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 501                 safe_close(fd);
 502                 munmap(p, st.st_size);
 503                 return -EBADMSG;
 504         }
 505
 506         *_fd = fd;
 507         *_st = st;
 508         *_p = p;
 509
 510         return 0;
 511 }
 512
 513 static const char *find_id(void *p, sd_id128_t id) {
 514         CatalogItem key, *f = NULL;
 515         const CatalogHeader *h = p;
 516         const char *loc;
 517
 518         zero(key);
 519         key.id = id;
 520
 521         loc = setlocale(LC_MESSAGES, NULL);
 522         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 523                 strncpy(key.language, loc, sizeof(key.language));
 524                 key.language[strcspn(key.language, ".@")] = 0;
 525
 526                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 527                 if (!f) {
 528                         char *e;
 529
 530                         e = strchr(key.language, '_');
 531                         if (e) {
 532                                 *e = 0;
 533                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 534                         }
 535                 }
 536         }
 537
 538         if (!f) {
 539                 zero(key.language);
 540                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 541         }
 542
 543         if (!f)
 544                 return NULL;
 545
 546         return (const char*) p +
 547                 le64toh(h->header_size) +
 548                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 549                 le64toh(f->offset);
 550 }
 551
 552 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 553         _cleanup_close_ int fd = -1;
 554         void *p = NULL;
 555         struct stat st = {};
 556         char *text = NULL;
 557         int r;
 558         const char *s;
 559
 560         assert(_text);
 561
 562         r = open_mmap(database, &fd, &st, &p);
 563         if (r < 0)
 564                 return r;
 565
 566         s = find_id(p, id);
 567         if (!s) {
 568                 r = -ENOENT;
 569                 goto finish;
 570         }
 571
 572         text = strdup(s);
 573         if (!text) {
 574                 r = -ENOMEM;
 575                 goto finish;
 576         }
 577
 578         *_text = text;
 579         r = 0;
 580
 581 finish:
 582         if (p)
 583                 munmap(p, st.st_size);
 584
 585         return r;
 586 }
 587
 588 static char *find_header(const char *s, const char *header) {
 589
 590         for (;;) {
 591                 const char *v, *e;
 592
 593                 v = startswith(s, header);
 594                 if (v) {
 595                         v += strspn(v, WHITESPACE);
 596                         return strndup(v, strcspn(v, NEWLINE));
 597                 }
 598
 599                 /* End of text */
 600                 e = strchr(s, '\n');
 601                 if (!e)
 602                         return NULL;
 603
 604                 /* End of header */
 605                 if (e == s)
 606                         return NULL;
 607
 608                 s = e + 1;
 609         }
 610 }
 611
 612 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 613         if (oneline) {
 614                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 615
 616                 subject = find_header(s, "Subject:");
 617                 defined_by = find_header(s, "Defined-By:");
 618
 619                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 620                         SD_ID128_FORMAT_VAL(id),
 621                         strna(defined_by), strna(subject));
 622         } else
 623                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 624                         SD_ID128_FORMAT_VAL(id), s);
 625 }
 626
 627
 628 int catalog_list(FILE *f, const char *database, bool oneline) {
 629         _cleanup_close_ int fd = -1;
 630         void *p = NULL;
 631         struct stat st;
 632         const CatalogHeader *h;
 633         const CatalogItem *items;
 634         int r;
 635         unsigned n;
 636         sd_id128_t last_id;
 637         bool last_id_set = false;
 638
 639         r = open_mmap(database, &fd, &st, &p);
 640         if (r < 0)
 641                 return r;
 642
 643         h = p;
 644         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 645
 646         for (n = 0; n < le64toh(h->n_items); n++) {
 647                 const char *s;
 648
 649                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 650                         continue;
 651
 652                 assert_se(s = find_id(p, items[n].id));
 653
 654                 dump_catalog_entry(f, items[n].id, s, oneline);
 655
 656                 last_id_set = true;
 657                 last_id = items[n].id;
 658         }
 659
 660         munmap(p, st.st_size);
 661
 662         return 0;
 663 }
 664
 665 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 666         char **item;
 667         int r = 0;
 668
 669         STRV_FOREACH(item, items) {
 670                 sd_id128_t id;
 671                 int k;
 672                 _cleanup_free_ char *msg = NULL;
 673
 674                 k = sd_id128_from_string(*item, &id);
 675                 if (k < 0) {
 676                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 677                         if (r == 0)
 678                                 r = k;
 679                         continue;
 680                 }
 681
 682                 k = catalog_get(database, id, &msg);
 683                 if (k < 0) {
 684                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 685                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 686                         if (r == 0)
 687                                 r = k;
 688                         continue;
 689                 }
 690
 691                 dump_catalog_entry(f, id, msg, oneline);
 692         }
 693
 694         return r;
 695 }