src/journal/catalog.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #include <errno.h>
   4 #include <fcntl.h>
   5 #include <locale.h>
   6 #include <stdio.h>
   7 #include <string.h>
   8 #include <sys/mman.h>
   9 #include <unistd.h>
  10
  11 #include "sd-id128.h"
  12
  13 #include "alloc-util.h"
  14 #include "catalog.h"
  15 #include "conf-files.h"
  16 #include "fd-util.h"
  17 #include "fileio.h"
  18 #include "hashmap.h"
  19 #include "log.h"
  20 #include "mkdir.h"
  21 #include "path-util.h"
  22 #include "siphash24.h"
  23 #include "sparse-endian.h"
  24 #include "strbuf.h"
  25 #include "string-util.h"
  26 #include "strv.h"
  27 #include "util.h"
  28
  29 const char * const catalog_file_dirs[] = {
  30         "/usr/local/lib/systemd/catalog/",
  31         "/usr/lib/systemd/catalog/",
  32         NULL
  33 };
  34
  35 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  36
  37 typedef struct CatalogHeader {
  38         uint8_t signature[8];  /* "RHHHKSLP" */
  39         le32_t compatible_flags;
  40         le32_t incompatible_flags;
  41         le64_t header_size;
  42         le64_t n_items;
  43         le64_t catalog_item_size;
  44 } CatalogHeader;
  45
  46 typedef struct CatalogItem {
  47         sd_id128_t id;
  48         char language[32];
  49         le64_t offset;
  50 } CatalogItem;
  51
  52 static void catalog_hash_func(const void *p, struct siphash *state) {
  53         const CatalogItem *i = p;
  54
  55         siphash24_compress(&i->id, sizeof(i->id), state);
  56         siphash24_compress(i->language, strlen(i->language), state);
  57 }
  58
  59 static int catalog_compare_func(const CatalogItem *a, const CatalogItem *b) {
  60         unsigned k;
  61         int r;
  62
  63         for (k = 0; k < ELEMENTSOF(b->id.bytes); k++) {
  64                 r = CMP(a->id.bytes[k], b->id.bytes[k]);
  65                 if (r != 0)
  66                         return r;
  67         }
  68
  69         return strcmp(a->language, b->language);
  70 }
  71
  72 const struct hash_ops catalog_hash_ops = {
  73         .hash = catalog_hash_func,
  74         .compare = (comparison_fn_t) catalog_compare_func,
  75 };
  76
  77 static bool next_header(const char **s) {
  78         const char *e;
  79
  80         e = strchr(*s, '\n');
  81
  82         /* Unexpected end */
  83         if (!e)
  84                 return false;
  85
  86         /* End of headers */
  87         if (e == *s)
  88                 return false;
  89
  90         *s = e + 1;
  91         return true;
  92 }
  93
  94 static const char *skip_header(const char *s) {
  95         while (next_header(&s))
  96                 ;
  97         return s;
  98 }
  99
 100 static char *combine_entries(const char *one, const char *two) {
 101         const char *b1, *b2;
 102         size_t l1, l2, n;
 103         char *dest, *p;
 104
 105         /* Find split point of headers to body */
 106         b1 = skip_header(one);
 107         b2 = skip_header(two);
 108
 109         l1 = strlen(one);
 110         l2 = strlen(two);
 111         dest = new(char, l1 + l2 + 1);
 112         if (!dest) {
 113                 log_oom();
 114                 return NULL;
 115         }
 116
 117         p = dest;
 118
 119         /* Headers from @one */
 120         n = b1 - one;
 121         p = mempcpy(p, one, n);
 122
 123         /* Headers from @two, these will only be found if not present above */
 124         n = b2 - two;
 125         p = mempcpy(p, two, n);
 126
 127         /* Body from @one */
 128         n = l1 - (b1 - one);
 129         if (n > 0) {
 130                 memcpy(p, b1, n);
 131                 p += n;
 132
 133         /* Body from @two */
 134         } else {
 135                 n = l2 - (b2 - two);
 136                 memcpy(p, b2, n);
 137                 p += n;
 138         }
 139
 140         assert(p - dest <= (ptrdiff_t)(l1 + l2));
 141         p[0] = '\0';
 142         return dest;
 143 }
 144
 145 static int finish_item(
 146                 Hashmap *h,
 147                 sd_id128_t id,
 148                 const char *language,
 149                 char *payload, size_t payload_size) {
 150
 151         _cleanup_free_ CatalogItem *i = NULL;
 152         _cleanup_free_ char *prev = NULL, *combined = NULL;
 153
 154         assert(h);
 155         assert(payload);
 156         assert(payload_size > 0);
 157
 158         i = new0(CatalogItem, 1);
 159         if (!i)
 160                 return log_oom();
 161
 162         i->id = id;
 163         if (language) {
 164                 assert(strlen(language) > 1 && strlen(language) < 32);
 165                 strcpy(i->language, language);
 166         }
 167
 168         prev = hashmap_get(h, i);
 169         if (prev) {
 170                 /* Already have such an item, combine them */
 171                 combined = combine_entries(payload, prev);
 172                 if (!combined)
 173                         return log_oom();
 174
 175                 if (hashmap_update(h, i, combined) < 0)
 176                         return log_oom();
 177                 combined = NULL;
 178         } else {
 179                 /* A new item */
 180                 combined = memdup(payload, payload_size + 1);
 181                 if (!combined)
 182                         return log_oom();
 183
 184                 if (hashmap_put(h, i, combined) < 0)
 185                         return log_oom();
 186                 i = NULL;
 187                 combined = NULL;
 188         }
 189
 190         return 0;
 191 }
 192
 193 int catalog_file_lang(const char* filename, char **lang) {
 194         char *beg, *end, *_lang;
 195
 196         end = endswith(filename, ".catalog");
 197         if (!end)
 198                 return 0;
 199
 200         beg = end - 1;
 201         while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
 202                 beg--;
 203
 204         if (*beg != '.' || end <= beg + 1)
 205                 return 0;
 206
 207         _lang = strndup(beg + 1, end - beg - 1);
 208         if (!_lang)
 209                 return -ENOMEM;
 210
 211         *lang = _lang;
 212         return 1;
 213 }
 214
 215 static int catalog_entry_lang(const char* filename, int line,
 216                               const char* t, const char* deflang, char **lang) {
 217         size_t c;
 218
 219         c = strlen(t);
 220         if (c == 0) {
 221                 log_error("[%s:%u] Language too short.", filename, line);
 222                 return -EINVAL;
 223         }
 224         if (c > 31) {
 225                 log_error("[%s:%u] language too long.", filename, line);
 226                 return -EINVAL;
 227         }
 228
 229         if (deflang) {
 230                 if (streq(t, deflang)) {
 231                         log_warning("[%s:%u] language specified unnecessarily",
 232                                     filename, line);
 233                         return 0;
 234                 } else
 235                         log_warning("[%s:%u] language differs from default for file",
 236                                     filename, line);
 237         }
 238
 239         *lang = strdup(t);
 240         if (!*lang)
 241                         return -ENOMEM;
 242
 243         return 0;
 244 }
 245
 246 int catalog_import_file(Hashmap *h, const char *path) {
 247         _cleanup_fclose_ FILE *f = NULL;
 248         _cleanup_free_ char *payload = NULL;
 249         size_t payload_size = 0, payload_allocated = 0;
 250         unsigned n = 0;
 251         sd_id128_t id;
 252         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 253         bool got_id = false, empty_line = true;
 254         int r;
 255
 256         assert(h);
 257         assert(path);
 258
 259         f = fopen(path, "re");
 260         if (!f)
 261                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 262
 263         r = catalog_file_lang(path, &deflang);
 264         if (r < 0)
 265                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 266         if (r == 1)
 267                 log_debug("File %s has language %s.", path, deflang);
 268
 269         for (;;) {
 270                 _cleanup_free_ char *line = NULL;
 271                 size_t line_len;
 272
 273                 r = read_line(f, LONG_LINE_MAX, &line);
 274                 if (r < 0)
 275                         return log_error_errno(r, "Failed to read file %s: %m", path);
 276                 if (r == 0)
 277                         break;
 278
 279                 n++;
 280
 281                 if (isempty(line)) {
 282                         empty_line = true;
 283                         continue;
 284                 }
 285
 286                 if (strchr(COMMENTS, line[0]))
 287                         continue;
 288
 289                 if (empty_line &&
 290                     strlen(line) >= 2+1+32 &&
 291                     line[0] == '-' &&
 292                     line[1] == '-' &&
 293                     line[2] == ' ' &&
 294                     IN_SET(line[2+1+32], ' ', '\0')) {
 295
 296                         bool with_language;
 297                         sd_id128_t jd;
 298
 299                         /* New entry */
 300
 301                         with_language = line[2+1+32] != '\0';
 302                         line[2+1+32] = '\0';
 303
 304                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 305
 306                                 if (got_id) {
 307                                         if (payload_size == 0) {
 308                                                 log_error("[%s:%u] No payload text.", path, n);
 309                                                 return -EINVAL;
 310                                         }
 311
 312                                         r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 313                                         if (r < 0)
 314                                                 return r;
 315
 316                                         lang = mfree(lang);
 317                                         payload_size = 0;
 318                                 }
 319
 320                                 if (with_language) {
 321                                         char *t;
 322
 323                                         t = strstrip(line + 2 + 1 + 32 + 1);
 324                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 325                                         if (r < 0)
 326                                                 return r;
 327                                 }
 328
 329                                 got_id = true;
 330                                 empty_line = false;
 331                                 id = jd;
 332
 333                                 continue;
 334                         }
 335                 }
 336
 337                 /* Payload */
 338                 if (!got_id) {
 339                         log_error("[%s:%u] Got payload before ID.", path, n);
 340                         return -EINVAL;
 341                 }
 342
 343                 line_len = strlen(line);
 344                 if (!GREEDY_REALLOC(payload, payload_allocated,
 345                                     payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
 346                         return log_oom();
 347
 348                 if (empty_line)
 349                         payload[payload_size++] = '\n';
 350                 memcpy(payload + payload_size, line, line_len);
 351                 payload_size += line_len;
 352                 payload[payload_size++] = '\n';
 353                 payload[payload_size] = '\0';
 354
 355                 empty_line = false;
 356         }
 357
 358         if (got_id) {
 359                 if (payload_size == 0) {
 360                         log_error("[%s:%u] No payload text.", path, n);
 361                         return -EINVAL;
 362                 }
 363
 364                 r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 365                 if (r < 0)
 366                         return r;
 367         }
 368
 369         return 0;
 370 }
 371
 372 static int64_t write_catalog(const char *database, struct strbuf *sb,
 373                              CatalogItem *items, size_t n) {
 374         CatalogHeader header;
 375         _cleanup_fclose_ FILE *w = NULL;
 376         int r;
 377         _cleanup_free_ char *d, *p = NULL;
 378         size_t k;
 379
 380         d = dirname_malloc(database);
 381         if (!d)
 382                 return log_oom();
 383
 384         r = mkdir_p(d, 0775);
 385         if (r < 0)
 386                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 387
 388         r = fopen_temporary(database, &w, &p);
 389         if (r < 0)
 390                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 391                                        database);
 392
 393         zero(header);
 394         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 395         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 396         header.catalog_item_size = htole64(sizeof(CatalogItem));
 397         header.n_items = htole64(n);
 398
 399         r = -EIO;
 400
 401         k = fwrite(&header, 1, sizeof(header), w);
 402         if (k != sizeof(header)) {
 403                 log_error("%s: failed to write header.", p);
 404                 goto error;
 405         }
 406
 407         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 408         if (k != n * sizeof(CatalogItem)) {
 409                 log_error("%s: failed to write database.", p);
 410                 goto error;
 411         }
 412
 413         k = fwrite(sb->buf, 1, sb->len, w);
 414         if (k != sb->len) {
 415                 log_error("%s: failed to write strings.", p);
 416                 goto error;
 417         }
 418
 419         r = fflush_and_check(w);
 420         if (r < 0) {
 421                 log_error_errno(r, "%s: failed to write database: %m", p);
 422                 goto error;
 423         }
 424
 425         fchmod(fileno(w), 0644);
 426
 427         if (rename(p, database) < 0) {
 428                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 429                 goto error;
 430         }
 431
 432         return ftello(w);
 433
 434 error:
 435         (void) unlink(p);
 436         return r;
 437 }
 438
 439 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 440         _cleanup_strv_free_ char **files = NULL;
 441         char **f;
 442         _cleanup_(strbuf_cleanupp) struct strbuf *sb = NULL;
 443         _cleanup_hashmap_free_free_free_ Hashmap *h = NULL;
 444         _cleanup_free_ CatalogItem *items = NULL;
 445         ssize_t offset;
 446         char *payload;
 447         CatalogItem *i;
 448         Iterator j;
 449         unsigned n;
 450         int r;
 451         int64_t sz;
 452
 453         h = hashmap_new(&catalog_hash_ops);
 454         sb = strbuf_new();
 455         if (!h || !sb)
 456                 return log_oom();
 457
 458         r = conf_files_list_strv(&files, ".catalog", root, 0, dirs);
 459         if (r < 0)
 460                 return log_error_errno(r, "Failed to get catalog files: %m");
 461
 462         STRV_FOREACH(f, files) {
 463                 log_debug("Reading file '%s'", *f);
 464                 r = catalog_import_file(h, *f);
 465                 if (r < 0)
 466                         return log_error_errno(r, "Failed to import file '%s': %m", *f);
 467         }
 468
 469         if (hashmap_size(h) <= 0) {
 470                 log_info("No items in catalog.");
 471                 return 0;
 472         } else
 473                 log_debug("Found %u items in catalog.", hashmap_size(h));
 474
 475         items = new(CatalogItem, hashmap_size(h));
 476         if (!items)
 477                 return log_oom();
 478
 479         n = 0;
 480         HASHMAP_FOREACH_KEY(payload, i, h, j) {
 481                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 482                           SD_ID128_FORMAT_VAL(i->id),
 483                           isempty(i->language) ? "C" : i->language);
 484
 485                 offset = strbuf_add_string(sb, payload, strlen(payload));
 486                 if (offset < 0)
 487                         return log_oom();
 488
 489                 i->offset = htole64((uint64_t) offset);
 490                 items[n++] = *i;
 491         }
 492
 493         assert(n == hashmap_size(h));
 494         typesafe_qsort(items, n, catalog_compare_func);
 495
 496         strbuf_complete(sb);
 497
 498         sz = write_catalog(database, sb, items, n);
 499         if (sz < 0)
 500                 return log_error_errno(sz, "Failed to write %s: %m", database);
 501
 502         log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 503                   database, n, sb->len, sz);
 504         return 0;
 505 }
 506
 507 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 508         const CatalogHeader *h;
 509         int fd;
 510         void *p;
 511         struct stat st;
 512
 513         assert(_fd);
 514         assert(_st);
 515         assert(_p);
 516
 517         fd = open(database, O_RDONLY|O_CLOEXEC);
 518         if (fd < 0)
 519                 return -errno;
 520
 521         if (fstat(fd, &st) < 0) {
 522                 safe_close(fd);
 523                 return -errno;
 524         }
 525
 526         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 527                 safe_close(fd);
 528                 return -EINVAL;
 529         }
 530
 531         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 532         if (p == MAP_FAILED) {
 533                 safe_close(fd);
 534                 return -errno;
 535         }
 536
 537         h = p;
 538         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 539             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 540             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 541             h->incompatible_flags != 0 ||
 542             le64toh(h->n_items) <= 0 ||
 543             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 544                 safe_close(fd);
 545                 munmap(p, st.st_size);
 546                 return -EBADMSG;
 547         }
 548
 549         *_fd = fd;
 550         *_st = st;
 551         *_p = p;
 552
 553         return 0;
 554 }
 555
 556 static const char *find_id(void *p, sd_id128_t id) {
 557         CatalogItem *f = NULL, key = { .id = id };
 558         const CatalogHeader *h = p;
 559         const char *loc;
 560
 561         loc = setlocale(LC_MESSAGES, NULL);
 562         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 563                 strncpy(key.language, loc, sizeof(key.language));
 564                 key.language[strcspn(key.language, ".@")] = 0;
 565
 566                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), (comparison_fn_t) catalog_compare_func);
 567                 if (!f) {
 568                         char *e;
 569
 570                         e = strchr(key.language, '_');
 571                         if (e) {
 572                                 *e = 0;
 573                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), (comparison_fn_t) catalog_compare_func);
 574                         }
 575                 }
 576         }
 577
 578         if (!f) {
 579                 zero(key.language);
 580                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), (comparison_fn_t) catalog_compare_func);
 581         }
 582
 583         if (!f)
 584                 return NULL;
 585
 586         return (const char*) p +
 587                 le64toh(h->header_size) +
 588                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 589                 le64toh(f->offset);
 590 }
 591
 592 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 593         _cleanup_close_ int fd = -1;
 594         void *p = NULL;
 595         struct stat st = {};
 596         char *text = NULL;
 597         int r;
 598         const char *s;
 599
 600         assert(_text);
 601
 602         r = open_mmap(database, &fd, &st, &p);
 603         if (r < 0)
 604                 return r;
 605
 606         s = find_id(p, id);
 607         if (!s) {
 608                 r = -ENOENT;
 609                 goto finish;
 610         }
 611
 612         text = strdup(s);
 613         if (!text) {
 614                 r = -ENOMEM;
 615                 goto finish;
 616         }
 617
 618         *_text = text;
 619         r = 0;
 620
 621 finish:
 622         if (p)
 623                 munmap(p, st.st_size);
 624
 625         return r;
 626 }
 627
 628 static char *find_header(const char *s, const char *header) {
 629
 630         for (;;) {
 631                 const char *v;
 632
 633                 v = startswith(s, header);
 634                 if (v) {
 635                         v += strspn(v, WHITESPACE);
 636                         return strndup(v, strcspn(v, NEWLINE));
 637                 }
 638
 639                 if (!next_header(&s))
 640                         return NULL;
 641         }
 642 }
 643
 644 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 645         if (oneline) {
 646                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 647
 648                 subject = find_header(s, "Subject:");
 649                 defined_by = find_header(s, "Defined-By:");
 650
 651                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 652                         SD_ID128_FORMAT_VAL(id),
 653                         strna(defined_by), strna(subject));
 654         } else
 655                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 656                         SD_ID128_FORMAT_VAL(id), s);
 657 }
 658
 659 int catalog_list(FILE *f, const char *database, bool oneline) {
 660         _cleanup_close_ int fd = -1;
 661         void *p = NULL;
 662         struct stat st;
 663         const CatalogHeader *h;
 664         const CatalogItem *items;
 665         int r;
 666         unsigned n;
 667         sd_id128_t last_id;
 668         bool last_id_set = false;
 669
 670         r = open_mmap(database, &fd, &st, &p);
 671         if (r < 0)
 672                 return r;
 673
 674         h = p;
 675         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 676
 677         for (n = 0; n < le64toh(h->n_items); n++) {
 678                 const char *s;
 679
 680                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 681                         continue;
 682
 683                 assert_se(s = find_id(p, items[n].id));
 684
 685                 dump_catalog_entry(f, items[n].id, s, oneline);
 686
 687                 last_id_set = true;
 688                 last_id = items[n].id;
 689         }
 690
 691         munmap(p, st.st_size);
 692
 693         return 0;
 694 }
 695
 696 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 697         char **item;
 698         int r = 0;
 699
 700         STRV_FOREACH(item, items) {
 701                 sd_id128_t id;
 702                 int k;
 703                 _cleanup_free_ char *msg = NULL;
 704
 705                 k = sd_id128_from_string(*item, &id);
 706                 if (k < 0) {
 707                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 708                         if (r == 0)
 709                                 r = k;
 710                         continue;
 711                 }
 712
 713                 k = catalog_get(database, id, &msg);
 714                 if (k < 0) {
 715                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 716                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 717                         if (r == 0)
 718                                 r = k;
 719                         continue;
 720                 }
 721
 722                 dump_catalog_entry(f, id, msg, oneline);
 723         }
 724
 725         return r;
 726 }