src/journal/catalog.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #include <errno.h>
   4 #include <fcntl.h>
   5 #include <locale.h>
   6 #include <stdio.h>
   7 #include <string.h>
   8 #include <sys/mman.h>
   9 #include <unistd.h>
  10
  11 #include "sd-id128.h"
  12
  13 #include "alloc-util.h"
  14 #include "catalog.h"
  15 #include "conf-files.h"
  16 #include "fd-util.h"
  17 #include "fileio.h"
  18 #include "hashmap.h"
  19 #include "log.h"
  20 #include "mkdir.h"
  21 #include "path-util.h"
  22 #include "siphash24.h"
  23 #include "sparse-endian.h"
  24 #include "strbuf.h"
  25 #include "string-util.h"
  26 #include "strv.h"
  27 #include "tmpfile-util.h"
  28 #include "util.h"
  29
  30 const char * const catalog_file_dirs[] = {
  31         "/usr/local/lib/systemd/catalog/",
  32         "/usr/lib/systemd/catalog/",
  33         NULL
  34 };
  35
  36 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  37
  38 typedef struct CatalogHeader {
  39         uint8_t signature[8];  /* "RHHHKSLP" */
  40         le32_t compatible_flags;
  41         le32_t incompatible_flags;
  42         le64_t header_size;
  43         le64_t n_items;
  44         le64_t catalog_item_size;
  45 } CatalogHeader;
  46
  47 typedef struct CatalogItem {
  48         sd_id128_t id;
  49         char language[32];
  50         le64_t offset;
  51 } CatalogItem;
  52
  53 static void catalog_hash_func(const CatalogItem *i, struct siphash *state) {
  54         siphash24_compress(&i->id, sizeof(i->id), state);
  55         siphash24_compress(i->language, strlen(i->language), state);
  56 }
  57
  58 static int catalog_compare_func(const CatalogItem *a, const CatalogItem *b) {
  59         unsigned k;
  60         int r;
  61
  62         for (k = 0; k < ELEMENTSOF(b->id.bytes); k++) {
  63                 r = CMP(a->id.bytes[k], b->id.bytes[k]);
  64                 if (r != 0)
  65                         return r;
  66         }
  67
  68         return strcmp(a->language, b->language);
  69 }
  70
  71 DEFINE_HASH_OPS(catalog_hash_ops, CatalogItem, catalog_hash_func, catalog_compare_func);
  72
  73 static bool next_header(const char **s) {
  74         const char *e;
  75
  76         e = strchr(*s, '\n');
  77
  78         /* Unexpected end */
  79         if (!e)
  80                 return false;
  81
  82         /* End of headers */
  83         if (e == *s)
  84                 return false;
  85
  86         *s = e + 1;
  87         return true;
  88 }
  89
  90 static const char *skip_header(const char *s) {
  91         while (next_header(&s))
  92                 ;
  93         return s;
  94 }
  95
  96 static char *combine_entries(const char *one, const char *two) {
  97         const char *b1, *b2;
  98         size_t l1, l2, n;
  99         char *dest, *p;
 100
 101         /* Find split point of headers to body */
 102         b1 = skip_header(one);
 103         b2 = skip_header(two);
 104
 105         l1 = strlen(one);
 106         l2 = strlen(two);
 107         dest = new(char, l1 + l2 + 1);
 108         if (!dest) {
 109                 log_oom();
 110                 return NULL;
 111         }
 112
 113         p = dest;
 114
 115         /* Headers from @one */
 116         n = b1 - one;
 117         p = mempcpy(p, one, n);
 118
 119         /* Headers from @two, these will only be found if not present above */
 120         n = b2 - two;
 121         p = mempcpy(p, two, n);
 122
 123         /* Body from @one */
 124         n = l1 - (b1 - one);
 125         if (n > 0) {
 126                 memcpy(p, b1, n);
 127                 p += n;
 128
 129         /* Body from @two */
 130         } else {
 131                 n = l2 - (b2 - two);
 132                 memcpy(p, b2, n);
 133                 p += n;
 134         }
 135
 136         assert(p - dest <= (ptrdiff_t)(l1 + l2));
 137         p[0] = '\0';
 138         return dest;
 139 }
 140
 141 static int finish_item(
 142                 Hashmap *h,
 143                 sd_id128_t id,
 144                 const char *language,
 145                 char *payload, size_t payload_size) {
 146
 147         _cleanup_free_ CatalogItem *i = NULL;
 148         _cleanup_free_ char *prev = NULL, *combined = NULL;
 149
 150         assert(h);
 151         assert(payload);
 152         assert(payload_size > 0);
 153
 154         i = new0(CatalogItem, 1);
 155         if (!i)
 156                 return log_oom();
 157
 158         i->id = id;
 159         if (language) {
 160                 assert(strlen(language) > 1 && strlen(language) < 32);
 161                 strcpy(i->language, language);
 162         }
 163
 164         prev = hashmap_get(h, i);
 165         if (prev) {
 166                 /* Already have such an item, combine them */
 167                 combined = combine_entries(payload, prev);
 168                 if (!combined)
 169                         return log_oom();
 170
 171                 if (hashmap_update(h, i, combined) < 0)
 172                         return log_oom();
 173                 combined = NULL;
 174         } else {
 175                 /* A new item */
 176                 combined = memdup(payload, payload_size + 1);
 177                 if (!combined)
 178                         return log_oom();
 179
 180                 if (hashmap_put(h, i, combined) < 0)
 181                         return log_oom();
 182                 i = NULL;
 183                 combined = NULL;
 184         }
 185
 186         return 0;
 187 }
 188
 189 int catalog_file_lang(const char* filename, char **lang) {
 190         char *beg, *end, *_lang;
 191
 192         end = endswith(filename, ".catalog");
 193         if (!end)
 194                 return 0;
 195
 196         beg = end - 1;
 197         while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
 198                 beg--;
 199
 200         if (*beg != '.' || end <= beg + 1)
 201                 return 0;
 202
 203         _lang = strndup(beg + 1, end - beg - 1);
 204         if (!_lang)
 205                 return -ENOMEM;
 206
 207         *lang = _lang;
 208         return 1;
 209 }
 210
 211 static int catalog_entry_lang(const char* filename, int line,
 212                               const char* t, const char* deflang, char **lang) {
 213         size_t c;
 214
 215         c = strlen(t);
 216         if (c < 2)
 217                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 218                                        "[%s:%u] Language too short.",
 219                                        filename, line);
 220         if (c > 31)
 221                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 222                                        "[%s:%u] language too long.", filename,
 223                                        line);
 224
 225         if (deflang) {
 226                 if (streq(t, deflang)) {
 227                         log_warning("[%s:%u] language specified unnecessarily",
 228                                     filename, line);
 229                         return 0;
 230                 } else
 231                         log_warning("[%s:%u] language differs from default for file",
 232                                     filename, line);
 233         }
 234
 235         *lang = strdup(t);
 236         if (!*lang)
 237                         return -ENOMEM;
 238
 239         return 0;
 240 }
 241
 242 int catalog_import_file(Hashmap *h, const char *path) {
 243         _cleanup_fclose_ FILE *f = NULL;
 244         _cleanup_free_ char *payload = NULL;
 245         size_t payload_size = 0, payload_allocated = 0;
 246         unsigned n = 0;
 247         sd_id128_t id;
 248         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 249         bool got_id = false, empty_line = true;
 250         int r;
 251
 252         assert(h);
 253         assert(path);
 254
 255         f = fopen(path, "re");
 256         if (!f)
 257                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 258
 259         r = catalog_file_lang(path, &deflang);
 260         if (r < 0)
 261                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 262         if (r == 1)
 263                 log_debug("File %s has language %s.", path, deflang);
 264
 265         for (;;) {
 266                 _cleanup_free_ char *line = NULL;
 267                 size_t line_len;
 268
 269                 r = read_line(f, LONG_LINE_MAX, &line);
 270                 if (r < 0)
 271                         return log_error_errno(r, "Failed to read file %s: %m", path);
 272                 if (r == 0)
 273                         break;
 274
 275                 n++;
 276
 277                 if (isempty(line)) {
 278                         empty_line = true;
 279                         continue;
 280                 }
 281
 282                 if (strchr(COMMENTS, line[0]))
 283                         continue;
 284
 285                 if (empty_line &&
 286                     strlen(line) >= 2+1+32 &&
 287                     line[0] == '-' &&
 288                     line[1] == '-' &&
 289                     line[2] == ' ' &&
 290                     IN_SET(line[2+1+32], ' ', '\0')) {
 291
 292                         bool with_language;
 293                         sd_id128_t jd;
 294
 295                         /* New entry */
 296
 297                         with_language = line[2+1+32] != '\0';
 298                         line[2+1+32] = '\0';
 299
 300                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 301
 302                                 if (got_id) {
 303                                         if (payload_size == 0)
 304                                                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 305                                                                        "[%s:%u] No payload text.",
 306                                                                        path,
 307                                                                        n);
 308
 309                                         r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 310                                         if (r < 0)
 311                                                 return r;
 312
 313                                         lang = mfree(lang);
 314                                         payload_size = 0;
 315                                 }
 316
 317                                 if (with_language) {
 318                                         char *t;
 319
 320                                         t = strstrip(line + 2 + 1 + 32 + 1);
 321                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 322                                         if (r < 0)
 323                                                 return r;
 324                                 }
 325
 326                                 got_id = true;
 327                                 empty_line = false;
 328                                 id = jd;
 329
 330                                 continue;
 331                         }
 332                 }
 333
 334                 /* Payload */
 335                 if (!got_id)
 336                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 337                                                "[%s:%u] Got payload before ID.",
 338                                                path, n);
 339
 340                 line_len = strlen(line);
 341                 if (!GREEDY_REALLOC(payload, payload_allocated,
 342                                     payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
 343                         return log_oom();
 344
 345                 if (empty_line)
 346                         payload[payload_size++] = '\n';
 347                 memcpy(payload + payload_size, line, line_len);
 348                 payload_size += line_len;
 349                 payload[payload_size++] = '\n';
 350                 payload[payload_size] = '\0';
 351
 352                 empty_line = false;
 353         }
 354
 355         if (got_id) {
 356                 if (payload_size == 0)
 357                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 358                                                "[%s:%u] No payload text.",
 359                                                path, n);
 360
 361                 r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 362                 if (r < 0)
 363                         return r;
 364         }
 365
 366         return 0;
 367 }
 368
 369 static int64_t write_catalog(const char *database, struct strbuf *sb,
 370                              CatalogItem *items, size_t n) {
 371         CatalogHeader header;
 372         _cleanup_fclose_ FILE *w = NULL;
 373         int r;
 374         _cleanup_free_ char *d, *p = NULL;
 375         size_t k;
 376
 377         d = dirname_malloc(database);
 378         if (!d)
 379                 return log_oom();
 380
 381         r = mkdir_p(d, 0775);
 382         if (r < 0)
 383                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 384
 385         r = fopen_temporary(database, &w, &p);
 386         if (r < 0)
 387                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 388                                        database);
 389
 390         zero(header);
 391         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 392         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 393         header.catalog_item_size = htole64(sizeof(CatalogItem));
 394         header.n_items = htole64(n);
 395
 396         r = -EIO;
 397
 398         k = fwrite(&header, 1, sizeof(header), w);
 399         if (k != sizeof(header)) {
 400                 log_error("%s: failed to write header.", p);
 401                 goto error;
 402         }
 403
 404         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 405         if (k != n * sizeof(CatalogItem)) {
 406                 log_error("%s: failed to write database.", p);
 407                 goto error;
 408         }
 409
 410         k = fwrite(sb->buf, 1, sb->len, w);
 411         if (k != sb->len) {
 412                 log_error("%s: failed to write strings.", p);
 413                 goto error;
 414         }
 415
 416         r = fflush_and_check(w);
 417         if (r < 0) {
 418                 log_error_errno(r, "%s: failed to write database: %m", p);
 419                 goto error;
 420         }
 421
 422         fchmod(fileno(w), 0644);
 423
 424         if (rename(p, database) < 0) {
 425                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 426                 goto error;
 427         }
 428
 429         return ftello(w);
 430
 431 error:
 432         (void) unlink(p);
 433         return r;
 434 }
 435
 436 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 437         _cleanup_strv_free_ char **files = NULL;
 438         char **f;
 439         _cleanup_(strbuf_cleanupp) struct strbuf *sb = NULL;
 440         _cleanup_hashmap_free_free_free_ Hashmap *h = NULL;
 441         _cleanup_free_ CatalogItem *items = NULL;
 442         ssize_t offset;
 443         char *payload;
 444         CatalogItem *i;
 445         Iterator j;
 446         unsigned n;
 447         int r;
 448         int64_t sz;
 449
 450         h = hashmap_new(&catalog_hash_ops);
 451         sb = strbuf_new();
 452         if (!h || !sb)
 453                 return log_oom();
 454
 455         r = conf_files_list_strv(&files, ".catalog", root, 0, dirs);
 456         if (r < 0)
 457                 return log_error_errno(r, "Failed to get catalog files: %m");
 458
 459         STRV_FOREACH(f, files) {
 460                 log_debug("Reading file '%s'", *f);
 461                 r = catalog_import_file(h, *f);
 462                 if (r < 0)
 463                         return log_error_errno(r, "Failed to import file '%s': %m", *f);
 464         }
 465
 466         if (hashmap_size(h) <= 0) {
 467                 log_info("No items in catalog.");
 468                 return 0;
 469         } else
 470                 log_debug("Found %u items in catalog.", hashmap_size(h));
 471
 472         items = new(CatalogItem, hashmap_size(h));
 473         if (!items)
 474                 return log_oom();
 475
 476         n = 0;
 477         HASHMAP_FOREACH_KEY(payload, i, h, j) {
 478                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 479                           SD_ID128_FORMAT_VAL(i->id),
 480                           isempty(i->language) ? "C" : i->language);
 481
 482                 offset = strbuf_add_string(sb, payload, strlen(payload));
 483                 if (offset < 0)
 484                         return log_oom();
 485
 486                 i->offset = htole64((uint64_t) offset);
 487                 items[n++] = *i;
 488         }
 489
 490         assert(n == hashmap_size(h));
 491         typesafe_qsort(items, n, catalog_compare_func);
 492
 493         strbuf_complete(sb);
 494
 495         sz = write_catalog(database, sb, items, n);
 496         if (sz < 0)
 497                 return log_error_errno(sz, "Failed to write %s: %m", database);
 498
 499         log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 500                   database, n, sb->len, sz);
 501         return 0;
 502 }
 503
 504 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 505         const CatalogHeader *h;
 506         int fd;
 507         void *p;
 508         struct stat st;
 509
 510         assert(_fd);
 511         assert(_st);
 512         assert(_p);
 513
 514         fd = open(database, O_RDONLY|O_CLOEXEC);
 515         if (fd < 0)
 516                 return -errno;
 517
 518         if (fstat(fd, &st) < 0) {
 519                 safe_close(fd);
 520                 return -errno;
 521         }
 522
 523         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 524                 safe_close(fd);
 525                 return -EINVAL;
 526         }
 527
 528         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 529         if (p == MAP_FAILED) {
 530                 safe_close(fd);
 531                 return -errno;
 532         }
 533
 534         h = p;
 535         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 536             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 537             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 538             h->incompatible_flags != 0 ||
 539             le64toh(h->n_items) <= 0 ||
 540             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 541                 safe_close(fd);
 542                 munmap(p, st.st_size);
 543                 return -EBADMSG;
 544         }
 545
 546         *_fd = fd;
 547         *_st = st;
 548         *_p = p;
 549
 550         return 0;
 551 }
 552
 553 static const char *find_id(void *p, sd_id128_t id) {
 554         CatalogItem *f = NULL, key = { .id = id };
 555         const CatalogHeader *h = p;
 556         const char *loc;
 557
 558         loc = setlocale(LC_MESSAGES, NULL);
 559         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 560                 strncpy(key.language, loc, sizeof(key.language));
 561                 key.language[strcspn(key.language, ".@")] = 0;
 562
 563                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), (comparison_fn_t) catalog_compare_func);
 564                 if (!f) {
 565                         char *e;
 566
 567                         e = strchr(key.language, '_');
 568                         if (e) {
 569                                 *e = 0;
 570                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), (comparison_fn_t) catalog_compare_func);
 571                         }
 572                 }
 573         }
 574
 575         if (!f) {
 576                 zero(key.language);
 577                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), (comparison_fn_t) catalog_compare_func);
 578         }
 579
 580         if (!f)
 581                 return NULL;
 582
 583         return (const char*) p +
 584                 le64toh(h->header_size) +
 585                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 586                 le64toh(f->offset);
 587 }
 588
 589 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 590         _cleanup_close_ int fd = -1;
 591         void *p = NULL;
 592         struct stat st = {};
 593         char *text = NULL;
 594         int r;
 595         const char *s;
 596
 597         assert(_text);
 598
 599         r = open_mmap(database, &fd, &st, &p);
 600         if (r < 0)
 601                 return r;
 602
 603         s = find_id(p, id);
 604         if (!s) {
 605                 r = -ENOENT;
 606                 goto finish;
 607         }
 608
 609         text = strdup(s);
 610         if (!text) {
 611                 r = -ENOMEM;
 612                 goto finish;
 613         }
 614
 615         *_text = text;
 616         r = 0;
 617
 618 finish:
 619         if (p)
 620                 munmap(p, st.st_size);
 621
 622         return r;
 623 }
 624
 625 static char *find_header(const char *s, const char *header) {
 626
 627         for (;;) {
 628                 const char *v;
 629
 630                 v = startswith(s, header);
 631                 if (v) {
 632                         v += strspn(v, WHITESPACE);
 633                         return strndup(v, strcspn(v, NEWLINE));
 634                 }
 635
 636                 if (!next_header(&s))
 637                         return NULL;
 638         }
 639 }
 640
 641 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 642         if (oneline) {
 643                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 644
 645                 subject = find_header(s, "Subject:");
 646                 defined_by = find_header(s, "Defined-By:");
 647
 648                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 649                         SD_ID128_FORMAT_VAL(id),
 650                         strna(defined_by), strna(subject));
 651         } else
 652                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 653                         SD_ID128_FORMAT_VAL(id), s);
 654 }
 655
 656 int catalog_list(FILE *f, const char *database, bool oneline) {
 657         _cleanup_close_ int fd = -1;
 658         void *p = NULL;
 659         struct stat st;
 660         const CatalogHeader *h;
 661         const CatalogItem *items;
 662         int r;
 663         unsigned n;
 664         sd_id128_t last_id;
 665         bool last_id_set = false;
 666
 667         r = open_mmap(database, &fd, &st, &p);
 668         if (r < 0)
 669                 return r;
 670
 671         h = p;
 672         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 673
 674         for (n = 0; n < le64toh(h->n_items); n++) {
 675                 const char *s;
 676
 677                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 678                         continue;
 679
 680                 assert_se(s = find_id(p, items[n].id));
 681
 682                 dump_catalog_entry(f, items[n].id, s, oneline);
 683
 684                 last_id_set = true;
 685                 last_id = items[n].id;
 686         }
 687
 688         munmap(p, st.st_size);
 689
 690         return 0;
 691 }
 692
 693 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 694         char **item;
 695         int r = 0;
 696
 697         STRV_FOREACH(item, items) {
 698                 sd_id128_t id;
 699                 int k;
 700                 _cleanup_free_ char *msg = NULL;
 701
 702                 k = sd_id128_from_string(*item, &id);
 703                 if (k < 0) {
 704                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 705                         if (r == 0)
 706                                 r = k;
 707                         continue;
 708                 }
 709
 710                 k = catalog_get(database, id, &msg);
 711                 if (k < 0) {
 712                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 713                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 714                         if (r == 0)
 715                                 r = k;
 716                         continue;
 717                 }
 718
 719                 dump_catalog_entry(f, id, msg, oneline);
 720         }
 721
 722         return r;
 723 }