src/journal/catalog.c

   1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
   2
   3 #include <errno.h>
   4 #include <fcntl.h>
   5 #include <locale.h>
   6 #include <stdio.h>
   7 #include <sys/mman.h>
   8 #include <sys/stat.h>
   9 #include <sys/types.h>
  10 #include <unistd.h>
  11
  12 #include "sd-id128.h"
  13
  14 #include "alloc-util.h"
  15 #include "catalog.h"
  16 #include "conf-files.h"
  17 #include "fd-util.h"
  18 #include "fileio.h"
  19 #include "hashmap.h"
  20 #include "log.h"
  21 #include "memory-util.h"
  22 #include "mkdir.h"
  23 #include "path-util.h"
  24 #include "siphash24.h"
  25 #include "sort-util.h"
  26 #include "sparse-endian.h"
  27 #include "strbuf.h"
  28 #include "string-util.h"
  29 #include "strv.h"
  30 #include "tmpfile-util.h"
  31
  32 const char * const catalog_file_dirs[] = {
  33         "/usr/local/lib/systemd/catalog/",
  34         "/usr/lib/systemd/catalog/",
  35         NULL
  36 };
  37
  38 #define CATALOG_SIGNATURE { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  39
  40 typedef struct CatalogHeader {
  41         uint8_t signature[8];  /* "RHHHKSLP" */
  42         le32_t compatible_flags;
  43         le32_t incompatible_flags;
  44         le64_t header_size;
  45         le64_t n_items;
  46         le64_t catalog_item_size;
  47 } CatalogHeader;
  48
  49 typedef struct CatalogItem {
  50         sd_id128_t id;
  51         char language[32]; /* One byte is used for termination, so the maximum allowed
  52                             * length of the string is actually 31 bytes. */
  53         le64_t offset;
  54 } CatalogItem;
  55
  56 static void catalog_hash_func(const CatalogItem *i, struct siphash *state) {
  57         siphash24_compress(&i->id, sizeof(i->id), state);
  58         siphash24_compress_string(i->language, state);
  59 }
  60
  61 static int catalog_compare_func(const CatalogItem *a, const CatalogItem *b) {
  62         unsigned k;
  63         int r;
  64
  65         for (k = 0; k < ELEMENTSOF(b->id.bytes); k++) {
  66                 r = CMP(a->id.bytes[k], b->id.bytes[k]);
  67                 if (r != 0)
  68                         return r;
  69         }
  70
  71         return strcmp(a->language, b->language);
  72 }
  73
  74 DEFINE_HASH_OPS(catalog_hash_ops, CatalogItem, catalog_hash_func, catalog_compare_func);
  75
  76 static bool next_header(const char **s) {
  77         const char *e;
  78
  79         e = strchr(*s, '\n');
  80
  81         /* Unexpected end */
  82         if (!e)
  83                 return false;
  84
  85         /* End of headers */
  86         if (e == *s)
  87                 return false;
  88
  89         *s = e + 1;
  90         return true;
  91 }
  92
  93 static const char *skip_header(const char *s) {
  94         while (next_header(&s))
  95                 ;
  96         return s;
  97 }
  98
  99 static char *combine_entries(const char *one, const char *two) {
 100         const char *b1, *b2;
 101         size_t l1, l2, n;
 102         char *dest, *p;
 103
 104         /* Find split point of headers to body */
 105         b1 = skip_header(one);
 106         b2 = skip_header(two);
 107
 108         l1 = strlen(one);
 109         l2 = strlen(two);
 110         dest = new(char, l1 + l2 + 1);
 111         if (!dest) {
 112                 log_oom();
 113                 return NULL;
 114         }
 115
 116         p = dest;
 117
 118         /* Headers from @one */
 119         n = b1 - one;
 120         p = mempcpy(p, one, n);
 121
 122         /* Headers from @two, these will only be found if not present above */
 123         n = b2 - two;
 124         p = mempcpy(p, two, n);
 125
 126         /* Body from @one */
 127         n = l1 - (b1 - one);
 128         if (n > 0) {
 129                 memcpy(p, b1, n);
 130                 p += n;
 131
 132         /* Body from @two */
 133         } else {
 134                 n = l2 - (b2 - two);
 135                 memcpy(p, b2, n);
 136                 p += n;
 137         }
 138
 139         assert(p - dest <= (ptrdiff_t)(l1 + l2));
 140         p[0] = '\0';
 141         return dest;
 142 }
 143
 144 static int finish_item(
 145                 OrderedHashmap *h,
 146                 sd_id128_t id,
 147                 const char *language,
 148                 char *payload, size_t payload_size) {
 149
 150         _cleanup_free_ CatalogItem *i = NULL;
 151         _cleanup_free_ char *prev = NULL, *combined = NULL;
 152
 153         assert(h);
 154         assert(payload);
 155         assert(payload_size > 0);
 156
 157         i = new0(CatalogItem, 1);
 158         if (!i)
 159                 return log_oom();
 160
 161         i->id = id;
 162         if (language) {
 163                 assert(strlen(language) > 1 && strlen(language) < 32);
 164                 strcpy(i->language, language);
 165         }
 166
 167         prev = ordered_hashmap_get(h, i);
 168         if (prev) {
 169                 /* Already have such an item, combine them */
 170                 combined = combine_entries(payload, prev);
 171                 if (!combined)
 172                         return log_oom();
 173
 174                 if (ordered_hashmap_update(h, i, combined) < 0)
 175                         return log_oom();
 176                 combined = NULL;
 177         } else {
 178                 /* A new item */
 179                 combined = memdup(payload, payload_size + 1);
 180                 if (!combined)
 181                         return log_oom();
 182
 183                 if (ordered_hashmap_put(h, i, combined) < 0)
 184                         return log_oom();
 185                 i = NULL;
 186                 combined = NULL;
 187         }
 188
 189         return 0;
 190 }
 191
 192 int catalog_file_lang(const char* filename, char **lang) {
 193         char *beg, *end, *_lang;
 194
 195         end = endswith(filename, ".catalog");
 196         if (!end)
 197                 return 0;
 198
 199         beg = end - 1;
 200         while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
 201                 beg--;
 202
 203         if (*beg != '.' || end <= beg + 1)
 204                 return 0;
 205
 206         _lang = strndup(beg + 1, end - beg - 1);
 207         if (!_lang)
 208                 return -ENOMEM;
 209
 210         *lang = _lang;
 211         return 1;
 212 }
 213
 214 static int catalog_entry_lang(
 215                 const char* filename,
 216                 unsigned line,
 217                 const char* t,
 218                 const char* deflang,
 219                 char **ret) {
 220
 221         size_t c;
 222         char *z;
 223
 224         c = strlen(t);
 225         if (c < 2)
 226                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 227                                        "[%s:%u] Language too short.", filename, line);
 228         if (c > 31)
 229                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 230                                        "[%s:%u] language too long.", filename, line);
 231
 232         if (deflang) {
 233                 if (streq(t, deflang)) {
 234                         log_warning("[%s:%u] language specified unnecessarily", filename, line);
 235                         return 0;
 236                 }
 237
 238                 log_warning("[%s:%u] language differs from default for file", filename, line);
 239         }
 240
 241         z = strdup(t);
 242         if (!z)
 243                 return -ENOMEM;
 244
 245         *ret = z;
 246         return 0;
 247 }
 248
 249 int catalog_import_file(OrderedHashmap *h, const char *path) {
 250         _cleanup_fclose_ FILE *f = NULL;
 251         _cleanup_free_ char *payload = NULL;
 252         size_t payload_size = 0, payload_allocated = 0;
 253         unsigned n = 0;
 254         sd_id128_t id;
 255         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 256         bool got_id = false, empty_line = true;
 257         int r;
 258
 259         assert(h);
 260         assert(path);
 261
 262         f = fopen(path, "re");
 263         if (!f)
 264                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 265
 266         r = catalog_file_lang(path, &deflang);
 267         if (r < 0)
 268                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 269         if (r == 1)
 270                 log_debug("File %s has language %s.", path, deflang);
 271
 272         for (;;) {
 273                 _cleanup_free_ char *line = NULL;
 274                 size_t line_len;
 275
 276                 r = read_line(f, LONG_LINE_MAX, &line);
 277                 if (r < 0)
 278                         return log_error_errno(r, "Failed to read file %s: %m", path);
 279                 if (r == 0)
 280                         break;
 281
 282                 n++;
 283
 284                 if (isempty(line)) {
 285                         empty_line = true;
 286                         continue;
 287                 }
 288
 289                 if (strchr(COMMENTS, line[0]))
 290                         continue;
 291
 292                 if (empty_line &&
 293                     strlen(line) >= 2+1+32 &&
 294                     line[0] == '-' &&
 295                     line[1] == '-' &&
 296                     line[2] == ' ' &&
 297                     IN_SET(line[2+1+32], ' ', '\0')) {
 298
 299                         bool with_language;
 300                         sd_id128_t jd;
 301
 302                         /* New entry */
 303
 304                         with_language = line[2+1+32] != '\0';
 305                         line[2+1+32] = '\0';
 306
 307                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 308
 309                                 if (got_id) {
 310                                         if (payload_size == 0)
 311                                                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 312                                                                        "[%s:%u] No payload text.",
 313                                                                        path,
 314                                                                        n);
 315
 316                                         r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 317                                         if (r < 0)
 318                                                 return r;
 319
 320                                         lang = mfree(lang);
 321                                         payload_size = 0;
 322                                 }
 323
 324                                 if (with_language) {
 325                                         char *t;
 326
 327                                         t = strstrip(line + 2 + 1 + 32 + 1);
 328                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 329                                         if (r < 0)
 330                                                 return r;
 331                                 }
 332
 333                                 got_id = true;
 334                                 empty_line = false;
 335                                 id = jd;
 336
 337                                 continue;
 338                         }
 339                 }
 340
 341                 /* Payload */
 342                 if (!got_id)
 343                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 344                                                "[%s:%u] Got payload before ID.",
 345                                                path, n);
 346
 347                 line_len = strlen(line);
 348                 if (!GREEDY_REALLOC(payload, payload_allocated,
 349                                     payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
 350                         return log_oom();
 351
 352                 if (empty_line)
 353                         payload[payload_size++] = '\n';
 354                 memcpy(payload + payload_size, line, line_len);
 355                 payload_size += line_len;
 356                 payload[payload_size++] = '\n';
 357                 payload[payload_size] = '\0';
 358
 359                 empty_line = false;
 360         }
 361
 362         if (got_id) {
 363                 if (payload_size == 0)
 364                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 365                                                "[%s:%u] No payload text.",
 366                                                path, n);
 367
 368                 r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 369                 if (r < 0)
 370                         return r;
 371         }
 372
 373         return 0;
 374 }
 375
 376 static int64_t write_catalog(
 377                 const char *database,
 378                 struct strbuf *sb,
 379                 CatalogItem *items,
 380                 size_t n) {
 381
 382         _cleanup_fclose_ FILE *w = NULL;
 383         _cleanup_free_ char *p = NULL;
 384         CatalogHeader header;
 385         size_t k;
 386         int r;
 387
 388         r = mkdir_parents(database, 0755);
 389         if (r < 0)
 390                 return log_error_errno(r, "Failed to create parent directories of %s: %m", database);
 391
 392         r = fopen_temporary(database, &w, &p);
 393         if (r < 0)
 394                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 395                                        database);
 396
 397         header = (CatalogHeader) {
 398                 .signature = CATALOG_SIGNATURE,
 399                 .header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8)),
 400                 .catalog_item_size = htole64(sizeof(CatalogItem)),
 401                 .n_items = htole64(n),
 402         };
 403
 404         r = -EIO;
 405
 406         k = fwrite(&header, 1, sizeof(header), w);
 407         if (k != sizeof(header)) {
 408                 log_error("%s: failed to write header.", p);
 409                 goto error;
 410         }
 411
 412         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 413         if (k != n * sizeof(CatalogItem)) {
 414                 log_error("%s: failed to write database.", p);
 415                 goto error;
 416         }
 417
 418         k = fwrite(sb->buf, 1, sb->len, w);
 419         if (k != sb->len) {
 420                 log_error("%s: failed to write strings.", p);
 421                 goto error;
 422         }
 423
 424         r = fflush_and_check(w);
 425         if (r < 0) {
 426                 log_error_errno(r, "%s: failed to write database: %m", p);
 427                 goto error;
 428         }
 429
 430         (void) fchmod(fileno(w), 0644);
 431
 432         if (rename(p, database) < 0) {
 433                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 434                 goto error;
 435         }
 436
 437         return ftello(w);
 438
 439 error:
 440         (void) unlink(p);
 441         return r;
 442 }
 443
 444 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 445         _cleanup_strv_free_ char **files = NULL;
 446         char **f;
 447         _cleanup_(strbuf_cleanupp) struct strbuf *sb = NULL;
 448         _cleanup_ordered_hashmap_free_free_free_ OrderedHashmap *h = NULL;
 449         _cleanup_free_ CatalogItem *items = NULL;
 450         ssize_t offset;
 451         char *payload;
 452         CatalogItem *i;
 453         unsigned n;
 454         int r;
 455         int64_t sz;
 456
 457         h = ordered_hashmap_new(&catalog_hash_ops);
 458         sb = strbuf_new();
 459         if (!h || !sb)
 460                 return log_oom();
 461
 462         r = conf_files_list_strv(&files, ".catalog", root, 0, dirs);
 463         if (r < 0)
 464                 return log_error_errno(r, "Failed to get catalog files: %m");
 465
 466         STRV_FOREACH(f, files) {
 467                 log_debug("Reading file '%s'", *f);
 468                 r = catalog_import_file(h, *f);
 469                 if (r < 0)
 470                         return log_error_errno(r, "Failed to import file '%s': %m", *f);
 471         }
 472
 473         if (ordered_hashmap_size(h) <= 0) {
 474                 log_info("No items in catalog.");
 475                 return 0;
 476         } else
 477                 log_debug("Found %u items in catalog.", ordered_hashmap_size(h));
 478
 479         items = new(CatalogItem, ordered_hashmap_size(h));
 480         if (!items)
 481                 return log_oom();
 482
 483         n = 0;
 484         ORDERED_HASHMAP_FOREACH_KEY(payload, i, h) {
 485                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 486                           SD_ID128_FORMAT_VAL(i->id),
 487                           isempty(i->language) ? "C" : i->language);
 488
 489                 offset = strbuf_add_string(sb, payload, strlen(payload));
 490                 if (offset < 0)
 491                         return log_oom();
 492
 493                 i->offset = htole64((uint64_t) offset);
 494                 items[n++] = *i;
 495         }
 496
 497         assert(n == ordered_hashmap_size(h));
 498         typesafe_qsort(items, n, catalog_compare_func);
 499
 500         strbuf_complete(sb);
 501
 502         sz = write_catalog(database, sb, items, n);
 503         if (sz < 0)
 504                 return log_error_errno(sz, "Failed to write %s: %m", database);
 505
 506         log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 507                   database, n, sb->len, sz);
 508         return 0;
 509 }
 510
 511 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 512         _cleanup_close_ int fd = -1;
 513         const CatalogHeader *h;
 514         struct stat st;
 515         void *p;
 516
 517         assert(_fd);
 518         assert(_st);
 519         assert(_p);
 520
 521         fd = open(database, O_RDONLY|O_CLOEXEC);
 522         if (fd < 0)
 523                 return -errno;
 524
 525         if (fstat(fd, &st) < 0)
 526                 return -errno;
 527
 528         if (st.st_size < (off_t) sizeof(CatalogHeader))
 529                 return -EINVAL;
 530
 531         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 532         if (p == MAP_FAILED)
 533                 return -errno;
 534
 535         h = p;
 536         if (memcmp(h->signature, (const uint8_t[]) CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 537             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 538             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 539             h->incompatible_flags != 0 ||
 540             le64toh(h->n_items) <= 0 ||
 541             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 542                 munmap(p, st.st_size);
 543                 return -EBADMSG;
 544         }
 545
 546         *_fd = TAKE_FD(fd);
 547         *_st = st;
 548         *_p = p;
 549
 550         return 0;
 551 }
 552
 553 static const char *find_id(void *p, sd_id128_t id) {
 554         CatalogItem *f = NULL, key = { .id = id };
 555         const CatalogHeader *h = p;
 556         const char *loc;
 557
 558         loc = setlocale(LC_MESSAGES, NULL);
 559         if (!isempty(loc) && !STR_IN_SET(loc, "C", "POSIX")) {
 560                 size_t len;
 561
 562                 len = strcspn(loc, ".@");
 563                 if (len > sizeof(key.language) - 1)
 564                         log_debug("LC_MESSAGES value too long, ignoring: \"%.*s\"", (int) len, loc);
 565                 else {
 566                         strncpy(key.language, loc, len);
 567                         key.language[len] = '\0';
 568
 569                         f = bsearch(&key,
 570                                     (const uint8_t*) p + le64toh(h->header_size),
 571                                     le64toh(h->n_items),
 572                                     le64toh(h->catalog_item_size),
 573                                     (comparison_fn_t) catalog_compare_func);
 574                         if (!f) {
 575                                 char *e;
 576
 577                                 e = strchr(key.language, '_');
 578                                 if (e) {
 579                                         *e = 0;
 580                                         f = bsearch(&key,
 581                                                     (const uint8_t*) p + le64toh(h->header_size),
 582                                                     le64toh(h->n_items),
 583                                                     le64toh(h->catalog_item_size),
 584                                                     (comparison_fn_t) catalog_compare_func);
 585                                 }
 586                         }
 587                 }
 588         }
 589
 590         if (!f) {
 591                 zero(key.language);
 592                 f = bsearch(&key,
 593                             (const uint8_t*) p + le64toh(h->header_size),
 594                             le64toh(h->n_items),
 595                             le64toh(h->catalog_item_size),
 596                             (comparison_fn_t) catalog_compare_func);
 597         }
 598
 599         if (!f)
 600                 return NULL;
 601
 602         return (const char*) p +
 603                 le64toh(h->header_size) +
 604                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 605                 le64toh(f->offset);
 606 }
 607
 608 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 609         _cleanup_close_ int fd = -1;
 610         void *p = NULL;
 611         struct stat st = {};
 612         char *text = NULL;
 613         int r;
 614         const char *s;
 615
 616         assert(_text);
 617
 618         r = open_mmap(database, &fd, &st, &p);
 619         if (r < 0)
 620                 return r;
 621
 622         s = find_id(p, id);
 623         if (!s) {
 624                 r = -ENOENT;
 625                 goto finish;
 626         }
 627
 628         text = strdup(s);
 629         if (!text) {
 630                 r = -ENOMEM;
 631                 goto finish;
 632         }
 633
 634         *_text = text;
 635         r = 0;
 636
 637 finish:
 638         if (p)
 639                 munmap(p, st.st_size);
 640
 641         return r;
 642 }
 643
 644 static char *find_header(const char *s, const char *header) {
 645
 646         for (;;) {
 647                 const char *v;
 648
 649                 v = startswith(s, header);
 650                 if (v) {
 651                         v += strspn(v, WHITESPACE);
 652                         return strndup(v, strcspn(v, NEWLINE));
 653                 }
 654
 655                 if (!next_header(&s))
 656                         return NULL;
 657         }
 658 }
 659
 660 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 661         if (oneline) {
 662                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 663
 664                 subject = find_header(s, "Subject:");
 665                 defined_by = find_header(s, "Defined-By:");
 666
 667                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 668                         SD_ID128_FORMAT_VAL(id),
 669                         strna(defined_by), strna(subject));
 670         } else
 671                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 672                         SD_ID128_FORMAT_VAL(id), s);
 673 }
 674
 675 int catalog_list(FILE *f, const char *database, bool oneline) {
 676         _cleanup_close_ int fd = -1;
 677         void *p = NULL;
 678         struct stat st;
 679         const CatalogHeader *h;
 680         const CatalogItem *items;
 681         int r;
 682         unsigned n;
 683         sd_id128_t last_id;
 684         bool last_id_set = false;
 685
 686         r = open_mmap(database, &fd, &st, &p);
 687         if (r < 0)
 688                 return r;
 689
 690         h = p;
 691         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 692
 693         for (n = 0; n < le64toh(h->n_items); n++) {
 694                 const char *s;
 695
 696                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 697                         continue;
 698
 699                 assert_se(s = find_id(p, items[n].id));
 700
 701                 dump_catalog_entry(f, items[n].id, s, oneline);
 702
 703                 last_id_set = true;
 704                 last_id = items[n].id;
 705         }
 706
 707         munmap(p, st.st_size);
 708
 709         return 0;
 710 }
 711
 712 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 713         char **item;
 714         int r = 0;
 715
 716         STRV_FOREACH(item, items) {
 717                 sd_id128_t id;
 718                 int k;
 719                 _cleanup_free_ char *msg = NULL;
 720
 721                 k = sd_id128_from_string(*item, &id);
 722                 if (k < 0) {
 723                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 724                         if (r == 0)
 725                                 r = k;
 726                         continue;
 727                 }
 728
 729                 k = catalog_get(database, id, &msg);
 730                 if (k < 0) {
 731                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 732                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 733                         if (r == 0)
 734                                 r = k;
 735                         continue;
 736                 }
 737
 738                 dump_catalog_entry(f, id, msg, oneline);
 739         }
 740
 741         return r;
 742 }