src/libsystemd/sd-journal/catalog.c

   1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
   2
   3 #include <errno.h>
   4 #include <fcntl.h>
   5 #include <locale.h>
   6 #include <stdio.h>
   7 #include <sys/mman.h>
   8 #include <sys/stat.h>
   9 #include <sys/types.h>
  10 #include <unistd.h>
  11
  12 #include "sd-id128.h"
  13
  14 #include "alloc-util.h"
  15 #include "catalog.h"
  16 #include "conf-files.h"
  17 #include "fd-util.h"
  18 #include "fileio.h"
  19 #include "fs-util.h"
  20 #include "hashmap.h"
  21 #include "log.h"
  22 #include "memory-util.h"
  23 #include "mkdir.h"
  24 #include "path-util.h"
  25 #include "siphash24.h"
  26 #include "sort-util.h"
  27 #include "sparse-endian.h"
  28 #include "strbuf.h"
  29 #include "string-util.h"
  30 #include "strv.h"
  31 #include "tmpfile-util.h"
  32
  33 const char * const catalog_file_dirs[] = {
  34         "/usr/local/lib/systemd/catalog/",
  35         "/usr/lib/systemd/catalog/",
  36         NULL
  37 };
  38
  39 #define CATALOG_SIGNATURE { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  40
  41 typedef struct CatalogHeader {
  42         uint8_t signature[8];  /* "RHHHKSLP" */
  43         le32_t compatible_flags;
  44         le32_t incompatible_flags;
  45         le64_t header_size;
  46         le64_t n_items;
  47         le64_t catalog_item_size;
  48 } CatalogHeader;
  49
  50 typedef struct CatalogItem {
  51         sd_id128_t id;
  52         char language[32]; /* One byte is used for termination, so the maximum allowed
  53                             * length of the string is actually 31 bytes. */
  54         le64_t offset;
  55 } CatalogItem;
  56
  57 static void catalog_hash_func(const CatalogItem *i, struct siphash *state) {
  58         siphash24_compress(&i->id, sizeof(i->id), state);
  59         siphash24_compress_string(i->language, state);
  60 }
  61
  62 static int catalog_compare_func(const CatalogItem *a, const CatalogItem *b) {
  63         unsigned k;
  64         int r;
  65
  66         for (k = 0; k < ELEMENTSOF(b->id.bytes); k++) {
  67                 r = CMP(a->id.bytes[k], b->id.bytes[k]);
  68                 if (r != 0)
  69                         return r;
  70         }
  71
  72         return strcmp(a->language, b->language);
  73 }
  74
  75 DEFINE_HASH_OPS(catalog_hash_ops, CatalogItem, catalog_hash_func, catalog_compare_func);
  76
  77 static bool next_header(const char **s) {
  78         const char *e;
  79
  80         e = strchr(*s, '\n');
  81
  82         /* Unexpected end */
  83         if (!e)
  84                 return false;
  85
  86         /* End of headers */
  87         if (e == *s)
  88                 return false;
  89
  90         *s = e + 1;
  91         return true;
  92 }
  93
  94 static const char *skip_header(const char *s) {
  95         while (next_header(&s))
  96                 ;
  97         return s;
  98 }
  99
 100 static char *combine_entries(const char *one, const char *two) {
 101         const char *b1, *b2;
 102         size_t l1, l2, n;
 103         char *dest, *p;
 104
 105         /* Find split point of headers to body */
 106         b1 = skip_header(one);
 107         b2 = skip_header(two);
 108
 109         l1 = strlen(one);
 110         l2 = strlen(two);
 111         dest = new(char, l1 + l2 + 1);
 112         if (!dest) {
 113                 log_oom();
 114                 return NULL;
 115         }
 116
 117         p = dest;
 118
 119         /* Headers from @one */
 120         n = b1 - one;
 121         p = mempcpy(p, one, n);
 122
 123         /* Headers from @two, these will only be found if not present above */
 124         n = b2 - two;
 125         p = mempcpy(p, two, n);
 126
 127         /* Body from @one */
 128         n = l1 - (b1 - one);
 129         if (n > 0)
 130                 p = mempcpy(p, b1, n);
 131         /* Body from @two */
 132         else {
 133                 n = l2 - (b2 - two);
 134                 p = mempcpy(p, b2, n);
 135         }
 136
 137         assert(p - dest <= (ptrdiff_t)(l1 + l2));
 138         p[0] = '\0';
 139         return dest;
 140 }
 141
 142 static int finish_item(
 143                 OrderedHashmap *h,
 144                 sd_id128_t id,
 145                 const char *language,
 146                 char *payload, size_t payload_size) {
 147
 148         _cleanup_free_ CatalogItem *i = NULL;
 149         _cleanup_free_ char *combined = NULL;
 150         char *prev;
 151         int r;
 152
 153         assert(h);
 154         assert(payload);
 155         assert(payload_size > 0);
 156
 157         i = new0(CatalogItem, 1);
 158         if (!i)
 159                 return log_oom();
 160
 161         i->id = id;
 162         if (language) {
 163                 assert(strlen(language) > 1 && strlen(language) < 32);
 164                 strcpy(i->language, language);
 165         }
 166
 167         prev = ordered_hashmap_get(h, i);
 168         if (prev) {
 169                 /* Already have such an item, combine them */
 170                 combined = combine_entries(payload, prev);
 171                 if (!combined)
 172                         return log_oom();
 173
 174                 r = ordered_hashmap_update(h, i, combined);
 175                 if (r < 0)
 176                         return log_error_errno(r, "Failed to update catalog item: %m");
 177
 178                 TAKE_PTR(combined);
 179                 free(prev);
 180         } else {
 181                 /* A new item */
 182                 combined = memdup(payload, payload_size + 1);
 183                 if (!combined)
 184                         return log_oom();
 185
 186                 r = ordered_hashmap_put(h, i, combined);
 187                 if (r < 0)
 188                         return log_error_errno(r, "Failed to insert catalog item: %m");
 189
 190                 TAKE_PTR(i);
 191                 TAKE_PTR(combined);
 192         }
 193
 194         return 0;
 195 }
 196
 197 int catalog_file_lang(const char* filename, char **lang) {
 198         char *beg, *end, *_lang;
 199
 200         end = endswith(filename, ".catalog");
 201         if (!end)
 202                 return 0;
 203
 204         beg = end - 1;
 205         while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
 206                 beg--;
 207
 208         if (*beg != '.' || end <= beg + 1)
 209                 return 0;
 210
 211         _lang = strndup(beg + 1, end - beg - 1);
 212         if (!_lang)
 213                 return -ENOMEM;
 214
 215         *lang = _lang;
 216         return 1;
 217 }
 218
 219 static int catalog_entry_lang(
 220                 const char* filename,
 221                 unsigned line,
 222                 const char* t,
 223                 const char* deflang,
 224                 char **ret) {
 225
 226         size_t c;
 227         char *z;
 228
 229         c = strlen(t);
 230         if (c < 2)
 231                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 232                                        "[%s:%u] Language too short.", filename, line);
 233         if (c > 31)
 234                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 235                                        "[%s:%u] language too long.", filename, line);
 236
 237         if (deflang) {
 238                 if (streq(t, deflang)) {
 239                         log_warning("[%s:%u] language specified unnecessarily", filename, line);
 240                         return 0;
 241                 }
 242
 243                 log_warning("[%s:%u] language differs from default for file", filename, line);
 244         }
 245
 246         z = strdup(t);
 247         if (!z)
 248                 return -ENOMEM;
 249
 250         *ret = z;
 251         return 0;
 252 }
 253
 254 int catalog_import_file(OrderedHashmap *h, const char *path) {
 255         _cleanup_fclose_ FILE *f = NULL;
 256         _cleanup_free_ char *payload = NULL;
 257         size_t payload_size = 0;
 258         unsigned n = 0;
 259         sd_id128_t id;
 260         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 261         bool got_id = false, empty_line = true;
 262         int r;
 263
 264         assert(h);
 265         assert(path);
 266
 267         f = fopen(path, "re");
 268         if (!f)
 269                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 270
 271         r = catalog_file_lang(path, &deflang);
 272         if (r < 0)
 273                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 274         if (r == 1)
 275                 log_debug("File %s has language %s.", path, deflang);
 276
 277         for (;;) {
 278                 _cleanup_free_ char *line = NULL;
 279                 size_t line_len;
 280
 281                 r = read_line(f, LONG_LINE_MAX, &line);
 282                 if (r < 0)
 283                         return log_error_errno(r, "Failed to read file %s: %m", path);
 284                 if (r == 0)
 285                         break;
 286
 287                 n++;
 288
 289                 if (isempty(line)) {
 290                         empty_line = true;
 291                         continue;
 292                 }
 293
 294                 if (strchr(COMMENTS, line[0]))
 295                         continue;
 296
 297                 if (empty_line &&
 298                     strlen(line) >= 2+1+32 &&
 299                     line[0] == '-' &&
 300                     line[1] == '-' &&
 301                     line[2] == ' ' &&
 302                     IN_SET(line[2+1+32], ' ', '\0')) {
 303
 304                         bool with_language;
 305                         sd_id128_t jd;
 306
 307                         /* New entry */
 308
 309                         with_language = line[2+1+32] != '\0';
 310                         line[2+1+32] = '\0';
 311
 312                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 313
 314                                 if (got_id) {
 315                                         if (payload_size == 0)
 316                                                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 317                                                                        "[%s:%u] No payload text.",
 318                                                                        path,
 319                                                                        n);
 320
 321                                         r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 322                                         if (r < 0)
 323                                                 return r;
 324
 325                                         lang = mfree(lang);
 326                                         payload_size = 0;
 327                                 }
 328
 329                                 if (with_language) {
 330                                         char *t;
 331
 332                                         t = strstrip(line + 2 + 1 + 32 + 1);
 333                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 334                                         if (r < 0)
 335                                                 return r;
 336                                 }
 337
 338                                 got_id = true;
 339                                 empty_line = false;
 340                                 id = jd;
 341
 342                                 continue;
 343                         }
 344                 }
 345
 346                 /* Payload */
 347                 if (!got_id)
 348                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 349                                                "[%s:%u] Got payload before ID.",
 350                                                path, n);
 351
 352                 line_len = strlen(line);
 353                 if (!GREEDY_REALLOC(payload, payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
 354                         return log_oom();
 355
 356                 if (empty_line)
 357                         payload[payload_size++] = '\n';
 358                 memcpy(payload + payload_size, line, line_len);
 359                 payload_size += line_len;
 360                 payload[payload_size++] = '\n';
 361                 payload[payload_size] = '\0';
 362
 363                 empty_line = false;
 364         }
 365
 366         if (got_id) {
 367                 if (payload_size == 0)
 368                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 369                                                "[%s:%u] No payload text.",
 370                                                path, n);
 371
 372                 r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 373                 if (r < 0)
 374                         return r;
 375         }
 376
 377         return 0;
 378 }
 379
 380 static int64_t write_catalog(
 381                 const char *database,
 382                 struct strbuf *sb,
 383                 CatalogItem *items,
 384                 size_t n) {
 385
 386         _cleanup_(unlink_and_freep) char *p = NULL;
 387         _cleanup_fclose_ FILE *w = NULL;
 388         int r;
 389
 390         r = mkdir_parents(database, 0755);
 391         if (r < 0)
 392                 return log_error_errno(r, "Failed to create parent directories of %s: %m", database);
 393
 394         r = fopen_temporary(database, &w, &p);
 395         if (r < 0)
 396                 return log_error_errno(r, "Failed to open database for writing: %s: %m", database);
 397
 398         CatalogHeader header = {
 399                 .signature = CATALOG_SIGNATURE,
 400                 .header_size = htole64(CONST_ALIGN_TO(sizeof(CatalogHeader), 8)),
 401                 .catalog_item_size = htole64(sizeof(CatalogItem)),
 402                 .n_items = htole64(n),
 403         };
 404
 405         if (fwrite(&header, sizeof(header), 1, w) != 1)
 406                 return log_error_errno(SYNTHETIC_ERRNO(EIO), "%s: failed to write header.", p);
 407
 408         if (fwrite(items, sizeof(CatalogItem), n, w) != n)
 409                 return log_error_errno(SYNTHETIC_ERRNO(EIO), "%s: failed to write database.", p);
 410
 411         if (fwrite(sb->buf, sb->len, 1, w) != 1)
 412                 return log_error_errno(SYNTHETIC_ERRNO(EIO), "%s: failed to write strings.", p);
 413
 414         r = fflush_and_check(w);
 415         if (r < 0)
 416                 return log_error_errno(r, "%s: failed to write database: %m", p);
 417
 418         (void) fchmod(fileno(w), 0644);
 419
 420         if (rename(p, database) < 0)
 421                 return log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 422
 423         p = mfree(p); /* free without unlinking */
 424         return ftello(w);
 425 }
 426
 427 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 428         _cleanup_strv_free_ char **files = NULL;
 429         _cleanup_(strbuf_freep) struct strbuf *sb = NULL;
 430         _cleanup_ordered_hashmap_free_free_free_ OrderedHashmap *h = NULL;
 431         _cleanup_free_ CatalogItem *items = NULL;
 432         ssize_t offset;
 433         char *payload;
 434         CatalogItem *i;
 435         unsigned n;
 436         int r;
 437         int64_t sz;
 438
 439         h = ordered_hashmap_new(&catalog_hash_ops);
 440         sb = strbuf_new();
 441         if (!h || !sb)
 442                 return log_oom();
 443
 444         r = conf_files_list_strv(&files, ".catalog", root, 0, dirs);
 445         if (r < 0)
 446                 return log_error_errno(r, "Failed to get catalog files: %m");
 447
 448         STRV_FOREACH(f, files) {
 449                 log_debug("Reading file '%s'", *f);
 450                 r = catalog_import_file(h, *f);
 451                 if (r < 0)
 452                         return log_error_errno(r, "Failed to import file '%s': %m", *f);
 453         }
 454
 455         if (ordered_hashmap_size(h) <= 0) {
 456                 log_info("No items in catalog.");
 457                 return 0;
 458         } else
 459                 log_debug("Found %u items in catalog.", ordered_hashmap_size(h));
 460
 461         items = new(CatalogItem, ordered_hashmap_size(h));
 462         if (!items)
 463                 return log_oom();
 464
 465         n = 0;
 466         ORDERED_HASHMAP_FOREACH_KEY(payload, i, h) {
 467                 log_trace("Found " SD_ID128_FORMAT_STR ", language %s",
 468                           SD_ID128_FORMAT_VAL(i->id),
 469                           isempty(i->language) ? "C" : i->language);
 470
 471                 offset = strbuf_add_string(sb, payload, strlen(payload));
 472                 if (offset < 0)
 473                         return log_oom();
 474
 475                 i->offset = htole64((uint64_t) offset);
 476                 items[n++] = *i;
 477         }
 478
 479         assert(n == ordered_hashmap_size(h));
 480         typesafe_qsort(items, n, catalog_compare_func);
 481
 482         strbuf_complete(sb);
 483
 484         sz = write_catalog(database, sb, items, n);
 485         if (sz < 0)
 486                 return log_error_errno(sz, "Failed to write %s: %m", database);
 487
 488         log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 489                   database, n, sb->len, sz);
 490         return 0;
 491 }
 492
 493 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 494         _cleanup_close_ int fd = -EBADF;
 495         const CatalogHeader *h;
 496         struct stat st;
 497         void *p;
 498
 499         assert(_fd);
 500         assert(_st);
 501         assert(_p);
 502
 503         fd = open(database, O_RDONLY|O_CLOEXEC);
 504         if (fd < 0)
 505                 return -errno;
 506
 507         if (fstat(fd, &st) < 0)
 508                 return -errno;
 509
 510         if (st.st_size < (off_t) sizeof(CatalogHeader) || file_offset_beyond_memory_size(st.st_size))
 511                 return -EINVAL;
 512
 513         p = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
 514         if (p == MAP_FAILED)
 515                 return -errno;
 516
 517         h = p;
 518         if (memcmp(h->signature, (const uint8_t[]) CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 519             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 520             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 521             h->incompatible_flags != 0 ||
 522             le64toh(h->n_items) <= 0 ||
 523             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 524                 munmap(p, st.st_size);
 525                 return -EBADMSG;
 526         }
 527
 528         *_fd = TAKE_FD(fd);
 529         *_st = st;
 530         *_p = p;
 531
 532         return 0;
 533 }
 534
 535 static const char *find_id(void *p, sd_id128_t id) {
 536         CatalogItem *f = NULL, key = { .id = id };
 537         const CatalogHeader *h = p;
 538         const char *loc;
 539
 540         loc = setlocale(LC_MESSAGES, NULL);
 541         if (!isempty(loc) && !STR_IN_SET(loc, "C", "POSIX")) {
 542                 size_t len;
 543
 544                 len = strcspn(loc, ".@");
 545                 if (len > sizeof(key.language) - 1)
 546                         log_debug("LC_MESSAGES value too long, ignoring: \"%.*s\"", (int) len, loc);
 547                 else {
 548                         strncpy(key.language, loc, len);
 549                         key.language[len] = '\0';
 550
 551                         f = bsearch(&key,
 552                                     (const uint8_t*) p + le64toh(h->header_size),
 553                                     le64toh(h->n_items),
 554                                     le64toh(h->catalog_item_size),
 555                                     (comparison_fn_t) catalog_compare_func);
 556                         if (!f) {
 557                                 char *e;
 558
 559                                 e = strchr(key.language, '_');
 560                                 if (e) {
 561                                         *e = 0;
 562                                         f = bsearch(&key,
 563                                                     (const uint8_t*) p + le64toh(h->header_size),
 564                                                     le64toh(h->n_items),
 565                                                     le64toh(h->catalog_item_size),
 566                                                     (comparison_fn_t) catalog_compare_func);
 567                                 }
 568                         }
 569                 }
 570         }
 571
 572         if (!f) {
 573                 zero(key.language);
 574                 f = bsearch(&key,
 575                             (const uint8_t*) p + le64toh(h->header_size),
 576                             le64toh(h->n_items),
 577                             le64toh(h->catalog_item_size),
 578                             (comparison_fn_t) catalog_compare_func);
 579         }
 580
 581         if (!f)
 582                 return NULL;
 583
 584         return (const char*) p +
 585                 le64toh(h->header_size) +
 586                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 587                 le64toh(f->offset);
 588 }
 589
 590 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 591         _cleanup_close_ int fd = -EBADF;
 592         void *p = NULL;
 593         struct stat st = {};
 594         char *text = NULL;
 595         int r;
 596         const char *s;
 597
 598         assert(_text);
 599
 600         r = open_mmap(database, &fd, &st, &p);
 601         if (r < 0)
 602                 return r;
 603
 604         s = find_id(p, id);
 605         if (!s) {
 606                 r = -ENOENT;
 607                 goto finish;
 608         }
 609
 610         text = strdup(s);
 611         if (!text) {
 612                 r = -ENOMEM;
 613                 goto finish;
 614         }
 615
 616         *_text = text;
 617         r = 0;
 618
 619 finish:
 620         if (p)
 621                 munmap(p, st.st_size);
 622
 623         return r;
 624 }
 625
 626 static char *find_header(const char *s, const char *header) {
 627
 628         for (;;) {
 629                 const char *v;
 630
 631                 v = startswith(s, header);
 632                 if (v) {
 633                         v += strspn(v, WHITESPACE);
 634                         return strndup(v, strcspn(v, NEWLINE));
 635                 }
 636
 637                 if (!next_header(&s))
 638                         return NULL;
 639         }
 640 }
 641
 642 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 643         if (oneline) {
 644                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 645
 646                 subject = find_header(s, "Subject:");
 647                 defined_by = find_header(s, "Defined-By:");
 648
 649                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 650                         SD_ID128_FORMAT_VAL(id),
 651                         strna(defined_by), strna(subject));
 652         } else
 653                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 654                         SD_ID128_FORMAT_VAL(id), s);
 655 }
 656
 657 int catalog_list(FILE *f, const char *database, bool oneline) {
 658         _cleanup_close_ int fd = -EBADF;
 659         void *p = NULL;
 660         struct stat st;
 661         const CatalogHeader *h;
 662         const CatalogItem *items;
 663         int r;
 664         unsigned n;
 665         sd_id128_t last_id;
 666         bool last_id_set = false;
 667
 668         r = open_mmap(database, &fd, &st, &p);
 669         if (r < 0)
 670                 return r;
 671
 672         h = p;
 673         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 674
 675         for (n = 0; n < le64toh(h->n_items); n++) {
 676                 const char *s;
 677
 678                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 679                         continue;
 680
 681                 assert_se(s = find_id(p, items[n].id));
 682
 683                 dump_catalog_entry(f, items[n].id, s, oneline);
 684
 685                 last_id_set = true;
 686                 last_id = items[n].id;
 687         }
 688
 689         munmap(p, st.st_size);
 690
 691         return 0;
 692 }
 693
 694 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 695         int r = 0;
 696
 697         STRV_FOREACH(item, items) {
 698                 sd_id128_t id;
 699                 int k;
 700                 _cleanup_free_ char *msg = NULL;
 701
 702                 k = sd_id128_from_string(*item, &id);
 703                 if (k < 0) {
 704                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 705                         if (r == 0)
 706                                 r = k;
 707                         continue;
 708                 }
 709
 710                 k = catalog_get(database, id, &msg);
 711                 if (k < 0) {
 712                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 713                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 714                         if (r == 0)
 715                                 r = k;
 716                         continue;
 717                 }
 718
 719                 dump_catalog_entry(f, id, msg, oneline);
 720         }
 721
 722         return r;
 723 }