src/libsystemd/sd-journal/catalog.c

   1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
   2
   3 #include <fcntl.h>
   4 #include <locale.h>
   5 #include <stdio.h>
   6 #include <stdlib.h>
   7 #include <sys/mman.h>
   8 #include <sys/stat.h>
   9
  10 #include "sd-id128.h"
  11
  12 #include "alloc-util.h"
  13 #include "catalog.h"
  14 #include "conf-files.h"
  15 #include "errno-util.h"
  16 #include "fd-util.h"
  17 #include "fileio.h"
  18 #include "fs-util.h"
  19 #include "hashmap.h"
  20 #include "log.h"
  21 #include "memory-util.h"
  22 #include "mkdir.h"
  23 #include "siphash24.h"
  24 #include "sort-util.h"
  25 #include "sparse-endian.h"
  26 #include "strbuf.h"
  27 #include "string-util.h"
  28 #include "strv.h"
  29 #include "tmpfile-util.h"
  30
  31 const char * const catalog_file_dirs[] = {
  32         "/usr/local/lib/systemd/catalog/",
  33         "/usr/lib/systemd/catalog/",
  34         NULL
  35 };
  36
  37 #define CATALOG_SIGNATURE { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  38
  39 typedef struct CatalogHeader {
  40         uint8_t signature[8];  /* "RHHHKSLP" */
  41         le32_t compatible_flags;
  42         le32_t incompatible_flags;
  43         le64_t header_size;
  44         le64_t n_items;
  45         le64_t catalog_item_size;
  46 } CatalogHeader;
  47
  48 typedef struct CatalogItem {
  49         sd_id128_t id;
  50         char language[32]; /* One byte is used for termination, so the maximum allowed
  51                             * length of the string is actually 31 bytes. */
  52         le64_t offset;
  53 } CatalogItem;
  54
  55 static void catalog_hash_func(const CatalogItem *i, struct siphash *state) {
  56         assert(i);
  57         assert(state);
  58
  59         siphash24_compress_typesafe(i->id, state);
  60         siphash24_compress_string(i->language, state);
  61 }
  62
  63 static int catalog_compare_func(const CatalogItem *a, const CatalogItem *b) {
  64         int r;
  65
  66         assert(a);
  67         assert(b);
  68
  69         for (size_t k = 0; k < ELEMENTSOF(b->id.bytes); k++) {
  70                 r = CMP(a->id.bytes[k], b->id.bytes[k]);
  71                 if (r != 0)
  72                         return r;
  73         }
  74
  75         return strcmp(a->language, b->language);
  76 }
  77
  78 DEFINE_PRIVATE_HASH_OPS_FULL(
  79                 catalog_hash_ops,
  80                 CatalogItem, catalog_hash_func, catalog_compare_func, free,
  81                 void, free);
  82
  83 static bool next_header(const char **s) {
  84         const char *e;
  85
  86         assert(s);
  87         assert(*s);
  88
  89         e = strchr(*s, '\n');
  90
  91         /* Unexpected end */
  92         if (!e)
  93                 return false;
  94
  95         /* End of headers */
  96         if (e == *s)
  97                 return false;
  98
  99         *s = e + 1;
 100         return true;
 101 }
 102
 103 static const char* skip_header(const char *s) {
 104         assert(s);
 105
 106         while (next_header(&s))
 107                 ;
 108         return s;
 109 }
 110
 111 static char* combine_entries(const char *one, const char *two) {
 112         const char *b1, *b2;
 113         size_t l1, l2, n;
 114         char *dest, *p;
 115
 116         assert(one);
 117         assert(two);
 118
 119         /* Find split point of headers to body */
 120         b1 = skip_header(one);
 121         b2 = skip_header(two);
 122
 123         l1 = strlen(one);
 124         l2 = strlen(two);
 125         dest = new(char, l1 + l2 + 1);
 126         if (!dest) {
 127                 log_oom();
 128                 return NULL;
 129         }
 130
 131         p = dest;
 132
 133         /* Headers from @one */
 134         n = b1 - one;
 135         p = mempcpy(p, one, n);
 136
 137         /* Headers from @two, these will only be found if not present above */
 138         n = b2 - two;
 139         p = mempcpy(p, two, n);
 140
 141         /* Body from @one */
 142         n = l1 - (b1 - one);
 143         if (n > 0)
 144                 p = mempcpy(p, b1, n);
 145         /* Body from @two */
 146         else {
 147                 n = l2 - (b2 - two);
 148                 p = mempcpy(p, b2, n);
 149         }
 150
 151         assert(p - dest <= (ptrdiff_t)(l1 + l2));
 152         p[0] = '\0';
 153         return dest;
 154 }
 155
 156 static int finish_item(
 157                 OrderedHashmap **h,
 158                 sd_id128_t id,
 159                 const char *language,
 160                 const char *payload,
 161                 size_t payload_size) {
 162
 163         _cleanup_free_ CatalogItem *i = NULL;
 164         _cleanup_free_ char *combined = NULL;
 165         char *prev;
 166         int r;
 167
 168         assert(h);
 169         assert(payload);
 170         assert(payload_size > 0);
 171
 172         i = new0(CatalogItem, 1);
 173         if (!i)
 174                 return log_oom();
 175
 176         i->id = id;
 177         if (language) {
 178                 assert(strlen(language) > 1 && strlen(language) < 32);
 179                 strcpy(i->language, language);
 180         }
 181
 182         prev = ordered_hashmap_get(*h, i);
 183         if (prev) {
 184                 /* Already have such an item, combine them */
 185                 combined = combine_entries(payload, prev);
 186                 if (!combined)
 187                         return log_oom();
 188
 189                 r = ordered_hashmap_update(*h, i, combined);
 190                 if (r < 0)
 191                         return log_error_errno(r, "Failed to update catalog item: %m");
 192
 193                 TAKE_PTR(combined);
 194                 free(prev);
 195         } else {
 196                 /* A new item */
 197                 combined = memdup(payload, payload_size + 1);
 198                 if (!combined)
 199                         return log_oom();
 200
 201                 r = ordered_hashmap_ensure_put(h, &catalog_hash_ops, i, combined);
 202                 if (r < 0)
 203                         return log_error_errno(r, "Failed to insert catalog item: %m");
 204
 205                 TAKE_PTR(i);
 206                 TAKE_PTR(combined);
 207         }
 208
 209         return 0;
 210 }
 211
 212 int catalog_file_lang(const char *filename, char **ret) {
 213         char *beg, *end, *lang;
 214
 215         assert(filename);
 216         assert(ret);
 217
 218         end = endswith(filename, ".catalog");
 219         if (!end)
 220                 return 0;
 221
 222         beg = end - 1;
 223         while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
 224                 beg--;
 225
 226         if (*beg != '.' || end <= beg + 1)
 227                 return 0;
 228
 229         lang = strndup(beg + 1, end - beg - 1);
 230         if (!lang)
 231                 return -ENOMEM;
 232
 233         *ret = lang;
 234         return 1;
 235 }
 236
 237 static int catalog_entry_lang(
 238                 const char *filename,
 239                 unsigned line,
 240                 const char *t,
 241                 const char *deflang,
 242                 char **ret) {
 243
 244         assert(t);
 245
 246         size_t c = strlen(t);
 247         if (c < 2)
 248                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 249                                        "[%s:%u] Language too short.", filename, line);
 250         if (c > 31)
 251                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 252                                        "[%s:%u] language too long.", filename, line);
 253
 254         if (deflang) {
 255                 if (streq(t, deflang)) {
 256                         log_warning("[%s:%u] language specified unnecessarily", filename, line);
 257                         return 0;
 258                 }
 259
 260                 log_warning("[%s:%u] language differs from default for file", filename, line);
 261         }
 262
 263         return strdup_to(ret, t);
 264 }
 265
 266 int catalog_import_file(OrderedHashmap **h, int fd, const char *path) {
 267         _cleanup_fclose_ FILE *f = NULL;
 268         _cleanup_free_ char *payload = NULL;
 269         size_t payload_size = 0;
 270         unsigned n = 0;
 271         sd_id128_t id;
 272         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 273         bool got_id = false, empty_line = true;
 274         int r;
 275
 276         assert(h);
 277         assert(fd >= 0);
 278         assert(path);
 279
 280         f = fopen(FORMAT_PROC_FD_PATH(fd), "re");
 281         if (!f)
 282                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 283
 284         r = catalog_file_lang(path, &deflang);
 285         if (r < 0)
 286                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 287         if (r == 1)
 288                 log_debug("File %s has language %s.", path, deflang);
 289
 290         for (;;) {
 291                 _cleanup_free_ char *line = NULL;
 292                 size_t line_len;
 293
 294                 r = read_line(f, LONG_LINE_MAX, &line);
 295                 if (r < 0)
 296                         return log_error_errno(r, "Failed to read file %s: %m", path);
 297                 if (r == 0)
 298                         break;
 299
 300                 n++;
 301
 302                 if (isempty(line)) {
 303                         empty_line = true;
 304                         continue;
 305                 }
 306
 307                 if (strchr(COMMENTS, line[0]))
 308                         continue;
 309
 310                 if (empty_line &&
 311                     strlen(line) >= 2+1+32 &&
 312                     line[0] == '-' &&
 313                     line[1] == '-' &&
 314                     line[2] == ' ' &&
 315                     IN_SET(line[2+1+32], ' ', '\0')) {
 316
 317                         bool with_language;
 318                         sd_id128_t jd;
 319
 320                         /* New entry */
 321
 322                         with_language = line[2+1+32] != '\0';
 323                         line[2+1+32] = '\0';
 324
 325                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 326
 327                                 if (got_id) {
 328                                         if (payload_size == 0)
 329                                                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 330                                                                        "[%s:%u] No payload text.",
 331                                                                        path,
 332                                                                        n);
 333
 334                                         r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 335                                         if (r < 0)
 336                                                 return r;
 337
 338                                         lang = mfree(lang);
 339                                         payload_size = 0;
 340                                 }
 341
 342                                 if (with_language) {
 343                                         char *t;
 344
 345                                         t = strstrip(line + 2 + 1 + 32 + 1);
 346                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 347                                         if (r < 0)
 348                                                 return r;
 349                                 }
 350
 351                                 got_id = true;
 352                                 empty_line = false;
 353                                 id = jd;
 354
 355                                 continue;
 356                         }
 357                 }
 358
 359                 /* Payload */
 360                 if (!got_id)
 361                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 362                                                "[%s:%u] Got payload before ID.",
 363                                                path, n);
 364
 365                 line_len = strlen(line);
 366                 if (!GREEDY_REALLOC(payload, payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
 367                         return log_oom();
 368
 369                 if (empty_line)
 370                         payload[payload_size++] = '\n';
 371                 memcpy(payload + payload_size, line, line_len);
 372                 payload_size += line_len;
 373                 payload[payload_size++] = '\n';
 374                 payload[payload_size] = '\0';
 375
 376                 empty_line = false;
 377         }
 378
 379         if (got_id) {
 380                 if (payload_size == 0)
 381                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 382                                                "[%s:%u] No payload text.",
 383                                                path, n);
 384
 385                 r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 386                 if (r < 0)
 387                         return r;
 388         }
 389
 390         return 0;
 391 }
 392
 393 static int64_t write_catalog(
 394                 const char *database,
 395                 const struct strbuf *sb,
 396                 const CatalogItem *items,
 397                 size_t n_items) {
 398
 399         _cleanup_(unlink_and_freep) char *p = NULL;
 400         _cleanup_fclose_ FILE *w = NULL;
 401         int r;
 402
 403         assert(database);
 404         assert(sb);
 405         assert(items);
 406         assert(n_items > 0);
 407
 408         r = mkdir_parents(database, 0755);
 409         if (r < 0)
 410                 return log_error_errno(r, "Failed to create parent directories of %s: %m", database);
 411
 412         r = fopen_temporary(database, &w, &p);
 413         if (r < 0)
 414                 return log_error_errno(r, "Failed to open database for writing: %s: %m", database);
 415
 416         CatalogHeader header = {
 417                 .signature = CATALOG_SIGNATURE,
 418                 .header_size = htole64(CONST_ALIGN_TO(sizeof(CatalogHeader), 8)),
 419                 .catalog_item_size = htole64(sizeof(CatalogItem)),
 420                 .n_items = htole64(n_items),
 421         };
 422
 423         if (fwrite(&header, sizeof(header), 1, w) != 1)
 424                 return log_error_errno(SYNTHETIC_ERRNO(EIO), "%s: failed to write header.", p);
 425
 426         if (fwrite(items, sizeof(CatalogItem), n_items, w) != n_items)
 427                 return log_error_errno(SYNTHETIC_ERRNO(EIO), "%s: failed to write database.", p);
 428
 429         if (fwrite(sb->buf, sb->len, 1, w) != 1)
 430                 return log_error_errno(SYNTHETIC_ERRNO(EIO), "%s: failed to write strings.", p);
 431
 432         r = fflush_and_check(w);
 433         if (r < 0)
 434                 return log_error_errno(r, "%s: failed to write database: %m", p);
 435
 436         (void) fchmod(fileno(w), 0644);
 437
 438         if (rename(p, database) < 0)
 439                 return log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 440
 441         p = mfree(p); /* free without unlinking */
 442         return ftello(w);
 443 }
 444
 445 int catalog_update(const char *database, const char *root, const char* const *dirs) {
 446         int r;
 447
 448         assert(database);
 449
 450         if (!dirs)
 451                 dirs = catalog_file_dirs;
 452
 453         ConfFile **files = NULL;
 454         size_t n_files = 0;
 455
 456         CLEANUP_ARRAY(files, n_files, conf_file_free_many);
 457
 458         r = conf_files_list_strv_full(".catalog", root, CONF_FILES_REGULAR | CONF_FILES_FILTER_MASKED, dirs, &files, &n_files);
 459         if (r < 0)
 460                 return log_error_errno(r, "Failed to get catalog files: %m");
 461
 462         _cleanup_ordered_hashmap_free_ OrderedHashmap *h = NULL;
 463         FOREACH_ARRAY(i, files, n_files) {
 464                 ConfFile *c = *i;
 465
 466                 log_debug("Reading file: '%s' -> '%s'", c->original_path, c->resolved_path);
 467                 r = catalog_import_file(&h, c->fd, c->original_path);
 468                 if (r < 0)
 469                         return log_error_errno(r, "Failed to import file '%s': %m", c->original_path);
 470         }
 471
 472         if (ordered_hashmap_isempty(h)) {
 473                 log_info("No items in catalog.");
 474                 return 0;
 475         }
 476
 477         log_debug("Found %u items in catalog.", ordered_hashmap_size(h));
 478
 479         _cleanup_free_ CatalogItem *items = new(CatalogItem, ordered_hashmap_size(h));
 480         if (!items)
 481                 return log_oom();
 482
 483         _cleanup_(strbuf_freep) struct strbuf *sb = strbuf_new();
 484         if (!sb)
 485                 return log_oom();
 486
 487         unsigned n = 0;
 488         char *payload;
 489         CatalogItem *i;
 490         ORDERED_HASHMAP_FOREACH_KEY(payload, i, h) {
 491                 log_trace("Found " SD_ID128_FORMAT_STR ", language %s",
 492                           SD_ID128_FORMAT_VAL(i->id),
 493                           isempty(i->language) ? "C" : i->language);
 494
 495                 ssize_t offset = strbuf_add_string(sb, payload);
 496                 if (offset < 0)
 497                         return log_oom();
 498
 499                 i->offset = htole64((uint64_t) offset);
 500                 items[n++] = *i;
 501         }
 502
 503         assert(n == ordered_hashmap_size(h));
 504         typesafe_qsort(items, n, catalog_compare_func);
 505
 506         strbuf_complete(sb);
 507
 508         int64_t sz = write_catalog(database, sb, items, n);
 509         if (sz < 0)
 510                 return log_error_errno(sz, "Failed to write %s: %m", database);
 511
 512         log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 513                   database, n, sb->len, sz);
 514         return 0;
 515 }
 516
 517 static int open_mmap(const char *database, int *ret_fd, struct stat *ret_st, void **ret_map) {
 518         assert(database);
 519         assert(ret_fd);
 520         assert(ret_st);
 521         assert(ret_map);
 522
 523         _cleanup_close_ int fd = open(database, O_RDONLY|O_CLOEXEC);
 524         if (fd < 0)
 525                 return -errno;
 526
 527         struct stat st;
 528         if (fstat(fd, &st) < 0)
 529                 return -errno;
 530
 531         if (st.st_size < (off_t) sizeof(CatalogHeader) || file_offset_beyond_memory_size(st.st_size))
 532                 return -EINVAL;
 533
 534         void *p = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
 535         if (p == MAP_FAILED)
 536                 return -errno;
 537
 538         const CatalogHeader *h = p;
 539         if (memcmp(h->signature, (const uint8_t[]) CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 540             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 541             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 542             h->incompatible_flags != 0 ||
 543             le64toh(h->n_items) <= 0 ||
 544             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 545                 munmap(p, st.st_size);
 546                 return -EBADMSG;
 547         }
 548
 549         *ret_fd = TAKE_FD(fd);
 550         *ret_st = st;
 551         *ret_map = p;
 552         return 0;
 553 }
 554
 555 static const char* find_id(const void *p, sd_id128_t id) {
 556         CatalogItem *f = NULL, key = { .id = id };
 557         const CatalogHeader *h = ASSERT_PTR(p);
 558         const char *loc;
 559
 560         loc = setlocale(LC_MESSAGES, NULL);
 561         if (!isempty(loc) && !STR_IN_SET(loc, "C", "POSIX")) {
 562                 size_t len;
 563
 564                 len = strcspn(loc, ".@");
 565                 if (len > sizeof(key.language) - 1)
 566                         log_debug("LC_MESSAGES value too long, ignoring: \"%.*s\"", (int) len, loc);
 567                 else {
 568                         strncpy(key.language, loc, len);
 569                         key.language[len] = '\0';
 570
 571                         f = bsearch(&key,
 572                                     (const uint8_t*) p + le64toh(h->header_size),
 573                                     le64toh(h->n_items),
 574                                     le64toh(h->catalog_item_size),
 575                                     (comparison_fn_t) catalog_compare_func);
 576                         if (!f) {
 577                                 char *e;
 578
 579                                 e = strchr(key.language, '_');
 580                                 if (e) {
 581                                         *e = 0;
 582                                         f = bsearch(&key,
 583                                                     (const uint8_t*) p + le64toh(h->header_size),
 584                                                     le64toh(h->n_items),
 585                                                     le64toh(h->catalog_item_size),
 586                                                     (comparison_fn_t) catalog_compare_func);
 587                                 }
 588                         }
 589                 }
 590         }
 591
 592         if (!f) {
 593                 zero(key.language);
 594                 f = bsearch(&key,
 595                             (const uint8_t*) p + le64toh(h->header_size),
 596                             le64toh(h->n_items),
 597                             le64toh(h->catalog_item_size),
 598                             (comparison_fn_t) catalog_compare_func);
 599         }
 600
 601         if (!f)
 602                 return NULL;
 603
 604         return (const char*) p +
 605                 le64toh(h->header_size) +
 606                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 607                 le64toh(f->offset);
 608 }
 609
 610 int catalog_get(const char *database, sd_id128_t id, char **ret_text) {
 611         int r;
 612
 613         assert(database);
 614         assert(ret_text);
 615
 616         _cleanup_close_ int fd = -EBADF;
 617         struct stat st;
 618         void *p;
 619         r = open_mmap(database, &fd, &st, &p);
 620         if (r < 0)
 621                 return r;
 622
 623         const char *s = find_id(p, id);
 624         if (!s)
 625                 r = -ENOENT;
 626         else
 627                 r = strdup_to(ret_text, s);
 628
 629         (void) munmap(p, st.st_size);
 630         return r;
 631 }
 632
 633 static char* find_header(const char *s, const char *header) {
 634
 635         assert(s);
 636         assert(header);
 637
 638         for (;;) {
 639                 const char *v;
 640
 641                 v = startswith(s, header);
 642                 if (v) {
 643                         v += strspn(v, WHITESPACE);
 644                         return strndup(v, strcspn(v, NEWLINE));
 645                 }
 646
 647                 if (!next_header(&s))
 648                         return NULL;
 649         }
 650 }
 651
 652 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 653         assert(s);
 654
 655         if (!f)
 656                 f = stdout;
 657
 658         if (oneline) {
 659                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 660
 661                 subject = find_header(s, "Subject:");
 662                 defined_by = find_header(s, "Defined-By:");
 663
 664                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 665                         SD_ID128_FORMAT_VAL(id),
 666                         strna(defined_by), strna(subject));
 667         } else
 668                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 669                         SD_ID128_FORMAT_VAL(id), s);
 670 }
 671
 672 int catalog_list(FILE *f, const char *database, bool oneline) {
 673         int r;
 674
 675         assert(database);
 676
 677         _cleanup_close_ int fd = -EBADF;
 678         struct stat st;
 679         void *p;
 680         r = open_mmap(database, &fd, &st, &p);
 681         if (r < 0)
 682                 return r;
 683
 684         const CatalogHeader *h = p;
 685         const CatalogItem *items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 686
 687         sd_id128_t last_id;
 688         bool last_id_set = false;
 689         for (size_t n = 0; n < le64toh(h->n_items); n++) {
 690                 const char *s;
 691
 692                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 693                         continue;
 694
 695                 assert_se(s = find_id(p, items[n].id));
 696
 697                 dump_catalog_entry(f, items[n].id, s, oneline);
 698
 699                 last_id_set = true;
 700                 last_id = items[n].id;
 701         }
 702
 703         (void) munmap(p, st.st_size);
 704         return 0;
 705 }
 706
 707 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 708         int r, ret = 0;
 709
 710         assert(database);
 711
 712         STRV_FOREACH(item, items) {
 713                 sd_id128_t id;
 714                 r = sd_id128_from_string(*item, &id);
 715                 if (r < 0) {
 716                         RET_GATHER(ret, log_error_errno(r, "Failed to parse id128 '%s': %m", *item));
 717                         continue;
 718                 }
 719
 720                 _cleanup_free_ char *msg = NULL;
 721                 r = catalog_get(database, id, &msg);
 722                 if (r < 0) {
 723                         RET_GATHER(ret, log_full_errno(r == -ENOENT ? LOG_NOTICE : LOG_ERR, r,
 724                                                        "Failed to retrieve catalog entry for '%s': %m", *item));
 725                         continue;
 726                 }
 727
 728                 dump_catalog_entry(f, id, msg, oneline);
 729         }
 730
 731         return ret;
 732 }