src/journal/catalog.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #include <errno.h>
   4 #include <fcntl.h>
   5 #include <locale.h>
   6 #include <stdio.h>
   7 #include <string.h>
   8 #include <sys/mman.h>
   9 #include <unistd.h>
  10
  11 #include "sd-id128.h"
  12
  13 #include "alloc-util.h"
  14 #include "catalog.h"
  15 #include "conf-files.h"
  16 #include "fd-util.h"
  17 #include "fileio.h"
  18 #include "hashmap.h"
  19 #include "log.h"
  20 #include "mkdir.h"
  21 #include "path-util.h"
  22 #include "siphash24.h"
  23 #include "sparse-endian.h"
  24 #include "strbuf.h"
  25 #include "string-util.h"
  26 #include "strv.h"
  27 #include "tmpfile-util.h"
  28 #include "util.h"
  29
  30 const char * const catalog_file_dirs[] = {
  31         "/usr/local/lib/systemd/catalog/",
  32         "/usr/lib/systemd/catalog/",
  33         NULL
  34 };
  35
  36 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  37
  38 typedef struct CatalogHeader {
  39         uint8_t signature[8];  /* "RHHHKSLP" */
  40         le32_t compatible_flags;
  41         le32_t incompatible_flags;
  42         le64_t header_size;
  43         le64_t n_items;
  44         le64_t catalog_item_size;
  45 } CatalogHeader;
  46
  47 typedef struct CatalogItem {
  48         sd_id128_t id;
  49         char language[32]; /* One byte is used for termination, so the maximum allowed
  50                             * length of the string is actually 31 bytes. */
  51         le64_t offset;
  52 } CatalogItem;
  53
  54 static void catalog_hash_func(const CatalogItem *i, struct siphash *state) {
  55         siphash24_compress(&i->id, sizeof(i->id), state);
  56         siphash24_compress(i->language, strlen(i->language), state);
  57 }
  58
  59 static int catalog_compare_func(const CatalogItem *a, const CatalogItem *b) {
  60         unsigned k;
  61         int r;
  62
  63         for (k = 0; k < ELEMENTSOF(b->id.bytes); k++) {
  64                 r = CMP(a->id.bytes[k], b->id.bytes[k]);
  65                 if (r != 0)
  66                         return r;
  67         }
  68
  69         return strcmp(a->language, b->language);
  70 }
  71
  72 DEFINE_HASH_OPS(catalog_hash_ops, CatalogItem, catalog_hash_func, catalog_compare_func);
  73
  74 static bool next_header(const char **s) {
  75         const char *e;
  76
  77         e = strchr(*s, '\n');
  78
  79         /* Unexpected end */
  80         if (!e)
  81                 return false;
  82
  83         /* End of headers */
  84         if (e == *s)
  85                 return false;
  86
  87         *s = e + 1;
  88         return true;
  89 }
  90
  91 static const char *skip_header(const char *s) {
  92         while (next_header(&s))
  93                 ;
  94         return s;
  95 }
  96
  97 static char *combine_entries(const char *one, const char *two) {
  98         const char *b1, *b2;
  99         size_t l1, l2, n;
 100         char *dest, *p;
 101
 102         /* Find split point of headers to body */
 103         b1 = skip_header(one);
 104         b2 = skip_header(two);
 105
 106         l1 = strlen(one);
 107         l2 = strlen(two);
 108         dest = new(char, l1 + l2 + 1);
 109         if (!dest) {
 110                 log_oom();
 111                 return NULL;
 112         }
 113
 114         p = dest;
 115
 116         /* Headers from @one */
 117         n = b1 - one;
 118         p = mempcpy(p, one, n);
 119
 120         /* Headers from @two, these will only be found if not present above */
 121         n = b2 - two;
 122         p = mempcpy(p, two, n);
 123
 124         /* Body from @one */
 125         n = l1 - (b1 - one);
 126         if (n > 0) {
 127                 memcpy(p, b1, n);
 128                 p += n;
 129
 130         /* Body from @two */
 131         } else {
 132                 n = l2 - (b2 - two);
 133                 memcpy(p, b2, n);
 134                 p += n;
 135         }
 136
 137         assert(p - dest <= (ptrdiff_t)(l1 + l2));
 138         p[0] = '\0';
 139         return dest;
 140 }
 141
 142 static int finish_item(
 143                 Hashmap *h,
 144                 sd_id128_t id,
 145                 const char *language,
 146                 char *payload, size_t payload_size) {
 147
 148         _cleanup_free_ CatalogItem *i = NULL;
 149         _cleanup_free_ char *prev = NULL, *combined = NULL;
 150
 151         assert(h);
 152         assert(payload);
 153         assert(payload_size > 0);
 154
 155         i = new0(CatalogItem, 1);
 156         if (!i)
 157                 return log_oom();
 158
 159         i->id = id;
 160         if (language) {
 161                 assert(strlen(language) > 1 && strlen(language) < 32);
 162                 strcpy(i->language, language);
 163         }
 164
 165         prev = hashmap_get(h, i);
 166         if (prev) {
 167                 /* Already have such an item, combine them */
 168                 combined = combine_entries(payload, prev);
 169                 if (!combined)
 170                         return log_oom();
 171
 172                 if (hashmap_update(h, i, combined) < 0)
 173                         return log_oom();
 174                 combined = NULL;
 175         } else {
 176                 /* A new item */
 177                 combined = memdup(payload, payload_size + 1);
 178                 if (!combined)
 179                         return log_oom();
 180
 181                 if (hashmap_put(h, i, combined) < 0)
 182                         return log_oom();
 183                 i = NULL;
 184                 combined = NULL;
 185         }
 186
 187         return 0;
 188 }
 189
 190 int catalog_file_lang(const char* filename, char **lang) {
 191         char *beg, *end, *_lang;
 192
 193         end = endswith(filename, ".catalog");
 194         if (!end)
 195                 return 0;
 196
 197         beg = end - 1;
 198         while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
 199                 beg--;
 200
 201         if (*beg != '.' || end <= beg + 1)
 202                 return 0;
 203
 204         _lang = strndup(beg + 1, end - beg - 1);
 205         if (!_lang)
 206                 return -ENOMEM;
 207
 208         *lang = _lang;
 209         return 1;
 210 }
 211
 212 static int catalog_entry_lang(const char* filename, int line,
 213                               const char* t, const char* deflang, char **lang) {
 214         size_t c;
 215
 216         c = strlen(t);
 217         if (c < 2)
 218                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 219                                        "[%s:%u] Language too short.",
 220                                        filename, line);
 221         if (c > 31)
 222                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 223                                        "[%s:%u] language too long.", filename,
 224                                        line);
 225
 226         if (deflang) {
 227                 if (streq(t, deflang)) {
 228                         log_warning("[%s:%u] language specified unnecessarily",
 229                                     filename, line);
 230                         return 0;
 231                 } else
 232                         log_warning("[%s:%u] language differs from default for file",
 233                                     filename, line);
 234         }
 235
 236         *lang = strdup(t);
 237         if (!*lang)
 238                         return -ENOMEM;
 239
 240         return 0;
 241 }
 242
 243 int catalog_import_file(Hashmap *h, const char *path) {
 244         _cleanup_fclose_ FILE *f = NULL;
 245         _cleanup_free_ char *payload = NULL;
 246         size_t payload_size = 0, payload_allocated = 0;
 247         unsigned n = 0;
 248         sd_id128_t id;
 249         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 250         bool got_id = false, empty_line = true;
 251         int r;
 252
 253         assert(h);
 254         assert(path);
 255
 256         f = fopen(path, "re");
 257         if (!f)
 258                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 259
 260         r = catalog_file_lang(path, &deflang);
 261         if (r < 0)
 262                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 263         if (r == 1)
 264                 log_debug("File %s has language %s.", path, deflang);
 265
 266         for (;;) {
 267                 _cleanup_free_ char *line = NULL;
 268                 size_t line_len;
 269
 270                 r = read_line(f, LONG_LINE_MAX, &line);
 271                 if (r < 0)
 272                         return log_error_errno(r, "Failed to read file %s: %m", path);
 273                 if (r == 0)
 274                         break;
 275
 276                 n++;
 277
 278                 if (isempty(line)) {
 279                         empty_line = true;
 280                         continue;
 281                 }
 282
 283                 if (strchr(COMMENTS, line[0]))
 284                         continue;
 285
 286                 if (empty_line &&
 287                     strlen(line) >= 2+1+32 &&
 288                     line[0] == '-' &&
 289                     line[1] == '-' &&
 290                     line[2] == ' ' &&
 291                     IN_SET(line[2+1+32], ' ', '\0')) {
 292
 293                         bool with_language;
 294                         sd_id128_t jd;
 295
 296                         /* New entry */
 297
 298                         with_language = line[2+1+32] != '\0';
 299                         line[2+1+32] = '\0';
 300
 301                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 302
 303                                 if (got_id) {
 304                                         if (payload_size == 0)
 305                                                 return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 306                                                                        "[%s:%u] No payload text.",
 307                                                                        path,
 308                                                                        n);
 309
 310                                         r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 311                                         if (r < 0)
 312                                                 return r;
 313
 314                                         lang = mfree(lang);
 315                                         payload_size = 0;
 316                                 }
 317
 318                                 if (with_language) {
 319                                         char *t;
 320
 321                                         t = strstrip(line + 2 + 1 + 32 + 1);
 322                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 323                                         if (r < 0)
 324                                                 return r;
 325                                 }
 326
 327                                 got_id = true;
 328                                 empty_line = false;
 329                                 id = jd;
 330
 331                                 continue;
 332                         }
 333                 }
 334
 335                 /* Payload */
 336                 if (!got_id)
 337                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 338                                                "[%s:%u] Got payload before ID.",
 339                                                path, n);
 340
 341                 line_len = strlen(line);
 342                 if (!GREEDY_REALLOC(payload, payload_allocated,
 343                                     payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
 344                         return log_oom();
 345
 346                 if (empty_line)
 347                         payload[payload_size++] = '\n';
 348                 memcpy(payload + payload_size, line, line_len);
 349                 payload_size += line_len;
 350                 payload[payload_size++] = '\n';
 351                 payload[payload_size] = '\0';
 352
 353                 empty_line = false;
 354         }
 355
 356         if (got_id) {
 357                 if (payload_size == 0)
 358                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
 359                                                "[%s:%u] No payload text.",
 360                                                path, n);
 361
 362                 r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 363                 if (r < 0)
 364                         return r;
 365         }
 366
 367         return 0;
 368 }
 369
 370 static int64_t write_catalog(const char *database, struct strbuf *sb,
 371                              CatalogItem *items, size_t n) {
 372         CatalogHeader header;
 373         _cleanup_fclose_ FILE *w = NULL;
 374         int r;
 375         _cleanup_free_ char *d, *p = NULL;
 376         size_t k;
 377
 378         d = dirname_malloc(database);
 379         if (!d)
 380                 return log_oom();
 381
 382         r = mkdir_p(d, 0775);
 383         if (r < 0)
 384                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 385
 386         r = fopen_temporary(database, &w, &p);
 387         if (r < 0)
 388                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 389                                        database);
 390
 391         zero(header);
 392         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 393         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 394         header.catalog_item_size = htole64(sizeof(CatalogItem));
 395         header.n_items = htole64(n);
 396
 397         r = -EIO;
 398
 399         k = fwrite(&header, 1, sizeof(header), w);
 400         if (k != sizeof(header)) {
 401                 log_error("%s: failed to write header.", p);
 402                 goto error;
 403         }
 404
 405         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 406         if (k != n * sizeof(CatalogItem)) {
 407                 log_error("%s: failed to write database.", p);
 408                 goto error;
 409         }
 410
 411         k = fwrite(sb->buf, 1, sb->len, w);
 412         if (k != sb->len) {
 413                 log_error("%s: failed to write strings.", p);
 414                 goto error;
 415         }
 416
 417         r = fflush_and_check(w);
 418         if (r < 0) {
 419                 log_error_errno(r, "%s: failed to write database: %m", p);
 420                 goto error;
 421         }
 422
 423         fchmod(fileno(w), 0644);
 424
 425         if (rename(p, database) < 0) {
 426                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 427                 goto error;
 428         }
 429
 430         return ftello(w);
 431
 432 error:
 433         (void) unlink(p);
 434         return r;
 435 }
 436
 437 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 438         _cleanup_strv_free_ char **files = NULL;
 439         char **f;
 440         _cleanup_(strbuf_cleanupp) struct strbuf *sb = NULL;
 441         _cleanup_hashmap_free_free_free_ Hashmap *h = NULL;
 442         _cleanup_free_ CatalogItem *items = NULL;
 443         ssize_t offset;
 444         char *payload;
 445         CatalogItem *i;
 446         Iterator j;
 447         unsigned n;
 448         int r;
 449         int64_t sz;
 450
 451         h = hashmap_new(&catalog_hash_ops);
 452         sb = strbuf_new();
 453         if (!h || !sb)
 454                 return log_oom();
 455
 456         r = conf_files_list_strv(&files, ".catalog", root, 0, dirs);
 457         if (r < 0)
 458                 return log_error_errno(r, "Failed to get catalog files: %m");
 459
 460         STRV_FOREACH(f, files) {
 461                 log_debug("Reading file '%s'", *f);
 462                 r = catalog_import_file(h, *f);
 463                 if (r < 0)
 464                         return log_error_errno(r, "Failed to import file '%s': %m", *f);
 465         }
 466
 467         if (hashmap_size(h) <= 0) {
 468                 log_info("No items in catalog.");
 469                 return 0;
 470         } else
 471                 log_debug("Found %u items in catalog.", hashmap_size(h));
 472
 473         items = new(CatalogItem, hashmap_size(h));
 474         if (!items)
 475                 return log_oom();
 476
 477         n = 0;
 478         HASHMAP_FOREACH_KEY(payload, i, h, j) {
 479                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 480                           SD_ID128_FORMAT_VAL(i->id),
 481                           isempty(i->language) ? "C" : i->language);
 482
 483                 offset = strbuf_add_string(sb, payload, strlen(payload));
 484                 if (offset < 0)
 485                         return log_oom();
 486
 487                 i->offset = htole64((uint64_t) offset);
 488                 items[n++] = *i;
 489         }
 490
 491         assert(n == hashmap_size(h));
 492         typesafe_qsort(items, n, catalog_compare_func);
 493
 494         strbuf_complete(sb);
 495
 496         sz = write_catalog(database, sb, items, n);
 497         if (sz < 0)
 498                 return log_error_errno(sz, "Failed to write %s: %m", database);
 499
 500         log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 501                   database, n, sb->len, sz);
 502         return 0;
 503 }
 504
 505 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 506         const CatalogHeader *h;
 507         int fd;
 508         void *p;
 509         struct stat st;
 510
 511         assert(_fd);
 512         assert(_st);
 513         assert(_p);
 514
 515         fd = open(database, O_RDONLY|O_CLOEXEC);
 516         if (fd < 0)
 517                 return -errno;
 518
 519         if (fstat(fd, &st) < 0) {
 520                 safe_close(fd);
 521                 return -errno;
 522         }
 523
 524         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 525                 safe_close(fd);
 526                 return -EINVAL;
 527         }
 528
 529         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 530         if (p == MAP_FAILED) {
 531                 safe_close(fd);
 532                 return -errno;
 533         }
 534
 535         h = p;
 536         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 537             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 538             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 539             h->incompatible_flags != 0 ||
 540             le64toh(h->n_items) <= 0 ||
 541             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 542                 safe_close(fd);
 543                 munmap(p, st.st_size);
 544                 return -EBADMSG;
 545         }
 546
 547         *_fd = fd;
 548         *_st = st;
 549         *_p = p;
 550
 551         return 0;
 552 }
 553
 554 static const char *find_id(void *p, sd_id128_t id) {
 555         CatalogItem *f = NULL, key = { .id = id };
 556         const CatalogHeader *h = p;
 557         const char *loc;
 558
 559         loc = setlocale(LC_MESSAGES, NULL);
 560         if (!isempty(loc) && !STR_IN_SET(loc, "C", "POSIX")) {
 561                 size_t len;
 562
 563                 len = strcspn(loc, ".@");
 564                 if (len > sizeof(key.language) - 1)
 565                         log_debug("LC_MESSAGES value too long, ignoring: \"%.*s\"", (int) len, loc);
 566                 else {
 567                         strncpy(key.language, loc, len);
 568                         key.language[len] = '\0';
 569
 570                         f = bsearch(&key,
 571                                     (const uint8_t*) p + le64toh(h->header_size),
 572                                     le64toh(h->n_items),
 573                                     le64toh(h->catalog_item_size),
 574                                     (comparison_fn_t) catalog_compare_func);
 575                         if (!f) {
 576                                 char *e;
 577
 578                                 e = strchr(key.language, '_');
 579                                 if (e) {
 580                                         *e = 0;
 581                                         f = bsearch(&key,
 582                                                     (const uint8_t*) p + le64toh(h->header_size),
 583                                                     le64toh(h->n_items),
 584                                                     le64toh(h->catalog_item_size),
 585                                                     (comparison_fn_t) catalog_compare_func);
 586                                 }
 587                         }
 588                 }
 589         }
 590
 591         if (!f) {
 592                 zero(key.language);
 593                 f = bsearch(&key,
 594                             (const uint8_t*) p + le64toh(h->header_size),
 595                             le64toh(h->n_items),
 596                             le64toh(h->catalog_item_size),
 597                             (comparison_fn_t) catalog_compare_func);
 598         }
 599
 600         if (!f)
 601                 return NULL;
 602
 603         return (const char*) p +
 604                 le64toh(h->header_size) +
 605                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 606                 le64toh(f->offset);
 607 }
 608
 609 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 610         _cleanup_close_ int fd = -1;
 611         void *p = NULL;
 612         struct stat st = {};
 613         char *text = NULL;
 614         int r;
 615         const char *s;
 616
 617         assert(_text);
 618
 619         r = open_mmap(database, &fd, &st, &p);
 620         if (r < 0)
 621                 return r;
 622
 623         s = find_id(p, id);
 624         if (!s) {
 625                 r = -ENOENT;
 626                 goto finish;
 627         }
 628
 629         text = strdup(s);
 630         if (!text) {
 631                 r = -ENOMEM;
 632                 goto finish;
 633         }
 634
 635         *_text = text;
 636         r = 0;
 637
 638 finish:
 639         if (p)
 640                 munmap(p, st.st_size);
 641
 642         return r;
 643 }
 644
 645 static char *find_header(const char *s, const char *header) {
 646
 647         for (;;) {
 648                 const char *v;
 649
 650                 v = startswith(s, header);
 651                 if (v) {
 652                         v += strspn(v, WHITESPACE);
 653                         return strndup(v, strcspn(v, NEWLINE));
 654                 }
 655
 656                 if (!next_header(&s))
 657                         return NULL;
 658         }
 659 }
 660
 661 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 662         if (oneline) {
 663                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 664
 665                 subject = find_header(s, "Subject:");
 666                 defined_by = find_header(s, "Defined-By:");
 667
 668                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 669                         SD_ID128_FORMAT_VAL(id),
 670                         strna(defined_by), strna(subject));
 671         } else
 672                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 673                         SD_ID128_FORMAT_VAL(id), s);
 674 }
 675
 676 int catalog_list(FILE *f, const char *database, bool oneline) {
 677         _cleanup_close_ int fd = -1;
 678         void *p = NULL;
 679         struct stat st;
 680         const CatalogHeader *h;
 681         const CatalogItem *items;
 682         int r;
 683         unsigned n;
 684         sd_id128_t last_id;
 685         bool last_id_set = false;
 686
 687         r = open_mmap(database, &fd, &st, &p);
 688         if (r < 0)
 689                 return r;
 690
 691         h = p;
 692         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 693
 694         for (n = 0; n < le64toh(h->n_items); n++) {
 695                 const char *s;
 696
 697                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 698                         continue;
 699
 700                 assert_se(s = find_id(p, items[n].id));
 701
 702                 dump_catalog_entry(f, items[n].id, s, oneline);
 703
 704                 last_id_set = true;
 705                 last_id = items[n].id;
 706         }
 707
 708         munmap(p, st.st_size);
 709
 710         return 0;
 711 }
 712
 713 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 714         char **item;
 715         int r = 0;
 716
 717         STRV_FOREACH(item, items) {
 718                 sd_id128_t id;
 719                 int k;
 720                 _cleanup_free_ char *msg = NULL;
 721
 722                 k = sd_id128_from_string(*item, &id);
 723                 if (k < 0) {
 724                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 725                         if (r == 0)
 726                                 r = k;
 727                         continue;
 728                 }
 729
 730                 k = catalog_get(database, id, &msg);
 731                 if (k < 0) {
 732                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 733                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 734                         if (r == 0)
 735                                 r = k;
 736                         continue;
 737                 }
 738
 739                 dump_catalog_entry(f, id, msg, oneline);
 740         }
 741
 742         return r;
 743 }