src/journal/catalog.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #include <errno.h>
   4 #include <fcntl.h>
   5 #include <locale.h>
   6 #include <stdio.h>
   7 #include <string.h>
   8 #include <sys/mman.h>
   9 #include <unistd.h>
  10
  11 #include "sd-id128.h"
  12
  13 #include "alloc-util.h"
  14 #include "catalog.h"
  15 #include "conf-files.h"
  16 #include "def.h"
  17 #include "fd-util.h"
  18 #include "fileio.h"
  19 #include "hashmap.h"
  20 #include "log.h"
  21 #include "mkdir.h"
  22 #include "path-util.h"
  23 #include "siphash24.h"
  24 #include "sparse-endian.h"
  25 #include "strbuf.h"
  26 #include "string-util.h"
  27 #include "strv.h"
  28 #include "util.h"
  29
  30 const char * const catalog_file_dirs[] = {
  31         "/usr/local/lib/systemd/catalog/",
  32         "/usr/lib/systemd/catalog/",
  33         NULL
  34 };
  35
  36 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  37
  38 typedef struct CatalogHeader {
  39         uint8_t signature[8];  /* "RHHHKSLP" */
  40         le32_t compatible_flags;
  41         le32_t incompatible_flags;
  42         le64_t header_size;
  43         le64_t n_items;
  44         le64_t catalog_item_size;
  45 } CatalogHeader;
  46
  47 typedef struct CatalogItem {
  48         sd_id128_t id;
  49         char language[32];
  50         le64_t offset;
  51 } CatalogItem;
  52
  53 static void catalog_hash_func(const void *p, struct siphash *state) {
  54         const CatalogItem *i = p;
  55
  56         siphash24_compress(&i->id, sizeof(i->id), state);
  57         siphash24_compress(i->language, strlen(i->language), state);
  58 }
  59
  60 static int catalog_compare_func(const CatalogItem *a, const CatalogItem *b) {
  61         unsigned k;
  62         int r;
  63
  64         for (k = 0; k < ELEMENTSOF(b->id.bytes); k++) {
  65                 r = CMP(a->id.bytes[k], b->id.bytes[k]);
  66                 if (r != 0)
  67                         return r;
  68         }
  69
  70         return strcmp(a->language, b->language);
  71 }
  72
  73 const struct hash_ops catalog_hash_ops = {
  74         .hash = catalog_hash_func,
  75         .compare = (comparison_fn_t) catalog_compare_func,
  76 };
  77
  78 static bool next_header(const char **s) {
  79         const char *e;
  80
  81         e = strchr(*s, '\n');
  82
  83         /* Unexpected end */
  84         if (!e)
  85                 return false;
  86
  87         /* End of headers */
  88         if (e == *s)
  89                 return false;
  90
  91         *s = e + 1;
  92         return true;
  93 }
  94
  95 static const char *skip_header(const char *s) {
  96         while (next_header(&s))
  97                 ;
  98         return s;
  99 }
 100
 101 static char *combine_entries(const char *one, const char *two) {
 102         const char *b1, *b2;
 103         size_t l1, l2, n;
 104         char *dest, *p;
 105
 106         /* Find split point of headers to body */
 107         b1 = skip_header(one);
 108         b2 = skip_header(two);
 109
 110         l1 = strlen(one);
 111         l2 = strlen(two);
 112         dest = new(char, l1 + l2 + 1);
 113         if (!dest) {
 114                 log_oom();
 115                 return NULL;
 116         }
 117
 118         p = dest;
 119
 120         /* Headers from @one */
 121         n = b1 - one;
 122         p = mempcpy(p, one, n);
 123
 124         /* Headers from @two, these will only be found if not present above */
 125         n = b2 - two;
 126         p = mempcpy(p, two, n);
 127
 128         /* Body from @one */
 129         n = l1 - (b1 - one);
 130         if (n > 0) {
 131                 memcpy(p, b1, n);
 132                 p += n;
 133
 134         /* Body from @two */
 135         } else {
 136                 n = l2 - (b2 - two);
 137                 memcpy(p, b2, n);
 138                 p += n;
 139         }
 140
 141         assert(p - dest <= (ptrdiff_t)(l1 + l2));
 142         p[0] = '\0';
 143         return dest;
 144 }
 145
 146 static int finish_item(
 147                 Hashmap *h,
 148                 sd_id128_t id,
 149                 const char *language,
 150                 char *payload, size_t payload_size) {
 151
 152         _cleanup_free_ CatalogItem *i = NULL;
 153         _cleanup_free_ char *prev = NULL, *combined = NULL;
 154
 155         assert(h);
 156         assert(payload);
 157         assert(payload_size > 0);
 158
 159         i = new0(CatalogItem, 1);
 160         if (!i)
 161                 return log_oom();
 162
 163         i->id = id;
 164         if (language) {
 165                 assert(strlen(language) > 1 && strlen(language) < 32);
 166                 strcpy(i->language, language);
 167         }
 168
 169         prev = hashmap_get(h, i);
 170         if (prev) {
 171                 /* Already have such an item, combine them */
 172                 combined = combine_entries(payload, prev);
 173                 if (!combined)
 174                         return log_oom();
 175
 176                 if (hashmap_update(h, i, combined) < 0)
 177                         return log_oom();
 178                 combined = NULL;
 179         } else {
 180                 /* A new item */
 181                 combined = memdup(payload, payload_size + 1);
 182                 if (!combined)
 183                         return log_oom();
 184
 185                 if (hashmap_put(h, i, combined) < 0)
 186                         return log_oom();
 187                 i = NULL;
 188                 combined = NULL;
 189         }
 190
 191         return 0;
 192 }
 193
 194 int catalog_file_lang(const char* filename, char **lang) {
 195         char *beg, *end, *_lang;
 196
 197         end = endswith(filename, ".catalog");
 198         if (!end)
 199                 return 0;
 200
 201         beg = end - 1;
 202         while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
 203                 beg--;
 204
 205         if (*beg != '.' || end <= beg + 1)
 206                 return 0;
 207
 208         _lang = strndup(beg + 1, end - beg - 1);
 209         if (!_lang)
 210                 return -ENOMEM;
 211
 212         *lang = _lang;
 213         return 1;
 214 }
 215
 216 static int catalog_entry_lang(const char* filename, int line,
 217                               const char* t, const char* deflang, char **lang) {
 218         size_t c;
 219
 220         c = strlen(t);
 221         if (c == 0) {
 222                 log_error("[%s:%u] Language too short.", filename, line);
 223                 return -EINVAL;
 224         }
 225         if (c > 31) {
 226                 log_error("[%s:%u] language too long.", filename, line);
 227                 return -EINVAL;
 228         }
 229
 230         if (deflang) {
 231                 if (streq(t, deflang)) {
 232                         log_warning("[%s:%u] language specified unnecessarily",
 233                                     filename, line);
 234                         return 0;
 235                 } else
 236                         log_warning("[%s:%u] language differs from default for file",
 237                                     filename, line);
 238         }
 239
 240         *lang = strdup(t);
 241         if (!*lang)
 242                         return -ENOMEM;
 243
 244         return 0;
 245 }
 246
 247 int catalog_import_file(Hashmap *h, const char *path) {
 248         _cleanup_fclose_ FILE *f = NULL;
 249         _cleanup_free_ char *payload = NULL;
 250         size_t payload_size = 0, payload_allocated = 0;
 251         unsigned n = 0;
 252         sd_id128_t id;
 253         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 254         bool got_id = false, empty_line = true;
 255         int r;
 256
 257         assert(h);
 258         assert(path);
 259
 260         f = fopen(path, "re");
 261         if (!f)
 262                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 263
 264         r = catalog_file_lang(path, &deflang);
 265         if (r < 0)
 266                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 267         if (r == 1)
 268                 log_debug("File %s has language %s.", path, deflang);
 269
 270         for (;;) {
 271                 _cleanup_free_ char *line = NULL;
 272                 size_t line_len;
 273
 274                 r = read_line(f, LONG_LINE_MAX, &line);
 275                 if (r < 0)
 276                         return log_error_errno(r, "Failed to read file %s: %m", path);
 277                 if (r == 0)
 278                         break;
 279
 280                 n++;
 281
 282                 if (isempty(line)) {
 283                         empty_line = true;
 284                         continue;
 285                 }
 286
 287                 if (strchr(COMMENTS, line[0]))
 288                         continue;
 289
 290                 if (empty_line &&
 291                     strlen(line) >= 2+1+32 &&
 292                     line[0] == '-' &&
 293                     line[1] == '-' &&
 294                     line[2] == ' ' &&
 295                     IN_SET(line[2+1+32], ' ', '\0')) {
 296
 297                         bool with_language;
 298                         sd_id128_t jd;
 299
 300                         /* New entry */
 301
 302                         with_language = line[2+1+32] != '\0';
 303                         line[2+1+32] = '\0';
 304
 305                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 306
 307                                 if (got_id) {
 308                                         if (payload_size == 0) {
 309                                                 log_error("[%s:%u] No payload text.", path, n);
 310                                                 return -EINVAL;
 311                                         }
 312
 313                                         r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 314                                         if (r < 0)
 315                                                 return r;
 316
 317                                         lang = mfree(lang);
 318                                         payload_size = 0;
 319                                 }
 320
 321                                 if (with_language) {
 322                                         char *t;
 323
 324                                         t = strstrip(line + 2 + 1 + 32 + 1);
 325                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 326                                         if (r < 0)
 327                                                 return r;
 328                                 }
 329
 330                                 got_id = true;
 331                                 empty_line = false;
 332                                 id = jd;
 333
 334                                 continue;
 335                         }
 336                 }
 337
 338                 /* Payload */
 339                 if (!got_id) {
 340                         log_error("[%s:%u] Got payload before ID.", path, n);
 341                         return -EINVAL;
 342                 }
 343
 344                 line_len = strlen(line);
 345                 if (!GREEDY_REALLOC(payload, payload_allocated,
 346                                     payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
 347                         return log_oom();
 348
 349                 if (empty_line)
 350                         payload[payload_size++] = '\n';
 351                 memcpy(payload + payload_size, line, line_len);
 352                 payload_size += line_len;
 353                 payload[payload_size++] = '\n';
 354                 payload[payload_size] = '\0';
 355
 356                 empty_line = false;
 357         }
 358
 359         if (got_id) {
 360                 if (payload_size == 0) {
 361                         log_error("[%s:%u] No payload text.", path, n);
 362                         return -EINVAL;
 363                 }
 364
 365                 r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 366                 if (r < 0)
 367                         return r;
 368         }
 369
 370         return 0;
 371 }
 372
 373 static int64_t write_catalog(const char *database, struct strbuf *sb,
 374                              CatalogItem *items, size_t n) {
 375         CatalogHeader header;
 376         _cleanup_fclose_ FILE *w = NULL;
 377         int r;
 378         _cleanup_free_ char *d, *p = NULL;
 379         size_t k;
 380
 381         d = dirname_malloc(database);
 382         if (!d)
 383                 return log_oom();
 384
 385         r = mkdir_p(d, 0775);
 386         if (r < 0)
 387                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 388
 389         r = fopen_temporary(database, &w, &p);
 390         if (r < 0)
 391                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 392                                        database);
 393
 394         zero(header);
 395         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 396         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 397         header.catalog_item_size = htole64(sizeof(CatalogItem));
 398         header.n_items = htole64(n);
 399
 400         r = -EIO;
 401
 402         k = fwrite(&header, 1, sizeof(header), w);
 403         if (k != sizeof(header)) {
 404                 log_error("%s: failed to write header.", p);
 405                 goto error;
 406         }
 407
 408         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 409         if (k != n * sizeof(CatalogItem)) {
 410                 log_error("%s: failed to write database.", p);
 411                 goto error;
 412         }
 413
 414         k = fwrite(sb->buf, 1, sb->len, w);
 415         if (k != sb->len) {
 416                 log_error("%s: failed to write strings.", p);
 417                 goto error;
 418         }
 419
 420         r = fflush_and_check(w);
 421         if (r < 0) {
 422                 log_error_errno(r, "%s: failed to write database: %m", p);
 423                 goto error;
 424         }
 425
 426         fchmod(fileno(w), 0644);
 427
 428         if (rename(p, database) < 0) {
 429                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 430                 goto error;
 431         }
 432
 433         return ftello(w);
 434
 435 error:
 436         (void) unlink(p);
 437         return r;
 438 }
 439
 440 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 441         _cleanup_strv_free_ char **files = NULL;
 442         char **f;
 443         _cleanup_(strbuf_cleanupp) struct strbuf *sb = NULL;
 444         _cleanup_hashmap_free_free_free_ Hashmap *h = NULL;
 445         _cleanup_free_ CatalogItem *items = NULL;
 446         ssize_t offset;
 447         char *payload;
 448         CatalogItem *i;
 449         Iterator j;
 450         unsigned n;
 451         int r;
 452         int64_t sz;
 453
 454         h = hashmap_new(&catalog_hash_ops);
 455         sb = strbuf_new();
 456         if (!h || !sb)
 457                 return log_oom();
 458
 459         r = conf_files_list_strv(&files, ".catalog", root, 0, dirs);
 460         if (r < 0)
 461                 return log_error_errno(r, "Failed to get catalog files: %m");
 462
 463         STRV_FOREACH(f, files) {
 464                 log_debug("Reading file '%s'", *f);
 465                 r = catalog_import_file(h, *f);
 466                 if (r < 0)
 467                         return log_error_errno(r, "Failed to import file '%s': %m", *f);
 468         }
 469
 470         if (hashmap_size(h) <= 0) {
 471                 log_info("No items in catalog.");
 472                 return 0;
 473         } else
 474                 log_debug("Found %u items in catalog.", hashmap_size(h));
 475
 476         items = new(CatalogItem, hashmap_size(h));
 477         if (!items)
 478                 return log_oom();
 479
 480         n = 0;
 481         HASHMAP_FOREACH_KEY(payload, i, h, j) {
 482                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 483                           SD_ID128_FORMAT_VAL(i->id),
 484                           isempty(i->language) ? "C" : i->language);
 485
 486                 offset = strbuf_add_string(sb, payload, strlen(payload));
 487                 if (offset < 0)
 488                         return log_oom();
 489
 490                 i->offset = htole64((uint64_t) offset);
 491                 items[n++] = *i;
 492         }
 493
 494         assert(n == hashmap_size(h));
 495         typesafe_qsort(items, n, catalog_compare_func);
 496
 497         strbuf_complete(sb);
 498
 499         sz = write_catalog(database, sb, items, n);
 500         if (sz < 0)
 501                 return log_error_errno(sz, "Failed to write %s: %m", database);
 502
 503         log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 504                   database, n, sb->len, sz);
 505         return 0;
 506 }
 507
 508 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 509         const CatalogHeader *h;
 510         int fd;
 511         void *p;
 512         struct stat st;
 513
 514         assert(_fd);
 515         assert(_st);
 516         assert(_p);
 517
 518         fd = open(database, O_RDONLY|O_CLOEXEC);
 519         if (fd < 0)
 520                 return -errno;
 521
 522         if (fstat(fd, &st) < 0) {
 523                 safe_close(fd);
 524                 return -errno;
 525         }
 526
 527         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 528                 safe_close(fd);
 529                 return -EINVAL;
 530         }
 531
 532         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 533         if (p == MAP_FAILED) {
 534                 safe_close(fd);
 535                 return -errno;
 536         }
 537
 538         h = p;
 539         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 540             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 541             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 542             h->incompatible_flags != 0 ||
 543             le64toh(h->n_items) <= 0 ||
 544             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 545                 safe_close(fd);
 546                 munmap(p, st.st_size);
 547                 return -EBADMSG;
 548         }
 549
 550         *_fd = fd;
 551         *_st = st;
 552         *_p = p;
 553
 554         return 0;
 555 }
 556
 557 static const char *find_id(void *p, sd_id128_t id) {
 558         CatalogItem *f = NULL, key = { .id = id };
 559         const CatalogHeader *h = p;
 560         const char *loc;
 561
 562         loc = setlocale(LC_MESSAGES, NULL);
 563         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 564                 strncpy(key.language, loc, sizeof(key.language));
 565                 key.language[strcspn(key.language, ".@")] = 0;
 566
 567                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), (comparison_fn_t) catalog_compare_func);
 568                 if (!f) {
 569                         char *e;
 570
 571                         e = strchr(key.language, '_');
 572                         if (e) {
 573                                 *e = 0;
 574                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), (comparison_fn_t) catalog_compare_func);
 575                         }
 576                 }
 577         }
 578
 579         if (!f) {
 580                 zero(key.language);
 581                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), (comparison_fn_t) catalog_compare_func);
 582         }
 583
 584         if (!f)
 585                 return NULL;
 586
 587         return (const char*) p +
 588                 le64toh(h->header_size) +
 589                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 590                 le64toh(f->offset);
 591 }
 592
 593 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 594         _cleanup_close_ int fd = -1;
 595         void *p = NULL;
 596         struct stat st = {};
 597         char *text = NULL;
 598         int r;
 599         const char *s;
 600
 601         assert(_text);
 602
 603         r = open_mmap(database, &fd, &st, &p);
 604         if (r < 0)
 605                 return r;
 606
 607         s = find_id(p, id);
 608         if (!s) {
 609                 r = -ENOENT;
 610                 goto finish;
 611         }
 612
 613         text = strdup(s);
 614         if (!text) {
 615                 r = -ENOMEM;
 616                 goto finish;
 617         }
 618
 619         *_text = text;
 620         r = 0;
 621
 622 finish:
 623         if (p)
 624                 munmap(p, st.st_size);
 625
 626         return r;
 627 }
 628
 629 static char *find_header(const char *s, const char *header) {
 630
 631         for (;;) {
 632                 const char *v;
 633
 634                 v = startswith(s, header);
 635                 if (v) {
 636                         v += strspn(v, WHITESPACE);
 637                         return strndup(v, strcspn(v, NEWLINE));
 638                 }
 639
 640                 if (!next_header(&s))
 641                         return NULL;
 642         }
 643 }
 644
 645 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 646         if (oneline) {
 647                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 648
 649                 subject = find_header(s, "Subject:");
 650                 defined_by = find_header(s, "Defined-By:");
 651
 652                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 653                         SD_ID128_FORMAT_VAL(id),
 654                         strna(defined_by), strna(subject));
 655         } else
 656                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 657                         SD_ID128_FORMAT_VAL(id), s);
 658 }
 659
 660 int catalog_list(FILE *f, const char *database, bool oneline) {
 661         _cleanup_close_ int fd = -1;
 662         void *p = NULL;
 663         struct stat st;
 664         const CatalogHeader *h;
 665         const CatalogItem *items;
 666         int r;
 667         unsigned n;
 668         sd_id128_t last_id;
 669         bool last_id_set = false;
 670
 671         r = open_mmap(database, &fd, &st, &p);
 672         if (r < 0)
 673                 return r;
 674
 675         h = p;
 676         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 677
 678         for (n = 0; n < le64toh(h->n_items); n++) {
 679                 const char *s;
 680
 681                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 682                         continue;
 683
 684                 assert_se(s = find_id(p, items[n].id));
 685
 686                 dump_catalog_entry(f, items[n].id, s, oneline);
 687
 688                 last_id_set = true;
 689                 last_id = items[n].id;
 690         }
 691
 692         munmap(p, st.st_size);
 693
 694         return 0;
 695 }
 696
 697 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 698         char **item;
 699         int r = 0;
 700
 701         STRV_FOREACH(item, items) {
 702                 sd_id128_t id;
 703                 int k;
 704                 _cleanup_free_ char *msg = NULL;
 705
 706                 k = sd_id128_from_string(*item, &id);
 707                 if (k < 0) {
 708                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 709                         if (r == 0)
 710                                 r = k;
 711                         continue;
 712                 }
 713
 714                 k = catalog_get(database, id, &msg);
 715                 if (k < 0) {
 716                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 717                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 718                         if (r == 0)
 719                                 r = k;
 720                         continue;
 721                 }
 722
 723                 dump_catalog_entry(f, id, msg, oneline);
 724         }
 725
 726         return r;
 727 }