src/journal/catalog.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2 /***
   3   This file is part of systemd.
   4
   5   Copyright 2012 Lennart Poettering
   6
   7   systemd is free software; you can redistribute it and/or modify it
   8   under the terms of the GNU Lesser General Public License as published by
   9   the Free Software Foundation; either version 2.1 of the License, or
  10   (at your option) any later version.
  11
  12   systemd is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   Lesser General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  19 ***/
  20
  21 #include <errno.h>
  22 #include <fcntl.h>
  23 #include <locale.h>
  24 #include <stdio.h>
  25 #include <string.h>
  26 #include <sys/mman.h>
  27 #include <unistd.h>
  28
  29 #include "sd-id128.h"
  30
  31 #include "alloc-util.h"
  32 #include "catalog.h"
  33 #include "conf-files.h"
  34 #include "fd-util.h"
  35 #include "fileio.h"
  36 #include "hashmap.h"
  37 #include "log.h"
  38 #include "mkdir.h"
  39 #include "path-util.h"
  40 #include "siphash24.h"
  41 #include "sparse-endian.h"
  42 #include "strbuf.h"
  43 #include "string-util.h"
  44 #include "strv.h"
  45 #include "util.h"
  46
  47 const char * const catalog_file_dirs[] = {
  48         "/usr/local/lib/systemd/catalog/",
  49         "/usr/lib/systemd/catalog/",
  50         NULL
  51 };
  52
  53 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  54
  55 typedef struct CatalogHeader {
  56         uint8_t signature[8];  /* "RHHHKSLP" */
  57         le32_t compatible_flags;
  58         le32_t incompatible_flags;
  59         le64_t header_size;
  60         le64_t n_items;
  61         le64_t catalog_item_size;
  62 } CatalogHeader;
  63
  64 typedef struct CatalogItem {
  65         sd_id128_t id;
  66         char language[32];
  67         le64_t offset;
  68 } CatalogItem;
  69
  70 static void catalog_hash_func(const void *p, struct siphash *state) {
  71         const CatalogItem *i = p;
  72
  73         siphash24_compress(&i->id, sizeof(i->id), state);
  74         siphash24_compress(i->language, strlen(i->language), state);
  75 }
  76
  77 static int catalog_compare_func(const void *a, const void *b) {
  78         const CatalogItem *i = a, *j = b;
  79         unsigned k;
  80
  81         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  82                  if (i->id.bytes[k] < j->id.bytes[k])
  83                         return -1;
  84                  if (i->id.bytes[k] > j->id.bytes[k])
  85                         return 1;
  86         }
  87
  88         return strcmp(i->language, j->language);
  89 }
  90
  91 const struct hash_ops catalog_hash_ops = {
  92         .hash = catalog_hash_func,
  93         .compare = catalog_compare_func
  94 };
  95
  96 static bool next_header(const char **s) {
  97         const char *e;
  98
  99         e = strchr(*s, '\n');
 100
 101         /* Unexpected end */
 102         if (!e)
 103                 return false;
 104
 105         /* End of headers */
 106         if (e == *s)
 107                 return false;
 108
 109         *s = e + 1;
 110         return true;
 111 }
 112
 113 static const char *skip_header(const char *s) {
 114         while (next_header(&s))
 115                 ;
 116         return s;
 117 }
 118
 119 static char *combine_entries(const char *one, const char *two) {
 120         const char *b1, *b2;
 121         size_t l1, l2, n;
 122         char *dest, *p;
 123
 124         /* Find split point of headers to body */
 125         b1 = skip_header(one);
 126         b2 = skip_header(two);
 127
 128         l1 = strlen(one);
 129         l2 = strlen(two);
 130         dest = new(char, l1 + l2 + 1);
 131         if (!dest) {
 132                 log_oom();
 133                 return NULL;
 134         }
 135
 136         p = dest;
 137
 138         /* Headers from @one */
 139         n = b1 - one;
 140         p = mempcpy(p, one, n);
 141
 142         /* Headers from @two, these will only be found if not present above */
 143         n = b2 - two;
 144         p = mempcpy(p, two, n);
 145
 146         /* Body from @one */
 147         n = l1 - (b1 - one);
 148         if (n > 0) {
 149                 memcpy(p, b1, n);
 150                 p += n;
 151
 152         /* Body from @two */
 153         } else {
 154                 n = l2 - (b2 - two);
 155                 memcpy(p, b2, n);
 156                 p += n;
 157         }
 158
 159         assert(p - dest <= (ptrdiff_t)(l1 + l2));
 160         p[0] = '\0';
 161         return dest;
 162 }
 163
 164 static int finish_item(
 165                 Hashmap *h,
 166                 sd_id128_t id,
 167                 const char *language,
 168                 char *payload, size_t payload_size) {
 169
 170         _cleanup_free_ CatalogItem *i = NULL;
 171         _cleanup_free_ char *prev = NULL, *combined = NULL;
 172
 173         assert(h);
 174         assert(payload);
 175         assert(payload_size > 0);
 176
 177         i = new0(CatalogItem, 1);
 178         if (!i)
 179                 return log_oom();
 180
 181         i->id = id;
 182         if (language) {
 183                 assert(strlen(language) > 1 && strlen(language) < 32);
 184                 strcpy(i->language, language);
 185         }
 186
 187         prev = hashmap_get(h, i);
 188         if (prev) {
 189                 /* Already have such an item, combine them */
 190                 combined = combine_entries(payload, prev);
 191                 if (!combined)
 192                         return log_oom();
 193
 194                 if (hashmap_update(h, i, combined) < 0)
 195                         return log_oom();
 196                 combined = NULL;
 197         } else {
 198                 /* A new item */
 199                 combined = memdup(payload, payload_size + 1);
 200                 if (!combined)
 201                         return log_oom();
 202
 203                 if (hashmap_put(h, i, combined) < 0)
 204                         return log_oom();
 205                 i = NULL;
 206                 combined = NULL;
 207         }
 208
 209         return 0;
 210 }
 211
 212 int catalog_file_lang(const char* filename, char **lang) {
 213         char *beg, *end, *_lang;
 214
 215         end = endswith(filename, ".catalog");
 216         if (!end)
 217                 return 0;
 218
 219         beg = end - 1;
 220         while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
 221                 beg--;
 222
 223         if (*beg != '.' || end <= beg + 1)
 224                 return 0;
 225
 226         _lang = strndup(beg + 1, end - beg - 1);
 227         if (!_lang)
 228                 return -ENOMEM;
 229
 230         *lang = _lang;
 231         return 1;
 232 }
 233
 234 static int catalog_entry_lang(const char* filename, int line,
 235                               const char* t, const char* deflang, char **lang) {
 236         size_t c;
 237
 238         c = strlen(t);
 239         if (c == 0) {
 240                 log_error("[%s:%u] Language too short.", filename, line);
 241                 return -EINVAL;
 242         }
 243         if (c > 31) {
 244                 log_error("[%s:%u] language too long.", filename, line);
 245                 return -EINVAL;
 246         }
 247
 248         if (deflang) {
 249                 if (streq(t, deflang)) {
 250                         log_warning("[%s:%u] language specified unnecessarily",
 251                                     filename, line);
 252                         return 0;
 253                 } else
 254                         log_warning("[%s:%u] language differs from default for file",
 255                                     filename, line);
 256         }
 257
 258         *lang = strdup(t);
 259         if (!*lang)
 260                         return -ENOMEM;
 261
 262         return 0;
 263 }
 264
 265 int catalog_import_file(Hashmap *h, const char *path) {
 266         _cleanup_fclose_ FILE *f = NULL;
 267         _cleanup_free_ char *payload = NULL;
 268         size_t payload_size = 0, payload_allocated = 0;
 269         unsigned n = 0;
 270         sd_id128_t id;
 271         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 272         bool got_id = false, empty_line = true;
 273         int r;
 274
 275         assert(h);
 276         assert(path);
 277
 278         f = fopen(path, "re");
 279         if (!f)
 280                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 281
 282         r = catalog_file_lang(path, &deflang);
 283         if (r < 0)
 284                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 285         if (r == 1)
 286                 log_debug("File %s has language %s.", path, deflang);
 287
 288         for (;;) {
 289                 char line[LINE_MAX];
 290                 size_t line_len;
 291
 292                 if (!fgets(line, sizeof(line), f)) {
 293                         if (feof(f))
 294                                 break;
 295
 296                         return log_error_errno(errno, "Failed to read file %s: %m", path);
 297                 }
 298
 299                 n++;
 300
 301                 truncate_nl(line);
 302
 303                 if (line[0] == 0) {
 304                         empty_line = true;
 305                         continue;
 306                 }
 307
 308                 if (strchr(COMMENTS "\n", line[0]))
 309                         continue;
 310
 311                 if (empty_line &&
 312                     strlen(line) >= 2+1+32 &&
 313                     line[0] == '-' &&
 314                     line[1] == '-' &&
 315                     line[2] == ' ' &&
 316                     IN_SET(line[2+1+32], ' ', '\0')) {
 317
 318                         bool with_language;
 319                         sd_id128_t jd;
 320
 321                         /* New entry */
 322
 323                         with_language = line[2+1+32] != '\0';
 324                         line[2+1+32] = '\0';
 325
 326                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 327
 328                                 if (got_id) {
 329                                         if (payload_size == 0) {
 330                                                 log_error("[%s:%u] No payload text.", path, n);
 331                                                 return -EINVAL;
 332                                         }
 333
 334                                         r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 335                                         if (r < 0)
 336                                                 return r;
 337
 338                                         lang = mfree(lang);
 339                                         payload_size = 0;
 340                                 }
 341
 342                                 if (with_language) {
 343                                         char *t;
 344
 345                                         t = strstrip(line + 2 + 1 + 32 + 1);
 346                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 347                                         if (r < 0)
 348                                                 return r;
 349                                 }
 350
 351                                 got_id = true;
 352                                 empty_line = false;
 353                                 id = jd;
 354
 355                                 continue;
 356                         }
 357                 }
 358
 359                 /* Payload */
 360                 if (!got_id) {
 361                         log_error("[%s:%u] Got payload before ID.", path, n);
 362                         return -EINVAL;
 363                 }
 364
 365                 line_len = strlen(line);
 366                 if (!GREEDY_REALLOC(payload, payload_allocated,
 367                                     payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
 368                         return log_oom();
 369
 370                 if (empty_line)
 371                         payload[payload_size++] = '\n';
 372                 memcpy(payload + payload_size, line, line_len);
 373                 payload_size += line_len;
 374                 payload[payload_size++] = '\n';
 375                 payload[payload_size] = '\0';
 376
 377                 empty_line = false;
 378         }
 379
 380         if (got_id) {
 381                 if (payload_size == 0) {
 382                         log_error("[%s:%u] No payload text.", path, n);
 383                         return -EINVAL;
 384                 }
 385
 386                 r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 387                 if (r < 0)
 388                         return r;
 389         }
 390
 391         return 0;
 392 }
 393
 394 static int64_t write_catalog(const char *database, struct strbuf *sb,
 395                              CatalogItem *items, size_t n) {
 396         CatalogHeader header;
 397         _cleanup_fclose_ FILE *w = NULL;
 398         int r;
 399         _cleanup_free_ char *d, *p = NULL;
 400         size_t k;
 401
 402         d = dirname_malloc(database);
 403         if (!d)
 404                 return log_oom();
 405
 406         r = mkdir_p(d, 0775);
 407         if (r < 0)
 408                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 409
 410         r = fopen_temporary(database, &w, &p);
 411         if (r < 0)
 412                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 413                                        database);
 414
 415         zero(header);
 416         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 417         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 418         header.catalog_item_size = htole64(sizeof(CatalogItem));
 419         header.n_items = htole64(n);
 420
 421         r = -EIO;
 422
 423         k = fwrite(&header, 1, sizeof(header), w);
 424         if (k != sizeof(header)) {
 425                 log_error("%s: failed to write header.", p);
 426                 goto error;
 427         }
 428
 429         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 430         if (k != n * sizeof(CatalogItem)) {
 431                 log_error("%s: failed to write database.", p);
 432                 goto error;
 433         }
 434
 435         k = fwrite(sb->buf, 1, sb->len, w);
 436         if (k != sb->len) {
 437                 log_error("%s: failed to write strings.", p);
 438                 goto error;
 439         }
 440
 441         r = fflush_and_check(w);
 442         if (r < 0) {
 443                 log_error_errno(r, "%s: failed to write database: %m", p);
 444                 goto error;
 445         }
 446
 447         fchmod(fileno(w), 0644);
 448
 449         if (rename(p, database) < 0) {
 450                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 451                 goto error;
 452         }
 453
 454         return ftello(w);
 455
 456 error:
 457         (void) unlink(p);
 458         return r;
 459 }
 460
 461 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 462         _cleanup_strv_free_ char **files = NULL;
 463         char **f;
 464         struct strbuf *sb = NULL;
 465         _cleanup_hashmap_free_free_free_ Hashmap *h = NULL;
 466         _cleanup_free_ CatalogItem *items = NULL;
 467         ssize_t offset;
 468         char *payload;
 469         CatalogItem *i;
 470         Iterator j;
 471         unsigned n;
 472         int r;
 473         int64_t sz;
 474
 475         h = hashmap_new(&catalog_hash_ops);
 476         sb = strbuf_new();
 477
 478         if (!h || !sb) {
 479                 r = log_oom();
 480                 goto finish;
 481         }
 482
 483         r = conf_files_list_strv(&files, ".catalog", root, 0, dirs);
 484         if (r < 0) {
 485                 log_error_errno(r, "Failed to get catalog files: %m");
 486                 goto finish;
 487         }
 488
 489         STRV_FOREACH(f, files) {
 490                 log_debug("Reading file '%s'", *f);
 491                 r = catalog_import_file(h, *f);
 492                 if (r < 0) {
 493                         log_error_errno(r, "Failed to import file '%s': %m", *f);
 494                         goto finish;
 495                 }
 496         }
 497
 498         if (hashmap_size(h) <= 0) {
 499                 log_info("No items in catalog.");
 500                 goto finish;
 501         } else
 502                 log_debug("Found %u items in catalog.", hashmap_size(h));
 503
 504         items = new(CatalogItem, hashmap_size(h));
 505         if (!items) {
 506                 r = log_oom();
 507                 goto finish;
 508         }
 509
 510         n = 0;
 511         HASHMAP_FOREACH_KEY(payload, i, h, j) {
 512                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 513                           SD_ID128_FORMAT_VAL(i->id),
 514                           isempty(i->language) ? "C" : i->language);
 515
 516                 offset = strbuf_add_string(sb, payload, strlen(payload));
 517                 if (offset < 0) {
 518                         r = log_oom();
 519                         goto finish;
 520                 }
 521                 i->offset = htole64((uint64_t) offset);
 522                 items[n++] = *i;
 523         }
 524
 525         assert(n == hashmap_size(h));
 526         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 527
 528         strbuf_complete(sb);
 529
 530         sz = write_catalog(database, sb, items, n);
 531         if (sz < 0)
 532                 r = log_error_errno(sz, "Failed to write %s: %m", database);
 533         else {
 534                 r = 0;
 535                 log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 536                           database, n, sb->len, sz);
 537         }
 538
 539 finish:
 540         strbuf_cleanup(sb);
 541
 542         return r;
 543 }
 544
 545 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 546         const CatalogHeader *h;
 547         int fd;
 548         void *p;
 549         struct stat st;
 550
 551         assert(_fd);
 552         assert(_st);
 553         assert(_p);
 554
 555         fd = open(database, O_RDONLY|O_CLOEXEC);
 556         if (fd < 0)
 557                 return -errno;
 558
 559         if (fstat(fd, &st) < 0) {
 560                 safe_close(fd);
 561                 return -errno;
 562         }
 563
 564         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 565                 safe_close(fd);
 566                 return -EINVAL;
 567         }
 568
 569         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 570         if (p == MAP_FAILED) {
 571                 safe_close(fd);
 572                 return -errno;
 573         }
 574
 575         h = p;
 576         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 577             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 578             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 579             h->incompatible_flags != 0 ||
 580             le64toh(h->n_items) <= 0 ||
 581             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 582                 safe_close(fd);
 583                 munmap(p, st.st_size);
 584                 return -EBADMSG;
 585         }
 586
 587         *_fd = fd;
 588         *_st = st;
 589         *_p = p;
 590
 591         return 0;
 592 }
 593
 594 static const char *find_id(void *p, sd_id128_t id) {
 595         CatalogItem key, *f = NULL;
 596         const CatalogHeader *h = p;
 597         const char *loc;
 598
 599         zero(key);
 600         key.id = id;
 601
 602         loc = setlocale(LC_MESSAGES, NULL);
 603         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 604                 strncpy(key.language, loc, sizeof(key.language));
 605                 key.language[strcspn(key.language, ".@")] = 0;
 606
 607                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 608                 if (!f) {
 609                         char *e;
 610
 611                         e = strchr(key.language, '_');
 612                         if (e) {
 613                                 *e = 0;
 614                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 615                         }
 616                 }
 617         }
 618
 619         if (!f) {
 620                 zero(key.language);
 621                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 622         }
 623
 624         if (!f)
 625                 return NULL;
 626
 627         return (const char*) p +
 628                 le64toh(h->header_size) +
 629                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 630                 le64toh(f->offset);
 631 }
 632
 633 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 634         _cleanup_close_ int fd = -1;
 635         void *p = NULL;
 636         struct stat st = {};
 637         char *text = NULL;
 638         int r;
 639         const char *s;
 640
 641         assert(_text);
 642
 643         r = open_mmap(database, &fd, &st, &p);
 644         if (r < 0)
 645                 return r;
 646
 647         s = find_id(p, id);
 648         if (!s) {
 649                 r = -ENOENT;
 650                 goto finish;
 651         }
 652
 653         text = strdup(s);
 654         if (!text) {
 655                 r = -ENOMEM;
 656                 goto finish;
 657         }
 658
 659         *_text = text;
 660         r = 0;
 661
 662 finish:
 663         if (p)
 664                 munmap(p, st.st_size);
 665
 666         return r;
 667 }
 668
 669 static char *find_header(const char *s, const char *header) {
 670
 671         for (;;) {
 672                 const char *v;
 673
 674                 v = startswith(s, header);
 675                 if (v) {
 676                         v += strspn(v, WHITESPACE);
 677                         return strndup(v, strcspn(v, NEWLINE));
 678                 }
 679
 680                 if (!next_header(&s))
 681                         return NULL;
 682         }
 683 }
 684
 685 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 686         if (oneline) {
 687                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 688
 689                 subject = find_header(s, "Subject:");
 690                 defined_by = find_header(s, "Defined-By:");
 691
 692                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 693                         SD_ID128_FORMAT_VAL(id),
 694                         strna(defined_by), strna(subject));
 695         } else
 696                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 697                         SD_ID128_FORMAT_VAL(id), s);
 698 }
 699
 700
 701 int catalog_list(FILE *f, const char *database, bool oneline) {
 702         _cleanup_close_ int fd = -1;
 703         void *p = NULL;
 704         struct stat st;
 705         const CatalogHeader *h;
 706         const CatalogItem *items;
 707         int r;
 708         unsigned n;
 709         sd_id128_t last_id;
 710         bool last_id_set = false;
 711
 712         r = open_mmap(database, &fd, &st, &p);
 713         if (r < 0)
 714                 return r;
 715
 716         h = p;
 717         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 718
 719         for (n = 0; n < le64toh(h->n_items); n++) {
 720                 const char *s;
 721
 722                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 723                         continue;
 724
 725                 assert_se(s = find_id(p, items[n].id));
 726
 727                 dump_catalog_entry(f, items[n].id, s, oneline);
 728
 729                 last_id_set = true;
 730                 last_id = items[n].id;
 731         }
 732
 733         munmap(p, st.st_size);
 734
 735         return 0;
 736 }
 737
 738 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 739         char **item;
 740         int r = 0;
 741
 742         STRV_FOREACH(item, items) {
 743                 sd_id128_t id;
 744                 int k;
 745                 _cleanup_free_ char *msg = NULL;
 746
 747                 k = sd_id128_from_string(*item, &id);
 748                 if (k < 0) {
 749                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 750                         if (r == 0)
 751                                 r = k;
 752                         continue;
 753                 }
 754
 755                 k = catalog_get(database, id, &msg);
 756                 if (k < 0) {
 757                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 758                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 759                         if (r == 0)
 760                                 r = k;
 761                         continue;
 762                 }
 763
 764                 dump_catalog_entry(f, id, msg, oneline);
 765         }
 766
 767         return r;
 768 }