src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <locale.h>
  25 #include <stdio.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <unistd.h>
  29
  30 #include "sd-id128.h"
  31
  32 #include "alloc-util.h"
  33 #include "catalog.h"
  34 #include "conf-files.h"
  35 #include "fd-util.h"
  36 #include "fileio.h"
  37 #include "hashmap.h"
  38 #include "log.h"
  39 #include "mkdir.h"
  40 #include "path-util.h"
  41 #include "siphash24.h"
  42 #include "sparse-endian.h"
  43 #include "strbuf.h"
  44 #include "string-util.h"
  45 #include "strv.h"
  46 #include "util.h"
  47
  48 const char * const catalog_file_dirs[] = {
  49         "/usr/local/lib/systemd/catalog/",
  50         "/usr/lib/systemd/catalog/",
  51         NULL
  52 };
  53
  54 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  55
  56 typedef struct CatalogHeader {
  57         uint8_t signature[8];  /* "RHHHKSLP" */
  58         le32_t compatible_flags;
  59         le32_t incompatible_flags;
  60         le64_t header_size;
  61         le64_t n_items;
  62         le64_t catalog_item_size;
  63 } CatalogHeader;
  64
  65 typedef struct CatalogItem {
  66         sd_id128_t id;
  67         char language[32];
  68         le64_t offset;
  69 } CatalogItem;
  70
  71 static void catalog_hash_func(const void *p, struct siphash *state) {
  72         const CatalogItem *i = p;
  73
  74         siphash24_compress(&i->id, sizeof(i->id), state);
  75         siphash24_compress(i->language, strlen(i->language), state);
  76 }
  77
  78 static int catalog_compare_func(const void *a, const void *b) {
  79         const CatalogItem *i = a, *j = b;
  80         unsigned k;
  81
  82         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  83                  if (i->id.bytes[k] < j->id.bytes[k])
  84                         return -1;
  85                  if (i->id.bytes[k] > j->id.bytes[k])
  86                         return 1;
  87         }
  88
  89         return strcmp(i->language, j->language);
  90 }
  91
  92 const struct hash_ops catalog_hash_ops = {
  93         .hash = catalog_hash_func,
  94         .compare = catalog_compare_func
  95 };
  96
  97 static int finish_item(
  98                 Hashmap *h,
  99                 struct strbuf *sb,
 100                 sd_id128_t id,
 101                 const char *language,
 102                 const char *payload) {
 103
 104         ssize_t offset;
 105         _cleanup_free_ CatalogItem *i = NULL;
 106         int r;
 107
 108         assert(h);
 109         assert(sb);
 110         assert(payload);
 111
 112         offset = strbuf_add_string(sb, payload, strlen(payload));
 113         if (offset < 0)
 114                 return log_oom();
 115
 116         i = new0(CatalogItem, 1);
 117         if (!i)
 118                 return log_oom();
 119
 120         i->id = id;
 121         if (language) {
 122                 assert(strlen(language) > 1 && strlen(language) < 32);
 123                 strcpy(i->language, language);
 124         }
 125         i->offset = htole64((uint64_t) offset);
 126
 127         r = hashmap_put(h, i, i);
 128         if (r == -EEXIST) {
 129                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 130                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 131                 return 0;
 132         } else if (r < 0)
 133                 return r;
 134
 135         i = NULL;
 136         return 0;
 137 }
 138
 139 int catalog_file_lang(const char* filename, char **lang) {
 140         char *beg, *end, *_lang;
 141
 142         end = endswith(filename, ".catalog");
 143         if (!end)
 144                 return 0;
 145
 146         beg = end - 1;
 147         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 148                 beg --;
 149
 150         if (*beg != '.' || end <= beg + 1)
 151                 return 0;
 152
 153         _lang = strndup(beg + 1, end - beg - 1);
 154         if (!_lang)
 155                 return -ENOMEM;
 156
 157         *lang = _lang;
 158         return 1;
 159 }
 160
 161 static int catalog_entry_lang(const char* filename, int line,
 162                               const char* t, const char* deflang, char **lang) {
 163         size_t c;
 164
 165         c = strlen(t);
 166         if (c == 0) {
 167                 log_error("[%s:%u] Language too short.", filename, line);
 168                 return -EINVAL;
 169         }
 170         if (c > 31) {
 171                 log_error("[%s:%u] language too long.", filename, line);
 172                 return -EINVAL;
 173         }
 174
 175         if (deflang) {
 176                 if (streq(t, deflang)) {
 177                         log_warning("[%s:%u] language specified unnecessarily",
 178                                     filename, line);
 179                         return 0;
 180                 } else
 181                         log_warning("[%s:%u] language differs from default for file",
 182                                     filename, line);
 183         }
 184
 185         *lang = strdup(t);
 186         if (!*lang)
 187                         return -ENOMEM;
 188
 189         return 0;
 190 }
 191
 192 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 193         _cleanup_fclose_ FILE *f = NULL;
 194         _cleanup_free_ char *payload = NULL;
 195         unsigned n = 0;
 196         sd_id128_t id;
 197         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 198         bool got_id = false, empty_line = true;
 199         int r;
 200
 201         assert(h);
 202         assert(sb);
 203         assert(path);
 204
 205         f = fopen(path, "re");
 206         if (!f)
 207                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 208
 209         r = catalog_file_lang(path, &deflang);
 210         if (r < 0)
 211                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 212         if (r == 1)
 213                 log_debug("File %s has language %s.", path, deflang);
 214
 215         for (;;) {
 216                 char line[LINE_MAX];
 217                 size_t a, b, c;
 218                 char *t;
 219
 220                 if (!fgets(line, sizeof(line), f)) {
 221                         if (feof(f))
 222                                 break;
 223
 224                         return log_error_errno(errno, "Failed to read file %s: %m", path);
 225                 }
 226
 227                 n++;
 228
 229                 truncate_nl(line);
 230
 231                 if (line[0] == 0) {
 232                         empty_line = true;
 233                         continue;
 234                 }
 235
 236                 if (strchr(COMMENTS "\n", line[0]))
 237                         continue;
 238
 239                 if (empty_line &&
 240                     strlen(line) >= 2+1+32 &&
 241                     line[0] == '-' &&
 242                     line[1] == '-' &&
 243                     line[2] == ' ' &&
 244                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 245
 246                         bool with_language;
 247                         sd_id128_t jd;
 248
 249                         /* New entry */
 250
 251                         with_language = line[2+1+32] != '\0';
 252                         line[2+1+32] = '\0';
 253
 254                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 255
 256                                 if (got_id) {
 257                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 258                                         if (r < 0)
 259                                                 return r;
 260
 261                                         lang = mfree(lang);
 262                                 }
 263
 264                                 if (with_language) {
 265                                         t = strstrip(line + 2 + 1 + 32 + 1);
 266
 267                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 268                                         if (r < 0)
 269                                                 return r;
 270                                 }
 271
 272                                 got_id = true;
 273                                 empty_line = false;
 274                                 id = jd;
 275
 276                                 if (payload)
 277                                         payload[0] = '\0';
 278
 279                                 continue;
 280                         }
 281                 }
 282
 283                 /* Payload */
 284                 if (!got_id) {
 285                         log_error("[%s:%u] Got payload before ID.", path, n);
 286                         return -EINVAL;
 287                 }
 288
 289                 a = payload ? strlen(payload) : 0;
 290                 b = strlen(line);
 291
 292                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 293                 t = realloc(payload, c);
 294                 if (!t)
 295                         return log_oom();
 296
 297                 if (empty_line) {
 298                         t[a] = '\n';
 299                         memcpy(t + a + 1, line, b);
 300                         t[a+b+1] = '\n';
 301                         t[a+b+2] = 0;
 302                 } else {
 303                         memcpy(t + a, line, b);
 304                         t[a+b] = '\n';
 305                         t[a+b+1] = 0;
 306                 }
 307
 308                 payload = t;
 309                 empty_line = false;
 310         }
 311
 312         if (got_id) {
 313                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 314                 if (r < 0)
 315                         return r;
 316         }
 317
 318         return 0;
 319 }
 320
 321 static int64_t write_catalog(const char *database, struct strbuf *sb,
 322                              CatalogItem *items, size_t n) {
 323         CatalogHeader header;
 324         _cleanup_fclose_ FILE *w = NULL;
 325         int r;
 326         _cleanup_free_ char *d, *p = NULL;
 327         size_t k;
 328
 329         d = dirname_malloc(database);
 330         if (!d)
 331                 return log_oom();
 332
 333         r = mkdir_p(d, 0775);
 334         if (r < 0)
 335                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 336
 337         r = fopen_temporary(database, &w, &p);
 338         if (r < 0)
 339                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 340                                        database);
 341
 342         zero(header);
 343         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 344         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 345         header.catalog_item_size = htole64(sizeof(CatalogItem));
 346         header.n_items = htole64(n);
 347
 348         r = -EIO;
 349
 350         k = fwrite(&header, 1, sizeof(header), w);
 351         if (k != sizeof(header)) {
 352                 log_error("%s: failed to write header.", p);
 353                 goto error;
 354         }
 355
 356         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 357         if (k != n * sizeof(CatalogItem)) {
 358                 log_error("%s: failed to write database.", p);
 359                 goto error;
 360         }
 361
 362         k = fwrite(sb->buf, 1, sb->len, w);
 363         if (k != sb->len) {
 364                 log_error("%s: failed to write strings.", p);
 365                 goto error;
 366         }
 367
 368         r = fflush_and_check(w);
 369         if (r < 0) {
 370                 log_error_errno(r, "%s: failed to write database: %m", p);
 371                 goto error;
 372         }
 373
 374         fchmod(fileno(w), 0644);
 375
 376         if (rename(p, database) < 0) {
 377                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 378                 goto error;
 379         }
 380
 381         return ftello(w);
 382
 383 error:
 384         (void) unlink(p);
 385         return r;
 386 }
 387
 388 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 389         _cleanup_strv_free_ char **files = NULL;
 390         char **f;
 391         struct strbuf *sb = NULL;
 392         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 393         _cleanup_free_ CatalogItem *items = NULL;
 394         CatalogItem *i;
 395         Iterator j;
 396         unsigned n;
 397         int r;
 398         int64_t sz;
 399
 400         h = hashmap_new(&catalog_hash_ops);
 401         sb = strbuf_new();
 402
 403         if (!h || !sb) {
 404                 r = log_oom();
 405                 goto finish;
 406         }
 407
 408         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 409         if (r < 0) {
 410                 log_error_errno(r, "Failed to get catalog files: %m");
 411                 goto finish;
 412         }
 413
 414         STRV_FOREACH(f, files) {
 415                 log_debug("Reading file '%s'", *f);
 416                 r = catalog_import_file(h, sb, *f);
 417                 if (r < 0) {
 418                         log_error_errno(r, "Failed to import file '%s': %m", *f);
 419                         goto finish;
 420                 }
 421         }
 422
 423         if (hashmap_size(h) <= 0) {
 424                 log_info("No items in catalog.");
 425                 goto finish;
 426         } else
 427                 log_debug("Found %u items in catalog.", hashmap_size(h));
 428
 429         strbuf_complete(sb);
 430
 431         items = new(CatalogItem, hashmap_size(h));
 432         if (!items) {
 433                 r = log_oom();
 434                 goto finish;
 435         }
 436
 437         n = 0;
 438         HASHMAP_FOREACH(i, h, j) {
 439                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 440                           SD_ID128_FORMAT_VAL(i->id),
 441                           isempty(i->language) ? "C" : i->language);
 442                 items[n++] = *i;
 443         }
 444
 445         assert(n == hashmap_size(h));
 446         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 447
 448         sz = write_catalog(database, sb, items, n);
 449         if (sz < 0)
 450                 r = log_error_errno(sz, "Failed to write %s: %m", database);
 451         else {
 452                 r = 0;
 453                 log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 454                           database, n, sb->len, sz);
 455         }
 456
 457 finish:
 458         strbuf_cleanup(sb);
 459
 460         return r;
 461 }
 462
 463 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 464         const CatalogHeader *h;
 465         int fd;
 466         void *p;
 467         struct stat st;
 468
 469         assert(_fd);
 470         assert(_st);
 471         assert(_p);
 472
 473         fd = open(database, O_RDONLY|O_CLOEXEC);
 474         if (fd < 0)
 475                 return -errno;
 476
 477         if (fstat(fd, &st) < 0) {
 478                 safe_close(fd);
 479                 return -errno;
 480         }
 481
 482         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 483                 safe_close(fd);
 484                 return -EINVAL;
 485         }
 486
 487         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 488         if (p == MAP_FAILED) {
 489                 safe_close(fd);
 490                 return -errno;
 491         }
 492
 493         h = p;
 494         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 495             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 496             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 497             h->incompatible_flags != 0 ||
 498             le64toh(h->n_items) <= 0 ||
 499             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 500                 safe_close(fd);
 501                 munmap(p, st.st_size);
 502                 return -EBADMSG;
 503         }
 504
 505         *_fd = fd;
 506         *_st = st;
 507         *_p = p;
 508
 509         return 0;
 510 }
 511
 512 static const char *find_id(void *p, sd_id128_t id) {
 513         CatalogItem key, *f = NULL;
 514         const CatalogHeader *h = p;
 515         const char *loc;
 516
 517         zero(key);
 518         key.id = id;
 519
 520         loc = setlocale(LC_MESSAGES, NULL);
 521         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 522                 strncpy(key.language, loc, sizeof(key.language));
 523                 key.language[strcspn(key.language, ".@")] = 0;
 524
 525                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 526                 if (!f) {
 527                         char *e;
 528
 529                         e = strchr(key.language, '_');
 530                         if (e) {
 531                                 *e = 0;
 532                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 533                         }
 534                 }
 535         }
 536
 537         if (!f) {
 538                 zero(key.language);
 539                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 540         }
 541
 542         if (!f)
 543                 return NULL;
 544
 545         return (const char*) p +
 546                 le64toh(h->header_size) +
 547                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 548                 le64toh(f->offset);
 549 }
 550
 551 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 552         _cleanup_close_ int fd = -1;
 553         void *p = NULL;
 554         struct stat st = {};
 555         char *text = NULL;
 556         int r;
 557         const char *s;
 558
 559         assert(_text);
 560
 561         r = open_mmap(database, &fd, &st, &p);
 562         if (r < 0)
 563                 return r;
 564
 565         s = find_id(p, id);
 566         if (!s) {
 567                 r = -ENOENT;
 568                 goto finish;
 569         }
 570
 571         text = strdup(s);
 572         if (!text) {
 573                 r = -ENOMEM;
 574                 goto finish;
 575         }
 576
 577         *_text = text;
 578         r = 0;
 579
 580 finish:
 581         if (p)
 582                 munmap(p, st.st_size);
 583
 584         return r;
 585 }
 586
 587 static char *find_header(const char *s, const char *header) {
 588
 589         for (;;) {
 590                 const char *v, *e;
 591
 592                 v = startswith(s, header);
 593                 if (v) {
 594                         v += strspn(v, WHITESPACE);
 595                         return strndup(v, strcspn(v, NEWLINE));
 596                 }
 597
 598                 /* End of text */
 599                 e = strchr(s, '\n');
 600                 if (!e)
 601                         return NULL;
 602
 603                 /* End of header */
 604                 if (e == s)
 605                         return NULL;
 606
 607                 s = e + 1;
 608         }
 609 }
 610
 611 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 612         if (oneline) {
 613                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 614
 615                 subject = find_header(s, "Subject:");
 616                 defined_by = find_header(s, "Defined-By:");
 617
 618                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 619                         SD_ID128_FORMAT_VAL(id),
 620                         strna(defined_by), strna(subject));
 621         } else
 622                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 623                         SD_ID128_FORMAT_VAL(id), s);
 624 }
 625
 626
 627 int catalog_list(FILE *f, const char *database, bool oneline) {
 628         _cleanup_close_ int fd = -1;
 629         void *p = NULL;
 630         struct stat st;
 631         const CatalogHeader *h;
 632         const CatalogItem *items;
 633         int r;
 634         unsigned n;
 635         sd_id128_t last_id;
 636         bool last_id_set = false;
 637
 638         r = open_mmap(database, &fd, &st, &p);
 639         if (r < 0)
 640                 return r;
 641
 642         h = p;
 643         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 644
 645         for (n = 0; n < le64toh(h->n_items); n++) {
 646                 const char *s;
 647
 648                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 649                         continue;
 650
 651                 assert_se(s = find_id(p, items[n].id));
 652
 653                 dump_catalog_entry(f, items[n].id, s, oneline);
 654
 655                 last_id_set = true;
 656                 last_id = items[n].id;
 657         }
 658
 659         munmap(p, st.st_size);
 660
 661         return 0;
 662 }
 663
 664 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 665         char **item;
 666         int r = 0;
 667
 668         STRV_FOREACH(item, items) {
 669                 sd_id128_t id;
 670                 int k;
 671                 _cleanup_free_ char *msg = NULL;
 672
 673                 k = sd_id128_from_string(*item, &id);
 674                 if (k < 0) {
 675                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 676                         if (r == 0)
 677                                 r = k;
 678                         continue;
 679                 }
 680
 681                 k = catalog_get(database, id, &msg);
 682                 if (k < 0) {
 683                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 684                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 685                         if (r == 0)
 686                                 r = k;
 687                         continue;
 688                 }
 689
 690                 dump_catalog_entry(f, id, msg, oneline);
 691         }
 692
 693         return r;
 694 }