src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <fcntl.h>
  23 #include <stdio.h>
  24 #include <unistd.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <locale.h>
  29
  30 #include "util.h"
  31 #include "log.h"
  32 #include "sparse-endian.h"
  33 #include "sd-id128.h"
  34 #include "hashmap.h"
  35 #include "strv.h"
  36 #include "strbuf.h"
  37 #include "conf-files.h"
  38 #include "mkdir.h"
  39 #include "catalog.h"
  40 #include "siphash24.h"
  41
  42 const char * const catalog_file_dirs[] = {
  43         "/usr/local/lib/systemd/catalog/",
  44         "/usr/lib/systemd/catalog/",
  45         NULL
  46 };
  47
  48 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  49
  50 typedef struct CatalogHeader {
  51         uint8_t signature[8];  /* "RHHHKSLP" */
  52         le32_t compatible_flags;
  53         le32_t incompatible_flags;
  54         le64_t header_size;
  55         le64_t n_items;
  56         le64_t catalog_item_size;
  57 } CatalogHeader;
  58
  59 typedef struct CatalogItem {
  60         sd_id128_t id;
  61         char language[32];
  62         le64_t offset;
  63 } CatalogItem;
  64
  65 static void catalog_hash_func(const void *p, struct siphash *state) {
  66         const CatalogItem *i = p;
  67
  68         siphash24_compress(&i->id, sizeof(i->id), state);
  69         siphash24_compress(i->language, strlen(i->language), state);
  70 }
  71
  72 static int catalog_compare_func(const void *a, const void *b) {
  73         const CatalogItem *i = a, *j = b;
  74         unsigned k;
  75
  76         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  77                  if (i->id.bytes[k] < j->id.bytes[k])
  78                         return -1;
  79                  if (i->id.bytes[k] > j->id.bytes[k])
  80                         return 1;
  81         }
  82
  83         return strcmp(i->language, j->language);
  84 }
  85
  86 const struct hash_ops catalog_hash_ops = {
  87         .hash = catalog_hash_func,
  88         .compare = catalog_compare_func
  89 };
  90
  91 static int finish_item(
  92                 Hashmap *h,
  93                 struct strbuf *sb,
  94                 sd_id128_t id,
  95                 const char *language,
  96                 const char *payload) {
  97
  98         ssize_t offset;
  99         _cleanup_free_ CatalogItem *i = NULL;
 100         int r;
 101
 102         assert(h);
 103         assert(sb);
 104         assert(payload);
 105
 106         offset = strbuf_add_string(sb, payload, strlen(payload));
 107         if (offset < 0)
 108                 return log_oom();
 109
 110         i = new0(CatalogItem, 1);
 111         if (!i)
 112                 return log_oom();
 113
 114         i->id = id;
 115         if (language) {
 116                 assert(strlen(language) > 1 && strlen(language) < 32);
 117                 strcpy(i->language, language);
 118         }
 119         i->offset = htole64((uint64_t) offset);
 120
 121         r = hashmap_put(h, i, i);
 122         if (r == -EEXIST) {
 123                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 124                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 125                 return 0;
 126         } else if (r < 0)
 127                 return r;
 128
 129         i = NULL;
 130         return 0;
 131 }
 132
 133 int catalog_file_lang(const char* filename, char **lang) {
 134         char *beg, *end, *_lang;
 135
 136         end = endswith(filename, ".catalog");
 137         if (!end)
 138                 return 0;
 139
 140         beg = end - 1;
 141         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 142                 beg --;
 143
 144         if (*beg != '.' || end <= beg + 1)
 145                 return 0;
 146
 147         _lang = strndup(beg + 1, end - beg - 1);
 148         if (!_lang)
 149                 return -ENOMEM;
 150
 151         *lang = _lang;
 152         return 1;
 153 }
 154
 155 static int catalog_entry_lang(const char* filename, int line,
 156                               const char* t, const char* deflang, char **lang) {
 157         size_t c;
 158
 159         c = strlen(t);
 160         if (c == 0) {
 161                 log_error("[%s:%u] Language too short.", filename, line);
 162                 return -EINVAL;
 163         }
 164         if (c > 31) {
 165                 log_error("[%s:%u] language too long.", filename, line);
 166                 return -EINVAL;
 167         }
 168
 169         if (deflang) {
 170                 if (streq(t, deflang)) {
 171                         log_warning("[%s:%u] language specified unnecessarily",
 172                                     filename, line);
 173                         return 0;
 174                 } else
 175                         log_warning("[%s:%u] language differs from default for file",
 176                                     filename, line);
 177         }
 178
 179         *lang = strdup(t);
 180         if (!*lang)
 181                         return -ENOMEM;
 182
 183         return 0;
 184 }
 185
 186 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 187         _cleanup_fclose_ FILE *f = NULL;
 188         _cleanup_free_ char *payload = NULL;
 189         unsigned n = 0;
 190         sd_id128_t id;
 191         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 192         bool got_id = false, empty_line = true;
 193         int r;
 194
 195         assert(h);
 196         assert(sb);
 197         assert(path);
 198
 199         f = fopen(path, "re");
 200         if (!f)
 201                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 202
 203         r = catalog_file_lang(path, &deflang);
 204         if (r < 0)
 205                 log_error_errno(errno, "Failed to determine language for file %s: %m", path);
 206         if (r == 1)
 207                 log_debug("File %s has language %s.", path, deflang);
 208
 209         for (;;) {
 210                 char line[LINE_MAX];
 211                 size_t a, b, c;
 212                 char *t;
 213
 214                 if (!fgets(line, sizeof(line), f)) {
 215                         if (feof(f))
 216                                 break;
 217
 218                         log_error_errno(errno, "Failed to read file %s: %m", path);
 219                         return -errno;
 220                 }
 221
 222                 n++;
 223
 224                 truncate_nl(line);
 225
 226                 if (line[0] == 0) {
 227                         empty_line = true;
 228                         continue;
 229                 }
 230
 231                 if (strchr(COMMENTS "\n", line[0]))
 232                         continue;
 233
 234                 if (empty_line &&
 235                     strlen(line) >= 2+1+32 &&
 236                     line[0] == '-' &&
 237                     line[1] == '-' &&
 238                     line[2] == ' ' &&
 239                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 240
 241                         bool with_language;
 242                         sd_id128_t jd;
 243
 244                         /* New entry */
 245
 246                         with_language = line[2+1+32] != '\0';
 247                         line[2+1+32] = '\0';
 248
 249                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 250
 251                                 if (got_id) {
 252                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 253                                         if (r < 0)
 254                                                 return r;
 255
 256                                         lang = mfree(lang);
 257                                 }
 258
 259                                 if (with_language) {
 260                                         t = strstrip(line + 2 + 1 + 32 + 1);
 261
 262                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 263                                         if (r < 0)
 264                                                 return r;
 265                                 }
 266
 267                                 got_id = true;
 268                                 empty_line = false;
 269                                 id = jd;
 270
 271                                 if (payload)
 272                                         payload[0] = '\0';
 273
 274                                 continue;
 275                         }
 276                 }
 277
 278                 /* Payload */
 279                 if (!got_id) {
 280                         log_error("[%s:%u] Got payload before ID.", path, n);
 281                         return -EINVAL;
 282                 }
 283
 284                 a = payload ? strlen(payload) : 0;
 285                 b = strlen(line);
 286
 287                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 288                 t = realloc(payload, c);
 289                 if (!t)
 290                         return log_oom();
 291
 292                 if (empty_line) {
 293                         t[a] = '\n';
 294                         memcpy(t + a + 1, line, b);
 295                         t[a+b+1] = '\n';
 296                         t[a+b+2] = 0;
 297                 } else {
 298                         memcpy(t + a, line, b);
 299                         t[a+b] = '\n';
 300                         t[a+b+1] = 0;
 301                 }
 302
 303                 payload = t;
 304                 empty_line = false;
 305         }
 306
 307         if (got_id) {
 308                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 309                 if (r < 0)
 310                         return r;
 311         }
 312
 313         return 0;
 314 }
 315
 316 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 317                           CatalogItem *items, size_t n) {
 318         CatalogHeader header;
 319         _cleanup_fclose_ FILE *w = NULL;
 320         int r;
 321         _cleanup_free_ char *d, *p = NULL;
 322         size_t k;
 323
 324         d = dirname_malloc(database);
 325         if (!d)
 326                 return log_oom();
 327
 328         r = mkdir_p(d, 0775);
 329         if (r < 0)
 330                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 331
 332         r = fopen_temporary(database, &w, &p);
 333         if (r < 0)
 334                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 335                                        database);
 336
 337         zero(header);
 338         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 339         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 340         header.catalog_item_size = htole64(sizeof(CatalogItem));
 341         header.n_items = htole64(hashmap_size(h));
 342
 343         r = -EIO;
 344
 345         k = fwrite(&header, 1, sizeof(header), w);
 346         if (k != sizeof(header)) {
 347                 log_error("%s: failed to write header.", p);
 348                 goto error;
 349         }
 350
 351         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 352         if (k != n * sizeof(CatalogItem)) {
 353                 log_error("%s: failed to write database.", p);
 354                 goto error;
 355         }
 356
 357         k = fwrite(sb->buf, 1, sb->len, w);
 358         if (k != sb->len) {
 359                 log_error("%s: failed to write strings.", p);
 360                 goto error;
 361         }
 362
 363         r = fflush_and_check(w);
 364         if (r < 0) {
 365                 log_error_errno(r, "%s: failed to write database: %m", p);
 366                 goto error;
 367         }
 368
 369         fchmod(fileno(w), 0644);
 370
 371         if (rename(p, database) < 0) {
 372                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 373                 goto error;
 374         }
 375
 376         return ftell(w);
 377
 378 error:
 379         (void) unlink(p);
 380         return r;
 381 }
 382
 383 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 384         _cleanup_strv_free_ char **files = NULL;
 385         char **f;
 386         struct strbuf *sb = NULL;
 387         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 388         _cleanup_free_ CatalogItem *items = NULL;
 389         CatalogItem *i;
 390         Iterator j;
 391         unsigned n;
 392         long r;
 393
 394         h = hashmap_new(&catalog_hash_ops);
 395         sb = strbuf_new();
 396
 397         if (!h || !sb) {
 398                 r = log_oom();
 399                 goto finish;
 400         }
 401
 402         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 403         if (r < 0) {
 404                 log_error_errno(r, "Failed to get catalog files: %m");
 405                 goto finish;
 406         }
 407
 408         STRV_FOREACH(f, files) {
 409                 log_debug("Reading file '%s'", *f);
 410                 r = catalog_import_file(h, sb, *f);
 411                 if (r < 0) {
 412                         log_error_errno(r, "Failed to import file '%s': %m", *f);
 413                         goto finish;
 414                 }
 415         }
 416
 417         if (hashmap_size(h) <= 0) {
 418                 log_info("No items in catalog.");
 419                 goto finish;
 420         } else
 421                 log_debug("Found %u items in catalog.", hashmap_size(h));
 422
 423         strbuf_complete(sb);
 424
 425         items = new(CatalogItem, hashmap_size(h));
 426         if (!items) {
 427                 r = log_oom();
 428                 goto finish;
 429         }
 430
 431         n = 0;
 432         HASHMAP_FOREACH(i, h, j) {
 433                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 434                           SD_ID128_FORMAT_VAL(i->id),
 435                           isempty(i->language) ? "C" : i->language);
 436                 items[n++] = *i;
 437         }
 438
 439         assert(n == hashmap_size(h));
 440         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 441
 442         r = write_catalog(database, h, sb, items, n);
 443         if (r < 0)
 444                 log_error_errno(r, "Failed to write %s: %m", database);
 445         else
 446                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 447                           database, n, sb->len, r);
 448
 449 finish:
 450         if (sb)
 451                 strbuf_cleanup(sb);
 452
 453         return r < 0 ? r : 0;
 454 }
 455
 456 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 457         const CatalogHeader *h;
 458         int fd;
 459         void *p;
 460         struct stat st;
 461
 462         assert(_fd);
 463         assert(_st);
 464         assert(_p);
 465
 466         fd = open(database, O_RDONLY|O_CLOEXEC);
 467         if (fd < 0)
 468                 return -errno;
 469
 470         if (fstat(fd, &st) < 0) {
 471                 safe_close(fd);
 472                 return -errno;
 473         }
 474
 475         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 476                 safe_close(fd);
 477                 return -EINVAL;
 478         }
 479
 480         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 481         if (p == MAP_FAILED) {
 482                 safe_close(fd);
 483                 return -errno;
 484         }
 485
 486         h = p;
 487         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 488             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 489             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 490             h->incompatible_flags != 0 ||
 491             le64toh(h->n_items) <= 0 ||
 492             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 493                 safe_close(fd);
 494                 munmap(p, st.st_size);
 495                 return -EBADMSG;
 496         }
 497
 498         *_fd = fd;
 499         *_st = st;
 500         *_p = p;
 501
 502         return 0;
 503 }
 504
 505 static const char *find_id(void *p, sd_id128_t id) {
 506         CatalogItem key, *f = NULL;
 507         const CatalogHeader *h = p;
 508         const char *loc;
 509
 510         zero(key);
 511         key.id = id;
 512
 513         loc = setlocale(LC_MESSAGES, NULL);
 514         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 515                 strncpy(key.language, loc, sizeof(key.language));
 516                 key.language[strcspn(key.language, ".@")] = 0;
 517
 518                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 519                 if (!f) {
 520                         char *e;
 521
 522                         e = strchr(key.language, '_');
 523                         if (e) {
 524                                 *e = 0;
 525                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 526                         }
 527                 }
 528         }
 529
 530         if (!f) {
 531                 zero(key.language);
 532                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 533         }
 534
 535         if (!f)
 536                 return NULL;
 537
 538         return (const char*) p +
 539                 le64toh(h->header_size) +
 540                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 541                 le64toh(f->offset);
 542 }
 543
 544 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 545         _cleanup_close_ int fd = -1;
 546         void *p = NULL;
 547         struct stat st = {};
 548         char *text = NULL;
 549         int r;
 550         const char *s;
 551
 552         assert(_text);
 553
 554         r = open_mmap(database, &fd, &st, &p);
 555         if (r < 0)
 556                 return r;
 557
 558         s = find_id(p, id);
 559         if (!s) {
 560                 r = -ENOENT;
 561                 goto finish;
 562         }
 563
 564         text = strdup(s);
 565         if (!text) {
 566                 r = -ENOMEM;
 567                 goto finish;
 568         }
 569
 570         *_text = text;
 571         r = 0;
 572
 573 finish:
 574         if (p)
 575                 munmap(p, st.st_size);
 576
 577         return r;
 578 }
 579
 580 static char *find_header(const char *s, const char *header) {
 581
 582         for (;;) {
 583                 const char *v, *e;
 584
 585                 v = startswith(s, header);
 586                 if (v) {
 587                         v += strspn(v, WHITESPACE);
 588                         return strndup(v, strcspn(v, NEWLINE));
 589                 }
 590
 591                 /* End of text */
 592                 e = strchr(s, '\n');
 593                 if (!e)
 594                         return NULL;
 595
 596                 /* End of header */
 597                 if (e == s)
 598                         return NULL;
 599
 600                 s = e + 1;
 601         }
 602 }
 603
 604 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 605         if (oneline) {
 606                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 607
 608                 subject = find_header(s, "Subject:");
 609                 defined_by = find_header(s, "Defined-By:");
 610
 611                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 612                         SD_ID128_FORMAT_VAL(id),
 613                         strna(defined_by), strna(subject));
 614         } else
 615                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 616                         SD_ID128_FORMAT_VAL(id), s);
 617 }
 618
 619
 620 int catalog_list(FILE *f, const char *database, bool oneline) {
 621         _cleanup_close_ int fd = -1;
 622         void *p = NULL;
 623         struct stat st;
 624         const CatalogHeader *h;
 625         const CatalogItem *items;
 626         int r;
 627         unsigned n;
 628         sd_id128_t last_id;
 629         bool last_id_set = false;
 630
 631         r = open_mmap(database, &fd, &st, &p);
 632         if (r < 0)
 633                 return r;
 634
 635         h = p;
 636         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 637
 638         for (n = 0; n < le64toh(h->n_items); n++) {
 639                 const char *s;
 640
 641                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 642                         continue;
 643
 644                 assert_se(s = find_id(p, items[n].id));
 645
 646                 dump_catalog_entry(f, items[n].id, s, oneline);
 647
 648                 last_id_set = true;
 649                 last_id = items[n].id;
 650         }
 651
 652         munmap(p, st.st_size);
 653
 654         return 0;
 655 }
 656
 657 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 658         char **item;
 659         int r = 0;
 660
 661         STRV_FOREACH(item, items) {
 662                 sd_id128_t id;
 663                 int k;
 664                 _cleanup_free_ char *msg = NULL;
 665
 666                 k = sd_id128_from_string(*item, &id);
 667                 if (k < 0) {
 668                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 669                         if (r == 0)
 670                                 r = k;
 671                         continue;
 672                 }
 673
 674                 k = catalog_get(database, id, &msg);
 675                 if (k < 0) {
 676                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 677                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 678                         if (r == 0)
 679                                 r = k;
 680                         continue;
 681                 }
 682
 683                 dump_catalog_entry(f, id, msg, oneline);
 684         }
 685
 686         return r;
 687 }