src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <locale.h>
  25 #include <stdio.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <unistd.h>
  29
  30 #include "sd-id128.h"
  31
  32 #include "alloc-util.h"
  33 #include "catalog.h"
  34 #include "conf-files.h"
  35 #include "fd-util.h"
  36 #include "fileio.h"
  37 #include "hashmap.h"
  38 #include "log.h"
  39 #include "mkdir.h"
  40 #include "path-util.h"
  41 #include "siphash24.h"
  42 #include "sparse-endian.h"
  43 #include "strbuf.h"
  44 #include "string-util.h"
  45 #include "strv.h"
  46 #include "util.h"
  47
  48 const char * const catalog_file_dirs[] = {
  49         "/usr/local/lib/systemd/catalog/",
  50         "/usr/lib/systemd/catalog/",
  51         NULL
  52 };
  53
  54 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  55
  56 typedef struct CatalogHeader {
  57         uint8_t signature[8];  /* "RHHHKSLP" */
  58         le32_t compatible_flags;
  59         le32_t incompatible_flags;
  60         le64_t header_size;
  61         le64_t n_items;
  62         le64_t catalog_item_size;
  63 } CatalogHeader;
  64
  65 typedef struct CatalogItem {
  66         sd_id128_t id;
  67         char language[32];
  68         le64_t offset;
  69 } CatalogItem;
  70
  71 static void catalog_hash_func(const void *p, struct siphash *state) {
  72         const CatalogItem *i = p;
  73
  74         siphash24_compress(&i->id, sizeof(i->id), state);
  75         siphash24_compress(i->language, strlen(i->language), state);
  76 }
  77
  78 static int catalog_compare_func(const void *a, const void *b) {
  79         const CatalogItem *i = a, *j = b;
  80         unsigned k;
  81
  82         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  83                  if (i->id.bytes[k] < j->id.bytes[k])
  84                         return -1;
  85                  if (i->id.bytes[k] > j->id.bytes[k])
  86                         return 1;
  87         }
  88
  89         return strcmp(i->language, j->language);
  90 }
  91
  92 const struct hash_ops catalog_hash_ops = {
  93         .hash = catalog_hash_func,
  94         .compare = catalog_compare_func
  95 };
  96
  97 static int finish_item(
  98                 Hashmap *h,
  99                 struct strbuf *sb,
 100                 sd_id128_t id,
 101                 const char *language,
 102                 const char *payload) {
 103
 104         ssize_t offset;
 105         _cleanup_free_ CatalogItem *i = NULL;
 106         int r;
 107
 108         assert(h);
 109         assert(sb);
 110         assert(payload);
 111
 112         offset = strbuf_add_string(sb, payload, strlen(payload));
 113         if (offset < 0)
 114                 return log_oom();
 115
 116         i = new0(CatalogItem, 1);
 117         if (!i)
 118                 return log_oom();
 119
 120         i->id = id;
 121         if (language) {
 122                 assert(strlen(language) > 1 && strlen(language) < 32);
 123                 strcpy(i->language, language);
 124         }
 125         i->offset = htole64((uint64_t) offset);
 126
 127         r = hashmap_put(h, i, i);
 128         if (r == -EEXIST) {
 129                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 130                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 131                 return 0;
 132         } else if (r < 0)
 133                 return r;
 134
 135         i = NULL;
 136         return 0;
 137 }
 138
 139 int catalog_file_lang(const char* filename, char **lang) {
 140         char *beg, *end, *_lang;
 141
 142         end = endswith(filename, ".catalog");
 143         if (!end)
 144                 return 0;
 145
 146         beg = end - 1;
 147         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 148                 beg --;
 149
 150         if (*beg != '.' || end <= beg + 1)
 151                 return 0;
 152
 153         _lang = strndup(beg + 1, end - beg - 1);
 154         if (!_lang)
 155                 return -ENOMEM;
 156
 157         *lang = _lang;
 158         return 1;
 159 }
 160
 161 static int catalog_entry_lang(const char* filename, int line,
 162                               const char* t, const char* deflang, char **lang) {
 163         size_t c;
 164
 165         c = strlen(t);
 166         if (c == 0) {
 167                 log_error("[%s:%u] Language too short.", filename, line);
 168                 return -EINVAL;
 169         }
 170         if (c > 31) {
 171                 log_error("[%s:%u] language too long.", filename, line);
 172                 return -EINVAL;
 173         }
 174
 175         if (deflang) {
 176                 if (streq(t, deflang)) {
 177                         log_warning("[%s:%u] language specified unnecessarily",
 178                                     filename, line);
 179                         return 0;
 180                 } else
 181                         log_warning("[%s:%u] language differs from default for file",
 182                                     filename, line);
 183         }
 184
 185         *lang = strdup(t);
 186         if (!*lang)
 187                         return -ENOMEM;
 188
 189         return 0;
 190 }
 191
 192 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 193         _cleanup_fclose_ FILE *f = NULL;
 194         _cleanup_free_ char *payload = NULL;
 195         unsigned n = 0;
 196         sd_id128_t id;
 197         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 198         bool got_id = false, empty_line = true;
 199         int r;
 200
 201         assert(h);
 202         assert(sb);
 203         assert(path);
 204
 205         f = fopen(path, "re");
 206         if (!f)
 207                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 208
 209         r = catalog_file_lang(path, &deflang);
 210         if (r < 0)
 211                 log_error_errno(errno, "Failed to determine language for file %s: %m", path);
 212         if (r == 1)
 213                 log_debug("File %s has language %s.", path, deflang);
 214
 215         for (;;) {
 216                 char line[LINE_MAX];
 217                 size_t a, b, c;
 218                 char *t;
 219
 220                 if (!fgets(line, sizeof(line), f)) {
 221                         if (feof(f))
 222                                 break;
 223
 224                         log_error_errno(errno, "Failed to read file %s: %m", path);
 225                         return -errno;
 226                 }
 227
 228                 n++;
 229
 230                 truncate_nl(line);
 231
 232                 if (line[0] == 0) {
 233                         empty_line = true;
 234                         continue;
 235                 }
 236
 237                 if (strchr(COMMENTS "\n", line[0]))
 238                         continue;
 239
 240                 if (empty_line &&
 241                     strlen(line) >= 2+1+32 &&
 242                     line[0] == '-' &&
 243                     line[1] == '-' &&
 244                     line[2] == ' ' &&
 245                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 246
 247                         bool with_language;
 248                         sd_id128_t jd;
 249
 250                         /* New entry */
 251
 252                         with_language = line[2+1+32] != '\0';
 253                         line[2+1+32] = '\0';
 254
 255                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 256
 257                                 if (got_id) {
 258                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 259                                         if (r < 0)
 260                                                 return r;
 261
 262                                         lang = mfree(lang);
 263                                 }
 264
 265                                 if (with_language) {
 266                                         t = strstrip(line + 2 + 1 + 32 + 1);
 267
 268                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 269                                         if (r < 0)
 270                                                 return r;
 271                                 }
 272
 273                                 got_id = true;
 274                                 empty_line = false;
 275                                 id = jd;
 276
 277                                 if (payload)
 278                                         payload[0] = '\0';
 279
 280                                 continue;
 281                         }
 282                 }
 283
 284                 /* Payload */
 285                 if (!got_id) {
 286                         log_error("[%s:%u] Got payload before ID.", path, n);
 287                         return -EINVAL;
 288                 }
 289
 290                 a = payload ? strlen(payload) : 0;
 291                 b = strlen(line);
 292
 293                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 294                 t = realloc(payload, c);
 295                 if (!t)
 296                         return log_oom();
 297
 298                 if (empty_line) {
 299                         t[a] = '\n';
 300                         memcpy(t + a + 1, line, b);
 301                         t[a+b+1] = '\n';
 302                         t[a+b+2] = 0;
 303                 } else {
 304                         memcpy(t + a, line, b);
 305                         t[a+b] = '\n';
 306                         t[a+b+1] = 0;
 307                 }
 308
 309                 payload = t;
 310                 empty_line = false;
 311         }
 312
 313         if (got_id) {
 314                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 315                 if (r < 0)
 316                         return r;
 317         }
 318
 319         return 0;
 320 }
 321
 322 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 323                           CatalogItem *items, size_t n) {
 324         CatalogHeader header;
 325         _cleanup_fclose_ FILE *w = NULL;
 326         int r;
 327         _cleanup_free_ char *d, *p = NULL;
 328         size_t k;
 329
 330         d = dirname_malloc(database);
 331         if (!d)
 332                 return log_oom();
 333
 334         r = mkdir_p(d, 0775);
 335         if (r < 0)
 336                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 337
 338         r = fopen_temporary(database, &w, &p);
 339         if (r < 0)
 340                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 341                                        database);
 342
 343         zero(header);
 344         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 345         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 346         header.catalog_item_size = htole64(sizeof(CatalogItem));
 347         header.n_items = htole64(hashmap_size(h));
 348
 349         r = -EIO;
 350
 351         k = fwrite(&header, 1, sizeof(header), w);
 352         if (k != sizeof(header)) {
 353                 log_error("%s: failed to write header.", p);
 354                 goto error;
 355         }
 356
 357         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 358         if (k != n * sizeof(CatalogItem)) {
 359                 log_error("%s: failed to write database.", p);
 360                 goto error;
 361         }
 362
 363         k = fwrite(sb->buf, 1, sb->len, w);
 364         if (k != sb->len) {
 365                 log_error("%s: failed to write strings.", p);
 366                 goto error;
 367         }
 368
 369         r = fflush_and_check(w);
 370         if (r < 0) {
 371                 log_error_errno(r, "%s: failed to write database: %m", p);
 372                 goto error;
 373         }
 374
 375         fchmod(fileno(w), 0644);
 376
 377         if (rename(p, database) < 0) {
 378                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 379                 goto error;
 380         }
 381
 382         return ftell(w);
 383
 384 error:
 385         (void) unlink(p);
 386         return r;
 387 }
 388
 389 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 390         _cleanup_strv_free_ char **files = NULL;
 391         char **f;
 392         struct strbuf *sb = NULL;
 393         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 394         _cleanup_free_ CatalogItem *items = NULL;
 395         CatalogItem *i;
 396         Iterator j;
 397         unsigned n;
 398         long r;
 399
 400         h = hashmap_new(&catalog_hash_ops);
 401         sb = strbuf_new();
 402
 403         if (!h || !sb) {
 404                 r = log_oom();
 405                 goto finish;
 406         }
 407
 408         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 409         if (r < 0) {
 410                 log_error_errno(r, "Failed to get catalog files: %m");
 411                 goto finish;
 412         }
 413
 414         STRV_FOREACH(f, files) {
 415                 log_debug("Reading file '%s'", *f);
 416                 r = catalog_import_file(h, sb, *f);
 417                 if (r < 0) {
 418                         log_error_errno(r, "Failed to import file '%s': %m", *f);
 419                         goto finish;
 420                 }
 421         }
 422
 423         if (hashmap_size(h) <= 0) {
 424                 log_info("No items in catalog.");
 425                 goto finish;
 426         } else
 427                 log_debug("Found %u items in catalog.", hashmap_size(h));
 428
 429         strbuf_complete(sb);
 430
 431         items = new(CatalogItem, hashmap_size(h));
 432         if (!items) {
 433                 r = log_oom();
 434                 goto finish;
 435         }
 436
 437         n = 0;
 438         HASHMAP_FOREACH(i, h, j) {
 439                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 440                           SD_ID128_FORMAT_VAL(i->id),
 441                           isempty(i->language) ? "C" : i->language);
 442                 items[n++] = *i;
 443         }
 444
 445         assert(n == hashmap_size(h));
 446         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 447
 448         r = write_catalog(database, h, sb, items, n);
 449         if (r < 0)
 450                 log_error_errno(r, "Failed to write %s: %m", database);
 451         else
 452                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 453                           database, n, sb->len, r);
 454
 455 finish:
 456         if (sb)
 457                 strbuf_cleanup(sb);
 458
 459         return r < 0 ? r : 0;
 460 }
 461
 462 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 463         const CatalogHeader *h;
 464         int fd;
 465         void *p;
 466         struct stat st;
 467
 468         assert(_fd);
 469         assert(_st);
 470         assert(_p);
 471
 472         fd = open(database, O_RDONLY|O_CLOEXEC);
 473         if (fd < 0)
 474                 return -errno;
 475
 476         if (fstat(fd, &st) < 0) {
 477                 safe_close(fd);
 478                 return -errno;
 479         }
 480
 481         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 482                 safe_close(fd);
 483                 return -EINVAL;
 484         }
 485
 486         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 487         if (p == MAP_FAILED) {
 488                 safe_close(fd);
 489                 return -errno;
 490         }
 491
 492         h = p;
 493         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 494             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 495             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 496             h->incompatible_flags != 0 ||
 497             le64toh(h->n_items) <= 0 ||
 498             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 499                 safe_close(fd);
 500                 munmap(p, st.st_size);
 501                 return -EBADMSG;
 502         }
 503
 504         *_fd = fd;
 505         *_st = st;
 506         *_p = p;
 507
 508         return 0;
 509 }
 510
 511 static const char *find_id(void *p, sd_id128_t id) {
 512         CatalogItem key, *f = NULL;
 513         const CatalogHeader *h = p;
 514         const char *loc;
 515
 516         zero(key);
 517         key.id = id;
 518
 519         loc = setlocale(LC_MESSAGES, NULL);
 520         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 521                 strncpy(key.language, loc, sizeof(key.language));
 522                 key.language[strcspn(key.language, ".@")] = 0;
 523
 524                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 525                 if (!f) {
 526                         char *e;
 527
 528                         e = strchr(key.language, '_');
 529                         if (e) {
 530                                 *e = 0;
 531                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 532                         }
 533                 }
 534         }
 535
 536         if (!f) {
 537                 zero(key.language);
 538                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 539         }
 540
 541         if (!f)
 542                 return NULL;
 543
 544         return (const char*) p +
 545                 le64toh(h->header_size) +
 546                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 547                 le64toh(f->offset);
 548 }
 549
 550 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 551         _cleanup_close_ int fd = -1;
 552         void *p = NULL;
 553         struct stat st = {};
 554         char *text = NULL;
 555         int r;
 556         const char *s;
 557
 558         assert(_text);
 559
 560         r = open_mmap(database, &fd, &st, &p);
 561         if (r < 0)
 562                 return r;
 563
 564         s = find_id(p, id);
 565         if (!s) {
 566                 r = -ENOENT;
 567                 goto finish;
 568         }
 569
 570         text = strdup(s);
 571         if (!text) {
 572                 r = -ENOMEM;
 573                 goto finish;
 574         }
 575
 576         *_text = text;
 577         r = 0;
 578
 579 finish:
 580         if (p)
 581                 munmap(p, st.st_size);
 582
 583         return r;
 584 }
 585
 586 static char *find_header(const char *s, const char *header) {
 587
 588         for (;;) {
 589                 const char *v, *e;
 590
 591                 v = startswith(s, header);
 592                 if (v) {
 593                         v += strspn(v, WHITESPACE);
 594                         return strndup(v, strcspn(v, NEWLINE));
 595                 }
 596
 597                 /* End of text */
 598                 e = strchr(s, '\n');
 599                 if (!e)
 600                         return NULL;
 601
 602                 /* End of header */
 603                 if (e == s)
 604                         return NULL;
 605
 606                 s = e + 1;
 607         }
 608 }
 609
 610 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 611         if (oneline) {
 612                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 613
 614                 subject = find_header(s, "Subject:");
 615                 defined_by = find_header(s, "Defined-By:");
 616
 617                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 618                         SD_ID128_FORMAT_VAL(id),
 619                         strna(defined_by), strna(subject));
 620         } else
 621                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 622                         SD_ID128_FORMAT_VAL(id), s);
 623 }
 624
 625
 626 int catalog_list(FILE *f, const char *database, bool oneline) {
 627         _cleanup_close_ int fd = -1;
 628         void *p = NULL;
 629         struct stat st;
 630         const CatalogHeader *h;
 631         const CatalogItem *items;
 632         int r;
 633         unsigned n;
 634         sd_id128_t last_id;
 635         bool last_id_set = false;
 636
 637         r = open_mmap(database, &fd, &st, &p);
 638         if (r < 0)
 639                 return r;
 640
 641         h = p;
 642         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 643
 644         for (n = 0; n < le64toh(h->n_items); n++) {
 645                 const char *s;
 646
 647                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 648                         continue;
 649
 650                 assert_se(s = find_id(p, items[n].id));
 651
 652                 dump_catalog_entry(f, items[n].id, s, oneline);
 653
 654                 last_id_set = true;
 655                 last_id = items[n].id;
 656         }
 657
 658         munmap(p, st.st_size);
 659
 660         return 0;
 661 }
 662
 663 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 664         char **item;
 665         int r = 0;
 666
 667         STRV_FOREACH(item, items) {
 668                 sd_id128_t id;
 669                 int k;
 670                 _cleanup_free_ char *msg = NULL;
 671
 672                 k = sd_id128_from_string(*item, &id);
 673                 if (k < 0) {
 674                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 675                         if (r == 0)
 676                                 r = k;
 677                         continue;
 678                 }
 679
 680                 k = catalog_get(database, id, &msg);
 681                 if (k < 0) {
 682                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 683                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 684                         if (r == 0)
 685                                 r = k;
 686                         continue;
 687                 }
 688
 689                 dump_catalog_entry(f, id, msg, oneline);
 690         }
 691
 692         return r;
 693 }