src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <fcntl.h>
  23 #include <stdio.h>
  24 #include <unistd.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <locale.h>
  29
  30 #include "util.h"
  31 #include "log.h"
  32 #include "sparse-endian.h"
  33 #include "sd-id128.h"
  34 #include "hashmap.h"
  35 #include "strv.h"
  36 #include "strbuf.h"
  37 #include "conf-files.h"
  38 #include "mkdir.h"
  39 #include "catalog.h"
  40 #include "siphash24.h"
  41
  42 const char * const catalog_file_dirs[] = {
  43         "/usr/local/lib/systemd/catalog/",
  44         "/usr/lib/systemd/catalog/",
  45         NULL
  46 };
  47
  48 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  49
  50 typedef struct CatalogHeader {
  51         uint8_t signature[8];  /* "RHHHKSLP" */
  52         le32_t compatible_flags;
  53         le32_t incompatible_flags;
  54         le64_t header_size;
  55         le64_t n_items;
  56         le64_t catalog_item_size;
  57 } CatalogHeader;
  58
  59 typedef struct CatalogItem {
  60         sd_id128_t id;
  61         char language[32];
  62         le64_t offset;
  63 } CatalogItem;
  64
  65 static unsigned long catalog_hash_func(const void *p, const uint8_t hash_key[HASH_KEY_SIZE]) {
  66         const CatalogItem *i = p;
  67         uint64_t u;
  68         size_t l, sz;
  69         void *v;
  70
  71         l = strlen(i->language);
  72         sz = sizeof(i->id) + l;
  73         v = alloca(sz);
  74
  75         memcpy(mempcpy(v, &i->id, sizeof(i->id)), i->language, l);
  76
  77         siphash24((uint8_t*) &u, v, sz, hash_key);
  78
  79         return (unsigned long) u;
  80 }
  81
  82 static int catalog_compare_func(const void *a, const void *b) {
  83         const CatalogItem *i = a, *j = b;
  84         unsigned k;
  85
  86         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  87                  if (i->id.bytes[k] < j->id.bytes[k])
  88                         return -1;
  89                  if (i->id.bytes[k] > j->id.bytes[k])
  90                         return 1;
  91         }
  92
  93         return strcmp(i->language, j->language);
  94 }
  95
  96 const struct hash_ops catalog_hash_ops = {
  97         .hash = catalog_hash_func,
  98         .compare = catalog_compare_func
  99 };
 100
 101 static int finish_item(
 102                 Hashmap *h,
 103                 struct strbuf *sb,
 104                 sd_id128_t id,
 105                 const char *language,
 106                 const char *payload) {
 107
 108         ssize_t offset;
 109         _cleanup_free_ CatalogItem *i = NULL;
 110         int r;
 111
 112         assert(h);
 113         assert(sb);
 114         assert(payload);
 115
 116         offset = strbuf_add_string(sb, payload, strlen(payload));
 117         if (offset < 0)
 118                 return log_oom();
 119
 120         i = new0(CatalogItem, 1);
 121         if (!i)
 122                 return log_oom();
 123
 124         i->id = id;
 125         if (language) {
 126                 assert(strlen(language) > 1 && strlen(language) < 32);
 127                 strcpy(i->language, language);
 128         }
 129         i->offset = htole64((uint64_t) offset);
 130
 131         r = hashmap_put(h, i, i);
 132         if (r == -EEXIST) {
 133                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 134                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 135                 return 0;
 136         } else if (r < 0)
 137                 return r;
 138
 139         i = NULL;
 140         return 0;
 141 }
 142
 143 int catalog_file_lang(const char* filename, char **lang) {
 144         char *beg, *end, *_lang;
 145
 146         end = endswith(filename, ".catalog");
 147         if (!end)
 148                 return 0;
 149
 150         beg = end - 1;
 151         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 152                 beg --;
 153
 154         if (*beg != '.' || end <= beg + 1)
 155                 return 0;
 156
 157         _lang = strndup(beg + 1, end - beg - 1);
 158         if (!_lang)
 159                 return -ENOMEM;
 160
 161         *lang = _lang;
 162         return 1;
 163 }
 164
 165 static int catalog_entry_lang(const char* filename, int line,
 166                               const char* t, const char* deflang, char **lang) {
 167         size_t c;
 168
 169         c = strlen(t);
 170         if (c == 0) {
 171                 log_error("[%s:%u] Language too short.", filename, line);
 172                 return -EINVAL;
 173         }
 174         if (c > 31) {
 175                 log_error("[%s:%u] language too long.", filename, line);
 176                 return -EINVAL;
 177         }
 178
 179         if (deflang) {
 180                 if (streq(t, deflang)) {
 181                         log_warning("[%s:%u] language specified unnecessarily",
 182                                     filename, line);
 183                         return 0;
 184                 } else
 185                         log_warning("[%s:%u] language differs from default for file",
 186                                     filename, line);
 187         }
 188
 189         *lang = strdup(t);
 190         if (!*lang)
 191                         return -ENOMEM;
 192
 193         return 0;
 194 }
 195
 196 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 197         _cleanup_fclose_ FILE *f = NULL;
 198         _cleanup_free_ char *payload = NULL;
 199         unsigned n = 0;
 200         sd_id128_t id;
 201         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 202         bool got_id = false, empty_line = true;
 203         int r;
 204
 205         assert(h);
 206         assert(sb);
 207         assert(path);
 208
 209         f = fopen(path, "re");
 210         if (!f)
 211                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 212
 213         r = catalog_file_lang(path, &deflang);
 214         if (r < 0)
 215                 log_error_errno(errno, "Failed to determine language for file %s: %m", path);
 216         if (r == 1)
 217                 log_debug("File %s has language %s.", path, deflang);
 218
 219         for (;;) {
 220                 char line[LINE_MAX];
 221                 size_t a, b, c;
 222                 char *t;
 223
 224                 if (!fgets(line, sizeof(line), f)) {
 225                         if (feof(f))
 226                                 break;
 227
 228                         log_error_errno(errno, "Failed to read file %s: %m", path);
 229                         return -errno;
 230                 }
 231
 232                 n++;
 233
 234                 truncate_nl(line);
 235
 236                 if (line[0] == 0) {
 237                         empty_line = true;
 238                         continue;
 239                 }
 240
 241                 if (strchr(COMMENTS "\n", line[0]))
 242                         continue;
 243
 244                 if (empty_line &&
 245                     strlen(line) >= 2+1+32 &&
 246                     line[0] == '-' &&
 247                     line[1] == '-' &&
 248                     line[2] == ' ' &&
 249                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 250
 251                         bool with_language;
 252                         sd_id128_t jd;
 253
 254                         /* New entry */
 255
 256                         with_language = line[2+1+32] != '\0';
 257                         line[2+1+32] = '\0';
 258
 259                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 260
 261                                 if (got_id) {
 262                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 263                                         if (r < 0)
 264                                                 return r;
 265
 266                                         lang = mfree(lang);
 267                                 }
 268
 269                                 if (with_language) {
 270                                         t = strstrip(line + 2 + 1 + 32 + 1);
 271
 272                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 273                                         if (r < 0)
 274                                                 return r;
 275                                 }
 276
 277                                 got_id = true;
 278                                 empty_line = false;
 279                                 id = jd;
 280
 281                                 if (payload)
 282                                         payload[0] = '\0';
 283
 284                                 continue;
 285                         }
 286                 }
 287
 288                 /* Payload */
 289                 if (!got_id) {
 290                         log_error("[%s:%u] Got payload before ID.", path, n);
 291                         return -EINVAL;
 292                 }
 293
 294                 a = payload ? strlen(payload) : 0;
 295                 b = strlen(line);
 296
 297                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 298                 t = realloc(payload, c);
 299                 if (!t)
 300                         return log_oom();
 301
 302                 if (empty_line) {
 303                         t[a] = '\n';
 304                         memcpy(t + a + 1, line, b);
 305                         t[a+b+1] = '\n';
 306                         t[a+b+2] = 0;
 307                 } else {
 308                         memcpy(t + a, line, b);
 309                         t[a+b] = '\n';
 310                         t[a+b+1] = 0;
 311                 }
 312
 313                 payload = t;
 314                 empty_line = false;
 315         }
 316
 317         if (got_id) {
 318                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 319                 if (r < 0)
 320                         return r;
 321         }
 322
 323         return 0;
 324 }
 325
 326 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 327                           CatalogItem *items, size_t n) {
 328         CatalogHeader header;
 329         _cleanup_fclose_ FILE *w = NULL;
 330         int r;
 331         _cleanup_free_ char *d, *p = NULL;
 332         size_t k;
 333
 334         d = dirname_malloc(database);
 335         if (!d)
 336                 return log_oom();
 337
 338         r = mkdir_p(d, 0775);
 339         if (r < 0)
 340                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 341
 342         r = fopen_temporary(database, &w, &p);
 343         if (r < 0)
 344                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 345                                        database);
 346
 347         zero(header);
 348         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 349         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 350         header.catalog_item_size = htole64(sizeof(CatalogItem));
 351         header.n_items = htole64(hashmap_size(h));
 352
 353         r = -EIO;
 354
 355         k = fwrite(&header, 1, sizeof(header), w);
 356         if (k != sizeof(header)) {
 357                 log_error("%s: failed to write header.", p);
 358                 goto error;
 359         }
 360
 361         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 362         if (k != n * sizeof(CatalogItem)) {
 363                 log_error("%s: failed to write database.", p);
 364                 goto error;
 365         }
 366
 367         k = fwrite(sb->buf, 1, sb->len, w);
 368         if (k != sb->len) {
 369                 log_error("%s: failed to write strings.", p);
 370                 goto error;
 371         }
 372
 373         r = fflush_and_check(w);
 374         if (r < 0) {
 375                 log_error_errno(r, "%s: failed to write database: %m", p);
 376                 goto error;
 377         }
 378
 379         fchmod(fileno(w), 0644);
 380
 381         if (rename(p, database) < 0) {
 382                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 383                 goto error;
 384         }
 385
 386         return ftell(w);
 387
 388 error:
 389         (void) unlink(p);
 390         return r;
 391 }
 392
 393 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 394         _cleanup_strv_free_ char **files = NULL;
 395         char **f;
 396         struct strbuf *sb = NULL;
 397         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 398         _cleanup_free_ CatalogItem *items = NULL;
 399         CatalogItem *i;
 400         Iterator j;
 401         unsigned n;
 402         long r;
 403
 404         h = hashmap_new(&catalog_hash_ops);
 405         sb = strbuf_new();
 406
 407         if (!h || !sb) {
 408                 r = log_oom();
 409                 goto finish;
 410         }
 411
 412         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 413         if (r < 0) {
 414                 log_error_errno(r, "Failed to get catalog files: %m");
 415                 goto finish;
 416         }
 417
 418         STRV_FOREACH(f, files) {
 419                 log_debug("Reading file '%s'", *f);
 420                 r = catalog_import_file(h, sb, *f);
 421                 if (r < 0) {
 422                         log_error("Failed to import file '%s': %s.",
 423                                   *f, strerror(-r));
 424                         goto finish;
 425                 }
 426         }
 427
 428         if (hashmap_size(h) <= 0) {
 429                 log_info("No items in catalog.");
 430                 goto finish;
 431         } else
 432                 log_debug("Found %u items in catalog.", hashmap_size(h));
 433
 434         strbuf_complete(sb);
 435
 436         items = new(CatalogItem, hashmap_size(h));
 437         if (!items) {
 438                 r = log_oom();
 439                 goto finish;
 440         }
 441
 442         n = 0;
 443         HASHMAP_FOREACH(i, h, j) {
 444                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 445                           SD_ID128_FORMAT_VAL(i->id),
 446                           isempty(i->language) ? "C" : i->language);
 447                 items[n++] = *i;
 448         }
 449
 450         assert(n == hashmap_size(h));
 451         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 452
 453         r = write_catalog(database, h, sb, items, n);
 454         if (r < 0)
 455                 log_error_errno(r, "Failed to write %s: %m", database);
 456         else
 457                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 458                           database, n, sb->len, r);
 459
 460 finish:
 461         if (sb)
 462                 strbuf_cleanup(sb);
 463
 464         return r < 0 ? r : 0;
 465 }
 466
 467 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 468         const CatalogHeader *h;
 469         int fd;
 470         void *p;
 471         struct stat st;
 472
 473         assert(_fd);
 474         assert(_st);
 475         assert(_p);
 476
 477         fd = open(database, O_RDONLY|O_CLOEXEC);
 478         if (fd < 0)
 479                 return -errno;
 480
 481         if (fstat(fd, &st) < 0) {
 482                 safe_close(fd);
 483                 return -errno;
 484         }
 485
 486         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 487                 safe_close(fd);
 488                 return -EINVAL;
 489         }
 490
 491         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 492         if (p == MAP_FAILED) {
 493                 safe_close(fd);
 494                 return -errno;
 495         }
 496
 497         h = p;
 498         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 499             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 500             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 501             h->incompatible_flags != 0 ||
 502             le64toh(h->n_items) <= 0 ||
 503             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 504                 safe_close(fd);
 505                 munmap(p, st.st_size);
 506                 return -EBADMSG;
 507         }
 508
 509         *_fd = fd;
 510         *_st = st;
 511         *_p = p;
 512
 513         return 0;
 514 }
 515
 516 static const char *find_id(void *p, sd_id128_t id) {
 517         CatalogItem key, *f = NULL;
 518         const CatalogHeader *h = p;
 519         const char *loc;
 520
 521         zero(key);
 522         key.id = id;
 523
 524         loc = setlocale(LC_MESSAGES, NULL);
 525         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 526                 strncpy(key.language, loc, sizeof(key.language));
 527                 key.language[strcspn(key.language, ".@")] = 0;
 528
 529                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 530                 if (!f) {
 531                         char *e;
 532
 533                         e = strchr(key.language, '_');
 534                         if (e) {
 535                                 *e = 0;
 536                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 537                         }
 538                 }
 539         }
 540
 541         if (!f) {
 542                 zero(key.language);
 543                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 544         }
 545
 546         if (!f)
 547                 return NULL;
 548
 549         return (const char*) p +
 550                 le64toh(h->header_size) +
 551                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 552                 le64toh(f->offset);
 553 }
 554
 555 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 556         _cleanup_close_ int fd = -1;
 557         void *p = NULL;
 558         struct stat st = {};
 559         char *text = NULL;
 560         int r;
 561         const char *s;
 562
 563         assert(_text);
 564
 565         r = open_mmap(database, &fd, &st, &p);
 566         if (r < 0)
 567                 return r;
 568
 569         s = find_id(p, id);
 570         if (!s) {
 571                 r = -ENOENT;
 572                 goto finish;
 573         }
 574
 575         text = strdup(s);
 576         if (!text) {
 577                 r = -ENOMEM;
 578                 goto finish;
 579         }
 580
 581         *_text = text;
 582         r = 0;
 583
 584 finish:
 585         if (p)
 586                 munmap(p, st.st_size);
 587
 588         return r;
 589 }
 590
 591 static char *find_header(const char *s, const char *header) {
 592
 593         for (;;) {
 594                 const char *v, *e;
 595
 596                 v = startswith(s, header);
 597                 if (v) {
 598                         v += strspn(v, WHITESPACE);
 599                         return strndup(v, strcspn(v, NEWLINE));
 600                 }
 601
 602                 /* End of text */
 603                 e = strchr(s, '\n');
 604                 if (!e)
 605                         return NULL;
 606
 607                 /* End of header */
 608                 if (e == s)
 609                         return NULL;
 610
 611                 s = e + 1;
 612         }
 613 }
 614
 615 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 616         if (oneline) {
 617                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 618
 619                 subject = find_header(s, "Subject:");
 620                 defined_by = find_header(s, "Defined-By:");
 621
 622                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 623                         SD_ID128_FORMAT_VAL(id),
 624                         strna(defined_by), strna(subject));
 625         } else
 626                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 627                         SD_ID128_FORMAT_VAL(id), s);
 628 }
 629
 630
 631 int catalog_list(FILE *f, const char *database, bool oneline) {
 632         _cleanup_close_ int fd = -1;
 633         void *p = NULL;
 634         struct stat st;
 635         const CatalogHeader *h;
 636         const CatalogItem *items;
 637         int r;
 638         unsigned n;
 639         sd_id128_t last_id;
 640         bool last_id_set = false;
 641
 642         r = open_mmap(database, &fd, &st, &p);
 643         if (r < 0)
 644                 return r;
 645
 646         h = p;
 647         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 648
 649         for (n = 0; n < le64toh(h->n_items); n++) {
 650                 const char *s;
 651
 652                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 653                         continue;
 654
 655                 assert_se(s = find_id(p, items[n].id));
 656
 657                 dump_catalog_entry(f, items[n].id, s, oneline);
 658
 659                 last_id_set = true;
 660                 last_id = items[n].id;
 661         }
 662
 663         munmap(p, st.st_size);
 664
 665         return 0;
 666 }
 667
 668 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 669         char **item;
 670         int r = 0;
 671
 672         STRV_FOREACH(item, items) {
 673                 sd_id128_t id;
 674                 int k;
 675                 _cleanup_free_ char *msg = NULL;
 676
 677                 k = sd_id128_from_string(*item, &id);
 678                 if (k < 0) {
 679                         log_error_errno(k, "Failed to parse id128 '%s': %m",
 680                                         *item);
 681                         if (r == 0)
 682                                 r = k;
 683                         continue;
 684                 }
 685
 686                 k = catalog_get(database, id, &msg);
 687                 if (k < 0) {
 688                         log_full(k == -ENOENT ? LOG_NOTICE : LOG_ERR,
 689                                  "Failed to retrieve catalog entry for '%s': %s",
 690                                   *item, strerror(-k));
 691                         if (r == 0)
 692                                 r = k;
 693                         continue;
 694                 }
 695
 696                 dump_catalog_entry(f, id, msg, oneline);
 697         }
 698
 699         return r;
 700 }