src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <fcntl.h>
  23 #include <stdio.h>
  24 #include <unistd.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <locale.h>
  29
  30 #include "util.h"
  31 #include "log.h"
  32 #include "sparse-endian.h"
  33 #include "sd-id128.h"
  34 #include "hashmap.h"
  35 #include "strv.h"
  36 #include "strbuf.h"
  37 #include "conf-files.h"
  38 #include "mkdir.h"
  39 #include "catalog.h"
  40 #include "siphash24.h"
  41
  42 const char * const catalog_file_dirs[] = {
  43         "/usr/local/lib/systemd/catalog/",
  44         "/usr/lib/systemd/catalog/",
  45         NULL
  46 };
  47
  48 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  49
  50 typedef struct CatalogHeader {
  51         uint8_t signature[8];  /* "RHHHKSLP" */
  52         le32_t compatible_flags;
  53         le32_t incompatible_flags;
  54         le64_t header_size;
  55         le64_t n_items;
  56         le64_t catalog_item_size;
  57 } CatalogHeader;
  58
  59 typedef struct CatalogItem {
  60         sd_id128_t id;
  61         char language[32];
  62         le64_t offset;
  63 } CatalogItem;
  64
  65 static unsigned long catalog_hash_func(const void *p, const uint8_t hash_key[HASH_KEY_SIZE]) {
  66         const CatalogItem *i = p;
  67         uint64_t u;
  68         size_t l, sz;
  69         void *v;
  70
  71         l = strlen(i->language);
  72         sz = sizeof(i->id) + l;
  73         v = alloca(sz);
  74
  75         memcpy(mempcpy(v, &i->id, sizeof(i->id)), i->language, l);
  76
  77         siphash24((uint8_t*) &u, v, sz, hash_key);
  78
  79         return (unsigned long) u;
  80 }
  81
  82 static int catalog_compare_func(const void *a, const void *b) {
  83         const CatalogItem *i = a, *j = b;
  84         unsigned k;
  85
  86         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  87                  if (i->id.bytes[k] < j->id.bytes[k])
  88                         return -1;
  89                  if (i->id.bytes[k] > j->id.bytes[k])
  90                         return 1;
  91         }
  92
  93         return strcmp(i->language, j->language);
  94 }
  95
  96 const struct hash_ops catalog_hash_ops = {
  97         .hash = catalog_hash_func,
  98         .compare = catalog_compare_func
  99 };
 100
 101 static int finish_item(
 102                 Hashmap *h,
 103                 struct strbuf *sb,
 104                 sd_id128_t id,
 105                 const char *language,
 106                 const char *payload) {
 107
 108         ssize_t offset;
 109         _cleanup_free_ CatalogItem *i = NULL;
 110         int r;
 111
 112         assert(h);
 113         assert(sb);
 114         assert(payload);
 115
 116         offset = strbuf_add_string(sb, payload, strlen(payload));
 117         if (offset < 0)
 118                 return log_oom();
 119
 120         i = new0(CatalogItem, 1);
 121         if (!i)
 122                 return log_oom();
 123
 124         i->id = id;
 125         if (language) {
 126                 assert(strlen(language) > 1 && strlen(language) < 32);
 127                 strcpy(i->language, language);
 128         }
 129         i->offset = htole64((uint64_t) offset);
 130
 131         r = hashmap_put(h, i, i);
 132         if (r == -EEXIST) {
 133                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 134                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 135                 return 0;
 136         } else if (r < 0)
 137                 return r;
 138
 139         i = NULL;
 140         return 0;
 141 }
 142
 143 int catalog_file_lang(const char* filename, char **lang) {
 144         char *beg, *end, *_lang;
 145
 146         end = endswith(filename, ".catalog");
 147         if (!end)
 148                 return 0;
 149
 150         beg = end - 1;
 151         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 152                 beg --;
 153
 154         if (*beg != '.' || end <= beg + 1)
 155                 return 0;
 156
 157         _lang = strndup(beg + 1, end - beg - 1);
 158         if (!_lang)
 159                 return -ENOMEM;
 160
 161         *lang = _lang;
 162         return 1;
 163 }
 164
 165 static int catalog_entry_lang(const char* filename, int line,
 166                               const char* t, const char* deflang, char **lang) {
 167         size_t c;
 168
 169         c = strlen(t);
 170         if (c == 0) {
 171                 log_error("[%s:%u] Language too short.", filename, line);
 172                 return -EINVAL;
 173         }
 174         if (c > 31) {
 175                 log_error("[%s:%u] language too long.", filename, line);
 176                 return -EINVAL;
 177         }
 178
 179         if (deflang) {
 180                 if (streq(t, deflang)) {
 181                         log_warning("[%s:%u] language specified unnecessarily",
 182                                     filename, line);
 183                         return 0;
 184                 } else
 185                         log_warning("[%s:%u] language differs from default for file",
 186                                     filename, line);
 187         }
 188
 189         *lang = strdup(t);
 190         if (!*lang)
 191                         return -ENOMEM;
 192
 193         return 0;
 194 }
 195
 196 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 197         _cleanup_fclose_ FILE *f = NULL;
 198         _cleanup_free_ char *payload = NULL;
 199         unsigned n = 0;
 200         sd_id128_t id;
 201         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 202         bool got_id = false, empty_line = true;
 203         int r;
 204
 205         assert(h);
 206         assert(sb);
 207         assert(path);
 208
 209         f = fopen(path, "re");
 210         if (!f)
 211                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 212
 213         r = catalog_file_lang(path, &deflang);
 214         if (r < 0)
 215                 log_error_errno(errno, "Failed to determine language for file %s: %m", path);
 216         if (r == 1)
 217                 log_debug("File %s has language %s.", path, deflang);
 218
 219         for (;;) {
 220                 char line[LINE_MAX];
 221                 size_t a, b, c;
 222                 char *t;
 223
 224                 if (!fgets(line, sizeof(line), f)) {
 225                         if (feof(f))
 226                                 break;
 227
 228                         log_error_errno(errno, "Failed to read file %s: %m", path);
 229                         return -errno;
 230                 }
 231
 232                 n++;
 233
 234                 truncate_nl(line);
 235
 236                 if (line[0] == 0) {
 237                         empty_line = true;
 238                         continue;
 239                 }
 240
 241                 if (strchr(COMMENTS "\n", line[0]))
 242                         continue;
 243
 244                 if (empty_line &&
 245                     strlen(line) >= 2+1+32 &&
 246                     line[0] == '-' &&
 247                     line[1] == '-' &&
 248                     line[2] == ' ' &&
 249                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 250
 251                         bool with_language;
 252                         sd_id128_t jd;
 253
 254                         /* New entry */
 255
 256                         with_language = line[2+1+32] != '\0';
 257                         line[2+1+32] = '\0';
 258
 259                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 260
 261                                 if (got_id) {
 262                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 263                                         if (r < 0)
 264                                                 return r;
 265
 266                                         free(lang);
 267                                         lang = NULL;
 268                                 }
 269
 270                                 if (with_language) {
 271                                         t = strstrip(line + 2 + 1 + 32 + 1);
 272
 273                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 274                                         if (r < 0)
 275                                                 return r;
 276                                 }
 277
 278                                 got_id = true;
 279                                 empty_line = false;
 280                                 id = jd;
 281
 282                                 if (payload)
 283                                         payload[0] = '\0';
 284
 285                                 continue;
 286                         }
 287                 }
 288
 289                 /* Payload */
 290                 if (!got_id) {
 291                         log_error("[%s:%u] Got payload before ID.", path, n);
 292                         return -EINVAL;
 293                 }
 294
 295                 a = payload ? strlen(payload) : 0;
 296                 b = strlen(line);
 297
 298                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 299                 t = realloc(payload, c);
 300                 if (!t)
 301                         return log_oom();
 302
 303                 if (empty_line) {
 304                         t[a] = '\n';
 305                         memcpy(t + a + 1, line, b);
 306                         t[a+b+1] = '\n';
 307                         t[a+b+2] = 0;
 308                 } else {
 309                         memcpy(t + a, line, b);
 310                         t[a+b] = '\n';
 311                         t[a+b+1] = 0;
 312                 }
 313
 314                 payload = t;
 315                 empty_line = false;
 316         }
 317
 318         if (got_id) {
 319                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 320                 if (r < 0)
 321                         return r;
 322         }
 323
 324         return 0;
 325 }
 326
 327 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 328                           CatalogItem *items, size_t n) {
 329         CatalogHeader header;
 330         _cleanup_fclose_ FILE *w = NULL;
 331         int r;
 332         _cleanup_free_ char *d, *p = NULL;
 333         size_t k;
 334
 335         d = dirname_malloc(database);
 336         if (!d)
 337                 return log_oom();
 338
 339         r = mkdir_p(d, 0775);
 340         if (r < 0)
 341                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 342
 343         r = fopen_temporary(database, &w, &p);
 344         if (r < 0)
 345                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 346                                        database);
 347
 348         zero(header);
 349         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 350         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 351         header.catalog_item_size = htole64(sizeof(CatalogItem));
 352         header.n_items = htole64(hashmap_size(h));
 353
 354         r = -EIO;
 355
 356         k = fwrite(&header, 1, sizeof(header), w);
 357         if (k != sizeof(header)) {
 358                 log_error("%s: failed to write header.", p);
 359                 goto error;
 360         }
 361
 362         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 363         if (k != n * sizeof(CatalogItem)) {
 364                 log_error("%s: failed to write database.", p);
 365                 goto error;
 366         }
 367
 368         k = fwrite(sb->buf, 1, sb->len, w);
 369         if (k != sb->len) {
 370                 log_error("%s: failed to write strings.", p);
 371                 goto error;
 372         }
 373
 374         fflush(w);
 375
 376         if (ferror(w)) {
 377                 log_error("%s: failed to write database.", p);
 378                 goto error;
 379         }
 380
 381         fchmod(fileno(w), 0644);
 382
 383         if (rename(p, database) < 0) {
 384                 log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 385                 r = -errno;
 386                 goto error;
 387         }
 388
 389         return ftell(w);
 390
 391 error:
 392         unlink(p);
 393         return r;
 394 }
 395
 396 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 397         _cleanup_strv_free_ char **files = NULL;
 398         char **f;
 399         struct strbuf *sb = NULL;
 400         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 401         _cleanup_free_ CatalogItem *items = NULL;
 402         CatalogItem *i;
 403         Iterator j;
 404         unsigned n;
 405         long r;
 406
 407         h = hashmap_new(&catalog_hash_ops);
 408         sb = strbuf_new();
 409
 410         if (!h || !sb) {
 411                 r = log_oom();
 412                 goto finish;
 413         }
 414
 415         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 416         if (r < 0) {
 417                 log_error_errno(r, "Failed to get catalog files: %m");
 418                 goto finish;
 419         }
 420
 421         STRV_FOREACH(f, files) {
 422                 log_debug("Reading file '%s'", *f);
 423                 r = catalog_import_file(h, sb, *f);
 424                 if (r < 0) {
 425                         log_error("Failed to import file '%s': %s.",
 426                                   *f, strerror(-r));
 427                         goto finish;
 428                 }
 429         }
 430
 431         if (hashmap_size(h) <= 0) {
 432                 log_info("No items in catalog.");
 433                 goto finish;
 434         } else
 435                 log_debug("Found %u items in catalog.", hashmap_size(h));
 436
 437         strbuf_complete(sb);
 438
 439         items = new(CatalogItem, hashmap_size(h));
 440         if (!items) {
 441                 r = log_oom();
 442                 goto finish;
 443         }
 444
 445         n = 0;
 446         HASHMAP_FOREACH(i, h, j) {
 447                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 448                           SD_ID128_FORMAT_VAL(i->id),
 449                           isempty(i->language) ? "C" : i->language);
 450                 items[n++] = *i;
 451         }
 452
 453         assert(n == hashmap_size(h));
 454         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 455
 456         r = write_catalog(database, h, sb, items, n);
 457         if (r < 0)
 458                 log_error_errno(r, "Failed to write %s: %m", database);
 459         else
 460                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 461                           database, n, sb->len, r);
 462
 463 finish:
 464         if (sb)
 465                 strbuf_cleanup(sb);
 466
 467         return r < 0 ? r : 0;
 468 }
 469
 470 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 471         const CatalogHeader *h;
 472         int fd;
 473         void *p;
 474         struct stat st;
 475
 476         assert(_fd);
 477         assert(_st);
 478         assert(_p);
 479
 480         fd = open(database, O_RDONLY|O_CLOEXEC);
 481         if (fd < 0)
 482                 return -errno;
 483
 484         if (fstat(fd, &st) < 0) {
 485                 safe_close(fd);
 486                 return -errno;
 487         }
 488
 489         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 490                 safe_close(fd);
 491                 return -EINVAL;
 492         }
 493
 494         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 495         if (p == MAP_FAILED) {
 496                 safe_close(fd);
 497                 return -errno;
 498         }
 499
 500         h = p;
 501         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 502             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 503             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 504             h->incompatible_flags != 0 ||
 505             le64toh(h->n_items) <= 0 ||
 506             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 507                 safe_close(fd);
 508                 munmap(p, st.st_size);
 509                 return -EBADMSG;
 510         }
 511
 512         *_fd = fd;
 513         *_st = st;
 514         *_p = p;
 515
 516         return 0;
 517 }
 518
 519 static const char *find_id(void *p, sd_id128_t id) {
 520         CatalogItem key, *f = NULL;
 521         const CatalogHeader *h = p;
 522         const char *loc;
 523
 524         zero(key);
 525         key.id = id;
 526
 527         loc = setlocale(LC_MESSAGES, NULL);
 528         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 529                 strncpy(key.language, loc, sizeof(key.language));
 530                 key.language[strcspn(key.language, ".@")] = 0;
 531
 532                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 533                 if (!f) {
 534                         char *e;
 535
 536                         e = strchr(key.language, '_');
 537                         if (e) {
 538                                 *e = 0;
 539                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 540                         }
 541                 }
 542         }
 543
 544         if (!f) {
 545                 zero(key.language);
 546                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 547         }
 548
 549         if (!f)
 550                 return NULL;
 551
 552         return (const char*) p +
 553                 le64toh(h->header_size) +
 554                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 555                 le64toh(f->offset);
 556 }
 557
 558 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 559         _cleanup_close_ int fd = -1;
 560         void *p = NULL;
 561         struct stat st;
 562         char *text = NULL;
 563         int r;
 564         const char *s;
 565
 566         assert(_text);
 567
 568         r = open_mmap(database, &fd, &st, &p);
 569         if (r < 0)
 570                 return r;
 571
 572         s = find_id(p, id);
 573         if (!s) {
 574                 r = -ENOENT;
 575                 goto finish;
 576         }
 577
 578         text = strdup(s);
 579         if (!text) {
 580                 r = -ENOMEM;
 581                 goto finish;
 582         }
 583
 584         *_text = text;
 585         r = 0;
 586
 587 finish:
 588         if (p)
 589                 munmap(p, st.st_size);
 590
 591         return r;
 592 }
 593
 594 static char *find_header(const char *s, const char *header) {
 595
 596         for (;;) {
 597                 const char *v, *e;
 598
 599                 v = startswith(s, header);
 600                 if (v) {
 601                         v += strspn(v, WHITESPACE);
 602                         return strndup(v, strcspn(v, NEWLINE));
 603                 }
 604
 605                 /* End of text */
 606                 e = strchr(s, '\n');
 607                 if (!e)
 608                         return NULL;
 609
 610                 /* End of header */
 611                 if (e == s)
 612                         return NULL;
 613
 614                 s = e + 1;
 615         }
 616 }
 617
 618 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 619         if (oneline) {
 620                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 621
 622                 subject = find_header(s, "Subject:");
 623                 defined_by = find_header(s, "Defined-By:");
 624
 625                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 626                         SD_ID128_FORMAT_VAL(id),
 627                         strna(defined_by), strna(subject));
 628         } else
 629                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 630                         SD_ID128_FORMAT_VAL(id), s);
 631 }
 632
 633
 634 int catalog_list(FILE *f, const char *database, bool oneline) {
 635         _cleanup_close_ int fd = -1;
 636         void *p = NULL;
 637         struct stat st;
 638         const CatalogHeader *h;
 639         const CatalogItem *items;
 640         int r;
 641         unsigned n;
 642         sd_id128_t last_id;
 643         bool last_id_set = false;
 644
 645         r = open_mmap(database, &fd, &st, &p);
 646         if (r < 0)
 647                 return r;
 648
 649         h = p;
 650         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 651
 652         for (n = 0; n < le64toh(h->n_items); n++) {
 653                 const char *s;
 654
 655                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 656                         continue;
 657
 658                 assert_se(s = find_id(p, items[n].id));
 659
 660                 dump_catalog_entry(f, items[n].id, s, oneline);
 661
 662                 last_id_set = true;
 663                 last_id = items[n].id;
 664         }
 665
 666         munmap(p, st.st_size);
 667
 668         return 0;
 669 }
 670
 671 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 672         char **item;
 673         int r = 0;
 674
 675         STRV_FOREACH(item, items) {
 676                 sd_id128_t id;
 677                 int k;
 678                 _cleanup_free_ char *msg = NULL;
 679
 680                 k = sd_id128_from_string(*item, &id);
 681                 if (k < 0) {
 682                         log_error_errno(k, "Failed to parse id128 '%s': %m",
 683                                         *item);
 684                         if (r == 0)
 685                                 r = k;
 686                         continue;
 687                 }
 688
 689                 k = catalog_get(database, id, &msg);
 690                 if (k < 0) {
 691                         log_full(k == -ENOENT ? LOG_NOTICE : LOG_ERR,
 692                                  "Failed to retrieve catalog entry for '%s': %s",
 693                                   *item, strerror(-k));
 694                         if (r == 0)
 695                                 r = k;
 696                         continue;
 697                 }
 698
 699                 dump_catalog_entry(f, id, msg, oneline);
 700         }
 701
 702         return r;
 703 }