src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <fcntl.h>
  23 #include <stdio.h>
  24 #include <unistd.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <locale.h>
  29
  30 #include "util.h"
  31 #include "log.h"
  32 #include "sparse-endian.h"
  33 #include "sd-id128.h"
  34 #include "hashmap.h"
  35 #include "strv.h"
  36 #include "strbuf.h"
  37 #include "conf-files.h"
  38 #include "mkdir.h"
  39 #include "catalog.h"
  40 #include "siphash24.h"
  41
  42 const char * const catalog_file_dirs[] = {
  43         "/usr/local/lib/systemd/catalog/",
  44         "/usr/lib/systemd/catalog/",
  45         NULL
  46 };
  47
  48 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  49
  50 typedef struct CatalogHeader {
  51         uint8_t signature[8];  /* "RHHHKSLP" */
  52         le32_t compatible_flags;
  53         le32_t incompatible_flags;
  54         le64_t header_size;
  55         le64_t n_items;
  56         le64_t catalog_item_size;
  57 } CatalogHeader;
  58
  59 typedef struct CatalogItem {
  60         sd_id128_t id;
  61         char language[32];
  62         le64_t offset;
  63 } CatalogItem;
  64
  65 static unsigned long catalog_hash_func(const void *p, const uint8_t hash_key[HASH_KEY_SIZE]) {
  66         const CatalogItem *i = p;
  67         uint64_t u;
  68         size_t l, sz;
  69         void *v;
  70
  71         l = strlen(i->language);
  72         sz = sizeof(i->id) + l;
  73         v = alloca(sz);
  74
  75         memcpy(mempcpy(v, &i->id, sizeof(i->id)), i->language, l);
  76
  77         siphash24((uint8_t*) &u, v, sz, hash_key);
  78
  79         return (unsigned long) u;
  80 }
  81
  82 static int catalog_compare_func(const void *a, const void *b) {
  83         const CatalogItem *i = a, *j = b;
  84         unsigned k;
  85
  86         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  87                  if (i->id.bytes[k] < j->id.bytes[k])
  88                         return -1;
  89                  if (i->id.bytes[k] > j->id.bytes[k])
  90                         return 1;
  91         }
  92
  93         return strcmp(i->language, j->language);
  94 }
  95
  96 const struct hash_ops catalog_hash_ops = {
  97         .hash = catalog_hash_func,
  98         .compare = catalog_compare_func
  99 };
 100
 101 static int finish_item(
 102                 Hashmap *h,
 103                 struct strbuf *sb,
 104                 sd_id128_t id,
 105                 const char *language,
 106                 const char *payload) {
 107
 108         ssize_t offset;
 109         _cleanup_free_ CatalogItem *i = NULL;
 110         int r;
 111
 112         assert(h);
 113         assert(sb);
 114         assert(payload);
 115
 116         offset = strbuf_add_string(sb, payload, strlen(payload));
 117         if (offset < 0)
 118                 return log_oom();
 119
 120         i = new0(CatalogItem, 1);
 121         if (!i)
 122                 return log_oom();
 123
 124         i->id = id;
 125         if (language) {
 126                 assert(strlen(language) > 1 && strlen(language) < 32);
 127                 strcpy(i->language, language);
 128         }
 129         i->offset = htole64((uint64_t) offset);
 130
 131         r = hashmap_put(h, i, i);
 132         if (r == -EEXIST) {
 133                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 134                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 135                 return 0;
 136         } else if (r < 0)
 137                 return r;
 138
 139         i = NULL;
 140         return 0;
 141 }
 142
 143 int catalog_file_lang(const char* filename, char **lang) {
 144         char *beg, *end, *_lang;
 145
 146         end = endswith(filename, ".catalog");
 147         if (!end)
 148                 return 0;
 149
 150         beg = end - 1;
 151         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 152                 beg --;
 153
 154         if (*beg != '.' || end <= beg + 1)
 155                 return 0;
 156
 157         _lang = strndup(beg + 1, end - beg - 1);
 158         if (!_lang)
 159                 return -ENOMEM;
 160
 161         *lang = _lang;
 162         return 1;
 163 }
 164
 165 static int catalog_entry_lang(const char* filename, int line,
 166                               const char* t, const char* deflang, char **lang) {
 167         size_t c;
 168
 169         c = strlen(t);
 170         if (c == 0) {
 171                 log_error("[%s:%u] Language too short.", filename, line);
 172                 return -EINVAL;
 173         }
 174         if (c > 31) {
 175                 log_error("[%s:%u] language too long.", filename, line);
 176                 return -EINVAL;
 177         }
 178
 179         if (deflang) {
 180                 if (streq(t, deflang)) {
 181                         log_warning("[%s:%u] language specified unnecessarily",
 182                                     filename, line);
 183                         return 0;
 184                 } else
 185                         log_warning("[%s:%u] language differs from default for file",
 186                                     filename, line);
 187         }
 188
 189         *lang = strdup(t);
 190         if (!*lang)
 191                         return -ENOMEM;
 192
 193         return 0;
 194 }
 195
 196 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 197         _cleanup_fclose_ FILE *f = NULL;
 198         _cleanup_free_ char *payload = NULL;
 199         unsigned n = 0;
 200         sd_id128_t id;
 201         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 202         bool got_id = false, empty_line = true;
 203         int r;
 204
 205         assert(h);
 206         assert(sb);
 207         assert(path);
 208
 209         f = fopen(path, "re");
 210         if (!f)
 211                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 212
 213         r = catalog_file_lang(path, &deflang);
 214         if (r < 0)
 215                 log_error_errno(errno, "Failed to determine language for file %s: %m", path);
 216         if (r == 1)
 217                 log_debug("File %s has language %s.", path, deflang);
 218
 219         for (;;) {
 220                 char line[LINE_MAX];
 221                 size_t a, b, c;
 222                 char *t;
 223
 224                 if (!fgets(line, sizeof(line), f)) {
 225                         if (feof(f))
 226                                 break;
 227
 228                         log_error_errno(errno, "Failed to read file %s: %m", path);
 229                         return -errno;
 230                 }
 231
 232                 n++;
 233
 234                 truncate_nl(line);
 235
 236                 if (line[0] == 0) {
 237                         empty_line = true;
 238                         continue;
 239                 }
 240
 241                 if (strchr(COMMENTS "\n", line[0]))
 242                         continue;
 243
 244                 if (empty_line &&
 245                     strlen(line) >= 2+1+32 &&
 246                     line[0] == '-' &&
 247                     line[1] == '-' &&
 248                     line[2] == ' ' &&
 249                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 250
 251                         bool with_language;
 252                         sd_id128_t jd;
 253
 254                         /* New entry */
 255
 256                         with_language = line[2+1+32] != '\0';
 257                         line[2+1+32] = '\0';
 258
 259                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 260
 261                                 if (got_id) {
 262                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 263                                         if (r < 0)
 264                                                 return r;
 265
 266                                         lang = mfree(lang);
 267                                 }
 268
 269                                 if (with_language) {
 270                                         t = strstrip(line + 2 + 1 + 32 + 1);
 271
 272                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 273                                         if (r < 0)
 274                                                 return r;
 275                                 }
 276
 277                                 got_id = true;
 278                                 empty_line = false;
 279                                 id = jd;
 280
 281                                 if (payload)
 282                                         payload[0] = '\0';
 283
 284                                 continue;
 285                         }
 286                 }
 287
 288                 /* Payload */
 289                 if (!got_id) {
 290                         log_error("[%s:%u] Got payload before ID.", path, n);
 291                         return -EINVAL;
 292                 }
 293
 294                 a = payload ? strlen(payload) : 0;
 295                 b = strlen(line);
 296
 297                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 298                 t = realloc(payload, c);
 299                 if (!t)
 300                         return log_oom();
 301
 302                 if (empty_line) {
 303                         t[a] = '\n';
 304                         memcpy(t + a + 1, line, b);
 305                         t[a+b+1] = '\n';
 306                         t[a+b+2] = 0;
 307                 } else {
 308                         memcpy(t + a, line, b);
 309                         t[a+b] = '\n';
 310                         t[a+b+1] = 0;
 311                 }
 312
 313                 payload = t;
 314                 empty_line = false;
 315         }
 316
 317         if (got_id) {
 318                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 319                 if (r < 0)
 320                         return r;
 321         }
 322
 323         return 0;
 324 }
 325
 326 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 327                           CatalogItem *items, size_t n) {
 328         CatalogHeader header;
 329         _cleanup_fclose_ FILE *w = NULL;
 330         int r;
 331         _cleanup_free_ char *d, *p = NULL;
 332         size_t k;
 333
 334         d = dirname_malloc(database);
 335         if (!d)
 336                 return log_oom();
 337
 338         r = mkdir_p(d, 0775);
 339         if (r < 0)
 340                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 341
 342         r = fopen_temporary(database, &w, &p);
 343         if (r < 0)
 344                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 345                                        database);
 346
 347         zero(header);
 348         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 349         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 350         header.catalog_item_size = htole64(sizeof(CatalogItem));
 351         header.n_items = htole64(hashmap_size(h));
 352
 353         r = -EIO;
 354
 355         k = fwrite(&header, 1, sizeof(header), w);
 356         if (k != sizeof(header)) {
 357                 log_error("%s: failed to write header.", p);
 358                 goto error;
 359         }
 360
 361         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 362         if (k != n * sizeof(CatalogItem)) {
 363                 log_error("%s: failed to write database.", p);
 364                 goto error;
 365         }
 366
 367         k = fwrite(sb->buf, 1, sb->len, w);
 368         if (k != sb->len) {
 369                 log_error("%s: failed to write strings.", p);
 370                 goto error;
 371         }
 372
 373         r = fflush_and_check(w);
 374         if (r < 0) {
 375                 log_error_errno(r, "%s: failed to write database: %m", p);
 376                 goto error;
 377         }
 378
 379         fchmod(fileno(w), 0644);
 380
 381         if (rename(p, database) < 0) {
 382                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 383                 goto error;
 384         }
 385
 386         return ftell(w);
 387
 388 error:
 389         (void) unlink(p);
 390         return r;
 391 }
 392
 393 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 394         _cleanup_strv_free_ char **files = NULL;
 395         char **f;
 396         struct strbuf *sb = NULL;
 397         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 398         _cleanup_free_ CatalogItem *items = NULL;
 399         CatalogItem *i;
 400         Iterator j;
 401         unsigned n;
 402         long r;
 403
 404         h = hashmap_new(&catalog_hash_ops);
 405         sb = strbuf_new();
 406
 407         if (!h || !sb) {
 408                 r = log_oom();
 409                 goto finish;
 410         }
 411
 412         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 413         if (r < 0) {
 414                 log_error_errno(r, "Failed to get catalog files: %m");
 415                 goto finish;
 416         }
 417
 418         STRV_FOREACH(f, files) {
 419                 log_debug("Reading file '%s'", *f);
 420                 r = catalog_import_file(h, sb, *f);
 421                 if (r < 0) {
 422                         log_error_errno(r, "Failed to import file '%s': %m", *f);
 423                         goto finish;
 424                 }
 425         }
 426
 427         if (hashmap_size(h) <= 0) {
 428                 log_info("No items in catalog.");
 429                 goto finish;
 430         } else
 431                 log_debug("Found %u items in catalog.", hashmap_size(h));
 432
 433         strbuf_complete(sb);
 434
 435         items = new(CatalogItem, hashmap_size(h));
 436         if (!items) {
 437                 r = log_oom();
 438                 goto finish;
 439         }
 440
 441         n = 0;
 442         HASHMAP_FOREACH(i, h, j) {
 443                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 444                           SD_ID128_FORMAT_VAL(i->id),
 445                           isempty(i->language) ? "C" : i->language);
 446                 items[n++] = *i;
 447         }
 448
 449         assert(n == hashmap_size(h));
 450         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 451
 452         r = write_catalog(database, h, sb, items, n);
 453         if (r < 0)
 454                 log_error_errno(r, "Failed to write %s: %m", database);
 455         else
 456                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 457                           database, n, sb->len, r);
 458
 459 finish:
 460         if (sb)
 461                 strbuf_cleanup(sb);
 462
 463         return r < 0 ? r : 0;
 464 }
 465
 466 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 467         const CatalogHeader *h;
 468         int fd;
 469         void *p;
 470         struct stat st;
 471
 472         assert(_fd);
 473         assert(_st);
 474         assert(_p);
 475
 476         fd = open(database, O_RDONLY|O_CLOEXEC);
 477         if (fd < 0)
 478                 return -errno;
 479
 480         if (fstat(fd, &st) < 0) {
 481                 safe_close(fd);
 482                 return -errno;
 483         }
 484
 485         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 486                 safe_close(fd);
 487                 return -EINVAL;
 488         }
 489
 490         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 491         if (p == MAP_FAILED) {
 492                 safe_close(fd);
 493                 return -errno;
 494         }
 495
 496         h = p;
 497         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 498             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 499             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 500             h->incompatible_flags != 0 ||
 501             le64toh(h->n_items) <= 0 ||
 502             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 503                 safe_close(fd);
 504                 munmap(p, st.st_size);
 505                 return -EBADMSG;
 506         }
 507
 508         *_fd = fd;
 509         *_st = st;
 510         *_p = p;
 511
 512         return 0;
 513 }
 514
 515 static const char *find_id(void *p, sd_id128_t id) {
 516         CatalogItem key, *f = NULL;
 517         const CatalogHeader *h = p;
 518         const char *loc;
 519
 520         zero(key);
 521         key.id = id;
 522
 523         loc = setlocale(LC_MESSAGES, NULL);
 524         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 525                 strncpy(key.language, loc, sizeof(key.language));
 526                 key.language[strcspn(key.language, ".@")] = 0;
 527
 528                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 529                 if (!f) {
 530                         char *e;
 531
 532                         e = strchr(key.language, '_');
 533                         if (e) {
 534                                 *e = 0;
 535                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 536                         }
 537                 }
 538         }
 539
 540         if (!f) {
 541                 zero(key.language);
 542                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 543         }
 544
 545         if (!f)
 546                 return NULL;
 547
 548         return (const char*) p +
 549                 le64toh(h->header_size) +
 550                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 551                 le64toh(f->offset);
 552 }
 553
 554 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 555         _cleanup_close_ int fd = -1;
 556         void *p = NULL;
 557         struct stat st = {};
 558         char *text = NULL;
 559         int r;
 560         const char *s;
 561
 562         assert(_text);
 563
 564         r = open_mmap(database, &fd, &st, &p);
 565         if (r < 0)
 566                 return r;
 567
 568         s = find_id(p, id);
 569         if (!s) {
 570                 r = -ENOENT;
 571                 goto finish;
 572         }
 573
 574         text = strdup(s);
 575         if (!text) {
 576                 r = -ENOMEM;
 577                 goto finish;
 578         }
 579
 580         *_text = text;
 581         r = 0;
 582
 583 finish:
 584         if (p)
 585                 munmap(p, st.st_size);
 586
 587         return r;
 588 }
 589
 590 static char *find_header(const char *s, const char *header) {
 591
 592         for (;;) {
 593                 const char *v, *e;
 594
 595                 v = startswith(s, header);
 596                 if (v) {
 597                         v += strspn(v, WHITESPACE);
 598                         return strndup(v, strcspn(v, NEWLINE));
 599                 }
 600
 601                 /* End of text */
 602                 e = strchr(s, '\n');
 603                 if (!e)
 604                         return NULL;
 605
 606                 /* End of header */
 607                 if (e == s)
 608                         return NULL;
 609
 610                 s = e + 1;
 611         }
 612 }
 613
 614 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 615         if (oneline) {
 616                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 617
 618                 subject = find_header(s, "Subject:");
 619                 defined_by = find_header(s, "Defined-By:");
 620
 621                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 622                         SD_ID128_FORMAT_VAL(id),
 623                         strna(defined_by), strna(subject));
 624         } else
 625                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 626                         SD_ID128_FORMAT_VAL(id), s);
 627 }
 628
 629
 630 int catalog_list(FILE *f, const char *database, bool oneline) {
 631         _cleanup_close_ int fd = -1;
 632         void *p = NULL;
 633         struct stat st;
 634         const CatalogHeader *h;
 635         const CatalogItem *items;
 636         int r;
 637         unsigned n;
 638         sd_id128_t last_id;
 639         bool last_id_set = false;
 640
 641         r = open_mmap(database, &fd, &st, &p);
 642         if (r < 0)
 643                 return r;
 644
 645         h = p;
 646         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 647
 648         for (n = 0; n < le64toh(h->n_items); n++) {
 649                 const char *s;
 650
 651                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 652                         continue;
 653
 654                 assert_se(s = find_id(p, items[n].id));
 655
 656                 dump_catalog_entry(f, items[n].id, s, oneline);
 657
 658                 last_id_set = true;
 659                 last_id = items[n].id;
 660         }
 661
 662         munmap(p, st.st_size);
 663
 664         return 0;
 665 }
 666
 667 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 668         char **item;
 669         int r = 0;
 670
 671         STRV_FOREACH(item, items) {
 672                 sd_id128_t id;
 673                 int k;
 674                 _cleanup_free_ char *msg = NULL;
 675
 676                 k = sd_id128_from_string(*item, &id);
 677                 if (k < 0) {
 678                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 679                         if (r == 0)
 680                                 r = k;
 681                         continue;
 682                 }
 683
 684                 k = catalog_get(database, id, &msg);
 685                 if (k < 0) {
 686                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 687                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 688                         if (r == 0)
 689                                 r = k;
 690                         continue;
 691                 }
 692
 693                 dump_catalog_entry(f, id, msg, oneline);
 694         }
 695
 696         return r;
 697 }