src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <fcntl.h>
  23 #include <stdio.h>
  24 #include <unistd.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <locale.h>
  29 #include <libgen.h>
  30
  31 #include "util.h"
  32 #include "log.h"
  33 #include "sparse-endian.h"
  34 #include "sd-id128.h"
  35 #include "hashmap.h"
  36 #include "strv.h"
  37 #include "strbuf.h"
  38 #include "strxcpyx.h"
  39 #include "conf-files.h"
  40 #include "mkdir.h"
  41 #include "catalog.h"
  42 #include "siphash24.h"
  43
  44 const char * const catalog_file_dirs[] = {
  45         "/usr/local/lib/systemd/catalog/",
  46         "/usr/lib/systemd/catalog/",
  47         NULL
  48 };
  49
  50 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  51
  52 typedef struct CatalogHeader {
  53         uint8_t signature[8];  /* "RHHHKSLP" */
  54         le32_t compatible_flags;
  55         le32_t incompatible_flags;
  56         le64_t header_size;
  57         le64_t n_items;
  58         le64_t catalog_item_size;
  59 } CatalogHeader;
  60
  61 typedef struct CatalogItem {
  62         sd_id128_t id;
  63         char language[32];
  64         le64_t offset;
  65 } CatalogItem;
  66
  67 static unsigned long catalog_hash_func(const void *p, const uint8_t hash_key[HASH_KEY_SIZE]) {
  68         const CatalogItem *i = p;
  69         uint64_t u;
  70         size_t l, sz;
  71         void *v;
  72
  73         l = strlen(i->language);
  74         sz = sizeof(i->id) + l;
  75         v = alloca(sz);
  76
  77         memcpy(mempcpy(v, &i->id, sizeof(i->id)), i->language, l);
  78
  79         siphash24((uint8_t*) &u, v, sz, hash_key);
  80
  81         return (unsigned long) u;
  82 }
  83
  84 static int catalog_compare_func(const void *a, const void *b) {
  85         const CatalogItem *i = a, *j = b;
  86         unsigned k;
  87
  88         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  89                  if (i->id.bytes[k] < j->id.bytes[k])
  90                         return -1;
  91                  if (i->id.bytes[k] > j->id.bytes[k])
  92                         return 1;
  93         }
  94
  95         return strcmp(i->language, j->language);
  96 }
  97
  98 const struct hash_ops catalog_hash_ops = {
  99         .hash = catalog_hash_func,
 100         .compare = catalog_compare_func
 101 };
 102
 103 static int finish_item(
 104                 Hashmap *h,
 105                 struct strbuf *sb,
 106                 sd_id128_t id,
 107                 const char *language,
 108                 const char *payload) {
 109
 110         ssize_t offset;
 111         _cleanup_free_ CatalogItem *i = NULL;
 112         int r;
 113
 114         assert(h);
 115         assert(sb);
 116         assert(payload);
 117
 118         offset = strbuf_add_string(sb, payload, strlen(payload));
 119         if (offset < 0)
 120                 return log_oom();
 121
 122         i = new0(CatalogItem, 1);
 123         if (!i)
 124                 return log_oom();
 125
 126         i->id = id;
 127         if (language) {
 128                 assert(strlen(language) > 1 && strlen(language) < 32);
 129                 strcpy(i->language, language);
 130         }
 131         i->offset = htole64((uint64_t) offset);
 132
 133         r = hashmap_put(h, i, i);
 134         if (r == -EEXIST) {
 135                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 136                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 137                 return 0;
 138         } else if (r < 0)
 139                 return r;
 140
 141         i = NULL;
 142         return 0;
 143 }
 144
 145 int catalog_file_lang(const char* filename, char **lang) {
 146         char *beg, *end, *_lang;
 147
 148         end = endswith(filename, ".catalog");
 149         if (!end)
 150                 return 0;
 151
 152         beg = end - 1;
 153         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 154                 beg --;
 155
 156         if (*beg != '.' || end <= beg + 1)
 157                 return 0;
 158
 159         _lang = strndup(beg + 1, end - beg - 1);
 160         if (!_lang)
 161                 return -ENOMEM;
 162
 163         *lang = _lang;
 164         return 1;
 165 }
 166
 167 static int catalog_entry_lang(const char* filename, int line,
 168                               const char* t, const char* deflang, char **lang) {
 169         size_t c;
 170
 171         c = strlen(t);
 172         if (c == 0) {
 173                 log_error("[%s:%u] Language too short.", filename, line);
 174                 return -EINVAL;
 175         }
 176         if (c > 31) {
 177                 log_error("[%s:%u] language too long.", filename, line);
 178                 return -EINVAL;
 179         }
 180
 181         if (deflang) {
 182                 if (streq(t, deflang)) {
 183                         log_warning("[%s:%u] language specified unnecessarily",
 184                                     filename, line);
 185                         return 0;
 186                 } else
 187                         log_warning("[%s:%u] language differs from default for file",
 188                                     filename, line);
 189         }
 190
 191         *lang = strdup(t);
 192         if (!*lang)
 193                         return -ENOMEM;
 194
 195         return 0;
 196 }
 197
 198 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 199         _cleanup_fclose_ FILE *f = NULL;
 200         _cleanup_free_ char *payload = NULL;
 201         unsigned n = 0;
 202         sd_id128_t id;
 203         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 204         bool got_id = false, empty_line = true;
 205         int r;
 206
 207         assert(h);
 208         assert(sb);
 209         assert(path);
 210
 211         f = fopen(path, "re");
 212         if (!f) {
 213                 log_error("Failed to open file %s: %m", path);
 214                 return -errno;
 215         }
 216
 217         r = catalog_file_lang(path, &deflang);
 218         if (r < 0)
 219                 log_error("Failed to determine language for file %s: %m", path);
 220         if (r == 1)
 221                 log_debug("File %s has language %s.", path, deflang);
 222
 223         for (;;) {
 224                 char line[LINE_MAX];
 225                 size_t a, b, c;
 226                 char *t;
 227
 228                 if (!fgets(line, sizeof(line), f)) {
 229                         if (feof(f))
 230                                 break;
 231
 232                         log_error("Failed to read file %s: %m", path);
 233                         return -errno;
 234                 }
 235
 236                 n++;
 237
 238                 truncate_nl(line);
 239
 240                 if (line[0] == 0) {
 241                         empty_line = true;
 242                         continue;
 243                 }
 244
 245                 if (strchr(COMMENTS "\n", line[0]))
 246                         continue;
 247
 248                 if (empty_line &&
 249                     strlen(line) >= 2+1+32 &&
 250                     line[0] == '-' &&
 251                     line[1] == '-' &&
 252                     line[2] == ' ' &&
 253                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 254
 255                         bool with_language;
 256                         sd_id128_t jd;
 257
 258                         /* New entry */
 259
 260                         with_language = line[2+1+32] != '\0';
 261                         line[2+1+32] = '\0';
 262
 263                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 264
 265                                 if (got_id) {
 266                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 267                                         if (r < 0)
 268                                                 return r;
 269
 270                                         free(lang);
 271                                         lang = NULL;
 272                                 }
 273
 274                                 if (with_language) {
 275                                         t = strstrip(line + 2 + 1 + 32 + 1);
 276
 277                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 278                                         if (r < 0)
 279                                                 return r;
 280                                 }
 281
 282                                 got_id = true;
 283                                 empty_line = false;
 284                                 id = jd;
 285
 286                                 if (payload)
 287                                         payload[0] = '\0';
 288
 289                                 continue;
 290                         }
 291                 }
 292
 293                 /* Payload */
 294                 if (!got_id) {
 295                         log_error("[%s:%u] Got payload before ID.", path, n);
 296                         return -EINVAL;
 297                 }
 298
 299                 a = payload ? strlen(payload) : 0;
 300                 b = strlen(line);
 301
 302                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 303                 t = realloc(payload, c);
 304                 if (!t)
 305                         return log_oom();
 306
 307                 if (empty_line) {
 308                         t[a] = '\n';
 309                         memcpy(t + a + 1, line, b);
 310                         t[a+b+1] = '\n';
 311                         t[a+b+2] = 0;
 312                 } else {
 313                         memcpy(t + a, line, b);
 314                         t[a+b] = '\n';
 315                         t[a+b+1] = 0;
 316                 }
 317
 318                 payload = t;
 319                 empty_line = false;
 320         }
 321
 322         if (got_id) {
 323                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 324                 if (r < 0)
 325                         return r;
 326         }
 327
 328         return 0;
 329 }
 330
 331 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 332                           CatalogItem *items, size_t n) {
 333         CatalogHeader header;
 334         _cleanup_fclose_ FILE *w = NULL;
 335         int r;
 336         _cleanup_free_ char *d, *p = NULL;
 337         size_t k;
 338
 339         d = dirname_malloc(database);
 340         if (!d)
 341                 return log_oom();
 342
 343         r = mkdir_p(d, 0775);
 344         if (r < 0)
 345                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 346
 347         r = fopen_temporary(database, &w, &p);
 348         if (r < 0)
 349                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 350                                        database);
 351
 352         zero(header);
 353         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 354         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 355         header.catalog_item_size = htole64(sizeof(CatalogItem));
 356         header.n_items = htole64(hashmap_size(h));
 357
 358         r = -EIO;
 359
 360         k = fwrite(&header, 1, sizeof(header), w);
 361         if (k != sizeof(header)) {
 362                 log_error("%s: failed to write header.", p);
 363                 goto error;
 364         }
 365
 366         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 367         if (k != n * sizeof(CatalogItem)) {
 368                 log_error("%s: failed to write database.", p);
 369                 goto error;
 370         }
 371
 372         k = fwrite(sb->buf, 1, sb->len, w);
 373         if (k != sb->len) {
 374                 log_error("%s: failed to write strings.", p);
 375                 goto error;
 376         }
 377
 378         fflush(w);
 379
 380         if (ferror(w)) {
 381                 log_error("%s: failed to write database.", p);
 382                 goto error;
 383         }
 384
 385         fchmod(fileno(w), 0644);
 386
 387         if (rename(p, database) < 0) {
 388                 log_error("rename (%s -> %s) failed: %m", p, database);
 389                 r = -errno;
 390                 goto error;
 391         }
 392
 393         return ftell(w);
 394
 395 error:
 396         unlink(p);
 397         return r;
 398 }
 399
 400 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 401         _cleanup_strv_free_ char **files = NULL;
 402         char **f;
 403         struct strbuf *sb = NULL;
 404         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 405         _cleanup_free_ CatalogItem *items = NULL;
 406         CatalogItem *i;
 407         Iterator j;
 408         unsigned n;
 409         long r;
 410
 411         h = hashmap_new(&catalog_hash_ops);
 412         sb = strbuf_new();
 413
 414         if (!h || !sb) {
 415                 r = log_oom();
 416                 goto finish;
 417         }
 418
 419         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 420         if (r < 0) {
 421                 log_error_errno(r, "Failed to get catalog files: %m");
 422                 goto finish;
 423         }
 424
 425         STRV_FOREACH(f, files) {
 426                 log_debug("Reading file '%s'", *f);
 427                 r = catalog_import_file(h, sb, *f);
 428                 if (r < 0) {
 429                         log_error("Failed to import file '%s': %s.",
 430                                   *f, strerror(-r));
 431                         goto finish;
 432                 }
 433         }
 434
 435         if (hashmap_size(h) <= 0) {
 436                 log_info("No items in catalog.");
 437                 goto finish;
 438         } else
 439                 log_debug("Found %u items in catalog.", hashmap_size(h));
 440
 441         strbuf_complete(sb);
 442
 443         items = new(CatalogItem, hashmap_size(h));
 444         if (!items) {
 445                 r = log_oom();
 446                 goto finish;
 447         }
 448
 449         n = 0;
 450         HASHMAP_FOREACH(i, h, j) {
 451                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 452                           SD_ID128_FORMAT_VAL(i->id),
 453                           isempty(i->language) ? "C" : i->language);
 454                 items[n++] = *i;
 455         }
 456
 457         assert(n == hashmap_size(h));
 458         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 459
 460         r = write_catalog(database, h, sb, items, n);
 461         if (r < 0)
 462                 log_error_errno(r, "Failed to write %s: %m", database);
 463         else
 464                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 465                           database, n, sb->len, r);
 466
 467 finish:
 468         if (sb)
 469                 strbuf_cleanup(sb);
 470
 471         return r < 0 ? r : 0;
 472 }
 473
 474 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 475         const CatalogHeader *h;
 476         int fd;
 477         void *p;
 478         struct stat st;
 479
 480         assert(_fd);
 481         assert(_st);
 482         assert(_p);
 483
 484         fd = open(database, O_RDONLY|O_CLOEXEC);
 485         if (fd < 0)
 486                 return -errno;
 487
 488         if (fstat(fd, &st) < 0) {
 489                 safe_close(fd);
 490                 return -errno;
 491         }
 492
 493         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 494                 safe_close(fd);
 495                 return -EINVAL;
 496         }
 497
 498         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 499         if (p == MAP_FAILED) {
 500                 safe_close(fd);
 501                 return -errno;
 502         }
 503
 504         h = p;
 505         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 506             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 507             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 508             h->incompatible_flags != 0 ||
 509             le64toh(h->n_items) <= 0 ||
 510             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 511                 safe_close(fd);
 512                 munmap(p, st.st_size);
 513                 return -EBADMSG;
 514         }
 515
 516         *_fd = fd;
 517         *_st = st;
 518         *_p = p;
 519
 520         return 0;
 521 }
 522
 523 static const char *find_id(void *p, sd_id128_t id) {
 524         CatalogItem key, *f = NULL;
 525         const CatalogHeader *h = p;
 526         const char *loc;
 527
 528         zero(key);
 529         key.id = id;
 530
 531         loc = setlocale(LC_MESSAGES, NULL);
 532         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 533                 strncpy(key.language, loc, sizeof(key.language));
 534                 key.language[strcspn(key.language, ".@")] = 0;
 535
 536                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 537                 if (!f) {
 538                         char *e;
 539
 540                         e = strchr(key.language, '_');
 541                         if (e) {
 542                                 *e = 0;
 543                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 544                         }
 545                 }
 546         }
 547
 548         if (!f) {
 549                 zero(key.language);
 550                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 551         }
 552
 553         if (!f)
 554                 return NULL;
 555
 556         return (const char*) p +
 557                 le64toh(h->header_size) +
 558                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 559                 le64toh(f->offset);
 560 }
 561
 562 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 563         _cleanup_close_ int fd = -1;
 564         void *p = NULL;
 565         struct stat st;
 566         char *text = NULL;
 567         int r;
 568         const char *s;
 569
 570         assert(_text);
 571
 572         r = open_mmap(database, &fd, &st, &p);
 573         if (r < 0)
 574                 return r;
 575
 576         s = find_id(p, id);
 577         if (!s) {
 578                 r = -ENOENT;
 579                 goto finish;
 580         }
 581
 582         text = strdup(s);
 583         if (!text) {
 584                 r = -ENOMEM;
 585                 goto finish;
 586         }
 587
 588         *_text = text;
 589         r = 0;
 590
 591 finish:
 592         if (p)
 593                 munmap(p, st.st_size);
 594
 595         return r;
 596 }
 597
 598 static char *find_header(const char *s, const char *header) {
 599
 600         for (;;) {
 601                 const char *v, *e;
 602
 603                 v = startswith(s, header);
 604                 if (v) {
 605                         v += strspn(v, WHITESPACE);
 606                         return strndup(v, strcspn(v, NEWLINE));
 607                 }
 608
 609                 /* End of text */
 610                 e = strchr(s, '\n');
 611                 if (!e)
 612                         return NULL;
 613
 614                 /* End of header */
 615                 if (e == s)
 616                         return NULL;
 617
 618                 s = e + 1;
 619         }
 620 }
 621
 622 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 623         if (oneline) {
 624                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 625
 626                 subject = find_header(s, "Subject:");
 627                 defined_by = find_header(s, "Defined-By:");
 628
 629                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 630                         SD_ID128_FORMAT_VAL(id),
 631                         strna(defined_by), strna(subject));
 632         } else
 633                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 634                         SD_ID128_FORMAT_VAL(id), s);
 635 }
 636
 637
 638 int catalog_list(FILE *f, const char *database, bool oneline) {
 639         _cleanup_close_ int fd = -1;
 640         void *p = NULL;
 641         struct stat st;
 642         const CatalogHeader *h;
 643         const CatalogItem *items;
 644         int r;
 645         unsigned n;
 646         sd_id128_t last_id;
 647         bool last_id_set = false;
 648
 649         r = open_mmap(database, &fd, &st, &p);
 650         if (r < 0)
 651                 return r;
 652
 653         h = p;
 654         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 655
 656         for (n = 0; n < le64toh(h->n_items); n++) {
 657                 const char *s;
 658
 659                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 660                         continue;
 661
 662                 assert_se(s = find_id(p, items[n].id));
 663
 664                 dump_catalog_entry(f, items[n].id, s, oneline);
 665
 666                 last_id_set = true;
 667                 last_id = items[n].id;
 668         }
 669
 670         munmap(p, st.st_size);
 671
 672         return 0;
 673 }
 674
 675 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 676         char **item;
 677         int r = 0;
 678
 679         STRV_FOREACH(item, items) {
 680                 sd_id128_t id;
 681                 int k;
 682                 _cleanup_free_ char *msg = NULL;
 683
 684                 k = sd_id128_from_string(*item, &id);
 685                 if (k < 0) {
 686                         log_error_errno(k, "Failed to parse id128 '%s': %m",
 687                                         *item);
 688                         if (r == 0)
 689                                 r = k;
 690                         continue;
 691                 }
 692
 693                 k = catalog_get(database, id, &msg);
 694                 if (k < 0) {
 695                         log_full(k == -ENOENT ? LOG_NOTICE : LOG_ERR,
 696                                  "Failed to retrieve catalog entry for '%s': %s",
 697                                   *item, strerror(-k));
 698                         if (r == 0)
 699                                 r = k;
 700                         continue;
 701                 }
 702
 703                 dump_catalog_entry(f, id, msg, oneline);
 704         }
 705
 706         return r;
 707 }