src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <locale.h>
  25 #include <stdio.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <unistd.h>
  29
  30 #include "sd-id128.h"
  31
  32 #include "catalog.h"
  33 #include "conf-files.h"
  34 #include "hashmap.h"
  35 #include "log.h"
  36 #include "mkdir.h"
  37 #include "siphash24.h"
  38 #include "sparse-endian.h"
  39 #include "strbuf.h"
  40 #include "string-util.h"
  41 #include "strv.h"
  42 #include "util.h"
  43
  44 const char * const catalog_file_dirs[] = {
  45         "/usr/local/lib/systemd/catalog/",
  46         "/usr/lib/systemd/catalog/",
  47         NULL
  48 };
  49
  50 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  51
  52 typedef struct CatalogHeader {
  53         uint8_t signature[8];  /* "RHHHKSLP" */
  54         le32_t compatible_flags;
  55         le32_t incompatible_flags;
  56         le64_t header_size;
  57         le64_t n_items;
  58         le64_t catalog_item_size;
  59 } CatalogHeader;
  60
  61 typedef struct CatalogItem {
  62         sd_id128_t id;
  63         char language[32];
  64         le64_t offset;
  65 } CatalogItem;
  66
  67 static void catalog_hash_func(const void *p, struct siphash *state) {
  68         const CatalogItem *i = p;
  69
  70         siphash24_compress(&i->id, sizeof(i->id), state);
  71         siphash24_compress(i->language, strlen(i->language), state);
  72 }
  73
  74 static int catalog_compare_func(const void *a, const void *b) {
  75         const CatalogItem *i = a, *j = b;
  76         unsigned k;
  77
  78         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  79                  if (i->id.bytes[k] < j->id.bytes[k])
  80                         return -1;
  81                  if (i->id.bytes[k] > j->id.bytes[k])
  82                         return 1;
  83         }
  84
  85         return strcmp(i->language, j->language);
  86 }
  87
  88 const struct hash_ops catalog_hash_ops = {
  89         .hash = catalog_hash_func,
  90         .compare = catalog_compare_func
  91 };
  92
  93 static int finish_item(
  94                 Hashmap *h,
  95                 struct strbuf *sb,
  96                 sd_id128_t id,
  97                 const char *language,
  98                 const char *payload) {
  99
 100         ssize_t offset;
 101         _cleanup_free_ CatalogItem *i = NULL;
 102         int r;
 103
 104         assert(h);
 105         assert(sb);
 106         assert(payload);
 107
 108         offset = strbuf_add_string(sb, payload, strlen(payload));
 109         if (offset < 0)
 110                 return log_oom();
 111
 112         i = new0(CatalogItem, 1);
 113         if (!i)
 114                 return log_oom();
 115
 116         i->id = id;
 117         if (language) {
 118                 assert(strlen(language) > 1 && strlen(language) < 32);
 119                 strcpy(i->language, language);
 120         }
 121         i->offset = htole64((uint64_t) offset);
 122
 123         r = hashmap_put(h, i, i);
 124         if (r == -EEXIST) {
 125                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 126                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 127                 return 0;
 128         } else if (r < 0)
 129                 return r;
 130
 131         i = NULL;
 132         return 0;
 133 }
 134
 135 int catalog_file_lang(const char* filename, char **lang) {
 136         char *beg, *end, *_lang;
 137
 138         end = endswith(filename, ".catalog");
 139         if (!end)
 140                 return 0;
 141
 142         beg = end - 1;
 143         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 144                 beg --;
 145
 146         if (*beg != '.' || end <= beg + 1)
 147                 return 0;
 148
 149         _lang = strndup(beg + 1, end - beg - 1);
 150         if (!_lang)
 151                 return -ENOMEM;
 152
 153         *lang = _lang;
 154         return 1;
 155 }
 156
 157 static int catalog_entry_lang(const char* filename, int line,
 158                               const char* t, const char* deflang, char **lang) {
 159         size_t c;
 160
 161         c = strlen(t);
 162         if (c == 0) {
 163                 log_error("[%s:%u] Language too short.", filename, line);
 164                 return -EINVAL;
 165         }
 166         if (c > 31) {
 167                 log_error("[%s:%u] language too long.", filename, line);
 168                 return -EINVAL;
 169         }
 170
 171         if (deflang) {
 172                 if (streq(t, deflang)) {
 173                         log_warning("[%s:%u] language specified unnecessarily",
 174                                     filename, line);
 175                         return 0;
 176                 } else
 177                         log_warning("[%s:%u] language differs from default for file",
 178                                     filename, line);
 179         }
 180
 181         *lang = strdup(t);
 182         if (!*lang)
 183                         return -ENOMEM;
 184
 185         return 0;
 186 }
 187
 188 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 189         _cleanup_fclose_ FILE *f = NULL;
 190         _cleanup_free_ char *payload = NULL;
 191         unsigned n = 0;
 192         sd_id128_t id;
 193         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 194         bool got_id = false, empty_line = true;
 195         int r;
 196
 197         assert(h);
 198         assert(sb);
 199         assert(path);
 200
 201         f = fopen(path, "re");
 202         if (!f)
 203                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 204
 205         r = catalog_file_lang(path, &deflang);
 206         if (r < 0)
 207                 log_error_errno(errno, "Failed to determine language for file %s: %m", path);
 208         if (r == 1)
 209                 log_debug("File %s has language %s.", path, deflang);
 210
 211         for (;;) {
 212                 char line[LINE_MAX];
 213                 size_t a, b, c;
 214                 char *t;
 215
 216                 if (!fgets(line, sizeof(line), f)) {
 217                         if (feof(f))
 218                                 break;
 219
 220                         log_error_errno(errno, "Failed to read file %s: %m", path);
 221                         return -errno;
 222                 }
 223
 224                 n++;
 225
 226                 truncate_nl(line);
 227
 228                 if (line[0] == 0) {
 229                         empty_line = true;
 230                         continue;
 231                 }
 232
 233                 if (strchr(COMMENTS "\n", line[0]))
 234                         continue;
 235
 236                 if (empty_line &&
 237                     strlen(line) >= 2+1+32 &&
 238                     line[0] == '-' &&
 239                     line[1] == '-' &&
 240                     line[2] == ' ' &&
 241                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 242
 243                         bool with_language;
 244                         sd_id128_t jd;
 245
 246                         /* New entry */
 247
 248                         with_language = line[2+1+32] != '\0';
 249                         line[2+1+32] = '\0';
 250
 251                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 252
 253                                 if (got_id) {
 254                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 255                                         if (r < 0)
 256                                                 return r;
 257
 258                                         lang = mfree(lang);
 259                                 }
 260
 261                                 if (with_language) {
 262                                         t = strstrip(line + 2 + 1 + 32 + 1);
 263
 264                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 265                                         if (r < 0)
 266                                                 return r;
 267                                 }
 268
 269                                 got_id = true;
 270                                 empty_line = false;
 271                                 id = jd;
 272
 273                                 if (payload)
 274                                         payload[0] = '\0';
 275
 276                                 continue;
 277                         }
 278                 }
 279
 280                 /* Payload */
 281                 if (!got_id) {
 282                         log_error("[%s:%u] Got payload before ID.", path, n);
 283                         return -EINVAL;
 284                 }
 285
 286                 a = payload ? strlen(payload) : 0;
 287                 b = strlen(line);
 288
 289                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 290                 t = realloc(payload, c);
 291                 if (!t)
 292                         return log_oom();
 293
 294                 if (empty_line) {
 295                         t[a] = '\n';
 296                         memcpy(t + a + 1, line, b);
 297                         t[a+b+1] = '\n';
 298                         t[a+b+2] = 0;
 299                 } else {
 300                         memcpy(t + a, line, b);
 301                         t[a+b] = '\n';
 302                         t[a+b+1] = 0;
 303                 }
 304
 305                 payload = t;
 306                 empty_line = false;
 307         }
 308
 309         if (got_id) {
 310                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 311                 if (r < 0)
 312                         return r;
 313         }
 314
 315         return 0;
 316 }
 317
 318 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 319                           CatalogItem *items, size_t n) {
 320         CatalogHeader header;
 321         _cleanup_fclose_ FILE *w = NULL;
 322         int r;
 323         _cleanup_free_ char *d, *p = NULL;
 324         size_t k;
 325
 326         d = dirname_malloc(database);
 327         if (!d)
 328                 return log_oom();
 329
 330         r = mkdir_p(d, 0775);
 331         if (r < 0)
 332                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 333
 334         r = fopen_temporary(database, &w, &p);
 335         if (r < 0)
 336                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 337                                        database);
 338
 339         zero(header);
 340         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 341         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 342         header.catalog_item_size = htole64(sizeof(CatalogItem));
 343         header.n_items = htole64(hashmap_size(h));
 344
 345         r = -EIO;
 346
 347         k = fwrite(&header, 1, sizeof(header), w);
 348         if (k != sizeof(header)) {
 349                 log_error("%s: failed to write header.", p);
 350                 goto error;
 351         }
 352
 353         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 354         if (k != n * sizeof(CatalogItem)) {
 355                 log_error("%s: failed to write database.", p);
 356                 goto error;
 357         }
 358
 359         k = fwrite(sb->buf, 1, sb->len, w);
 360         if (k != sb->len) {
 361                 log_error("%s: failed to write strings.", p);
 362                 goto error;
 363         }
 364
 365         r = fflush_and_check(w);
 366         if (r < 0) {
 367                 log_error_errno(r, "%s: failed to write database: %m", p);
 368                 goto error;
 369         }
 370
 371         fchmod(fileno(w), 0644);
 372
 373         if (rename(p, database) < 0) {
 374                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 375                 goto error;
 376         }
 377
 378         return ftell(w);
 379
 380 error:
 381         (void) unlink(p);
 382         return r;
 383 }
 384
 385 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 386         _cleanup_strv_free_ char **files = NULL;
 387         char **f;
 388         struct strbuf *sb = NULL;
 389         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 390         _cleanup_free_ CatalogItem *items = NULL;
 391         CatalogItem *i;
 392         Iterator j;
 393         unsigned n;
 394         long r;
 395
 396         h = hashmap_new(&catalog_hash_ops);
 397         sb = strbuf_new();
 398
 399         if (!h || !sb) {
 400                 r = log_oom();
 401                 goto finish;
 402         }
 403
 404         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 405         if (r < 0) {
 406                 log_error_errno(r, "Failed to get catalog files: %m");
 407                 goto finish;
 408         }
 409
 410         STRV_FOREACH(f, files) {
 411                 log_debug("Reading file '%s'", *f);
 412                 r = catalog_import_file(h, sb, *f);
 413                 if (r < 0) {
 414                         log_error_errno(r, "Failed to import file '%s': %m", *f);
 415                         goto finish;
 416                 }
 417         }
 418
 419         if (hashmap_size(h) <= 0) {
 420                 log_info("No items in catalog.");
 421                 goto finish;
 422         } else
 423                 log_debug("Found %u items in catalog.", hashmap_size(h));
 424
 425         strbuf_complete(sb);
 426
 427         items = new(CatalogItem, hashmap_size(h));
 428         if (!items) {
 429                 r = log_oom();
 430                 goto finish;
 431         }
 432
 433         n = 0;
 434         HASHMAP_FOREACH(i, h, j) {
 435                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 436                           SD_ID128_FORMAT_VAL(i->id),
 437                           isempty(i->language) ? "C" : i->language);
 438                 items[n++] = *i;
 439         }
 440
 441         assert(n == hashmap_size(h));
 442         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 443
 444         r = write_catalog(database, h, sb, items, n);
 445         if (r < 0)
 446                 log_error_errno(r, "Failed to write %s: %m", database);
 447         else
 448                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 449                           database, n, sb->len, r);
 450
 451 finish:
 452         if (sb)
 453                 strbuf_cleanup(sb);
 454
 455         return r < 0 ? r : 0;
 456 }
 457
 458 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 459         const CatalogHeader *h;
 460         int fd;
 461         void *p;
 462         struct stat st;
 463
 464         assert(_fd);
 465         assert(_st);
 466         assert(_p);
 467
 468         fd = open(database, O_RDONLY|O_CLOEXEC);
 469         if (fd < 0)
 470                 return -errno;
 471
 472         if (fstat(fd, &st) < 0) {
 473                 safe_close(fd);
 474                 return -errno;
 475         }
 476
 477         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 478                 safe_close(fd);
 479                 return -EINVAL;
 480         }
 481
 482         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 483         if (p == MAP_FAILED) {
 484                 safe_close(fd);
 485                 return -errno;
 486         }
 487
 488         h = p;
 489         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 490             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 491             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 492             h->incompatible_flags != 0 ||
 493             le64toh(h->n_items) <= 0 ||
 494             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 495                 safe_close(fd);
 496                 munmap(p, st.st_size);
 497                 return -EBADMSG;
 498         }
 499
 500         *_fd = fd;
 501         *_st = st;
 502         *_p = p;
 503
 504         return 0;
 505 }
 506
 507 static const char *find_id(void *p, sd_id128_t id) {
 508         CatalogItem key, *f = NULL;
 509         const CatalogHeader *h = p;
 510         const char *loc;
 511
 512         zero(key);
 513         key.id = id;
 514
 515         loc = setlocale(LC_MESSAGES, NULL);
 516         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 517                 strncpy(key.language, loc, sizeof(key.language));
 518                 key.language[strcspn(key.language, ".@")] = 0;
 519
 520                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 521                 if (!f) {
 522                         char *e;
 523
 524                         e = strchr(key.language, '_');
 525                         if (e) {
 526                                 *e = 0;
 527                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 528                         }
 529                 }
 530         }
 531
 532         if (!f) {
 533                 zero(key.language);
 534                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 535         }
 536
 537         if (!f)
 538                 return NULL;
 539
 540         return (const char*) p +
 541                 le64toh(h->header_size) +
 542                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 543                 le64toh(f->offset);
 544 }
 545
 546 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 547         _cleanup_close_ int fd = -1;
 548         void *p = NULL;
 549         struct stat st = {};
 550         char *text = NULL;
 551         int r;
 552         const char *s;
 553
 554         assert(_text);
 555
 556         r = open_mmap(database, &fd, &st, &p);
 557         if (r < 0)
 558                 return r;
 559
 560         s = find_id(p, id);
 561         if (!s) {
 562                 r = -ENOENT;
 563                 goto finish;
 564         }
 565
 566         text = strdup(s);
 567         if (!text) {
 568                 r = -ENOMEM;
 569                 goto finish;
 570         }
 571
 572         *_text = text;
 573         r = 0;
 574
 575 finish:
 576         if (p)
 577                 munmap(p, st.st_size);
 578
 579         return r;
 580 }
 581
 582 static char *find_header(const char *s, const char *header) {
 583
 584         for (;;) {
 585                 const char *v, *e;
 586
 587                 v = startswith(s, header);
 588                 if (v) {
 589                         v += strspn(v, WHITESPACE);
 590                         return strndup(v, strcspn(v, NEWLINE));
 591                 }
 592
 593                 /* End of text */
 594                 e = strchr(s, '\n');
 595                 if (!e)
 596                         return NULL;
 597
 598                 /* End of header */
 599                 if (e == s)
 600                         return NULL;
 601
 602                 s = e + 1;
 603         }
 604 }
 605
 606 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 607         if (oneline) {
 608                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 609
 610                 subject = find_header(s, "Subject:");
 611                 defined_by = find_header(s, "Defined-By:");
 612
 613                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 614                         SD_ID128_FORMAT_VAL(id),
 615                         strna(defined_by), strna(subject));
 616         } else
 617                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 618                         SD_ID128_FORMAT_VAL(id), s);
 619 }
 620
 621
 622 int catalog_list(FILE *f, const char *database, bool oneline) {
 623         _cleanup_close_ int fd = -1;
 624         void *p = NULL;
 625         struct stat st;
 626         const CatalogHeader *h;
 627         const CatalogItem *items;
 628         int r;
 629         unsigned n;
 630         sd_id128_t last_id;
 631         bool last_id_set = false;
 632
 633         r = open_mmap(database, &fd, &st, &p);
 634         if (r < 0)
 635                 return r;
 636
 637         h = p;
 638         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 639
 640         for (n = 0; n < le64toh(h->n_items); n++) {
 641                 const char *s;
 642
 643                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 644                         continue;
 645
 646                 assert_se(s = find_id(p, items[n].id));
 647
 648                 dump_catalog_entry(f, items[n].id, s, oneline);
 649
 650                 last_id_set = true;
 651                 last_id = items[n].id;
 652         }
 653
 654         munmap(p, st.st_size);
 655
 656         return 0;
 657 }
 658
 659 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 660         char **item;
 661         int r = 0;
 662
 663         STRV_FOREACH(item, items) {
 664                 sd_id128_t id;
 665                 int k;
 666                 _cleanup_free_ char *msg = NULL;
 667
 668                 k = sd_id128_from_string(*item, &id);
 669                 if (k < 0) {
 670                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 671                         if (r == 0)
 672                                 r = k;
 673                         continue;
 674                 }
 675
 676                 k = catalog_get(database, id, &msg);
 677                 if (k < 0) {
 678                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 679                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 680                         if (r == 0)
 681                                 r = k;
 682                         continue;
 683                 }
 684
 685                 dump_catalog_entry(f, id, msg, oneline);
 686         }
 687
 688         return r;
 689 }