src/journal/catalog.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2012 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <fcntl.h>
  23 #include <stdio.h>
  24 #include <unistd.h>
  25 #include <errno.h>
  26 #include <string.h>
  27 #include <sys/mman.h>
  28 #include <locale.h>
  29 #include <libgen.h>
  30
  31 #include "util.h"
  32 #include "log.h"
  33 #include "sparse-endian.h"
  34 #include "sd-id128.h"
  35 #include "hashmap.h"
  36 #include "strv.h"
  37 #include "strbuf.h"
  38 #include "strxcpyx.h"
  39 #include "conf-files.h"
  40 #include "mkdir.h"
  41 #include "catalog.h"
  42 #include "siphash24.h"
  43
  44 const char * const catalog_file_dirs[] = {
  45         "/usr/local/lib/systemd/catalog/",
  46         "/usr/lib/systemd/catalog/",
  47         NULL
  48 };
  49
  50 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  51
  52 typedef struct CatalogHeader {
  53         uint8_t signature[8];  /* "RHHHKSLP" */
  54         le32_t compatible_flags;
  55         le32_t incompatible_flags;
  56         le64_t header_size;
  57         le64_t n_items;
  58         le64_t catalog_item_size;
  59 } CatalogHeader;
  60
  61 typedef struct CatalogItem {
  62         sd_id128_t id;
  63         char language[32];
  64         le64_t offset;
  65 } CatalogItem;
  66
  67 static unsigned long catalog_hash_func(const void *p, const uint8_t hash_key[HASH_KEY_SIZE]) {
  68         const CatalogItem *i = p;
  69         uint64_t u;
  70         size_t l, sz;
  71         void *v;
  72
  73         l = strlen(i->language);
  74         sz = sizeof(i->id) + l;
  75         v = alloca(sz);
  76
  77         memcpy(mempcpy(v, &i->id, sizeof(i->id)), i->language, l);
  78
  79         siphash24((uint8_t*) &u, v, sz, hash_key);
  80
  81         return (unsigned long) u;
  82 }
  83
  84 static int catalog_compare_func(const void *a, const void *b) {
  85         const CatalogItem *i = a, *j = b;
  86         unsigned k;
  87
  88         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  89                  if (i->id.bytes[k] < j->id.bytes[k])
  90                         return -1;
  91                  if (i->id.bytes[k] > j->id.bytes[k])
  92                         return 1;
  93         }
  94
  95         return strcmp(i->language, j->language);
  96 }
  97
  98 const struct hash_ops catalog_hash_ops = {
  99         .hash = catalog_hash_func,
 100         .compare = catalog_compare_func
 101 };
 102
 103 static int finish_item(
 104                 Hashmap *h,
 105                 struct strbuf *sb,
 106                 sd_id128_t id,
 107                 const char *language,
 108                 const char *payload) {
 109
 110         ssize_t offset;
 111         _cleanup_free_ CatalogItem *i = NULL;
 112         int r;
 113
 114         assert(h);
 115         assert(sb);
 116         assert(payload);
 117
 118         offset = strbuf_add_string(sb, payload, strlen(payload));
 119         if (offset < 0)
 120                 return log_oom();
 121
 122         i = new0(CatalogItem, 1);
 123         if (!i)
 124                 return log_oom();
 125
 126         i->id = id;
 127         if (language) {
 128                 assert(strlen(language) > 1 && strlen(language) < 32);
 129                 strcpy(i->language, language);
 130         }
 131         i->offset = htole64((uint64_t) offset);
 132
 133         r = hashmap_put(h, i, i);
 134         if (r == -EEXIST) {
 135                 log_warning("Duplicate entry for " SD_ID128_FORMAT_STR ".%s, ignoring.",
 136                             SD_ID128_FORMAT_VAL(id), language ? language : "C");
 137                 return 0;
 138         } else if (r < 0)
 139                 return r;
 140
 141         i = NULL;
 142         return 0;
 143 }
 144
 145 int catalog_file_lang(const char* filename, char **lang) {
 146         char *beg, *end, *_lang;
 147
 148         end = endswith(filename, ".catalog");
 149         if (!end)
 150                 return 0;
 151
 152         beg = end - 1;
 153         while (beg > filename && *beg != '.' && *beg != '/' && end - beg < 32)
 154                 beg --;
 155
 156         if (*beg != '.' || end <= beg + 1)
 157                 return 0;
 158
 159         _lang = strndup(beg + 1, end - beg - 1);
 160         if (!_lang)
 161                 return -ENOMEM;
 162
 163         *lang = _lang;
 164         return 1;
 165 }
 166
 167 static int catalog_entry_lang(const char* filename, int line,
 168                               const char* t, const char* deflang, char **lang) {
 169         size_t c;
 170
 171         c = strlen(t);
 172         if (c == 0) {
 173                 log_error("[%s:%u] Language too short.", filename, line);
 174                 return -EINVAL;
 175         }
 176         if (c > 31) {
 177                 log_error("[%s:%u] language too long.", filename, line);
 178                 return -EINVAL;
 179         }
 180
 181         if (deflang) {
 182                 if (streq(t, deflang)) {
 183                         log_warning("[%s:%u] language specified unnecessarily",
 184                                     filename, line);
 185                         return 0;
 186                 } else
 187                         log_warning("[%s:%u] language differs from default for file",
 188                                     filename, line);
 189         }
 190
 191         *lang = strdup(t);
 192         if (!*lang)
 193                         return -ENOMEM;
 194
 195         return 0;
 196 }
 197
 198 int catalog_import_file(Hashmap *h, struct strbuf *sb, const char *path) {
 199         _cleanup_fclose_ FILE *f = NULL;
 200         _cleanup_free_ char *payload = NULL;
 201         unsigned n = 0;
 202         sd_id128_t id;
 203         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 204         bool got_id = false, empty_line = true;
 205         int r;
 206
 207         assert(h);
 208         assert(sb);
 209         assert(path);
 210
 211         f = fopen(path, "re");
 212         if (!f)
 213                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 214
 215         r = catalog_file_lang(path, &deflang);
 216         if (r < 0)
 217                 log_error_errno(errno, "Failed to determine language for file %s: %m", path);
 218         if (r == 1)
 219                 log_debug("File %s has language %s.", path, deflang);
 220
 221         for (;;) {
 222                 char line[LINE_MAX];
 223                 size_t a, b, c;
 224                 char *t;
 225
 226                 if (!fgets(line, sizeof(line), f)) {
 227                         if (feof(f))
 228                                 break;
 229
 230                         log_error_errno(errno, "Failed to read file %s: %m", path);
 231                         return -errno;
 232                 }
 233
 234                 n++;
 235
 236                 truncate_nl(line);
 237
 238                 if (line[0] == 0) {
 239                         empty_line = true;
 240                         continue;
 241                 }
 242
 243                 if (strchr(COMMENTS "\n", line[0]))
 244                         continue;
 245
 246                 if (empty_line &&
 247                     strlen(line) >= 2+1+32 &&
 248                     line[0] == '-' &&
 249                     line[1] == '-' &&
 250                     line[2] == ' ' &&
 251                     (line[2+1+32] == ' ' || line[2+1+32] == '\0')) {
 252
 253                         bool with_language;
 254                         sd_id128_t jd;
 255
 256                         /* New entry */
 257
 258                         with_language = line[2+1+32] != '\0';
 259                         line[2+1+32] = '\0';
 260
 261                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 262
 263                                 if (got_id) {
 264                                         r = finish_item(h, sb, id, lang ?: deflang, payload);
 265                                         if (r < 0)
 266                                                 return r;
 267
 268                                         free(lang);
 269                                         lang = NULL;
 270                                 }
 271
 272                                 if (with_language) {
 273                                         t = strstrip(line + 2 + 1 + 32 + 1);
 274
 275                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 276                                         if (r < 0)
 277                                                 return r;
 278                                 }
 279
 280                                 got_id = true;
 281                                 empty_line = false;
 282                                 id = jd;
 283
 284                                 if (payload)
 285                                         payload[0] = '\0';
 286
 287                                 continue;
 288                         }
 289                 }
 290
 291                 /* Payload */
 292                 if (!got_id) {
 293                         log_error("[%s:%u] Got payload before ID.", path, n);
 294                         return -EINVAL;
 295                 }
 296
 297                 a = payload ? strlen(payload) : 0;
 298                 b = strlen(line);
 299
 300                 c = a + (empty_line ? 1 : 0) + b + 1 + 1;
 301                 t = realloc(payload, c);
 302                 if (!t)
 303                         return log_oom();
 304
 305                 if (empty_line) {
 306                         t[a] = '\n';
 307                         memcpy(t + a + 1, line, b);
 308                         t[a+b+1] = '\n';
 309                         t[a+b+2] = 0;
 310                 } else {
 311                         memcpy(t + a, line, b);
 312                         t[a+b] = '\n';
 313                         t[a+b+1] = 0;
 314                 }
 315
 316                 payload = t;
 317                 empty_line = false;
 318         }
 319
 320         if (got_id) {
 321                 r = finish_item(h, sb, id, lang ?: deflang, payload);
 322                 if (r < 0)
 323                         return r;
 324         }
 325
 326         return 0;
 327 }
 328
 329 static long write_catalog(const char *database, Hashmap *h, struct strbuf *sb,
 330                           CatalogItem *items, size_t n) {
 331         CatalogHeader header;
 332         _cleanup_fclose_ FILE *w = NULL;
 333         int r;
 334         _cleanup_free_ char *d, *p = NULL;
 335         size_t k;
 336
 337         d = dirname_malloc(database);
 338         if (!d)
 339                 return log_oom();
 340
 341         r = mkdir_p(d, 0775);
 342         if (r < 0)
 343                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 344
 345         r = fopen_temporary(database, &w, &p);
 346         if (r < 0)
 347                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 348                                        database);
 349
 350         zero(header);
 351         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 352         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 353         header.catalog_item_size = htole64(sizeof(CatalogItem));
 354         header.n_items = htole64(hashmap_size(h));
 355
 356         r = -EIO;
 357
 358         k = fwrite(&header, 1, sizeof(header), w);
 359         if (k != sizeof(header)) {
 360                 log_error("%s: failed to write header.", p);
 361                 goto error;
 362         }
 363
 364         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 365         if (k != n * sizeof(CatalogItem)) {
 366                 log_error("%s: failed to write database.", p);
 367                 goto error;
 368         }
 369
 370         k = fwrite(sb->buf, 1, sb->len, w);
 371         if (k != sb->len) {
 372                 log_error("%s: failed to write strings.", p);
 373                 goto error;
 374         }
 375
 376         fflush(w);
 377
 378         if (ferror(w)) {
 379                 log_error("%s: failed to write database.", p);
 380                 goto error;
 381         }
 382
 383         fchmod(fileno(w), 0644);
 384
 385         if (rename(p, database) < 0) {
 386                 log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 387                 r = -errno;
 388                 goto error;
 389         }
 390
 391         return ftell(w);
 392
 393 error:
 394         unlink(p);
 395         return r;
 396 }
 397
 398 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 399         _cleanup_strv_free_ char **files = NULL;
 400         char **f;
 401         struct strbuf *sb = NULL;
 402         _cleanup_hashmap_free_free_ Hashmap *h = NULL;
 403         _cleanup_free_ CatalogItem *items = NULL;
 404         CatalogItem *i;
 405         Iterator j;
 406         unsigned n;
 407         long r;
 408
 409         h = hashmap_new(&catalog_hash_ops);
 410         sb = strbuf_new();
 411
 412         if (!h || !sb) {
 413                 r = log_oom();
 414                 goto finish;
 415         }
 416
 417         r = conf_files_list_strv(&files, ".catalog", root, dirs);
 418         if (r < 0) {
 419                 log_error_errno(r, "Failed to get catalog files: %m");
 420                 goto finish;
 421         }
 422
 423         STRV_FOREACH(f, files) {
 424                 log_debug("Reading file '%s'", *f);
 425                 r = catalog_import_file(h, sb, *f);
 426                 if (r < 0) {
 427                         log_error("Failed to import file '%s': %s.",
 428                                   *f, strerror(-r));
 429                         goto finish;
 430                 }
 431         }
 432
 433         if (hashmap_size(h) <= 0) {
 434                 log_info("No items in catalog.");
 435                 goto finish;
 436         } else
 437                 log_debug("Found %u items in catalog.", hashmap_size(h));
 438
 439         strbuf_complete(sb);
 440
 441         items = new(CatalogItem, hashmap_size(h));
 442         if (!items) {
 443                 r = log_oom();
 444                 goto finish;
 445         }
 446
 447         n = 0;
 448         HASHMAP_FOREACH(i, h, j) {
 449                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 450                           SD_ID128_FORMAT_VAL(i->id),
 451                           isempty(i->language) ? "C" : i->language);
 452                 items[n++] = *i;
 453         }
 454
 455         assert(n == hashmap_size(h));
 456         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 457
 458         r = write_catalog(database, h, sb, items, n);
 459         if (r < 0)
 460                 log_error_errno(r, "Failed to write %s: %m", database);
 461         else
 462                 log_debug("%s: wrote %u items, with %zu bytes of strings, %ld total size.",
 463                           database, n, sb->len, r);
 464
 465 finish:
 466         if (sb)
 467                 strbuf_cleanup(sb);
 468
 469         return r < 0 ? r : 0;
 470 }
 471
 472 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 473         const CatalogHeader *h;
 474         int fd;
 475         void *p;
 476         struct stat st;
 477
 478         assert(_fd);
 479         assert(_st);
 480         assert(_p);
 481
 482         fd = open(database, O_RDONLY|O_CLOEXEC);
 483         if (fd < 0)
 484                 return -errno;
 485
 486         if (fstat(fd, &st) < 0) {
 487                 safe_close(fd);
 488                 return -errno;
 489         }
 490
 491         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 492                 safe_close(fd);
 493                 return -EINVAL;
 494         }
 495
 496         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 497         if (p == MAP_FAILED) {
 498                 safe_close(fd);
 499                 return -errno;
 500         }
 501
 502         h = p;
 503         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 504             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 505             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 506             h->incompatible_flags != 0 ||
 507             le64toh(h->n_items) <= 0 ||
 508             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 509                 safe_close(fd);
 510                 munmap(p, st.st_size);
 511                 return -EBADMSG;
 512         }
 513
 514         *_fd = fd;
 515         *_st = st;
 516         *_p = p;
 517
 518         return 0;
 519 }
 520
 521 static const char *find_id(void *p, sd_id128_t id) {
 522         CatalogItem key, *f = NULL;
 523         const CatalogHeader *h = p;
 524         const char *loc;
 525
 526         zero(key);
 527         key.id = id;
 528
 529         loc = setlocale(LC_MESSAGES, NULL);
 530         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 531                 strncpy(key.language, loc, sizeof(key.language));
 532                 key.language[strcspn(key.language, ".@")] = 0;
 533
 534                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 535                 if (!f) {
 536                         char *e;
 537
 538                         e = strchr(key.language, '_');
 539                         if (e) {
 540                                 *e = 0;
 541                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 542                         }
 543                 }
 544         }
 545
 546         if (!f) {
 547                 zero(key.language);
 548                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 549         }
 550
 551         if (!f)
 552                 return NULL;
 553
 554         return (const char*) p +
 555                 le64toh(h->header_size) +
 556                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 557                 le64toh(f->offset);
 558 }
 559
 560 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 561         _cleanup_close_ int fd = -1;
 562         void *p = NULL;
 563         struct stat st;
 564         char *text = NULL;
 565         int r;
 566         const char *s;
 567
 568         assert(_text);
 569
 570         r = open_mmap(database, &fd, &st, &p);
 571         if (r < 0)
 572                 return r;
 573
 574         s = find_id(p, id);
 575         if (!s) {
 576                 r = -ENOENT;
 577                 goto finish;
 578         }
 579
 580         text = strdup(s);
 581         if (!text) {
 582                 r = -ENOMEM;
 583                 goto finish;
 584         }
 585
 586         *_text = text;
 587         r = 0;
 588
 589 finish:
 590         if (p)
 591                 munmap(p, st.st_size);
 592
 593         return r;
 594 }
 595
 596 static char *find_header(const char *s, const char *header) {
 597
 598         for (;;) {
 599                 const char *v, *e;
 600
 601                 v = startswith(s, header);
 602                 if (v) {
 603                         v += strspn(v, WHITESPACE);
 604                         return strndup(v, strcspn(v, NEWLINE));
 605                 }
 606
 607                 /* End of text */
 608                 e = strchr(s, '\n');
 609                 if (!e)
 610                         return NULL;
 611
 612                 /* End of header */
 613                 if (e == s)
 614                         return NULL;
 615
 616                 s = e + 1;
 617         }
 618 }
 619
 620 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 621         if (oneline) {
 622                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 623
 624                 subject = find_header(s, "Subject:");
 625                 defined_by = find_header(s, "Defined-By:");
 626
 627                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 628                         SD_ID128_FORMAT_VAL(id),
 629                         strna(defined_by), strna(subject));
 630         } else
 631                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 632                         SD_ID128_FORMAT_VAL(id), s);
 633 }
 634
 635
 636 int catalog_list(FILE *f, const char *database, bool oneline) {
 637         _cleanup_close_ int fd = -1;
 638         void *p = NULL;
 639         struct stat st;
 640         const CatalogHeader *h;
 641         const CatalogItem *items;
 642         int r;
 643         unsigned n;
 644         sd_id128_t last_id;
 645         bool last_id_set = false;
 646
 647         r = open_mmap(database, &fd, &st, &p);
 648         if (r < 0)
 649                 return r;
 650
 651         h = p;
 652         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 653
 654         for (n = 0; n < le64toh(h->n_items); n++) {
 655                 const char *s;
 656
 657                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 658                         continue;
 659
 660                 assert_se(s = find_id(p, items[n].id));
 661
 662                 dump_catalog_entry(f, items[n].id, s, oneline);
 663
 664                 last_id_set = true;
 665                 last_id = items[n].id;
 666         }
 667
 668         munmap(p, st.st_size);
 669
 670         return 0;
 671 }
 672
 673 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 674         char **item;
 675         int r = 0;
 676
 677         STRV_FOREACH(item, items) {
 678                 sd_id128_t id;
 679                 int k;
 680                 _cleanup_free_ char *msg = NULL;
 681
 682                 k = sd_id128_from_string(*item, &id);
 683                 if (k < 0) {
 684                         log_error_errno(k, "Failed to parse id128 '%s': %m",
 685                                         *item);
 686                         if (r == 0)
 687                                 r = k;
 688                         continue;
 689                 }
 690
 691                 k = catalog_get(database, id, &msg);
 692                 if (k < 0) {
 693                         log_full(k == -ENOENT ? LOG_NOTICE : LOG_ERR,
 694                                  "Failed to retrieve catalog entry for '%s': %s",
 695                                   *item, strerror(-k));
 696                         if (r == 0)
 697                                 r = k;
 698                         continue;
 699                 }
 700
 701                 dump_catalog_entry(f, id, msg, oneline);
 702         }
 703
 704         return r;
 705 }