src/journal/catalog.c

   1 /***
   2   This file is part of systemd.
   3
   4   Copyright 2012 Lennart Poettering
   5
   6   systemd is free software; you can redistribute it and/or modify it
   7   under the terms of the GNU Lesser General Public License as published by
   8   the Free Software Foundation; either version 2.1 of the License, or
   9   (at your option) any later version.
  10
  11   systemd is distributed in the hope that it will be useful, but
  12   WITHOUT ANY WARRANTY; without even the implied warranty of
  13   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14   Lesser General Public License for more details.
  15
  16   You should have received a copy of the GNU Lesser General Public License
  17   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  18 ***/
  19
  20 #include <errno.h>
  21 #include <fcntl.h>
  22 #include <locale.h>
  23 #include <stdio.h>
  24 #include <string.h>
  25 #include <sys/mman.h>
  26 #include <unistd.h>
  27
  28 #include "sd-id128.h"
  29
  30 #include "alloc-util.h"
  31 #include "catalog.h"
  32 #include "conf-files.h"
  33 #include "fd-util.h"
  34 #include "fileio.h"
  35 #include "hashmap.h"
  36 #include "log.h"
  37 #include "mkdir.h"
  38 #include "path-util.h"
  39 #include "siphash24.h"
  40 #include "sparse-endian.h"
  41 #include "strbuf.h"
  42 #include "string-util.h"
  43 #include "strv.h"
  44 #include "util.h"
  45
  46 const char * const catalog_file_dirs[] = {
  47         "/usr/local/lib/systemd/catalog/",
  48         "/usr/lib/systemd/catalog/",
  49         NULL
  50 };
  51
  52 #define CATALOG_SIGNATURE (uint8_t[]) { 'R', 'H', 'H', 'H', 'K', 'S', 'L', 'P' }
  53
  54 typedef struct CatalogHeader {
  55         uint8_t signature[8];  /* "RHHHKSLP" */
  56         le32_t compatible_flags;
  57         le32_t incompatible_flags;
  58         le64_t header_size;
  59         le64_t n_items;
  60         le64_t catalog_item_size;
  61 } CatalogHeader;
  62
  63 typedef struct CatalogItem {
  64         sd_id128_t id;
  65         char language[32];
  66         le64_t offset;
  67 } CatalogItem;
  68
  69 static void catalog_hash_func(const void *p, struct siphash *state) {
  70         const CatalogItem *i = p;
  71
  72         siphash24_compress(&i->id, sizeof(i->id), state);
  73         siphash24_compress(i->language, strlen(i->language), state);
  74 }
  75
  76 static int catalog_compare_func(const void *a, const void *b) {
  77         const CatalogItem *i = a, *j = b;
  78         unsigned k;
  79
  80         for (k = 0; k < ELEMENTSOF(j->id.bytes); k++) {
  81                  if (i->id.bytes[k] < j->id.bytes[k])
  82                         return -1;
  83                  if (i->id.bytes[k] > j->id.bytes[k])
  84                         return 1;
  85         }
  86
  87         return strcmp(i->language, j->language);
  88 }
  89
  90 const struct hash_ops catalog_hash_ops = {
  91         .hash = catalog_hash_func,
  92         .compare = catalog_compare_func
  93 };
  94
  95 static bool next_header(const char **s) {
  96         const char *e;
  97
  98         e = strchr(*s, '\n');
  99
 100         /* Unexpected end */
 101         if (!e)
 102                 return false;
 103
 104         /* End of headers */
 105         if (e == *s)
 106                 return false;
 107
 108         *s = e + 1;
 109         return true;
 110 }
 111
 112 static const char *skip_header(const char *s) {
 113         while (next_header(&s))
 114                 ;
 115         return s;
 116 }
 117
 118 static char *combine_entries(const char *one, const char *two) {
 119         const char *b1, *b2;
 120         size_t l1, l2, n;
 121         char *dest, *p;
 122
 123         /* Find split point of headers to body */
 124         b1 = skip_header(one);
 125         b2 = skip_header(two);
 126
 127         l1 = strlen(one);
 128         l2 = strlen(two);
 129         dest = new(char, l1 + l2 + 1);
 130         if (!dest) {
 131                 log_oom();
 132                 return NULL;
 133         }
 134
 135         p = dest;
 136
 137         /* Headers from @one */
 138         n = b1 - one;
 139         p = mempcpy(p, one, n);
 140
 141         /* Headers from @two, these will only be found if not present above */
 142         n = b2 - two;
 143         p = mempcpy(p, two, n);
 144
 145         /* Body from @one */
 146         n = l1 - (b1 - one);
 147         if (n > 0) {
 148                 memcpy(p, b1, n);
 149                 p += n;
 150
 151         /* Body from @two */
 152         } else {
 153                 n = l2 - (b2 - two);
 154                 memcpy(p, b2, n);
 155                 p += n;
 156         }
 157
 158         assert(p - dest <= (ptrdiff_t)(l1 + l2));
 159         p[0] = '\0';
 160         return dest;
 161 }
 162
 163 static int finish_item(
 164                 Hashmap *h,
 165                 sd_id128_t id,
 166                 const char *language,
 167                 char *payload, size_t payload_size) {
 168
 169         _cleanup_free_ CatalogItem *i = NULL;
 170         _cleanup_free_ char *prev = NULL, *combined = NULL;
 171
 172         assert(h);
 173         assert(payload);
 174         assert(payload_size > 0);
 175
 176         i = new0(CatalogItem, 1);
 177         if (!i)
 178                 return log_oom();
 179
 180         i->id = id;
 181         if (language) {
 182                 assert(strlen(language) > 1 && strlen(language) < 32);
 183                 strcpy(i->language, language);
 184         }
 185
 186         prev = hashmap_get(h, i);
 187         if (prev) {
 188                 /* Already have such an item, combine them */
 189                 combined = combine_entries(payload, prev);
 190                 if (!combined)
 191                         return log_oom();
 192
 193                 if (hashmap_update(h, i, combined) < 0)
 194                         return log_oom();
 195                 combined = NULL;
 196         } else {
 197                 /* A new item */
 198                 combined = memdup(payload, payload_size + 1);
 199                 if (!combined)
 200                         return log_oom();
 201
 202                 if (hashmap_put(h, i, combined) < 0)
 203                         return log_oom();
 204                 i = NULL;
 205                 combined = NULL;
 206         }
 207
 208         return 0;
 209 }
 210
 211 int catalog_file_lang(const char* filename, char **lang) {
 212         char *beg, *end, *_lang;
 213
 214         end = endswith(filename, ".catalog");
 215         if (!end)
 216                 return 0;
 217
 218         beg = end - 1;
 219         while (beg > filename && !IN_SET(*beg, '.', '/') && end - beg < 32)
 220                 beg--;
 221
 222         if (*beg != '.' || end <= beg + 1)
 223                 return 0;
 224
 225         _lang = strndup(beg + 1, end - beg - 1);
 226         if (!_lang)
 227                 return -ENOMEM;
 228
 229         *lang = _lang;
 230         return 1;
 231 }
 232
 233 static int catalog_entry_lang(const char* filename, int line,
 234                               const char* t, const char* deflang, char **lang) {
 235         size_t c;
 236
 237         c = strlen(t);
 238         if (c == 0) {
 239                 log_error("[%s:%u] Language too short.", filename, line);
 240                 return -EINVAL;
 241         }
 242         if (c > 31) {
 243                 log_error("[%s:%u] language too long.", filename, line);
 244                 return -EINVAL;
 245         }
 246
 247         if (deflang) {
 248                 if (streq(t, deflang)) {
 249                         log_warning("[%s:%u] language specified unnecessarily",
 250                                     filename, line);
 251                         return 0;
 252                 } else
 253                         log_warning("[%s:%u] language differs from default for file",
 254                                     filename, line);
 255         }
 256
 257         *lang = strdup(t);
 258         if (!*lang)
 259                         return -ENOMEM;
 260
 261         return 0;
 262 }
 263
 264 int catalog_import_file(Hashmap *h, const char *path) {
 265         _cleanup_fclose_ FILE *f = NULL;
 266         _cleanup_free_ char *payload = NULL;
 267         size_t payload_size = 0, payload_allocated = 0;
 268         unsigned n = 0;
 269         sd_id128_t id;
 270         _cleanup_free_ char *deflang = NULL, *lang = NULL;
 271         bool got_id = false, empty_line = true;
 272         int r;
 273
 274         assert(h);
 275         assert(path);
 276
 277         f = fopen(path, "re");
 278         if (!f)
 279                 return log_error_errno(errno, "Failed to open file %s: %m", path);
 280
 281         r = catalog_file_lang(path, &deflang);
 282         if (r < 0)
 283                 log_error_errno(r, "Failed to determine language for file %s: %m", path);
 284         if (r == 1)
 285                 log_debug("File %s has language %s.", path, deflang);
 286
 287         for (;;) {
 288                 char line[LINE_MAX];
 289                 size_t line_len;
 290
 291                 if (!fgets(line, sizeof(line), f)) {
 292                         if (feof(f))
 293                                 break;
 294
 295                         return log_error_errno(errno, "Failed to read file %s: %m", path);
 296                 }
 297
 298                 n++;
 299
 300                 truncate_nl(line);
 301
 302                 if (line[0] == 0) {
 303                         empty_line = true;
 304                         continue;
 305                 }
 306
 307                 if (strchr(COMMENTS "\n", line[0]))
 308                         continue;
 309
 310                 if (empty_line &&
 311                     strlen(line) >= 2+1+32 &&
 312                     line[0] == '-' &&
 313                     line[1] == '-' &&
 314                     line[2] == ' ' &&
 315                     IN_SET(line[2+1+32], ' ', '\0')) {
 316
 317                         bool with_language;
 318                         sd_id128_t jd;
 319
 320                         /* New entry */
 321
 322                         with_language = line[2+1+32] != '\0';
 323                         line[2+1+32] = '\0';
 324
 325                         if (sd_id128_from_string(line + 2 + 1, &jd) >= 0) {
 326
 327                                 if (got_id) {
 328                                         if (payload_size == 0) {
 329                                                 log_error("[%s:%u] No payload text.", path, n);
 330                                                 return -EINVAL;
 331                                         }
 332
 333                                         r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 334                                         if (r < 0)
 335                                                 return r;
 336
 337                                         lang = mfree(lang);
 338                                         payload_size = 0;
 339                                 }
 340
 341                                 if (with_language) {
 342                                         char *t;
 343
 344                                         t = strstrip(line + 2 + 1 + 32 + 1);
 345                                         r = catalog_entry_lang(path, n, t, deflang, &lang);
 346                                         if (r < 0)
 347                                                 return r;
 348                                 }
 349
 350                                 got_id = true;
 351                                 empty_line = false;
 352                                 id = jd;
 353
 354                                 continue;
 355                         }
 356                 }
 357
 358                 /* Payload */
 359                 if (!got_id) {
 360                         log_error("[%s:%u] Got payload before ID.", path, n);
 361                         return -EINVAL;
 362                 }
 363
 364                 line_len = strlen(line);
 365                 if (!GREEDY_REALLOC(payload, payload_allocated,
 366                                     payload_size + (empty_line ? 1 : 0) + line_len + 1 + 1))
 367                         return log_oom();
 368
 369                 if (empty_line)
 370                         payload[payload_size++] = '\n';
 371                 memcpy(payload + payload_size, line, line_len);
 372                 payload_size += line_len;
 373                 payload[payload_size++] = '\n';
 374                 payload[payload_size] = '\0';
 375
 376                 empty_line = false;
 377         }
 378
 379         if (got_id) {
 380                 if (payload_size == 0) {
 381                         log_error("[%s:%u] No payload text.", path, n);
 382                         return -EINVAL;
 383                 }
 384
 385                 r = finish_item(h, id, lang ?: deflang, payload, payload_size);
 386                 if (r < 0)
 387                         return r;
 388         }
 389
 390         return 0;
 391 }
 392
 393 static int64_t write_catalog(const char *database, struct strbuf *sb,
 394                              CatalogItem *items, size_t n) {
 395         CatalogHeader header;
 396         _cleanup_fclose_ FILE *w = NULL;
 397         int r;
 398         _cleanup_free_ char *d, *p = NULL;
 399         size_t k;
 400
 401         d = dirname_malloc(database);
 402         if (!d)
 403                 return log_oom();
 404
 405         r = mkdir_p(d, 0775);
 406         if (r < 0)
 407                 return log_error_errno(r, "Recursive mkdir %s: %m", d);
 408
 409         r = fopen_temporary(database, &w, &p);
 410         if (r < 0)
 411                 return log_error_errno(r, "Failed to open database for writing: %s: %m",
 412                                        database);
 413
 414         zero(header);
 415         memcpy(header.signature, CATALOG_SIGNATURE, sizeof(header.signature));
 416         header.header_size = htole64(ALIGN_TO(sizeof(CatalogHeader), 8));
 417         header.catalog_item_size = htole64(sizeof(CatalogItem));
 418         header.n_items = htole64(n);
 419
 420         r = -EIO;
 421
 422         k = fwrite(&header, 1, sizeof(header), w);
 423         if (k != sizeof(header)) {
 424                 log_error("%s: failed to write header.", p);
 425                 goto error;
 426         }
 427
 428         k = fwrite(items, 1, n * sizeof(CatalogItem), w);
 429         if (k != n * sizeof(CatalogItem)) {
 430                 log_error("%s: failed to write database.", p);
 431                 goto error;
 432         }
 433
 434         k = fwrite(sb->buf, 1, sb->len, w);
 435         if (k != sb->len) {
 436                 log_error("%s: failed to write strings.", p);
 437                 goto error;
 438         }
 439
 440         r = fflush_and_check(w);
 441         if (r < 0) {
 442                 log_error_errno(r, "%s: failed to write database: %m", p);
 443                 goto error;
 444         }
 445
 446         fchmod(fileno(w), 0644);
 447
 448         if (rename(p, database) < 0) {
 449                 r = log_error_errno(errno, "rename (%s -> %s) failed: %m", p, database);
 450                 goto error;
 451         }
 452
 453         return ftello(w);
 454
 455 error:
 456         (void) unlink(p);
 457         return r;
 458 }
 459
 460 int catalog_update(const char* database, const char* root, const char* const* dirs) {
 461         _cleanup_strv_free_ char **files = NULL;
 462         char **f;
 463         struct strbuf *sb = NULL;
 464         _cleanup_hashmap_free_free_free_ Hashmap *h = NULL;
 465         _cleanup_free_ CatalogItem *items = NULL;
 466         ssize_t offset;
 467         char *payload;
 468         CatalogItem *i;
 469         Iterator j;
 470         unsigned n;
 471         int r;
 472         int64_t sz;
 473
 474         h = hashmap_new(&catalog_hash_ops);
 475         sb = strbuf_new();
 476
 477         if (!h || !sb) {
 478                 r = log_oom();
 479                 goto finish;
 480         }
 481
 482         r = conf_files_list_strv(&files, ".catalog", root, 0, dirs);
 483         if (r < 0) {
 484                 log_error_errno(r, "Failed to get catalog files: %m");
 485                 goto finish;
 486         }
 487
 488         STRV_FOREACH(f, files) {
 489                 log_debug("Reading file '%s'", *f);
 490                 r = catalog_import_file(h, *f);
 491                 if (r < 0) {
 492                         log_error_errno(r, "Failed to import file '%s': %m", *f);
 493                         goto finish;
 494                 }
 495         }
 496
 497         if (hashmap_size(h) <= 0) {
 498                 log_info("No items in catalog.");
 499                 goto finish;
 500         } else
 501                 log_debug("Found %u items in catalog.", hashmap_size(h));
 502
 503         items = new(CatalogItem, hashmap_size(h));
 504         if (!items) {
 505                 r = log_oom();
 506                 goto finish;
 507         }
 508
 509         n = 0;
 510         HASHMAP_FOREACH_KEY(payload, i, h, j) {
 511                 log_debug("Found " SD_ID128_FORMAT_STR ", language %s",
 512                           SD_ID128_FORMAT_VAL(i->id),
 513                           isempty(i->language) ? "C" : i->language);
 514
 515                 offset = strbuf_add_string(sb, payload, strlen(payload));
 516                 if (offset < 0) {
 517                         r = log_oom();
 518                         goto finish;
 519                 }
 520                 i->offset = htole64((uint64_t) offset);
 521                 items[n++] = *i;
 522         }
 523
 524         assert(n == hashmap_size(h));
 525         qsort_safe(items, n, sizeof(CatalogItem), catalog_compare_func);
 526
 527         strbuf_complete(sb);
 528
 529         sz = write_catalog(database, sb, items, n);
 530         if (sz < 0)
 531                 r = log_error_errno(sz, "Failed to write %s: %m", database);
 532         else {
 533                 r = 0;
 534                 log_debug("%s: wrote %u items, with %zu bytes of strings, %"PRIi64" total size.",
 535                           database, n, sb->len, sz);
 536         }
 537
 538 finish:
 539         strbuf_cleanup(sb);
 540
 541         return r;
 542 }
 543
 544 static int open_mmap(const char *database, int *_fd, struct stat *_st, void **_p) {
 545         const CatalogHeader *h;
 546         int fd;
 547         void *p;
 548         struct stat st;
 549
 550         assert(_fd);
 551         assert(_st);
 552         assert(_p);
 553
 554         fd = open(database, O_RDONLY|O_CLOEXEC);
 555         if (fd < 0)
 556                 return -errno;
 557
 558         if (fstat(fd, &st) < 0) {
 559                 safe_close(fd);
 560                 return -errno;
 561         }
 562
 563         if (st.st_size < (off_t) sizeof(CatalogHeader)) {
 564                 safe_close(fd);
 565                 return -EINVAL;
 566         }
 567
 568         p = mmap(NULL, PAGE_ALIGN(st.st_size), PROT_READ, MAP_SHARED, fd, 0);
 569         if (p == MAP_FAILED) {
 570                 safe_close(fd);
 571                 return -errno;
 572         }
 573
 574         h = p;
 575         if (memcmp(h->signature, CATALOG_SIGNATURE, sizeof(h->signature)) != 0 ||
 576             le64toh(h->header_size) < sizeof(CatalogHeader) ||
 577             le64toh(h->catalog_item_size) < sizeof(CatalogItem) ||
 578             h->incompatible_flags != 0 ||
 579             le64toh(h->n_items) <= 0 ||
 580             st.st_size < (off_t) (le64toh(h->header_size) + le64toh(h->catalog_item_size) * le64toh(h->n_items))) {
 581                 safe_close(fd);
 582                 munmap(p, st.st_size);
 583                 return -EBADMSG;
 584         }
 585
 586         *_fd = fd;
 587         *_st = st;
 588         *_p = p;
 589
 590         return 0;
 591 }
 592
 593 static const char *find_id(void *p, sd_id128_t id) {
 594         CatalogItem key, *f = NULL;
 595         const CatalogHeader *h = p;
 596         const char *loc;
 597
 598         zero(key);
 599         key.id = id;
 600
 601         loc = setlocale(LC_MESSAGES, NULL);
 602         if (loc && loc[0] && !streq(loc, "C") && !streq(loc, "POSIX")) {
 603                 strncpy(key.language, loc, sizeof(key.language));
 604                 key.language[strcspn(key.language, ".@")] = 0;
 605
 606                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 607                 if (!f) {
 608                         char *e;
 609
 610                         e = strchr(key.language, '_');
 611                         if (e) {
 612                                 *e = 0;
 613                                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 614                         }
 615                 }
 616         }
 617
 618         if (!f) {
 619                 zero(key.language);
 620                 f = bsearch(&key, (const uint8_t*) p + le64toh(h->header_size), le64toh(h->n_items), le64toh(h->catalog_item_size), catalog_compare_func);
 621         }
 622
 623         if (!f)
 624                 return NULL;
 625
 626         return (const char*) p +
 627                 le64toh(h->header_size) +
 628                 le64toh(h->n_items) * le64toh(h->catalog_item_size) +
 629                 le64toh(f->offset);
 630 }
 631
 632 int catalog_get(const char* database, sd_id128_t id, char **_text) {
 633         _cleanup_close_ int fd = -1;
 634         void *p = NULL;
 635         struct stat st = {};
 636         char *text = NULL;
 637         int r;
 638         const char *s;
 639
 640         assert(_text);
 641
 642         r = open_mmap(database, &fd, &st, &p);
 643         if (r < 0)
 644                 return r;
 645
 646         s = find_id(p, id);
 647         if (!s) {
 648                 r = -ENOENT;
 649                 goto finish;
 650         }
 651
 652         text = strdup(s);
 653         if (!text) {
 654                 r = -ENOMEM;
 655                 goto finish;
 656         }
 657
 658         *_text = text;
 659         r = 0;
 660
 661 finish:
 662         if (p)
 663                 munmap(p, st.st_size);
 664
 665         return r;
 666 }
 667
 668 static char *find_header(const char *s, const char *header) {
 669
 670         for (;;) {
 671                 const char *v;
 672
 673                 v = startswith(s, header);
 674                 if (v) {
 675                         v += strspn(v, WHITESPACE);
 676                         return strndup(v, strcspn(v, NEWLINE));
 677                 }
 678
 679                 if (!next_header(&s))
 680                         return NULL;
 681         }
 682 }
 683
 684 static void dump_catalog_entry(FILE *f, sd_id128_t id, const char *s, bool oneline) {
 685         if (oneline) {
 686                 _cleanup_free_ char *subject = NULL, *defined_by = NULL;
 687
 688                 subject = find_header(s, "Subject:");
 689                 defined_by = find_header(s, "Defined-By:");
 690
 691                 fprintf(f, SD_ID128_FORMAT_STR " %s: %s\n",
 692                         SD_ID128_FORMAT_VAL(id),
 693                         strna(defined_by), strna(subject));
 694         } else
 695                 fprintf(f, "-- " SD_ID128_FORMAT_STR "\n%s\n",
 696                         SD_ID128_FORMAT_VAL(id), s);
 697 }
 698
 699
 700 int catalog_list(FILE *f, const char *database, bool oneline) {
 701         _cleanup_close_ int fd = -1;
 702         void *p = NULL;
 703         struct stat st;
 704         const CatalogHeader *h;
 705         const CatalogItem *items;
 706         int r;
 707         unsigned n;
 708         sd_id128_t last_id;
 709         bool last_id_set = false;
 710
 711         r = open_mmap(database, &fd, &st, &p);
 712         if (r < 0)
 713                 return r;
 714
 715         h = p;
 716         items = (const CatalogItem*) ((const uint8_t*) p + le64toh(h->header_size));
 717
 718         for (n = 0; n < le64toh(h->n_items); n++) {
 719                 const char *s;
 720
 721                 if (last_id_set && sd_id128_equal(last_id, items[n].id))
 722                         continue;
 723
 724                 assert_se(s = find_id(p, items[n].id));
 725
 726                 dump_catalog_entry(f, items[n].id, s, oneline);
 727
 728                 last_id_set = true;
 729                 last_id = items[n].id;
 730         }
 731
 732         munmap(p, st.st_size);
 733
 734         return 0;
 735 }
 736
 737 int catalog_list_items(FILE *f, const char *database, bool oneline, char **items) {
 738         char **item;
 739         int r = 0;
 740
 741         STRV_FOREACH(item, items) {
 742                 sd_id128_t id;
 743                 int k;
 744                 _cleanup_free_ char *msg = NULL;
 745
 746                 k = sd_id128_from_string(*item, &id);
 747                 if (k < 0) {
 748                         log_error_errno(k, "Failed to parse id128 '%s': %m", *item);
 749                         if (r == 0)
 750                                 r = k;
 751                         continue;
 752                 }
 753
 754                 k = catalog_get(database, id, &msg);
 755                 if (k < 0) {
 756                         log_full_errno(k == -ENOENT ? LOG_NOTICE : LOG_ERR, k,
 757                                        "Failed to retrieve catalog entry for '%s': %m", *item);
 758                         if (r == 0)
 759                                 r = k;
 760                         continue;
 761                 }
 762
 763                 dump_catalog_entry(f, id, msg, oneline);
 764         }
 765
 766         return r;
 767 }