src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "path-util.h"
  37 #include "lookup3.h"
  38 #include "compress.h"
  39 #include "journal-internal.h"
  40 #include "missing.h"
  41 #include "catalog.h"
  42 #include "replace-var.h"
  43
  44 #define JOURNAL_FILES_MAX 1024
  45
  46 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  47
  48 #define REPLACE_VAR_MAX 256
  49
  50 #define DEFAULT_DATA_THRESHOLD (64*1024)
  51
  52 /* We return an error here only if we didn't manage to
  53    memorize the real error. */
  54 static int set_put_error(sd_journal *j, int r) {
  55         int k;
  56
  57         if (r >= 0)
  58                 return r;
  59
  60         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  61         if (k < 0)
  62                 return k;
  63
  64         return set_put(j->errors, INT_TO_PTR(r));
  65 }
  66
  67 static void detach_location(sd_journal *j) {
  68         Iterator i;
  69         JournalFile *f;
  70
  71         assert(j);
  72
  73         j->current_file = NULL;
  74         j->current_field = 0;
  75
  76         HASHMAP_FOREACH(f, j->files, i)
  77                 f->current_offset = 0;
  78 }
  79
  80 static void reset_location(sd_journal *j) {
  81         assert(j);
  82
  83         detach_location(j);
  84         zero(j->current_location);
  85 }
  86
  87 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  88         assert(l);
  89         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
  90         assert(f);
  91         assert(o->object.type == OBJECT_ENTRY);
  92
  93         l->type = type;
  94         l->seqnum = le64toh(o->entry.seqnum);
  95         l->seqnum_id = f->header->seqnum_id;
  96         l->realtime = le64toh(o->entry.realtime);
  97         l->monotonic = le64toh(o->entry.monotonic);
  98         l->boot_id = o->entry.boot_id;
  99         l->xor_hash = le64toh(o->entry.xor_hash);
 100
 101         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 102 }
 103
 104 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o, uint64_t offset) {
 105         assert(j);
 106         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 107         assert(f);
 108         assert(o);
 109
 110         init_location(&j->current_location, type, f, o);
 111
 112         j->current_file = f;
 113         j->current_field = 0;
 114
 115         f->current_offset = offset;
 116 }
 117
 118 static int match_is_valid(const void *data, size_t size) {
 119         const char *b, *p;
 120
 121         assert(data);
 122
 123         if (size < 2)
 124                 return false;
 125
 126         if (startswith(data, "__"))
 127                 return false;
 128
 129         b = data;
 130         for (p = b; p < b + size; p++) {
 131
 132                 if (*p == '=')
 133                         return p > b;
 134
 135                 if (*p == '_')
 136                         continue;
 137
 138                 if (*p >= 'A' && *p <= 'Z')
 139                         continue;
 140
 141                 if (*p >= '0' && *p <= '9')
 142                         continue;
 143
 144                 return false;
 145         }
 146
 147         return false;
 148 }
 149
 150 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 151         const uint8_t *a = _a, *b = _b;
 152         size_t j;
 153
 154         for (j = 0; j < s && j < t; j++) {
 155
 156                 if (a[j] != b[j])
 157                         return false;
 158
 159                 if (a[j] == '=')
 160                         return true;
 161         }
 162
 163         return true;
 164 }
 165
 166 static Match *match_new(Match *p, MatchType t) {
 167         Match *m;
 168
 169         m = new0(Match, 1);
 170         if (!m)
 171                 return NULL;
 172
 173         m->type = t;
 174
 175         if (p) {
 176                 m->parent = p;
 177                 LIST_PREPEND(Match, matches, p->matches, m);
 178         }
 179
 180         return m;
 181 }
 182
 183 static void match_free(Match *m) {
 184         assert(m);
 185
 186         while (m->matches)
 187                 match_free(m->matches);
 188
 189         if (m->parent)
 190                 LIST_REMOVE(Match, matches, m->parent->matches, m);
 191
 192         free(m->data);
 193         free(m);
 194 }
 195
 196 static void match_free_if_empty(Match *m) {
 197         assert(m);
 198
 199         if (m->matches)
 200                 return;
 201
 202         match_free(m);
 203 }
 204
 205 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 206         Match *l3, *l4, *add_here = NULL, *m;
 207         le64_t le_hash;
 208
 209         if (!j)
 210                 return -EINVAL;
 211
 212         if (!data)
 213                 return -EINVAL;
 214
 215         if (size == 0)
 216                 size = strlen(data);
 217
 218         if (!match_is_valid(data, size))
 219                 return -EINVAL;
 220
 221         /* level 0: AND term
 222          * level 1: OR terms
 223          * level 2: AND terms
 224          * level 3: OR terms
 225          * level 4: concrete matches */
 226
 227         if (!j->level0) {
 228                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 229                 if (!j->level0)
 230                         return -ENOMEM;
 231         }
 232
 233         if (!j->level1) {
 234                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 235                 if (!j->level1)
 236                         return -ENOMEM;
 237         }
 238
 239         if (!j->level2) {
 240                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 241                 if (!j->level2)
 242                         return -ENOMEM;
 243         }
 244
 245         assert(j->level0->type == MATCH_AND_TERM);
 246         assert(j->level1->type == MATCH_OR_TERM);
 247         assert(j->level2->type == MATCH_AND_TERM);
 248
 249         le_hash = htole64(hash64(data, size));
 250
 251         LIST_FOREACH(matches, l3, j->level2->matches) {
 252                 assert(l3->type == MATCH_OR_TERM);
 253
 254                 LIST_FOREACH(matches, l4, l3->matches) {
 255                         assert(l4->type == MATCH_DISCRETE);
 256
 257                         /* Exactly the same match already? Then ignore
 258                          * this addition */
 259                         if (l4->le_hash == le_hash &&
 260                             l4->size == size &&
 261                             memcmp(l4->data, data, size) == 0)
 262                                 return 0;
 263
 264                         /* Same field? Then let's add this to this OR term */
 265                         if (same_field(data, size, l4->data, l4->size)) {
 266                                 add_here = l3;
 267                                 break;
 268                         }
 269                 }
 270
 271                 if (add_here)
 272                         break;
 273         }
 274
 275         if (!add_here) {
 276                 add_here = match_new(j->level2, MATCH_OR_TERM);
 277                 if (!add_here)
 278                         goto fail;
 279         }
 280
 281         m = match_new(add_here, MATCH_DISCRETE);
 282         if (!m)
 283                 goto fail;
 284
 285         m->le_hash = le_hash;
 286         m->size = size;
 287         m->data = memdup(data, size);
 288         if (!m->data)
 289                 goto fail;
 290
 291         detach_location(j);
 292
 293         return 0;
 294
 295 fail:
 296         if (add_here)
 297                 match_free_if_empty(add_here);
 298
 299         if (j->level2)
 300                 match_free_if_empty(j->level2);
 301
 302         if (j->level1)
 303                 match_free_if_empty(j->level1);
 304
 305         if (j->level0)
 306                 match_free_if_empty(j->level0);
 307
 308         return -ENOMEM;
 309 }
 310
 311 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 312         assert(j);
 313
 314         if (!j->level0)
 315                 return 0;
 316
 317         if (!j->level1)
 318                 return 0;
 319
 320         if (!j->level1->matches)
 321                 return 0;
 322
 323         j->level1 = NULL;
 324         j->level2 = NULL;
 325
 326         return 0;
 327 }
 328
 329 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 330         assert(j);
 331
 332         if (!j->level0)
 333                 return 0;
 334
 335         if (!j->level1)
 336                 return 0;
 337
 338         if (!j->level2)
 339                 return 0;
 340
 341         if (!j->level2->matches)
 342                 return 0;
 343
 344         j->level2 = NULL;
 345         return 0;
 346 }
 347
 348 static char *match_make_string(Match *m) {
 349         char *p, *r;
 350         Match *i;
 351         bool enclose = false;
 352
 353         if (!m)
 354                 return strdup("");
 355
 356         if (m->type == MATCH_DISCRETE)
 357                 return strndup(m->data, m->size);
 358
 359         p = NULL;
 360         LIST_FOREACH(matches, i, m->matches) {
 361                 char *t, *k;
 362
 363                 t = match_make_string(i);
 364                 if (!t) {
 365                         free(p);
 366                         return NULL;
 367                 }
 368
 369                 if (p) {
 370                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 371                         free(p);
 372                         free(t);
 373
 374                         if (!k)
 375                                 return NULL;
 376
 377                         p = k;
 378
 379                         enclose = true;
 380                 } else {
 381                         free(p);
 382                         p = t;
 383                 }
 384         }
 385
 386         if (enclose) {
 387                 r = strjoin("(", p, ")", NULL);
 388                 free(p);
 389                 return r;
 390         }
 391
 392         return p;
 393 }
 394
 395 char *journal_make_match_string(sd_journal *j) {
 396         assert(j);
 397
 398         return match_make_string(j->level0);
 399 }
 400
 401 _public_ void sd_journal_flush_matches(sd_journal *j) {
 402
 403         if (!j)
 404                 return;
 405
 406         if (j->level0)
 407                 match_free(j->level0);
 408
 409         j->level0 = j->level1 = j->level2 = NULL;
 410
 411         detach_location(j);
 412 }
 413
 414 static int compare_entry_order(JournalFile *af, Object *_ao,
 415                          JournalFile *bf, uint64_t bp) {
 416
 417         uint64_t a, b;
 418         Object *ao, *bo;
 419         int r;
 420
 421         assert(af);
 422         assert(bf);
 423         assert(_ao);
 424
 425         /* The mmap cache might invalidate the object from the first
 426          * file if we look at the one from the second file. Hence
 427          * temporarily copy the header of the first one, and look at
 428          * that only. */
 429         ao = alloca(offsetof(EntryObject, items));
 430         memcpy(ao, _ao, offsetof(EntryObject, items));
 431
 432         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 433         if (r < 0)
 434                 return strcmp(af->path, bf->path);
 435
 436         /* We operate on two different files here, hence we can access
 437          * two objects at the same time, which we normally can't.
 438          *
 439          * If contents and timestamps match, these entries are
 440          * identical, even if the seqnum does not match */
 441
 442         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 443             ao->entry.monotonic == bo->entry.monotonic &&
 444             ao->entry.realtime == bo->entry.realtime &&
 445             ao->entry.xor_hash == bo->entry.xor_hash)
 446                 return 0;
 447
 448         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 449
 450                 /* If this is from the same seqnum source, compare
 451                  * seqnums */
 452                 a = le64toh(ao->entry.seqnum);
 453                 b = le64toh(bo->entry.seqnum);
 454
 455                 if (a < b)
 456                         return -1;
 457                 if (a > b)
 458                         return 1;
 459
 460                 /* Wow! This is weird, different data but the same
 461                  * seqnums? Something is borked, but let's make the
 462                  * best of it and compare by time. */
 463         }
 464
 465         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 466
 467                 /* If the boot id matches compare monotonic time */
 468                 a = le64toh(ao->entry.monotonic);
 469                 b = le64toh(bo->entry.monotonic);
 470
 471                 if (a < b)
 472                         return -1;
 473                 if (a > b)
 474                         return 1;
 475         }
 476
 477         /* Otherwise compare UTC time */
 478         a = le64toh(ao->entry.realtime);
 479         b = le64toh(bo->entry.realtime);
 480
 481         if (a < b)
 482                 return -1;
 483         if (a > b)
 484                 return 1;
 485
 486         /* Finally, compare by contents */
 487         a = le64toh(ao->entry.xor_hash);
 488         b = le64toh(bo->entry.xor_hash);
 489
 490         if (a < b)
 491                 return -1;
 492         if (a > b)
 493                 return 1;
 494
 495         return 0;
 496 }
 497
 498 static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 499         uint64_t a;
 500
 501         assert(af);
 502         assert(ao);
 503         assert(l);
 504         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 505
 506         if (l->monotonic_set &&
 507             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 508             l->realtime_set &&
 509             le64toh(ao->entry.realtime) == l->realtime &&
 510             l->xor_hash_set &&
 511             le64toh(ao->entry.xor_hash) == l->xor_hash)
 512                 return 0;
 513
 514         if (l->seqnum_set &&
 515             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 516
 517                 a = le64toh(ao->entry.seqnum);
 518
 519                 if (a < l->seqnum)
 520                         return -1;
 521                 if (a > l->seqnum)
 522                         return 1;
 523         }
 524
 525         if (l->monotonic_set &&
 526             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 527
 528                 a = le64toh(ao->entry.monotonic);
 529
 530                 if (a < l->monotonic)
 531                         return -1;
 532                 if (a > l->monotonic)
 533                         return 1;
 534         }
 535
 536         if (l->realtime_set) {
 537
 538                 a = le64toh(ao->entry.realtime);
 539
 540                 if (a < l->realtime)
 541                         return -1;
 542                 if (a > l->realtime)
 543                         return 1;
 544         }
 545
 546         if (l->xor_hash_set) {
 547                 a = le64toh(ao->entry.xor_hash);
 548
 549                 if (a < l->xor_hash)
 550                         return -1;
 551                 if (a > l->xor_hash)
 552                         return 1;
 553         }
 554
 555         return 0;
 556 }
 557
 558 static int next_for_match(
 559                 sd_journal *j,
 560                 Match *m,
 561                 JournalFile *f,
 562                 uint64_t after_offset,
 563                 direction_t direction,
 564                 Object **ret,
 565                 uint64_t *offset) {
 566
 567         int r;
 568         uint64_t np = 0;
 569         Object *n;
 570
 571         assert(j);
 572         assert(m);
 573         assert(f);
 574
 575         if (m->type == MATCH_DISCRETE) {
 576                 uint64_t dp;
 577
 578                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 579                 if (r <= 0)
 580                         return r;
 581
 582                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 583
 584         } else if (m->type == MATCH_OR_TERM) {
 585                 Match *i;
 586
 587                 /* Find the earliest match beyond after_offset */
 588
 589                 LIST_FOREACH(matches, i, m->matches) {
 590                         uint64_t cp;
 591
 592                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 593                         if (r < 0)
 594                                 return r;
 595                         else if (r > 0) {
 596                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 597                                         np = cp;
 598                         }
 599                 }
 600
 601         } else if (m->type == MATCH_AND_TERM) {
 602                 Match *i;
 603                 bool continue_looking;
 604
 605                 /* Always jump to the next matching entry and repeat
 606                  * this until we fine and offset that matches for all
 607                  * matches. */
 608
 609                 if (!m->matches)
 610                         return 0;
 611
 612                 np = 0;
 613                 do {
 614                         continue_looking = false;
 615
 616                         LIST_FOREACH(matches, i, m->matches) {
 617                                 uint64_t cp, limit;
 618
 619                                 if (np == 0)
 620                                         limit = after_offset;
 621                                 else if (direction == DIRECTION_DOWN)
 622                                         limit = MAX(np, after_offset);
 623                                 else
 624                                         limit = MIN(np, after_offset);
 625
 626                                 r = next_for_match(j, i, f, limit, direction, NULL, &cp);
 627                                 if (r <= 0)
 628                                         return r;
 629
 630                                 if ((direction == DIRECTION_DOWN ? cp >= after_offset : cp <= after_offset) &&
 631                                     (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))) {
 632                                         np = cp;
 633                                         continue_looking = true;
 634                                 }
 635                         }
 636
 637                 } while (continue_looking);
 638         }
 639
 640         if (np == 0)
 641                 return 0;
 642
 643         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 644         if (r < 0)
 645                 return r;
 646
 647         if (ret)
 648                 *ret = n;
 649         if (offset)
 650                 *offset = np;
 651
 652         return 1;
 653 }
 654
 655 static int find_location_for_match(
 656                 sd_journal *j,
 657                 Match *m,
 658                 JournalFile *f,
 659                 direction_t direction,
 660                 Object **ret,
 661                 uint64_t *offset) {
 662
 663         int r;
 664
 665         assert(j);
 666         assert(m);
 667         assert(f);
 668
 669         if (m->type == MATCH_DISCRETE) {
 670                 uint64_t dp;
 671
 672                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 673                 if (r <= 0)
 674                         return r;
 675
 676                 /* FIXME: missing: find by monotonic */
 677
 678                 if (j->current_location.type == LOCATION_HEAD)
 679                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 680                 if (j->current_location.type == LOCATION_TAIL)
 681                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 682                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 683                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 684                 if (j->current_location.monotonic_set) {
 685                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 686                         if (r != -ENOENT)
 687                                 return r;
 688                 }
 689                 if (j->current_location.realtime_set)
 690                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 691
 692                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 693
 694         } else if (m->type == MATCH_OR_TERM) {
 695                 uint64_t np = 0;
 696                 Object *n;
 697                 Match *i;
 698
 699                 /* Find the earliest match */
 700
 701                 LIST_FOREACH(matches, i, m->matches) {
 702                         uint64_t cp;
 703
 704                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 705                         if (r < 0)
 706                                 return r;
 707                         else if (r > 0) {
 708                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 709                                         np = cp;
 710                         }
 711                 }
 712
 713                 if (np == 0)
 714                         return 0;
 715
 716                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 717                 if (r < 0)
 718                         return r;
 719
 720                 if (ret)
 721                         *ret = n;
 722                 if (offset)
 723                         *offset = np;
 724
 725                 return 1;
 726
 727         } else {
 728                 Match *i;
 729                 uint64_t np = 0;
 730
 731                 assert(m->type == MATCH_AND_TERM);
 732
 733                 /* First jump to the last match, and then find the
 734                  * next one where all matches match */
 735
 736                 if (!m->matches)
 737                         return 0;
 738
 739                 LIST_FOREACH(matches, i, m->matches) {
 740                         uint64_t cp;
 741
 742                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 743                         if (r <= 0)
 744                                 return r;
 745
 746                         if (np == 0 || (direction == DIRECTION_DOWN ? np < cp : np > cp))
 747                                 np = cp;
 748                 }
 749
 750                 return next_for_match(j, m, f, np, direction, ret, offset);
 751         }
 752 }
 753
 754 static int find_location_with_matches(
 755                 sd_journal *j,
 756                 JournalFile *f,
 757                 direction_t direction,
 758                 Object **ret,
 759                 uint64_t *offset) {
 760
 761         int r;
 762
 763         assert(j);
 764         assert(f);
 765         assert(ret);
 766         assert(offset);
 767
 768         if (!j->level0) {
 769                 /* No matches is simple */
 770
 771                 if (j->current_location.type == LOCATION_HEAD)
 772                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 773                 if (j->current_location.type == LOCATION_TAIL)
 774                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 775                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 776                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 777                 if (j->current_location.monotonic_set) {
 778                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 779                         if (r != -ENOENT)
 780                                 return r;
 781                 }
 782                 if (j->current_location.realtime_set)
 783                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 784
 785                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 786         } else
 787                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 788 }
 789
 790 static int next_with_matches(
 791                 sd_journal *j,
 792                 JournalFile *f,
 793                 direction_t direction,
 794                 Object **ret,
 795                 uint64_t *offset) {
 796
 797         Object *c;
 798         uint64_t cp;
 799
 800         assert(j);
 801         assert(f);
 802         assert(ret);
 803         assert(offset);
 804
 805         c = *ret;
 806         cp = *offset;
 807
 808         /* No matches is easy. We simple advance the file
 809          * pointer by one. */
 810         if (!j->level0)
 811                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 812
 813         /* If we have a match then we look for the next matching entry
 814          * with an offset at least one step larger */
 815         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 816 }
 817
 818 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 819         Object *c;
 820         uint64_t cp;
 821         int r;
 822
 823         assert(j);
 824         assert(f);
 825
 826         if (f->current_offset > 0) {
 827                 cp = f->current_offset;
 828
 829                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 830                 if (r < 0)
 831                         return r;
 832
 833                 r = next_with_matches(j, f, direction, &c, &cp);
 834                 if (r <= 0)
 835                         return r;
 836         } else {
 837                 r = find_location_with_matches(j, f, direction, &c, &cp);
 838                 if (r <= 0)
 839                         return r;
 840         }
 841
 842         /* OK, we found the spot, now let's advance until to an entry
 843          * that is actually different from what we were previously
 844          * looking at. This is necessary to handle entries which exist
 845          * in two (or more) journal files, and which shall all be
 846          * suppressed but one. */
 847
 848         for (;;) {
 849                 bool found;
 850
 851                 if (j->current_location.type == LOCATION_DISCRETE) {
 852                         int k;
 853
 854                         k = compare_with_location(f, c, &j->current_location);
 855                         if (direction == DIRECTION_DOWN)
 856                                 found = k > 0;
 857                         else
 858                                 found = k < 0;
 859                 } else
 860                         found = true;
 861
 862                 if (found) {
 863                         if (ret)
 864                                 *ret = c;
 865                         if (offset)
 866                                 *offset = cp;
 867                         return 1;
 868                 }
 869
 870                 r = next_with_matches(j, f, direction, &c, &cp);
 871                 if (r <= 0)
 872                         return r;
 873         }
 874 }
 875
 876 static int real_journal_next(sd_journal *j, direction_t direction) {
 877         JournalFile *f, *new_file = NULL;
 878         uint64_t new_offset = 0;
 879         Object *o;
 880         uint64_t p;
 881         Iterator i;
 882         int r;
 883
 884         if (!j)
 885                 return -EINVAL;
 886
 887         HASHMAP_FOREACH(f, j->files, i) {
 888                 bool found;
 889
 890                 r = next_beyond_location(j, f, direction, &o, &p);
 891                 if (r < 0) {
 892                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 893                         continue;
 894                 } else if (r == 0)
 895                         continue;
 896
 897                 if (!new_file)
 898                         found = true;
 899                 else {
 900                         int k;
 901
 902                         k = compare_entry_order(f, o, new_file, new_offset);
 903
 904                         if (direction == DIRECTION_DOWN)
 905                                 found = k < 0;
 906                         else
 907                                 found = k > 0;
 908                 }
 909
 910                 if (found) {
 911                         new_file = f;
 912                         new_offset = p;
 913                 }
 914         }
 915
 916         if (!new_file)
 917                 return 0;
 918
 919         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 920         if (r < 0)
 921                 return r;
 922
 923         set_location(j, LOCATION_DISCRETE, new_file, o, new_offset);
 924
 925         return 1;
 926 }
 927
 928 _public_ int sd_journal_next(sd_journal *j) {
 929         return real_journal_next(j, DIRECTION_DOWN);
 930 }
 931
 932 _public_ int sd_journal_previous(sd_journal *j) {
 933         return real_journal_next(j, DIRECTION_UP);
 934 }
 935
 936 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 937         int c = 0, r;
 938
 939         if (!j)
 940                 return -EINVAL;
 941
 942         if (skip == 0) {
 943                 /* If this is not a discrete skip, then at least
 944                  * resolve the current location */
 945                 if (j->current_location.type != LOCATION_DISCRETE)
 946                         return real_journal_next(j, direction);
 947
 948                 return 0;
 949         }
 950
 951         do {
 952                 r = real_journal_next(j, direction);
 953                 if (r < 0)
 954                         return r;
 955
 956                 if (r == 0)
 957                         return c;
 958
 959                 skip--;
 960                 c++;
 961         } while (skip > 0);
 962
 963         return c;
 964 }
 965
 966 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 967         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 968 }
 969
 970 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 971         return real_journal_next_skip(j, DIRECTION_UP, skip);
 972 }
 973
 974 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 975         Object *o;
 976         int r;
 977         char bid[33], sid[33];
 978
 979         if (!j)
 980                 return -EINVAL;
 981         if (!cursor)
 982                 return -EINVAL;
 983
 984         if (!j->current_file || j->current_file->current_offset <= 0)
 985                 return -EADDRNOTAVAIL;
 986
 987         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 988         if (r < 0)
 989                 return r;
 990
 991         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 992         sd_id128_to_string(o->entry.boot_id, bid);
 993
 994         if (asprintf(cursor,
 995                      "s=%s;i=%llx;b=%s;m=%llx;t=%llx;x=%llx",
 996                      sid, (unsigned long long) le64toh(o->entry.seqnum),
 997                      bid, (unsigned long long) le64toh(o->entry.monotonic),
 998                      (unsigned long long) le64toh(o->entry.realtime),
 999                      (unsigned long long) le64toh(o->entry.xor_hash)) < 0)
1000                 return -ENOMEM;
1001
1002         return 0;
1003 }
1004
1005 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
1006         char *w, *state;
1007         size_t l;
1008         unsigned long long seqnum, monotonic, realtime, xor_hash;
1009         bool
1010                 seqnum_id_set = false,
1011                 seqnum_set = false,
1012                 boot_id_set = false,
1013                 monotonic_set = false,
1014                 realtime_set = false,
1015                 xor_hash_set = false;
1016         sd_id128_t seqnum_id, boot_id;
1017
1018         if (!j)
1019                 return -EINVAL;
1020         if (isempty(cursor))
1021                 return -EINVAL;
1022
1023         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1024                 char *item;
1025                 int k = 0;
1026
1027                 if (l < 2 || w[1] != '=')
1028                         return -EINVAL;
1029
1030                 item = strndup(w, l);
1031                 if (!item)
1032                         return -ENOMEM;
1033
1034                 switch (w[0]) {
1035
1036                 case 's':
1037                         seqnum_id_set = true;
1038                         k = sd_id128_from_string(item+2, &seqnum_id);
1039                         break;
1040
1041                 case 'i':
1042                         seqnum_set = true;
1043                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1044                                 k = -EINVAL;
1045                         break;
1046
1047                 case 'b':
1048                         boot_id_set = true;
1049                         k = sd_id128_from_string(item+2, &boot_id);
1050                         break;
1051
1052                 case 'm':
1053                         monotonic_set = true;
1054                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1055                                 k = -EINVAL;
1056                         break;
1057
1058                 case 't':
1059                         realtime_set = true;
1060                         if (sscanf(item+2, "%llx", &realtime) != 1)
1061                                 k = -EINVAL;
1062                         break;
1063
1064                 case 'x':
1065                         xor_hash_set = true;
1066                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1067                                 k = -EINVAL;
1068                         break;
1069                 }
1070
1071                 free(item);
1072
1073                 if (k < 0)
1074                         return k;
1075         }
1076
1077         if ((!seqnum_set || !seqnum_id_set) &&
1078             (!monotonic_set || !boot_id_set) &&
1079             !realtime_set)
1080                 return -EINVAL;
1081
1082         reset_location(j);
1083
1084         j->current_location.type = LOCATION_SEEK;
1085
1086         if (realtime_set) {
1087                 j->current_location.realtime = (uint64_t) realtime;
1088                 j->current_location.realtime_set = true;
1089         }
1090
1091         if (seqnum_set && seqnum_id_set) {
1092                 j->current_location.seqnum = (uint64_t) seqnum;
1093                 j->current_location.seqnum_id = seqnum_id;
1094                 j->current_location.seqnum_set = true;
1095         }
1096
1097         if (monotonic_set && boot_id_set) {
1098                 j->current_location.monotonic = (uint64_t) monotonic;
1099                 j->current_location.boot_id = boot_id;
1100                 j->current_location.monotonic_set = true;
1101         }
1102
1103         if (xor_hash_set) {
1104                 j->current_location.xor_hash = (uint64_t) xor_hash;
1105                 j->current_location.xor_hash_set = true;
1106         }
1107
1108         return 0;
1109 }
1110
1111 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1112         int r;
1113         char *w, *state;
1114         size_t l;
1115         Object *o;
1116
1117         if (!j)
1118                 return -EINVAL;
1119         if (isempty(cursor))
1120                 return -EINVAL;
1121
1122         if (!j->current_file || j->current_file->current_offset <= 0)
1123                 return -EADDRNOTAVAIL;
1124
1125         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1126         if (r < 0)
1127                 return r;
1128
1129         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1130                 _cleanup_free_ char *item = NULL;
1131                 sd_id128_t id;
1132                 unsigned long long ll;
1133                 int k = 0;
1134
1135                 if (l < 2 || w[1] != '=')
1136                         return -EINVAL;
1137
1138                 item = strndup(w, l);
1139                 if (!item)
1140                         return -ENOMEM;
1141
1142                 switch (w[0]) {
1143
1144                 case 's':
1145                         k = sd_id128_from_string(item+2, &id);
1146                         if (k < 0)
1147                                 return k;
1148                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1149                                 return 0;
1150                         break;
1151
1152                 case 'i':
1153                         if (sscanf(item+2, "%llx", &ll) != 1)
1154                                 return -EINVAL;
1155                         if (ll != le64toh(o->entry.seqnum))
1156                                 return 0;
1157                         break;
1158
1159                 case 'b':
1160                         k = sd_id128_from_string(item+2, &id);
1161                         if (k < 0)
1162                                 return k;
1163                         if (!sd_id128_equal(id, o->entry.boot_id))
1164                                 return 0;
1165                         break;
1166
1167                 case 'm':
1168                         if (sscanf(item+2, "%llx", &ll) != 1)
1169                                 return -EINVAL;
1170                         if (ll != le64toh(o->entry.monotonic))
1171                                 return 0;
1172                         break;
1173
1174                 case 't':
1175                         if (sscanf(item+2, "%llx", &ll) != 1)
1176                                 return -EINVAL;
1177                         if (ll != le64toh(o->entry.realtime))
1178                                 return 0;
1179                         break;
1180
1181                 case 'x':
1182                         if (sscanf(item+2, "%llx", &ll) != 1)
1183                                 return -EINVAL;
1184                         if (ll != le64toh(o->entry.xor_hash))
1185                                 return 0;
1186                         break;
1187                 }
1188         }
1189
1190         return 1;
1191 }
1192
1193
1194 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1195         if (!j)
1196                 return -EINVAL;
1197
1198         reset_location(j);
1199         j->current_location.type = LOCATION_SEEK;
1200         j->current_location.boot_id = boot_id;
1201         j->current_location.monotonic = usec;
1202         j->current_location.monotonic_set = true;
1203
1204         return 0;
1205 }
1206
1207 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1208         if (!j)
1209                 return -EINVAL;
1210
1211         reset_location(j);
1212         j->current_location.type = LOCATION_SEEK;
1213         j->current_location.realtime = usec;
1214         j->current_location.realtime_set = true;
1215
1216         return 0;
1217 }
1218
1219 _public_ int sd_journal_seek_head(sd_journal *j) {
1220         if (!j)
1221                 return -EINVAL;
1222
1223         reset_location(j);
1224         j->current_location.type = LOCATION_HEAD;
1225
1226         return 0;
1227 }
1228
1229 _public_ int sd_journal_seek_tail(sd_journal *j) {
1230         if (!j)
1231                 return -EINVAL;
1232
1233         reset_location(j);
1234         j->current_location.type = LOCATION_TAIL;
1235
1236         return 0;
1237 }
1238
1239 static void check_network(sd_journal *j, int fd) {
1240         struct statfs sfs;
1241
1242         assert(j);
1243
1244         if (j->on_network)
1245                 return;
1246
1247         if (fstatfs(fd, &sfs) < 0)
1248                 return;
1249
1250         j->on_network =
1251                 CMP_F_TYPE(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1252                 CMP_F_TYPE(sfs.f_type, CODA_SUPER_MAGIC) ||
1253                 CMP_F_TYPE(sfs.f_type, NCP_SUPER_MAGIC) ||
1254                 CMP_F_TYPE(sfs.f_type, NFS_SUPER_MAGIC) ||
1255                 CMP_F_TYPE(sfs.f_type, SMB_SUPER_MAGIC);
1256 }
1257
1258 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1259         char _cleanup_free_ *path = NULL;
1260         int r;
1261         JournalFile *f;
1262
1263         assert(j);
1264         assert(prefix);
1265         assert(filename);
1266
1267         if ((j->flags & SD_JOURNAL_SYSTEM_ONLY) &&
1268             !(streq(filename, "system.journal") ||
1269               streq(filename, "system.journal~") ||
1270               (startswith(filename, "system@") &&
1271                (endswith(filename, ".journal") || endswith(filename, ".journal~")))))
1272                 return 0;
1273
1274         path = strjoin(prefix, "/", filename, NULL);
1275         if (!path)
1276                 return -ENOMEM;
1277
1278         if (hashmap_get(j->files, path))
1279                 return 0;
1280
1281         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1282                 log_debug("Too many open journal files, not adding %s, ignoring.", path);
1283                 return set_put_error(j, -ETOOMANYREFS);
1284         }
1285
1286         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1287         if (r < 0) {
1288                 if (errno == ENOENT)
1289                         return 0;
1290
1291                 return r;
1292         }
1293
1294         /* journal_file_dump(f); */
1295
1296         r = hashmap_put(j->files, f->path, f);
1297         if (r < 0) {
1298                 journal_file_close(f);
1299                 return r;
1300         }
1301
1302         log_debug("File %s got added.", f->path);
1303
1304         check_network(j, f->fd);
1305
1306         j->current_invalidate_counter ++;
1307
1308         return 0;
1309 }
1310
1311 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1312         char *path;
1313         JournalFile *f;
1314
1315         assert(j);
1316         assert(prefix);
1317         assert(filename);
1318
1319         path = strjoin(prefix, "/", filename, NULL);
1320         if (!path)
1321                 return -ENOMEM;
1322
1323         f = hashmap_get(j->files, path);
1324         free(path);
1325         if (!f)
1326                 return 0;
1327
1328         hashmap_remove(j->files, f->path);
1329
1330         log_debug("File %s got removed.", f->path);
1331
1332         if (j->current_file == f) {
1333                 j->current_file = NULL;
1334                 j->current_field = 0;
1335         }
1336
1337         if (j->unique_file == f) {
1338                 j->unique_file = NULL;
1339                 j->unique_offset = 0;
1340         }
1341
1342         journal_file_close(f);
1343
1344         j->current_invalidate_counter ++;
1345
1346         return 0;
1347 }
1348
1349 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1350         char _cleanup_free_ *path = NULL;
1351         int r;
1352         DIR _cleanup_closedir_ *d = NULL;
1353         sd_id128_t id, mid;
1354         Directory *m;
1355
1356         assert(j);
1357         assert(prefix);
1358         assert(dirname);
1359
1360         log_debug("Considering %s/%s.", prefix, dirname);
1361
1362         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1363             (sd_id128_from_string(dirname, &id) < 0 ||
1364              sd_id128_get_machine(&mid) < 0 ||
1365              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1366             return 0;
1367
1368         path = strjoin(prefix, "/", dirname, NULL);
1369         if (!path)
1370                 return -ENOMEM;
1371
1372         d = opendir(path);
1373         if (!d) {
1374                 log_debug("Failed to open %s: %m", path);
1375                 if (errno == ENOENT)
1376                         return 0;
1377                 return -errno;
1378         }
1379
1380         m = hashmap_get(j->directories_by_path, path);
1381         if (!m) {
1382                 m = new0(Directory, 1);
1383                 if (!m)
1384                         return -ENOMEM;
1385
1386                 m->is_root = false;
1387                 m->path = path;
1388
1389                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1390                         free(m);
1391                         return -ENOMEM;
1392                 }
1393
1394                 path = NULL; /* avoid freeing in cleanup */
1395                 j->current_invalidate_counter ++;
1396
1397                 log_debug("Directory %s got added.", m->path);
1398
1399         } else if (m->is_root)
1400                 return 0;
1401
1402         if (m->wd <= 0 && j->inotify_fd >= 0) {
1403
1404                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1405                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1406                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1407                                           IN_ONLYDIR);
1408
1409                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1410                         inotify_rm_watch(j->inotify_fd, m->wd);
1411         }
1412
1413         for (;;) {
1414                 struct dirent *de;
1415                 union dirent_storage buf;
1416
1417                 r = readdir_r(d, &buf.de, &de);
1418                 if (r != 0 || !de)
1419                         break;
1420
1421                 if (dirent_is_file_with_suffix(de, ".journal") ||
1422                     dirent_is_file_with_suffix(de, ".journal~")) {
1423                         r = add_file(j, m->path, de->d_name);
1424                         if (r < 0) {
1425                                 log_debug("Failed to add file %s/%s: %s",
1426                                           m->path, de->d_name, strerror(-r));
1427                                 r = set_put_error(j, r);
1428                                 if (r < 0)
1429                                         return r;
1430                         }
1431                 }
1432         }
1433
1434         check_network(j, dirfd(d));
1435
1436         return 0;
1437 }
1438
1439 static int add_root_directory(sd_journal *j, const char *p) {
1440         DIR _cleanup_closedir_ *d = NULL;
1441         Directory *m;
1442         int r;
1443
1444         assert(j);
1445         assert(p);
1446
1447         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1448             !path_startswith(p, "/run"))
1449                 return -EINVAL;
1450
1451         d = opendir(p);
1452         if (!d)
1453                 return -errno;
1454
1455         m = hashmap_get(j->directories_by_path, p);
1456         if (!m) {
1457                 m = new0(Directory, 1);
1458                 if (!m)
1459                         return -ENOMEM;
1460
1461                 m->is_root = true;
1462                 m->path = strdup(p);
1463                 if (!m->path) {
1464                         free(m);
1465                         return -ENOMEM;
1466                 }
1467
1468                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1469                         free(m->path);
1470                         free(m);
1471                         return -ENOMEM;
1472                 }
1473
1474                 j->current_invalidate_counter ++;
1475
1476                 log_debug("Root directory %s got added.", m->path);
1477
1478         } else if (!m->is_root)
1479                 return 0;
1480
1481         if (m->wd <= 0 && j->inotify_fd >= 0) {
1482
1483                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1484                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1485                                           IN_ONLYDIR);
1486
1487                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1488                         inotify_rm_watch(j->inotify_fd, m->wd);
1489         }
1490
1491         for (;;) {
1492                 struct dirent *de;
1493                 union dirent_storage buf;
1494                 sd_id128_t id;
1495
1496                 r = readdir_r(d, &buf.de, &de);
1497                 if (r != 0 || !de)
1498                         break;
1499
1500                 if (dirent_is_file_with_suffix(de, ".journal") ||
1501                     dirent_is_file_with_suffix(de, ".journal~")) {
1502                         r = add_file(j, m->path, de->d_name);
1503                         if (r < 0) {
1504                                 log_debug("Failed to add file %s/%s: %s",
1505                                           m->path, de->d_name, strerror(-r));
1506                                 r = set_put_error(j, r);
1507                                 if (r < 0)
1508                                         return r;
1509                         }
1510                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1511                            sd_id128_from_string(de->d_name, &id) >= 0) {
1512
1513                         r = add_directory(j, m->path, de->d_name);
1514                         if (r < 0)
1515                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1516                 }
1517         }
1518
1519         check_network(j, dirfd(d));
1520
1521         return 0;
1522 }
1523
1524 static int remove_directory(sd_journal *j, Directory *d) {
1525         assert(j);
1526
1527         if (d->wd > 0) {
1528                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1529
1530                 if (j->inotify_fd >= 0)
1531                         inotify_rm_watch(j->inotify_fd, d->wd);
1532         }
1533
1534         hashmap_remove(j->directories_by_path, d->path);
1535
1536         if (d->is_root)
1537                 log_debug("Root directory %s got removed.", d->path);
1538         else
1539                 log_debug("Directory %s got removed.", d->path);
1540
1541         free(d->path);
1542         free(d);
1543
1544         return 0;
1545 }
1546
1547 static int add_search_paths(sd_journal *j) {
1548         int r;
1549         const char search_paths[] =
1550                 "/run/log/journal\0"
1551                 "/var/log/journal\0";
1552         const char *p;
1553
1554         assert(j);
1555
1556         /* We ignore most errors here, since the idea is to only open
1557          * what's actually accessible, and ignore the rest. */
1558
1559         NULSTR_FOREACH(p, search_paths) {
1560                 r = add_root_directory(j, p);
1561                 if (r < 0 && r != -ENOENT) {
1562                         r = set_put_error(j, r);
1563                         if (r < 0)
1564                                 return r;
1565                 }
1566         }
1567
1568         return 0;
1569 }
1570
1571 static int allocate_inotify(sd_journal *j) {
1572         assert(j);
1573
1574         if (j->inotify_fd < 0) {
1575                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1576                 if (j->inotify_fd < 0)
1577                         return -errno;
1578         }
1579
1580         if (!j->directories_by_wd) {
1581                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1582                 if (!j->directories_by_wd)
1583                         return -ENOMEM;
1584         }
1585
1586         return 0;
1587 }
1588
1589 static sd_journal *journal_new(int flags, const char *path) {
1590         sd_journal *j;
1591
1592         j = new0(sd_journal, 1);
1593         if (!j)
1594                 return NULL;
1595
1596         j->inotify_fd = -1;
1597         j->flags = flags;
1598         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1599
1600         if (path) {
1601                 j->path = strdup(path);
1602                 if (!j->path)
1603                         goto fail;
1604         }
1605
1606         j->files = hashmap_new(string_hash_func, string_compare_func);
1607         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1608         j->mmap = mmap_cache_new();
1609         if (!j->files || !j->directories_by_path || !j->mmap)
1610                 goto fail;
1611
1612         return j;
1613
1614 fail:
1615         sd_journal_close(j);
1616         return NULL;
1617 }
1618
1619 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1620         sd_journal *j;
1621         int r;
1622
1623         if (!ret)
1624                 return -EINVAL;
1625
1626         if (flags & ~(SD_JOURNAL_LOCAL_ONLY|
1627                       SD_JOURNAL_RUNTIME_ONLY|
1628                       SD_JOURNAL_SYSTEM_ONLY))
1629                 return -EINVAL;
1630
1631         j = journal_new(flags, NULL);
1632         if (!j)
1633                 return -ENOMEM;
1634
1635         r = add_search_paths(j);
1636         if (r < 0)
1637                 goto fail;
1638
1639         *ret = j;
1640         return 0;
1641
1642 fail:
1643         sd_journal_close(j);
1644
1645         return r;
1646 }
1647
1648 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1649         sd_journal *j;
1650         int r;
1651
1652         if (!ret)
1653                 return -EINVAL;
1654
1655         if (!path)
1656                 return -EINVAL;
1657
1658         if (flags != 0)
1659                 return -EINVAL;
1660
1661         j = journal_new(flags, path);
1662         if (!j)
1663                 return -ENOMEM;
1664
1665         r = add_root_directory(j, path);
1666         if (r < 0) {
1667                 set_put_error(j, r);
1668                 goto fail;
1669         }
1670
1671         *ret = j;
1672         return 0;
1673
1674 fail:
1675         sd_journal_close(j);
1676
1677         return r;
1678 }
1679
1680 _public_ void sd_journal_close(sd_journal *j) {
1681         Directory *d;
1682         JournalFile *f;
1683
1684         if (!j)
1685                 return;
1686
1687         sd_journal_flush_matches(j);
1688
1689         while ((f = hashmap_steal_first(j->files)))
1690                 journal_file_close(f);
1691
1692         hashmap_free(j->files);
1693
1694         while ((d = hashmap_first(j->directories_by_path)))
1695                 remove_directory(j, d);
1696
1697         while ((d = hashmap_first(j->directories_by_wd)))
1698                 remove_directory(j, d);
1699
1700         hashmap_free(j->directories_by_path);
1701         hashmap_free(j->directories_by_wd);
1702
1703         if (j->inotify_fd >= 0)
1704                 close_nointr_nofail(j->inotify_fd);
1705
1706         if (j->mmap)
1707                 mmap_cache_unref(j->mmap);
1708
1709         free(j->path);
1710         free(j->unique_field);
1711         set_free(j->errors);
1712         free(j);
1713 }
1714
1715 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1716         Object *o;
1717         JournalFile *f;
1718         int r;
1719
1720         if (!j)
1721                 return -EINVAL;
1722         if (!ret)
1723                 return -EINVAL;
1724
1725         f = j->current_file;
1726         if (!f)
1727                 return -EADDRNOTAVAIL;
1728
1729         if (f->current_offset <= 0)
1730                 return -EADDRNOTAVAIL;
1731
1732         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1733         if (r < 0)
1734                 return r;
1735
1736         *ret = le64toh(o->entry.realtime);
1737         return 0;
1738 }
1739
1740 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1741         Object *o;
1742         JournalFile *f;
1743         int r;
1744         sd_id128_t id;
1745
1746         if (!j)
1747                 return -EINVAL;
1748
1749         f = j->current_file;
1750         if (!f)
1751                 return -EADDRNOTAVAIL;
1752
1753         if (f->current_offset <= 0)
1754                 return -EADDRNOTAVAIL;
1755
1756         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1757         if (r < 0)
1758                 return r;
1759
1760         if (ret_boot_id)
1761                 *ret_boot_id = o->entry.boot_id;
1762         else {
1763                 r = sd_id128_get_boot(&id);
1764                 if (r < 0)
1765                         return r;
1766
1767                 if (!sd_id128_equal(id, o->entry.boot_id))
1768                         return -ESTALE;
1769         }
1770
1771         if (ret)
1772                 *ret = le64toh(o->entry.monotonic);
1773
1774         return 0;
1775 }
1776
1777 static bool field_is_valid(const char *field) {
1778         const char *p;
1779
1780         assert(field);
1781
1782         if (isempty(field))
1783                 return false;
1784
1785         if (startswith(field, "__"))
1786                 return false;
1787
1788         for (p = field; *p; p++) {
1789
1790                 if (*p == '_')
1791                         continue;
1792
1793                 if (*p >= 'A' && *p <= 'Z')
1794                         continue;
1795
1796                 if (*p >= '0' && *p <= '9')
1797                         continue;
1798
1799                 return false;
1800         }
1801
1802         return true;
1803 }
1804
1805 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1806         JournalFile *f;
1807         uint64_t i, n;
1808         size_t field_length;
1809         int r;
1810         Object *o;
1811
1812         if (!j)
1813                 return -EINVAL;
1814         if (!field)
1815                 return -EINVAL;
1816         if (!data)
1817                 return -EINVAL;
1818         if (!size)
1819                 return -EINVAL;
1820
1821         if (!field_is_valid(field))
1822                 return -EINVAL;
1823
1824         f = j->current_file;
1825         if (!f)
1826                 return -EADDRNOTAVAIL;
1827
1828         if (f->current_offset <= 0)
1829                 return -EADDRNOTAVAIL;
1830
1831         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1832         if (r < 0)
1833                 return r;
1834
1835         field_length = strlen(field);
1836
1837         n = journal_file_entry_n_items(o);
1838         for (i = 0; i < n; i++) {
1839                 uint64_t p, l;
1840                 le64_t le_hash;
1841                 size_t t;
1842
1843                 p = le64toh(o->entry.items[i].object_offset);
1844                 le_hash = o->entry.items[i].hash;
1845                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1846                 if (r < 0)
1847                         return r;
1848
1849                 if (le_hash != o->data.hash)
1850                         return -EBADMSG;
1851
1852                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1853
1854                 if (o->object.flags & OBJECT_COMPRESSED) {
1855
1856 #ifdef HAVE_XZ
1857                         if (uncompress_startswith(o->data.payload, l,
1858                                                   &f->compress_buffer, &f->compress_buffer_size,
1859                                                   field, field_length, '=')) {
1860
1861                                 uint64_t rsize;
1862
1863                                 if (!uncompress_blob(o->data.payload, l,
1864                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1865                                                      j->data_threshold))
1866                                         return -EBADMSG;
1867
1868                                 *data = f->compress_buffer;
1869                                 *size = (size_t) rsize;
1870
1871                                 return 0;
1872                         }
1873 #else
1874                         return -EPROTONOSUPPORT;
1875 #endif
1876
1877                 } else if (l >= field_length+1 &&
1878                            memcmp(o->data.payload, field, field_length) == 0 &&
1879                            o->data.payload[field_length] == '=') {
1880
1881                         t = (size_t) l;
1882
1883                         if ((uint64_t) t != l)
1884                                 return -E2BIG;
1885
1886                         *data = o->data.payload;
1887                         *size = t;
1888
1889                         return 0;
1890                 }
1891
1892                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1893                 if (r < 0)
1894                         return r;
1895         }
1896
1897         return -ENOENT;
1898 }
1899
1900 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1901         size_t t;
1902         uint64_t l;
1903
1904         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1905         t = (size_t) l;
1906
1907         /* We can't read objects larger than 4G on a 32bit machine */
1908         if ((uint64_t) t != l)
1909                 return -E2BIG;
1910
1911         if (o->object.flags & OBJECT_COMPRESSED) {
1912 #ifdef HAVE_XZ
1913                 uint64_t rsize;
1914
1915                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
1916                         return -EBADMSG;
1917
1918                 *data = f->compress_buffer;
1919                 *size = (size_t) rsize;
1920 #else
1921                 return -EPROTONOSUPPORT;
1922 #endif
1923         } else {
1924                 *data = o->data.payload;
1925                 *size = t;
1926         }
1927
1928         return 0;
1929 }
1930
1931 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
1932         JournalFile *f;
1933         uint64_t p, n;
1934         le64_t le_hash;
1935         int r;
1936         Object *o;
1937
1938         if (!j)
1939                 return -EINVAL;
1940         if (!data)
1941                 return -EINVAL;
1942         if (!size)
1943                 return -EINVAL;
1944
1945         f = j->current_file;
1946         if (!f)
1947                 return -EADDRNOTAVAIL;
1948
1949         if (f->current_offset <= 0)
1950                 return -EADDRNOTAVAIL;
1951
1952         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1953         if (r < 0)
1954                 return r;
1955
1956         n = journal_file_entry_n_items(o);
1957         if (j->current_field >= n)
1958                 return 0;
1959
1960         p = le64toh(o->entry.items[j->current_field].object_offset);
1961         le_hash = o->entry.items[j->current_field].hash;
1962         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1963         if (r < 0)
1964                 return r;
1965
1966         if (le_hash != o->data.hash)
1967                 return -EBADMSG;
1968
1969         r = return_data(j, f, o, data, size);
1970         if (r < 0)
1971                 return r;
1972
1973         j->current_field ++;
1974
1975         return 1;
1976 }
1977
1978 _public_ void sd_journal_restart_data(sd_journal *j) {
1979         if (!j)
1980                 return;
1981
1982         j->current_field = 0;
1983 }
1984
1985 _public_ int sd_journal_get_fd(sd_journal *j) {
1986         int r;
1987
1988         if (!j)
1989                 return -EINVAL;
1990
1991         if (j->inotify_fd >= 0)
1992                 return j->inotify_fd;
1993
1994         r = allocate_inotify(j);
1995         if (r < 0)
1996                 return r;
1997
1998         /* Iterate through all dirs again, to add them to the
1999          * inotify */
2000         if (j->path)
2001                 r = add_root_directory(j, j->path);
2002         else
2003                 r = add_search_paths(j);
2004         if (r < 0)
2005                 return r;
2006
2007         return j->inotify_fd;
2008 }
2009
2010 _public_ int sd_journal_get_events(sd_journal *j) {
2011         int fd;
2012
2013         if (!j)
2014                 return -EINVAL;
2015
2016         fd = sd_journal_get_fd(j);
2017         if (fd < 0)
2018                 return fd;
2019
2020         return POLLIN;
2021 }
2022
2023 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2024         int fd;
2025
2026         if (!j)
2027                 return -EINVAL;
2028         if (!timeout_usec)
2029                 return -EINVAL;
2030
2031         fd = sd_journal_get_fd(j);
2032         if (fd < 0)
2033                 return fd;
2034
2035         if (!j->on_network) {
2036                 *timeout_usec = (uint64_t) -1;
2037                 return 0;
2038         }
2039
2040         /* If we are on the network we need to regularly check for
2041          * changes manually */
2042
2043         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2044         return 1;
2045 }
2046
2047 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2048         Directory *d;
2049         int r;
2050
2051         assert(j);
2052         assert(e);
2053
2054         /* Is this a subdirectory we watch? */
2055         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2056         if (d) {
2057                 sd_id128_t id;
2058
2059                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2060                     (endswith(e->name, ".journal") ||
2061                      endswith(e->name, ".journal~"))) {
2062
2063                         /* Event for a journal file */
2064
2065                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2066                                 r = add_file(j, d->path, e->name);
2067                                 if (r < 0) {
2068                                         log_debug("Failed to add file %s/%s: %s",
2069                                                   d->path, e->name, strerror(-r));
2070                                         set_put_error(j, r);
2071                                 }
2072
2073                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2074
2075                                 r = remove_file(j, d->path, e->name);
2076                                 if (r < 0)
2077                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2078                         }
2079
2080                 } else if (!d->is_root && e->len == 0) {
2081
2082                         /* Event for a subdirectory */
2083
2084                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2085                                 r = remove_directory(j, d);
2086                                 if (r < 0)
2087                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2088                         }
2089
2090
2091                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2092
2093                         /* Event for root directory */
2094
2095                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2096                                 r = add_directory(j, d->path, e->name);
2097                                 if (r < 0)
2098                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2099                         }
2100                 }
2101
2102                 return;
2103         }
2104
2105         if (e->mask & IN_IGNORED)
2106                 return;
2107
2108         log_warning("Unknown inotify event.");
2109 }
2110
2111 static int determine_change(sd_journal *j) {
2112         bool b;
2113
2114         assert(j);
2115
2116         b = j->current_invalidate_counter != j->last_invalidate_counter;
2117         j->last_invalidate_counter = j->current_invalidate_counter;
2118
2119         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2120 }
2121
2122 _public_ int sd_journal_process(sd_journal *j) {
2123         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2124         bool got_something = false;
2125
2126         if (!j)
2127                 return -EINVAL;
2128
2129         j->last_process_usec = now(CLOCK_MONOTONIC);
2130
2131         for (;;) {
2132                 struct inotify_event *e;
2133                 ssize_t l;
2134
2135                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2136                 if (l < 0) {
2137                         if (errno == EAGAIN || errno == EINTR)
2138                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2139
2140                         return -errno;
2141                 }
2142
2143                 got_something = true;
2144
2145                 e = (struct inotify_event*) buffer;
2146                 while (l > 0) {
2147                         size_t step;
2148
2149                         process_inotify_event(j, e);
2150
2151                         step = sizeof(struct inotify_event) + e->len;
2152                         assert(step <= (size_t) l);
2153
2154                         e = (struct inotify_event*) ((uint8_t*) e + step);
2155                         l -= step;
2156                 }
2157         }
2158
2159         return determine_change(j);
2160 }
2161
2162 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2163         int r;
2164         uint64_t t;
2165
2166         assert(j);
2167
2168         if (j->inotify_fd < 0) {
2169
2170                 /* This is the first invocation, hence create the
2171                  * inotify watch */
2172                 r = sd_journal_get_fd(j);
2173                 if (r < 0)
2174                         return r;
2175
2176                 /* The journal might have changed since the context
2177                  * object was created and we weren't watching before,
2178                  * hence don't wait for anything, and return
2179                  * immediately. */
2180                 return determine_change(j);
2181         }
2182
2183         r = sd_journal_get_timeout(j, &t);
2184         if (r < 0)
2185                 return r;
2186
2187         if (t != (uint64_t) -1) {
2188                 usec_t n;
2189
2190                 n = now(CLOCK_MONOTONIC);
2191                 t = t > n ? t - n : 0;
2192
2193                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2194                         timeout_usec = t;
2195         }
2196
2197         do {
2198                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2199         } while (r == -EINTR);
2200
2201         if (r < 0)
2202                 return r;
2203
2204         return sd_journal_process(j);
2205 }
2206
2207 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2208         Iterator i;
2209         JournalFile *f;
2210         bool first = true;
2211         int r;
2212
2213         if (!j)
2214                 return -EINVAL;
2215         if (!from && !to)
2216                 return -EINVAL;
2217
2218         HASHMAP_FOREACH(f, j->files, i) {
2219                 usec_t fr, t;
2220
2221                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2222                 if (r == -ENOENT)
2223                         continue;
2224                 if (r < 0)
2225                         return r;
2226                 if (r == 0)
2227                         continue;
2228
2229                 if (first) {
2230                         if (from)
2231                                 *from = fr;
2232                         if (to)
2233                                 *to = t;
2234                         first = false;
2235                 } else {
2236                         if (from)
2237                                 *from = MIN(fr, *from);
2238                         if (to)
2239                                 *to = MAX(t, *to);
2240                 }
2241         }
2242
2243         return first ? 0 : 1;
2244 }
2245
2246 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2247         Iterator i;
2248         JournalFile *f;
2249         bool first = true;
2250         int r;
2251
2252         if (!j)
2253                 return -EINVAL;
2254         if (!from && !to)
2255                 return -EINVAL;
2256
2257         HASHMAP_FOREACH(f, j->files, i) {
2258                 usec_t fr, t;
2259
2260                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2261                 if (r == -ENOENT)
2262                         continue;
2263                 if (r < 0)
2264                         return r;
2265                 if (r == 0)
2266                         continue;
2267
2268                 if (first) {
2269                         if (from)
2270                                 *from = fr;
2271                         if (to)
2272                                 *to = t;
2273                         first = false;
2274                 } else {
2275                         if (from)
2276                                 *from = MIN(fr, *from);
2277                         if (to)
2278                                 *to = MAX(t, *to);
2279                 }
2280         }
2281
2282         return first ? 0 : 1;
2283 }
2284
2285 void journal_print_header(sd_journal *j) {
2286         Iterator i;
2287         JournalFile *f;
2288         bool newline = false;
2289
2290         assert(j);
2291
2292         HASHMAP_FOREACH(f, j->files, i) {
2293                 if (newline)
2294                         putchar('\n');
2295                 else
2296                         newline = true;
2297
2298                 journal_file_print_header(f);
2299         }
2300 }
2301
2302 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2303         Iterator i;
2304         JournalFile *f;
2305         uint64_t sum = 0;
2306
2307         if (!j)
2308                 return -EINVAL;
2309         if (!bytes)
2310                 return -EINVAL;
2311
2312         HASHMAP_FOREACH(f, j->files, i) {
2313                 struct stat st;
2314
2315                 if (fstat(f->fd, &st) < 0)
2316                         return -errno;
2317
2318                 sum += (uint64_t) st.st_blocks * 512ULL;
2319         }
2320
2321         *bytes = sum;
2322         return 0;
2323 }
2324
2325 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2326         char *f;
2327
2328         if (!j)
2329                 return -EINVAL;
2330         if (isempty(field))
2331                 return -EINVAL;
2332         if (!field_is_valid(field))
2333                 return -EINVAL;
2334
2335         f = strdup(field);
2336         if (!f)
2337                 return -ENOMEM;
2338
2339         free(j->unique_field);
2340         j->unique_field = f;
2341         j->unique_file = NULL;
2342         j->unique_offset = 0;
2343
2344         return 0;
2345 }
2346
2347 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2348         Object *o;
2349         size_t k;
2350         int r;
2351
2352         if (!j)
2353                 return -EINVAL;
2354         if (!data)
2355                 return -EINVAL;
2356         if (!l)
2357                 return -EINVAL;
2358         if (!j->unique_field)
2359                 return -EINVAL;
2360
2361         k = strlen(j->unique_field);
2362
2363         if (!j->unique_file) {
2364                 j->unique_file = hashmap_first(j->files);
2365                 if (!j->unique_file)
2366                         return 0;
2367                 j->unique_offset = 0;
2368         }
2369
2370         for (;;) {
2371                 JournalFile *of;
2372                 Iterator i;
2373                 const void *odata;
2374                 size_t ol;
2375                 bool found;
2376
2377                 /* Proceed to next data object in the field's linked list */
2378                 if (j->unique_offset == 0) {
2379                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2380                         if (r < 0)
2381                                 return r;
2382
2383                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2384                 } else {
2385                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2386                         if (r < 0)
2387                                 return r;
2388
2389                         j->unique_offset = le64toh(o->data.next_field_offset);
2390                 }
2391
2392                 /* We reached the end of the list? Then start again, with the next file */
2393                 if (j->unique_offset == 0) {
2394                         JournalFile *n;
2395
2396                         n = hashmap_next(j->files, j->unique_file->path);
2397                         if (!n)
2398                                 return 0;
2399
2400                         j->unique_file = n;
2401                         continue;
2402                 }
2403
2404                 /* We do not use the type context here, but 0 instead,
2405                  * so that we can look at this data object at the same
2406                  * time as one on another file */
2407                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2408                 if (r < 0)
2409                         return r;
2410
2411                 /* Let's do the type check by hand, since we used 0 context above. */
2412                 if (o->object.type != OBJECT_DATA)
2413                         return -EBADMSG;
2414
2415                 r = return_data(j, j->unique_file, o, &odata, &ol);
2416                 if (r < 0)
2417                         return r;
2418
2419                 /* OK, now let's see if we already returned this data
2420                  * object by checking if it exists in the earlier
2421                  * traversed files. */
2422                 found = false;
2423                 HASHMAP_FOREACH(of, j->files, i) {
2424                         Object *oo;
2425                         uint64_t op;
2426
2427                         if (of == j->unique_file)
2428                                 break;
2429
2430                         /* Skip this file it didn't have any fields
2431                          * indexed */
2432                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2433                             le64toh(of->header->n_fields) <= 0)
2434                                 continue;
2435
2436                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2437                         if (r < 0)
2438                                 return r;
2439
2440                         if (r > 0)
2441                                 found = true;
2442                 }
2443
2444                 if (found)
2445                         continue;
2446
2447                 r = return_data(j, j->unique_file, o, data, l);
2448                 if (r < 0)
2449                         return r;
2450
2451                 return 1;
2452         }
2453 }
2454
2455 _public_ void sd_journal_restart_unique(sd_journal *j) {
2456         if (!j)
2457                 return;
2458
2459         j->unique_file = NULL;
2460         j->unique_offset = 0;
2461 }
2462
2463 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2464         if (!j)
2465                 return -EINVAL;
2466
2467         return !j->on_network;
2468 }
2469
2470 static char *lookup_field(const char *field, void *userdata) {
2471         sd_journal *j = userdata;
2472         const void *data;
2473         size_t size, d;
2474         int r;
2475
2476         assert(field);
2477         assert(j);
2478
2479         r = sd_journal_get_data(j, field, &data, &size);
2480         if (r < 0 ||
2481             size > REPLACE_VAR_MAX)
2482                 return strdup(field);
2483
2484         d = strlen(field) + 1;
2485
2486         return strndup((const char*) data + d, size - d);
2487 }
2488
2489 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2490         const void *data;
2491         size_t size;
2492         sd_id128_t id;
2493         _cleanup_free_ char *text = NULL, *cid = NULL;
2494         char *t;
2495         int r;
2496
2497         if (!j)
2498                 return -EINVAL;
2499         if (!ret)
2500                 return -EINVAL;
2501
2502         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2503         if (r < 0)
2504                 return r;
2505
2506         cid = strndup((const char*) data + 11, size - 11);
2507         if (!cid)
2508                 return -ENOMEM;
2509
2510         r = sd_id128_from_string(cid, &id);
2511         if (r < 0)
2512                 return r;
2513
2514         r = catalog_get(CATALOG_DATABASE, id, &text);
2515         if (r < 0)
2516                 return r;
2517
2518         t = replace_var(text, lookup_field, j);
2519         if (!t)
2520                 return -ENOMEM;
2521
2522         *ret = t;
2523         return 0;
2524 }
2525
2526 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2527         if (!ret)
2528                 return -EINVAL;
2529
2530         return catalog_get(CATALOG_DATABASE, id, ret);
2531 }
2532
2533 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2534         if (!j)
2535                 return -EINVAL;
2536
2537         j->data_threshold = sz;
2538         return 0;
2539 }
2540
2541 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2542         if (!j)
2543                 return -EINVAL;
2544         if (!sz)
2545                 return -EINVAL;
2546
2547         *sz = j->data_threshold;
2548         return 0;
2549 }