src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <stddef.h>
  25 #include <unistd.h>
  26 #include <sys/inotify.h>
  27 #include <sys/poll.h>
  28 #include <sys/vfs.h>
  29 #include <linux/magic.h>
  30
  31 #include "sd-journal.h"
  32 #include "journal-def.h"
  33 #include "journal-file.h"
  34 #include "hashmap.h"
  35 #include "list.h"
  36 #include "strv.h"
  37 #include "path-util.h"
  38 #include "lookup3.h"
  39 #include "compress.h"
  40 #include "journal-internal.h"
  41 #include "missing.h"
  42 #include "catalog.h"
  43 #include "replace-var.h"
  44 #include "fileio.h"
  45
  46 #define JOURNAL_FILES_MAX 1024
  47
  48 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  49
  50 #define REPLACE_VAR_MAX 256
  51
  52 #define DEFAULT_DATA_THRESHOLD (64*1024)
  53
  54 static bool journal_pid_changed(sd_journal *j) {
  55         assert(j);
  56
  57         /* We don't support people creating a journal object and
  58          * keeping it around over a fork(). Let's complain. */
  59
  60         return j->original_pid != getpid();
  61 }
  62
  63 /* We return an error here only if we didn't manage to
  64    memorize the real error. */
  65 static int set_put_error(sd_journal *j, int r) {
  66         int k;
  67
  68         if (r >= 0)
  69                 return r;
  70
  71         k = set_ensure_allocated(&j->errors, trivial_hash_func, trivial_compare_func);
  72         if (k < 0)
  73                 return k;
  74
  75         return set_put(j->errors, INT_TO_PTR(r));
  76 }
  77
  78 static void detach_location(sd_journal *j) {
  79         Iterator i;
  80         JournalFile *f;
  81
  82         assert(j);
  83
  84         j->current_file = NULL;
  85         j->current_field = 0;
  86
  87         HASHMAP_FOREACH(f, j->files, i)
  88                 f->current_offset = 0;
  89 }
  90
  91 static void reset_location(sd_journal *j) {
  92         assert(j);
  93
  94         detach_location(j);
  95         zero(j->current_location);
  96 }
  97
  98 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
  99         assert(l);
 100         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 101         assert(f);
 102         assert(o->object.type == OBJECT_ENTRY);
 103
 104         l->type = type;
 105         l->seqnum = le64toh(o->entry.seqnum);
 106         l->seqnum_id = f->header->seqnum_id;
 107         l->realtime = le64toh(o->entry.realtime);
 108         l->monotonic = le64toh(o->entry.monotonic);
 109         l->boot_id = o->entry.boot_id;
 110         l->xor_hash = le64toh(o->entry.xor_hash);
 111
 112         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 113 }
 114
 115 static void set_location(sd_journal *j, LocationType type, JournalFile *f, Object *o,
 116                          direction_t direction, uint64_t offset) {
 117         assert(j);
 118         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 119         assert(f);
 120         assert(o);
 121
 122         init_location(&j->current_location, type, f, o);
 123
 124         j->current_file = f;
 125         j->current_field = 0;
 126
 127         f->last_direction = direction;
 128         f->current_offset = offset;
 129 }
 130
 131 static int match_is_valid(const void *data, size_t size) {
 132         const char *b, *p;
 133
 134         assert(data);
 135
 136         if (size < 2)
 137                 return false;
 138
 139         if (startswith(data, "__"))
 140                 return false;
 141
 142         b = data;
 143         for (p = b; p < b + size; p++) {
 144
 145                 if (*p == '=')
 146                         return p > b;
 147
 148                 if (*p == '_')
 149                         continue;
 150
 151                 if (*p >= 'A' && *p <= 'Z')
 152                         continue;
 153
 154                 if (*p >= '0' && *p <= '9')
 155                         continue;
 156
 157                 return false;
 158         }
 159
 160         return false;
 161 }
 162
 163 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 164         const uint8_t *a = _a, *b = _b;
 165         size_t j;
 166
 167         for (j = 0; j < s && j < t; j++) {
 168
 169                 if (a[j] != b[j])
 170                         return false;
 171
 172                 if (a[j] == '=')
 173                         return true;
 174         }
 175
 176         assert_not_reached("\"=\" not found");
 177 }
 178
 179 static Match *match_new(Match *p, MatchType t) {
 180         Match *m;
 181
 182         m = new0(Match, 1);
 183         if (!m)
 184                 return NULL;
 185
 186         m->type = t;
 187
 188         if (p) {
 189                 m->parent = p;
 190                 LIST_PREPEND(matches, p->matches, m);
 191         }
 192
 193         return m;
 194 }
 195
 196 static void match_free(Match *m) {
 197         assert(m);
 198
 199         while (m->matches)
 200                 match_free(m->matches);
 201
 202         if (m->parent)
 203                 LIST_REMOVE(matches, m->parent->matches, m);
 204
 205         free(m->data);
 206         free(m);
 207 }
 208
 209 static void match_free_if_empty(Match *m) {
 210         if (!m || m->matches)
 211                 return;
 212
 213         match_free(m);
 214 }
 215
 216 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 217         Match *l3, *l4, *add_here = NULL, *m;
 218         le64_t le_hash;
 219
 220         assert_return(j, -EINVAL);
 221         assert_return(!journal_pid_changed(j), -ECHILD);
 222         assert_return(data, -EINVAL);
 223
 224         if (size == 0)
 225                 size = strlen(data);
 226
 227         assert_return(match_is_valid(data, size), -EINVAL);
 228
 229         /* level 0: AND term
 230          * level 1: OR terms
 231          * level 2: AND terms
 232          * level 3: OR terms
 233          * level 4: concrete matches */
 234
 235         if (!j->level0) {
 236                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 237                 if (!j->level0)
 238                         return -ENOMEM;
 239         }
 240
 241         if (!j->level1) {
 242                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 243                 if (!j->level1)
 244                         return -ENOMEM;
 245         }
 246
 247         if (!j->level2) {
 248                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 249                 if (!j->level2)
 250                         return -ENOMEM;
 251         }
 252
 253         assert(j->level0->type == MATCH_AND_TERM);
 254         assert(j->level1->type == MATCH_OR_TERM);
 255         assert(j->level2->type == MATCH_AND_TERM);
 256
 257         le_hash = htole64(hash64(data, size));
 258
 259         LIST_FOREACH(matches, l3, j->level2->matches) {
 260                 assert(l3->type == MATCH_OR_TERM);
 261
 262                 LIST_FOREACH(matches, l4, l3->matches) {
 263                         assert(l4->type == MATCH_DISCRETE);
 264
 265                         /* Exactly the same match already? Then ignore
 266                          * this addition */
 267                         if (l4->le_hash == le_hash &&
 268                             l4->size == size &&
 269                             memcmp(l4->data, data, size) == 0)
 270                                 return 0;
 271
 272                         /* Same field? Then let's add this to this OR term */
 273                         if (same_field(data, size, l4->data, l4->size)) {
 274                                 add_here = l3;
 275                                 break;
 276                         }
 277                 }
 278
 279                 if (add_here)
 280                         break;
 281         }
 282
 283         if (!add_here) {
 284                 add_here = match_new(j->level2, MATCH_OR_TERM);
 285                 if (!add_here)
 286                         goto fail;
 287         }
 288
 289         m = match_new(add_here, MATCH_DISCRETE);
 290         if (!m)
 291                 goto fail;
 292
 293         m->le_hash = le_hash;
 294         m->size = size;
 295         m->data = memdup(data, size);
 296         if (!m->data)
 297                 goto fail;
 298
 299         detach_location(j);
 300
 301         return 0;
 302
 303 fail:
 304         match_free_if_empty(add_here);
 305         match_free_if_empty(j->level2);
 306         match_free_if_empty(j->level1);
 307         match_free_if_empty(j->level0);
 308
 309         return -ENOMEM;
 310 }
 311
 312 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 313         assert_return(j, -EINVAL);
 314         assert_return(!journal_pid_changed(j), -ECHILD);
 315
 316         if (!j->level0)
 317                 return 0;
 318
 319         if (!j->level1)
 320                 return 0;
 321
 322         if (!j->level1->matches)
 323                 return 0;
 324
 325         j->level1 = NULL;
 326         j->level2 = NULL;
 327
 328         return 0;
 329 }
 330
 331 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 332         assert_return(j, -EINVAL);
 333         assert_return(!journal_pid_changed(j), -ECHILD);
 334
 335         if (!j->level0)
 336                 return 0;
 337
 338         if (!j->level1)
 339                 return 0;
 340
 341         if (!j->level2)
 342                 return 0;
 343
 344         if (!j->level2->matches)
 345                 return 0;
 346
 347         j->level2 = NULL;
 348         return 0;
 349 }
 350
 351 static char *match_make_string(Match *m) {
 352         char *p, *r;
 353         Match *i;
 354         bool enclose = false;
 355
 356         if (!m)
 357                 return strdup("none");
 358
 359         if (m->type == MATCH_DISCRETE)
 360                 return strndup(m->data, m->size);
 361
 362         p = NULL;
 363         LIST_FOREACH(matches, i, m->matches) {
 364                 char *t, *k;
 365
 366                 t = match_make_string(i);
 367                 if (!t) {
 368                         free(p);
 369                         return NULL;
 370                 }
 371
 372                 if (p) {
 373                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 374                         free(p);
 375                         free(t);
 376
 377                         if (!k)
 378                                 return NULL;
 379
 380                         p = k;
 381
 382                         enclose = true;
 383                 } else
 384                         p = t;
 385         }
 386
 387         if (enclose) {
 388                 r = strjoin("(", p, ")", NULL);
 389                 free(p);
 390                 return r;
 391         }
 392
 393         return p;
 394 }
 395
 396 char *journal_make_match_string(sd_journal *j) {
 397         assert(j);
 398
 399         return match_make_string(j->level0);
 400 }
 401
 402 _public_ void sd_journal_flush_matches(sd_journal *j) {
 403         if (!j)
 404                 return;
 405
 406         if (j->level0)
 407                 match_free(j->level0);
 408
 409         j->level0 = j->level1 = j->level2 = NULL;
 410
 411         detach_location(j);
 412 }
 413
 414 static int compare_entry_order(JournalFile *af, Object *_ao,
 415                                JournalFile *bf, uint64_t bp) {
 416
 417         uint64_t a, b;
 418         Object *ao, *bo;
 419         int r;
 420
 421         assert(af);
 422         assert(bf);
 423         assert(_ao);
 424
 425         /* The mmap cache might invalidate the object from the first
 426          * file if we look at the one from the second file. Hence
 427          * temporarily copy the header of the first one, and look at
 428          * that only. */
 429         ao = alloca(offsetof(EntryObject, items));
 430         memcpy(ao, _ao, offsetof(EntryObject, items));
 431
 432         r = journal_file_move_to_object(bf, OBJECT_ENTRY, bp, &bo);
 433         if (r < 0)
 434                 return strcmp(af->path, bf->path);
 435
 436         /* We operate on two different files here, hence we can access
 437          * two objects at the same time, which we normally can't.
 438          *
 439          * If contents and timestamps match, these entries are
 440          * identical, even if the seqnum does not match */
 441
 442         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id) &&
 443             ao->entry.monotonic == bo->entry.monotonic &&
 444             ao->entry.realtime == bo->entry.realtime &&
 445             ao->entry.xor_hash == bo->entry.xor_hash)
 446                 return 0;
 447
 448         if (sd_id128_equal(af->header->seqnum_id, bf->header->seqnum_id)) {
 449
 450                 /* If this is from the same seqnum source, compare
 451                  * seqnums */
 452                 a = le64toh(ao->entry.seqnum);
 453                 b = le64toh(bo->entry.seqnum);
 454
 455                 if (a < b)
 456                         return -1;
 457                 if (a > b)
 458                         return 1;
 459
 460                 /* Wow! This is weird, different data but the same
 461                  * seqnums? Something is borked, but let's make the
 462                  * best of it and compare by time. */
 463         }
 464
 465         if (sd_id128_equal(ao->entry.boot_id, bo->entry.boot_id)) {
 466
 467                 /* If the boot id matches compare monotonic time */
 468                 a = le64toh(ao->entry.monotonic);
 469                 b = le64toh(bo->entry.monotonic);
 470
 471                 if (a < b)
 472                         return -1;
 473                 if (a > b)
 474                         return 1;
 475         }
 476
 477         /* Otherwise compare UTC time */
 478         a = le64toh(ao->entry.realtime);
 479         b = le64toh(bo->entry.realtime);
 480
 481         if (a < b)
 482                 return -1;
 483         if (a > b)
 484                 return 1;
 485
 486         /* Finally, compare by contents */
 487         a = le64toh(ao->entry.xor_hash);
 488         b = le64toh(bo->entry.xor_hash);
 489
 490         if (a < b)
 491                 return -1;
 492         if (a > b)
 493                 return 1;
 494
 495         return 0;
 496 }
 497
 498 _pure_ static int compare_with_location(JournalFile *af, Object *ao, Location *l) {
 499         uint64_t a;
 500
 501         assert(af);
 502         assert(ao);
 503         assert(l);
 504         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 505
 506         if (l->monotonic_set &&
 507             sd_id128_equal(ao->entry.boot_id, l->boot_id) &&
 508             l->realtime_set &&
 509             le64toh(ao->entry.realtime) == l->realtime &&
 510             l->xor_hash_set &&
 511             le64toh(ao->entry.xor_hash) == l->xor_hash)
 512                 return 0;
 513
 514         if (l->seqnum_set &&
 515             sd_id128_equal(af->header->seqnum_id, l->seqnum_id)) {
 516
 517                 a = le64toh(ao->entry.seqnum);
 518
 519                 if (a < l->seqnum)
 520                         return -1;
 521                 if (a > l->seqnum)
 522                         return 1;
 523         }
 524
 525         if (l->monotonic_set &&
 526             sd_id128_equal(ao->entry.boot_id, l->boot_id)) {
 527
 528                 a = le64toh(ao->entry.monotonic);
 529
 530                 if (a < l->monotonic)
 531                         return -1;
 532                 if (a > l->monotonic)
 533                         return 1;
 534         }
 535
 536         if (l->realtime_set) {
 537
 538                 a = le64toh(ao->entry.realtime);
 539
 540                 if (a < l->realtime)
 541                         return -1;
 542                 if (a > l->realtime)
 543                         return 1;
 544         }
 545
 546         if (l->xor_hash_set) {
 547                 a = le64toh(ao->entry.xor_hash);
 548
 549                 if (a < l->xor_hash)
 550                         return -1;
 551                 if (a > l->xor_hash)
 552                         return 1;
 553         }
 554
 555         return 0;
 556 }
 557
 558 static int next_for_match(
 559                 sd_journal *j,
 560                 Match *m,
 561                 JournalFile *f,
 562                 uint64_t after_offset,
 563                 direction_t direction,
 564                 Object **ret,
 565                 uint64_t *offset) {
 566
 567         int r;
 568         uint64_t np = 0;
 569         Object *n;
 570
 571         assert(j);
 572         assert(m);
 573         assert(f);
 574
 575         if (m->type == MATCH_DISCRETE) {
 576                 uint64_t dp;
 577
 578                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 579                 if (r <= 0)
 580                         return r;
 581
 582                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 583
 584         } else if (m->type == MATCH_OR_TERM) {
 585                 Match *i;
 586
 587                 /* Find the earliest match beyond after_offset */
 588
 589                 LIST_FOREACH(matches, i, m->matches) {
 590                         uint64_t cp;
 591
 592                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 593                         if (r < 0)
 594                                 return r;
 595                         else if (r > 0) {
 596                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 597                                         np = cp;
 598                         }
 599                 }
 600
 601                 if (np == 0)
 602                         return 0;
 603
 604         } else if (m->type == MATCH_AND_TERM) {
 605                 Match *i, *last_moved;
 606
 607                 /* Always jump to the next matching entry and repeat
 608                  * this until we find an offset that matches for all
 609                  * matches. */
 610
 611                 if (!m->matches)
 612                         return 0;
 613
 614                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 615                 if (r <= 0)
 616                         return r;
 617
 618                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 619                 last_moved = m->matches;
 620
 621                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 622                         uint64_t cp;
 623
 624                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 625                         if (r <= 0)
 626                                 return r;
 627
 628                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 629                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 630                                 np = cp;
 631                                 last_moved = i;
 632                         }
 633                 }
 634         }
 635
 636         assert(np > 0);
 637
 638         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 639         if (r < 0)
 640                 return r;
 641
 642         if (ret)
 643                 *ret = n;
 644         if (offset)
 645                 *offset = np;
 646
 647         return 1;
 648 }
 649
 650 static int find_location_for_match(
 651                 sd_journal *j,
 652                 Match *m,
 653                 JournalFile *f,
 654                 direction_t direction,
 655                 Object **ret,
 656                 uint64_t *offset) {
 657
 658         int r;
 659
 660         assert(j);
 661         assert(m);
 662         assert(f);
 663
 664         if (m->type == MATCH_DISCRETE) {
 665                 uint64_t dp;
 666
 667                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 668                 if (r <= 0)
 669                         return r;
 670
 671                 /* FIXME: missing: find by monotonic */
 672
 673                 if (j->current_location.type == LOCATION_HEAD)
 674                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 675                 if (j->current_location.type == LOCATION_TAIL)
 676                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 677                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 678                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 679                 if (j->current_location.monotonic_set) {
 680                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 681                         if (r != -ENOENT)
 682                                 return r;
 683                 }
 684                 if (j->current_location.realtime_set)
 685                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 686
 687                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 688
 689         } else if (m->type == MATCH_OR_TERM) {
 690                 uint64_t np = 0;
 691                 Object *n;
 692                 Match *i;
 693
 694                 /* Find the earliest match */
 695
 696                 LIST_FOREACH(matches, i, m->matches) {
 697                         uint64_t cp;
 698
 699                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 700                         if (r < 0)
 701                                 return r;
 702                         else if (r > 0) {
 703                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 704                                         np = cp;
 705                         }
 706                 }
 707
 708                 if (np == 0)
 709                         return 0;
 710
 711                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 712                 if (r < 0)
 713                         return r;
 714
 715                 if (ret)
 716                         *ret = n;
 717                 if (offset)
 718                         *offset = np;
 719
 720                 return 1;
 721
 722         } else {
 723                 Match *i;
 724                 uint64_t np = 0;
 725
 726                 assert(m->type == MATCH_AND_TERM);
 727
 728                 /* First jump to the last match, and then find the
 729                  * next one where all matches match */
 730
 731                 if (!m->matches)
 732                         return 0;
 733
 734                 LIST_FOREACH(matches, i, m->matches) {
 735                         uint64_t cp;
 736
 737                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 738                         if (r <= 0)
 739                                 return r;
 740
 741                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 742                                 np = cp;
 743                 }
 744
 745                 return next_for_match(j, m, f, np, direction, ret, offset);
 746         }
 747 }
 748
 749 static int find_location_with_matches(
 750                 sd_journal *j,
 751                 JournalFile *f,
 752                 direction_t direction,
 753                 Object **ret,
 754                 uint64_t *offset) {
 755
 756         int r;
 757
 758         assert(j);
 759         assert(f);
 760         assert(ret);
 761         assert(offset);
 762
 763         if (!j->level0) {
 764                 /* No matches is simple */
 765
 766                 if (j->current_location.type == LOCATION_HEAD)
 767                         return journal_file_next_entry(f, NULL, 0, DIRECTION_DOWN, ret, offset);
 768                 if (j->current_location.type == LOCATION_TAIL)
 769                         return journal_file_next_entry(f, NULL, 0, DIRECTION_UP, ret, offset);
 770                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 771                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 772                 if (j->current_location.monotonic_set) {
 773                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 774                         if (r != -ENOENT)
 775                                 return r;
 776                 }
 777                 if (j->current_location.realtime_set)
 778                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 779
 780                 return journal_file_next_entry(f, NULL, 0, direction, ret, offset);
 781         } else
 782                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 783 }
 784
 785 static int next_with_matches(
 786                 sd_journal *j,
 787                 JournalFile *f,
 788                 direction_t direction,
 789                 Object **ret,
 790                 uint64_t *offset) {
 791
 792         Object *c;
 793         uint64_t cp;
 794
 795         assert(j);
 796         assert(f);
 797         assert(ret);
 798         assert(offset);
 799
 800         c = *ret;
 801         cp = *offset;
 802
 803         /* No matches is easy. We simple advance the file
 804          * pointer by one. */
 805         if (!j->level0)
 806                 return journal_file_next_entry(f, c, cp, direction, ret, offset);
 807
 808         /* If we have a match then we look for the next matching entry
 809          * with an offset at least one step larger */
 810         return next_for_match(j, j->level0, f, direction == DIRECTION_DOWN ? cp+1 : cp-1, direction, ret, offset);
 811 }
 812
 813 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction, Object **ret, uint64_t *offset) {
 814         Object *c;
 815         uint64_t cp;
 816         int r;
 817
 818         assert(j);
 819         assert(f);
 820
 821         if (f->last_direction == direction && f->current_offset > 0) {
 822                 cp = f->current_offset;
 823
 824                 r = journal_file_move_to_object(f, OBJECT_ENTRY, cp, &c);
 825                 if (r < 0)
 826                         return r;
 827
 828                 r = next_with_matches(j, f, direction, &c, &cp);
 829                 if (r <= 0)
 830                         return r;
 831         } else {
 832                 r = find_location_with_matches(j, f, direction, &c, &cp);
 833                 if (r <= 0)
 834                         return r;
 835         }
 836
 837         /* OK, we found the spot, now let's advance until an entry
 838          * that is actually different from what we were previously
 839          * looking at. This is necessary to handle entries which exist
 840          * in two (or more) journal files, and which shall all be
 841          * suppressed but one. */
 842
 843         for (;;) {
 844                 bool found;
 845
 846                 if (j->current_location.type == LOCATION_DISCRETE) {
 847                         int k;
 848
 849                         k = compare_with_location(f, c, &j->current_location);
 850                         if (direction == DIRECTION_DOWN)
 851                                 found = k > 0;
 852                         else
 853                                 found = k < 0;
 854                 } else
 855                         found = true;
 856
 857                 if (found) {
 858                         if (ret)
 859                                 *ret = c;
 860                         if (offset)
 861                                 *offset = cp;
 862                         return 1;
 863                 }
 864
 865                 r = next_with_matches(j, f, direction, &c, &cp);
 866                 if (r <= 0)
 867                         return r;
 868         }
 869 }
 870
 871 static int real_journal_next(sd_journal *j, direction_t direction) {
 872         JournalFile *f, *new_file = NULL;
 873         uint64_t new_offset = 0;
 874         Object *o;
 875         uint64_t p;
 876         Iterator i;
 877         int r;
 878
 879         assert_return(j, -EINVAL);
 880         assert_return(!journal_pid_changed(j), -ECHILD);
 881
 882         HASHMAP_FOREACH(f, j->files, i) {
 883                 bool found;
 884
 885                 r = next_beyond_location(j, f, direction, &o, &p);
 886                 if (r < 0) {
 887                         log_debug("Can't iterate through %s, ignoring: %s", f->path, strerror(-r));
 888                         continue;
 889                 } else if (r == 0)
 890                         continue;
 891
 892                 if (!new_file)
 893                         found = true;
 894                 else {
 895                         int k;
 896
 897                         k = compare_entry_order(f, o, new_file, new_offset);
 898
 899                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 900                 }
 901
 902                 if (found) {
 903                         new_file = f;
 904                         new_offset = p;
 905                 }
 906         }
 907
 908         if (!new_file)
 909                 return 0;
 910
 911         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_offset, &o);
 912         if (r < 0)
 913                 return r;
 914
 915         set_location(j, LOCATION_DISCRETE, new_file, o, direction, new_offset);
 916
 917         return 1;
 918 }
 919
 920 _public_ int sd_journal_next(sd_journal *j) {
 921         return real_journal_next(j, DIRECTION_DOWN);
 922 }
 923
 924 _public_ int sd_journal_previous(sd_journal *j) {
 925         return real_journal_next(j, DIRECTION_UP);
 926 }
 927
 928 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 929         int c = 0, r;
 930
 931         assert_return(j, -EINVAL);
 932         assert_return(!journal_pid_changed(j), -ECHILD);
 933
 934         if (skip == 0) {
 935                 /* If this is not a discrete skip, then at least
 936                  * resolve the current location */
 937                 if (j->current_location.type != LOCATION_DISCRETE)
 938                         return real_journal_next(j, direction);
 939
 940                 return 0;
 941         }
 942
 943         do {
 944                 r = real_journal_next(j, direction);
 945                 if (r < 0)
 946                         return r;
 947
 948                 if (r == 0)
 949                         return c;
 950
 951                 skip--;
 952                 c++;
 953         } while (skip > 0);
 954
 955         return c;
 956 }
 957
 958 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 959         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 960 }
 961
 962 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 963         return real_journal_next_skip(j, DIRECTION_UP, skip);
 964 }
 965
 966 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 967         Object *o;
 968         int r;
 969         char bid[33], sid[33];
 970
 971         assert_return(j, -EINVAL);
 972         assert_return(!journal_pid_changed(j), -ECHILD);
 973         assert_return(cursor, -EINVAL);
 974
 975         if (!j->current_file || j->current_file->current_offset <= 0)
 976                 return -EADDRNOTAVAIL;
 977
 978         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 979         if (r < 0)
 980                 return r;
 981
 982         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 983         sd_id128_to_string(o->entry.boot_id, bid);
 984
 985         if (asprintf(cursor,
 986                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 987                      sid, le64toh(o->entry.seqnum),
 988                      bid, le64toh(o->entry.monotonic),
 989                      le64toh(o->entry.realtime),
 990                      le64toh(o->entry.xor_hash)) < 0)
 991                 return -ENOMEM;
 992
 993         return 0;
 994 }
 995
 996 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 997         char *w, *state;
 998         size_t l;
 999         unsigned long long seqnum, monotonic, realtime, xor_hash;
1000         bool
1001                 seqnum_id_set = false,
1002                 seqnum_set = false,
1003                 boot_id_set = false,
1004                 monotonic_set = false,
1005                 realtime_set = false,
1006                 xor_hash_set = false;
1007         sd_id128_t seqnum_id, boot_id;
1008
1009         assert_return(j, -EINVAL);
1010         assert_return(!journal_pid_changed(j), -ECHILD);
1011         assert_return(!isempty(cursor), -EINVAL);
1012
1013         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1014                 char *item;
1015                 int k = 0;
1016
1017                 if (l < 2 || w[1] != '=')
1018                         return -EINVAL;
1019
1020                 item = strndup(w, l);
1021                 if (!item)
1022                         return -ENOMEM;
1023
1024                 switch (w[0]) {
1025
1026                 case 's':
1027                         seqnum_id_set = true;
1028                         k = sd_id128_from_string(item+2, &seqnum_id);
1029                         break;
1030
1031                 case 'i':
1032                         seqnum_set = true;
1033                         if (sscanf(item+2, "%llx", &seqnum) != 1)
1034                                 k = -EINVAL;
1035                         break;
1036
1037                 case 'b':
1038                         boot_id_set = true;
1039                         k = sd_id128_from_string(item+2, &boot_id);
1040                         break;
1041
1042                 case 'm':
1043                         monotonic_set = true;
1044                         if (sscanf(item+2, "%llx", &monotonic) != 1)
1045                                 k = -EINVAL;
1046                         break;
1047
1048                 case 't':
1049                         realtime_set = true;
1050                         if (sscanf(item+2, "%llx", &realtime) != 1)
1051                                 k = -EINVAL;
1052                         break;
1053
1054                 case 'x':
1055                         xor_hash_set = true;
1056                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1057                                 k = -EINVAL;
1058                         break;
1059                 }
1060
1061                 free(item);
1062
1063                 if (k < 0)
1064                         return k;
1065         }
1066
1067         if ((!seqnum_set || !seqnum_id_set) &&
1068             (!monotonic_set || !boot_id_set) &&
1069             !realtime_set)
1070                 return -EINVAL;
1071
1072         reset_location(j);
1073
1074         j->current_location.type = LOCATION_SEEK;
1075
1076         if (realtime_set) {
1077                 j->current_location.realtime = (uint64_t) realtime;
1078                 j->current_location.realtime_set = true;
1079         }
1080
1081         if (seqnum_set && seqnum_id_set) {
1082                 j->current_location.seqnum = (uint64_t) seqnum;
1083                 j->current_location.seqnum_id = seqnum_id;
1084                 j->current_location.seqnum_set = true;
1085         }
1086
1087         if (monotonic_set && boot_id_set) {
1088                 j->current_location.monotonic = (uint64_t) monotonic;
1089                 j->current_location.boot_id = boot_id;
1090                 j->current_location.monotonic_set = true;
1091         }
1092
1093         if (xor_hash_set) {
1094                 j->current_location.xor_hash = (uint64_t) xor_hash;
1095                 j->current_location.xor_hash_set = true;
1096         }
1097
1098         return 0;
1099 }
1100
1101 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1102         int r;
1103         char *w, *state;
1104         size_t l;
1105         Object *o;
1106
1107         assert_return(j, -EINVAL);
1108         assert_return(!journal_pid_changed(j), -ECHILD);
1109         assert_return(!isempty(cursor), -EINVAL);
1110
1111         if (!j->current_file || j->current_file->current_offset <= 0)
1112                 return -EADDRNOTAVAIL;
1113
1114         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1115         if (r < 0)
1116                 return r;
1117
1118         FOREACH_WORD_SEPARATOR(w, l, cursor, ";", state) {
1119                 _cleanup_free_ char *item = NULL;
1120                 sd_id128_t id;
1121                 unsigned long long ll;
1122                 int k = 0;
1123
1124                 if (l < 2 || w[1] != '=')
1125                         return -EINVAL;
1126
1127                 item = strndup(w, l);
1128                 if (!item)
1129                         return -ENOMEM;
1130
1131                 switch (w[0]) {
1132
1133                 case 's':
1134                         k = sd_id128_from_string(item+2, &id);
1135                         if (k < 0)
1136                                 return k;
1137                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1138                                 return 0;
1139                         break;
1140
1141                 case 'i':
1142                         if (sscanf(item+2, "%llx", &ll) != 1)
1143                                 return -EINVAL;
1144                         if (ll != le64toh(o->entry.seqnum))
1145                                 return 0;
1146                         break;
1147
1148                 case 'b':
1149                         k = sd_id128_from_string(item+2, &id);
1150                         if (k < 0)
1151                                 return k;
1152                         if (!sd_id128_equal(id, o->entry.boot_id))
1153                                 return 0;
1154                         break;
1155
1156                 case 'm':
1157                         if (sscanf(item+2, "%llx", &ll) != 1)
1158                                 return -EINVAL;
1159                         if (ll != le64toh(o->entry.monotonic))
1160                                 return 0;
1161                         break;
1162
1163                 case 't':
1164                         if (sscanf(item+2, "%llx", &ll) != 1)
1165                                 return -EINVAL;
1166                         if (ll != le64toh(o->entry.realtime))
1167                                 return 0;
1168                         break;
1169
1170                 case 'x':
1171                         if (sscanf(item+2, "%llx", &ll) != 1)
1172                                 return -EINVAL;
1173                         if (ll != le64toh(o->entry.xor_hash))
1174                                 return 0;
1175                         break;
1176                 }
1177         }
1178
1179         return 1;
1180 }
1181
1182
1183 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1184         assert_return(j, -EINVAL);
1185         assert_return(!journal_pid_changed(j), -ECHILD);
1186
1187         reset_location(j);
1188         j->current_location.type = LOCATION_SEEK;
1189         j->current_location.boot_id = boot_id;
1190         j->current_location.monotonic = usec;
1191         j->current_location.monotonic_set = true;
1192
1193         return 0;
1194 }
1195
1196 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1197         assert_return(j, -EINVAL);
1198         assert_return(!journal_pid_changed(j), -ECHILD);
1199
1200         reset_location(j);
1201         j->current_location.type = LOCATION_SEEK;
1202         j->current_location.realtime = usec;
1203         j->current_location.realtime_set = true;
1204
1205         return 0;
1206 }
1207
1208 _public_ int sd_journal_seek_head(sd_journal *j) {
1209         assert_return(j, -EINVAL);
1210         assert_return(!journal_pid_changed(j), -ECHILD);
1211
1212         reset_location(j);
1213         j->current_location.type = LOCATION_HEAD;
1214
1215         return 0;
1216 }
1217
1218 _public_ int sd_journal_seek_tail(sd_journal *j) {
1219         assert_return(j, -EINVAL);
1220         assert_return(!journal_pid_changed(j), -ECHILD);
1221
1222         reset_location(j);
1223         j->current_location.type = LOCATION_TAIL;
1224
1225         return 0;
1226 }
1227
1228 static void check_network(sd_journal *j, int fd) {
1229         struct statfs sfs;
1230
1231         assert(j);
1232
1233         if (j->on_network)
1234                 return;
1235
1236         if (fstatfs(fd, &sfs) < 0)
1237                 return;
1238
1239         j->on_network =
1240                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1241                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1242                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1243                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1244                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1245 }
1246
1247 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1248         const char *full, *tilded, *atted;
1249
1250         full = strappenda(prefix, ".journal");
1251         tilded = strappenda(full, "~");
1252         atted = strappenda(prefix, "@");
1253
1254         return streq(filename, full) ||
1255                streq(filename, tilded) ||
1256                startswith(filename, atted);
1257 }
1258
1259 static bool file_type_wanted(int flags, const char *filename) {
1260         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1261                 return false;
1262
1263         /* no flags set → every type is OK */
1264         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1265                 return true;
1266
1267         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1268                 return true;
1269
1270         if (flags & SD_JOURNAL_CURRENT_USER) {
1271                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1272
1273                 assert_se(snprintf(prefix, sizeof(prefix), "user-%lu", (unsigned long) getuid())
1274                           < (int) sizeof(prefix));
1275
1276                 if (file_has_type_prefix(prefix, filename))
1277                         return true;
1278         }
1279
1280         return false;
1281 }
1282
1283 static int add_any_file(sd_journal *j, const char *path) {
1284         JournalFile *f;
1285         int r;
1286
1287         assert(j);
1288         assert(path);
1289
1290         if (hashmap_get(j->files, path))
1291                 return 0;
1292
1293         if (hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1294                 log_warning("Too many open journal files, not adding %s.", path);
1295                 return set_put_error(j, -ETOOMANYREFS);
1296         }
1297
1298         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1299         if (r < 0)
1300                 return r;
1301
1302         /* journal_file_dump(f); */
1303
1304         r = hashmap_put(j->files, f->path, f);
1305         if (r < 0) {
1306                 journal_file_close(f);
1307                 return r;
1308         }
1309
1310         log_debug("File %s added.", f->path);
1311
1312         check_network(j, f->fd);
1313
1314         j->current_invalidate_counter ++;
1315
1316         return 0;
1317 }
1318
1319 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1320         _cleanup_free_ char *path = NULL;
1321         int r;
1322
1323         assert(j);
1324         assert(prefix);
1325         assert(filename);
1326
1327         if (j->no_new_files ||
1328             !file_type_wanted(j->flags, filename))
1329                 return 0;
1330
1331         path = strjoin(prefix, "/", filename, NULL);
1332         if (!path)
1333                 return -ENOMEM;
1334
1335         r = add_any_file(j, path);
1336         if (r == -ENOENT)
1337                 return 0;
1338         return 0;
1339 }
1340
1341 static int remove_file(sd_journal *j, const char *prefix, const char *filename) {
1342         char *path;
1343         JournalFile *f;
1344
1345         assert(j);
1346         assert(prefix);
1347         assert(filename);
1348
1349         path = strjoin(prefix, "/", filename, NULL);
1350         if (!path)
1351                 return -ENOMEM;
1352
1353         f = hashmap_get(j->files, path);
1354         free(path);
1355         if (!f)
1356                 return 0;
1357
1358         hashmap_remove(j->files, f->path);
1359
1360         log_debug("File %s removed.", f->path);
1361
1362         if (j->current_file == f) {
1363                 j->current_file = NULL;
1364                 j->current_field = 0;
1365         }
1366
1367         if (j->unique_file == f) {
1368                 j->unique_file = NULL;
1369                 j->unique_offset = 0;
1370         }
1371
1372         journal_file_close(f);
1373
1374         j->current_invalidate_counter ++;
1375
1376         return 0;
1377 }
1378
1379 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1380         _cleanup_free_ char *path = NULL;
1381         int r;
1382         _cleanup_closedir_ DIR *d = NULL;
1383         sd_id128_t id, mid;
1384         Directory *m;
1385
1386         assert(j);
1387         assert(prefix);
1388         assert(dirname);
1389
1390         log_debug("Considering %s/%s.", prefix, dirname);
1391
1392         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1393             (sd_id128_from_string(dirname, &id) < 0 ||
1394              sd_id128_get_machine(&mid) < 0 ||
1395              !(sd_id128_equal(id, mid) || path_startswith(prefix, "/run"))))
1396             return 0;
1397
1398         path = strjoin(prefix, "/", dirname, NULL);
1399         if (!path)
1400                 return -ENOMEM;
1401
1402         d = opendir(path);
1403         if (!d) {
1404                 log_debug("Failed to open %s: %m", path);
1405                 if (errno == ENOENT)
1406                         return 0;
1407                 return -errno;
1408         }
1409
1410         m = hashmap_get(j->directories_by_path, path);
1411         if (!m) {
1412                 m = new0(Directory, 1);
1413                 if (!m)
1414                         return -ENOMEM;
1415
1416                 m->is_root = false;
1417                 m->path = path;
1418
1419                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1420                         free(m);
1421                         return -ENOMEM;
1422                 }
1423
1424                 path = NULL; /* avoid freeing in cleanup */
1425                 j->current_invalidate_counter ++;
1426
1427                 log_debug("Directory %s added.", m->path);
1428
1429         } else if (m->is_root)
1430                 return 0;
1431
1432         if (m->wd <= 0 && j->inotify_fd >= 0) {
1433
1434                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1435                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1436                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1437                                           IN_ONLYDIR);
1438
1439                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1440                         inotify_rm_watch(j->inotify_fd, m->wd);
1441         }
1442
1443         for (;;) {
1444                 struct dirent *de;
1445                 union dirent_storage buf;
1446
1447                 r = readdir_r(d, &buf.de, &de);
1448                 if (r != 0 || !de)
1449                         break;
1450
1451                 if (dirent_is_file_with_suffix(de, ".journal") ||
1452                     dirent_is_file_with_suffix(de, ".journal~")) {
1453                         r = add_file(j, m->path, de->d_name);
1454                         if (r < 0) {
1455                                 log_debug("Failed to add file %s/%s: %s",
1456                                           m->path, de->d_name, strerror(-r));
1457                                 r = set_put_error(j, r);
1458                                 if (r < 0)
1459                                         return r;
1460                         }
1461                 }
1462         }
1463
1464         check_network(j, dirfd(d));
1465
1466         return 0;
1467 }
1468
1469 static int add_root_directory(sd_journal *j, const char *p) {
1470         _cleanup_closedir_ DIR *d = NULL;
1471         Directory *m;
1472         int r;
1473
1474         assert(j);
1475         assert(p);
1476
1477         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1478             !path_startswith(p, "/run"))
1479                 return -EINVAL;
1480
1481         if (j->prefix)
1482                 p = strappenda(j->prefix, p);
1483
1484         d = opendir(p);
1485         if (!d)
1486                 return -errno;
1487
1488         m = hashmap_get(j->directories_by_path, p);
1489         if (!m) {
1490                 m = new0(Directory, 1);
1491                 if (!m)
1492                         return -ENOMEM;
1493
1494                 m->is_root = true;
1495                 m->path = strdup(p);
1496                 if (!m->path) {
1497                         free(m);
1498                         return -ENOMEM;
1499                 }
1500
1501                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1502                         free(m->path);
1503                         free(m);
1504                         return -ENOMEM;
1505                 }
1506
1507                 j->current_invalidate_counter ++;
1508
1509                 log_debug("Root directory %s added.", m->path);
1510
1511         } else if (!m->is_root)
1512                 return 0;
1513
1514         if (m->wd <= 0 && j->inotify_fd >= 0) {
1515
1516                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1517                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1518                                           IN_ONLYDIR);
1519
1520                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1521                         inotify_rm_watch(j->inotify_fd, m->wd);
1522         }
1523
1524         if (j->no_new_files)
1525                 return 0;
1526
1527         for (;;) {
1528                 struct dirent *de;
1529                 union dirent_storage buf;
1530                 sd_id128_t id;
1531
1532                 r = readdir_r(d, &buf.de, &de);
1533                 if (r != 0 || !de)
1534                         break;
1535
1536                 if (dirent_is_file_with_suffix(de, ".journal") ||
1537                     dirent_is_file_with_suffix(de, ".journal~")) {
1538                         r = add_file(j, m->path, de->d_name);
1539                         if (r < 0) {
1540                                 log_debug("Failed to add file %s/%s: %s",
1541                                           m->path, de->d_name, strerror(-r));
1542                                 r = set_put_error(j, r);
1543                                 if (r < 0)
1544                                         return r;
1545                         }
1546                 } else if ((de->d_type == DT_DIR || de->d_type == DT_LNK || de->d_type == DT_UNKNOWN) &&
1547                            sd_id128_from_string(de->d_name, &id) >= 0) {
1548
1549                         r = add_directory(j, m->path, de->d_name);
1550                         if (r < 0)
1551                                 log_debug("Failed to add directory %s/%s: %s", m->path, de->d_name, strerror(-r));
1552                 }
1553         }
1554
1555         check_network(j, dirfd(d));
1556
1557         return 0;
1558 }
1559
1560 static int remove_directory(sd_journal *j, Directory *d) {
1561         assert(j);
1562
1563         if (d->wd > 0) {
1564                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1565
1566                 if (j->inotify_fd >= 0)
1567                         inotify_rm_watch(j->inotify_fd, d->wd);
1568         }
1569
1570         hashmap_remove(j->directories_by_path, d->path);
1571
1572         if (d->is_root)
1573                 log_debug("Root directory %s removed.", d->path);
1574         else
1575                 log_debug("Directory %s removed.", d->path);
1576
1577         free(d->path);
1578         free(d);
1579
1580         return 0;
1581 }
1582
1583 static int add_search_paths(sd_journal *j) {
1584         int r;
1585         const char search_paths[] =
1586                 "/run/log/journal\0"
1587                 "/var/log/journal\0";
1588         const char *p;
1589
1590         assert(j);
1591
1592         /* We ignore most errors here, since the idea is to only open
1593          * what's actually accessible, and ignore the rest. */
1594
1595         NULSTR_FOREACH(p, search_paths) {
1596                 r = add_root_directory(j, p);
1597                 if (r < 0 && r != -ENOENT) {
1598                         r = set_put_error(j, r);
1599                         if (r < 0)
1600                                 return r;
1601                 }
1602         }
1603
1604         return 0;
1605 }
1606
1607 static int add_current_paths(sd_journal *j) {
1608         Iterator i;
1609         JournalFile *f;
1610
1611         assert(j);
1612         assert(j->no_new_files);
1613
1614         /* Simply adds all directories for files we have open as
1615          * "root" directories. We don't expect errors here, so we
1616          * treat them as fatal. */
1617
1618         HASHMAP_FOREACH(f, j->files, i) {
1619                 _cleanup_free_ char *dir;
1620                 int r;
1621
1622                 dir = dirname_malloc(f->path);
1623                 if (!dir)
1624                         return -ENOMEM;
1625
1626                 r = add_root_directory(j, dir);
1627                 if (r < 0) {
1628                         set_put_error(j, r);
1629                         return r;
1630                 }
1631         }
1632
1633         return 0;
1634 }
1635
1636
1637 static int allocate_inotify(sd_journal *j) {
1638         assert(j);
1639
1640         if (j->inotify_fd < 0) {
1641                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1642                 if (j->inotify_fd < 0)
1643                         return -errno;
1644         }
1645
1646         if (!j->directories_by_wd) {
1647                 j->directories_by_wd = hashmap_new(trivial_hash_func, trivial_compare_func);
1648                 if (!j->directories_by_wd)
1649                         return -ENOMEM;
1650         }
1651
1652         return 0;
1653 }
1654
1655 static sd_journal *journal_new(int flags, const char *path) {
1656         sd_journal *j;
1657
1658         j = new0(sd_journal, 1);
1659         if (!j)
1660                 return NULL;
1661
1662         j->original_pid = getpid();
1663         j->inotify_fd = -1;
1664         j->flags = flags;
1665         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1666
1667         if (path) {
1668                 j->path = strdup(path);
1669                 if (!j->path)
1670                         goto fail;
1671         }
1672
1673         j->files = hashmap_new(string_hash_func, string_compare_func);
1674         j->directories_by_path = hashmap_new(string_hash_func, string_compare_func);
1675         j->mmap = mmap_cache_new();
1676         if (!j->files || !j->directories_by_path || !j->mmap)
1677                 goto fail;
1678
1679         return j;
1680
1681 fail:
1682         sd_journal_close(j);
1683         return NULL;
1684 }
1685
1686 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1687         sd_journal *j;
1688         int r;
1689
1690         assert_return(ret, -EINVAL);
1691         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_RUNTIME_ONLY|SD_JOURNAL_SYSTEM|SD_JOURNAL_CURRENT_USER)) == 0, -EINVAL);
1692
1693         j = journal_new(flags, NULL);
1694         if (!j)
1695                 return -ENOMEM;
1696
1697         r = add_search_paths(j);
1698         if (r < 0)
1699                 goto fail;
1700
1701         *ret = j;
1702         return 0;
1703
1704 fail:
1705         sd_journal_close(j);
1706
1707         return r;
1708 }
1709
1710 _public_ int sd_journal_open_container(sd_journal **ret, const char *machine, int flags) {
1711         _cleanup_free_ char *root = NULL, *class = NULL;
1712         sd_journal *j;
1713         char *p;
1714         int r;
1715
1716         assert_return(machine, -EINVAL);
1717         assert_return(ret, -EINVAL);
1718         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_SYSTEM)) == 0, -EINVAL);
1719         assert_return(filename_is_safe(machine), -EINVAL);
1720
1721         p = strappenda("/run/systemd/machines/", machine);
1722         r = parse_env_file(p, NEWLINE, "ROOT", &root, "CLASS", &class, NULL);
1723         if (r == -ENOENT)
1724                 return -EHOSTDOWN;
1725         if (r < 0)
1726                 return r;
1727         if (!root)
1728                 return -ENODATA;
1729
1730         if (!streq_ptr(class, "container"))
1731                 return -EIO;
1732
1733         j = journal_new(flags, NULL);
1734         if (!j)
1735                 return -ENOMEM;
1736
1737         j->prefix = root;
1738         root = NULL;
1739
1740         r = add_search_paths(j);
1741         if (r < 0)
1742                 goto fail;
1743
1744         *ret = j;
1745         return 0;
1746
1747 fail:
1748         sd_journal_close(j);
1749         return r;
1750 }
1751
1752 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1753         sd_journal *j;
1754         int r;
1755
1756         assert_return(ret, -EINVAL);
1757         assert_return(path, -EINVAL);
1758         assert_return(flags == 0, -EINVAL);
1759
1760         j = journal_new(flags, path);
1761         if (!j)
1762                 return -ENOMEM;
1763
1764         r = add_root_directory(j, path);
1765         if (r < 0) {
1766                 set_put_error(j, r);
1767                 goto fail;
1768         }
1769
1770         *ret = j;
1771         return 0;
1772
1773 fail:
1774         sd_journal_close(j);
1775
1776         return r;
1777 }
1778
1779 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1780         sd_journal *j;
1781         const char **path;
1782         int r;
1783
1784         assert_return(ret, -EINVAL);
1785         assert_return(flags == 0, -EINVAL);
1786
1787         j = journal_new(flags, NULL);
1788         if (!j)
1789                 return -ENOMEM;
1790
1791         STRV_FOREACH(path, paths) {
1792                 r = add_any_file(j, *path);
1793                 if (r < 0) {
1794                         log_error("Failed to open %s: %s", *path, strerror(-r));
1795                         goto fail;
1796                 }
1797         }
1798
1799         j->no_new_files = true;
1800
1801         *ret = j;
1802         return 0;
1803
1804 fail:
1805         sd_journal_close(j);
1806
1807         return r;
1808 }
1809
1810 _public_ void sd_journal_close(sd_journal *j) {
1811         Directory *d;
1812         JournalFile *f;
1813
1814         if (!j)
1815                 return;
1816
1817         sd_journal_flush_matches(j);
1818
1819         while ((f = hashmap_steal_first(j->files)))
1820                 journal_file_close(f);
1821
1822         hashmap_free(j->files);
1823
1824         while ((d = hashmap_first(j->directories_by_path)))
1825                 remove_directory(j, d);
1826
1827         while ((d = hashmap_first(j->directories_by_wd)))
1828                 remove_directory(j, d);
1829
1830         hashmap_free(j->directories_by_path);
1831         hashmap_free(j->directories_by_wd);
1832
1833         if (j->inotify_fd >= 0)
1834                 close_nointr_nofail(j->inotify_fd);
1835
1836         if (j->mmap) {
1837                 log_debug("mmap cache statistics: %u hit, %u miss", mmap_cache_get_hit(j->mmap), mmap_cache_get_missed(j->mmap));
1838                 mmap_cache_unref(j->mmap);
1839         }
1840
1841         free(j->path);
1842         free(j->prefix);
1843         free(j->unique_field);
1844         set_free(j->errors);
1845         free(j);
1846 }
1847
1848 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1849         Object *o;
1850         JournalFile *f;
1851         int r;
1852
1853         assert_return(j, -EINVAL);
1854         assert_return(!journal_pid_changed(j), -ECHILD);
1855         assert_return(ret, -EINVAL);
1856
1857         f = j->current_file;
1858         if (!f)
1859                 return -EADDRNOTAVAIL;
1860
1861         if (f->current_offset <= 0)
1862                 return -EADDRNOTAVAIL;
1863
1864         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1865         if (r < 0)
1866                 return r;
1867
1868         *ret = le64toh(o->entry.realtime);
1869         return 0;
1870 }
1871
1872 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1873         Object *o;
1874         JournalFile *f;
1875         int r;
1876         sd_id128_t id;
1877
1878         assert_return(j, -EINVAL);
1879         assert_return(!journal_pid_changed(j), -ECHILD);
1880
1881         f = j->current_file;
1882         if (!f)
1883                 return -EADDRNOTAVAIL;
1884
1885         if (f->current_offset <= 0)
1886                 return -EADDRNOTAVAIL;
1887
1888         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1889         if (r < 0)
1890                 return r;
1891
1892         if (ret_boot_id)
1893                 *ret_boot_id = o->entry.boot_id;
1894         else {
1895                 r = sd_id128_get_boot(&id);
1896                 if (r < 0)
1897                         return r;
1898
1899                 if (!sd_id128_equal(id, o->entry.boot_id))
1900                         return -ESTALE;
1901         }
1902
1903         if (ret)
1904                 *ret = le64toh(o->entry.monotonic);
1905
1906         return 0;
1907 }
1908
1909 static bool field_is_valid(const char *field) {
1910         const char *p;
1911
1912         assert(field);
1913
1914         if (isempty(field))
1915                 return false;
1916
1917         if (startswith(field, "__"))
1918                 return false;
1919
1920         for (p = field; *p; p++) {
1921
1922                 if (*p == '_')
1923                         continue;
1924
1925                 if (*p >= 'A' && *p <= 'Z')
1926                         continue;
1927
1928                 if (*p >= '0' && *p <= '9')
1929                         continue;
1930
1931                 return false;
1932         }
1933
1934         return true;
1935 }
1936
1937 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1938         JournalFile *f;
1939         uint64_t i, n;
1940         size_t field_length;
1941         int r;
1942         Object *o;
1943
1944         assert_return(j, -EINVAL);
1945         assert_return(!journal_pid_changed(j), -ECHILD);
1946         assert_return(field, -EINVAL);
1947         assert_return(data, -EINVAL);
1948         assert_return(size, -EINVAL);
1949         assert_return(field_is_valid(field), -EINVAL);
1950
1951         f = j->current_file;
1952         if (!f)
1953                 return -EADDRNOTAVAIL;
1954
1955         if (f->current_offset <= 0)
1956                 return -EADDRNOTAVAIL;
1957
1958         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1959         if (r < 0)
1960                 return r;
1961
1962         field_length = strlen(field);
1963
1964         n = journal_file_entry_n_items(o);
1965         for (i = 0; i < n; i++) {
1966                 uint64_t p, l;
1967                 le64_t le_hash;
1968                 size_t t;
1969
1970                 p = le64toh(o->entry.items[i].object_offset);
1971                 le_hash = o->entry.items[i].hash;
1972                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1973                 if (r < 0)
1974                         return r;
1975
1976                 if (le_hash != o->data.hash)
1977                         return -EBADMSG;
1978
1979                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1980
1981                 if (o->object.flags & OBJECT_COMPRESSED) {
1982
1983 #ifdef HAVE_XZ
1984                         if (uncompress_startswith(o->data.payload, l,
1985                                                   &f->compress_buffer, &f->compress_buffer_size,
1986                                                   field, field_length, '=')) {
1987
1988                                 uint64_t rsize;
1989
1990                                 if (!uncompress_blob(o->data.payload, l,
1991                                                      &f->compress_buffer, &f->compress_buffer_size, &rsize,
1992                                                      j->data_threshold))
1993                                         return -EBADMSG;
1994
1995                                 *data = f->compress_buffer;
1996                                 *size = (size_t) rsize;
1997
1998                                 return 0;
1999                         }
2000 #else
2001                         return -EPROTONOSUPPORT;
2002 #endif
2003
2004                 } else if (l >= field_length+1 &&
2005                            memcmp(o->data.payload, field, field_length) == 0 &&
2006                            o->data.payload[field_length] == '=') {
2007
2008                         t = (size_t) l;
2009
2010                         if ((uint64_t) t != l)
2011                                 return -E2BIG;
2012
2013                         *data = o->data.payload;
2014                         *size = t;
2015
2016                         return 0;
2017                 }
2018
2019                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2020                 if (r < 0)
2021                         return r;
2022         }
2023
2024         return -ENOENT;
2025 }
2026
2027 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
2028         size_t t;
2029         uint64_t l;
2030
2031         l = le64toh(o->object.size) - offsetof(Object, data.payload);
2032         t = (size_t) l;
2033
2034         /* We can't read objects larger than 4G on a 32bit machine */
2035         if ((uint64_t) t != l)
2036                 return -E2BIG;
2037
2038         if (o->object.flags & OBJECT_COMPRESSED) {
2039 #ifdef HAVE_XZ
2040                 uint64_t rsize;
2041
2042                 if (!uncompress_blob(o->data.payload, l, &f->compress_buffer, &f->compress_buffer_size, &rsize, j->data_threshold))
2043                         return -EBADMSG;
2044
2045                 *data = f->compress_buffer;
2046                 *size = (size_t) rsize;
2047 #else
2048                 return -EPROTONOSUPPORT;
2049 #endif
2050         } else {
2051                 *data = o->data.payload;
2052                 *size = t;
2053         }
2054
2055         return 0;
2056 }
2057
2058 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2059         JournalFile *f;
2060         uint64_t p, n;
2061         le64_t le_hash;
2062         int r;
2063         Object *o;
2064
2065         assert_return(j, -EINVAL);
2066         assert_return(!journal_pid_changed(j), -ECHILD);
2067         assert_return(data, -EINVAL);
2068         assert_return(size, -EINVAL);
2069
2070         f = j->current_file;
2071         if (!f)
2072                 return -EADDRNOTAVAIL;
2073
2074         if (f->current_offset <= 0)
2075                 return -EADDRNOTAVAIL;
2076
2077         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2078         if (r < 0)
2079                 return r;
2080
2081         n = journal_file_entry_n_items(o);
2082         if (j->current_field >= n)
2083                 return 0;
2084
2085         p = le64toh(o->entry.items[j->current_field].object_offset);
2086         le_hash = o->entry.items[j->current_field].hash;
2087         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2088         if (r < 0)
2089                 return r;
2090
2091         if (le_hash != o->data.hash)
2092                 return -EBADMSG;
2093
2094         r = return_data(j, f, o, data, size);
2095         if (r < 0)
2096                 return r;
2097
2098         j->current_field ++;
2099
2100         return 1;
2101 }
2102
2103 _public_ void sd_journal_restart_data(sd_journal *j) {
2104         if (!j)
2105                 return;
2106
2107         j->current_field = 0;
2108 }
2109
2110 _public_ int sd_journal_get_fd(sd_journal *j) {
2111         int r;
2112
2113         assert_return(j, -EINVAL);
2114         assert_return(!journal_pid_changed(j), -ECHILD);
2115
2116         if (j->inotify_fd >= 0)
2117                 return j->inotify_fd;
2118
2119         r = allocate_inotify(j);
2120         if (r < 0)
2121                 return r;
2122
2123         /* Iterate through all dirs again, to add them to the
2124          * inotify */
2125         if (j->no_new_files)
2126                 r = add_current_paths(j);
2127         else if (j->path)
2128                 r = add_root_directory(j, j->path);
2129         else
2130                 r = add_search_paths(j);
2131         if (r < 0)
2132                 return r;
2133
2134         return j->inotify_fd;
2135 }
2136
2137 _public_ int sd_journal_get_events(sd_journal *j) {
2138         int fd;
2139
2140         assert_return(j, -EINVAL);
2141         assert_return(!journal_pid_changed(j), -ECHILD);
2142
2143         fd = sd_journal_get_fd(j);
2144         if (fd < 0)
2145                 return fd;
2146
2147         return POLLIN;
2148 }
2149
2150 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2151         int fd;
2152
2153         assert_return(j, -EINVAL);
2154         assert_return(!journal_pid_changed(j), -ECHILD);
2155         assert_return(timeout_usec, -EINVAL);
2156
2157         fd = sd_journal_get_fd(j);
2158         if (fd < 0)
2159                 return fd;
2160
2161         if (!j->on_network) {
2162                 *timeout_usec = (uint64_t) -1;
2163                 return 0;
2164         }
2165
2166         /* If we are on the network we need to regularly check for
2167          * changes manually */
2168
2169         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2170         return 1;
2171 }
2172
2173 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2174         Directory *d;
2175         int r;
2176
2177         assert(j);
2178         assert(e);
2179
2180         /* Is this a subdirectory we watch? */
2181         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2182         if (d) {
2183                 sd_id128_t id;
2184
2185                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2186                     (endswith(e->name, ".journal") ||
2187                      endswith(e->name, ".journal~"))) {
2188
2189                         /* Event for a journal file */
2190
2191                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2192                                 r = add_file(j, d->path, e->name);
2193                                 if (r < 0) {
2194                                         log_debug("Failed to add file %s/%s: %s",
2195                                                   d->path, e->name, strerror(-r));
2196                                         set_put_error(j, r);
2197                                 }
2198
2199                         } else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT)) {
2200
2201                                 r = remove_file(j, d->path, e->name);
2202                                 if (r < 0)
2203                                         log_debug("Failed to remove file %s/%s: %s", d->path, e->name, strerror(-r));
2204                         }
2205
2206                 } else if (!d->is_root && e->len == 0) {
2207
2208                         /* Event for a subdirectory */
2209
2210                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT)) {
2211                                 r = remove_directory(j, d);
2212                                 if (r < 0)
2213                                         log_debug("Failed to remove directory %s: %s", d->path, strerror(-r));
2214                         }
2215
2216
2217                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2218
2219                         /* Event for root directory */
2220
2221                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB)) {
2222                                 r = add_directory(j, d->path, e->name);
2223                                 if (r < 0)
2224                                         log_debug("Failed to add directory %s/%s: %s", d->path, e->name, strerror(-r));
2225                         }
2226                 }
2227
2228                 return;
2229         }
2230
2231         if (e->mask & IN_IGNORED)
2232                 return;
2233
2234         log_warning("Unknown inotify event.");
2235 }
2236
2237 static int determine_change(sd_journal *j) {
2238         bool b;
2239
2240         assert(j);
2241
2242         b = j->current_invalidate_counter != j->last_invalidate_counter;
2243         j->last_invalidate_counter = j->current_invalidate_counter;
2244
2245         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2246 }
2247
2248 _public_ int sd_journal_process(sd_journal *j) {
2249         uint8_t buffer[sizeof(struct inotify_event) + FILENAME_MAX] _alignas_(struct inotify_event);
2250         bool got_something = false;
2251
2252         assert_return(j, -EINVAL);
2253         assert_return(!journal_pid_changed(j), -ECHILD);
2254
2255         j->last_process_usec = now(CLOCK_MONOTONIC);
2256
2257         for (;;) {
2258                 struct inotify_event *e;
2259                 ssize_t l;
2260
2261                 l = read(j->inotify_fd, buffer, sizeof(buffer));
2262                 if (l < 0) {
2263                         if (errno == EAGAIN || errno == EINTR)
2264                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2265
2266                         return -errno;
2267                 }
2268
2269                 got_something = true;
2270
2271                 e = (struct inotify_event*) buffer;
2272                 while (l > 0) {
2273                         size_t step;
2274
2275                         process_inotify_event(j, e);
2276
2277                         step = sizeof(struct inotify_event) + e->len;
2278                         assert(step <= (size_t) l);
2279
2280                         e = (struct inotify_event*) ((uint8_t*) e + step);
2281                         l -= step;
2282                 }
2283         }
2284
2285         return determine_change(j);
2286 }
2287
2288 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2289         int r;
2290         uint64_t t;
2291
2292         assert_return(j, -EINVAL);
2293         assert_return(!journal_pid_changed(j), -ECHILD);
2294
2295         if (j->inotify_fd < 0) {
2296
2297                 /* This is the first invocation, hence create the
2298                  * inotify watch */
2299                 r = sd_journal_get_fd(j);
2300                 if (r < 0)
2301                         return r;
2302
2303                 /* The journal might have changed since the context
2304                  * object was created and we weren't watching before,
2305                  * hence don't wait for anything, and return
2306                  * immediately. */
2307                 return determine_change(j);
2308         }
2309
2310         r = sd_journal_get_timeout(j, &t);
2311         if (r < 0)
2312                 return r;
2313
2314         if (t != (uint64_t) -1) {
2315                 usec_t n;
2316
2317                 n = now(CLOCK_MONOTONIC);
2318                 t = t > n ? t - n : 0;
2319
2320                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2321                         timeout_usec = t;
2322         }
2323
2324         do {
2325                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2326         } while (r == -EINTR);
2327
2328         if (r < 0)
2329                 return r;
2330
2331         return sd_journal_process(j);
2332 }
2333
2334 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2335         Iterator i;
2336         JournalFile *f;
2337         bool first = true;
2338         int r;
2339
2340         assert_return(j, -EINVAL);
2341         assert_return(!journal_pid_changed(j), -ECHILD);
2342         assert_return(from || to, -EINVAL);
2343         assert_return(from != to, -EINVAL);
2344
2345         HASHMAP_FOREACH(f, j->files, i) {
2346                 usec_t fr, t;
2347
2348                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2349                 if (r == -ENOENT)
2350                         continue;
2351                 if (r < 0)
2352                         return r;
2353                 if (r == 0)
2354                         continue;
2355
2356                 if (first) {
2357                         if (from)
2358                                 *from = fr;
2359                         if (to)
2360                                 *to = t;
2361                         first = false;
2362                 } else {
2363                         if (from)
2364                                 *from = MIN(fr, *from);
2365                         if (to)
2366                                 *to = MAX(t, *to);
2367                 }
2368         }
2369
2370         return first ? 0 : 1;
2371 }
2372
2373 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2374         Iterator i;
2375         JournalFile *f;
2376         bool first = true;
2377         int r;
2378
2379         assert_return(j, -EINVAL);
2380         assert_return(!journal_pid_changed(j), -ECHILD);
2381         assert_return(from || to, -EINVAL);
2382         assert_return(from != to, -EINVAL);
2383
2384         HASHMAP_FOREACH(f, j->files, i) {
2385                 usec_t fr, t;
2386
2387                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2388                 if (r == -ENOENT)
2389                         continue;
2390                 if (r < 0)
2391                         return r;
2392                 if (r == 0)
2393                         continue;
2394
2395                 if (first) {
2396                         if (from)
2397                                 *from = fr;
2398                         if (to)
2399                                 *to = t;
2400                         first = false;
2401                 } else {
2402                         if (from)
2403                                 *from = MIN(fr, *from);
2404                         if (to)
2405                                 *to = MAX(t, *to);
2406                 }
2407         }
2408
2409         return first ? 0 : 1;
2410 }
2411
2412 void journal_print_header(sd_journal *j) {
2413         Iterator i;
2414         JournalFile *f;
2415         bool newline = false;
2416
2417         assert(j);
2418
2419         HASHMAP_FOREACH(f, j->files, i) {
2420                 if (newline)
2421                         putchar('\n');
2422                 else
2423                         newline = true;
2424
2425                 journal_file_print_header(f);
2426         }
2427 }
2428
2429 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2430         Iterator i;
2431         JournalFile *f;
2432         uint64_t sum = 0;
2433
2434         assert_return(j, -EINVAL);
2435         assert_return(!journal_pid_changed(j), -ECHILD);
2436         assert_return(bytes, -EINVAL);
2437
2438         HASHMAP_FOREACH(f, j->files, i) {
2439                 struct stat st;
2440
2441                 if (fstat(f->fd, &st) < 0)
2442                         return -errno;
2443
2444                 sum += (uint64_t) st.st_blocks * 512ULL;
2445         }
2446
2447         *bytes = sum;
2448         return 0;
2449 }
2450
2451 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2452         char *f;
2453
2454         assert_return(j, -EINVAL);
2455         assert_return(!journal_pid_changed(j), -ECHILD);
2456         assert_return(!isempty(field), -EINVAL);
2457         assert_return(field_is_valid(field), -EINVAL);
2458
2459         f = strdup(field);
2460         if (!f)
2461                 return -ENOMEM;
2462
2463         free(j->unique_field);
2464         j->unique_field = f;
2465         j->unique_file = NULL;
2466         j->unique_offset = 0;
2467
2468         return 0;
2469 }
2470
2471 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2472         Object *o;
2473         size_t k;
2474         int r;
2475
2476         assert_return(j, -EINVAL);
2477         assert_return(!journal_pid_changed(j), -ECHILD);
2478         assert_return(data, -EINVAL);
2479         assert_return(l, -EINVAL);
2480         assert_return(j->unique_field, -EINVAL);
2481
2482         k = strlen(j->unique_field);
2483
2484         if (!j->unique_file) {
2485                 j->unique_file = hashmap_first(j->files);
2486                 if (!j->unique_file)
2487                         return 0;
2488                 j->unique_offset = 0;
2489         }
2490
2491         for (;;) {
2492                 JournalFile *of;
2493                 Iterator i;
2494                 const void *odata;
2495                 size_t ol;
2496                 bool found;
2497
2498                 /* Proceed to next data object in the field's linked list */
2499                 if (j->unique_offset == 0) {
2500                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2501                         if (r < 0)
2502                                 return r;
2503
2504                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2505                 } else {
2506                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2507                         if (r < 0)
2508                                 return r;
2509
2510                         j->unique_offset = le64toh(o->data.next_field_offset);
2511                 }
2512
2513                 /* We reached the end of the list? Then start again, with the next file */
2514                 if (j->unique_offset == 0) {
2515                         JournalFile *n;
2516
2517                         n = hashmap_next(j->files, j->unique_file->path);
2518                         if (!n)
2519                                 return 0;
2520
2521                         j->unique_file = n;
2522                         continue;
2523                 }
2524
2525                 /* We do not use the type context here, but 0 instead,
2526                  * so that we can look at this data object at the same
2527                  * time as one on another file */
2528                 r = journal_file_move_to_object(j->unique_file, 0, j->unique_offset, &o);
2529                 if (r < 0)
2530                         return r;
2531
2532                 /* Let's do the type check by hand, since we used 0 context above. */
2533                 if (o->object.type != OBJECT_DATA)
2534                         return -EBADMSG;
2535
2536                 r = return_data(j, j->unique_file, o, &odata, &ol);
2537                 if (r < 0)
2538                         return r;
2539
2540                 /* OK, now let's see if we already returned this data
2541                  * object by checking if it exists in the earlier
2542                  * traversed files. */
2543                 found = false;
2544                 HASHMAP_FOREACH(of, j->files, i) {
2545                         Object *oo;
2546                         uint64_t op;
2547
2548                         if (of == j->unique_file)
2549                                 break;
2550
2551                         /* Skip this file it didn't have any fields
2552                          * indexed */
2553                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2554                             le64toh(of->header->n_fields) <= 0)
2555                                 continue;
2556
2557                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2558                         if (r < 0)
2559                                 return r;
2560
2561                         if (r > 0)
2562                                 found = true;
2563                 }
2564
2565                 if (found)
2566                         continue;
2567
2568                 r = return_data(j, j->unique_file, o, data, l);
2569                 if (r < 0)
2570                         return r;
2571
2572                 return 1;
2573         }
2574 }
2575
2576 _public_ void sd_journal_restart_unique(sd_journal *j) {
2577         if (!j)
2578                 return;
2579
2580         j->unique_file = NULL;
2581         j->unique_offset = 0;
2582 }
2583
2584 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2585         assert_return(j, -EINVAL);
2586         assert_return(!journal_pid_changed(j), -ECHILD);
2587
2588         return !j->on_network;
2589 }
2590
2591 static char *lookup_field(const char *field, void *userdata) {
2592         sd_journal *j = userdata;
2593         const void *data;
2594         size_t size, d;
2595         int r;
2596
2597         assert(field);
2598         assert(j);
2599
2600         r = sd_journal_get_data(j, field, &data, &size);
2601         if (r < 0 ||
2602             size > REPLACE_VAR_MAX)
2603                 return strdup(field);
2604
2605         d = strlen(field) + 1;
2606
2607         return strndup((const char*) data + d, size - d);
2608 }
2609
2610 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2611         const void *data;
2612         size_t size;
2613         sd_id128_t id;
2614         _cleanup_free_ char *text = NULL, *cid = NULL;
2615         char *t;
2616         int r;
2617
2618         assert_return(j, -EINVAL);
2619         assert_return(!journal_pid_changed(j), -ECHILD);
2620         assert_return(ret, -EINVAL);
2621
2622         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2623         if (r < 0)
2624                 return r;
2625
2626         cid = strndup((const char*) data + 11, size - 11);
2627         if (!cid)
2628                 return -ENOMEM;
2629
2630         r = sd_id128_from_string(cid, &id);
2631         if (r < 0)
2632                 return r;
2633
2634         r = catalog_get(CATALOG_DATABASE, id, &text);
2635         if (r < 0)
2636                 return r;
2637
2638         t = replace_var(text, lookup_field, j);
2639         if (!t)
2640                 return -ENOMEM;
2641
2642         *ret = t;
2643         return 0;
2644 }
2645
2646 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2647         assert_return(ret, -EINVAL);
2648
2649         return catalog_get(CATALOG_DATABASE, id, ret);
2650 }
2651
2652 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2653         assert_return(j, -EINVAL);
2654         assert_return(!journal_pid_changed(j), -ECHILD);
2655
2656         j->data_threshold = sz;
2657         return 0;
2658 }
2659
2660 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2661         assert_return(j, -EINVAL);
2662         assert_return(!journal_pid_changed(j), -ECHILD);
2663         assert_return(sz, -EINVAL);
2664
2665         *sz = j->data_threshold;
2666         return 0;
2667 }