src/journal/sd-journal.c

   1 /*-*- Mode: C; c-basic-offset: 8; indent-tabs-mode: nil -*-*/
   2
   3 /***
   4   This file is part of systemd.
   5
   6   Copyright 2011 Lennart Poettering
   7
   8   systemd is free software; you can redistribute it and/or modify it
   9   under the terms of the GNU Lesser General Public License as published by
  10   the Free Software Foundation; either version 2.1 of the License, or
  11   (at your option) any later version.
  12
  13   systemd is distributed in the hope that it will be useful, but
  14   WITHOUT ANY WARRANTY; without even the implied warranty of
  15   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  16   Lesser General Public License for more details.
  17
  18   You should have received a copy of the GNU Lesser General Public License
  19   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  20 ***/
  21
  22 #include <errno.h>
  23 #include <fcntl.h>
  24 #include <linux/magic.h>
  25 #include <poll.h>
  26 #include <stddef.h>
  27 #include <sys/inotify.h>
  28 #include <sys/vfs.h>
  29 #include <unistd.h>
  30
  31 #include "sd-journal.h"
  32
  33 #include "alloc-util.h"
  34 #include "catalog.h"
  35 #include "compress.h"
  36 #include "dirent-util.h"
  37 #include "fd-util.h"
  38 #include "fileio.h"
  39 #include "formats-util.h"
  40 #include "fs-util.h"
  41 #include "hashmap.h"
  42 #include "hostname-util.h"
  43 #include "io-util.h"
  44 #include "journal-def.h"
  45 #include "journal-file.h"
  46 #include "journal-internal.h"
  47 #include "list.h"
  48 #include "lookup3.h"
  49 #include "missing.h"
  50 #include "path-util.h"
  51 #include "replace-var.h"
  52 #include "stat-util.h"
  53 #include "stdio-util.h"
  54 #include "string-util.h"
  55 #include "strv.h"
  56
  57 #define JOURNAL_FILES_MAX 7168
  58
  59 #define JOURNAL_FILES_RECHECK_USEC (2 * USEC_PER_SEC)
  60
  61 #define REPLACE_VAR_MAX 256
  62
  63 #define DEFAULT_DATA_THRESHOLD (64*1024)
  64
  65 static void remove_file_real(sd_journal *j, JournalFile *f);
  66
  67 static bool journal_pid_changed(sd_journal *j) {
  68         assert(j);
  69
  70         /* We don't support people creating a journal object and
  71          * keeping it around over a fork(). Let's complain. */
  72
  73         return j->original_pid != getpid();
  74 }
  75
  76 static int journal_put_error(sd_journal *j, int r, const char *path) {
  77         char *copy;
  78         int k;
  79
  80         /* Memorize an error we encountered, and store which
  81          * file/directory it was generated from. Note that we store
  82          * only *one* path per error code, as the error code is the
  83          * key into the hashmap, and the path is the value. This means
  84          * we keep track only of all error kinds, but not of all error
  85          * locations. This has the benefit that the hashmap cannot
  86          * grow beyond bounds.
  87          *
  88          * We return an error here only if we didn't manage to
  89          * memorize the real error. */
  90
  91         if (r >= 0)
  92                 return r;
  93
  94         k = hashmap_ensure_allocated(&j->errors, NULL);
  95         if (k < 0)
  96                 return k;
  97
  98         if (path) {
  99                 copy = strdup(path);
 100                 if (!copy)
 101                         return -ENOMEM;
 102         } else
 103                 copy = NULL;
 104
 105         k = hashmap_put(j->errors, INT_TO_PTR(r), copy);
 106         if (k < 0) {
 107                 free(copy);
 108
 109                 if (k == -EEXIST)
 110                         return 0;
 111
 112                 return k;
 113         }
 114
 115         return 0;
 116 }
 117
 118 static void detach_location(sd_journal *j) {
 119         Iterator i;
 120         JournalFile *f;
 121
 122         assert(j);
 123
 124         j->current_file = NULL;
 125         j->current_field = 0;
 126
 127         ORDERED_HASHMAP_FOREACH(f, j->files, i)
 128                 journal_file_reset_location(f);
 129 }
 130
 131 static void reset_location(sd_journal *j) {
 132         assert(j);
 133
 134         detach_location(j);
 135         zero(j->current_location);
 136 }
 137
 138 static void init_location(Location *l, LocationType type, JournalFile *f, Object *o) {
 139         assert(l);
 140         assert(type == LOCATION_DISCRETE || type == LOCATION_SEEK);
 141         assert(f);
 142         assert(o->object.type == OBJECT_ENTRY);
 143
 144         l->type = type;
 145         l->seqnum = le64toh(o->entry.seqnum);
 146         l->seqnum_id = f->header->seqnum_id;
 147         l->realtime = le64toh(o->entry.realtime);
 148         l->monotonic = le64toh(o->entry.monotonic);
 149         l->boot_id = o->entry.boot_id;
 150         l->xor_hash = le64toh(o->entry.xor_hash);
 151
 152         l->seqnum_set = l->realtime_set = l->monotonic_set = l->xor_hash_set = true;
 153 }
 154
 155 static void set_location(sd_journal *j, JournalFile *f, Object *o) {
 156         assert(j);
 157         assert(f);
 158         assert(o);
 159
 160         init_location(&j->current_location, LOCATION_DISCRETE, f, o);
 161
 162         j->current_file = f;
 163         j->current_field = 0;
 164
 165         /* Let f know its candidate entry was picked. */
 166         assert(f->location_type == LOCATION_SEEK);
 167         f->location_type = LOCATION_DISCRETE;
 168 }
 169
 170 static int match_is_valid(const void *data, size_t size) {
 171         const char *b, *p;
 172
 173         assert(data);
 174
 175         if (size < 2)
 176                 return false;
 177
 178         if (startswith(data, "__"))
 179                 return false;
 180
 181         b = data;
 182         for (p = b; p < b + size; p++) {
 183
 184                 if (*p == '=')
 185                         return p > b;
 186
 187                 if (*p == '_')
 188                         continue;
 189
 190                 if (*p >= 'A' && *p <= 'Z')
 191                         continue;
 192
 193                 if (*p >= '0' && *p <= '9')
 194                         continue;
 195
 196                 return false;
 197         }
 198
 199         return false;
 200 }
 201
 202 static bool same_field(const void *_a, size_t s, const void *_b, size_t t) {
 203         const uint8_t *a = _a, *b = _b;
 204         size_t j;
 205
 206         for (j = 0; j < s && j < t; j++) {
 207
 208                 if (a[j] != b[j])
 209                         return false;
 210
 211                 if (a[j] == '=')
 212                         return true;
 213         }
 214
 215         assert_not_reached("\"=\" not found");
 216 }
 217
 218 static Match *match_new(Match *p, MatchType t) {
 219         Match *m;
 220
 221         m = new0(Match, 1);
 222         if (!m)
 223                 return NULL;
 224
 225         m->type = t;
 226
 227         if (p) {
 228                 m->parent = p;
 229                 LIST_PREPEND(matches, p->matches, m);
 230         }
 231
 232         return m;
 233 }
 234
 235 static void match_free(Match *m) {
 236         assert(m);
 237
 238         while (m->matches)
 239                 match_free(m->matches);
 240
 241         if (m->parent)
 242                 LIST_REMOVE(matches, m->parent->matches, m);
 243
 244         free(m->data);
 245         free(m);
 246 }
 247
 248 static void match_free_if_empty(Match *m) {
 249         if (!m || m->matches)
 250                 return;
 251
 252         match_free(m);
 253 }
 254
 255 _public_ int sd_journal_add_match(sd_journal *j, const void *data, size_t size) {
 256         Match *l3, *l4, *add_here = NULL, *m;
 257         le64_t le_hash;
 258
 259         assert_return(j, -EINVAL);
 260         assert_return(!journal_pid_changed(j), -ECHILD);
 261         assert_return(data, -EINVAL);
 262
 263         if (size == 0)
 264                 size = strlen(data);
 265
 266         assert_return(match_is_valid(data, size), -EINVAL);
 267
 268         /* level 0: AND term
 269          * level 1: OR terms
 270          * level 2: AND terms
 271          * level 3: OR terms
 272          * level 4: concrete matches */
 273
 274         if (!j->level0) {
 275                 j->level0 = match_new(NULL, MATCH_AND_TERM);
 276                 if (!j->level0)
 277                         return -ENOMEM;
 278         }
 279
 280         if (!j->level1) {
 281                 j->level1 = match_new(j->level0, MATCH_OR_TERM);
 282                 if (!j->level1)
 283                         return -ENOMEM;
 284         }
 285
 286         if (!j->level2) {
 287                 j->level2 = match_new(j->level1, MATCH_AND_TERM);
 288                 if (!j->level2)
 289                         return -ENOMEM;
 290         }
 291
 292         assert(j->level0->type == MATCH_AND_TERM);
 293         assert(j->level1->type == MATCH_OR_TERM);
 294         assert(j->level2->type == MATCH_AND_TERM);
 295
 296         le_hash = htole64(hash64(data, size));
 297
 298         LIST_FOREACH(matches, l3, j->level2->matches) {
 299                 assert(l3->type == MATCH_OR_TERM);
 300
 301                 LIST_FOREACH(matches, l4, l3->matches) {
 302                         assert(l4->type == MATCH_DISCRETE);
 303
 304                         /* Exactly the same match already? Then ignore
 305                          * this addition */
 306                         if (l4->le_hash == le_hash &&
 307                             l4->size == size &&
 308                             memcmp(l4->data, data, size) == 0)
 309                                 return 0;
 310
 311                         /* Same field? Then let's add this to this OR term */
 312                         if (same_field(data, size, l4->data, l4->size)) {
 313                                 add_here = l3;
 314                                 break;
 315                         }
 316                 }
 317
 318                 if (add_here)
 319                         break;
 320         }
 321
 322         if (!add_here) {
 323                 add_here = match_new(j->level2, MATCH_OR_TERM);
 324                 if (!add_here)
 325                         goto fail;
 326         }
 327
 328         m = match_new(add_here, MATCH_DISCRETE);
 329         if (!m)
 330                 goto fail;
 331
 332         m->le_hash = le_hash;
 333         m->size = size;
 334         m->data = memdup(data, size);
 335         if (!m->data)
 336                 goto fail;
 337
 338         detach_location(j);
 339
 340         return 0;
 341
 342 fail:
 343         match_free_if_empty(add_here);
 344         match_free_if_empty(j->level2);
 345         match_free_if_empty(j->level1);
 346         match_free_if_empty(j->level0);
 347
 348         return -ENOMEM;
 349 }
 350
 351 _public_ int sd_journal_add_conjunction(sd_journal *j) {
 352         assert_return(j, -EINVAL);
 353         assert_return(!journal_pid_changed(j), -ECHILD);
 354
 355         if (!j->level0)
 356                 return 0;
 357
 358         if (!j->level1)
 359                 return 0;
 360
 361         if (!j->level1->matches)
 362                 return 0;
 363
 364         j->level1 = NULL;
 365         j->level2 = NULL;
 366
 367         return 0;
 368 }
 369
 370 _public_ int sd_journal_add_disjunction(sd_journal *j) {
 371         assert_return(j, -EINVAL);
 372         assert_return(!journal_pid_changed(j), -ECHILD);
 373
 374         if (!j->level0)
 375                 return 0;
 376
 377         if (!j->level1)
 378                 return 0;
 379
 380         if (!j->level2)
 381                 return 0;
 382
 383         if (!j->level2->matches)
 384                 return 0;
 385
 386         j->level2 = NULL;
 387         return 0;
 388 }
 389
 390 static char *match_make_string(Match *m) {
 391         char *p, *r;
 392         Match *i;
 393         bool enclose = false;
 394
 395         if (!m)
 396                 return strdup("none");
 397
 398         if (m->type == MATCH_DISCRETE)
 399                 return strndup(m->data, m->size);
 400
 401         p = NULL;
 402         LIST_FOREACH(matches, i, m->matches) {
 403                 char *t, *k;
 404
 405                 t = match_make_string(i);
 406                 if (!t) {
 407                         free(p);
 408                         return NULL;
 409                 }
 410
 411                 if (p) {
 412                         k = strjoin(p, m->type == MATCH_OR_TERM ? " OR " : " AND ", t, NULL);
 413                         free(p);
 414                         free(t);
 415
 416                         if (!k)
 417                                 return NULL;
 418
 419                         p = k;
 420
 421                         enclose = true;
 422                 } else
 423                         p = t;
 424         }
 425
 426         if (enclose) {
 427                 r = strjoin("(", p, ")", NULL);
 428                 free(p);
 429                 return r;
 430         }
 431
 432         return p;
 433 }
 434
 435 char *journal_make_match_string(sd_journal *j) {
 436         assert(j);
 437
 438         return match_make_string(j->level0);
 439 }
 440
 441 _public_ void sd_journal_flush_matches(sd_journal *j) {
 442         if (!j)
 443                 return;
 444
 445         if (j->level0)
 446                 match_free(j->level0);
 447
 448         j->level0 = j->level1 = j->level2 = NULL;
 449
 450         detach_location(j);
 451 }
 452
 453 _pure_ static int compare_with_location(JournalFile *f, Location *l) {
 454         assert(f);
 455         assert(l);
 456         assert(f->location_type == LOCATION_SEEK);
 457         assert(l->type == LOCATION_DISCRETE || l->type == LOCATION_SEEK);
 458
 459         if (l->monotonic_set &&
 460             sd_id128_equal(f->current_boot_id, l->boot_id) &&
 461             l->realtime_set &&
 462             f->current_realtime == l->realtime &&
 463             l->xor_hash_set &&
 464             f->current_xor_hash == l->xor_hash)
 465                 return 0;
 466
 467         if (l->seqnum_set &&
 468             sd_id128_equal(f->header->seqnum_id, l->seqnum_id)) {
 469
 470                 if (f->current_seqnum < l->seqnum)
 471                         return -1;
 472                 if (f->current_seqnum > l->seqnum)
 473                         return 1;
 474         }
 475
 476         if (l->monotonic_set &&
 477             sd_id128_equal(f->current_boot_id, l->boot_id)) {
 478
 479                 if (f->current_monotonic < l->monotonic)
 480                         return -1;
 481                 if (f->current_monotonic > l->monotonic)
 482                         return 1;
 483         }
 484
 485         if (l->realtime_set) {
 486
 487                 if (f->current_realtime < l->realtime)
 488                         return -1;
 489                 if (f->current_realtime > l->realtime)
 490                         return 1;
 491         }
 492
 493         if (l->xor_hash_set) {
 494
 495                 if (f->current_xor_hash < l->xor_hash)
 496                         return -1;
 497                 if (f->current_xor_hash > l->xor_hash)
 498                         return 1;
 499         }
 500
 501         return 0;
 502 }
 503
 504 static int next_for_match(
 505                 sd_journal *j,
 506                 Match *m,
 507                 JournalFile *f,
 508                 uint64_t after_offset,
 509                 direction_t direction,
 510                 Object **ret,
 511                 uint64_t *offset) {
 512
 513         int r;
 514         uint64_t np = 0;
 515         Object *n;
 516
 517         assert(j);
 518         assert(m);
 519         assert(f);
 520
 521         if (m->type == MATCH_DISCRETE) {
 522                 uint64_t dp;
 523
 524                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 525                 if (r <= 0)
 526                         return r;
 527
 528                 return journal_file_move_to_entry_by_offset_for_data(f, dp, after_offset, direction, ret, offset);
 529
 530         } else if (m->type == MATCH_OR_TERM) {
 531                 Match *i;
 532
 533                 /* Find the earliest match beyond after_offset */
 534
 535                 LIST_FOREACH(matches, i, m->matches) {
 536                         uint64_t cp;
 537
 538                         r = next_for_match(j, i, f, after_offset, direction, NULL, &cp);
 539                         if (r < 0)
 540                                 return r;
 541                         else if (r > 0) {
 542                                 if (np == 0 || (direction == DIRECTION_DOWN ? cp < np : cp > np))
 543                                         np = cp;
 544                         }
 545                 }
 546
 547                 if (np == 0)
 548                         return 0;
 549
 550         } else if (m->type == MATCH_AND_TERM) {
 551                 Match *i, *last_moved;
 552
 553                 /* Always jump to the next matching entry and repeat
 554                  * this until we find an offset that matches for all
 555                  * matches. */
 556
 557                 if (!m->matches)
 558                         return 0;
 559
 560                 r = next_for_match(j, m->matches, f, after_offset, direction, NULL, &np);
 561                 if (r <= 0)
 562                         return r;
 563
 564                 assert(direction == DIRECTION_DOWN ? np >= after_offset : np <= after_offset);
 565                 last_moved = m->matches;
 566
 567                 LIST_LOOP_BUT_ONE(matches, i, m->matches, last_moved) {
 568                         uint64_t cp;
 569
 570                         r = next_for_match(j, i, f, np, direction, NULL, &cp);
 571                         if (r <= 0)
 572                                 return r;
 573
 574                         assert(direction == DIRECTION_DOWN ? cp >= np : cp <= np);
 575                         if (direction == DIRECTION_DOWN ? cp > np : cp < np) {
 576                                 np = cp;
 577                                 last_moved = i;
 578                         }
 579                 }
 580         }
 581
 582         assert(np > 0);
 583
 584         r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 585         if (r < 0)
 586                 return r;
 587
 588         if (ret)
 589                 *ret = n;
 590         if (offset)
 591                 *offset = np;
 592
 593         return 1;
 594 }
 595
 596 static int find_location_for_match(
 597                 sd_journal *j,
 598                 Match *m,
 599                 JournalFile *f,
 600                 direction_t direction,
 601                 Object **ret,
 602                 uint64_t *offset) {
 603
 604         int r;
 605
 606         assert(j);
 607         assert(m);
 608         assert(f);
 609
 610         if (m->type == MATCH_DISCRETE) {
 611                 uint64_t dp;
 612
 613                 r = journal_file_find_data_object_with_hash(f, m->data, m->size, le64toh(m->le_hash), NULL, &dp);
 614                 if (r <= 0)
 615                         return r;
 616
 617                 /* FIXME: missing: find by monotonic */
 618
 619                 if (j->current_location.type == LOCATION_HEAD)
 620                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_DOWN, ret, offset);
 621                 if (j->current_location.type == LOCATION_TAIL)
 622                         return journal_file_next_entry_for_data(f, NULL, 0, dp, DIRECTION_UP, ret, offset);
 623                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 624                         return journal_file_move_to_entry_by_seqnum_for_data(f, dp, j->current_location.seqnum, direction, ret, offset);
 625                 if (j->current_location.monotonic_set) {
 626                         r = journal_file_move_to_entry_by_monotonic_for_data(f, dp, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 627                         if (r != -ENOENT)
 628                                 return r;
 629                 }
 630                 if (j->current_location.realtime_set)
 631                         return journal_file_move_to_entry_by_realtime_for_data(f, dp, j->current_location.realtime, direction, ret, offset);
 632
 633                 return journal_file_next_entry_for_data(f, NULL, 0, dp, direction, ret, offset);
 634
 635         } else if (m->type == MATCH_OR_TERM) {
 636                 uint64_t np = 0;
 637                 Object *n;
 638                 Match *i;
 639
 640                 /* Find the earliest match */
 641
 642                 LIST_FOREACH(matches, i, m->matches) {
 643                         uint64_t cp;
 644
 645                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 646                         if (r < 0)
 647                                 return r;
 648                         else if (r > 0) {
 649                                 if (np == 0 || (direction == DIRECTION_DOWN ? np > cp : np < cp))
 650                                         np = cp;
 651                         }
 652                 }
 653
 654                 if (np == 0)
 655                         return 0;
 656
 657                 r = journal_file_move_to_object(f, OBJECT_ENTRY, np, &n);
 658                 if (r < 0)
 659                         return r;
 660
 661                 if (ret)
 662                         *ret = n;
 663                 if (offset)
 664                         *offset = np;
 665
 666                 return 1;
 667
 668         } else {
 669                 Match *i;
 670                 uint64_t np = 0;
 671
 672                 assert(m->type == MATCH_AND_TERM);
 673
 674                 /* First jump to the last match, and then find the
 675                  * next one where all matches match */
 676
 677                 if (!m->matches)
 678                         return 0;
 679
 680                 LIST_FOREACH(matches, i, m->matches) {
 681                         uint64_t cp;
 682
 683                         r = find_location_for_match(j, i, f, direction, NULL, &cp);
 684                         if (r <= 0)
 685                                 return r;
 686
 687                         if (np == 0 || (direction == DIRECTION_DOWN ? cp > np : cp < np))
 688                                 np = cp;
 689                 }
 690
 691                 return next_for_match(j, m, f, np, direction, ret, offset);
 692         }
 693 }
 694
 695 static int find_location_with_matches(
 696                 sd_journal *j,
 697                 JournalFile *f,
 698                 direction_t direction,
 699                 Object **ret,
 700                 uint64_t *offset) {
 701
 702         int r;
 703
 704         assert(j);
 705         assert(f);
 706         assert(ret);
 707         assert(offset);
 708
 709         if (!j->level0) {
 710                 /* No matches is simple */
 711
 712                 if (j->current_location.type == LOCATION_HEAD)
 713                         return journal_file_next_entry(f, 0, DIRECTION_DOWN, ret, offset);
 714                 if (j->current_location.type == LOCATION_TAIL)
 715                         return journal_file_next_entry(f, 0, DIRECTION_UP, ret, offset);
 716                 if (j->current_location.seqnum_set && sd_id128_equal(j->current_location.seqnum_id, f->header->seqnum_id))
 717                         return journal_file_move_to_entry_by_seqnum(f, j->current_location.seqnum, direction, ret, offset);
 718                 if (j->current_location.monotonic_set) {
 719                         r = journal_file_move_to_entry_by_monotonic(f, j->current_location.boot_id, j->current_location.monotonic, direction, ret, offset);
 720                         if (r != -ENOENT)
 721                                 return r;
 722                 }
 723                 if (j->current_location.realtime_set)
 724                         return journal_file_move_to_entry_by_realtime(f, j->current_location.realtime, direction, ret, offset);
 725
 726                 return journal_file_next_entry(f, 0, direction, ret, offset);
 727         } else
 728                 return find_location_for_match(j, j->level0, f, direction, ret, offset);
 729 }
 730
 731 static int next_with_matches(
 732                 sd_journal *j,
 733                 JournalFile *f,
 734                 direction_t direction,
 735                 Object **ret,
 736                 uint64_t *offset) {
 737
 738         assert(j);
 739         assert(f);
 740         assert(ret);
 741         assert(offset);
 742
 743         /* No matches is easy. We simple advance the file
 744          * pointer by one. */
 745         if (!j->level0)
 746                 return journal_file_next_entry(f, f->current_offset, direction, ret, offset);
 747
 748         /* If we have a match then we look for the next matching entry
 749          * with an offset at least one step larger */
 750         return next_for_match(j, j->level0, f,
 751                               direction == DIRECTION_DOWN ? f->current_offset + 1
 752                                                           : f->current_offset - 1,
 753                               direction, ret, offset);
 754 }
 755
 756 static int next_beyond_location(sd_journal *j, JournalFile *f, direction_t direction) {
 757         Object *c;
 758         uint64_t cp, n_entries;
 759         int r;
 760
 761         assert(j);
 762         assert(f);
 763
 764         n_entries = le64toh(f->header->n_entries);
 765
 766         /* If we hit EOF before, we don't need to look into this file again
 767          * unless direction changed or new entries appeared. */
 768         if (f->last_direction == direction && f->location_type == LOCATION_TAIL &&
 769             n_entries == f->last_n_entries)
 770                 return 0;
 771
 772         f->last_n_entries = n_entries;
 773
 774         if (f->last_direction == direction && f->current_offset > 0) {
 775                 /* LOCATION_SEEK here means we did the work in a previous
 776                  * iteration and the current location already points to a
 777                  * candidate entry. */
 778                 if (f->location_type != LOCATION_SEEK) {
 779                         r = next_with_matches(j, f, direction, &c, &cp);
 780                         if (r <= 0)
 781                                 return r;
 782
 783                         journal_file_save_location(f, c, cp);
 784                 }
 785         } else {
 786                 f->last_direction = direction;
 787
 788                 r = find_location_with_matches(j, f, direction, &c, &cp);
 789                 if (r <= 0)
 790                         return r;
 791
 792                 journal_file_save_location(f, c, cp);
 793         }
 794
 795         /* OK, we found the spot, now let's advance until an entry
 796          * that is actually different from what we were previously
 797          * looking at. This is necessary to handle entries which exist
 798          * in two (or more) journal files, and which shall all be
 799          * suppressed but one. */
 800
 801         for (;;) {
 802                 bool found;
 803
 804                 if (j->current_location.type == LOCATION_DISCRETE) {
 805                         int k;
 806
 807                         k = compare_with_location(f, &j->current_location);
 808
 809                         found = direction == DIRECTION_DOWN ? k > 0 : k < 0;
 810                 } else
 811                         found = true;
 812
 813                 if (found)
 814                         return 1;
 815
 816                 r = next_with_matches(j, f, direction, &c, &cp);
 817                 if (r <= 0)
 818                         return r;
 819
 820                 journal_file_save_location(f, c, cp);
 821         }
 822 }
 823
 824 static int real_journal_next(sd_journal *j, direction_t direction) {
 825         JournalFile *f, *new_file = NULL;
 826         Iterator i;
 827         Object *o;
 828         int r;
 829
 830         assert_return(j, -EINVAL);
 831         assert_return(!journal_pid_changed(j), -ECHILD);
 832
 833         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
 834                 bool found;
 835
 836                 r = next_beyond_location(j, f, direction);
 837                 if (r < 0) {
 838                         log_debug_errno(r, "Can't iterate through %s, ignoring: %m", f->path);
 839                         remove_file_real(j, f);
 840                         continue;
 841                 } else if (r == 0) {
 842                         f->location_type = LOCATION_TAIL;
 843                         continue;
 844                 }
 845
 846                 if (!new_file)
 847                         found = true;
 848                 else {
 849                         int k;
 850
 851                         k = journal_file_compare_locations(f, new_file);
 852
 853                         found = direction == DIRECTION_DOWN ? k < 0 : k > 0;
 854                 }
 855
 856                 if (found)
 857                         new_file = f;
 858         }
 859
 860         if (!new_file)
 861                 return 0;
 862
 863         r = journal_file_move_to_object(new_file, OBJECT_ENTRY, new_file->current_offset, &o);
 864         if (r < 0)
 865                 return r;
 866
 867         set_location(j, new_file, o);
 868
 869         return 1;
 870 }
 871
 872 _public_ int sd_journal_next(sd_journal *j) {
 873         return real_journal_next(j, DIRECTION_DOWN);
 874 }
 875
 876 _public_ int sd_journal_previous(sd_journal *j) {
 877         return real_journal_next(j, DIRECTION_UP);
 878 }
 879
 880 static int real_journal_next_skip(sd_journal *j, direction_t direction, uint64_t skip) {
 881         int c = 0, r;
 882
 883         assert_return(j, -EINVAL);
 884         assert_return(!journal_pid_changed(j), -ECHILD);
 885
 886         if (skip == 0) {
 887                 /* If this is not a discrete skip, then at least
 888                  * resolve the current location */
 889                 if (j->current_location.type != LOCATION_DISCRETE)
 890                         return real_journal_next(j, direction);
 891
 892                 return 0;
 893         }
 894
 895         do {
 896                 r = real_journal_next(j, direction);
 897                 if (r < 0)
 898                         return r;
 899
 900                 if (r == 0)
 901                         return c;
 902
 903                 skip--;
 904                 c++;
 905         } while (skip > 0);
 906
 907         return c;
 908 }
 909
 910 _public_ int sd_journal_next_skip(sd_journal *j, uint64_t skip) {
 911         return real_journal_next_skip(j, DIRECTION_DOWN, skip);
 912 }
 913
 914 _public_ int sd_journal_previous_skip(sd_journal *j, uint64_t skip) {
 915         return real_journal_next_skip(j, DIRECTION_UP, skip);
 916 }
 917
 918 _public_ int sd_journal_get_cursor(sd_journal *j, char **cursor) {
 919         Object *o;
 920         int r;
 921         char bid[33], sid[33];
 922
 923         assert_return(j, -EINVAL);
 924         assert_return(!journal_pid_changed(j), -ECHILD);
 925         assert_return(cursor, -EINVAL);
 926
 927         if (!j->current_file || j->current_file->current_offset <= 0)
 928                 return -EADDRNOTAVAIL;
 929
 930         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
 931         if (r < 0)
 932                 return r;
 933
 934         sd_id128_to_string(j->current_file->header->seqnum_id, sid);
 935         sd_id128_to_string(o->entry.boot_id, bid);
 936
 937         if (asprintf(cursor,
 938                      "s=%s;i=%"PRIx64";b=%s;m=%"PRIx64";t=%"PRIx64";x=%"PRIx64,
 939                      sid, le64toh(o->entry.seqnum),
 940                      bid, le64toh(o->entry.monotonic),
 941                      le64toh(o->entry.realtime),
 942                      le64toh(o->entry.xor_hash)) < 0)
 943                 return -ENOMEM;
 944
 945         return 0;
 946 }
 947
 948 _public_ int sd_journal_seek_cursor(sd_journal *j, const char *cursor) {
 949         const char *word, *state;
 950         size_t l;
 951         unsigned long long seqnum, monotonic, realtime, xor_hash;
 952         bool
 953                 seqnum_id_set = false,
 954                 seqnum_set = false,
 955                 boot_id_set = false,
 956                 monotonic_set = false,
 957                 realtime_set = false,
 958                 xor_hash_set = false;
 959         sd_id128_t seqnum_id, boot_id;
 960
 961         assert_return(j, -EINVAL);
 962         assert_return(!journal_pid_changed(j), -ECHILD);
 963         assert_return(!isempty(cursor), -EINVAL);
 964
 965         FOREACH_WORD_SEPARATOR(word, l, cursor, ";", state) {
 966                 char *item;
 967                 int k = 0;
 968
 969                 if (l < 2 || word[1] != '=')
 970                         return -EINVAL;
 971
 972                 item = strndup(word, l);
 973                 if (!item)
 974                         return -ENOMEM;
 975
 976                 switch (word[0]) {
 977
 978                 case 's':
 979                         seqnum_id_set = true;
 980                         k = sd_id128_from_string(item+2, &seqnum_id);
 981                         break;
 982
 983                 case 'i':
 984                         seqnum_set = true;
 985                         if (sscanf(item+2, "%llx", &seqnum) != 1)
 986                                 k = -EINVAL;
 987                         break;
 988
 989                 case 'b':
 990                         boot_id_set = true;
 991                         k = sd_id128_from_string(item+2, &boot_id);
 992                         break;
 993
 994                 case 'm':
 995                         monotonic_set = true;
 996                         if (sscanf(item+2, "%llx", &monotonic) != 1)
 997                                 k = -EINVAL;
 998                         break;
 999
1000                 case 't':
1001                         realtime_set = true;
1002                         if (sscanf(item+2, "%llx", &realtime) != 1)
1003                                 k = -EINVAL;
1004                         break;
1005
1006                 case 'x':
1007                         xor_hash_set = true;
1008                         if (sscanf(item+2, "%llx", &xor_hash) != 1)
1009                                 k = -EINVAL;
1010                         break;
1011                 }
1012
1013                 free(item);
1014
1015                 if (k < 0)
1016                         return k;
1017         }
1018
1019         if ((!seqnum_set || !seqnum_id_set) &&
1020             (!monotonic_set || !boot_id_set) &&
1021             !realtime_set)
1022                 return -EINVAL;
1023
1024         reset_location(j);
1025
1026         j->current_location.type = LOCATION_SEEK;
1027
1028         if (realtime_set) {
1029                 j->current_location.realtime = (uint64_t) realtime;
1030                 j->current_location.realtime_set = true;
1031         }
1032
1033         if (seqnum_set && seqnum_id_set) {
1034                 j->current_location.seqnum = (uint64_t) seqnum;
1035                 j->current_location.seqnum_id = seqnum_id;
1036                 j->current_location.seqnum_set = true;
1037         }
1038
1039         if (monotonic_set && boot_id_set) {
1040                 j->current_location.monotonic = (uint64_t) monotonic;
1041                 j->current_location.boot_id = boot_id;
1042                 j->current_location.monotonic_set = true;
1043         }
1044
1045         if (xor_hash_set) {
1046                 j->current_location.xor_hash = (uint64_t) xor_hash;
1047                 j->current_location.xor_hash_set = true;
1048         }
1049
1050         return 0;
1051 }
1052
1053 _public_ int sd_journal_test_cursor(sd_journal *j, const char *cursor) {
1054         int r;
1055         Object *o;
1056
1057         assert_return(j, -EINVAL);
1058         assert_return(!journal_pid_changed(j), -ECHILD);
1059         assert_return(!isempty(cursor), -EINVAL);
1060
1061         if (!j->current_file || j->current_file->current_offset <= 0)
1062                 return -EADDRNOTAVAIL;
1063
1064         r = journal_file_move_to_object(j->current_file, OBJECT_ENTRY, j->current_file->current_offset, &o);
1065         if (r < 0)
1066                 return r;
1067
1068         for(;;) {
1069                 _cleanup_free_ char *item = NULL;
1070                 unsigned long long ll;
1071                 sd_id128_t id;
1072                 int k = 0;
1073
1074                 r = extract_first_word(&cursor, &item, ";", EXTRACT_DONT_COALESCE_SEPARATORS);
1075                 if (r < 0)
1076                         return r;
1077
1078                 if (r == 0)
1079                         break;
1080
1081                 if (strlen(item) < 2 || item[1] != '=')
1082                         return -EINVAL;
1083
1084                 switch (item[0]) {
1085
1086                 case 's':
1087                         k = sd_id128_from_string(item+2, &id);
1088                         if (k < 0)
1089                                 return k;
1090                         if (!sd_id128_equal(id, j->current_file->header->seqnum_id))
1091                                 return 0;
1092                         break;
1093
1094                 case 'i':
1095                         if (sscanf(item+2, "%llx", &ll) != 1)
1096                                 return -EINVAL;
1097                         if (ll != le64toh(o->entry.seqnum))
1098                                 return 0;
1099                         break;
1100
1101                 case 'b':
1102                         k = sd_id128_from_string(item+2, &id);
1103                         if (k < 0)
1104                                 return k;
1105                         if (!sd_id128_equal(id, o->entry.boot_id))
1106                                 return 0;
1107                         break;
1108
1109                 case 'm':
1110                         if (sscanf(item+2, "%llx", &ll) != 1)
1111                                 return -EINVAL;
1112                         if (ll != le64toh(o->entry.monotonic))
1113                                 return 0;
1114                         break;
1115
1116                 case 't':
1117                         if (sscanf(item+2, "%llx", &ll) != 1)
1118                                 return -EINVAL;
1119                         if (ll != le64toh(o->entry.realtime))
1120                                 return 0;
1121                         break;
1122
1123                 case 'x':
1124                         if (sscanf(item+2, "%llx", &ll) != 1)
1125                                 return -EINVAL;
1126                         if (ll != le64toh(o->entry.xor_hash))
1127                                 return 0;
1128                         break;
1129                 }
1130         }
1131
1132         return 1;
1133 }
1134
1135
1136 _public_ int sd_journal_seek_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t usec) {
1137         assert_return(j, -EINVAL);
1138         assert_return(!journal_pid_changed(j), -ECHILD);
1139
1140         reset_location(j);
1141         j->current_location.type = LOCATION_SEEK;
1142         j->current_location.boot_id = boot_id;
1143         j->current_location.monotonic = usec;
1144         j->current_location.monotonic_set = true;
1145
1146         return 0;
1147 }
1148
1149 _public_ int sd_journal_seek_realtime_usec(sd_journal *j, uint64_t usec) {
1150         assert_return(j, -EINVAL);
1151         assert_return(!journal_pid_changed(j), -ECHILD);
1152
1153         reset_location(j);
1154         j->current_location.type = LOCATION_SEEK;
1155         j->current_location.realtime = usec;
1156         j->current_location.realtime_set = true;
1157
1158         return 0;
1159 }
1160
1161 _public_ int sd_journal_seek_head(sd_journal *j) {
1162         assert_return(j, -EINVAL);
1163         assert_return(!journal_pid_changed(j), -ECHILD);
1164
1165         reset_location(j);
1166         j->current_location.type = LOCATION_HEAD;
1167
1168         return 0;
1169 }
1170
1171 _public_ int sd_journal_seek_tail(sd_journal *j) {
1172         assert_return(j, -EINVAL);
1173         assert_return(!journal_pid_changed(j), -ECHILD);
1174
1175         reset_location(j);
1176         j->current_location.type = LOCATION_TAIL;
1177
1178         return 0;
1179 }
1180
1181 static void check_network(sd_journal *j, int fd) {
1182         struct statfs sfs;
1183
1184         assert(j);
1185
1186         if (j->on_network)
1187                 return;
1188
1189         if (fstatfs(fd, &sfs) < 0)
1190                 return;
1191
1192         j->on_network =
1193                 F_TYPE_EQUAL(sfs.f_type, CIFS_MAGIC_NUMBER) ||
1194                 F_TYPE_EQUAL(sfs.f_type, CODA_SUPER_MAGIC) ||
1195                 F_TYPE_EQUAL(sfs.f_type, NCP_SUPER_MAGIC) ||
1196                 F_TYPE_EQUAL(sfs.f_type, NFS_SUPER_MAGIC) ||
1197                 F_TYPE_EQUAL(sfs.f_type, SMB_SUPER_MAGIC);
1198 }
1199
1200 static bool file_has_type_prefix(const char *prefix, const char *filename) {
1201         const char *full, *tilded, *atted;
1202
1203         full = strjoina(prefix, ".journal");
1204         tilded = strjoina(full, "~");
1205         atted = strjoina(prefix, "@");
1206
1207         return streq(filename, full) ||
1208                streq(filename, tilded) ||
1209                startswith(filename, atted);
1210 }
1211
1212 static bool file_type_wanted(int flags, const char *filename) {
1213         assert(filename);
1214
1215         if (!endswith(filename, ".journal") && !endswith(filename, ".journal~"))
1216                 return false;
1217
1218         /* no flags set → every type is OK */
1219         if (!(flags & (SD_JOURNAL_SYSTEM | SD_JOURNAL_CURRENT_USER)))
1220                 return true;
1221
1222         if (flags & SD_JOURNAL_SYSTEM && file_has_type_prefix("system", filename))
1223                 return true;
1224
1225         if (flags & SD_JOURNAL_CURRENT_USER) {
1226                 char prefix[5 + DECIMAL_STR_MAX(uid_t) + 1];
1227
1228                 xsprintf(prefix, "user-"UID_FMT, getuid());
1229
1230                 if (file_has_type_prefix(prefix, filename))
1231                         return true;
1232         }
1233
1234         return false;
1235 }
1236
1237 static int add_any_file(sd_journal *j, const char *path) {
1238         JournalFile *f = NULL;
1239         int r, k;
1240
1241         assert(j);
1242         assert(path);
1243
1244         if (ordered_hashmap_get(j->files, path))
1245                 return 0;
1246
1247         if (ordered_hashmap_size(j->files) >= JOURNAL_FILES_MAX) {
1248                 log_debug("Too many open journal files, not adding %s.", path);
1249                 r = -ETOOMANYREFS;
1250                 goto fail;
1251         }
1252
1253         r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
1254         if (r < 0) {
1255                 log_debug_errno(r, "Failed to open journal file %s: %m", path);
1256                 goto fail;
1257         }
1258
1259         /* journal_file_dump(f); */
1260
1261         r = ordered_hashmap_put(j->files, f->path, f);
1262         if (r < 0) {
1263                 journal_file_close(f);
1264                 goto fail;
1265         }
1266
1267         log_debug("File %s added.", f->path);
1268
1269         check_network(j, f->fd);
1270
1271         j->current_invalidate_counter ++;
1272
1273         return 0;
1274
1275 fail:
1276         k = journal_put_error(j, r, path);
1277         if (k < 0)
1278                 return k;
1279
1280         return r;
1281 }
1282
1283 static int add_file(sd_journal *j, const char *prefix, const char *filename) {
1284         const char *path;
1285
1286         assert(j);
1287         assert(prefix);
1288         assert(filename);
1289
1290         if (j->no_new_files ||
1291             !file_type_wanted(j->flags, filename))
1292                 return 0;
1293
1294         path = strjoina(prefix, "/", filename);
1295         return add_any_file(j, path);
1296 }
1297
1298 static void remove_file(sd_journal *j, const char *prefix, const char *filename) {
1299         const char *path;
1300         JournalFile *f;
1301
1302         assert(j);
1303         assert(prefix);
1304         assert(filename);
1305
1306         path = strjoina(prefix, "/", filename);
1307         f = ordered_hashmap_get(j->files, path);
1308         if (!f)
1309                 return;
1310
1311         remove_file_real(j, f);
1312 }
1313
1314 static void remove_file_real(sd_journal *j, JournalFile *f) {
1315         assert(j);
1316         assert(f);
1317
1318         ordered_hashmap_remove(j->files, f->path);
1319
1320         log_debug("File %s removed.", f->path);
1321
1322         if (j->current_file == f) {
1323                 j->current_file = NULL;
1324                 j->current_field = 0;
1325         }
1326
1327         if (j->unique_file == f) {
1328                 /* Jump to the next unique_file or NULL if that one was last */
1329                 j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
1330                 j->unique_offset = 0;
1331                 if (!j->unique_file)
1332                         j->unique_file_lost = true;
1333         }
1334
1335         journal_file_close(f);
1336
1337         j->current_invalidate_counter ++;
1338 }
1339
1340 static int dirname_is_machine_id(const char *fn) {
1341         sd_id128_t id, machine;
1342         int r;
1343
1344         r = sd_id128_get_machine(&machine);
1345         if (r < 0)
1346                 return r;
1347
1348         r = sd_id128_from_string(fn, &id);
1349         if (r < 0)
1350                 return r;
1351
1352         return sd_id128_equal(id, machine);
1353 }
1354
1355 static int add_directory(sd_journal *j, const char *prefix, const char *dirname) {
1356         _cleanup_free_ char *path = NULL;
1357         _cleanup_closedir_ DIR *d = NULL;
1358         struct dirent *de = NULL;
1359         Directory *m;
1360         int r, k;
1361
1362         assert(j);
1363         assert(prefix);
1364         assert(dirname);
1365
1366         log_debug("Considering %s/%s.", prefix, dirname);
1367
1368         if ((j->flags & SD_JOURNAL_LOCAL_ONLY) &&
1369             !(dirname_is_machine_id(dirname) > 0 || path_startswith(prefix, "/run")))
1370             return 0;
1371
1372         path = strjoin(prefix, "/", dirname, NULL);
1373         if (!path) {
1374                 r = -ENOMEM;
1375                 goto fail;
1376         }
1377
1378         d = opendir(path);
1379         if (!d) {
1380                 r = log_debug_errno(errno, "Failed to open directory %s: %m", path);
1381                 goto fail;
1382         }
1383
1384         m = hashmap_get(j->directories_by_path, path);
1385         if (!m) {
1386                 m = new0(Directory, 1);
1387                 if (!m) {
1388                         r = -ENOMEM;
1389                         goto fail;
1390                 }
1391
1392                 m->is_root = false;
1393                 m->path = path;
1394
1395                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1396                         free(m);
1397                         r = -ENOMEM;
1398                         goto fail;
1399                 }
1400
1401                 path = NULL; /* avoid freeing in cleanup */
1402                 j->current_invalidate_counter ++;
1403
1404                 log_debug("Directory %s added.", m->path);
1405
1406         } else if (m->is_root)
1407                 return 0;
1408
1409         if (m->wd <= 0 && j->inotify_fd >= 0) {
1410
1411                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1412                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1413                                           IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT|IN_MOVED_FROM|
1414                                           IN_ONLYDIR);
1415
1416                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1417                         inotify_rm_watch(j->inotify_fd, m->wd);
1418         }
1419
1420         FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {
1421
1422                 if (dirent_is_file_with_suffix(de, ".journal") ||
1423                     dirent_is_file_with_suffix(de, ".journal~"))
1424                         (void) add_file(j, m->path, de->d_name);
1425         }
1426
1427         check_network(j, dirfd(d));
1428
1429         return 0;
1430
1431 fail:
1432         k = journal_put_error(j, r, path ?: dirname);
1433         if (k < 0)
1434                 return k;
1435
1436         return r;
1437 }
1438
1439 static int add_root_directory(sd_journal *j, const char *p, bool missing_ok) {
1440         _cleanup_closedir_ DIR *d = NULL;
1441         struct dirent *de;
1442         Directory *m;
1443         int r, k;
1444
1445         assert(j);
1446         assert(p);
1447
1448         if ((j->flags & SD_JOURNAL_RUNTIME_ONLY) &&
1449             !path_startswith(p, "/run"))
1450                 return -EINVAL;
1451
1452         if (j->prefix)
1453                 p = strjoina(j->prefix, p);
1454
1455         d = opendir(p);
1456         if (!d) {
1457                 if (errno == ENOENT && missing_ok)
1458                         return 0;
1459
1460                 r = log_debug_errno(errno, "Failed to open root directory %s: %m", p);
1461                 goto fail;
1462         }
1463
1464         m = hashmap_get(j->directories_by_path, p);
1465         if (!m) {
1466                 m = new0(Directory, 1);
1467                 if (!m) {
1468                         r = -ENOMEM;
1469                         goto fail;
1470                 }
1471
1472                 m->is_root = true;
1473                 m->path = strdup(p);
1474                 if (!m->path) {
1475                         free(m);
1476                         r = -ENOMEM;
1477                         goto fail;
1478                 }
1479
1480                 if (hashmap_put(j->directories_by_path, m->path, m) < 0) {
1481                         free(m->path);
1482                         free(m);
1483                         r = -ENOMEM;
1484                         goto fail;
1485                 }
1486
1487                 j->current_invalidate_counter ++;
1488
1489                 log_debug("Root directory %s added.", m->path);
1490
1491         } else if (!m->is_root)
1492                 return 0;
1493
1494         if (m->wd <= 0 && j->inotify_fd >= 0) {
1495
1496                 m->wd = inotify_add_watch(j->inotify_fd, m->path,
1497                                           IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB|IN_DELETE|
1498                                           IN_ONLYDIR);
1499
1500                 if (m->wd > 0 && hashmap_put(j->directories_by_wd, INT_TO_PTR(m->wd), m) < 0)
1501                         inotify_rm_watch(j->inotify_fd, m->wd);
1502         }
1503
1504         if (j->no_new_files)
1505                 return 0;
1506
1507         FOREACH_DIRENT_ALL(de, d, return log_debug_errno(errno, "Failed to read directory %s: %m", m->path)) {
1508                 sd_id128_t id;
1509
1510                 if (dirent_is_file_with_suffix(de, ".journal") ||
1511                     dirent_is_file_with_suffix(de, ".journal~"))
1512                         (void) add_file(j, m->path, de->d_name);
1513                 else if (IN_SET(de->d_type, DT_DIR, DT_LNK, DT_UNKNOWN) &&
1514                          sd_id128_from_string(de->d_name, &id) >= 0)
1515                         (void) add_directory(j, m->path, de->d_name);
1516         }
1517
1518         check_network(j, dirfd(d));
1519
1520         return 0;
1521
1522 fail:
1523         k = journal_put_error(j, r, p);
1524         if (k < 0)
1525                 return k;
1526
1527         return r;
1528 }
1529
1530 static void remove_directory(sd_journal *j, Directory *d) {
1531         assert(j);
1532
1533         if (d->wd > 0) {
1534                 hashmap_remove(j->directories_by_wd, INT_TO_PTR(d->wd));
1535
1536                 if (j->inotify_fd >= 0)
1537                         inotify_rm_watch(j->inotify_fd, d->wd);
1538         }
1539
1540         hashmap_remove(j->directories_by_path, d->path);
1541
1542         if (d->is_root)
1543                 log_debug("Root directory %s removed.", d->path);
1544         else
1545                 log_debug("Directory %s removed.", d->path);
1546
1547         free(d->path);
1548         free(d);
1549 }
1550
1551 static int add_search_paths(sd_journal *j) {
1552
1553         static const char search_paths[] =
1554                 "/run/log/journal\0"
1555                 "/var/log/journal\0";
1556         const char *p;
1557
1558         assert(j);
1559
1560         /* We ignore most errors here, since the idea is to only open
1561          * what's actually accessible, and ignore the rest. */
1562
1563         NULSTR_FOREACH(p, search_paths)
1564                 (void) add_root_directory(j, p, true);
1565
1566         return 0;
1567 }
1568
1569 static int add_current_paths(sd_journal *j) {
1570         Iterator i;
1571         JournalFile *f;
1572
1573         assert(j);
1574         assert(j->no_new_files);
1575
1576         /* Simply adds all directories for files we have open as
1577          * "root" directories. We don't expect errors here, so we
1578          * treat them as fatal. */
1579
1580         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
1581                 _cleanup_free_ char *dir;
1582                 int r;
1583
1584                 dir = dirname_malloc(f->path);
1585                 if (!dir)
1586                         return -ENOMEM;
1587
1588                 r = add_root_directory(j, dir, true);
1589                 if (r < 0)
1590                         return r;
1591         }
1592
1593         return 0;
1594 }
1595
1596 static int allocate_inotify(sd_journal *j) {
1597         assert(j);
1598
1599         if (j->inotify_fd < 0) {
1600                 j->inotify_fd = inotify_init1(IN_NONBLOCK|IN_CLOEXEC);
1601                 if (j->inotify_fd < 0)
1602                         return -errno;
1603         }
1604
1605         if (!j->directories_by_wd) {
1606                 j->directories_by_wd = hashmap_new(NULL);
1607                 if (!j->directories_by_wd)
1608                         return -ENOMEM;
1609         }
1610
1611         return 0;
1612 }
1613
1614 static sd_journal *journal_new(int flags, const char *path) {
1615         sd_journal *j;
1616
1617         j = new0(sd_journal, 1);
1618         if (!j)
1619                 return NULL;
1620
1621         j->original_pid = getpid();
1622         j->inotify_fd = -1;
1623         j->flags = flags;
1624         j->data_threshold = DEFAULT_DATA_THRESHOLD;
1625
1626         if (path) {
1627                 j->path = strdup(path);
1628                 if (!j->path)
1629                         goto fail;
1630         }
1631
1632         j->files = ordered_hashmap_new(&string_hash_ops);
1633         j->directories_by_path = hashmap_new(&string_hash_ops);
1634         j->mmap = mmap_cache_new();
1635         if (!j->files || !j->directories_by_path || !j->mmap)
1636                 goto fail;
1637
1638         return j;
1639
1640 fail:
1641         sd_journal_close(j);
1642         return NULL;
1643 }
1644
1645 _public_ int sd_journal_open(sd_journal **ret, int flags) {
1646         sd_journal *j;
1647         int r;
1648
1649         assert_return(ret, -EINVAL);
1650         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_RUNTIME_ONLY|SD_JOURNAL_SYSTEM|SD_JOURNAL_CURRENT_USER)) == 0, -EINVAL);
1651
1652         j = journal_new(flags, NULL);
1653         if (!j)
1654                 return -ENOMEM;
1655
1656         r = add_search_paths(j);
1657         if (r < 0)
1658                 goto fail;
1659
1660         *ret = j;
1661         return 0;
1662
1663 fail:
1664         sd_journal_close(j);
1665
1666         return r;
1667 }
1668
1669 _public_ int sd_journal_open_container(sd_journal **ret, const char *machine, int flags) {
1670         _cleanup_free_ char *root = NULL, *class = NULL;
1671         sd_journal *j;
1672         char *p;
1673         int r;
1674
1675         assert_return(machine, -EINVAL);
1676         assert_return(ret, -EINVAL);
1677         assert_return((flags & ~(SD_JOURNAL_LOCAL_ONLY|SD_JOURNAL_SYSTEM)) == 0, -EINVAL);
1678         assert_return(machine_name_is_valid(machine), -EINVAL);
1679
1680         p = strjoina("/run/systemd/machines/", machine);
1681         r = parse_env_file(p, NEWLINE, "ROOT", &root, "CLASS", &class, NULL);
1682         if (r == -ENOENT)
1683                 return -EHOSTDOWN;
1684         if (r < 0)
1685                 return r;
1686         if (!root)
1687                 return -ENODATA;
1688
1689         if (!streq_ptr(class, "container"))
1690                 return -EIO;
1691
1692         j = journal_new(flags, NULL);
1693         if (!j)
1694                 return -ENOMEM;
1695
1696         j->prefix = root;
1697         root = NULL;
1698
1699         r = add_search_paths(j);
1700         if (r < 0)
1701                 goto fail;
1702
1703         *ret = j;
1704         return 0;
1705
1706 fail:
1707         sd_journal_close(j);
1708         return r;
1709 }
1710
1711 _public_ int sd_journal_open_directory(sd_journal **ret, const char *path, int flags) {
1712         sd_journal *j;
1713         int r;
1714
1715         assert_return(ret, -EINVAL);
1716         assert_return(path, -EINVAL);
1717         assert_return(flags == 0, -EINVAL);
1718
1719         j = journal_new(flags, path);
1720         if (!j)
1721                 return -ENOMEM;
1722
1723         r = add_root_directory(j, path, false);
1724         if (r < 0)
1725                 goto fail;
1726
1727         *ret = j;
1728         return 0;
1729
1730 fail:
1731         sd_journal_close(j);
1732
1733         return r;
1734 }
1735
1736 _public_ int sd_journal_open_files(sd_journal **ret, const char **paths, int flags) {
1737         sd_journal *j;
1738         const char **path;
1739         int r;
1740
1741         assert_return(ret, -EINVAL);
1742         assert_return(flags == 0, -EINVAL);
1743
1744         j = journal_new(flags, NULL);
1745         if (!j)
1746                 return -ENOMEM;
1747
1748         STRV_FOREACH(path, paths) {
1749                 r = add_any_file(j, *path);
1750                 if (r < 0)
1751                         goto fail;
1752         }
1753
1754         j->no_new_files = true;
1755
1756         *ret = j;
1757         return 0;
1758
1759 fail:
1760         sd_journal_close(j);
1761
1762         return r;
1763 }
1764
1765 _public_ void sd_journal_close(sd_journal *j) {
1766         Directory *d;
1767         JournalFile *f;
1768         char *p;
1769
1770         if (!j)
1771                 return;
1772
1773         sd_journal_flush_matches(j);
1774
1775         while ((f = ordered_hashmap_steal_first(j->files)))
1776                 journal_file_close(f);
1777
1778         ordered_hashmap_free(j->files);
1779
1780         while ((d = hashmap_first(j->directories_by_path)))
1781                 remove_directory(j, d);
1782
1783         while ((d = hashmap_first(j->directories_by_wd)))
1784                 remove_directory(j, d);
1785
1786         hashmap_free(j->directories_by_path);
1787         hashmap_free(j->directories_by_wd);
1788
1789         safe_close(j->inotify_fd);
1790
1791         if (j->mmap) {
1792                 log_debug("mmap cache statistics: %u hit, %u miss", mmap_cache_get_hit(j->mmap), mmap_cache_get_missed(j->mmap));
1793                 mmap_cache_unref(j->mmap);
1794         }
1795
1796         while ((p = hashmap_steal_first(j->errors)))
1797                 free(p);
1798         hashmap_free(j->errors);
1799
1800         free(j->path);
1801         free(j->prefix);
1802         free(j->unique_field);
1803         free(j);
1804 }
1805
1806 _public_ int sd_journal_get_realtime_usec(sd_journal *j, uint64_t *ret) {
1807         Object *o;
1808         JournalFile *f;
1809         int r;
1810
1811         assert_return(j, -EINVAL);
1812         assert_return(!journal_pid_changed(j), -ECHILD);
1813         assert_return(ret, -EINVAL);
1814
1815         f = j->current_file;
1816         if (!f)
1817                 return -EADDRNOTAVAIL;
1818
1819         if (f->current_offset <= 0)
1820                 return -EADDRNOTAVAIL;
1821
1822         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1823         if (r < 0)
1824                 return r;
1825
1826         *ret = le64toh(o->entry.realtime);
1827         return 0;
1828 }
1829
1830 _public_ int sd_journal_get_monotonic_usec(sd_journal *j, uint64_t *ret, sd_id128_t *ret_boot_id) {
1831         Object *o;
1832         JournalFile *f;
1833         int r;
1834         sd_id128_t id;
1835
1836         assert_return(j, -EINVAL);
1837         assert_return(!journal_pid_changed(j), -ECHILD);
1838
1839         f = j->current_file;
1840         if (!f)
1841                 return -EADDRNOTAVAIL;
1842
1843         if (f->current_offset <= 0)
1844                 return -EADDRNOTAVAIL;
1845
1846         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1847         if (r < 0)
1848                 return r;
1849
1850         if (ret_boot_id)
1851                 *ret_boot_id = o->entry.boot_id;
1852         else {
1853                 r = sd_id128_get_boot(&id);
1854                 if (r < 0)
1855                         return r;
1856
1857                 if (!sd_id128_equal(id, o->entry.boot_id))
1858                         return -ESTALE;
1859         }
1860
1861         if (ret)
1862                 *ret = le64toh(o->entry.monotonic);
1863
1864         return 0;
1865 }
1866
1867 static bool field_is_valid(const char *field) {
1868         const char *p;
1869
1870         assert(field);
1871
1872         if (isempty(field))
1873                 return false;
1874
1875         if (startswith(field, "__"))
1876                 return false;
1877
1878         for (p = field; *p; p++) {
1879
1880                 if (*p == '_')
1881                         continue;
1882
1883                 if (*p >= 'A' && *p <= 'Z')
1884                         continue;
1885
1886                 if (*p >= '0' && *p <= '9')
1887                         continue;
1888
1889                 return false;
1890         }
1891
1892         return true;
1893 }
1894
1895 _public_ int sd_journal_get_data(sd_journal *j, const char *field, const void **data, size_t *size) {
1896         JournalFile *f;
1897         uint64_t i, n;
1898         size_t field_length;
1899         int r;
1900         Object *o;
1901
1902         assert_return(j, -EINVAL);
1903         assert_return(!journal_pid_changed(j), -ECHILD);
1904         assert_return(field, -EINVAL);
1905         assert_return(data, -EINVAL);
1906         assert_return(size, -EINVAL);
1907         assert_return(field_is_valid(field), -EINVAL);
1908
1909         f = j->current_file;
1910         if (!f)
1911                 return -EADDRNOTAVAIL;
1912
1913         if (f->current_offset <= 0)
1914                 return -EADDRNOTAVAIL;
1915
1916         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1917         if (r < 0)
1918                 return r;
1919
1920         field_length = strlen(field);
1921
1922         n = journal_file_entry_n_items(o);
1923         for (i = 0; i < n; i++) {
1924                 uint64_t p, l;
1925                 le64_t le_hash;
1926                 size_t t;
1927                 int compression;
1928
1929                 p = le64toh(o->entry.items[i].object_offset);
1930                 le_hash = o->entry.items[i].hash;
1931                 r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
1932                 if (r < 0)
1933                         return r;
1934
1935                 if (le_hash != o->data.hash)
1936                         return -EBADMSG;
1937
1938                 l = le64toh(o->object.size) - offsetof(Object, data.payload);
1939
1940                 compression = o->object.flags & OBJECT_COMPRESSION_MASK;
1941                 if (compression) {
1942 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
1943                         r = decompress_startswith(compression,
1944                                                   o->data.payload, l,
1945                                                   &f->compress_buffer, &f->compress_buffer_size,
1946                                                   field, field_length, '=');
1947                         if (r < 0)
1948                                 log_debug_errno(r, "Cannot decompress %s object of length %zu at offset "OFSfmt": %m",
1949                                                 object_compressed_to_string(compression), l, p);
1950                         else if (r > 0) {
1951
1952                                 size_t rsize;
1953
1954                                 r = decompress_blob(compression,
1955                                                     o->data.payload, l,
1956                                                     &f->compress_buffer, &f->compress_buffer_size, &rsize,
1957                                                     j->data_threshold);
1958                                 if (r < 0)
1959                                         return r;
1960
1961                                 *data = f->compress_buffer;
1962                                 *size = (size_t) rsize;
1963
1964                                 return 0;
1965                         }
1966 #else
1967                         return -EPROTONOSUPPORT;
1968 #endif
1969                 } else if (l >= field_length+1 &&
1970                            memcmp(o->data.payload, field, field_length) == 0 &&
1971                            o->data.payload[field_length] == '=') {
1972
1973                         t = (size_t) l;
1974
1975                         if ((uint64_t) t != l)
1976                                 return -E2BIG;
1977
1978                         *data = o->data.payload;
1979                         *size = t;
1980
1981                         return 0;
1982                 }
1983
1984                 r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
1985                 if (r < 0)
1986                         return r;
1987         }
1988
1989         return -ENOENT;
1990 }
1991
1992 static int return_data(sd_journal *j, JournalFile *f, Object *o, const void **data, size_t *size) {
1993         size_t t;
1994         uint64_t l;
1995         int compression;
1996
1997         l = le64toh(o->object.size) - offsetof(Object, data.payload);
1998         t = (size_t) l;
1999
2000         /* We can't read objects larger than 4G on a 32bit machine */
2001         if ((uint64_t) t != l)
2002                 return -E2BIG;
2003
2004         compression = o->object.flags & OBJECT_COMPRESSION_MASK;
2005         if (compression) {
2006 #if defined(HAVE_XZ) || defined(HAVE_LZ4)
2007                 size_t rsize;
2008                 int r;
2009
2010                 r = decompress_blob(compression,
2011                                     o->data.payload, l, &f->compress_buffer,
2012                                     &f->compress_buffer_size, &rsize, j->data_threshold);
2013                 if (r < 0)
2014                         return r;
2015
2016                 *data = f->compress_buffer;
2017                 *size = (size_t) rsize;
2018 #else
2019                 return -EPROTONOSUPPORT;
2020 #endif
2021         } else {
2022                 *data = o->data.payload;
2023                 *size = t;
2024         }
2025
2026         return 0;
2027 }
2028
2029 _public_ int sd_journal_enumerate_data(sd_journal *j, const void **data, size_t *size) {
2030         JournalFile *f;
2031         uint64_t p, n;
2032         le64_t le_hash;
2033         int r;
2034         Object *o;
2035
2036         assert_return(j, -EINVAL);
2037         assert_return(!journal_pid_changed(j), -ECHILD);
2038         assert_return(data, -EINVAL);
2039         assert_return(size, -EINVAL);
2040
2041         f = j->current_file;
2042         if (!f)
2043                 return -EADDRNOTAVAIL;
2044
2045         if (f->current_offset <= 0)
2046                 return -EADDRNOTAVAIL;
2047
2048         r = journal_file_move_to_object(f, OBJECT_ENTRY, f->current_offset, &o);
2049         if (r < 0)
2050                 return r;
2051
2052         n = journal_file_entry_n_items(o);
2053         if (j->current_field >= n)
2054                 return 0;
2055
2056         p = le64toh(o->entry.items[j->current_field].object_offset);
2057         le_hash = o->entry.items[j->current_field].hash;
2058         r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
2059         if (r < 0)
2060                 return r;
2061
2062         if (le_hash != o->data.hash)
2063                 return -EBADMSG;
2064
2065         r = return_data(j, f, o, data, size);
2066         if (r < 0)
2067                 return r;
2068
2069         j->current_field ++;
2070
2071         return 1;
2072 }
2073
2074 _public_ void sd_journal_restart_data(sd_journal *j) {
2075         if (!j)
2076                 return;
2077
2078         j->current_field = 0;
2079 }
2080
2081 _public_ int sd_journal_get_fd(sd_journal *j) {
2082         int r;
2083
2084         assert_return(j, -EINVAL);
2085         assert_return(!journal_pid_changed(j), -ECHILD);
2086
2087         if (j->inotify_fd >= 0)
2088                 return j->inotify_fd;
2089
2090         r = allocate_inotify(j);
2091         if (r < 0)
2092                 return r;
2093
2094         /* Iterate through all dirs again, to add them to the
2095          * inotify */
2096         if (j->no_new_files)
2097                 r = add_current_paths(j);
2098         else if (j->path)
2099                 r = add_root_directory(j, j->path, true);
2100         else
2101                 r = add_search_paths(j);
2102         if (r < 0)
2103                 return r;
2104
2105         return j->inotify_fd;
2106 }
2107
2108 _public_ int sd_journal_get_events(sd_journal *j) {
2109         int fd;
2110
2111         assert_return(j, -EINVAL);
2112         assert_return(!journal_pid_changed(j), -ECHILD);
2113
2114         fd = sd_journal_get_fd(j);
2115         if (fd < 0)
2116                 return fd;
2117
2118         return POLLIN;
2119 }
2120
2121 _public_ int sd_journal_get_timeout(sd_journal *j, uint64_t *timeout_usec) {
2122         int fd;
2123
2124         assert_return(j, -EINVAL);
2125         assert_return(!journal_pid_changed(j), -ECHILD);
2126         assert_return(timeout_usec, -EINVAL);
2127
2128         fd = sd_journal_get_fd(j);
2129         if (fd < 0)
2130                 return fd;
2131
2132         if (!j->on_network) {
2133                 *timeout_usec = (uint64_t) -1;
2134                 return 0;
2135         }
2136
2137         /* If we are on the network we need to regularly check for
2138          * changes manually */
2139
2140         *timeout_usec = j->last_process_usec + JOURNAL_FILES_RECHECK_USEC;
2141         return 1;
2142 }
2143
2144 static void process_inotify_event(sd_journal *j, struct inotify_event *e) {
2145         Directory *d;
2146
2147         assert(j);
2148         assert(e);
2149
2150         /* Is this a subdirectory we watch? */
2151         d = hashmap_get(j->directories_by_wd, INT_TO_PTR(e->wd));
2152         if (d) {
2153                 sd_id128_t id;
2154
2155                 if (!(e->mask & IN_ISDIR) && e->len > 0 &&
2156                     (endswith(e->name, ".journal") ||
2157                      endswith(e->name, ".journal~"))) {
2158
2159                         /* Event for a journal file */
2160
2161                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB))
2162                                 (void) add_file(j, d->path, e->name);
2163                         else if (e->mask & (IN_DELETE|IN_MOVED_FROM|IN_UNMOUNT))
2164                                 remove_file(j, d->path, e->name);
2165
2166                 } else if (!d->is_root && e->len == 0) {
2167
2168                         /* Event for a subdirectory */
2169
2170                         if (e->mask & (IN_DELETE_SELF|IN_MOVE_SELF|IN_UNMOUNT))
2171                                 remove_directory(j, d);
2172
2173                 } else if (d->is_root && (e->mask & IN_ISDIR) && e->len > 0 && sd_id128_from_string(e->name, &id) >= 0) {
2174
2175                         /* Event for root directory */
2176
2177                         if (e->mask & (IN_CREATE|IN_MOVED_TO|IN_MODIFY|IN_ATTRIB))
2178                                 (void) add_directory(j, d->path, e->name);
2179                 }
2180
2181                 return;
2182         }
2183
2184         if (e->mask & IN_IGNORED)
2185                 return;
2186
2187         log_debug("Unknown inotify event.");
2188 }
2189
2190 static int determine_change(sd_journal *j) {
2191         bool b;
2192
2193         assert(j);
2194
2195         b = j->current_invalidate_counter != j->last_invalidate_counter;
2196         j->last_invalidate_counter = j->current_invalidate_counter;
2197
2198         return b ? SD_JOURNAL_INVALIDATE : SD_JOURNAL_APPEND;
2199 }
2200
2201 _public_ int sd_journal_process(sd_journal *j) {
2202         bool got_something = false;
2203
2204         assert_return(j, -EINVAL);
2205         assert_return(!journal_pid_changed(j), -ECHILD);
2206
2207         j->last_process_usec = now(CLOCK_MONOTONIC);
2208
2209         for (;;) {
2210                 union inotify_event_buffer buffer;
2211                 struct inotify_event *e;
2212                 ssize_t l;
2213
2214                 l = read(j->inotify_fd, &buffer, sizeof(buffer));
2215                 if (l < 0) {
2216                         if (errno == EAGAIN || errno == EINTR)
2217                                 return got_something ? determine_change(j) : SD_JOURNAL_NOP;
2218
2219                         return -errno;
2220                 }
2221
2222                 got_something = true;
2223
2224                 FOREACH_INOTIFY_EVENT(e, buffer, l)
2225                         process_inotify_event(j, e);
2226         }
2227 }
2228
2229 _public_ int sd_journal_wait(sd_journal *j, uint64_t timeout_usec) {
2230         int r;
2231         uint64_t t;
2232
2233         assert_return(j, -EINVAL);
2234         assert_return(!journal_pid_changed(j), -ECHILD);
2235
2236         if (j->inotify_fd < 0) {
2237
2238                 /* This is the first invocation, hence create the
2239                  * inotify watch */
2240                 r = sd_journal_get_fd(j);
2241                 if (r < 0)
2242                         return r;
2243
2244                 /* The journal might have changed since the context
2245                  * object was created and we weren't watching before,
2246                  * hence don't wait for anything, and return
2247                  * immediately. */
2248                 return determine_change(j);
2249         }
2250
2251         r = sd_journal_get_timeout(j, &t);
2252         if (r < 0)
2253                 return r;
2254
2255         if (t != (uint64_t) -1) {
2256                 usec_t n;
2257
2258                 n = now(CLOCK_MONOTONIC);
2259                 t = t > n ? t - n : 0;
2260
2261                 if (timeout_usec == (uint64_t) -1 || timeout_usec > t)
2262                         timeout_usec = t;
2263         }
2264
2265         do {
2266                 r = fd_wait_for_event(j->inotify_fd, POLLIN, timeout_usec);
2267         } while (r == -EINTR);
2268
2269         if (r < 0)
2270                 return r;
2271
2272         return sd_journal_process(j);
2273 }
2274
2275 _public_ int sd_journal_get_cutoff_realtime_usec(sd_journal *j, uint64_t *from, uint64_t *to) {
2276         Iterator i;
2277         JournalFile *f;
2278         bool first = true;
2279         uint64_t fmin = 0, tmax = 0;
2280         int r;
2281
2282         assert_return(j, -EINVAL);
2283         assert_return(!journal_pid_changed(j), -ECHILD);
2284         assert_return(from || to, -EINVAL);
2285         assert_return(from != to, -EINVAL);
2286
2287         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2288                 usec_t fr, t;
2289
2290                 r = journal_file_get_cutoff_realtime_usec(f, &fr, &t);
2291                 if (r == -ENOENT)
2292                         continue;
2293                 if (r < 0)
2294                         return r;
2295                 if (r == 0)
2296                         continue;
2297
2298                 if (first) {
2299                         fmin = fr;
2300                         tmax = t;
2301                         first = false;
2302                 } else {
2303                         fmin = MIN(fr, fmin);
2304                         tmax = MAX(t, tmax);
2305                 }
2306         }
2307
2308         if (from)
2309                 *from = fmin;
2310         if (to)
2311                 *to = tmax;
2312
2313         return first ? 0 : 1;
2314 }
2315
2316 _public_ int sd_journal_get_cutoff_monotonic_usec(sd_journal *j, sd_id128_t boot_id, uint64_t *from, uint64_t *to) {
2317         Iterator i;
2318         JournalFile *f;
2319         bool found = false;
2320         int r;
2321
2322         assert_return(j, -EINVAL);
2323         assert_return(!journal_pid_changed(j), -ECHILD);
2324         assert_return(from || to, -EINVAL);
2325         assert_return(from != to, -EINVAL);
2326
2327         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2328                 usec_t fr, t;
2329
2330                 r = journal_file_get_cutoff_monotonic_usec(f, boot_id, &fr, &t);
2331                 if (r == -ENOENT)
2332                         continue;
2333                 if (r < 0)
2334                         return r;
2335                 if (r == 0)
2336                         continue;
2337
2338                 if (found) {
2339                         if (from)
2340                                 *from = MIN(fr, *from);
2341                         if (to)
2342                                 *to = MAX(t, *to);
2343                 } else {
2344                         if (from)
2345                                 *from = fr;
2346                         if (to)
2347                                 *to = t;
2348                         found = true;
2349                 }
2350         }
2351
2352         return found;
2353 }
2354
2355 void journal_print_header(sd_journal *j) {
2356         Iterator i;
2357         JournalFile *f;
2358         bool newline = false;
2359
2360         assert(j);
2361
2362         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2363                 if (newline)
2364                         putchar('\n');
2365                 else
2366                         newline = true;
2367
2368                 journal_file_print_header(f);
2369         }
2370 }
2371
2372 _public_ int sd_journal_get_usage(sd_journal *j, uint64_t *bytes) {
2373         Iterator i;
2374         JournalFile *f;
2375         uint64_t sum = 0;
2376
2377         assert_return(j, -EINVAL);
2378         assert_return(!journal_pid_changed(j), -ECHILD);
2379         assert_return(bytes, -EINVAL);
2380
2381         ORDERED_HASHMAP_FOREACH(f, j->files, i) {
2382                 struct stat st;
2383
2384                 if (fstat(f->fd, &st) < 0)
2385                         return -errno;
2386
2387                 sum += (uint64_t) st.st_blocks * 512ULL;
2388         }
2389
2390         *bytes = sum;
2391         return 0;
2392 }
2393
2394 _public_ int sd_journal_query_unique(sd_journal *j, const char *field) {
2395         char *f;
2396
2397         assert_return(j, -EINVAL);
2398         assert_return(!journal_pid_changed(j), -ECHILD);
2399         assert_return(!isempty(field), -EINVAL);
2400         assert_return(field_is_valid(field), -EINVAL);
2401
2402         f = strdup(field);
2403         if (!f)
2404                 return -ENOMEM;
2405
2406         free(j->unique_field);
2407         j->unique_field = f;
2408         j->unique_file = NULL;
2409         j->unique_offset = 0;
2410         j->unique_file_lost = false;
2411
2412         return 0;
2413 }
2414
2415 _public_ int sd_journal_enumerate_unique(sd_journal *j, const void **data, size_t *l) {
2416         size_t k;
2417
2418         assert_return(j, -EINVAL);
2419         assert_return(!journal_pid_changed(j), -ECHILD);
2420         assert_return(data, -EINVAL);
2421         assert_return(l, -EINVAL);
2422         assert_return(j->unique_field, -EINVAL);
2423
2424         k = strlen(j->unique_field);
2425
2426         if (!j->unique_file) {
2427                 if (j->unique_file_lost)
2428                         return 0;
2429
2430                 j->unique_file = ordered_hashmap_first(j->files);
2431                 if (!j->unique_file)
2432                         return 0;
2433
2434                 j->unique_offset = 0;
2435         }
2436
2437         for (;;) {
2438                 JournalFile *of;
2439                 Iterator i;
2440                 Object *o;
2441                 const void *odata;
2442                 size_t ol;
2443                 bool found;
2444                 int r;
2445
2446                 /* Proceed to next data object in the field's linked list */
2447                 if (j->unique_offset == 0) {
2448                         r = journal_file_find_field_object(j->unique_file, j->unique_field, k, &o, NULL);
2449                         if (r < 0)
2450                                 return r;
2451
2452                         j->unique_offset = r > 0 ? le64toh(o->field.head_data_offset) : 0;
2453                 } else {
2454                         r = journal_file_move_to_object(j->unique_file, OBJECT_DATA, j->unique_offset, &o);
2455                         if (r < 0)
2456                                 return r;
2457
2458                         j->unique_offset = le64toh(o->data.next_field_offset);
2459                 }
2460
2461                 /* We reached the end of the list? Then start again, with the next file */
2462                 if (j->unique_offset == 0) {
2463                         j->unique_file = ordered_hashmap_next(j->files, j->unique_file->path);
2464                         if (!j->unique_file)
2465                                 return 0;
2466
2467                         continue;
2468                 }
2469
2470                 /* We do not use OBJECT_DATA context here, but OBJECT_UNUSED
2471                  * instead, so that we can look at this data object at the same
2472                  * time as one on another file */
2473                 r = journal_file_move_to_object(j->unique_file, OBJECT_UNUSED, j->unique_offset, &o);
2474                 if (r < 0)
2475                         return r;
2476
2477                 /* Let's do the type check by hand, since we used 0 context above. */
2478                 if (o->object.type != OBJECT_DATA) {
2479                         log_debug("%s:offset " OFSfmt ": object has type %d, expected %d",
2480                                   j->unique_file->path, j->unique_offset,
2481                                   o->object.type, OBJECT_DATA);
2482                         return -EBADMSG;
2483                 }
2484
2485                 r = return_data(j, j->unique_file, o, &odata, &ol);
2486                 if (r < 0)
2487                         return r;
2488
2489                 /* Check if we have at least the field name and "=". */
2490                 if (ol <= k) {
2491                         log_debug("%s:offset " OFSfmt ": object has size %zu, expected at least %zu",
2492                                   j->unique_file->path, j->unique_offset,
2493                                   ol, k + 1);
2494                         return -EBADMSG;
2495                 }
2496
2497                 if (memcmp(odata, j->unique_field, k) || ((const char*) odata)[k] != '=') {
2498                         log_debug("%s:offset " OFSfmt ": object does not start with \"%s=\"",
2499                                   j->unique_file->path, j->unique_offset,
2500                                   j->unique_field);
2501                         return -EBADMSG;
2502                 }
2503
2504                 /* OK, now let's see if we already returned this data
2505                  * object by checking if it exists in the earlier
2506                  * traversed files. */
2507                 found = false;
2508                 ORDERED_HASHMAP_FOREACH(of, j->files, i) {
2509                         Object *oo;
2510                         uint64_t op;
2511
2512                         if (of == j->unique_file)
2513                                 break;
2514
2515                         /* Skip this file it didn't have any fields
2516                          * indexed */
2517                         if (JOURNAL_HEADER_CONTAINS(of->header, n_fields) &&
2518                             le64toh(of->header->n_fields) <= 0)
2519                                 continue;
2520
2521                         r = journal_file_find_data_object_with_hash(of, odata, ol, le64toh(o->data.hash), &oo, &op);
2522                         if (r < 0)
2523                                 return r;
2524
2525                         if (r > 0)
2526                                 found = true;
2527                 }
2528
2529                 if (found)
2530                         continue;
2531
2532                 r = return_data(j, j->unique_file, o, data, l);
2533                 if (r < 0)
2534                         return r;
2535
2536                 return 1;
2537         }
2538 }
2539
2540 _public_ void sd_journal_restart_unique(sd_journal *j) {
2541         if (!j)
2542                 return;
2543
2544         j->unique_file = NULL;
2545         j->unique_offset = 0;
2546         j->unique_file_lost = false;
2547 }
2548
2549 _public_ int sd_journal_reliable_fd(sd_journal *j) {
2550         assert_return(j, -EINVAL);
2551         assert_return(!journal_pid_changed(j), -ECHILD);
2552
2553         return !j->on_network;
2554 }
2555
2556 static char *lookup_field(const char *field, void *userdata) {
2557         sd_journal *j = userdata;
2558         const void *data;
2559         size_t size, d;
2560         int r;
2561
2562         assert(field);
2563         assert(j);
2564
2565         r = sd_journal_get_data(j, field, &data, &size);
2566         if (r < 0 ||
2567             size > REPLACE_VAR_MAX)
2568                 return strdup(field);
2569
2570         d = strlen(field) + 1;
2571
2572         return strndup((const char*) data + d, size - d);
2573 }
2574
2575 _public_ int sd_journal_get_catalog(sd_journal *j, char **ret) {
2576         const void *data;
2577         size_t size;
2578         sd_id128_t id;
2579         _cleanup_free_ char *text = NULL, *cid = NULL;
2580         char *t;
2581         int r;
2582
2583         assert_return(j, -EINVAL);
2584         assert_return(!journal_pid_changed(j), -ECHILD);
2585         assert_return(ret, -EINVAL);
2586
2587         r = sd_journal_get_data(j, "MESSAGE_ID", &data, &size);
2588         if (r < 0)
2589                 return r;
2590
2591         cid = strndup((const char*) data + 11, size - 11);
2592         if (!cid)
2593                 return -ENOMEM;
2594
2595         r = sd_id128_from_string(cid, &id);
2596         if (r < 0)
2597                 return r;
2598
2599         r = catalog_get(CATALOG_DATABASE, id, &text);
2600         if (r < 0)
2601                 return r;
2602
2603         t = replace_var(text, lookup_field, j);
2604         if (!t)
2605                 return -ENOMEM;
2606
2607         *ret = t;
2608         return 0;
2609 }
2610
2611 _public_ int sd_journal_get_catalog_for_message_id(sd_id128_t id, char **ret) {
2612         assert_return(ret, -EINVAL);
2613
2614         return catalog_get(CATALOG_DATABASE, id, ret);
2615 }
2616
2617 _public_ int sd_journal_set_data_threshold(sd_journal *j, size_t sz) {
2618         assert_return(j, -EINVAL);
2619         assert_return(!journal_pid_changed(j), -ECHILD);
2620
2621         j->data_threshold = sz;
2622         return 0;
2623 }
2624
2625 _public_ int sd_journal_get_data_threshold(sd_journal *j, size_t *sz) {
2626         assert_return(j, -EINVAL);
2627         assert_return(!journal_pid_changed(j), -ECHILD);
2628         assert_return(sz, -EINVAL);
2629
2630         *sz = j->data_threshold;
2631         return 0;
2632 }