journald: automatically rotate journal files when the clock jumps backwards

As soon as we notice that the clock jumps backwards, rotate journal files. This
is beneficial, as this makes sure that the entries in journal files remain
strictly ordered internally, and thus the bisection algorithm applied on it is
not confused.

This should help avoiding borked wallclock-based bisection on journal files as
witnessed in #4278.

diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c
index 7227c80c866fadd572e5506c3e636479bf2043b6..28aea35d18f2f7a0c775e378919297260ac96aab 100644 (file)
--- a/src/journal/journald-server.c
+++ b/src/journal/journald-server.c
@@ -627,8 +627,8 @@ static bool shall_try_append_again(JournalFile *f, int r) {
 }
 
 static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned n, int priority) {
+        bool vacuumed = false, rotate = false;
         struct dual_timestamp ts;
-        bool vacuumed = false;
         JournalFile *f;
         int r;
 
@@ -642,12 +642,27 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned
         assert_se(sd_event_now(s->event, CLOCK_REALTIME, &ts.realtime) >= 0);
         assert_se(sd_event_now(s->event, CLOCK_MONOTONIC, &ts.monotonic) >= 0);
 
-        f = find_journal(s, uid);
-        if (!f)
-                return;
+        if (ts.realtime < s->last_realtime_clock) {
+                /* When the time jumps backwards, let's immediately rotate. Of course, this should not happen during
+                 * regular operation. However, when it does happen, then we should make sure that we start fresh files
+                 * to ensure that the entries in the journal files are strictly ordered by time, in order to ensure
+                 * bisection works correctly. */
 
-        if (journal_file_rotate_suggested(f, s->max_file_usec)) {
-                log_debug("%s: Journal header limits reached or header out-of-date, rotating.", f->path);
+                log_debug("Time jumped backwards, rotating.");
+                rotate = true;
+        } else {
+
+                f = find_journal(s, uid);
+                if (!f)
+                        return;
+
+                if (journal_file_rotate_suggested(f, s->max_file_usec)) {
+                        log_debug("%s: Journal header limits reached or header out-of-date, rotating.", f->path);
+                        rotate = true;
+                }
+        }
+
+        if (rotate) {
                 server_rotate(s);
                 server_vacuum(s, false, false);
                 vacuumed = true;
@@ -657,6 +672,8 @@ static void write_to_journal(Server *s, uid_t uid, struct iovec *iovec, unsigned
                         return;
         }
 
+        s->last_realtime_clock = ts.realtime;
+
         r = journal_file_append_entry(f, &ts, iovec, n, &s->seqnum, NULL, NULL);
         if (r >= 0) {
                 server_schedule_sync(s, priority);

diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h
index dfb572479421c38c354650e7832d43492ab77c93..cc68a0a690fb8de7e77a6653af7a79b4f008996a 100644 (file)
--- a/src/journal/journald-server.h
+++ b/src/journal/journald-server.h
@@ -149,6 +149,8 @@ struct Server {
         char *cgroup_root;
 
         usec_t watchdog_usec;
+
+        usec_t last_realtime_clock;
 };
 
 #define SERVER_MACHINE_ID(s) ((s)->machine_id_field + strlen("_MACHINE_ID="))