journal: defer journal closes on rotate

When we rotate journals, we must set offline and close the current one,
but don't generally need to wait for this to complete.

Instead, we'll initiate an asynchronous offline via
journal_file_set_offline(oldfile, false), and add the file to a
per-server set of deferred closes to be closed later when they
won't block.

There's one complication however; journal_file_open() via
journal_file_verify_header() assumes that any writable journal in the
online state is the product of an unclean shutdown or other form of
corruption.

Thus there's a need for journal_file_open() to be aware of deferred
closes and synchronize with their completion when opening preexisting
journals for writing.  To facilitate this the deferred closes set is
supplied to the journal_file_open() function where the deferred closes
may be closed synchronously before verifying the header in such
circumstances.

diff --git a/src/journal-remote/journal-remote-write.c b/src/journal-remote/journal-remote-write.c
index 5fab74e5ccda1594fcf98eb8f2988efaa0a1ebcc..7bba52566ea58b94c0e9ec4404af7978a992c083 100644 (file)
--- a/src/journal-remote/journal-remote-write.c
+++ b/src/journal-remote/journal-remote-write.c
@@ -54,7 +54,7 @@ void iovw_rebase(struct iovec_wrapper *iovw, char *old, char *new) {
  **********************************************************************/
 
 static int do_rotate(JournalFile **f, bool compress, bool seal) {
-        int r = journal_file_rotate(f, compress, seal);
+        int r = journal_file_rotate(f, compress, seal, NULL);
         if (r < 0) {
                 if (*f)
                         log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);

diff --git a/src/journal-remote/journal-remote.c b/src/journal-remote/journal-remote.c
index 3ce6fe27b3354a3362a7172ff99a4d5f80d477ad..44f9a9b44f49e6c69dde926b5ac84b0a8ad68644 100644 (file)
--- a/src/journal-remote/journal-remote.c
+++ b/src/journal-remote/journal-remote.c
@@ -203,7 +203,7 @@ static int open_output(Writer *w, const char* host) {
                                        O_RDWR|O_CREAT, 0640,
                                        arg_compress, arg_seal,
                                        &w->metrics,
-                                       w->mmap,
+                                       w->mmap, NULL,
                                        NULL, &w->journal);
         if (r < 0)
                 log_error_errno(r, "Failed to open output journal %s: %m",

diff --git a/src/journal/journal-file.c b/src/journal/journal-file.c
index 96be339d5ba9d727e7ce4fb29fcc8f964afd0ec1..f5e2952c997a7ab4f2143fc266e2a6291e71ef46 100644 (file)
--- a/src/journal/journal-file.c
+++ b/src/journal/journal-file.c
@@ -39,6 +39,7 @@
 #include "parse-util.h"
 #include "random-util.h"
 #include "sd-event.h"
+#include "set.h"
 #include "string-util.h"
 #include "xattr-util.h"
 
@@ -310,6 +311,18 @@ static int journal_file_set_online(JournalFile *f) {
         }
 }
 
+bool journal_file_is_offlining(JournalFile *f) {
+        assert(f);
+
+        __sync_synchronize();
+
+        if (f->offline_state == OFFLINE_DONE ||
+            f->offline_state == OFFLINE_JOINED)
+                return false;
+
+        return true;
+}
+
 JournalFile* journal_file_close(JournalFile *f) {
         assert(f);
 
@@ -374,6 +387,15 @@ JournalFile* journal_file_close(JournalFile *f) {
         return NULL;
 }
 
+void journal_file_close_set(Set *s) {
+        JournalFile *f;
+
+        assert(s);
+
+        while ((f = set_steal_first(s)))
+                (void) journal_file_close(f);
+}
+
 static int journal_file_init_header(JournalFile *f, JournalFile *template) {
         Header h = {};
         ssize_t k;
@@ -2881,6 +2903,7 @@ int journal_file_open(
                 bool seal,
                 JournalMetrics *metrics,
                 MMapCache *mmap_cache,
+                Set *deferred_closes,
                 JournalFile *template,
                 JournalFile **ret) {
 
@@ -3000,6 +3023,9 @@ int journal_file_open(
         f->header = h;
 
         if (!newly_created) {
+                if (deferred_closes)
+                        journal_file_close_set(deferred_closes);
+
                 r = journal_file_verify_header(f);
                 if (r < 0)
                         goto fail;
@@ -3074,7 +3100,7 @@ fail:
         return r;
 }
 
-int journal_file_rotate(JournalFile **f, bool compress, bool seal) {
+int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes) {
         _cleanup_free_ char *p = NULL;
         size_t l;
         JournalFile *old_file, *new_file = NULL;
@@ -3114,8 +3140,13 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal) {
          * we archive them */
         old_file->defrag_on_close = true;
 
-        r = journal_file_open(old_file->path, old_file->flags, old_file->mode, compress, seal, NULL, old_file->mmap, old_file, &new_file);
-        journal_file_close(old_file);
+        r = journal_file_open(old_file->path, old_file->flags, old_file->mode, compress, seal, NULL, old_file->mmap, deferred_closes, old_file, &new_file);
+
+        if (deferred_closes &&
+            set_put(deferred_closes, old_file) >= 0)
+                (void) journal_file_set_offline(old_file, false);
+        else
+                (void) journal_file_close(old_file);
 
         *f = new_file;
         return r;
@@ -3129,6 +3160,7 @@ int journal_file_open_reliably(
                 bool seal,
                 JournalMetrics *metrics,
                 MMapCache *mmap_cache,
+                Set *deferred_closes,
                 JournalFile *template,
                 JournalFile **ret) {
 
@@ -3136,7 +3168,7 @@ int journal_file_open_reliably(
         size_t l;
         _cleanup_free_ char *p = NULL;
 
-        r = journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, template, ret);
+        r = journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
         if (!IN_SET(r,
                     -EBADMSG,           /* corrupted */
                     -ENODATA,           /* truncated */
@@ -3177,7 +3209,7 @@ int journal_file_open_reliably(
 
         log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname);
 
-        return journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, template, ret);
+        return journal_file_open(fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
 }
 
 int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) {

diff --git a/src/journal/journal-file.h b/src/journal/journal-file.h
index fad4f78bdcb2ee15d59ff076ba525d37fae9ae20..9ad601335928de88bbdd1f7178ed4115c2876222 100644 (file)
--- a/src/journal/journal-file.h
+++ b/src/journal/journal-file.h
@@ -149,11 +149,14 @@ int journal_file_open(
                 bool seal,
                 JournalMetrics *metrics,
                 MMapCache *mmap_cache,
+                Set *deferred_closes,
                 JournalFile *template,
                 JournalFile **ret);
 
 int journal_file_set_offline(JournalFile *f, bool wait);
+bool journal_file_is_offlining(JournalFile *f);
 JournalFile* journal_file_close(JournalFile *j);
+void journal_file_close_set(Set *s);
 
 int journal_file_open_reliably(
                 const char *fname,
@@ -163,6 +166,7 @@ int journal_file_open_reliably(
                 bool seal,
                 JournalMetrics *metrics,
                 MMapCache *mmap_cache,
+                Set *deferred_closes,
                 JournalFile *template,
                 JournalFile **ret);
 
@@ -236,7 +240,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
 void journal_file_dump(JournalFile *f);
 void journal_file_print_header(JournalFile *f);
 
-int journal_file_rotate(JournalFile **f, bool compress, bool seal);
+int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes);
 
 void journal_file_post_change(JournalFile *f);
 int journal_file_enable_post_change_timer(JournalFile *f, sd_event *e, usec_t t);

diff --git a/src/journal/journald-server.c b/src/journal/journald-server.c
index ac992a8b54bc371d0319ae6a006d01ab2a00d5b4..d5937bd013c88515b17ac1da4fc54f6f3daebb2b 100644 (file)
--- a/src/journal/journald-server.c
+++ b/src/journal/journald-server.c
@@ -251,9 +251,9 @@ static int open_journal(
         assert(ret);
 
         if (reliably)
-                r = journal_file_open_reliably(fname, flags, 0640, s->compress, seal, metrics, s->mmap, NULL, &f);
+                r = journal_file_open_reliably(fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
         else
-                r = journal_file_open(fname, flags, 0640, s->compress, seal, metrics, s->mmap, NULL, &f);
+                r = journal_file_open(fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
         if (r < 0)
                 return r;
 
@@ -333,7 +333,7 @@ static int do_rotate(
         if (!*f)
                 return -EINVAL;
 
-        r = journal_file_rotate(f, s->compress, seal);
+        r = journal_file_rotate(f, s->compress, seal, s->deferred_closes);
         if (r < 0)
                 if (*f)
                         log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);
@@ -364,6 +364,13 @@ void server_rotate(Server *s) {
                         /* Old file has been closed and deallocated */
                         ordered_hashmap_remove(s->user_journals, k);
         }
+
+        /* Perform any deferred closes which aren't still offlining. */
+        SET_FOREACH(f, s->deferred_closes, i)
+                if (!journal_file_is_offlining(f)) {
+                        (void) set_remove(s->deferred_closes, f);
+                        (void) journal_file_close(f);
+                }
 }
 
 void server_sync(Server *s) {
@@ -1765,6 +1772,10 @@ int server_init(Server *s) {
         if (!s->mmap)
                 return log_oom();
 
+        s->deferred_closes = set_new(NULL);
+        if (!s->deferred_closes)
+                return log_oom();
+
         r = sd_event_default(&s->event);
         if (r < 0)
                 return log_error_errno(r, "Failed to create event loop: %m");
@@ -1918,6 +1929,11 @@ void server_done(Server *s) {
         JournalFile *f;
         assert(s);
 
+        if (s->deferred_closes) {
+                journal_file_close_set(s->deferred_closes);
+                set_free(s->deferred_closes);
+        }
+
         while (s->stdout_streams)
                 stdout_stream_free(s->stdout_streams);
 

diff --git a/src/journal/journald-server.h b/src/journal/journald-server.h
index b9551dda1bea4b8287ca63f0868aa27927fab018..e025a4cf905a8b68ce1e2a3dde4a3d20162960a9 100644 (file)
--- a/src/journal/journald-server.h
+++ b/src/journal/journald-server.h
@@ -130,6 +130,8 @@ struct Server {
 
         MMapCache *mmap;
 
+        Set *deferred_closes;
+
         struct udev *udev;
 
         uint64_t *kernel_seqnum;

diff --git a/src/journal/sd-journal.c b/src/journal/sd-journal.c
index 9bc4215f2e307d03af87bae9ba01e710484ea799..6ff1c67f5fce1c59a296a5a27cc65a4c610e54be 100644 (file)
--- a/src/journal/sd-journal.c
+++ b/src/journal/sd-journal.c
@@ -1248,7 +1248,7 @@ static int add_any_file(sd_journal *j, const char *path) {
                 goto fail;
         }
 
-        r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, &f);
+        r = journal_file_open(path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, NULL, &f);
         if (r < 0) {
                 log_debug_errno(r, "Failed to open journal file %s: %m", path);
                 goto fail;

diff --git a/src/journal/test-journal-flush.c b/src/journal/test-journal-flush.c
index 0c35d532c6e3ba8fc37fcdfae793934169ff3d37..93dc0e0d817bf60da2619cf72ce4a19385aa1925 100644 (file)
--- a/src/journal/test-journal-flush.c
+++ b/src/journal/test-journal-flush.c
@@ -38,7 +38,7 @@ int main(int argc, char *argv[]) {
         assert_se(mkdtemp(dn));
         fn = strappend(dn, "/test.journal");
 
-        r = journal_file_open(fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, &new_journal);
+        r = journal_file_open(fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, NULL, &new_journal);
         assert_se(r >= 0);
 
         r = sd_journal_open(&j, 0);

diff --git a/src/journal/test-journal-interleaving.c b/src/journal/test-journal-interleaving.c
index 6c6238cc4be75d28e4addee6c5b9c8f285a968da..f887f43f0dd61de1f8fe67877549eca467c4bc0b 100644 (file)
--- a/src/journal/test-journal-interleaving.c
+++ b/src/journal/test-journal-interleaving.c
@@ -52,7 +52,7 @@ noreturn static void log_assert_errno(const char *text, int eno, const char *fil
 
 static JournalFile *test_open(const char *name) {
         JournalFile *f;
-        assert_ret(journal_file_open(name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, &f));
+        assert_ret(journal_file_open(name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, NULL, &f));
         return f;
 }
 
@@ -217,7 +217,7 @@ static void test_sequence_numbers(void) {
         assert_se(chdir(t) >= 0);
 
         assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0644,
-                                    true, false, NULL, NULL, NULL, &one) == 0);
+                                    true, false, NULL, NULL, NULL, NULL, &one) == 0);
 
         append_number(one, 1, &seqnum);
         printf("seqnum=%"PRIu64"\n", seqnum);
@@ -234,7 +234,7 @@ static void test_sequence_numbers(void) {
         memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t));
 
         assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0644,
-                                    true, false, NULL, NULL, one, &two) == 0);
+                                    true, false, NULL, NULL, NULL, one, &two) == 0);
 
         assert_se(two->header->state == STATE_ONLINE);
         assert_se(!sd_id128_equal(two->header->file_id, one->header->file_id));
@@ -265,7 +265,7 @@ static void test_sequence_numbers(void) {
         seqnum = 0;
 
         assert_se(journal_file_open("two.journal", O_RDWR, 0,
-                                    true, false, NULL, NULL, NULL, &two) == 0);
+                                    true, false, NULL, NULL, NULL, NULL, &two) == 0);
 
         assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id));
 

diff --git a/src/journal/test-journal-stream.c b/src/journal/test-journal-stream.c
index 82543bb5f958461c5ca906d1e9fbf89aece9541b..839ea5a9a56b32c4c91923389c9bbffce2fc5fb2 100644 (file)
--- a/src/journal/test-journal-stream.c
+++ b/src/journal/test-journal-stream.c
@@ -92,9 +92,9 @@ int main(int argc, char *argv[]) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, &one) == 0);
-        assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, &two) == 0);
-        assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, &three) == 0);
+        assert_se(journal_file_open("one.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &one) == 0);
+        assert_se(journal_file_open("two.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &two) == 0);
+        assert_se(journal_file_open("three.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &three) == 0);
 
         for (i = 0; i < N_ENTRIES; i++) {
                 char *p, *q;

diff --git a/src/journal/test-journal-verify.c b/src/journal/test-journal-verify.c
index 3fec18c480c55ff9012fefdd134304e45b3cfe80..6b4643cd250553e5198e28b7129212cbf17a50cb 100644 (file)
--- a/src/journal/test-journal-verify.c
+++ b/src/journal/test-journal-verify.c
@@ -55,7 +55,7 @@ static int raw_verify(const char *fn, const char *verification_key) {
         JournalFile *f;
         int r;
 
-        r = journal_file_open(fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, &f);
+        r = journal_file_open(fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f);
         if (r < 0)
                 return r;
 
@@ -88,7 +88,7 @@ int main(int argc, char *argv[]) {
 
         log_info("Generating...");
 
-        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
 
         for (n = 0; n < N_ENTRIES; n++) {
                 struct iovec iovec;
@@ -111,7 +111,7 @@ int main(int argc, char *argv[]) {
 
         log_info("Verifying...");
 
-        assert_se(journal_file_open("test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open("test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
         /* journal_file_print_header(f); */
         journal_file_dump(f);
 

diff --git a/src/journal/test-journal.c b/src/journal/test-journal.c
index 94d26a257370e34ebb0bc31546f9466ab568f9aa..ea685af782d3445fb1f45e3595805daee2b771fb 100644 (file)
--- a/src/journal/test-journal.c
+++ b/src/journal/test-journal.c
@@ -42,7 +42,7 @@ static void test_non_empty(void) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f) == 0);
 
         dual_timestamp_get(&ts);
 
@@ -104,8 +104,8 @@ static void test_non_empty(void) {
 
         assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0);
 
-        journal_file_rotate(&f, true, true);
-        journal_file_rotate(&f, true, true);
+        journal_file_rotate(&f, true, true, NULL);
+        journal_file_rotate(&f, true, true, NULL);
 
         (void) journal_file_close(f);
 
@@ -131,13 +131,13 @@ static void test_empty(void) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, &f1) == 0);
+        assert_se(journal_file_open("test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, NULL, &f1) == 0);
 
-        assert_se(journal_file_open("test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, &f2) == 0);
+        assert_se(journal_file_open("test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &f2) == 0);
 
-        assert_se(journal_file_open("test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, &f3) == 0);
+        assert_se(journal_file_open("test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, NULL, &f3) == 0);
 
-        assert_se(journal_file_open("test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, &f4) == 0);
+        assert_se(journal_file_open("test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f4) == 0);
 
         journal_file_print_header(f1);
         puts("");