]> git.ipfire.org Git - thirdparty/systemd.git/commitdiff
journal: provide compress_threshold_bytes parameter
authorAlex Gartrell <agartrell@fb.com>
Tue, 27 Feb 2018 05:56:35 +0000 (21:56 -0800)
committerAlex Gartrell <agartrell@fb.com>
Tue, 20 Mar 2018 18:48:52 +0000 (11:48 -0700)
Previously the compression threshold was hardcoded to 512, which meant that
smaller values wouldn't be compressed. This left some storage savings on the
table, so instead, we make that number tunable.

src/journal-remote/journal-remote-write.c
src/journal-remote/journal-remote.c
src/journal/journal-file.c
src/journal/journal-file.h
src/journal/journald-server.c
src/journal/sd-journal.c
src/journal/test-journal-flush.c
src/journal/test-journal-interleaving.c
src/journal/test-journal-stream.c
src/journal/test-journal-verify.c
src/journal/test-journal.c

index c5c6abbd5c39c14e77e3a2aa77ed64974f4fc64e..2836ba8b48e08ac76482cb19daf7903316ebd290 100644 (file)
@@ -22,7 +22,7 @@
 #include "journal-remote.h"
 
 static int do_rotate(JournalFile **f, bool compress, bool seal) {
-        int r = journal_file_rotate(f, compress, seal, NULL);
+        int r = journal_file_rotate(f, compress, (uint64_t) -1, seal, NULL);
         if (r < 0) {
                 if (*f)
                         log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);
index 428725223d95bf3b78ad8dd8c07b6834244de84a..ff46023920216962a82af809b0d56ba5f24da6ee 100644 (file)
@@ -185,7 +185,7 @@ static int open_output(Writer *w, const char* host) {
 
         r = journal_file_open_reliably(output,
                                        O_RDWR|O_CREAT, 0640,
-                                       arg_compress, arg_seal,
+                                       arg_compress, (uint64_t) -1, arg_seal,
                                        &w->metrics,
                                        w->mmap, NULL,
                                        NULL, &w->journal);
@@ -729,7 +729,7 @@ static int setup_microhttpd_server(RemoteServer *s,
         }
 
         r = sd_event_add_time(s->events, &d->timer_event,
-                              CLOCK_MONOTONIC, UINT64_MAX, 0,
+                              CLOCK_MONOTONIC, (uint64_t) -1, 0,
                               null_timer_event_handler, d);
         if (r < 0) {
                 log_error_errno(r, "Failed to add timer_event: %m");
index 5643c0578d6904fbc9a8dafe25e67844fdf642c6..5fc61b16e05883a52d6a5f9d90e27440538e1db0 100644 (file)
@@ -51,7 +51,8 @@
 #define DEFAULT_DATA_HASH_TABLE_SIZE (2047ULL*sizeof(HashItem))
 #define DEFAULT_FIELD_HASH_TABLE_SIZE (333ULL*sizeof(HashItem))
 
-#define COMPRESSION_SIZE_THRESHOLD (512ULL)
+#define DEFAULT_COMPRESS_THRESHOLD (512ULL)
+#define MIN_COMPRESS_THRESHOLD (8ULL)
 
 /* This is the minimum journal file size */
 #define JOURNAL_FILE_SIZE_MIN (512ULL*1024ULL)                 /* 512 KiB */
@@ -1552,7 +1553,7 @@ static int journal_file_append_data(
         o->data.hash = htole64(hash);
 
 #if HAVE_XZ || HAVE_LZ4
-        if (JOURNAL_FILE_COMPRESS(f) && size >= COMPRESSION_SIZE_THRESHOLD) {
+        if (JOURNAL_FILE_COMPRESS(f) && size >= f->compress_threshold_bytes) {
                 size_t rsize = 0;
 
                 compression = compress_blob(data, size, o->data.payload, size - 1, &rsize);
@@ -3211,6 +3212,7 @@ int journal_file_open(
                 int flags,
                 mode_t mode,
                 bool compress,
+                uint64_t compress_threshold_bytes,
                 bool seal,
                 JournalMetrics *metrics,
                 MMapCache *mmap_cache,
@@ -3222,6 +3224,7 @@ int journal_file_open(
         JournalFile *f;
         void *h;
         int r;
+        char bytes[FORMAT_BYTES_MAX];
 
         assert(ret);
         assert(fd >= 0 || fname);
@@ -3247,10 +3250,20 @@ int journal_file_open(
 #elif HAVE_XZ
         f->compress_xz = compress;
 #endif
+
+        if (compress_threshold_bytes == (uint64_t) -1)
+                f->compress_threshold_bytes = DEFAULT_COMPRESS_THRESHOLD;
+        else
+                f->compress_threshold_bytes = MAX(MIN_COMPRESS_THRESHOLD, compress_threshold_bytes);
+
 #if HAVE_GCRYPT
         f->seal = seal;
 #endif
 
+        log_debug("Journal effective settings seal=%s compress=%s compress_threshold_bytes=%s",
+                  yes_no(f->seal), yes_no(JOURNAL_FILE_COMPRESS(f)),
+                  format_bytes(bytes, sizeof(bytes), f->compress_threshold_bytes));
+
         if (mmap_cache)
                 f->mmap = mmap_cache_ref(mmap_cache);
         else {
@@ -3435,7 +3448,7 @@ fail:
         return r;
 }
 
-int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes) {
+int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes) {
         _cleanup_free_ char *p = NULL;
         size_t l;
         JournalFile *old_file, *new_file = NULL;
@@ -3489,7 +3502,9 @@ int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred
          * we archive them */
         old_file->defrag_on_close = true;
 
-        r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress, seal, NULL, old_file->mmap, deferred_closes, old_file, &new_file);
+        r = journal_file_open(-1, old_file->path, old_file->flags, old_file->mode, compress,
+                              compress_threshold_bytes, seal, NULL, old_file->mmap, deferred_closes,
+                              old_file, &new_file);
 
         if (deferred_closes &&
             set_put(deferred_closes, old_file) >= 0)
@@ -3506,6 +3521,7 @@ int journal_file_open_reliably(
                 int flags,
                 mode_t mode,
                 bool compress,
+                uint64_t compress_threshold_bytes,
                 bool seal,
                 JournalMetrics *metrics,
                 MMapCache *mmap_cache,
@@ -3517,7 +3533,8 @@ int journal_file_open_reliably(
         size_t l;
         _cleanup_free_ char *p = NULL;
 
-        r = journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
+        r = journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache,
+                              deferred_closes, template, ret);
         if (!IN_SET(r,
                     -EBADMSG,           /* Corrupted */
                     -ENODATA,           /* Truncated */
@@ -3559,7 +3576,8 @@ int journal_file_open_reliably(
 
         log_warning_errno(r, "File %s corrupted or uncleanly shut down, renaming and replacing.", fname);
 
-        return journal_file_open(-1, fname, flags, mode, compress, seal, metrics, mmap_cache, deferred_closes, template, ret);
+        return journal_file_open(-1, fname, flags, mode, compress, compress_threshold_bytes, seal, metrics, mmap_cache,
+                                 deferred_closes, template, ret);
 }
 
 int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint64_t p, uint64_t *seqnum, Object **ret, uint64_t *offset) {
index 67abf8da498b3dfa0b6458b4cf6db74f2681c71f..04d35f58131b7dbeb1e4b76a0e11dc014ffce1c3 100644 (file)
@@ -124,6 +124,7 @@ typedef struct JournalFile {
 
         unsigned last_seen_generation;
 
+        uint64_t compress_threshold_bytes;
 #if HAVE_XZ || HAVE_LZ4
         void *compress_buffer;
         size_t compress_buffer_size;
@@ -153,6 +154,7 @@ int journal_file_open(
                 int flags,
                 mode_t mode,
                 bool compress,
+                uint64_t compress_threshold_bytes,
                 bool seal,
                 JournalMetrics *metrics,
                 MMapCache *mmap_cache,
@@ -169,6 +171,7 @@ int journal_file_open_reliably(
                 int flags,
                 mode_t mode,
                 bool compress,
+                uint64_t compress_threshold_bytes,
                 bool seal,
                 JournalMetrics *metrics,
                 MMapCache *mmap_cache,
@@ -246,7 +249,7 @@ int journal_file_copy_entry(JournalFile *from, JournalFile *to, Object *o, uint6
 void journal_file_dump(JournalFile *f);
 void journal_file_print_header(JournalFile *f);
 
-int journal_file_rotate(JournalFile **f, bool compress, bool seal, Set *deferred_closes);
+int journal_file_rotate(JournalFile **f, bool compress, uint64_t compress_threshold_bytes, bool seal, Set *deferred_closes);
 
 void journal_file_post_change(JournalFile *f);
 int journal_file_enable_post_change_timer(JournalFile *f, sd_event *e, usec_t t);
index 5cd58e8a7794f18cc44d167329aaea50e4323bba..570450d321e3b209828709df2f6543ad16193f7a 100644 (file)
@@ -280,9 +280,9 @@ static int open_journal(
         assert(ret);
 
         if (reliably)
-                r = journal_file_open_reliably(fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
+                r = journal_file_open_reliably(fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
         else
-                r = journal_file_open(-1, fname, flags, 0640, s->compress, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
+                r = journal_file_open(-1, fname, flags, 0640, s->compress, (uint64_t) -1, seal, metrics, s->mmap, s->deferred_closes, NULL, &f);
         if (r < 0)
                 return r;
 
@@ -463,7 +463,7 @@ static int do_rotate(
         if (!*f)
                 return -EINVAL;
 
-        r = journal_file_rotate(f, s->compress, seal, s->deferred_closes);
+        r = journal_file_rotate(f, s->compress, (uint64_t) -1, seal, s->deferred_closes);
         if (r < 0) {
                 if (*f)
                         return log_error_errno(r, "Failed to rotate %s: %m", (*f)->path);
index 11dbd83f2debe63762854d43dd9f5ec710eff695..79ffd9cb3fa215baf8ec2423d9fd8582d9ad6685 100644 (file)
@@ -1337,7 +1337,7 @@ static int add_any_file(
                 goto finish;
         }
 
-        r = journal_file_open(fd, path, O_RDONLY, 0, false, false, NULL, j->mmap, NULL, NULL, &f);
+        r = journal_file_open(fd, path, O_RDONLY, 0, false, 0, false, NULL, j->mmap, NULL, NULL, &f);
         if (r < 0) {
                 log_debug_errno(r, "Failed to open journal file %s: %m", path);
                 goto finish;
index f9b7b75ef1b085314dde748bc7179f773c4697be..e302fd61f1f94e2b623219c29f1683696cb459aa 100644 (file)
@@ -39,7 +39,7 @@ int main(int argc, char *argv[]) {
         assert_se(mkdtemp(dn));
         fn = strappend(dn, "/test.journal");
 
-        r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, false, NULL, NULL, NULL, NULL, &new_journal);
+        r = journal_file_open(-1, fn, O_CREAT|O_RDWR, 0644, false, 0, false, NULL, NULL, NULL, NULL, &new_journal);
         assert_se(r >= 0);
 
         r = sd_journal_open(&j, 0);
index d87bdbdd32ad6b0eba5a56942ddd4f838412c93f..f8b8d1e6d873408ada6f6d4f9c682eefe8886231 100644 (file)
@@ -52,7 +52,7 @@ _noreturn_ static void log_assert_errno(const char *text, int error, const char
 
 static JournalFile *test_open(const char *name) {
         JournalFile *f;
-        assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, false, NULL, NULL, NULL, NULL, &f));
+        assert_ret(journal_file_open(-1, name, O_RDWR|O_CREAT, 0644, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f));
         return f;
 }
 
@@ -217,7 +217,7 @@ static void test_sequence_numbers(void) {
         assert_se(chdir(t) >= 0);
 
         assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0644,
-                                    true, false, NULL, NULL, NULL, NULL, &one) == 0);
+                                    true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
 
         append_number(one, 1, &seqnum);
         printf("seqnum=%"PRIu64"\n", seqnum);
@@ -234,7 +234,7 @@ static void test_sequence_numbers(void) {
         memcpy(&seqnum_id, &one->header->seqnum_id, sizeof(sd_id128_t));
 
         assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0644,
-                                    true, false, NULL, NULL, NULL, one, &two) == 0);
+                                    true, (uint64_t) -1, false, NULL, NULL, NULL, one, &two) == 0);
 
         assert_se(two->header->state == STATE_ONLINE);
         assert_se(!sd_id128_equal(two->header->file_id, one->header->file_id));
@@ -265,7 +265,7 @@ static void test_sequence_numbers(void) {
         seqnum = 0;
 
         assert_se(journal_file_open(-1, "two.journal", O_RDWR, 0,
-                                    true, false, NULL, NULL, NULL, NULL, &two) == 0);
+                                    true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
 
         assert_se(sd_id128_equal(two->header->seqnum_id, seqnum_id));
 
index 73ed6e5dcbd994c77ad9523fc69c973c4d874ea2..279301c67d04ed7c97a09e02c982e8416e06d022 100644 (file)
@@ -93,9 +93,9 @@ int main(int argc, char *argv[]) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &one) == 0);
-        assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &two) == 0);
-        assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &three) == 0);
+        assert_se(journal_file_open(-1, "one.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &one) == 0);
+        assert_se(journal_file_open(-1, "two.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &two) == 0);
+        assert_se(journal_file_open(-1, "three.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &three) == 0);
 
         for (i = 0; i < N_ENTRIES; i++) {
                 char *p, *q;
index fbb75e43e3a1755d4408332637bf9c3e75ac2247..89389579f337540baf4ece9e0be53842c1043153 100644 (file)
@@ -56,7 +56,7 @@ static int raw_verify(const char *fn, const char *verification_key) {
         JournalFile *f;
         int r;
 
-        r = journal_file_open(-1, fn, O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f);
+        r = journal_file_open(-1, fn, O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f);
         if (r < 0)
                 return r;
 
@@ -89,7 +89,7 @@ int main(int argc, char *argv[]) {
 
         log_info("Generating...");
 
-        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
 
         for (n = 0; n < N_ENTRIES; n++) {
                 struct iovec iovec;
@@ -112,7 +112,7 @@ int main(int argc, char *argv[]) {
 
         log_info("Verifying...");
 
-        assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDONLY, 0666, true, (uint64_t) -1, !!verification_key, NULL, NULL, NULL, NULL, &f) == 0);
         /* journal_file_print_header(f); */
         journal_file_dump(f);
 
index 517c9102a643e1b49c941172620ff6640a21ec07..3b9917b4a8c522afa1be95c79b53f25b95a8544f 100644 (file)
@@ -43,7 +43,7 @@ static void test_non_empty(void) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f) == 0);
 
         dual_timestamp_get(&ts);
 
@@ -105,8 +105,8 @@ static void test_non_empty(void) {
 
         assert_se(journal_file_move_to_entry_by_seqnum(f, 10, DIRECTION_DOWN, &o, NULL) == 0);
 
-        journal_file_rotate(&f, true, true, NULL);
-        journal_file_rotate(&f, true, true, NULL);
+        journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
+        journal_file_rotate(&f, true, (uint64_t) -1, true, NULL);
 
         (void) journal_file_close(f);
 
@@ -132,13 +132,13 @@ static void test_empty(void) {
         assert_se(mkdtemp(t));
         assert_se(chdir(t) >= 0);
 
-        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, false, NULL, NULL, NULL, NULL, &f1) == 0);
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f1) == 0);
 
-        assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, false, NULL, NULL, NULL, NULL, &f2) == 0);
+        assert_se(journal_file_open(-1, "test-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, false, NULL, NULL, NULL, NULL, &f2) == 0);
 
-        assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, true, NULL, NULL, NULL, NULL, &f3) == 0);
+        assert_se(journal_file_open(-1, "test-seal.journal", O_RDWR|O_CREAT, 0666, false, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f3) == 0);
 
-        assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, true, NULL, NULL, NULL, NULL, &f4) == 0);
+        assert_se(journal_file_open(-1, "test-seal-compress.journal", O_RDWR|O_CREAT, 0666, true, (uint64_t) -1, true, NULL, NULL, NULL, NULL, &f4) == 0);
 
         journal_file_print_header(f1);
         puts("");
@@ -165,6 +165,91 @@ static void test_empty(void) {
         (void) journal_file_close(f4);
 }
 
+#if HAVE_XZ || HAVE_LZ4
+static bool check_compressed(uint64_t compress_threshold, uint64_t data_size) {
+        dual_timestamp ts;
+        JournalFile *f;
+        struct iovec iovec;
+        Object *o;
+        uint64_t p;
+        char t[] = "/tmp/journal-XXXXXX";
+        char data[2048] = {0};
+        bool is_compressed;
+        int r;
+
+        assert_se(data_size <= sizeof(data));
+
+        log_set_max_level(LOG_DEBUG);
+
+        assert_se(mkdtemp(t));
+        assert_se(chdir(t) >= 0);
+
+        assert_se(journal_file_open(-1, "test.journal", O_RDWR|O_CREAT, 0666, true, compress_threshold, true, NULL, NULL, NULL, NULL, &f) == 0);
+
+        dual_timestamp_get(&ts);
+
+        iovec.iov_base = (void*) data;
+        iovec.iov_len = data_size;
+        assert_se(journal_file_append_entry(f, &ts, &iovec, 1, NULL, NULL, NULL) == 0);
+
+#if HAVE_GCRYPT
+        journal_file_append_tag(f);
+#endif
+        journal_file_dump(f);
+
+        /* We have to partially reimplement some of the dump logic, because the normal next_entry does the
+         * decompression for us. */
+        p = le64toh(f->header->header_size);
+        while (true) {
+                r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o);
+                assert_se(r == 0);
+                if (o->object.type == OBJECT_DATA)
+                        break;
+
+                assert_se(p < le64toh(f->header->tail_object_offset));
+                p = p + ALIGN64(le64toh(o->object.size));
+        }
+
+        is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0;
+
+        (void) journal_file_close(f);
+
+        log_info("Done...");
+
+        if (arg_keep)
+                log_info("Not removing %s", t);
+        else {
+                journal_directory_vacuum(".", 3000000, 0, 0, NULL, true);
+
+                assert_se(rm_rf(t, REMOVE_ROOT|REMOVE_PHYSICAL) >= 0);
+        }
+
+        puts("------------------------------------------------------------");
+
+        return is_compressed;
+}
+
+static void test_min_compress_size(void) {
+        /* Note that XZ will actually fail to compress anything under 80 bytes, so you have to choose the limits
+         * carefully */
+
+        /* DEFAULT_MIN_COMPRESS_SIZE is 512 */
+        assert_se(!check_compressed((uint64_t) -1, 255));
+        assert_se(check_compressed((uint64_t) -1, 513));
+
+        /* compress everything */
+        assert_se(check_compressed(0, 96));
+        assert_se(check_compressed(8, 96));
+
+        /* Ensure we don't try to compress less than 8 bytes */
+        assert_se(!check_compressed(0, 7));
+
+        /* check boundary conditions */
+        assert_se(check_compressed(256, 256));
+        assert_se(!check_compressed(256, 255));
+}
+#endif
+
 int main(int argc, char *argv[]) {
         arg_keep = argc > 1;
 
@@ -174,6 +259,9 @@ int main(int argc, char *argv[]) {
 
         test_non_empty();
         test_empty();
+#if HAVE_XZ || HAVE_LZ4
+        test_min_compress_size();
+#endif
 
         return 0;
 }