lib-index: mail_cache_compress*() - Add compress_file_seq parameter

This can be used to specify when the cache file should be compressed.
This was previously hidden in the cache->compress_file_seq field, but
now it's more explicit.

This commit also makes sure that cache is compressed when index rebuilding
is done.

diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c
index c9ceaad2298840cf76bf6ec309a60defc1120fbd..dd43b1ffef0f9dc8164927957521b3636790dbe9 100644 (file)
--- a/src/lib-index/mail-cache-compress.c
+++ b/src/lib-index/mail-cache-compress.c
@@ -330,12 +330,14 @@ mail_cache_compress_write(struct mail_cache *cache,
                          int fd, const char *temp_path, bool *unlock)
 {
        struct stat st;
-       uint32_t file_seq, old_offset, max_uid;
+       uint32_t prev_file_seq, file_seq, old_offset, max_uid;
        ARRAY_TYPE(uint32_t) ext_offsets;
        const uint32_t *offsets;
        uoff_t file_size;
        unsigned int i, count;
 
+       prev_file_seq = cache->hdr == NULL ? 0 : cache->hdr->file_seq;
+
        if (mail_cache_copy(cache, trans, fd, &file_seq, &file_size,
                            &max_uid, &ext_offsets) < 0)
                return -1;
@@ -353,8 +355,7 @@ mail_cache_compress_write(struct mail_cache *cache,
 
        e_debug(cache->index->event, "%s: Compressed, file_seq changed %u -> %u, "
                "size=%"PRIuUOFF_T", max_uid=%u", cache->filepath,
-               cache->need_compress_file_seq, file_seq,
-               file_size, max_uid);
+               prev_file_seq, file_seq, file_size, max_uid);
 
        /* once we're sure that the compression was successful,
           update the offsets */
@@ -382,7 +383,9 @@ mail_cache_compress_write(struct mail_cache *cache,
        return 0;
 }
 
-static int mail_cache_compress_has_file_changed(struct mail_cache *cache)
+static int
+mail_cache_compress_has_file_changed(struct mail_cache *cache,
+                                    uint32_t compress_file_seq)
 {
        struct mail_cache_header hdr;
        unsigned int i;
@@ -404,12 +407,12 @@ static int mail_cache_compress_has_file_changed(struct mail_cache *cache)
                if (ret >= 0) {
                        if (ret == 0)
                                return 0;
-                       if (cache->need_compress_file_seq == 0) {
+                       if (compress_file_seq == 0) {
                                /* previously it didn't exist or it
                                   was unusable and was just unlinked */
                                return 1;
                        }
-                       return hdr.file_seq != cache->need_compress_file_seq ? 1 : 0;
+                       return hdr.file_seq != compress_file_seq ? 1 : 0;
                } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
                        mail_cache_set_syscall_error(cache, "read()");
                        return -1;
@@ -429,7 +432,8 @@ static int mail_cache_compress_dotlock(struct mail_cache *cache,
        return 0;
 }
 
-static int mail_cache_compress_locked(struct mail_cache *cache, bool forced,
+static int mail_cache_compress_locked(struct mail_cache *cache,
+                                     uint32_t compress_file_seq,
                                      struct mail_index_transaction *trans,
                                      bool *unlock, struct dotlock **dotlock_r)
 {
@@ -447,8 +451,8 @@ static int mail_cache_compress_locked(struct mail_cache *cache, bool forced,
                return -1;
        /* we've locked the cache compression now. if somebody else had just
           recreated the cache, reopen the cache and return success. */
-       if (!forced &&
-           (ret = mail_cache_compress_has_file_changed(cache)) != 0) {
+       if (compress_file_seq != (uint32_t)-1 &&
+           (ret = mail_cache_compress_has_file_changed(cache, compress_file_seq)) != 0) {
                if (ret < 0)
                        return -1;
 
@@ -491,8 +495,9 @@ static int mail_cache_compress_locked(struct mail_cache *cache, bool forced,
 }
 
 static int
-mail_cache_compress_full(struct mail_cache *cache, bool forced,
+mail_cache_compress_full(struct mail_cache *cache,
                         struct mail_index_transaction *trans,
+                        uint32_t compress_file_seq,
                         struct mail_cache_compress_lock **lock_r)
 {
        struct dotlock *dotlock = NULL;
@@ -540,7 +545,7 @@ mail_cache_compress_full(struct mail_cache *cache, bool forced,
                }
        }
        cache->compressing = TRUE;
-       ret = mail_cache_compress_locked(cache, forced, trans, &unlock, &dotlock);
+       ret = mail_cache_compress_locked(cache, compress_file_seq, trans, &unlock, &dotlock);
        cache->compressing = FALSE;
        if (unlock) {
                if (mail_cache_unlock(cache) < 0)
@@ -561,12 +566,13 @@ mail_cache_compress_full(struct mail_cache *cache, bool forced,
 
 int mail_cache_compress_with_trans(struct mail_cache *cache,
                                   struct mail_index_transaction *trans,
+                                  uint32_t compress_file_seq,
                                   struct mail_cache_compress_lock **lock_r)
 {
-       return mail_cache_compress_full(cache, FALSE, trans, lock_r);
+       return mail_cache_compress_full(cache, trans, compress_file_seq, lock_r);
 }
 
-int mail_cache_compress(struct mail_cache *cache)
+int mail_cache_compress(struct mail_cache *cache, uint32_t compress_file_seq)
 {
        struct mail_index_view *view;
        struct mail_index_transaction *trans;
@@ -576,7 +582,7 @@ int mail_cache_compress(struct mail_cache *cache)
        view = mail_index_view_open(cache->index);
        trans = mail_index_transaction_begin(view,
                MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
-       if ((ret = mail_cache_compress_full(cache, FALSE, trans, &lock)) < 0)
+       if ((ret = mail_cache_compress_full(cache, trans, compress_file_seq, &lock)) < 0)
                mail_index_transaction_rollback(&trans);
        else {
                if (mail_index_transaction_commit(&trans) < 0)
@@ -591,7 +597,7 @@ int mail_cache_compress_forced(struct mail_cache *cache,
                               struct mail_index_transaction *trans,
                               struct mail_cache_compress_lock **lock_r)
 {
-       return mail_cache_compress_full(cache, TRUE, trans, lock_r);
+       return mail_cache_compress_full(cache, trans, (uint32_t)-1, lock_r);
 }
 
 void mail_cache_compress_unlock(struct mail_cache_compress_lock **_lock)

diff --git a/src/lib-index/mail-cache-decisions.c b/src/lib-index/mail-cache-decisions.c
index 5b605afa8e5687b431894863a243a1af6807548b..904268eb35802b52faec12548c7fb8cabb90ad42 100644 (file)
--- a/src/lib-index/mail-cache-decisions.c
+++ b/src/lib-index/mail-cache-decisions.c
@@ -171,5 +171,5 @@ int mail_cache_decisions_copy(struct mail_cache *src, struct mail_cache *dst)
           that the fields are updated even if the cache was already created
           and no compression was done. */
        dst->field_header_write_pending = TRUE;
-       return mail_cache_compress(dst);
+       return mail_cache_compress(dst, 0);
 }

diff --git a/src/lib-index/mail-cache-transaction.c b/src/lib-index/mail-cache-transaction.c
index 5c2301d63ffb88c94411206e2895b9c31ae0f19e..8e8224cbc314b9ee3c164719dfc074312ea39692 100644 (file)
--- a/src/lib-index/mail-cache-transaction.c
+++ b/src/lib-index/mail-cache-transaction.c
@@ -182,10 +182,10 @@ mail_cache_transaction_compress(struct mail_cache_transaction_ctx *ctx)
 
        ctx->tried_compression = TRUE;
 
-       cache->need_compress_file_seq =
+       uint32_t compress_file_seq =
                MAIL_CACHE_IS_UNUSABLE(cache) ? 0 : cache->hdr->file_seq;
 
-       int ret = mail_cache_compress(cache);
+       int ret = mail_cache_compress(cache, compress_file_seq);
        /* already written cache records must be forgotten, but records in
           memory can still be written to the new cache file */
        mail_cache_transaction_forget_flushed(ctx);

diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h
index f28de80da39dbe0b2a00221958ffd52720dfeb64..3c62188cbc6c3a2ffb4eb91de5a6c28d9c0b1eb9 100644 (file)
--- a/src/lib-index/mail-cache.h
+++ b/src/lib-index/mail-cache.h
@@ -71,11 +71,18 @@ bool mail_cache_need_compress(struct mail_cache *cache);
    compression lock should be kept until the transaction is committed.
    mail_cache_compress_unlock() needs to be called afterwards. The lock doesn't
    prevent updates to the cache while it's held, it only prevents another cache
-   compression. */
+   compression.
+
+   The cache compression is done only if the current cache file's file_seq
+   matches compress_file_seq. The idea is that compression isn't done if
+   another process had just compressed it. 0 means the cache file is created
+   only if it didn't already exist. (uint32_t)-1 means that compression is
+   done always regardless of file_seq. */
 int mail_cache_compress_with_trans(struct mail_cache *cache,
                                   struct mail_index_transaction *trans,
+                                  uint32_t compress_file_seq,
                                   struct mail_cache_compress_lock **lock_r);
-int mail_cache_compress(struct mail_cache *cache);
+int mail_cache_compress(struct mail_cache *cache, uint32_t compress_file_seq);
 int mail_cache_compress_forced(struct mail_cache *cache,
                               struct mail_index_transaction *trans,
                               struct mail_cache_compress_lock **lock_r);

diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c
index 61e5197cb99bcba250830d55db1c9fa302d4fe7d..247bd1d352f56c2abd58819accf839106c796c6f 100644 (file)
--- a/src/lib-index/mail-index-sync.c
+++ b/src/lib-index/mail-index-sync.c
@@ -949,6 +949,7 @@ int mail_index_sync_commit(struct mail_index_sync_ctx **_ctx)
                        trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_FSYNC;
                cache_trans = mail_index_transaction_begin(ctx->view, trans_flags);
                if (mail_cache_compress_with_trans(index->cache, cache_trans,
+                                                  index->cache->need_compress_file_seq,
                                                   &cache_lock) < 0)
                        mail_index_transaction_rollback(&cache_trans);
                else {

diff --git a/src/lib-storage/index/index-rebuild.c b/src/lib-storage/index/index-rebuild.c
index ee9ce2c4b20bb53bcb3f701532d97f406f5e99b1..01d2028d6f624874788c3f7884cafdf181e68847 100644 (file)
--- a/src/lib-storage/index/index-rebuild.c
+++ b/src/lib-storage/index/index-rebuild.c
@@ -240,7 +240,8 @@ void index_index_rebuild_deinit(struct index_rebuild_context **_ctx,
        *_ctx = NULL;
 
        /* initialize cache file with the old field decisions */
-       (void)mail_cache_compress_with_trans(ctx->box->cache, ctx->trans, &lock);
+       (void)mail_cache_compress_with_trans(ctx->box->cache, ctx->trans,
+                                            (uint32_t)-1, &lock);
        if (lock != NULL) {
                /* FIXME: this is a bit too early. ideally we should return it
                   from this function and unlock only after the transaction is