From: Timo Sirainen <timo.sirainen@open-xchange.com>
Date: Fri, 27 Mar 2020 10:36:40 +0000 (+0200)
Subject: lib-index: mail_cache_compress*() - Add compress_file_seq parameter
X-Git-Tag: 2.3.11.2~415
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=ea21a6e0973ad31a79ba6df6bc421e481889a79e;p=thirdparty%2Fdovecot%2Fcore.git

lib-index: mail_cache_compress*() - Add compress_file_seq parameter

This can be used to specify when the cache file should be compressed.
This was previously hidden in the cache->compress_file_seq field, but
now it's more explicit.

This commit also makes sure that cache is compressed when index rebuilding
is done.
---

diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c
index c9ceaad229..dd43b1ffef 100644
--- a/src/lib-index/mail-cache-compress.c
+++ b/src/lib-index/mail-cache-compress.c
@@ -330,12 +330,14 @@ mail_cache_compress_write(struct mail_cache *cache,
 			  int fd, const char *temp_path, bool *unlock)
 {
 	struct stat st;
-	uint32_t file_seq, old_offset, max_uid;
+	uint32_t prev_file_seq, file_seq, old_offset, max_uid;
 	ARRAY_TYPE(uint32_t) ext_offsets;
 	const uint32_t *offsets;
 	uoff_t file_size;
 	unsigned int i, count;
 
+	prev_file_seq = cache->hdr == NULL ? 0 : cache->hdr->file_seq;
+
 	if (mail_cache_copy(cache, trans, fd, &file_seq, &file_size,
 			    &max_uid, &ext_offsets) < 0)
 		return -1;
@@ -353,8 +355,7 @@ mail_cache_compress_write(struct mail_cache *cache,
 
 	e_debug(cache->index->event, "%s: Compressed, file_seq changed %u -> %u, "
 		"size=%"PRIuUOFF_T", max_uid=%u", cache->filepath,
-		cache->need_compress_file_seq, file_seq,
-		file_size, max_uid);
+		prev_file_seq, file_seq, file_size, max_uid);
 
 	/* once we're sure that the compression was successful,
 	   update the offsets */
@@ -382,7 +383,9 @@ mail_cache_compress_write(struct mail_cache *cache,
 	return 0;
 }
 
-static int mail_cache_compress_has_file_changed(struct mail_cache *cache)
+static int
+mail_cache_compress_has_file_changed(struct mail_cache *cache,
+				     uint32_t compress_file_seq)
 {
 	struct mail_cache_header hdr;
 	unsigned int i;
@@ -404,12 +407,12 @@ static int mail_cache_compress_has_file_changed(struct mail_cache *cache)
 		if (ret >= 0) {
 			if (ret == 0)
 				return 0;
-			if (cache->need_compress_file_seq == 0) {
+			if (compress_file_seq == 0) {
 				/* previously it didn't exist or it
 				   was unusable and was just unlinked */
 				return 1;
 			}
-			return hdr.file_seq != cache->need_compress_file_seq ? 1 : 0;
+			return hdr.file_seq != compress_file_seq ? 1 : 0;
 		} else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
 			mail_cache_set_syscall_error(cache, "read()");
 			return -1;
@@ -429,7 +432,8 @@ static int mail_cache_compress_dotlock(struct mail_cache *cache,
 	return 0;
 }
 
-static int mail_cache_compress_locked(struct mail_cache *cache, bool forced,
+static int mail_cache_compress_locked(struct mail_cache *cache,
+				      uint32_t compress_file_seq,
 				      struct mail_index_transaction *trans,
 				      bool *unlock, struct dotlock **dotlock_r)
 {
@@ -447,8 +451,8 @@ static int mail_cache_compress_locked(struct mail_cache *cache, bool forced,
 		return -1;
 	/* we've locked the cache compression now. if somebody else had just
 	   recreated the cache, reopen the cache and return success. */
-	if (!forced &&
-	    (ret = mail_cache_compress_has_file_changed(cache)) != 0) {
+	if (compress_file_seq != (uint32_t)-1 &&
+	    (ret = mail_cache_compress_has_file_changed(cache, compress_file_seq)) != 0) {
 		if (ret < 0)
 			return -1;
 
@@ -491,8 +495,9 @@ static int mail_cache_compress_locked(struct mail_cache *cache, bool forced,
 }
 
 static int
-mail_cache_compress_full(struct mail_cache *cache, bool forced,
+mail_cache_compress_full(struct mail_cache *cache,
 			 struct mail_index_transaction *trans,
+			 uint32_t compress_file_seq,
 			 struct mail_cache_compress_lock **lock_r)
 {
 	struct dotlock *dotlock = NULL;
@@ -540,7 +545,7 @@ mail_cache_compress_full(struct mail_cache *cache, bool forced,
 		}
 	}
 	cache->compressing = TRUE;
-	ret = mail_cache_compress_locked(cache, forced, trans, &unlock, &dotlock);
+	ret = mail_cache_compress_locked(cache, compress_file_seq, trans, &unlock, &dotlock);
 	cache->compressing = FALSE;
 	if (unlock) {
 		if (mail_cache_unlock(cache) < 0)
@@ -561,12 +566,13 @@ mail_cache_compress_full(struct mail_cache *cache, bool forced,
 
 int mail_cache_compress_with_trans(struct mail_cache *cache,
 				   struct mail_index_transaction *trans,
+				   uint32_t compress_file_seq,
 				   struct mail_cache_compress_lock **lock_r)
 {
-	return mail_cache_compress_full(cache, FALSE, trans, lock_r);
+	return mail_cache_compress_full(cache, trans, compress_file_seq, lock_r);
 }
 
-int mail_cache_compress(struct mail_cache *cache)
+int mail_cache_compress(struct mail_cache *cache, uint32_t compress_file_seq)
 {
 	struct mail_index_view *view;
 	struct mail_index_transaction *trans;
@@ -576,7 +582,7 @@ int mail_cache_compress(struct mail_cache *cache)
 	view = mail_index_view_open(cache->index);
 	trans = mail_index_transaction_begin(view,
 		MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
-	if ((ret = mail_cache_compress_full(cache, FALSE, trans, &lock)) < 0)
+	if ((ret = mail_cache_compress_full(cache, trans, compress_file_seq, &lock)) < 0)
 		mail_index_transaction_rollback(&trans);
 	else {
 		if (mail_index_transaction_commit(&trans) < 0)
@@ -591,7 +597,7 @@ int mail_cache_compress_forced(struct mail_cache *cache,
 			       struct mail_index_transaction *trans,
 			       struct mail_cache_compress_lock **lock_r)
 {
-	return mail_cache_compress_full(cache, TRUE, trans, lock_r);
+	return mail_cache_compress_full(cache, trans, (uint32_t)-1, lock_r);
 }
 
 void mail_cache_compress_unlock(struct mail_cache_compress_lock **_lock)
diff --git a/src/lib-index/mail-cache-decisions.c b/src/lib-index/mail-cache-decisions.c
index 5b605afa8e..904268eb35 100644
--- a/src/lib-index/mail-cache-decisions.c
+++ b/src/lib-index/mail-cache-decisions.c
@@ -171,5 +171,5 @@ int mail_cache_decisions_copy(struct mail_cache *src, struct mail_cache *dst)
 	   that the fields are updated even if the cache was already created
 	   and no compression was done. */
 	dst->field_header_write_pending = TRUE;
-	return mail_cache_compress(dst);
+	return mail_cache_compress(dst, 0);
 }
diff --git a/src/lib-index/mail-cache-transaction.c b/src/lib-index/mail-cache-transaction.c
index 5c2301d63f..8e8224cbc3 100644
--- a/src/lib-index/mail-cache-transaction.c
+++ b/src/lib-index/mail-cache-transaction.c
@@ -182,10 +182,10 @@ mail_cache_transaction_compress(struct mail_cache_transaction_ctx *ctx)
 
 	ctx->tried_compression = TRUE;
 
-	cache->need_compress_file_seq =
+	uint32_t compress_file_seq =
 		MAIL_CACHE_IS_UNUSABLE(cache) ? 0 : cache->hdr->file_seq;
 
-	int ret = mail_cache_compress(cache);
+	int ret = mail_cache_compress(cache, compress_file_seq);
 	/* already written cache records must be forgotten, but records in
 	   memory can still be written to the new cache file */
 	mail_cache_transaction_forget_flushed(ctx);
diff --git a/src/lib-index/mail-cache.h b/src/lib-index/mail-cache.h
index f28de80da3..3c62188cbc 100644
--- a/src/lib-index/mail-cache.h
+++ b/src/lib-index/mail-cache.h
@@ -71,11 +71,18 @@ bool mail_cache_need_compress(struct mail_cache *cache);
    compression lock should be kept until the transaction is committed.
    mail_cache_compress_unlock() needs to be called afterwards. The lock doesn't
    prevent updates to the cache while it's held, it only prevents another cache
-   compression. */
+   compression.
+
+   The cache compression is done only if the current cache file's file_seq
+   matches compress_file_seq. The idea is that compression isn't done if
+   another process had just compressed it. 0 means the cache file is created
+   only if it didn't already exist. (uint32_t)-1 means that compression is
+   done always regardless of file_seq. */
 int mail_cache_compress_with_trans(struct mail_cache *cache,
 				   struct mail_index_transaction *trans,
+				   uint32_t compress_file_seq,
 				   struct mail_cache_compress_lock **lock_r);
-int mail_cache_compress(struct mail_cache *cache);
+int mail_cache_compress(struct mail_cache *cache, uint32_t compress_file_seq);
 int mail_cache_compress_forced(struct mail_cache *cache,
 			       struct mail_index_transaction *trans,
 			       struct mail_cache_compress_lock **lock_r);
diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c
index 61e5197cb9..247bd1d352 100644
--- a/src/lib-index/mail-index-sync.c
+++ b/src/lib-index/mail-index-sync.c
@@ -949,6 +949,7 @@ int mail_index_sync_commit(struct mail_index_sync_ctx **_ctx)
 			trans_flags |= MAIL_INDEX_TRANSACTION_FLAG_FSYNC;
 		cache_trans = mail_index_transaction_begin(ctx->view, trans_flags);
 		if (mail_cache_compress_with_trans(index->cache, cache_trans,
+						   index->cache->need_compress_file_seq,
 						   &cache_lock) < 0)
 			mail_index_transaction_rollback(&cache_trans);
 		else {
diff --git a/src/lib-storage/index/index-rebuild.c b/src/lib-storage/index/index-rebuild.c
index ee9ce2c4b2..01d2028d6f 100644
--- a/src/lib-storage/index/index-rebuild.c
+++ b/src/lib-storage/index/index-rebuild.c
@@ -240,7 +240,8 @@ void index_index_rebuild_deinit(struct index_rebuild_context **_ctx,
 	*_ctx = NULL;
 
 	/* initialize cache file with the old field decisions */
-	(void)mail_cache_compress_with_trans(ctx->box->cache, ctx->trans, &lock);
+	(void)mail_cache_compress_with_trans(ctx->box->cache, ctx->trans,
+					     (uint32_t)-1, &lock);
 	if (lock != NULL) {
 		/* FIXME: this is a bit too early. ideally we should return it
 		   from this function and unlock only after the transaction is