From: Timo Sirainen <tss@iki.fi>
Date: Thu, 18 Oct 2012 02:22:36 +0000 (+0300)
Subject: lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache... 
X-Git-Tag: 2.2.alpha1~20^2~23
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=18d92dbbb752c79dc461514e52f7ef11847e636b;p=thirdparty%2Fdovecot%2Fcore.git

lib-index: Added MAIL_INDEX_OPEN_FLAG_SAVEONLY to do only minimal reads from cache file.
---

diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c
index cc8ec9cb60..e3bdf7cdac 100644
--- a/src/lib-index/mail-cache-compress.c
+++ b/src/lib-index/mail-cache-compress.c
@@ -461,6 +461,9 @@ int mail_cache_compress(struct mail_cache *cache,
 	if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly)
 		return 0;
 
+	/* compression isn't very efficient with small read()s */
+	cache->map_with_read = FALSE;
+
 	if (cache->index->lock_method == FILE_LOCK_METHOD_DOTLOCK) {
 		/* we're using dotlocking, cache file creation itself creates
 		   the dotlock file we need. */
@@ -495,5 +498,6 @@ int mail_cache_compress(struct mail_cache *cache,
 bool mail_cache_need_compress(struct mail_cache *cache)
 {
 	return cache->need_compress_file_seq != 0 &&
+		(cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) == 0 &&
 		!cache->index->readonly;
 }
diff --git a/src/lib-index/mail-cache-private.h b/src/lib-index/mail-cache-private.h
index 2b96cb55b8..46feda4042 100644
--- a/src/lib-index/mail-cache-private.h
+++ b/src/lib-index/mail-cache-private.h
@@ -130,9 +130,14 @@ struct mail_cache {
 	ino_t st_ino;
 	dev_t st_dev;
 
-	void *mmap_base;
 	size_t mmap_length;
+	/* a) mmaping the whole file */
+	void *mmap_base;
+	/* b) using file cache */
 	struct file_cache *file_cache;
+	/* c) using small read() calls with MAIL_INDEX_OPEN_FLAG_SAVEONLY */
+	uoff_t read_offset;
+	buffer_t *read_buf;
 	/* mail_cache_map() increases this always. */
 	unsigned int remap_counter;
 
@@ -169,6 +174,7 @@ struct mail_cache {
 	unsigned int hdr_modified:1;
 	unsigned int field_header_write_pending:1;
 	unsigned int compressing:1;
+	unsigned int map_with_read:1;
 };
 
 struct mail_cache_loop_track {
diff --git a/src/lib-index/mail-cache.c b/src/lib-index/mail-cache.c
index 1ea3d1402f..8505f1225d 100644
--- a/src/lib-index/mail-cache.c
+++ b/src/lib-index/mail-cache.c
@@ -7,6 +7,7 @@
 #include "nfs-workarounds.h"
 #include "file-cache.h"
 #include "mmap-util.h"
+#include "read-full.h"
 #include "write-full.h"
 #include "mail-cache-private.h"
 
@@ -264,17 +265,87 @@ static bool mail_cache_verify_header(struct mail_cache *cache,
 	return TRUE;
 }
 
+static int
+mail_cache_map_finish(struct mail_cache *cache, uoff_t offset, size_t size,
+		      const void *data, bool copy_hdr)
+{
+	if (offset == 0) {
+		const struct mail_cache_header *hdr = data;
+
+		if (!mail_cache_verify_header(cache, hdr)) {
+			cache->need_compress_file_seq =
+				!MAIL_CACHE_IS_UNUSABLE(cache) &&
+				cache->hdr->file_seq != 0 ?
+				cache->hdr->file_seq : 0;
+			return -1;
+		}
+		if (!copy_hdr)
+			cache->hdr = data;
+		else {
+			memcpy(&cache->hdr_ro_copy, data,
+			       sizeof(cache->hdr_ro_copy));
+			cache->hdr = &cache->hdr_ro_copy;
+		}
+		mail_cache_update_need_compress(cache);
+	}
+
+	if (offset + size > cache->mmap_length)
+		return 0;
+	return 1;
+}
+
+static int
+mail_cache_map_with_read(struct mail_cache *cache, size_t offset, size_t size,
+			 const void **data_r)
+{
+	void *data;
+	ssize_t ret;
+
+	if (cache->read_buf == NULL) {
+		cache->read_buf =
+			buffer_create_dynamic(default_pool, size);
+	} else if (cache->read_offset <= offset &&
+		   cache->read_offset + cache->read_buf->used >= offset+size) {
+		/* already mapped */
+		*data_r = CONST_PTR_OFFSET(cache->mmap_base,
+					   offset - cache->read_offset);
+		return mail_cache_map_finish(cache, offset, size, *data_r, TRUE);
+	} else {
+		buffer_set_used_size(cache->read_buf, 0);
+	}
+	data = buffer_append_space_unsafe(cache->read_buf, size);
+	ret = pread(cache->fd, data, size, offset);
+	if (ret < 0) {
+		if (errno != ESTALE)
+			mail_cache_set_syscall_error(cache, "read()");
+
+		buffer_set_used_size(cache->read_buf, 0);
+		cache->hdr = NULL;
+		cache->mmap_length = 0;
+		return -1;
+	}
+	buffer_set_used_size(cache->read_buf, ret);
+
+	cache->read_offset = offset;
+	cache->mmap_length = offset + size;
+
+	*data_r = data;
+	return mail_cache_map_finish(cache, offset, size, data, TRUE);
+}
+
 int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
 		   const void **data_r)
 {
 	const void *data;
 	ssize_t ret;
 
-	cache->remap_counter++;
-
 	if (size == 0)
 		size = sizeof(struct mail_cache_header);
 
+	cache->remap_counter++;
+	if (cache->map_with_read)
+		return mail_cache_map_with_read(cache, offset, size, data_r);
+
 	if (cache->file_cache != NULL) {
 		cache->hdr = NULL;
 
@@ -294,28 +365,9 @@ int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
 
 		data = file_cache_get_map(cache->file_cache,
 					  &cache->mmap_length);
-
-		if (offset == 0) {
-			if (!mail_cache_verify_header(cache, data)) {
-				cache->need_compress_file_seq =
-					!MAIL_CACHE_IS_UNUSABLE(cache) &&
-					cache->hdr->file_seq != 0 ?
-					cache->hdr->file_seq : 0;
-				return -1;
-			}
-			memcpy(&cache->hdr_ro_copy, data,
-			       sizeof(cache->hdr_ro_copy));
-		}
-		cache->hdr = &cache->hdr_ro_copy;
-		if (offset == 0)
-			mail_cache_update_need_compress(cache);
-
-		if (offset > cache->mmap_length) {
-			*data_r = NULL;
-			return 0;
-		}
-		*data_r = CONST_PTR_OFFSET(data, offset);
-		return offset + size > cache->mmap_length ? 0 : 1;
+		*data_r = offset > cache->mmap_length ? NULL :
+			CONST_PTR_OFFSET(data, offset);
+		return mail_cache_map_finish(cache, offset, size, *data_r, TRUE);
 	}
 
 	if (offset < cache->mmap_length &&
@@ -348,24 +400,9 @@ int mail_cache_map(struct mail_cache *cache, size_t offset, size_t size,
 		mail_cache_set_syscall_error(cache, "mmap()");
 		return -1;
 	}
-
-	if (!mail_cache_verify_header(cache, cache->mmap_base)) {
-		cache->need_compress_file_seq =
-			!MAIL_CACHE_IS_UNUSABLE(cache) &&
-			cache->hdr->file_seq != 0 ?
-			cache->hdr->file_seq : 0;
-		return -1;
-	}
-
-	cache->hdr = cache->mmap_base;
-	if (offset == 0)
-		mail_cache_update_need_compress(cache);
-	if (offset > cache->mmap_length) {
-		*data_r = NULL;
-		return 0;
-	}
-	*data_r = CONST_PTR_OFFSET(cache->mmap_base, offset);
-	return offset + size > cache->mmap_length ? 0 : 1;
+	*data_r = offset > cache->mmap_length ? NULL :
+		CONST_PTR_OFFSET(cache->mmap_base, offset);
+	return mail_cache_map_finish(cache, offset, size, *data_r, FALSE);
 }
 
 static int mail_cache_try_open(struct mail_cache *cache)
@@ -437,6 +474,8 @@ static struct mail_cache *mail_cache_alloc(struct mail_index *index)
 	if (!MAIL_INDEX_IS_IN_MEMORY(index) &&
 	    (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) != 0)
 		cache->file_cache = file_cache_new(-1);
+	cache->map_with_read =
+		(cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0;
 
 	cache->ext_id =
 		mail_index_ext_register(index, "cache", 0,
diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h
index 89723cef89..1082bb7fbe 100644
--- a/src/lib-index/mail-index.h
+++ b/src/lib-index/mail-index.h
@@ -27,7 +27,10 @@ enum mail_index_open_flags {
 	MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS	= 0x100,
 	/* If we run out of disk space, fail modifications instead of moving
 	   indexes to memory. */
-	MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY	= 0x200
+	MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY	= 0x200,
+	/* We're only going to save new messages to the index.
+	   Avoid unnecessary reads. */
+	MAIL_INDEX_OPEN_FLAG_SAVEONLY		= 0x400
 };
 
 enum mail_index_header_compat_flags {