We didn't notice cache compression immediately in other processes. Also some

other bugfixes.

--HG--
branch : HEAD

diff --git a/src/lib-index/mail-cache.c b/src/lib-index/mail-cache.c
index 00c85d92e05c5c49a6e5a2a919c439c677a9479a..f7e67323a3994b9c7bb3c6a139d7f2fa7271035f 100644 (file)
--- a/src/lib-index/mail-cache.c
+++ b/src/lib-index/mail-cache.c
@@ -42,6 +42,8 @@
 
 struct mail_cache_header {
        uint32_t indexid;
+       uint32_t sync_id;
+
        uint32_t continued_record_count;
 
        uint32_t used_file_size;
@@ -70,6 +72,7 @@ struct mail_cache {
        void *mmap_base;
        size_t mmap_length;
        uint32_t used_file_size;
+       uint32_t sync_id;
 
        struct mail_cache_header *header;
 
@@ -249,6 +252,7 @@ static int mmap_verify_header(struct mail_cache *cache)
        if (cache->mmap_length < sizeof(struct mail_cache_header))
                return mail_cache_set_corrupted(cache, "File too small");
        cache->header = hdr = cache->mmap_base;
+       cache->sync_id = hdr->sync_id;
 
        if (cache->header->indexid != cache->index->indexid) {
                /* index id changed */
@@ -290,25 +294,33 @@ static int mmap_verify_header(struct mail_cache *cache)
 static int mmap_update_nocheck(struct mail_cache *cache,
                               size_t offset, size_t size)
 {
-       if (cache->header != NULL &&
-           cache->header->indexid != cache->index->indexid) {
-               /* indexid changed, most likely it was rebuilt.
-                  try reopening. */
+       struct stat st;
+
+       /* if sync id has changed, the file has to be reopened.
+          note that if main index isn't locked, it may change again */
+       if (cache->sync_id != cache->index->cache_sync_id &&
+           cache->mmap_base != NULL) {
                if (!mail_cache_file_reopen(cache))
                        return -1;
-
-               /* force mmap refresh */
-               size = 0;
        }
 
-       if (size != 0 && offset < cache->mmap_length &&
-           size <= cache->mmap_length - offset) {
+       if (offset < cache->mmap_length &&
+           size <= cache->mmap_length - offset &&
+           !cache->mmap_refresh) {
                /* already mapped */
-               if (!cache->mmap_refresh)
+               if (size != 0 || cache->anon_mmap)
                        return 1;
 
-               cache->mmap_refresh = FALSE;
+               /* requesting the whole file - see if we need to
+                  re-mmap */
+               if (fstat(cache->fd, &st) < 0) {
+                       mail_cache_set_syscall_error(cache, "fstat()");
+                       return -1;
+               }
+               if ((uoff_t)st.st_size == cache->mmap_length)
+                       return 1;
        }
+       cache->mmap_refresh = FALSE;
 
        if (cache->anon_mmap)
                return 1;
@@ -346,14 +358,28 @@ static int mmap_update_nocheck(struct mail_cache *cache,
 
 static int mmap_update(struct mail_cache *cache, size_t offset, size_t size)
 {
-       int ret;
+       int synced, ret;
 
-       ret = mmap_update_nocheck(cache, offset, size);
-       if (ret > 0)
-               return TRUE;
-       if (ret < 0)
-               return FALSE;
-       return mmap_verify_header(cache);
+       for (synced = FALSE;; synced = TRUE) {
+               ret = mmap_update_nocheck(cache, offset, size);
+               if (ret > 0)
+                       return TRUE;
+               if (ret < 0)
+                       return FALSE;
+
+               if (!mmap_verify_header(cache))
+                       return FALSE;
+
+               /* see if cache file was rebuilt - do it only once to avoid
+                  infinite looping */
+               if (cache->header->sync_id == cache->index->cache_sync_id ||
+                   synced)
+                       break;
+
+               if (!mail_cache_file_reopen(cache))
+                       return FALSE;
+       }
+       return TRUE;
 }
 
 static int mail_cache_open_and_verify(struct mail_cache *cache, int silent)
@@ -475,6 +501,7 @@ int mail_cache_open_or_create(struct mail_index *index)
 
        memset(&hdr, 0, sizeof(hdr));
        hdr.indexid = index->indexid;
+       hdr.sync_id = index->cache_sync_id;
        hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
 
        cache = i_new(struct mail_cache, 1);
@@ -688,6 +715,8 @@ static int mail_cache_copy(struct mail_cache *cache, int fd)
 
        /* update header */
        hdr->indexid = cache->index->indexid;
+       hdr->sync_id = cache->sync_id = cache->index->cache_sync_id =
+               ++cache->index->header->cache_sync_id;
        hdr->used_file_size = uint32_to_nbo(offset);
        hdr->used_fields = used_fields;
        hdr->field_usage_start = uint32_to_nbo(ioloop_time);
@@ -710,12 +739,19 @@ int mail_cache_compress(struct mail_cache *cache)
 
        i_assert(cache->trans_ctx == NULL);
 
+       if (cache->anon_mmap)
+               return TRUE;
+
        if (!cache->index->set_lock(cache->index, MAIL_LOCK_EXCLUSIVE))
                return FALSE;
 
        if (mail_cache_lock(cache, TRUE) <= 0)
                return FALSE;
 
+#ifdef DEBUG
+       i_warning("Compressing cache file %s", cache->filepath);
+#endif
+
        fd = file_dotlock_open(cache->filepath, NULL, MAIL_CACHE_LOCK_TIMEOUT,
                               MAIL_CACHE_LOCK_STALE_TIMEOUT, NULL, NULL);
        if (fd == -1) {
@@ -759,8 +795,12 @@ int mail_cache_truncate(struct mail_cache *cache)
        struct mail_cache_header hdr;
        int ret, fd;
 
+       i_assert(cache->index->lock_type == MAIL_LOCK_EXCLUSIVE);
+
        memset(&hdr, 0, sizeof(hdr));
        hdr.indexid = cache->index->indexid;
+       hdr.sync_id = cache->sync_id = cache->index->cache_sync_id
+               =++cache->index->header->cache_sync_id;
        hdr.used_file_size = uint32_to_nbo(sizeof(hdr));
        cache->used_file_size = sizeof(hdr);
 
@@ -840,9 +880,19 @@ int mail_cache_lock(struct mail_cache *cache, int nonblock)
        }
 
        if (ret > 0) {
-               if (!mmap_verify_header(cache)) {
+               if (!mmap_update(cache, 0, 0)) {
                        (void)mail_cache_unlock(cache);
-                       ret = -1;
+                       return -1;
+               }
+               if (cache->sync_id != cache->index->cache_sync_id) {
+                       /* we have the cache file locked and sync_id still
+                          doesn't match. it means we crashed between updating
+                          cache file and updating sync_id in index header.
+                          just update the sync_ids so they match. */
+                       i_warning("Updating broken sync_id in cache file %s",
+                                 cache->filepath);
+                       cache->sync_id = cache->header->sync_id =
+                               cache->index->cache_sync_id;
                }
        }
        return ret;

diff --git a/src/lib-index/mail-index-file.c b/src/lib-index/mail-index-file.c
index e86b6589817ab7d313d257d681ff2bd5bbe5b3b5..87fb733f990f9d089246c2101336e86741c3e5b6 100644 (file)
--- a/src/lib-index/mail-index-file.c
+++ b/src/lib-index/mail-index-file.c
@@ -142,7 +142,7 @@ int mail_index_compress(struct mail_index *index)
                        return FALSE;
                }
 
-               index->header->sync_id++;
+               index->header->master_sync_id++;
                if (!mail_index_mmap_update(index))
                        return FALSE;
        }
@@ -162,7 +162,7 @@ int mail_index_compress(struct mail_index *index)
 
        index->header->header_size = sizeof(struct mail_index_header);
        index->header->used_file_size += diff;
-       index->header->sync_id++;
+       index->header->master_sync_id++;
  
        if (!mail_index_fmdatasync(index, index->mmap_used_length))
                return FALSE;
@@ -201,7 +201,7 @@ int mail_index_truncate(struct mail_index *index)
                if (ftruncate(index->fd, (off_t)index->mmap_full_length) < 0)
                        return index_set_syscall_error(index, "ftruncate()");
 
-               index->header->sync_id++;
+               index->header->master_sync_id++;
        }
 
        return TRUE;

diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c
index c6a6d662138a31c0aea0e9e3d999e3e604af3ed5..c4b4ec17910592c589e34b8d15129f8854a6cdea 100644 (file)
--- a/src/lib-index/mail-index.c
+++ b/src/lib-index/mail-index.c
@@ -74,7 +74,9 @@ static int mmap_verify(struct mail_index *index)
                return FALSE;
        }
 
-       index->sync_id = hdr->sync_id;
+       index->master_sync_id = hdr->master_sync_id;
+       index->cache_sync_id = hdr->cache_sync_id;
+       index->log_sync_id = hdr->log_sync_id;
        index->sync_stamp = hdr->sync_stamp;
        index->sync_size = hdr->sync_size;
        index->mmap_used_length = hdr->used_file_size;
@@ -89,6 +91,9 @@ int mail_index_mmap_update(struct mail_index *index)
        if (index->mmap_base != NULL) {
                index->header = (struct mail_index_header *) index->mmap_base;
 
+               index->cache_sync_id = index->header->cache_sync_id;
+               index->log_sync_id = index->header->log_sync_id;
+
                if (index->mmap_invalidate) {
                        if (msync(index->mmap_base,
                                  index->mmap_used_length,
@@ -99,7 +104,7 @@ int mail_index_mmap_update(struct mail_index *index)
                }
 
                /* make sure file size hasn't changed */
-               if (index->header->sync_id == index->sync_id) {
+               if (index->header->master_sync_id == index->master_sync_id) {
                        index->mmap_used_length = index->header->used_file_size;
                        if (index->mmap_used_length > index->mmap_full_length) {
                                i_panic("Index file size was grown without "
@@ -647,7 +652,7 @@ static int mail_index_grow(struct mail_index *index)
 
        /* file size changed, let others know about it too by changing
           sync_id in header. */
-       index->header->sync_id++;
+       index->header->master_sync_id++;
 
        if (!mail_index_mmap_update(index))
                return FALSE;

diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h
index f234028fa891c6469887b00aa7f26e675dbe7770..c15629b1d38baad8dc358d634879d672db82e8c7 100644 (file)
--- a/src/lib-index/mail-index.h
+++ b/src/lib-index/mail-index.h
@@ -115,10 +115,12 @@ struct mail_index_header {
        uint8_t compat_data[4];
 
        uint32_t indexid;
-
        uint32_t used_file_size;
-       uint32_t sync_id; /* re-mmap() when changed, required only
-                            if file size is shrinked */
+
+       /* file needs to be reopened if sync_ids change. */
+       uint32_t master_sync_id;
+       uint32_t cache_sync_id;
+       uint32_t log_sync_id;
 
        uint32_t flags;
 
@@ -281,7 +283,7 @@ struct mail_index {
        char *mailbox_path; /* file/directory for mailbox location */
        char *control_dir; /* destination for control files */
        unsigned int indexid;
-       unsigned int sync_id;
+       unsigned int master_sync_id, cache_sync_id, log_sync_id;
 
         /* updated whenever exclusive lock is set/unset */
        unsigned int excl_lock_counter;
@@ -358,11 +360,11 @@ struct mail_index {
    members.. */
 #define MAIL_INDEX_PRIVATE_FILL \
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
-       0, 0, 0, 0, { 0, 0, 0 }, 0, 0, 0, \
+       0, 0, 0, 0, 0, 0, { 0, 0, 0 }, 0, \
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \
-       0, 0, 0, 0, 0, 0, 0
+       0, 0, 0, 0, 0, 0, 0, 0, 0
 #endif
 
 /* defaults - same as above but prefixed with mail_index_. */