fixes

--HG--
branch : HEAD

diff --git a/src/lib-index/mail-index-lock.c b/src/lib-index/mail-index-lock.c
index b3ef7e3d964adf276789666957ed099eb4c73702..2d0195e04428e03f9718ed285d018e2744fc9f58 100644 (file)
--- a/src/lib-index/mail-index-lock.c
+++ b/src/lib-index/mail-index-lock.c
@@ -123,7 +123,7 @@ static int mail_index_lock_mprotect(struct mail_index *index, int lock_type)
                        lock_type == F_WRLCK ? (PROT_READ|PROT_WRITE) :
                        PROT_READ;
                if (mprotect(index->map->mmap_base,
-                            index->map->file_size, prot) < 0) {
+                            index->map->mmap_size, prot) < 0) {
                        mail_index_set_syscall_error(index, "mprotect()");
                        return -1;
                }

diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h
index dcb04c36b1f36a56462eee959a7e253c10f3e627..342ee7e5c6b2789b22b6aac4464e84af91f7f197 100644 (file)
--- a/src/lib-index/mail-index-private.h
+++ b/src/lib-index/mail-index-private.h
@@ -41,7 +41,7 @@ struct mail_index_map {
        unsigned int records_count;
 
        void *mmap_base;
-       size_t file_size, file_used_size;
+       size_t mmap_size, mmap_used_size;
 
        buffer_t *buffer;
 

diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c
index 03a1287e5e8ab3c75c7a10149f5d73a598acd85f..1db0c3f083ca6a16272617e7cd9bd7f8d02b368b 100644 (file)
--- a/src/lib-index/mail-index-sync-update.c
+++ b/src/lib-index/mail-index-sync-update.c
@@ -98,34 +98,39 @@ static void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx,
 static int mail_index_grow(struct mail_index *index, unsigned int count)
 {
        struct mail_index_map *map = index->map;
-       size_t size, file_used_size;
        unsigned int records_count;
+       size_t size;
 
        if (MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
                (void)buffer_append_space_unsafe(map->buffer,
                        count * sizeof(struct mail_index_record));
+               map->records = buffer_get_modifyable_data(map->buffer, NULL);
                return 0;
        }
 
+       size = map->hdr->header_size +
+               (map->records_count + count) * sizeof(struct mail_index_record);
+       if (size <= map->mmap_size)
+               return 0;
+
        /* when we grow fast, do it exponentially */
        if (count < index->last_grow_count)
                count = index->last_grow_count;
        count = nearest_power(count);
        index->last_grow_count = count;
 
-       size = map->file_used_size + count * sizeof(struct mail_index_record);
+       size = map->hdr->header_size +
+               (map->records_count + count) * sizeof(struct mail_index_record);
        if (file_set_size(index->fd, (off_t)size) < 0)
                return mail_index_set_syscall_error(index, "file_set_size()");
 
        records_count = map->records_count;
-       file_used_size = map->file_used_size;
 
        if (mail_index_map(index, TRUE) <= 0)
                return -1;
 
-       i_assert(map->file_size >= size);
+       i_assert(map->mmap_size >= size);
        map->records_count = records_count;
-       map->file_used_size = file_used_size;
        return 0;
 }
 
@@ -135,14 +140,10 @@ static int mail_index_sync_appends(struct mail_index_update_ctx *ctx,
 {
        struct mail_index_map *map = ctx->index->map;
        unsigned int i;
-       size_t space;
        uint32_t next_uid;
 
-       space = (map->file_size - map->file_used_size) / sizeof(*appends);
-       if (space < count) {
-               if (mail_index_grow(ctx->index, count) < 0)
-                       return -1;
-       }
+       if (mail_index_grow(ctx->index, count) < 0)
+               return -1;
 
        next_uid = ctx->hdr.next_uid;
        for (i = 0; i < count; i++) {
@@ -166,7 +167,6 @@ static int mail_index_sync_appends(struct mail_index_update_ctx *ctx,
        memcpy(map->records + map->records_count, appends,
               count * sizeof(*appends));
        map->records_count += count;
-       map->file_used_size += count * sizeof(struct mail_index_record);
        return 0;
 }
 
@@ -254,8 +254,6 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
                dest_idx += count;
 
                map->records_count = dest_idx;
-               map->file_used_size = index->hdr->header_size +
-                       map->records_count * sizeof(struct mail_index_record);
        }
 
        ret = 0;
@@ -269,8 +267,11 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
        ctx.hdr.log_file_offset = file_offset;
 
        if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
+               map->mmap_used_size = index->hdr->header_size +
+                       map->records_count * sizeof(struct mail_index_record);
+
                memcpy(map->mmap_base, &ctx.hdr, sizeof(ctx.hdr));
-               if (msync(map->mmap_base, map->file_used_size, MS_SYNC) < 0) {
+               if (msync(map->mmap_base, map->mmap_used_size, MS_SYNC) < 0) {
                        mail_index_set_syscall_error(index, "msync()");
                        ret = -1;
                }

diff --git a/src/lib-index/mail-index.c b/src/lib-index/mail-index.c
index f6e6924dcaf56c259ef0f8feb5bb4946bffe117a..71f74d375bcb3a049b4d8aa0f867c4c7c0eca42e 100644 (file)
--- a/src/lib-index/mail-index.c
+++ b/src/lib-index/mail-index.c
@@ -36,36 +36,11 @@ void mail_index_free(struct mail_index *index)
        i_free(index);
 }
 
-static int mail_index_check_quick_header(struct mail_index *index,
-                                        struct mail_index_map *map)
-{
-       if ((map->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
-               /* either a crash or we've already complained about it */
-               return -1;
-       }
-
-       if (map->file_used_size > map->file_size) {
-               map->records_count =
-                       (map->file_size - map->hdr->header_size) /
-                       sizeof(struct mail_index_record);
-               map->file_used_size = map->file_size;
-
-               mail_index_set_error(index, "Corrupted index file %s: "
-                                    "messages_count too large (%u > %u)",
-                                    index->filepath, map->hdr->messages_count,
-                                    map->records_count);
-               return 0;
-       }
-
-       return 1;
-}
-
 static int mail_index_check_header(struct mail_index *index,
                                   struct mail_index_map *map)
 {
        const struct mail_index_header *hdr = map->hdr;
        unsigned char compat_data[3];
-       int ret;
 
 #ifndef WORDS_BIGENDIAN
        compat_data[0] = MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
@@ -84,8 +59,10 @@ static int mail_index_check_header(struct mail_index *index,
                return -1;
        }
 
-       if ((ret = mail_index_check_quick_header(index, map)) <= 0)
-               return ret;
+       if ((map->hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
+               /* either a crash or we've already complained about it */
+               return -1;
+       }
 
        /* following some extra checks that only take a bit of CPU */
        if (hdr->uid_validity == 0) {
@@ -117,14 +94,14 @@ static void mail_index_map_clear(struct mail_index *index,
                map->buffer = NULL;
        } else if (map->mmap_base != NULL) {
                i_assert(map->buffer == NULL);
-               if (munmap(map->mmap_base, map->file_size) < 0)
+               if (munmap(map->mmap_base, map->mmap_size) < 0)
                        mail_index_set_syscall_error(index, "munmap()");
                map->mmap_base = NULL;
        }
 
        if (map->refcount > 0) {
-               map->file_size = 0;
-               map->file_used_size = 0;
+               map->mmap_size = 0;
+               map->mmap_used_size = 0;
                map->hdr = NULL;
                map->records = NULL;
                map->records_count = 0;
@@ -148,31 +125,92 @@ static void mail_index_unmap_forced(struct mail_index *index,
        mail_index_unmap(index, map);
 }
 
-static int mail_index_read_map(struct mail_index *index,
-                              struct mail_index_map *map)
+static int mail_index_mmap(struct mail_index *index, struct mail_index_map *map)
 {
-       struct stat st;
-       void *data;
-       size_t file_size;
-       int ret;
+       const struct mail_index_header *hdr;
+       unsigned int records_count;
 
-       if (fstat(index->fd, &st) < 0) {
-               if (errno == ESTALE)
-                       return 0;
-               mail_index_set_syscall_error(index, "fstat()");
+       map->mmap_base = index->lock_type != F_WRLCK ?
+               mmap_ro_file(index->fd, &map->mmap_size) :
+               mmap_rw_file(index->fd, &map->mmap_size);
+       if (map->mmap_base == MAP_FAILED) {
+               map->mmap_base = NULL;
+               mail_index_set_syscall_error(index, "mmap()");
                return -1;
        }
-       file_size = st.st_size;
 
-       if (map->buffer == NULL) {
-               map->buffer = buffer_create_dynamic(default_pool,
-                                                   file_size, (size_t)-1);
+       if (map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
+               mail_index_set_error(index, "Corrupted index file %s: "
+                                    "File too small (%"PRIuSIZE_T")",
+                                    index->filepath, map->mmap_size);
+               return 0;
+       }
+
+       hdr = map->mmap_base;
+       map->mmap_used_size = hdr->header_size +
+               hdr->messages_count * sizeof(struct mail_index_record);
+
+       if (map->mmap_used_size > map->mmap_size) {
+               records_count = (map->mmap_size - hdr->header_size) /
+                       sizeof(struct mail_index_record);
+               mail_index_set_error(index, "Corrupted index file %s: "
+                                    "messages_count too large (%u > %u)",
+                                    index->filepath, map->hdr->messages_count,
+                                    records_count);
+               return 0;
+       }
+
+       map->hdr = hdr;
+       if (map->hdr->header_size < sizeof(*map->hdr)) {
+               /* header smaller than ours, make a copy so our newer headers
+                  won't have garbage in them */
+               memcpy(&map->hdr_copy, map->hdr, map->hdr->header_size);
+               map->hdr = &map->hdr_copy;
        }
 
-       /* @UNSAFE */
-       buffer_set_used_size(map->buffer, 0);
-       data = buffer_append_space_unsafe(map->buffer, file_size);
-       ret = pread_full(index->fd, data, file_size, 0);
+       map->records = PTR_OFFSET(map->mmap_base, map->hdr->header_size);
+       map->records_count = map->hdr->messages_count;
+       return 1;
+}
+
+static int mail_index_read_map(struct mail_index *index,
+                              struct mail_index_map *map)
+{
+       struct mail_index_header hdr;
+       void *data;
+       ssize_t ret;
+       size_t pos, records_size;
+
+       do {
+               memset(&hdr, 0, sizeof(hdr));
+
+               ret = 1;
+               for (pos = 0; ret > 0 && pos < sizeof(hdr); ) {
+                       ret = pread(index->fd, PTR_OFFSET(&hdr, pos),
+                                   sizeof(hdr) - pos, pos);
+                       if (ret > 0)
+                               pos += ret;
+               }
+               if (ret < 0 || pos < MAIL_INDEX_HEADER_MIN_SIZE)
+                       break;
+
+               records_size = hdr.messages_count *
+                       sizeof(struct mail_index_record);
+
+               if (map->buffer == NULL) {
+                       map->buffer = buffer_create_dynamic(default_pool,
+                                                           records_size,
+                                                           (size_t)-1);
+               }
+
+               /* @UNSAFE */
+               buffer_set_used_size(map->buffer, 0);
+               data = buffer_append_space_unsafe(map->buffer, records_size);
+
+               ret = pread_full(index->fd, data, records_size,
+                                hdr.header_size);
+       } while (0);
+
        if (ret < 0) {
                if (errno == ESTALE)
                        return 0;
@@ -185,7 +223,11 @@ static int mail_index_read_map(struct mail_index *index,
                return -1;
        }
 
-       map->file_size = file_size;
+       map->records = data;
+       map->records_count = hdr.messages_count;
+
+       map->hdr_copy = hdr;
+       map->hdr = &map->hdr_copy;
        return 1;
 }
 
@@ -224,17 +266,19 @@ int mail_index_map(struct mail_index *index, int force)
 {
        struct mail_index_map *map;
        size_t used_size;
-       void *base;
        int ret;
 
-       if (index->map != NULL) {
+       if (index->map != NULL && MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
+               /* FIXME: can we avoid reading it? */
+               map = index->map;
+       } else if (index->map != NULL) {
                map = index->map;
 
                /* see if re-mmaping is needed (file has grown) */
                 used_size = map->hdr->header_size +
                        map->hdr->messages_count *
                        sizeof(struct mail_index_record);
-               if (map->file_size >= used_size && !force) {
+               if (map->mmap_size >= used_size && !force) {
                        /* update log file position in case it has changed */
                        map->log_file_seq = map->hdr->log_file_seq;
                        map->log_file_offset = map->hdr->log_file_offset;
@@ -242,7 +286,7 @@ int mail_index_map(struct mail_index *index, int force)
                }
 
                if (map->mmap_base != NULL) {
-                       if (munmap(map->mmap_base, map->file_size) < 0)
+                       if (munmap(map->mmap_base, map->mmap_size) < 0)
                                mail_index_set_syscall_error(index, "munmap()");
                        map->mmap_base = NULL;
                }
@@ -254,20 +298,10 @@ int mail_index_map(struct mail_index *index, int force)
        index->hdr = NULL;
        index->map = NULL;
 
-       /* make sure if we fail we don't try to access anything outside the
-          buffer */
-       map->file_size = 0;
-       map->file_used_size = 0;
-
        if (!index->mmap_disable) {
-               map->mmap_base = index->lock_type != F_WRLCK ?
-                       mmap_ro_file(index->fd, &map->file_size) :
-                       mmap_rw_file(index->fd, &map->file_size);
-               if (map->mmap_base == MAP_FAILED) {
-                       map->mmap_base = NULL;
-                       mail_index_set_syscall_error(index, "mmap()");
+               if ((ret = mail_index_mmap(index, map)) <= 0) {
                        mail_index_unmap_forced(index, map);
-                       return -1;
+                       return ret;
                }
        } else {
                if (mail_index_read_map_with_retry(index, map) < 0) {
@@ -276,30 +310,6 @@ int mail_index_map(struct mail_index *index, int force)
                }
        }
 
-       if (map->file_size < MAIL_INDEX_HEADER_MIN_SIZE) {
-               mail_index_set_error(index, "Corrupted index file %s: "
-                                    "File too small (%"PRIuSIZE_T")",
-                                    index->filepath, map->file_size);
-               mail_index_unmap_forced(index, map);
-               return 0;
-       }
-
-       base = !MAIL_INDEX_MAP_IS_IN_MEMORY(map) ? map->mmap_base :
-               buffer_get_modifyable_data(map->buffer, NULL);
-       map->hdr = base;
-
-       if (map->hdr->header_size < sizeof(*map->hdr)) {
-               /* header smaller than ours, make a copy so our newer headers
-                  won't have garbage in them */
-               memcpy(&map->hdr_copy, map->hdr, map->hdr->header_size);
-               map->hdr = &map->hdr_copy;
-       }
-
-       map->records = PTR_OFFSET(base, map->hdr->header_size);
-       map->records_count = map->hdr->messages_count;
-       map->file_used_size = map->hdr->header_size +
-               map->records_count * sizeof(struct mail_index_record);
-
        ret = mail_index_check_header(index, map);
        if (ret < 0) {
                mail_index_unmap_forced(index, map);
@@ -613,8 +623,10 @@ void mail_index_close(struct mail_index *index)
                index->log = NULL;
        }
 
-       mail_index_unmap(index, index->map);
-       index->map = NULL;
+       if (index->map != NULL) {
+               mail_index_unmap(index, index->map);
+               index->map = NULL;
+       }
 
        if (index->fd != -1) {
                if (close(index->fd) < 0)
@@ -623,6 +635,7 @@ void mail_index_close(struct mail_index *index)
        }
 
        i_free(index->copy_lock_path);
+       index->copy_lock_path = NULL;
        i_free(index->filepath);
        index->filepath = NULL;
 

diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c
index a5646c5fd723e667df69410bd4edaa3d4832be26..60007ef2b056ff87d0fc839af2555578af60c33d 100644 (file)
--- a/src/lib-storage/index/index-storage.c
+++ b/src/lib-storage/index/index-storage.c
@@ -363,7 +363,8 @@ void index_storage_mailbox_free(struct mailbox *box)
        struct index_mailbox *ibox = (struct index_mailbox *) box;
 
        /* make sure we're unlocked */
-       mail_index_view_unlock(ibox->view);
+       if (ibox->view != NULL)
+               mail_index_view_unlock(ibox->view);
 
        index_mailbox_check_remove_all(ibox);
        if (ibox->index != NULL)
@@ -419,7 +420,8 @@ int mail_storage_set_index_error(struct index_mailbox *ibox)
                break;
        }
 
-       mail_index_view_unlock(ibox->view);
+       if (ibox->view != NULL)
+               mail_index_view_unlock(ibox->view);
        mail_index_reset_error(ibox->index);
        return FALSE;
 }