fixes

--HG--
branch : HEAD

diff --git a/src/lib-index/mail-index-lock.c b/src/lib-index/mail-index-lock.c
index 2d0195e04428e03f9718ed285d018e2744fc9f58..15a38f301771dad5d200a87c6c920243fd9bd8af 100644 (file)
--- a/src/lib-index/mail-index-lock.c
+++ b/src/lib-index/mail-index-lock.c
@@ -113,17 +113,16 @@ static int mail_index_has_changed(struct mail_index *index)
        }
 }
 
-static int mail_index_lock_mprotect(struct mail_index *index, int lock_type)
+int mail_index_map_lock_mprotect(struct mail_index *index,
+                                struct mail_index_map *map, int lock_type)
 {
        int prot;
 
-       if (index->map != NULL &&
-           !MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) {
+       if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) {
                prot = lock_type == F_UNLCK ? PROT_NONE :
                        lock_type == F_WRLCK ? (PROT_READ|PROT_WRITE) :
                        PROT_READ;
-               if (mprotect(index->map->mmap_base,
-                            index->map->mmap_size, prot) < 0) {
+               if (mprotect(map->mmap_base, map->mmap_size, prot) < 0) {
                        mail_index_set_syscall_error(index, "mprotect()");
                        return -1;
                }
@@ -131,6 +130,14 @@ static int mail_index_lock_mprotect(struct mail_index *index, int lock_type)
        return 0;
 }
 
+static int mail_index_lock_mprotect(struct mail_index *index, int lock_type)
+{
+       if (index->map == NULL)
+               return 0;
+
+       return mail_index_map_lock_mprotect(index, index->map, lock_type);
+}
+
 static int mail_index_lock(struct mail_index *index, int lock_type,
                           unsigned int timeout_secs, int update_index,
                           unsigned int *lock_id_r)
@@ -142,14 +149,23 @@ static int mail_index_lock(struct mail_index *index, int lock_type,
        if (lock_type == F_RDLCK && index->lock_type != F_UNLCK) {
                index->shared_lock_count++;
                *lock_id_r = index->lock_id;
-               return 1;
-       }
-       if (lock_type == F_WRLCK && index->lock_type == F_WRLCK) {
+               ret = 1;
+       } else if (lock_type == F_WRLCK && index->lock_type == F_WRLCK) {
                index->excl_lock_count++;
                *lock_id_r = index->lock_id + 1;
-               return 1;
+               ret = 1;
+       } else {
+               ret = 0;
        }
 
+       if (update_index && index->excl_lock_count == 0) {
+               if (mail_index_has_changed(index) < 0)
+                       return -1;
+       }
+
+       if (ret > 0)
+               return 1;
+
        if (index->fcntl_locks_disable) {
                /* FIXME: exclusive locking will rewrite the index file every
                   time. shouldn't really be needed.. reading doesn't require
@@ -169,11 +185,6 @@ static int mail_index_lock(struct mail_index *index, int lock_type,
                return 1;
        }
 
-       if (update_index) {
-               if (mail_index_has_changed(index) < 0)
-                       return -1;
-       }
-
        ret = file_wait_lock_full(index->fd, lock_type, timeout_secs,
                                  NULL, NULL);
        if (ret <= 0) {

diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h
index 8beb3ea48da61c853c4aee386b68332705b1a8da..b6dbef10ba4e02afd7cfe16c38d5420d03cdb5e1 100644 (file)
--- a/src/lib-index/mail-index-private.h
+++ b/src/lib-index/mail-index-private.h
@@ -103,6 +103,8 @@ int mail_index_lock_exclusive(struct mail_index *index,
 void mail_index_unlock(struct mail_index *index, unsigned int lock_id);
 /* Returns 1 if given lock_id is valid, 0 if not. */
 int mail_index_is_locked(struct mail_index *index, unsigned int lock_id);
+int mail_index_map_lock_mprotect(struct mail_index *index,
+                                struct mail_index_map *map, int lock_type);
 
 /* Map index file to memory, replacing the previous mapping for index.
    Returns 1 = ok, 0 = corrupted, -1 = error. If index needs fscking, it

diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c
index e4408c68ed2e4963ab0c21f6e769bac4f59d1699..d78336ec51cfd6365ada9c9327fead9bf57a6dc0 100644 (file)
--- a/src/lib-index/mail-index-sync.c
+++ b/src/lib-index/mail-index-sync.c
@@ -344,6 +344,16 @@ int mail_index_sync_next(struct mail_index_sync_ctx *ctx,
                   it's quite unlikely this expunge was caused by some bug. */
                uint32_t uid1, uid2;
 
+               if (next_exp->seq1 > ctx->view->map->records_count ||
+                   next_exp->seq2 > ctx->view->map->records_count) {
+                       mail_transaction_log_view_set_corrupted(
+                               ctx->view->log_view, "Expunge range %u..%u "
+                               "larger than message count %u",
+                               next_exp->seq1, next_exp->seq2,
+                               ctx->view->map->records_count);
+                       return -1;
+               }
+
                if (mail_index_lookup_uid(ctx->view, next_exp->seq1, &uid1) < 0)
                        return -1;
                if (mail_index_lookup_uid(ctx->view, next_exp->seq2, &uid2) < 0)

diff --git a/src/lib-index/mail-index-view-private.h b/src/lib-index/mail-index-view-private.h
index 8cd675ad8eab01a11d9a0f02f917f68d718b2bf8..024d8829cdc286792332067765a6c332d8366e0d 100644 (file)
--- a/src/lib-index/mail-index-view-private.h
+++ b/src/lib-index/mail-index-view-private.h
@@ -7,6 +7,7 @@ struct mail_index_view {
        struct mail_index *index;
         struct mail_transaction_log_view *log_view;
 
+       unsigned int indexid;
        struct mail_index_map *map;
 
        uint32_t log_file_seq;
@@ -19,6 +20,7 @@ struct mail_index_view {
        unsigned int inconsistent:1;
        unsigned int syncing:1;
        unsigned int external:1;
+       unsigned int map_protected:1;
 };
 
 int mail_index_view_lock(struct mail_index_view *view);

diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c
index 4e0dafa31f6884fe3987fa666f42d3186983ea70..bf72d808a21cd36f984ca8edbb870692b425b42a 100644 (file)
--- a/src/lib-index/mail-index-view-sync.c
+++ b/src/lib-index/mail-index-view-sync.c
@@ -160,6 +160,7 @@ static int sync_append(const struct mail_index_record *rec, void *context)
 
        map->records_count++;
        map->hdr_copy.messages_count++;
+       map->hdr_copy.next_uid = rec->uid+1;
 
        mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags);
        mail_index_header_update_lowwaters(&map->hdr_copy, rec);

diff --git a/src/lib-index/mail-index-view.c b/src/lib-index/mail-index-view.c
index 59ff2fff7262be56773dbc5d071b12b99b65b6cd..979736b1ffd7e7af6f76eac292707ea06c64627c 100644 (file)
--- a/src/lib-index/mail-index-view.c
+++ b/src/lib-index/mail-index-view.c
@@ -14,6 +14,7 @@ struct mail_index_view *mail_index_view_open(struct mail_index *index)
        view->index = index;
        view->log_view = mail_transaction_log_view_open(index->log);
 
+       view->indexid = index->indexid;
        view->map = index->map;
        view->map->refcount++;
 
@@ -33,8 +34,27 @@ void mail_index_view_close(struct mail_index_view *view)
        i_free(view);
 }
 
+static int mail_index_view_map_protect(struct mail_index_view *view)
+{
+       /* not head mapping, no need to lock */
+       if (!view->map_protected) {
+               if (mail_index_map_lock_mprotect(view->index, view->map,
+                                                F_RDLCK) < 0)
+                       return -1;
+               view->map_protected = TRUE;
+       }
+       return 0;
+}
+
 int mail_index_view_lock_head(struct mail_index_view *view, int update_index)
 {
+       unsigned int lock_id;
+
+       if (view->map != view->index->map) {
+               if (mail_index_view_map_protect(view) < 0)
+                       return -1;
+       }
+
        if (!mail_index_is_locked(view->index, view->lock_id)) {
                if (mail_index_lock_shared(view->index, update_index,
                                           &view->lock_id) < 0)
@@ -45,13 +65,17 @@ int mail_index_view_lock_head(struct mail_index_view *view, int update_index)
                        return -1;
                }
 
-               if (view->index->indexid != view->map->hdr->indexid) {
+               if (view->index->indexid != view->indexid) {
                        /* index was rebuilt */
                        view->inconsistent = TRUE;
                        return -1;
                }
        } else if (update_index) {
-               // FIXME: check if we need to reopen it!
+               if (mail_index_lock_shared(view->index, TRUE, &lock_id) < 0)
+                       return -1;
+
+               mail_index_unlock(view->index, view->lock_id);
+               view->lock_id = lock_id;
        }
 
        return 0;
@@ -63,7 +87,8 @@ int mail_index_view_lock(struct mail_index_view *view)
                return -1;
 
        if (view->map != view->index->map) {
-               /* not head mapping, no need to lock */
+               if (mail_index_view_map_protect(view) < 0)
+                       return -1;
                return 0;
        }
 
@@ -72,6 +97,12 @@ int mail_index_view_lock(struct mail_index_view *view)
 
 void mail_index_view_unlock(struct mail_index_view *view)
 {
+       if (view->map_protected) {
+               (void)mail_index_map_lock_mprotect(view->index, view->map,
+                                                  F_UNLCK);
+               view->map_protected = FALSE;
+       }
+
        if (view->lock_id != 0) {
                mail_index_unlock(view->index, view->lock_id);
                view->lock_id = 0;

diff --git a/src/lib-index/mail-transaction-log-view.c b/src/lib-index/mail-transaction-log-view.c
index 56e9c9d275868d1e883cea1b66b453e7454bb29c..7f51ea7d73937946ea79f171b5a83770ce024bec 100644 (file)
--- a/src/lib-index/mail-transaction-log-view.c
+++ b/src/lib-index/mail-transaction-log-view.c
@@ -90,10 +90,10 @@ mail_transaction_log_view_set(struct mail_transaction_log_view *view,
        ret = mail_transaction_log_file_find(view->log, min_file_seq, &file);
        if (ret <= 0) {
                if (ret == 0 &&
-                   min_file_seq == view->log->tail->hdr.file_seq-1 &&
+                   min_file_seq == view->log->tail->hdr.prev_file_seq &&
                    min_file_offset == view->log->tail->hdr.prev_file_offset) {
                        /* we can skip this */
-                       min_file_seq++;
+                       min_file_seq = view->log->tail->hdr.file_seq;
                        min_file_offset =
                                sizeof(struct mail_transaction_log_header);
                        ret = mail_transaction_log_file_find(view->log,
@@ -197,19 +197,26 @@ static int log_view_get_next(struct mail_transaction_log_view *view,
                             const void **data_r)
 {
        const struct mail_transaction_header *hdr;
-       struct mail_transaction_log_file *file = view->file;
+       struct mail_transaction_log_file *file;
        const struct mail_transaction_type_map *type_rec;
        const void *data;
        unsigned int record_size;
        size_t file_size;
 
-       view->prev_file_seq = file->hdr.file_seq;
-       view->prev_file_offset = view->file_offset;
+       for (;;) {
+               file = view->file;
+
+               view->prev_file_seq = file->hdr.file_seq;
+               view->prev_file_offset = view->file_offset;
+
+               if (view->file_offset != file->hdr.used_size)
+                       break;
 
-       if (view->file_offset == file->hdr.used_size) {
                view->file = file->next;
                view->file_offset = sizeof(struct mail_transaction_log_header);
-               return 0;
+
+               if (view->file == NULL)
+                       return 0;
        }
 
        data = buffer_get_data(file->buffer, &file_size);

diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c
index f1faaaf868567d0121d4afac50767552863defa3..fdbff0b89c95cb9f6ccf0cfbddebdf4dc0be3ae8 100644 (file)
--- a/src/lib-index/mail-transaction-log.c
+++ b/src/lib-index/mail-transaction-log.c
@@ -63,7 +63,7 @@ mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file,
 
 #define INDEX_HAS_MISSING_LOGS(index, file) \
        ((file)->hdr.file_seq != (index)->hdr->log_file_seq && \
-        ((file)->hdr.file_seq != (index)->hdr->log_file_seq+1 || \
+        ((file)->hdr.prev_file_seq != (index)->hdr->log_file_seq || \
          (file)->hdr.prev_file_offset != (index)->hdr->log_file_offset))
 
 
@@ -78,6 +78,8 @@ static int mail_transaction_log_check_file_seq(struct mail_transaction_log *log)
                return -1;
 
        file = log->head;
+       file->refcount++;
+
        ret = mail_index_lock_shared(index, TRUE, &lock_id);
        if (ret == 0) {
                ret = mail_index_map(index, FALSE);
@@ -88,7 +90,11 @@ static int mail_transaction_log_check_file_seq(struct mail_transaction_log *log)
                        ret = mail_transaction_log_rotate(log);
                }
        }
-       (void)mail_transaction_log_file_lock(file, F_UNLCK);
+
+       if (--file->refcount == 0)
+               mail_transaction_logs_clean(log);
+       else
+               (void)mail_transaction_log_file_lock(file, F_UNLCK);
        return ret;
 }
 
@@ -330,11 +336,16 @@ mail_transaction_log_file_create(struct mail_transaction_log *log,
        hdr.indexid = index->indexid;
        hdr.used_size = sizeof(hdr);
 
-       if (index->fd != -1)
+       if (index->fd != -1) {
+               hdr.prev_file_seq = index->hdr->log_file_seq;
                hdr.prev_file_offset = index->hdr->log_file_offset;
+       }
        hdr.file_seq = index->hdr->log_file_seq+1;
 
-       i_assert(log->head == NULL || hdr.file_seq > log->head->hdr.file_seq);
+       if (log->head != NULL && hdr.file_seq <= log->head->hdr.file_seq) {
+               /* make sure the sequence grows */
+               hdr.file_seq = log->head->hdr.file_seq+1;
+       }
 
        if (write_full(fd, &hdr, sizeof(hdr)) < 0) {
                mail_index_file_set_syscall_error(index, path, "write_full()");
@@ -439,14 +450,15 @@ mail_transaction_log_file_open_or_create(struct mail_transaction_log *log,
 
 void mail_transaction_logs_clean(struct mail_transaction_log *log)
 {
-       struct mail_transaction_log_file **p;
+       struct mail_transaction_log_file **p, *next;
 
        for (p = &log->tail; *p != NULL; ) {
                if ((*p)->refcount != 0)
                         p = &(*p)->next;
                else {
-                        mail_transaction_log_file_close(*p);
-                       *p = (*p)->next;
+                       next = (*p)->next;
+                       mail_transaction_log_file_close(*p);
+                       *p = next;
                }
        }
 }
@@ -455,7 +467,7 @@ static int mail_transaction_log_rotate(struct mail_transaction_log *log)
 {
        struct mail_transaction_log_file *file;
        struct stat st;
-       int fd;
+       int fd, lock_type;
 
        if (fstat(log->head->fd, &st) < 0) {
                mail_index_file_set_syscall_error(log->index,
@@ -473,11 +485,17 @@ static int mail_transaction_log_rotate(struct mail_transaction_log *log)
        if (file == NULL)
                return -1;
 
-       if (log->head != NULL) {
-               if (--log->head->refcount == 0)
-                       mail_transaction_logs_clean(log);
+       lock_type = log->head->lock_type;
+       if (lock_type != F_UNLCK) {
+               if (mail_transaction_log_file_lock(file, lock_type) < 0)
+                       return -1;
        }
 
+       if (--log->head->refcount == 0)
+               mail_transaction_logs_clean(log);
+       else
+               (void)mail_transaction_log_file_lock(log->head, F_UNLCK);
+
        log->head = file;
        return 0;
 }
@@ -988,6 +1006,7 @@ int mail_transaction_log_append(struct mail_index_transaction *t,
                                uoff_t *log_file_offset_r)
 {
        struct mail_index_view *view = t->view;
+       struct mail_index *index;
        struct mail_transaction_log *log;
        struct mail_transaction_log_file *file;
        size_t offset;
@@ -1000,7 +1019,8 @@ int mail_transaction_log_append(struct mail_index_transaction *t,
                return 0;
        }
 
-       log = mail_index_view_get_index(view)->log;
+       index = mail_index_view_get_index(view);
+       log = index->log;
 
        if (log->index->log_locked) {
                i_assert(view->external);
@@ -1008,6 +1028,19 @@ int mail_transaction_log_append(struct mail_index_transaction *t,
                if (mail_transaction_log_lock_head(log) < 0)
                        return -1;
        }
+
+       if (log->head->hdr.file_seq == index->hdr->log_file_seq &&
+           log->head->hdr.used_size > MAIL_TRANSACTION_LOG_ROTATE_SIZE) {
+               /* everything synced in index, we can rotate. */
+               if (mail_transaction_log_rotate(log) < 0) {
+                       if (!log->index->log_locked) {
+                               (void)mail_transaction_log_file_lock(log->head,
+                                                                    F_UNLCK);
+                       }
+                       return -1;
+               }
+       }
+
        file = log->head;
        append_offset = file->hdr.used_size;
 

diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h
index bc70e600a566e1a477479451e999cc3885c2118a..aa9de233b2a9f2faae0d2a240241b25bf806bd56 100644 (file)
--- a/src/lib-index/mail-transaction-log.h
+++ b/src/lib-index/mail-transaction-log.h
@@ -2,10 +2,12 @@
 #define __MAIL_TRANSACTION_LOG_H
 
 #define MAIL_TRANSACTION_LOG_PREFIX ".log"
+#define MAIL_TRANSACTION_LOG_ROTATE_SIZE (1024*128)
 
 struct mail_transaction_log_header {
        uint32_t indexid;
        uint32_t file_seq;
+       uint32_t prev_file_seq;
        uint32_t prev_file_offset;
        uint32_t used_size;
 };

diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c
index f852063a402b5964c70bbcb07f9f851af8052e1e..7e255287c173a0cc20b6f7568929b09c5649a4ff 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-sync.c
+++ b/src/lib-storage/index/maildir/maildir-sync.c
@@ -132,7 +132,6 @@ int maildir_sync_last_commit(struct index_mailbox *ibox)
                ibox->commit_log_file_seq = 0;
                ibox->commit_log_file_offset = 0;
        } else {
-               // FIXME: this is bad - we have to fix this in some way
                mail_storage_set_index_error(ibox);
        }
        return ret;
@@ -307,7 +306,7 @@ static int maildir_sync_index(struct maildir_sync_context *ctx)
 
        if (mail_index_sync_begin(ibox->index, &sync_ctx, &view,
                                  (uint32_t)-1, (uoff_t)-1) <= 0) {
-               // FIXME: ?
+               mail_storage_set_index_error(ibox);
                return -1;
        }
 
@@ -393,7 +392,6 @@ static int maildir_sync_index(struct maildir_sync_context *ctx)
                ibox->commit_log_file_seq = 0;
                ibox->commit_log_file_offset = 0;
        } else {
-               // FIXME: this is bad - we have to fix this in some way
                mail_storage_set_index_error(ibox);
        }
 

diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c
index 5a82f946d4f1558f44d0ae055d71050e2598c8bd..c76a9e351fc95c6418ac949689d1d8a45c34db79 100644 (file)
--- a/src/lib-storage/index/maildir/maildir-uidlist.c
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c
@@ -570,11 +570,9 @@ int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx)
 {
        int ret;
 
-       if (ctx->failed) {
-               if (UIDLIST_IS_LOCKED(ctx->uidlist))
-                       maildir_uidlist_unlock(ctx->uidlist);
+       if (ctx->failed)
                ret = -1;
-       } else {
+       else {
                maildir_uidlist_swap(ctx);
                if (!ctx->new_files)
                        ret = 0;
@@ -582,6 +580,9 @@ int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx)
                        ret = maildir_uidlist_rewrite(ctx->uidlist);
        }
 
+       if (UIDLIST_IS_LOCKED(ctx->uidlist))
+               maildir_uidlist_unlock(ctx->uidlist);
+
        if (ctx->files != NULL)
                hash_destroy(ctx->files);
        if (ctx->filename_pool != NULL)