From: Timo Sirainen Date: Sat, 1 May 2004 14:30:25 +0000 (+0300) Subject: fixes X-Git-Tag: 1.1.alpha1~4166 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0add8c99ca65e56dbf613595fc37c41aafff3f7f;p=thirdparty%2Fdovecot%2Fcore.git fixes --HG-- branch : HEAD --- diff --git a/src/lib-index/mail-index-lock.c b/src/lib-index/mail-index-lock.c index 2d0195e044..15a38f3017 100644 --- a/src/lib-index/mail-index-lock.c +++ b/src/lib-index/mail-index-lock.c @@ -113,17 +113,16 @@ static int mail_index_has_changed(struct mail_index *index) } } -static int mail_index_lock_mprotect(struct mail_index *index, int lock_type) +int mail_index_map_lock_mprotect(struct mail_index *index, + struct mail_index_map *map, int lock_type) { int prot; - if (index->map != NULL && - !MAIL_INDEX_MAP_IS_IN_MEMORY(index->map)) { + if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { prot = lock_type == F_UNLCK ? PROT_NONE : lock_type == F_WRLCK ? (PROT_READ|PROT_WRITE) : PROT_READ; - if (mprotect(index->map->mmap_base, - index->map->mmap_size, prot) < 0) { + if (mprotect(map->mmap_base, map->mmap_size, prot) < 0) { mail_index_set_syscall_error(index, "mprotect()"); return -1; } @@ -131,6 +130,14 @@ static int mail_index_lock_mprotect(struct mail_index *index, int lock_type) return 0; } +static int mail_index_lock_mprotect(struct mail_index *index, int lock_type) +{ + if (index->map == NULL) + return 0; + + return mail_index_map_lock_mprotect(index, index->map, lock_type); +} + static int mail_index_lock(struct mail_index *index, int lock_type, unsigned int timeout_secs, int update_index, unsigned int *lock_id_r) @@ -142,14 +149,23 @@ static int mail_index_lock(struct mail_index *index, int lock_type, if (lock_type == F_RDLCK && index->lock_type != F_UNLCK) { index->shared_lock_count++; *lock_id_r = index->lock_id; - return 1; - } - if (lock_type == F_WRLCK && index->lock_type == F_WRLCK) { + ret = 1; + } else if (lock_type == F_WRLCK && index->lock_type == F_WRLCK) { index->excl_lock_count++; *lock_id_r = index->lock_id + 1; - return 1; + ret = 1; + } else { + ret = 0; } + if (update_index && index->excl_lock_count == 0) { + if (mail_index_has_changed(index) < 0) + return -1; + } + + if (ret > 0) + return 1; + if (index->fcntl_locks_disable) { /* FIXME: exclusive locking will rewrite the index file every time. shouldn't really be needed.. reading doesn't require @@ -169,11 +185,6 @@ static int mail_index_lock(struct mail_index *index, int lock_type, return 1; } - if (update_index) { - if (mail_index_has_changed(index) < 0) - return -1; - } - ret = file_wait_lock_full(index->fd, lock_type, timeout_secs, NULL, NULL); if (ret <= 0) { diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h index 8beb3ea48d..b6dbef10ba 100644 --- a/src/lib-index/mail-index-private.h +++ b/src/lib-index/mail-index-private.h @@ -103,6 +103,8 @@ int mail_index_lock_exclusive(struct mail_index *index, void mail_index_unlock(struct mail_index *index, unsigned int lock_id); /* Returns 1 if given lock_id is valid, 0 if not. */ int mail_index_is_locked(struct mail_index *index, unsigned int lock_id); +int mail_index_map_lock_mprotect(struct mail_index *index, + struct mail_index_map *map, int lock_type); /* Map index file to memory, replacing the previous mapping for index. Returns 1 = ok, 0 = corrupted, -1 = error. If index needs fscking, it diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c index e4408c68ed..d78336ec51 100644 --- a/src/lib-index/mail-index-sync.c +++ b/src/lib-index/mail-index-sync.c @@ -344,6 +344,16 @@ int mail_index_sync_next(struct mail_index_sync_ctx *ctx, it's quite unlikely this expunge was caused by some bug. */ uint32_t uid1, uid2; + if (next_exp->seq1 > ctx->view->map->records_count || + next_exp->seq2 > ctx->view->map->records_count) { + mail_transaction_log_view_set_corrupted( + ctx->view->log_view, "Expunge range %u..%u " + "larger than message count %u", + next_exp->seq1, next_exp->seq2, + ctx->view->map->records_count); + return -1; + } + if (mail_index_lookup_uid(ctx->view, next_exp->seq1, &uid1) < 0) return -1; if (mail_index_lookup_uid(ctx->view, next_exp->seq2, &uid2) < 0) diff --git a/src/lib-index/mail-index-view-private.h b/src/lib-index/mail-index-view-private.h index 8cd675ad8e..024d8829cd 100644 --- a/src/lib-index/mail-index-view-private.h +++ b/src/lib-index/mail-index-view-private.h @@ -7,6 +7,7 @@ struct mail_index_view { struct mail_index *index; struct mail_transaction_log_view *log_view; + unsigned int indexid; struct mail_index_map *map; uint32_t log_file_seq; @@ -19,6 +20,7 @@ struct mail_index_view { unsigned int inconsistent:1; unsigned int syncing:1; unsigned int external:1; + unsigned int map_protected:1; }; int mail_index_view_lock(struct mail_index_view *view); diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c index 4e0dafa31f..bf72d808a2 100644 --- a/src/lib-index/mail-index-view-sync.c +++ b/src/lib-index/mail-index-view-sync.c @@ -160,6 +160,7 @@ static int sync_append(const struct mail_index_record *rec, void *context) map->records_count++; map->hdr_copy.messages_count++; + map->hdr_copy.next_uid = rec->uid+1; mail_index_header_update_counts(&map->hdr_copy, 0, rec->flags); mail_index_header_update_lowwaters(&map->hdr_copy, rec); diff --git a/src/lib-index/mail-index-view.c b/src/lib-index/mail-index-view.c index 59ff2fff72..979736b1ff 100644 --- a/src/lib-index/mail-index-view.c +++ b/src/lib-index/mail-index-view.c @@ -14,6 +14,7 @@ struct mail_index_view *mail_index_view_open(struct mail_index *index) view->index = index; view->log_view = mail_transaction_log_view_open(index->log); + view->indexid = index->indexid; view->map = index->map; view->map->refcount++; @@ -33,8 +34,27 @@ void mail_index_view_close(struct mail_index_view *view) i_free(view); } +static int mail_index_view_map_protect(struct mail_index_view *view) +{ + /* not head mapping, no need to lock */ + if (!view->map_protected) { + if (mail_index_map_lock_mprotect(view->index, view->map, + F_RDLCK) < 0) + return -1; + view->map_protected = TRUE; + } + return 0; +} + int mail_index_view_lock_head(struct mail_index_view *view, int update_index) { + unsigned int lock_id; + + if (view->map != view->index->map) { + if (mail_index_view_map_protect(view) < 0) + return -1; + } + if (!mail_index_is_locked(view->index, view->lock_id)) { if (mail_index_lock_shared(view->index, update_index, &view->lock_id) < 0) @@ -45,13 +65,17 @@ int mail_index_view_lock_head(struct mail_index_view *view, int update_index) return -1; } - if (view->index->indexid != view->map->hdr->indexid) { + if (view->index->indexid != view->indexid) { /* index was rebuilt */ view->inconsistent = TRUE; return -1; } } else if (update_index) { - // FIXME: check if we need to reopen it! + if (mail_index_lock_shared(view->index, TRUE, &lock_id) < 0) + return -1; + + mail_index_unlock(view->index, view->lock_id); + view->lock_id = lock_id; } return 0; @@ -63,7 +87,8 @@ int mail_index_view_lock(struct mail_index_view *view) return -1; if (view->map != view->index->map) { - /* not head mapping, no need to lock */ + if (mail_index_view_map_protect(view) < 0) + return -1; return 0; } @@ -72,6 +97,12 @@ int mail_index_view_lock(struct mail_index_view *view) void mail_index_view_unlock(struct mail_index_view *view) { + if (view->map_protected) { + (void)mail_index_map_lock_mprotect(view->index, view->map, + F_UNLCK); + view->map_protected = FALSE; + } + if (view->lock_id != 0) { mail_index_unlock(view->index, view->lock_id); view->lock_id = 0; diff --git a/src/lib-index/mail-transaction-log-view.c b/src/lib-index/mail-transaction-log-view.c index 56e9c9d275..7f51ea7d73 100644 --- a/src/lib-index/mail-transaction-log-view.c +++ b/src/lib-index/mail-transaction-log-view.c @@ -90,10 +90,10 @@ mail_transaction_log_view_set(struct mail_transaction_log_view *view, ret = mail_transaction_log_file_find(view->log, min_file_seq, &file); if (ret <= 0) { if (ret == 0 && - min_file_seq == view->log->tail->hdr.file_seq-1 && + min_file_seq == view->log->tail->hdr.prev_file_seq && min_file_offset == view->log->tail->hdr.prev_file_offset) { /* we can skip this */ - min_file_seq++; + min_file_seq = view->log->tail->hdr.file_seq; min_file_offset = sizeof(struct mail_transaction_log_header); ret = mail_transaction_log_file_find(view->log, @@ -197,19 +197,26 @@ static int log_view_get_next(struct mail_transaction_log_view *view, const void **data_r) { const struct mail_transaction_header *hdr; - struct mail_transaction_log_file *file = view->file; + struct mail_transaction_log_file *file; const struct mail_transaction_type_map *type_rec; const void *data; unsigned int record_size; size_t file_size; - view->prev_file_seq = file->hdr.file_seq; - view->prev_file_offset = view->file_offset; + for (;;) { + file = view->file; + + view->prev_file_seq = file->hdr.file_seq; + view->prev_file_offset = view->file_offset; + + if (view->file_offset != file->hdr.used_size) + break; - if (view->file_offset == file->hdr.used_size) { view->file = file->next; view->file_offset = sizeof(struct mail_transaction_log_header); - return 0; + + if (view->file == NULL) + return 0; } data = buffer_get_data(file->buffer, &file_size); diff --git a/src/lib-index/mail-transaction-log.c b/src/lib-index/mail-transaction-log.c index f1faaaf868..fdbff0b89c 100644 --- a/src/lib-index/mail-transaction-log.c +++ b/src/lib-index/mail-transaction-log.c @@ -63,7 +63,7 @@ mail_transaction_log_file_set_corrupted(struct mail_transaction_log_file *file, #define INDEX_HAS_MISSING_LOGS(index, file) \ ((file)->hdr.file_seq != (index)->hdr->log_file_seq && \ - ((file)->hdr.file_seq != (index)->hdr->log_file_seq+1 || \ + ((file)->hdr.prev_file_seq != (index)->hdr->log_file_seq || \ (file)->hdr.prev_file_offset != (index)->hdr->log_file_offset)) @@ -78,6 +78,8 @@ static int mail_transaction_log_check_file_seq(struct mail_transaction_log *log) return -1; file = log->head; + file->refcount++; + ret = mail_index_lock_shared(index, TRUE, &lock_id); if (ret == 0) { ret = mail_index_map(index, FALSE); @@ -88,7 +90,11 @@ static int mail_transaction_log_check_file_seq(struct mail_transaction_log *log) ret = mail_transaction_log_rotate(log); } } - (void)mail_transaction_log_file_lock(file, F_UNLCK); + + if (--file->refcount == 0) + mail_transaction_logs_clean(log); + else + (void)mail_transaction_log_file_lock(file, F_UNLCK); return ret; } @@ -330,11 +336,16 @@ mail_transaction_log_file_create(struct mail_transaction_log *log, hdr.indexid = index->indexid; hdr.used_size = sizeof(hdr); - if (index->fd != -1) + if (index->fd != -1) { + hdr.prev_file_seq = index->hdr->log_file_seq; hdr.prev_file_offset = index->hdr->log_file_offset; + } hdr.file_seq = index->hdr->log_file_seq+1; - i_assert(log->head == NULL || hdr.file_seq > log->head->hdr.file_seq); + if (log->head != NULL && hdr.file_seq <= log->head->hdr.file_seq) { + /* make sure the sequence grows */ + hdr.file_seq = log->head->hdr.file_seq+1; + } if (write_full(fd, &hdr, sizeof(hdr)) < 0) { mail_index_file_set_syscall_error(index, path, "write_full()"); @@ -439,14 +450,15 @@ mail_transaction_log_file_open_or_create(struct mail_transaction_log *log, void mail_transaction_logs_clean(struct mail_transaction_log *log) { - struct mail_transaction_log_file **p; + struct mail_transaction_log_file **p, *next; for (p = &log->tail; *p != NULL; ) { if ((*p)->refcount != 0) p = &(*p)->next; else { - mail_transaction_log_file_close(*p); - *p = (*p)->next; + next = (*p)->next; + mail_transaction_log_file_close(*p); + *p = next; } } } @@ -455,7 +467,7 @@ static int mail_transaction_log_rotate(struct mail_transaction_log *log) { struct mail_transaction_log_file *file; struct stat st; - int fd; + int fd, lock_type; if (fstat(log->head->fd, &st) < 0) { mail_index_file_set_syscall_error(log->index, @@ -473,11 +485,17 @@ static int mail_transaction_log_rotate(struct mail_transaction_log *log) if (file == NULL) return -1; - if (log->head != NULL) { - if (--log->head->refcount == 0) - mail_transaction_logs_clean(log); + lock_type = log->head->lock_type; + if (lock_type != F_UNLCK) { + if (mail_transaction_log_file_lock(file, lock_type) < 0) + return -1; } + if (--log->head->refcount == 0) + mail_transaction_logs_clean(log); + else + (void)mail_transaction_log_file_lock(log->head, F_UNLCK); + log->head = file; return 0; } @@ -988,6 +1006,7 @@ int mail_transaction_log_append(struct mail_index_transaction *t, uoff_t *log_file_offset_r) { struct mail_index_view *view = t->view; + struct mail_index *index; struct mail_transaction_log *log; struct mail_transaction_log_file *file; size_t offset; @@ -1000,7 +1019,8 @@ int mail_transaction_log_append(struct mail_index_transaction *t, return 0; } - log = mail_index_view_get_index(view)->log; + index = mail_index_view_get_index(view); + log = index->log; if (log->index->log_locked) { i_assert(view->external); @@ -1008,6 +1028,19 @@ int mail_transaction_log_append(struct mail_index_transaction *t, if (mail_transaction_log_lock_head(log) < 0) return -1; } + + if (log->head->hdr.file_seq == index->hdr->log_file_seq && + log->head->hdr.used_size > MAIL_TRANSACTION_LOG_ROTATE_SIZE) { + /* everything synced in index, we can rotate. */ + if (mail_transaction_log_rotate(log) < 0) { + if (!log->index->log_locked) { + (void)mail_transaction_log_file_lock(log->head, + F_UNLCK); + } + return -1; + } + } + file = log->head; append_offset = file->hdr.used_size; diff --git a/src/lib-index/mail-transaction-log.h b/src/lib-index/mail-transaction-log.h index bc70e600a5..aa9de233b2 100644 --- a/src/lib-index/mail-transaction-log.h +++ b/src/lib-index/mail-transaction-log.h @@ -2,10 +2,12 @@ #define __MAIL_TRANSACTION_LOG_H #define MAIL_TRANSACTION_LOG_PREFIX ".log" +#define MAIL_TRANSACTION_LOG_ROTATE_SIZE (1024*128) struct mail_transaction_log_header { uint32_t indexid; uint32_t file_seq; + uint32_t prev_file_seq; uint32_t prev_file_offset; uint32_t used_size; }; diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c index f852063a40..7e255287c1 100644 --- a/src/lib-storage/index/maildir/maildir-sync.c +++ b/src/lib-storage/index/maildir/maildir-sync.c @@ -132,7 +132,6 @@ int maildir_sync_last_commit(struct index_mailbox *ibox) ibox->commit_log_file_seq = 0; ibox->commit_log_file_offset = 0; } else { - // FIXME: this is bad - we have to fix this in some way mail_storage_set_index_error(ibox); } return ret; @@ -307,7 +306,7 @@ static int maildir_sync_index(struct maildir_sync_context *ctx) if (mail_index_sync_begin(ibox->index, &sync_ctx, &view, (uint32_t)-1, (uoff_t)-1) <= 0) { - // FIXME: ? + mail_storage_set_index_error(ibox); return -1; } @@ -393,7 +392,6 @@ static int maildir_sync_index(struct maildir_sync_context *ctx) ibox->commit_log_file_seq = 0; ibox->commit_log_file_offset = 0; } else { - // FIXME: this is bad - we have to fix this in some way mail_storage_set_index_error(ibox); } diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c index 5a82f946d4..c76a9e351f 100644 --- a/src/lib-storage/index/maildir/maildir-uidlist.c +++ b/src/lib-storage/index/maildir/maildir-uidlist.c @@ -570,11 +570,9 @@ int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx) { int ret; - if (ctx->failed) { - if (UIDLIST_IS_LOCKED(ctx->uidlist)) - maildir_uidlist_unlock(ctx->uidlist); + if (ctx->failed) ret = -1; - } else { + else { maildir_uidlist_swap(ctx); if (!ctx->new_files) ret = 0; @@ -582,6 +580,9 @@ int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx *ctx) ret = maildir_uidlist_rewrite(ctx->uidlist); } + if (UIDLIST_IS_LOCKED(ctx->uidlist)) + maildir_uidlist_unlock(ctx->uidlist); + if (ctx->files != NULL) hash_destroy(ctx->files); if (ctx->filename_pool != NULL)