From: Timo Sirainen Date: Sat, 22 May 2004 21:30:42 +0000 (+0300) Subject: Set dirty flags through transaction log, not directly. Some other flag X-Git-Tag: 1.1.alpha1~4069 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b35f7104715edee0cfac6d46ab0b342033867eb7;p=thirdparty%2Fdovecot%2Fcore.git Set dirty flags through transaction log, not directly. Some other flag fixes etc. --HG-- branch : HEAD --- diff --git a/src/lib-index/mail-index-sync-private.h b/src/lib-index/mail-index-sync-private.h index 7513f23aaf..66c4ff645f 100644 --- a/src/lib-index/mail-index-sync-private.h +++ b/src/lib-index/mail-index-sync-private.h @@ -17,10 +17,9 @@ struct mail_index_sync_ctx { size_t expunge_idx, update_idx; uint32_t next_uid; - unsigned int lock_id, dirty_lock_id; + unsigned int lock_id; unsigned int sync_appends:1; - unsigned int have_dirty:1; }; int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx, diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c index 2cf0014cc5..c46f038e6b 100644 --- a/src/lib-index/mail-index-sync-update.c +++ b/src/lib-index/mail-index-sync-update.c @@ -13,6 +13,8 @@ struct mail_index_update_ctx { struct mail_index_view *view; struct mail_index_header hdr; struct mail_transaction_log_view *log_view; + + unsigned int have_dirty:1; }; void mail_index_header_update_counts(struct mail_index_header *hdr, @@ -75,6 +77,11 @@ static void mail_index_sync_update_flags(struct mail_index_update_ctx *ctx, if (seq1 == 0) return; + if ((syncrec->add_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + ctx->hdr.flags |= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + ctx->have_dirty = TRUE; + } + update_keywords = FALSE; for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) { if (syncrec->add_keywords[i] != 0) @@ -185,7 +192,7 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx, struct mail_index_sync_rec rec; const struct mail_index_record *appends; unsigned int append_count; - uint32_t count, file_seq, src_idx, dest_idx, dirty_flag; + uint32_t count, file_seq, src_idx, dest_idx, i; uint32_t seq1, seq2; uoff_t file_offset; unsigned int lock_id; @@ -204,12 +211,6 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx, ctx.hdr = *index->hdr; ctx.log_view = sync_ctx->view->log_view; - dirty_flag = sync_ctx->have_dirty ? MAIL_INDEX_HDR_FLAG_HAVE_DIRTY : 0; - if ((ctx.hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != dirty_flag) { - ctx.hdr.flags ^= MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; - changed = TRUE; - } - /* see if we need to update sync headers */ if (ctx.hdr.sync_stamp != sync_stamp && sync_stamp != 0) { ctx.hdr.sync_stamp = sync_stamp; @@ -301,6 +302,17 @@ int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx, ctx.hdr.log_file_seq = file_seq; ctx.hdr.log_file_offset = file_offset; + if ((ctx.hdr.flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) && + !ctx.have_dirty) { + /* do we have dirty flags anymore? */ + for (i = 0; i < map->records_count; i++) { + if (map->records[i].flags & MAIL_INDEX_MAIL_FLAG_DIRTY) + break; + } + if (i == map->records_count) + ctx.hdr.flags &= ~MAIL_INDEX_HDR_FLAG_HAVE_DIRTY; + } + if (!MAIL_INDEX_MAP_IS_IN_MEMORY(map)) { map->mmap_used_size = index->hdr->header_size + map->records_count * sizeof(struct mail_index_record); diff --git a/src/lib-index/mail-index-sync.c b/src/lib-index/mail-index-sync.c index 967e691d12..38c9f79cf3 100644 --- a/src/lib-index/mail-index-sync.c +++ b/src/lib-index/mail-index-sync.c @@ -25,7 +25,7 @@ static void mail_index_sync_sort_flags(struct mail_index_sync_ctx *ctx) dest = buffer_get_data(ctx->updates_buf, &dest_count); dest_count /= sizeof(*dest); - for (i = 0; src != src_end; ) { + for (i = 0; src != src_end; src++) { new_update = *src; /* insert it into buffer, split it in multiple parts if needed @@ -312,22 +312,6 @@ int mail_index_sync_have_more(struct mail_index_sync_ctx *ctx) ctx->sync_appends; } -int mail_index_sync_set_dirty(struct mail_index_sync_ctx *ctx, uint32_t seq) -{ - if (ctx->dirty_lock_id == 0) { - if (mail_index_lock_exclusive(ctx->index, - &ctx->dirty_lock_id) < 0) - return -1; - } - - /* FIXME: maybe this should go through transaction log anyway? - doesn't work well with non-mmaped indexes.. */ - i_assert(seq <= ctx->view->map->records_count); - ctx->view->map->records[seq-1].flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; - ctx->have_dirty = TRUE; - return 0; -} - int mail_index_sync_end(struct mail_index_sync_ctx *ctx, uint32_t sync_stamp, uint64_t sync_size) { @@ -358,9 +342,6 @@ int mail_index_sync_end(struct mail_index_sync_ctx *ctx, ret = -1; } - if (ctx->dirty_lock_id == 0) - mail_index_unlock(ctx->index, ctx->dirty_lock_id); - mail_index_unlock(ctx->index, ctx->lock_id); mail_transaction_log_sync_unlock(ctx->index->log); mail_index_view_close(ctx->view); diff --git a/src/lib-index/mail-index-transaction.c b/src/lib-index/mail-index-transaction.c index 2f4cef45e7..6272ec71eb 100644 --- a/src/lib-index/mail-index-transaction.c +++ b/src/lib-index/mail-index-transaction.c @@ -359,7 +359,7 @@ static void mail_index_transaction_add_last(struct mail_index_transaction *t) if (idx < size && data[idx].uid2 < update.uid1) idx++; - i_assert(idx == size || data[idx].uid1 < update.uid1); + i_assert(idx == size || data[idx].uid1 <= update.uid1); /* insert it into buffer, split it in multiple parts if needed to make sure the ordering stays the same */ @@ -371,10 +371,12 @@ static void mail_index_transaction_add_last(struct mail_index_transaction *t) last = update.uid2; update.uid2 = data[idx].uid1-1; - buffer_insert(t->updates, idx * sizeof(update), - &update, sizeof(update)); - data = buffer_get_modifyable_data(t->updates, NULL); - size++; + if (update.uid1 <= update.uid2) { + buffer_insert(t->updates, idx * sizeof(update), + &update, sizeof(update)); + data = buffer_get_modifyable_data(t->updates, NULL); + size++; + } update.uid1 = update.uid2+1; update.uid2 = last; diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c index 2c391c8d51..e935145107 100644 --- a/src/lib-index/mail-index-view-sync.c +++ b/src/lib-index/mail-index-view-sync.c @@ -292,10 +292,16 @@ static int mail_index_view_sync_next_trans(struct mail_index_view_sync_ctx *ctx, return 1; } -static void +#define FLAG_UPDATE_IS_INTERNAL(u, empty) \ + (((u)->add_flags | (u)->remove_flags) == MAIL_INDEX_MAIL_FLAG_DIRTY && \ + memcmp((u)->add_keywords, empty, INDEX_KEYWORDS_BYTE_COUNT) == 0 && \ + memcmp((u)->add_keywords, empty, INDEX_KEYWORDS_BYTE_COUNT) == 0) + +static int mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx, struct mail_index_sync_rec *rec) { + static keywords_mask_t empty_keywords = { 0, }; const struct mail_transaction_header *hdr = ctx->hdr; const void *data = ctx->data; @@ -318,13 +324,21 @@ mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx, const struct mail_transaction_flag_update *update = CONST_PTR_OFFSET(data, ctx->data_offset); - ctx->data_offset += sizeof(*update); + for (;;) { + ctx->data_offset += sizeof(*update); + if (!FLAG_UPDATE_IS_INTERNAL(update, empty_keywords)) + break; + + if (ctx->data_offset == ctx->hdr->size) + return 0; + } mail_index_sync_get_update(rec, update); break; } default: i_unreached(); } + return 1; } int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx, @@ -335,31 +349,34 @@ int mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx, uoff_t offset; int ret; - if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) { - ctx->data_offset = 0; - do { - ret = mail_index_view_sync_next_trans(ctx, &seq, - &offset); - if (ret < 0) - return -1; - - if (ctx->last_read) - return 0; - - if (!ctx->skipped) { - view->log_file_seq = seq; - view->log_file_offset = offset + - sizeof(*ctx->hdr) + ctx->hdr->size; + do { + if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) { + ctx->data_offset = 0; + do { + ret = mail_index_view_sync_next_trans(ctx, &seq, + &offset); + if (ret < 0) + return -1; + + if (ctx->last_read) + return 0; + + if (!ctx->skipped) { + view->log_file_seq = seq; + view->log_file_offset = offset + + sizeof(*ctx->hdr) + + ctx->hdr->size; + } + } while (ret == 0); + + if (ctx->skipped) { + mail_index_view_add_synced_transaction(view, + seq, + offset); } - } while (ret == 0); - - if (ctx->skipped) { - mail_index_view_add_synced_transaction(view, seq, - offset); } - } + } while (!mail_index_view_sync_get_rec(ctx, sync_rec)); - mail_index_view_sync_get_rec(ctx, sync_rec); return 1; } diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h index 5e03622312..5faf316f68 100644 --- a/src/lib-index/mail-index.h +++ b/src/lib-index/mail-index.h @@ -196,10 +196,6 @@ int mail_index_sync_next(struct mail_index_sync_ctx *ctx, struct mail_index_sync_rec *sync_rec); /* Returns 1 if there's more to sync, 0 if not. */ int mail_index_sync_have_more(struct mail_index_sync_ctx *ctx); -/* Mark given message to be dirty, ie. we couldn't temporarily change the - message flags in storage. Dirty messages are tried to be synced again in - next sync. */ -int mail_index_sync_set_dirty(struct mail_index_sync_ctx *ctx, uint32_t seq); /* End synchronization by unlocking the index and closing the view. sync_stamp/sync_size in header is updated to given values. */ int mail_index_sync_end(struct mail_index_sync_ctx *ctx, @@ -207,8 +203,8 @@ int mail_index_sync_end(struct mail_index_sync_ctx *ctx, /* Mark index file corrupted. Invalidates all views. */ void mail_index_mark_corrupted(struct mail_index *index); -/* Check and fix any found problems. If index is broken beyond repair, calls - mail_index_reset() and returns 0. Otherwise returns -1 if there was some +/* Check and fix any found problems. If index is broken beyond repair, it's + marked corrupted and 0 is returned. Otherwise returns -1 if there was some I/O error or 1 if everything went ok. */ int mail_index_fsck(struct mail_index *index); diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c index 0747c1c970..dcb1aa73ea 100644 --- a/src/lib-storage/index/index-sync.c +++ b/src/lib-storage/index/index-sync.c @@ -83,7 +83,6 @@ int index_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags) mail_storage_set_index_error(ibox); if (sc->expunge != NULL) { - // FIXME: these are UIDs now.. for (i = expunges_count*2; i > 0; i -= 2) { seq = expunges[i-1]; if (seq > messages_count) diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c index d8e8c361be..e9e24c9564 100644 --- a/src/lib-storage/index/maildir/maildir-sync.c +++ b/src/lib-storage/index/maildir/maildir-sync.c @@ -197,11 +197,13 @@ struct maildir_sync_context { struct maildir_index_sync_context { struct index_mailbox *ibox; - struct mail_index_view *view; + struct mail_index_view *view; struct mail_index_sync_ctx *sync_ctx; + struct mail_index_transaction *trans; struct mail_index_sync_rec sync_rec; uint32_t seq; + int have_dirty, last_dirty; }; static int maildir_expunge(struct index_mailbox *ibox, const char *path, @@ -228,6 +230,8 @@ static int maildir_sync_flags(struct index_mailbox *ibox, const char *path, uint8_t flags8; keywords_mask_t keywords; + ctx->last_dirty = FALSE; + (void)maildir_filename_get_flags(path, &flags, keywords); flags8 = flags; @@ -241,9 +245,11 @@ static int maildir_sync_flags(struct index_mailbox *ibox, const char *path, if (errno == ENOENT) return 0; - if (ENOSPACE(errno)) { - if (mail_index_sync_set_dirty(ctx->sync_ctx, ctx->seq) < 0) - return -1; + if (ENOSPACE(errno) || errno == EACCES) { + memset(keywords, 0, sizeof(keywords)); + mail_index_update_flags(ctx->trans, ctx->seq, MODIFY_ADD, + MAIL_INDEX_MAIL_FLAG_DIRTY, keywords); + ctx->last_dirty = TRUE; return 1; } @@ -257,6 +263,7 @@ static int maildir_sync_record(struct index_mailbox *ibox, { struct mail_index_sync_rec *sync_rec = &ctx->sync_rec; struct mail_index_view *view = ctx->view; + const struct mail_index_record *rec; uint32_t seq, seq1, seq2, uid; switch (sync_rec->type) { @@ -293,9 +300,23 @@ static int maildir_sync_record(struct index_mailbox *ibox, for (ctx->seq = seq1; ctx->seq <= seq2; ctx->seq++) { if (mail_index_lookup_uid(view, ctx->seq, &uid) < 0) return -1; - if (maildir_file_do(ibox, uid, maildir_sync_flags, - ctx) < 0) + if (maildir_file_do(ibox, uid, + maildir_sync_flags, ctx) < 0) return -1; + if (!ctx->last_dirty) { + /* if this flag was dirty, drop it */ + if (mail_index_lookup(view, ctx->seq, &rec) < 0) + return -1; + if (rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) { + keywords_mask_t keywords; + + memset(keywords, 0, sizeof(keywords)); + mail_index_update_flags(ctx->trans, + ctx->seq, MODIFY_REMOVE, + MAIL_INDEX_MAIL_FLAG_DIRTY, + keywords); + } + } } break; } @@ -306,6 +327,9 @@ static int maildir_sync_record(struct index_mailbox *ibox, int maildir_sync_last_commit(struct index_mailbox *ibox) { struct maildir_index_sync_context ctx; + const struct mail_index_header *hdr; + uint32_t seq; + uoff_t offset; int ret; if (ibox->commit_log_file_seq == 0) @@ -318,6 +342,12 @@ int maildir_sync_last_commit(struct index_mailbox *ibox) ibox->commit_log_file_seq, ibox->commit_log_file_offset); if (ret > 0) { + if (mail_index_get_header(ctx.view, &hdr) == 0 && + (hdr->flags & MAIL_INDEX_HDR_FLAG_HAVE_DIRTY) != 0) + ctx.have_dirty = TRUE; + + ctx.trans = mail_index_transaction_begin(ctx.view, FALSE); + while ((ret = mail_index_sync_next(ctx.sync_ctx, &ctx.sync_rec)) > 0) { if (maildir_sync_record(ibox, &ctx) < 0) { @@ -325,6 +355,8 @@ int maildir_sync_last_commit(struct index_mailbox *ibox) break; } } + if (mail_index_transaction_commit(ctx.trans, &seq, &offset) < 0) + ret = -1; if (mail_index_sync_end(ctx.sync_ctx, 0, 0) < 0) ret = -1; } @@ -557,6 +589,7 @@ static int maildir_sync_index(struct maildir_sync_context *ctx) i_assert(ret == 0); /* view is locked, can't happen */ trans = mail_index_transaction_begin(view, FALSE); + sync_ctx.trans = trans; seq = 0; iter = maildir_uidlist_iter_init(ibox->uidlist); @@ -658,6 +691,15 @@ static int maildir_sync_index(struct maildir_sync_context *ctx) mail_index_expunge(trans, seq); } + /* now, sync the index */ + while ((ret = mail_index_sync_next(sync_ctx.sync_ctx, + &sync_ctx.sync_rec)) > 0) { + if (maildir_sync_record(ibox, &sync_ctx) < 0) { + ret = -1; + break; + } + } + if (ret < 0) mail_index_transaction_rollback(trans); else { @@ -672,15 +714,6 @@ static int maildir_sync_index(struct maildir_sync_context *ctx) } } - /* now, sync the index */ - while ((ret = mail_index_sync_next(sync_ctx.sync_ctx, - &sync_ctx.sync_rec)) > 0) { - if (maildir_sync_record(ibox, &sync_ctx) < 0) { - ret = -1; - break; - } - } - sync_stamp = ibox->dirty_cur_time != 0 ? 0 : ibox->last_cur_mtime; if (mail_index_sync_end(sync_ctx.sync_ctx, sync_stamp, 0) < 0) ret = -1;