From: Timo Sirainen Date: Mon, 14 Jun 2004 04:30:32 +0000 (+0300) Subject: Save mbox offsets to index file using extra_records. Some other fixes. X-Git-Tag: 1.1.alpha1~3988 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=01cbf4ac5d44137ab434791be7f838d98d0fcf3b;p=thirdparty%2Fdovecot%2Fcore.git Save mbox offsets to index file using extra_records. Some other fixes. --HG-- branch : HEAD --- diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h index c3ea5a30ad..ba77a16399 100644 --- a/src/lib-storage/index/index-storage.h +++ b/src/lib-storage/index/index-storage.h @@ -87,9 +87,7 @@ struct index_mailbox { struct dotlock mbox_dotlock; unsigned int mbox_lock_id; - buffer_t *mbox_data_buf; - const uoff_t *mbox_data; - uint32_t mbox_data_count; + uint32_t mbox_extra_idx; /* maildir sync: */ struct maildir_uidlist *uidlist; @@ -125,8 +123,8 @@ void index_storage_lock_notify(struct index_mailbox *ibox, void index_storage_lock_notify_reset(struct index_mailbox *ibox); struct mail_index * -index_storage_alloc(const char *index_dir, - const char *mailbox_path, const char *prefix); +index_storage_alloc(const char *index_dir, const char *mailbox_path, + const char *prefix); void index_storage_unref(struct mail_index *index); void index_storage_destroy_unrefed(void); diff --git a/src/lib-storage/index/mbox/mbox-mail.c b/src/lib-storage/index/mbox/mbox-mail.c index fe076e65de..2ac5cd4ccc 100644 --- a/src/lib-storage/index/mbox/mbox-mail.c +++ b/src/lib-storage/index/mbox/mbox-mail.c @@ -13,21 +13,28 @@ static int mbox_mail_seek(struct index_mail *mail) { - i_assert(mail->mail.seq <= mail->ibox->mbox_data_count); + struct index_mailbox *ibox = mail->ibox; + const void *data; + + if (mail_index_lookup_extra(ibox->view, mail->mail.seq, + ibox->mbox_extra_idx, &data) < 0) { + mail_storage_set_index_error(ibox); + return -1; + } // FIXME: lock the file. sync if needed. - if (mbox_file_open_stream(mail->ibox) < 0) + if (mbox_file_open_stream(ibox) < 0) return -1; - istream_raw_mbox_seek(mail->ibox->mbox_stream, - mail->ibox->mbox_data[mail->mail.seq-1] >> 1); + istream_raw_mbox_seek(ibox->mbox_stream, *((const uint64_t *)data)); return 0; } static const struct mail_full_flags *mbox_mail_get_flags(struct mail *_mail) { - struct index_mail *mail = (struct index_mail *)_mail; + return index_mail_get_flags(_mail); + /*FIXME:struct index_mail *mail = (struct index_mail *)_mail; struct index_mail_data *data = &mail->data; i_assert(_mail->seq <= mail->ibox->mbox_data_count); @@ -36,7 +43,7 @@ static const struct mail_full_flags *mbox_mail_get_flags(struct mail *_mail) if ((mail->ibox->mbox_data[_mail->seq-1] & 1) != 0) data->flags.flags |= MAIL_RECENT; - return &data->flags; + return &data->flags;*/ } static time_t mbox_mail_get_received_date(struct mail *_mail) diff --git a/src/lib-storage/index/mbox/mbox-storage.c b/src/lib-storage/index/mbox/mbox-storage.c index b8e2372513..ea96b814de 100644 --- a/src/lib-storage/index/mbox/mbox-storage.c +++ b/src/lib-storage/index/mbox/mbox-storage.c @@ -397,6 +397,7 @@ mbox_open(struct index_storage *storage, const char *name, struct index_mailbox *ibox; struct mail_index *index; const char *path, *index_dir; + uint32_t mbox_extra_idx; if (strcasecmp(name, "INBOX") == 0) { /* name = "INBOX" @@ -413,6 +414,8 @@ mbox_open(struct index_storage *storage, const char *name, } index = index_storage_alloc(index_dir, path, MBOX_INDEX_PREFIX); + mbox_extra_idx = mail_index_register_record_extra(index, "mbox", + sizeof(uint64_t)); ibox = index_storage_mailbox_init(storage, &mbox_mailbox, index, name, flags); if (ibox == NULL) @@ -421,6 +424,7 @@ mbox_open(struct index_storage *storage, const char *name, ibox->path = i_strdup(path); ibox->mbox_fd = -1; ibox->mbox_lock_type = F_UNLCK; + ibox->mbox_extra_idx = mbox_extra_idx; ibox->get_recent_count = mbox_get_recent_count; ibox->mail_interface = &mbox_mail; @@ -775,10 +779,6 @@ static int mbox_get_mailbox_name_status(struct mail_storage *_storage, static int mbox_storage_close(struct mailbox *box) { - struct index_mailbox *ibox = (struct index_mailbox *)box; - - if (ibox->mbox_data_buf != NULL) - buffer_free(ibox->mbox_data_buf); index_storage_mailbox_free(box); return 0; } diff --git a/src/lib-storage/index/mbox/mbox-sync-parse.c b/src/lib-storage/index/mbox/mbox-sync-parse.c index e4c2e06293..1117763635 100644 --- a/src/lib-storage/index/mbox/mbox-sync-parse.c +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c @@ -124,6 +124,17 @@ static int parse_x_imap_base(struct mbox_sync_mail_context *ctx, return TRUE; } +static int parse_x_imap(struct mbox_sync_mail_context *ctx, + struct message_header_line *hdr) +{ + if (!parse_x_imap_base(ctx, hdr)) + return FALSE; + + /* this is the UW-IMAP style "FOLDER INTERNAL DATA" message. skip it. */ + ctx->pseudo = TRUE; + return TRUE; +} + static int parse_x_keywords(struct mbox_sync_mail_context *ctx, struct message_header_line *hdr) { @@ -226,6 +237,7 @@ static int parse_content_length(struct mbox_sync_mail_context *ctx, static struct header_func header_funcs[] = { { "Content-Length", parse_content_length }, { "Status", parse_status }, + { "X-IMAP", parse_x_imap }, { "X-IMAPbase", parse_x_imap_base }, { "X-Keywords", parse_x_keywords }, { "X-Status", parse_x_status }, diff --git a/src/lib-storage/index/mbox/mbox-sync-private.h b/src/lib-storage/index/mbox/mbox-sync-private.h index 73a620e64d..4e3d991f1e 100644 --- a/src/lib-storage/index/mbox/mbox-sync-private.h +++ b/src/lib-storage/index/mbox/mbox-sync-private.h @@ -30,6 +30,7 @@ struct mbox_sync_mail { uint8_t flags; keywords_mask_t keywords; + uoff_t from_offset; uoff_t offset; /* if space <= 0, points to beginning */ off_t space; uoff_t body_size; @@ -52,6 +53,7 @@ struct mbox_sync_mail_context { unsigned int have_eoh:1; unsigned int need_rewrite:1; unsigned int seen_imapbase:1; + unsigned int pseudo:1; unsigned int updated:1; }; diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c index 261e7a9cea..2903489794 100644 --- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c @@ -196,17 +196,6 @@ int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff) return 1; } -static void mbox_sync_fix_from_offset(struct mbox_sync_context *sync_ctx, - uint32_t idx, off_t diff) -{ - uoff_t *offset_p; - - offset_p = buffer_get_space_unsafe(sync_ctx->ibox->mbox_data_buf, - idx * sizeof(*offset_p), - sizeof(*offset_p)); - *offset_p = (*offset_p & 1) | (((*offset_p >> 1) + diff) << 1); -} - static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx, struct mbox_sync_mail *mails, uint32_t seq, uint32_t idx, @@ -245,9 +234,6 @@ static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx, i_assert(mail_ctx.mail.space == mails[idx].space); sync_ctx->prev_msg_uid = old_prev_msg_uid; - /* we're moving next message - update it's from_offset */ - mbox_sync_fix_from_offset(sync_ctx, idx+1, mails[idx+1].space); - if (mail_ctx.mail.space <= 0) mbox_sync_headers_add_space(&mail_ctx, extra_per_mail); else if (mail_ctx.mail.space <= extra_per_mail) { @@ -267,6 +253,7 @@ static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx, // FIXME: error handling return -1; } + mails[idx+1].from_offset += mails[idx+1].space; *end_offset = offset + mails[idx+1].space - str_len(mail_ctx.header); @@ -334,7 +321,7 @@ int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK); mails = buffer_get_modifyable_data(mails_buf, &size); - size /= sizeof(*mails); + i_assert(size / sizeof(*mails) == last_seq - first_seq + 1); /* if there's expunges in mails[], we would get more correct balancing by counting only them here. however, that might make us overwrite @@ -386,8 +373,7 @@ int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, break; } - mbox_sync_fix_from_offset(sync_ctx, idx+1, - mails[idx+1].space); + mails[idx+1].from_offset += mails[idx+1].space; mails[idx].space += mails[idx+1].space; if (mails[idx].uid != 0) @@ -408,8 +394,7 @@ int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, // FIXME: error handling ret = -1; } - mbox_sync_fix_from_offset(sync_ctx, 1, - (off_t)end_offset - offset); + mails[1].from_offset -= offset - end_offset; idx++; start_offset += offset - end_offset; diff --git a/src/lib-storage/index/mbox/mbox-sync-update.c b/src/lib-storage/index/mbox/mbox-sync-update.c index bdcaf51c61..5faba2c984 100644 --- a/src/lib-storage/index/mbox/mbox-sync-update.c +++ b/src/lib-storage/index/mbox/mbox-sync-update.c @@ -90,7 +90,7 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) if (ctx->mail.uid == ctx->sync_ctx->first_uid && ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1) { ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); - str_printfa(ctx->header, "X-IMAPbase: %u %u", + str_printfa(ctx->header, "X-IMAPbase: %u %010u", ctx->sync_ctx->base_uid_validity, ctx->sync_ctx->next_uid-1); //FIXME:keywords_append(ctx, all_keywords); diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c index 6745228f72..91be657d8e 100644 --- a/src/lib-storage/index/mbox/mbox-sync.c +++ b/src/lib-storage/index/mbox/mbox-sync.c @@ -106,8 +106,6 @@ static void mbox_sync_next_mail(struct mbox_sync_context *sync_ctx, struct mbox_sync_mail_context *mail_ctx, uint32_t seq) { - uoff_t from_offset; - memset(mail_ctx, 0, sizeof(*mail_ctx)); mail_ctx->sync_ctx = sync_ctx; mail_ctx->seq = seq; @@ -132,14 +130,12 @@ mbox_sync_next_mail(struct mbox_sync_context *sync_ctx, mail_ctx->content_length); /* save the offset permanently with recent flag state */ - from_offset = (mail_ctx->from_offset - sync_ctx->expunged_space) << 1; + mail_ctx->mail.from_offset = mail_ctx->from_offset; if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) { /* need to add 'O' flag to Status-header */ mail_ctx->need_rewrite = TRUE; - from_offset |= 1; + // FIXME: save it somewhere } - buffer_append(sync_ctx->ibox->mbox_data_buf, &from_offset, - sizeof(from_offset)); } static void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags, @@ -202,10 +198,27 @@ mbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff) return 0; } +static void +update_from_offsets(struct index_mailbox *ibox, + struct mail_index_transaction *t, buffer_t *mails_buf, + uint32_t seq1, uint32_t seq2) +{ + uint32_t extra_idx = ibox->mbox_extra_idx; + const struct mbox_sync_mail *mails; + + mails = buffer_get_modifyable_data(mails_buf, NULL); + + for (; seq1 <= seq2; seq1++, mails++) { + uint64_t offset = mails->from_offset; + mail_index_update_extra_rec(t, seq1, extra_idx, &offset); + } +} + static int mbox_sync_do(struct index_mailbox *ibox, struct mail_index_sync_ctx *index_sync_ctx, struct mail_index_view *sync_view, - buffer_t *syncs, struct mail_index_sync_rec *sync_rec) + buffer_t *syncs, struct mail_index_sync_rec *sync_rec, + int index_synced) { /* a horrible function. needs some serious cleanups. */ struct mbox_sync_context sync_ctx; @@ -219,18 +232,13 @@ static int mbox_sync_do(struct index_mailbox *ibox, off_t space_diff, move_diff; uoff_t offset, extra_space, trailer_size; buffer_t *mails; - size_t size; struct stat st; - int sync_expunge, ret = 0; + int sync_expunge, update_from_offset, ret = 0; - t = mail_index_transaction_begin(sync_view, FALSE); + if (mail_index_get_header(sync_view, &hdr) < 0) + return -1; - if (ibox->mbox_data_buf == NULL) { - ibox->mbox_data_buf = - buffer_create_dynamic(default_pool, 512, (size_t)-1); - } else { - buffer_set_used_size(ibox->mbox_data_buf, 0); - } + t = mail_index_transaction_begin(sync_view, FALSE); memset(&sync_ctx, 0, sizeof(sync_ctx)); sync_ctx.ibox = ibox; @@ -279,10 +287,6 @@ static int mbox_sync_do(struct index_mailbox *ibox, break; if (seq == 1 && sync_ctx.base_uid_validity == 0) { - if (mail_index_get_header(sync_view, &hdr) < 0) { - ret = -1; - break; - } sync_ctx.base_uid_validity = hdr->uid_validity == 0 ? (uint32_t)ioloop_time : hdr->uid_validity; @@ -327,6 +331,7 @@ static int mbox_sync_do(struct index_mailbox *ibox, move_diff); if (ret > 0) { + mail_ctx.mail.from_offset += move_diff; mail_ctx.mail.offset += move_diff; ret = mbox_write_from_line(&mail_ctx, move_diff); @@ -393,6 +398,7 @@ static int mbox_sync_do(struct index_mailbox *ibox, if (sync_expunge) { if (rec != NULL) mail_index_expunge(t, idx_seq); + update_from_offset = FALSE; } else if (rec != NULL) { /* see if flags changed */ keywords_mask_t old_keywords; @@ -417,6 +423,8 @@ static int mbox_sync_do(struct index_mailbox *ibox, mail_ctx.mail.keywords); } + update_from_offset = sync_ctx.expunged_space > 0; + /* we used this record */ rec = NULL; } else { @@ -427,6 +435,34 @@ static int mbox_sync_do(struct index_mailbox *ibox, mail_index_update_flags(t, idx_seq, MODIFY_REPLACE, new_flags, mail_ctx.mail.keywords); + update_from_offset = TRUE; + } + + if (update_from_offset) { + /* from_offset needs updating */ + uint64_t offset; + + offset = mail_ctx.mail.from_offset; + mail_index_update_extra_rec(t, idx_seq, + ibox->mbox_extra_idx, &offset); + } else if (need_space_seq == 0 && !sync_expunge) { + /* see if from_offset needs updating */ + const void *data; + uint64_t offset; + + if (mail_index_lookup_extra(sync_view, idx_seq, + ibox->mbox_extra_idx, + &data) < 0) { + ret = -1; + break; + } + + offset = *((const uint64_t *)data); + if (offset != mail_ctx.mail.from_offset) { + offset = mail_ctx.mail.from_offset; + mail_index_update_extra_rec(t, idx_seq, + ibox->mbox_extra_idx, &offset); + } } istream_raw_mbox_next(input, mail_ctx.mail.body_size); @@ -474,6 +510,10 @@ static int mbox_sync_do(struct index_mailbox *ibox, ret = -1; break; } + + update_from_offsets(ibox, t, mails, + need_space_seq, seq); + /* mail_ctx may contain wrong data after rewrite, so make sure we don't try to access it */ @@ -513,6 +553,9 @@ static int mbox_sync_do(struct index_mailbox *ibox, if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq, seq, extra_space) < 0) ret = -1; + + update_from_offsets(ibox, t, mails, + need_space_seq, seq); } } @@ -562,9 +605,6 @@ static int mbox_sync_do(struct index_mailbox *ibox, st.st_size = 0; } - if (mail_index_get_header(sync_view, &hdr) < 0) - ret = -1; - if (sync_ctx.base_uid_validity != hdr->uid_validity) { mail_index_update_header(t, offsetof(struct mail_index_header, uid_validity), @@ -615,9 +655,6 @@ static int mbox_sync_do(struct index_mailbox *ibox, mail_storage_set_index_error(ibox); } - ibox->mbox_data = buffer_get_data(ibox->mbox_data_buf, &size); - ibox->mbox_data_count = size / sizeof(*ibox->mbox_data); - str_free(sync_ctx.header); str_free(sync_ctx.from_line); buffer_free(mails); @@ -652,10 +689,13 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit) uoff_t offset; struct mail_index_sync_rec sync_rec; buffer_t *syncs; - int ret, lock_type; + int ret, lock_type, index_synced; - if ((ret = mbox_sync_has_changed(ibox)) <= 0) - return ret; + if ((ret = mbox_sync_has_changed(ibox)) < 0) + return -1; + if (ret == 0 && !last_commit) + return 0; + index_synced = ret > 0; if (last_commit) { seq = ibox->commit_log_file_seq; @@ -681,7 +721,7 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit) if (mbox_lock(ibox, lock_type, &lock_id) > 0 && mbox_file_open_stream(ibox) == 0) { ret = mbox_sync_do(ibox, index_sync_ctx, sync_view, - syncs, &sync_rec); + syncs, &sync_rec, index_synced); if (ret == -2) { /* read lock -> write lock. do it again. */ (void)mbox_unlock(ibox, lock_id); @@ -692,7 +732,8 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit) ret = -1; else { ret = mbox_sync_do(ibox, index_sync_ctx, - sync_view, syncs, &sync_rec); + sync_view, syncs, &sync_rec, + FALSE); } } } else {