From e68309fcfa2eaa88217fd51e7b4900fc9c20ef5d Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Fri, 11 Jun 2004 06:20:10 +0300 Subject: [PATCH] expunging is somewhat working --HG-- branch : HEAD --- src/lib-storage/index/mbox/istream-raw-mbox.c | 5 +- .../index/mbox/mbox-sync-private.h | 7 +- .../index/mbox/mbox-sync-rewrite.c | 55 ++++-- src/lib-storage/index/mbox/mbox-sync-update.c | 5 - src/lib-storage/index/mbox/mbox-sync.c | 157 ++++++++++++++++-- 5 files changed, 187 insertions(+), 42 deletions(-) diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.c b/src/lib-storage/index/mbox/istream-raw-mbox.c index 0eebf230c1..228f6050cb 100644 --- a/src/lib-storage/index/mbox/istream-raw-mbox.c +++ b/src/lib-storage/index/mbox/istream-raw-mbox.c @@ -201,8 +201,9 @@ static ssize_t _read(struct _istream *stream) stream->buffer = buf; stream->pos = new_pos; - if (i < pos && new_pos == stream->pos) { - /* beginning from From-line, try again */ + if (i < pos) { + /* beginning from From-line, try again + FIXME: loops forever if we don't skip forward */ ret = 0; } diff --git a/src/lib-storage/index/mbox/mbox-sync-private.h b/src/lib-storage/index/mbox/mbox-sync-private.h index 180b7cc3c0..3863949274 100644 --- a/src/lib-storage/index/mbox/mbox-sync-private.h +++ b/src/lib-storage/index/mbox/mbox-sync-private.h @@ -40,7 +40,7 @@ struct mbox_sync_mail_context { struct mbox_sync_mail mail; uint32_t seq; - uoff_t hdr_offset, body_offset; + uoff_t from_offset, hdr_offset, body_offset; size_t header_first_change, header_last_change; string_t *header; @@ -59,9 +59,10 @@ struct mbox_sync_context { struct istream *input, *file_input; int fd; - buffer_t *header; + string_t *header, *from_line; uint32_t base_uid_validity, base_uid_last; uint32_t prev_msg_uid, next_uid; + off_t expunged_space; }; int mbox_sync(struct index_mailbox *ibox, int last_commit); @@ -72,7 +73,7 @@ void mbox_sync_update_header(struct mbox_sync_mail_context *ctx, buffer_t *syncs_buf); void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx, const struct mbox_sync_mail *mail); -int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx); +int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff); int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, uint32_t first_seq, uint32_t last_seq, off_t extra_space); diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c index cf170e6714..704fc925cd 100644 --- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c @@ -16,6 +16,9 @@ int mbox_move(struct mbox_sync_context *sync_ctx, struct ostream *output; off_t ret; + if (size == 0 || source == dest) + return 0; + istream_raw_mbox_flush(sync_ctx->input); output = o_stream_create_file(sync_ctx->fd, default_pool, 4096, FALSE); @@ -24,15 +27,18 @@ int mbox_move(struct mbox_sync_context *sync_ctx, if (size == (uoff_t)-1) { input = sync_ctx->file_input; - return o_stream_send_istream(output, input) < 0 ? -1 : 0; + ret = o_stream_send_istream(output, input) < 0 ? -1 : 0; } else { input = i_stream_create_limit(default_pool, sync_ctx->file_input, source, size); ret = o_stream_send_istream(output, input); i_stream_unref(input); - return ret == (off_t)size ? 0 : -1; + ret = ret == (off_t)size ? 0 : -1; } + + o_stream_unref(output); + return (int)ret; } static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, @@ -134,7 +140,7 @@ static void mbox_sync_headers_remove_space(struct mbox_sync_mail_context *ctx, i_assert(size == 0); } -int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx) +int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff) { size_t old_hdr_size, new_hdr_size; const unsigned char *data; @@ -149,13 +155,27 @@ int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx) mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size); } else if (new_hdr_size > old_hdr_size) { size_t needed = new_hdr_size - old_hdr_size; - if (ctx->mail.space < 0) - return 0; - mbox_sync_headers_remove_space(ctx, needed); + if (ctx->mail.space >= 0) + mbox_sync_headers_remove_space(ctx, needed); + else if (move_diff < 0 && needed <= -move_diff) { + /* moving backwards - we can use the extra space from + it, just update expunged_space accordingly */ + i_assert(ctx->sync_ctx->expunged_space >= needed); + ctx->sync_ctx->expunged_space -= needed; + } else { + return 0; + } } - i_assert(ctx->header_first_change != (size_t)-1); + i_assert(ctx->header_first_change != (size_t)-1 || move_diff != 0); + + if (move_diff != 0) { + /* we're moving the header, forget about partial write + optimizations */ + ctx->header_first_change = 0; + ctx->header_last_change = 0; + } /* FIXME: last_change should rather just tell if we want to truncate to beginning of extra whitespace */ @@ -164,10 +184,11 @@ int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx) str_truncate(ctx->header, ctx->header_last_change); data = str_data(ctx->header); - new_hdr_size = str_len(ctx->header); + new_hdr_size = str_len(ctx->header); if (pwrite_full(ctx->sync_ctx->fd, data + ctx->header_first_change, new_hdr_size - ctx->header_first_change, - ctx->hdr_offset + ctx->header_first_change) < 0) { + ctx->hdr_offset + move_diff + + ctx->header_first_change) < 0) { // FIXME: error handling return -1; } @@ -253,6 +274,7 @@ int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, uint32_t idx, extra_per_mail; int ret = 0; + i_assert(first_seq != last_seq); i_assert(sync_ctx->ibox->mbox_lock_type == F_WRLCK); mails = buffer_get_modifyable_data(mails_buf, &size); @@ -263,13 +285,16 @@ int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, extra_per_mail = (extra_space / (last_seq - first_seq + 1)); - mails[last_seq-1].space -= extra_per_mail; - i_assert(mails[last_seq-1].space >= 0); - end_offset = mails[last_seq-1].offset + mails[last_seq-1].space; + last_seq--; + idx = last_seq - first_seq; + + mails[idx].space -= extra_per_mail; + i_assert(mails[idx].space >= 0); + end_offset = mails[idx].offset + mails[idx].space; /* start moving backwards */ - while (--last_seq >= first_seq) { - idx = last_seq-1; + do { + idx--; if (mails[idx].space <= 0) { /* offset points to beginning of headers. read the header again, update it and give enough space to @@ -303,7 +328,7 @@ int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, i_assert(mails[idx].space > 0); end_offset = mails[idx].offset + mails[idx].space; } - } + } while (idx > 0); istream_raw_mbox_flush(sync_ctx->input); return ret; diff --git a/src/lib-storage/index/mbox/mbox-sync-update.c b/src/lib-storage/index/mbox/mbox-sync-update.c index e3ed62f481..394257cfaf 100644 --- a/src/lib-storage/index/mbox/mbox-sync-update.c +++ b/src/lib-storage/index/mbox/mbox-sync-update.c @@ -155,11 +155,6 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) } } - if (ctx->header_first_change == (size_t)-1) { - /* no headers had to be modified */ - return; - } - if (ctx->have_eoh) str_append_c(ctx->header, '\n'); } diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c index d6749b027c..b913b6a243 100644 --- a/src/lib-storage/index/mbox/mbox-sync.c +++ b/src/lib-storage/index/mbox/mbox-sync.c @@ -113,19 +113,20 @@ mbox_sync_next_mail(struct mbox_sync_context *sync_ctx, mail_ctx->seq = seq; mail_ctx->header = sync_ctx->header; - from_offset = istream_raw_mbox_get_start_offset(sync_ctx->input); + mail_ctx->from_offset = + istream_raw_mbox_get_start_offset(sync_ctx->input); mail_ctx->mail.offset = istream_raw_mbox_get_header_offset(sync_ctx->input); mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx, FALSE); - i_assert(sync_ctx->input->v_offset != from_offset); + i_assert(sync_ctx->input->v_offset != mail_ctx->from_offset); mail_ctx->mail.body_size = istream_raw_mbox_get_body_size(sync_ctx->input, mail_ctx->content_length); /* save the offset permanently with recent flag state */ - from_offset <<= 1; + from_offset = (mail_ctx->from_offset - sync_ctx->expunged_space) << 1; if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) { /* need to add 'O' flag to Status-header */ mail_ctx->need_rewrite = TRUE; @@ -148,10 +149,66 @@ static void mbox_sync_apply_index_syncs(buffer_t *syncs_buf, uint8_t *flags, mail_index_sync_flags_apply(&sync[i], flags, keywords); } +static int mbox_read_from_line(struct mbox_sync_mail_context *ctx) +{ + struct istream *input = ctx->sync_ctx->file_input; + const unsigned char *data; + size_t size, from_line_size; + + buffer_set_used_size(ctx->sync_ctx->from_line, 0); + from_line_size = ctx->hdr_offset - ctx->from_offset; + + i_stream_seek(input, ctx->from_offset); + for (;;) { + data = i_stream_get_data(input, &size); + if (size >= from_line_size) + size = from_line_size; + + buffer_append(ctx->sync_ctx->from_line, data, size); + i_stream_skip(input, size); + from_line_size -= size; + + if (from_line_size == 0) + break; + + if (i_stream_read(input) < 0) + return -1; + } + + return 0; +} + +static int +mbox_write_from_line(struct mbox_sync_mail_context *ctx, off_t move_diff) +{ + string_t *str = ctx->sync_ctx->from_line; + + if (move_diff == 0) + return 0; + + if (ctx->from_offset + move_diff == 0) { + /* FIXME: kludge: we're writing the first header, + change the \n prefix into space suffix */ + buffer_copy(str, 0, str, 1, (size_t)-1); + str_truncate(str, str_len(str)-2); + str_append(str, " \n"); + } + + if (pwrite_full(ctx->sync_ctx->fd, str_data(str), str_len(str), + ctx->from_offset + move_diff) < 0) { + // FIXME: error handling + return -1; + } + + istream_raw_mbox_flush(ctx->sync_ctx->input); + return 0; +} + static int mbox_sync_do(struct index_mailbox *ibox, struct mail_index_sync_ctx *index_sync_ctx, struct mail_index_view *sync_view) { + /* FIXME: expunging + mbox_sync_rewrite() is broken within same sync */ struct mbox_sync_context sync_ctx; struct mbox_sync_mail_context mail_ctx; struct mail_index_sync_rec sync_rec; @@ -161,8 +218,8 @@ static int mbox_sync_do(struct index_mailbox *ibox, struct istream *input; uint32_t seq, need_space_seq, idx_seq, messages_count; uint8_t new_flags; - off_t space_diff; - uoff_t offset, extra_space; + off_t space_diff, move_diff; + uoff_t offset, extra_space, trailer_size; buffer_t *mails, *syncs; size_t size; struct stat st; @@ -182,6 +239,7 @@ static int mbox_sync_do(struct index_mailbox *ibox, sync_ctx.file_input = ibox->mbox_file_stream; sync_ctx.input = ibox->mbox_stream; sync_ctx.fd = ibox->mbox_fd; + sync_ctx.from_line = str_new(default_pool, 256); sync_ctx.header = str_new(default_pool, 4096); sync_ctx.next_uid = 1; @@ -214,10 +272,8 @@ static int mbox_sync_do(struct index_mailbox *ibox, sizeof(sync_rec)); if (sync_rec.type == - MAIL_INDEX_SYNC_TYPE_EXPUNGE) { + MAIL_INDEX_SYNC_TYPE_EXPUNGE) sync_expunge = TRUE; - break; - } } ret = mail_index_sync_next(index_sync_ctx, &sync_rec); if (ret == 0) @@ -236,16 +292,45 @@ static int mbox_sync_do(struct index_mailbox *ibox, hdr->uid_validity; } - if ((mail_ctx.need_rewrite || + if ((mail_ctx.need_rewrite || sync_ctx.expunged_space > 0 || buffer_get_used_size(syncs) != 0) && !ibox->readonly) { if (ibox->mbox_lock_type == F_RDLCK) { ret = -2; break; } - mbox_sync_update_header(&mail_ctx, syncs); - if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0) - return -1; + if (sync_expunge) { + ret = 1; + sync_ctx.expunged_space += + mail_ctx.body_offset - + mail_ctx.from_offset + + mail_ctx.mail.body_size; + } else { + move_diff = need_space_seq != 0 ? 0 : + -sync_ctx.expunged_space; + + /* read the From-line */ + if (move_diff != 0 && + mbox_read_from_line(&mail_ctx) < 0) { + ret = -1; + break; + } + + mbox_sync_update_header(&mail_ctx, syncs); + ret = mbox_sync_try_rewrite(&mail_ctx, + move_diff); + + if (ret > 0) { + mail_ctx.mail.offset += move_diff; + ret = mbox_write_from_line(&mail_ctx, + move_diff); + if (ret == 0) + ret = 1; + } + + if (ret < 0) + break; + } if (ret == 0 && need_space_seq == 0) { /* first mail with no space to write it */ @@ -285,7 +370,8 @@ static int mbox_sync_do(struct index_mailbox *ibox, } if (sync_expunge) { - /* .. */ + if (rec != NULL) + mail_index_expunge(t, idx_seq); } else if (rec != NULL) { /* see if flags changed */ keywords_mask_t old_keywords; @@ -325,12 +411,26 @@ static int mbox_sync_do(struct index_mailbox *ibox, istream_raw_mbox_next(input, mail_ctx.mail.body_size); offset = istream_raw_mbox_get_start_offset(input); + if (sync_ctx.expunged_space > 0 && !sync_expunge && + need_space_seq == 0) { + /* move the body */ + if (mbox_move(&sync_ctx, + mail_ctx.body_offset - + sync_ctx.expunged_space, + mail_ctx.body_offset, + mail_ctx.mail.body_size) < 0) { + ret = -1; + break; + } + i_stream_seek(input, offset); + } + if (need_space_seq != 0) { buffer_append(mails, &mail_ctx.mail, sizeof(mail_ctx.mail)); space_diff += mail_ctx.mail.space; - if (space_diff >= 0) { + if (space_diff + sync_ctx.expunged_space >= 0) { /* we have enough space now */ if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq, seq, @@ -343,20 +443,27 @@ static int mbox_sync_do(struct index_mailbox *ibox, it */ memset(&mail_ctx, 0, sizeof(mail_ctx)); i_stream_seek(input, offset); + need_space_seq = 0; + buffer_set_used_size(mails, 0); } } } + trailer_size = i_stream_get_size(sync_ctx.file_input) - offset; if (need_space_seq != 0 && ret >= 0) { i_assert(space_diff < 0); extra_space = MBOX_HEADER_EXTRA_SPACE * (seq - need_space_seq + 1); + space_diff -= extra_space; + + space_diff += sync_ctx.expunged_space; + sync_ctx.expunged_space -= -space_diff; - if (mbox_sync_grow_file(&sync_ctx, &mail_ctx, - -space_diff + extra_space) < 0) + if (space_diff < 0 && + mbox_sync_grow_file(&sync_ctx, &mail_ctx, -space_diff) < 0) ret = -1; - else if (mbox_sync_try_rewrite(&mail_ctx) < 0) + else if (mbox_sync_try_rewrite(&mail_ctx, 0) < 0) ret = -1; else if (seq != need_space_seq) { buffer_set_used_size(mails, @@ -370,6 +477,21 @@ static int mbox_sync_do(struct index_mailbox *ibox, } } + if (sync_ctx.expunged_space > 0) { + /* copy trailer, then truncate the file */ + offset = i_stream_get_size(sync_ctx.file_input) - + sync_ctx.expunged_space - trailer_size; + + if (mbox_move(&sync_ctx, offset, + offset + sync_ctx.expunged_space, + trailer_size) < 0) + ret = -1; + else if (ftruncate(ibox->mbox_fd, offset + trailer_size) < 0) + ret = -1; + + istream_raw_mbox_flush(input); + } + if (ret >= 0) { if (rec != NULL) mail_index_expunge(t, idx_seq); @@ -460,6 +582,7 @@ static int mbox_sync_do(struct index_mailbox *ibox, ibox->mbox_data_count = size / sizeof(*ibox->mbox_data); str_free(sync_ctx.header); + str_free(sync_ctx.from_line); buffer_free(mails); buffer_free(syncs); return ret < 0 ? ret : 0; -- 2.47.3