From: Timo Sirainen Date: Sun, 9 May 2004 17:06:06 +0000 (+0300) Subject: mbox rewriting is almost working - the hard part is done. X-Git-Tag: 1.1.alpha1~4125 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=313fe89df4d91cd0cd7f3558dc6d7fd21ad39eee;p=thirdparty%2Fdovecot%2Fcore.git mbox rewriting is almost working - the hard part is done. --HG-- branch : HEAD --- diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.c b/src/lib-storage/index/mbox/istream-raw-mbox.c index e0f0799848..261c130280 100644 --- a/src/lib-storage/index/mbox/istream-raw-mbox.c +++ b/src/lib-storage/index/mbox/istream-raw-mbox.c @@ -12,7 +12,7 @@ struct raw_mbox_istream { time_t received_time, next_received_time; char *sender, *next_sender; - uoff_t from_offset, hdr_offset, next_from_offset, body_size; + uoff_t from_offset, hdr_offset, body_offset, mail_size; struct istream *input; }; @@ -103,7 +103,7 @@ static ssize_t _read(struct _istream *stream) struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream; const unsigned char *buf; const char *fromp; - char *sender; + char *sender, eoh_char; time_t received_time; size_t i, pos; ssize_t ret; @@ -119,10 +119,12 @@ static ssize_t _read(struct _istream *stream) buf = i_stream_get_data(rstream->input, &pos); } while (ret > 0 && pos <= 6); - if (pos == 1 && buf[0] == '\n') { + if (pos == 0 || (pos == 1 && buf[0] == '\n')) { /* EOF */ stream->pos = 0; stream->istream.eof = TRUE; + rstream->mail_size = stream->istream.v_offset - + rstream->hdr_offset; return -1; } @@ -140,6 +142,8 @@ static ssize_t _read(struct _istream *stream) mbox_from_parse(buf+6, pos-6, &received_time, &sender) == 0) { rstream->next_received_time = received_time; + rstream->mail_size = stream->istream.v_offset - + rstream->hdr_offset; i_free(rstream->next_sender); rstream->next_sender = sender; @@ -148,9 +152,17 @@ static ssize_t _read(struct _istream *stream) } } else if (ret == -1) { /* last few bytes, can't contain From-line */ + if (buf[pos-1] == '\n') { + /* last LF doesn't belong to last message */ + pos--; + } + ret = pos <= stream->pos ? -1 : (ssize_t) (pos - stream->pos); + rstream->mail_size = stream->istream.v_offset + pos - + rstream->hdr_offset; + stream->buffer = buf; stream->pos = pos; stream->istream.eof = ret == -1; @@ -159,7 +171,12 @@ static ssize_t _read(struct _istream *stream) /* See if we have From-line here - note that it works right only because all characters are different in mbox_from. */ + eoh_char = rstream->body_offset == (uoff_t)-1 ? '\n' : '\0'; for (i = 0, fromp = mbox_from; i < pos; i++) { + if (buf[i] == eoh_char && i > 0 && buf[i-1] == '\n') { + rstream->body_offset = stream->istream.v_offset + i + 1; + eoh_char = '\0'; + } if (buf[i] == *fromp) { if (*++fromp == '\0') { /* potential From-line - stop here */ @@ -197,7 +214,10 @@ struct istream *i_stream_create_raw_mbox(pool_t pool, struct istream *input) rstream = p_new(pool, struct raw_mbox_istream, 1); rstream->input = input; - rstream->body_size = (uoff_t)-1; + rstream->body_offset = (uoff_t)-1; + rstream->mail_size = (uoff_t)-1; + rstream->received_time = (time_t)-1; + rstream->next_received_time = (time_t)-1; rstream->istream.iostream.close = _close; rstream->istream.iostream.destroy = _destroy; @@ -244,7 +264,26 @@ static int istream_raw_mbox_is_valid_from(struct raw_mbox_istream *rstream) return TRUE; } -uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size) +uoff_t istream_raw_mbox_get_start_offset(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + return rstream->from_offset; +} + +uoff_t istream_raw_mbox_get_header_offset(struct istream *stream) +{ + struct raw_mbox_istream *rstream = + (struct raw_mbox_istream *)stream->real_stream; + + if (rstream->hdr_offset == rstream->from_offset) + (void)_read(&rstream->istream); + + return rstream->hdr_offset; +} + +uoff_t istream_raw_mbox_get_body_size(struct istream *stream, uoff_t body_size) { struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream->real_stream; @@ -252,26 +291,29 @@ uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size) const unsigned char *data; size_t size; - if (rstream->body_size != (uoff_t)-1) - return rstream->body_size; + if (rstream->mail_size != (uoff_t)-1) { + return rstream->mail_size - + (rstream->body_offset - rstream->hdr_offset); + } if (body_size != (uoff_t)-1) { - i_stream_seek(rstream->input, rstream->from_offset + body_size); + i_assert(rstream->body_offset != (uoff_t)-1); + i_stream_seek(rstream->input, rstream->body_offset + body_size); if (istream_raw_mbox_is_valid_from(rstream) > 0) { - rstream->body_size = body_size; + rstream->mail_size = body_size + + (rstream->body_offset - rstream->hdr_offset); return body_size; } } - old_offset = stream->v_offset; - /* have to read through the message body */ while (i_stream_read_data(stream, &data, &size, 0) > 0) i_stream_skip(stream, size); - rstream->body_size = stream->v_offset - old_offset; + i_assert(rstream->mail_size != (uoff_t)-1); i_stream_seek(stream, old_offset); - return rstream->body_size; + return rstream->mail_size - + (rstream->body_offset - rstream->hdr_offset); } time_t istream_raw_mbox_get_received_time(struct istream *stream) @@ -279,7 +321,8 @@ time_t istream_raw_mbox_get_received_time(struct istream *stream) struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream->real_stream; - (void)_read(&rstream->istream); + if (rstream->received_time == (time_t)-1) + (void)_read(&rstream->istream); return rstream->received_time; } @@ -288,7 +331,8 @@ const char *istream_raw_mbox_get_sender(struct istream *stream) struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream->real_stream; - (void)_read(&rstream->istream); + if (rstream->sender == NULL) + (void)_read(&rstream->istream); return rstream->sender == NULL ? "" : rstream->sender; } @@ -297,8 +341,8 @@ void istream_raw_mbox_next(struct istream *stream, uoff_t body_size) struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream->real_stream; - body_size = istream_raw_mbox_get_size(stream, body_size); - rstream->body_size = (uoff_t)-1; + body_size = istream_raw_mbox_get_body_size(stream, body_size); + rstream->mail_size = (uoff_t)-1; rstream->received_time = rstream->next_received_time; rstream->next_received_time = (time_t)-1; @@ -307,8 +351,9 @@ void istream_raw_mbox_next(struct istream *stream, uoff_t body_size) rstream->sender = rstream->next_sender; rstream->next_sender = NULL; - rstream->from_offset = stream->v_offset + body_size; + rstream->from_offset = rstream->body_offset + body_size; rstream->hdr_offset = rstream->from_offset; + rstream->body_offset = (uoff_t)-1; /* don't clear stream->eof if we don't have to */ if (stream->v_offset != rstream->from_offset) @@ -321,7 +366,8 @@ void istream_raw_mbox_seek(struct istream *stream, uoff_t offset) struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream->real_stream; - if (offset == rstream->next_from_offset) { + if (rstream->mail_size != (uoff_t)-1 && + rstream->hdr_offset + rstream->mail_size == offset) { istream_raw_mbox_next(stream, (uoff_t)-1); return; } @@ -330,7 +376,8 @@ void istream_raw_mbox_seek(struct istream *stream, uoff_t offset) /* back to beginning of current message */ offset = rstream->hdr_offset; } else { - rstream->body_size = (uoff_t)-1; + rstream->body_offset = (uoff_t)-1; + rstream->mail_size = (uoff_t)-1; rstream->received_time = (time_t)-1; rstream->next_received_time = (time_t)-1; @@ -338,9 +385,11 @@ void istream_raw_mbox_seek(struct istream *stream, uoff_t offset) rstream->sender = NULL; i_free(rstream->next_sender); rstream->next_sender = NULL; + + rstream->from_offset = offset; + rstream->hdr_offset = offset; } - rstream->from_offset = rstream->hdr_offset = offset; i_stream_seek(stream, offset); i_stream_seek(rstream->input, offset); } diff --git a/src/lib-storage/index/mbox/istream-raw-mbox.h b/src/lib-storage/index/mbox/istream-raw-mbox.h index bc66adc16c..9d47fe0c82 100644 --- a/src/lib-storage/index/mbox/istream-raw-mbox.h +++ b/src/lib-storage/index/mbox/istream-raw-mbox.h @@ -5,10 +5,15 @@ you'll have to call istream_raw_mbox_next() to get to next message. */ struct istream *i_stream_create_raw_mbox(pool_t pool, struct istream *input); -/* Return number of bytes in this message after current offset. - If body_size isn't (uoff_t)-1, we'll use it as potentially valid body size - to avoid actually reading through the whole message. */ -uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size); +/* Return offset to beginning of the "\nFrom"-line. */ +uoff_t istream_raw_mbox_get_start_offset(struct istream *stream); +/* Return offset to beginning of the headers. */ +uoff_t istream_raw_mbox_get_header_offset(struct istream *stream); + +/* Return the number of bytes in the body of this message. If body_size isn't + (uoff_t)-1, we'll use it as potentially valid body size to avoid actually + reading through the whole message. */ +uoff_t istream_raw_mbox_get_body_size(struct istream *stream, uoff_t body_size); /* Return received time of current message, or (time_t)-1 if the timestamp is broken. */ diff --git a/src/lib-storage/index/mbox/mbox-sync-parse.c b/src/lib-storage/index/mbox/mbox-sync-parse.c index 70a2db6acf..05398a18ce 100644 --- a/src/lib-storage/index/mbox/mbox-sync-parse.c +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c @@ -52,7 +52,7 @@ static void parse_status_flags(struct mbox_sync_mail_context *ctx, size_t i; for (i = 0; i < hdr->full_value_len; i++) { - ctx->mail->flags |= + ctx->mail.flags |= mbox_flag_find(flags_list, hdr->full_value[i]); } } @@ -79,8 +79,9 @@ static int parse_x_imap_base(struct mbox_sync_mail_context *ctx, const char *str; char *end; size_t pos; + uint32_t uid_validity, uid_last; - if (ctx->seq != 1 || ctx->base_uid_validity != 0) { + if (ctx->seq != 1 || ctx->seen_imapbase) { /* Valid only in first message */ return FALSE; } @@ -88,15 +89,15 @@ static int parse_x_imap_base(struct mbox_sync_mail_context *ctx, /* */ t_push(); str = t_strndup(hdr->full_value, hdr->full_value_len); - ctx->base_uid_validity = strtoul(str, &end, 10); - ctx->base_uid_last = strtoul(end, &end, 10); + uid_validity = strtoul(str, &end, 10); + uid_last = strtoul(end, &end, 10); pos = end - str; t_pop(); while (pos < hdr->full_value_len && IS_LWSP_LF(hdr->full_value[pos])) pos++; - if (ctx->base_uid_validity == 0) { + if (uid_validity == 0) { /* broken */ return FALSE; } @@ -106,7 +107,14 @@ static int parse_x_imap_base(struct mbox_sync_mail_context *ctx, // FIXME: save keywords + if (ctx->sync_ctx->base_uid_validity == 0) { + ctx->sync_ctx->base_uid_validity = uid_validity; + ctx->sync_ctx->base_uid_last = uid_last; + ctx->sync_ctx->next_uid = uid_last+1; + } + ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); + ctx->seen_imapbase = TRUE; return TRUE; } @@ -115,15 +123,16 @@ static int parse_x_keywords(struct mbox_sync_mail_context *ctx, { size_t i, space = 0; - for (i = hdr->full_value_len; i > 0; i++) { + for (i = hdr->full_value_len; i > 0; i--) { if (!IS_LWSP_LF(hdr->full_value[i-1])) break; space++; } - if (space > ctx->mail->space) { - ctx->mail->space_offset = hdr->full_value_offset + i; - ctx->mail->space = space; + if (space > ctx->mail.space) { + ctx->mail.offset = ctx->hdr_offset + + hdr->full_value_offset + i; + ctx->mail.space = space; } // FIXME: parse them @@ -138,7 +147,7 @@ static int parse_x_uid(struct mbox_sync_mail_context *ctx, uint32_t value = 0; size_t i, space_pos, extra_space = 0; - if (ctx->mail->uid != 0) { + if (ctx->mail.uid != 0) { /* duplicate */ return FALSE; } @@ -162,15 +171,17 @@ static int parse_x_uid(struct mbox_sync_mail_context *ctx, /* broken - UIDs must be growing */ return FALSE; } + ctx->sync_ctx->prev_msg_uid = value; ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); - ctx->mail->uid = value; - if (ctx->mail->space == 0) { + ctx->mail.uid = value; + if (extra_space != 0 && ctx->mail.space == 0) { /* set it only if X-Keywords hasn't been seen. spaces in X-UID should be removed when writing X-Keywords. */ - ctx->mail->space_offset = hdr->full_value_offset + space_pos; - ctx->mail->space = extra_space; + ctx->mail.offset = ctx->hdr_offset + + hdr->full_value_offset + space_pos; + ctx->mail.space = extra_space; } return TRUE; } @@ -233,8 +244,7 @@ void mbox_sync_parse_next_mail(struct istream *input, size_t line_start_pos; int i; - ctx->hdr_offset = input->v_offset; - ctx->mail->space_offset = input->v_offset; + ctx->hdr_offset = ctx->mail.offset; ctx->header_first_change = (size_t)-1; ctx->header_last_change = (size_t)-1; @@ -253,6 +263,12 @@ void mbox_sync_parse_next_mail(struct istream *input, break; } + if (!hdr->continued) { + line_start_pos = str_len(ctx->header); + str_append(ctx->header, hdr->name); + str_append(ctx->header, ": "); + } + func = header_func_find(hdr->name); if (func != NULL) { if (hdr->continues) @@ -260,23 +276,15 @@ void mbox_sync_parse_next_mail(struct istream *input, else if (!func->func(ctx, hdr)) { /* this header is broken, remove it */ ctx->need_rewrite = TRUE; - if (hdr->continued) { - str_truncate(ctx->header, - line_start_pos); - } + str_truncate(ctx->header, line_start_pos); if (ctx->header_first_change == (size_t)-1) { ctx->header_first_change = - str_len(ctx->header); + line_start_pos; } continue; } } - if (!hdr->continued) { - line_start_pos = str_len(ctx->header); - str_append(ctx->header, hdr->name); - str_append(ctx->header, ": "); - } buffer_append(ctx->header, hdr->full_value, hdr->full_value_len); if (!hdr->no_newline) @@ -284,10 +292,14 @@ void mbox_sync_parse_next_mail(struct istream *input, } message_parse_header_deinit(hdr_ctx); - if (ctx->seq == 1 && ctx->base_uid_validity == 0) { + if (ctx->seq == 1 && ctx->sync_ctx->base_uid_validity == 0) { /* missing X-IMAPbase */ ctx->need_rewrite = TRUE; } + if (ctx->mail.uid == 0) { + /* missing X-UID */ + ctx->need_rewrite = TRUE; + } ctx->body_offset = input->v_offset; } diff --git a/src/lib-storage/index/mbox/mbox-sync-private.h b/src/lib-storage/index/mbox/mbox-sync-private.h index f1e797d47a..b65d39f81f 100644 --- a/src/lib-storage/index/mbox/mbox-sync-private.h +++ b/src/lib-storage/index/mbox/mbox-sync-private.h @@ -30,14 +30,14 @@ struct mbox_sync_mail { uint8_t flags; keywords_mask_t keywords; - uoff_t space_offset; /* if space is negative, points to beginning */ + uoff_t offset; /* if space <= 0, points to beginning */ off_t space; uoff_t body_size; }; struct mbox_sync_mail_context { struct mbox_sync_context *sync_ctx; - struct mbox_sync_mail *mail; + struct mbox_sync_mail mail; uint32_t seq; uoff_t hdr_offset, body_offset; @@ -45,13 +45,13 @@ struct mbox_sync_mail_context { size_t header_first_change, header_last_change; string_t *header; - uint32_t base_uid_validity, base_uid_last; uoff_t content_length; size_t hdr_pos[MBOX_HDR_COUNT]; unsigned int have_eoh:1; unsigned int need_rewrite:1; + unsigned int seen_imapbase:1; }; struct mbox_sync_context { @@ -61,6 +61,8 @@ struct mbox_sync_context { const struct mail_index_header *hdr; + buffer_t *header; + uint32_t base_uid_validity, base_uid_last; uint32_t prev_msg_uid, next_uid; }; @@ -68,7 +70,9 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit); void mbox_sync_parse_next_mail(struct istream *input, struct mbox_sync_mail_context *ctx); void mbox_sync_update_header(struct mbox_sync_mail_context *ctx, - struct mail_index_sync_rec *update); + buffer_t *syncs_buf); +void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx, + const struct mbox_sync_mail *mail); int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx); int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, uint32_t first_seq, uint32_t last_seq, off_t extra_space); diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c index 175854e85b..f335f6d135 100644 --- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c @@ -5,6 +5,7 @@ #include "str.h" #include "write-full.h" #include "message-parser.h" +#include "mbox-storage.h" #include "mbox-sync-private.h" #include "istream-raw-mbox.h" @@ -15,12 +16,12 @@ int mbox_move(struct mbox_sync_context *sync_ctx, struct ostream *output; off_t ret; + istream_raw_mbox_flush(sync_ctx->input); + output = o_stream_create_file(sync_ctx->fd, default_pool, 4096, FALSE); i_stream_seek(sync_ctx->file_input, source); o_stream_seek(output, dest); - istream_raw_mbox_flush(sync_ctx->file_input); - if (size == (uoff_t)-1) { input = sync_ctx->file_input; return o_stream_send_istream(output, input) < 0 ? -1 : 0; @@ -56,6 +57,9 @@ static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx, p = buffer_get_space_unsafe(ctx->header, pos, size); memset(p, ' ', size); + ctx->mail.offset = ctx->hdr_offset + pos; + ctx->mail.space += size; + if (ctx->header_first_change > pos) ctx->header_first_change = pos; ctx->header_last_change = (size_t)-1; @@ -68,7 +72,7 @@ static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx, size_t data_size, end, nonspace; /* find the end of the lwsp */ - nonspace = pos; + nonspace = pos-1; data = str_data(ctx->header); data_size = str_len(ctx->header); for (end = pos; end < data_size; end++) { @@ -88,7 +92,13 @@ static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx, *size -= end-nonspace; } else { str_delete(ctx->header, nonspace, *size); + end -= *size; *size = 0; + + if (ctx->mail.space < end-nonspace) { + ctx->mail.space = end-nonspace; + ctx->mail.offset = ctx->hdr_offset + nonspace; + } } } @@ -103,9 +113,16 @@ static void mbox_sync_headers_remove_space(struct mbox_sync_mail_context *ctx, enum header_position pos; int i; + ctx->header_last_change = (size_t)-1; + + ctx->mail.space = 0; + ctx->mail.offset = ctx->hdr_offset; + for (i = 0; i < 3 && size > 0; i++) { pos = space_positions[i]; if (ctx->hdr_pos[pos] != (size_t)-1) { + if (ctx->header_first_change > ctx->hdr_pos[pos]) + ctx->header_first_change = ctx->hdr_pos[pos]; mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos], &size); } @@ -123,17 +140,13 @@ int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx) new_hdr_size = str_len(ctx->header); /* do we have enough space? */ - if (new_hdr_size < old_hdr_size) { + if (new_hdr_size < old_hdr_size) mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size); - ctx->mail->space += old_hdr_size - new_hdr_size; - } else if (new_hdr_size > old_hdr_size) { + else if (new_hdr_size > old_hdr_size) { size_t needed = new_hdr_size - old_hdr_size; - if (ctx->mail->space < needed) { - ctx->mail->space -= needed; + if (ctx->mail.space < needed) return 0; - } - ctx->mail->space -= needed; mbox_sync_headers_remove_space(ctx, needed); } @@ -145,7 +158,7 @@ int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx) data = str_data(ctx->header); new_hdr_size = str_len(ctx->header); if (pwrite_full(ctx->sync_ctx->fd, data + ctx->header_first_change, - new_hdr_size, + new_hdr_size - ctx->header_first_change, ctx->hdr_offset + ctx->header_first_change) < 0) { // FIXME: error handling return -1; @@ -154,30 +167,134 @@ int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx) return 1; } +static void mbox_sync_fix_from_offset(struct mbox_sync_context *sync_ctx, + uint32_t idx, off_t diff) +{ + uoff_t *offset_p; + + offset_p = buffer_get_space_unsafe(sync_ctx->ibox->mbox_data_buf, + idx * sizeof(*offset_p), + sizeof(*offset_p)); + *offset_p = (*offset_p & 1) | (((*offset_p >> 1) + diff) << 1); +} + +static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail *mails, uint32_t idx, + uint32_t extra_per_mail, + uoff_t *end_offset) +{ + struct mbox_sync_mail_context mail_ctx; + uoff_t offset; + + i_stream_seek(sync_ctx->file_input, mails[idx].offset); + + memset(&mail_ctx, 0, sizeof(mail_ctx)); + mail_ctx.sync_ctx = sync_ctx; + mail_ctx.seq = idx+1; + mail_ctx.header = sync_ctx->header; + + mail_ctx.mail.offset = mails[idx].offset; + mail_ctx.mail.body_size = mails[idx].body_size; + + mbox_sync_parse_next_mail(sync_ctx->file_input, &mail_ctx); + mbox_sync_update_header_from(&mail_ctx, &mails[idx]); + + i_assert(mail_ctx.mail.space == mails[idx].space); + + /* we're moving next message - update it's from_offset */ + mbox_sync_fix_from_offset(sync_ctx, idx+1, mails[idx+1].space); + + if (mail_ctx.mail.space <= 0) { + mail_ctx.mail.space = 0; + mbox_sync_headers_add_space(&mail_ctx, extra_per_mail); + } else if (mail_ctx.mail.space <= extra_per_mail) { + mbox_sync_headers_add_space(&mail_ctx, extra_per_mail - + mail_ctx.mail.space); + } else { + mbox_sync_headers_remove_space(&mail_ctx, mail_ctx.mail.space - + extra_per_mail); + } + + /* now we have to move it. first move the body of the message, + then write the header and leave the extra space to beginning of + headers. */ + offset = sync_ctx->file_input->v_offset; + if (mbox_move(sync_ctx, offset + mails[idx+1].space, offset, + *end_offset - offset - mails[idx+1].space)) { + // FIXME: error handling + return -1; + } + + *end_offset = offset + mails[idx+1].space - str_len(mail_ctx.header); + + if (pwrite_full(sync_ctx->fd, str_data(mail_ctx.header), + str_len(mail_ctx.header), *end_offset) < 0) { + // FIXME: error handling + return -1; + } + + mails[idx].space += mails[idx+1].space - extra_per_mail; + return 0; +} + int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf, uint32_t first_seq, uint32_t last_seq, off_t extra_space) { struct mbox_sync_mail *mails; size_t size; - uint32_t first_idx, last_idx, extra_per_mail; - - first_idx = first_seq-1; - last_idx = last_seq-1; + uoff_t offset, end_offset; + uint32_t idx, extra_per_mail; + int ret = 0; mails = buffer_get_modifyable_data(mails_buf, &size); size /= sizeof(*mails); /* FIXME: see if we can be faster by going back a few mails - (update first_seq and last_seq) */ - /*while (mails[last_idx].space > 0) { - }*/ + (update first_seq and last_seq). */ + + extra_per_mail = (extra_space / (last_seq - first_seq + 1)); + + mails[last_seq-1].space -= extra_per_mail; + i_assert(mails[last_seq-1].space > 0); + end_offset = mails[last_seq-1].offset + mails[last_seq-1].space; -#if 0 /* start moving backwards */ - extra_per_mail = (extra_space / (last_seq - first_seq + 1)) + 1; - space_diff = 0; - while (last_seq > first_seq) { - dest = mails[last_seq].space_offset + mails[last_seq].space + while (--last_seq >= first_seq) { + idx = last_seq-1; + if (mails[idx].space <= 0) { + /* offset points to beginning of headers. read the + header again, update it and give enough space to + it */ + if (mbox_sync_read_and_move(sync_ctx, mails, idx, + extra_per_mail, + &end_offset) < 0) { + ret = -1; + break; + } + } else { + /* X-Keywords: xx [offset] \n + ... + X-Keywords: xx [end_offset] \n + + move data forward so mails before us gets the extra + space (ie. we temporarily get more space to us) */ + offset = mails[idx].offset + mails[idx].space; + if (mbox_move(sync_ctx, offset + mails[idx+1].space, + offset, end_offset - offset)) { + // FIXME: error handling + ret = -1; + break; + } + + mbox_sync_fix_from_offset(sync_ctx, idx+1, + mails[idx+1].space); + + mails[idx].space += mails[idx+1].space - extra_per_mail; + i_assert(mails[idx].space > 0); + end_offset = mails[idx].offset + mails[idx].space; + } } -#endif + + istream_raw_mbox_flush(sync_ctx->input); + return ret; } diff --git a/src/lib-storage/index/mbox/mbox-sync-update.c b/src/lib-storage/index/mbox/mbox-sync-update.c index 11f9010728..68f83c1097 100644 --- a/src/lib-storage/index/mbox/mbox-sync-update.c +++ b/src/lib-storage/index/mbox/mbox-sync-update.c @@ -5,15 +5,65 @@ #include "mbox-sync-private.h" static void status_flags_append(struct mbox_sync_mail_context *ctx, - struct mbox_flag_type *flags_list) + const struct mbox_flag_type *flags_list) { int i; for (i = 0; flags_list[i].chr != 0; i++) { - if ((ctx->mail->flags & flags_list[i].flag) != 0) + if ((ctx->mail.flags & flags_list[i].flag) != 0) str_append_c(ctx->header, flags_list[i].chr); } } + +static void status_flags_replace(struct mbox_sync_mail_context *ctx, size_t pos, + const struct mbox_flag_type *flags_list) +{ + unsigned char *data; + size_t size; + int i, need, have; + + /* how many bytes do we need? */ + for (i = 0, need = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail.flags & flags_list[i].flag) != 0) + need++; + } + + /* how many bytes do we have now? */ + data = buffer_get_modifyable_data(ctx->header, &size); + for (have = 0; pos < size; pos++) { + if (data[pos] == '\n') + break; + + /* see if this is unknown flag for us */ + for (i = 0; flags_list[i].chr != 0; i++) { + if (flags_list[i].chr == data[pos]) + break; + } + + if (flags_list[i].chr == 0) + have++; + else { + /* save this one */ + data[pos-have] = data[pos]; + } + } + pos -= have; + + if (need < have) + str_delete(ctx->header, pos, have-need); + else if (need > have) { + buffer_copy(ctx->header, pos + (have-need), + ctx->header, pos, (size_t)-1); + } + + /* @UNSAFE */ + data = buffer_get_space_unsafe(ctx->header, pos, need); + for (i = 0, need = 0; flags_list[i].chr != 0; i++) { + if ((ctx->mail.flags & flags_list[i].flag) != 0) + *data++ = flags_list[i].chr; + } +} + static void keywords_append(struct mbox_sync_mail_context *ctx, keywords_mask_t keywords) { @@ -28,7 +78,7 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) old_hdr_size = ctx->body_offset - ctx->hdr_offset; new_hdr_size = str_len(ctx->header) + ctx->have_eoh; - if (ctx->seq == 1 && ctx->base_uid_validity == 0) { + if (ctx->seq == 1 && ctx->sync_ctx->base_uid_validity == 0) { ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header); str_printfa(ctx->header, "X-IMAPbase: %u %u", ctx->sync_ctx->hdr->uid_validity, @@ -37,14 +87,16 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) str_append_c(ctx->header, '\n'); } - if (ctx->mail->uid == 0) { + if (ctx->hdr_pos[MBOX_HDR_X_UID] == (size_t)-1) { + if (ctx->mail.uid == 0) + ctx->mail.uid = ctx->sync_ctx->next_uid++; ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header); - str_printfa(ctx->header, "X-UID: %u\n", - ctx->sync_ctx->next_uid++); + str_printfa(ctx->header, "X-UID: %u\n", ctx->mail.uid); } + i_assert(ctx->mail.uid != 0); if (ctx->hdr_pos[MBOX_HDR_STATUS] == (size_t)-1 && - (ctx->mail->flags & STATUS_FLAGS_MASK) != 0) { + (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) { ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header); str_append(ctx->header, "Status: "); status_flags_append(ctx, mbox_status_flags); @@ -52,7 +104,7 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) } if (ctx->hdr_pos[MBOX_HDR_X_STATUS] == (size_t)-1 && - (ctx->mail->flags & XSTATUS_FLAGS_MASK) != 0) { + (ctx->mail.flags & XSTATUS_FLAGS_MASK) != 0) { ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header); str_append(ctx->header, "X-Status: "); status_flags_append(ctx, mbox_xstatus_flags); @@ -61,7 +113,7 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) have_keywords = FALSE; for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) { - if (ctx->mail->keywords[i] != 0) { + if (ctx->mail.keywords[i] != 0) { have_keywords = TRUE; break; } @@ -70,21 +122,30 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == (size_t)-1 && have_keywords) { ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header); str_append(ctx->header, "X-Keywords: "); - keywords_append(ctx, ctx->mail->keywords); + keywords_append(ctx, ctx->mail.keywords); str_append_c(ctx->header, '\n'); } if (ctx->content_length == (uoff_t)-1) { str_printfa(ctx->header, "Content-Length: %"PRIuUOFF_T"\n", - ctx->mail->body_size); + ctx->mail.body_size); } if (str_len(ctx->header) != new_hdr_size) { if (ctx->header_first_change == (size_t)-1) ctx->header_first_change = new_hdr_size; ctx->header_last_change = (size_t)-1; - ctx->mail->space -= str_len(ctx->header) - + ctx->mail.space -= str_len(ctx->header) - (new_hdr_size - ctx->have_eoh); + if (ctx->mail.space > 0) { + /* we should rewrite this header, so offset + must be broken if it's used anymore. */ + ctx->mail.offset = (uoff_t)-1; + } else { + /* we don't have enough space for this header, change + offset to point back to beginning of headers */ + ctx->mail.offset = ctx->hdr_offset; + } new_hdr_size = str_len(ctx->header) + ctx->have_eoh; } @@ -99,39 +160,82 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) static void mbox_sync_update_status(struct mbox_sync_mail_context *ctx) { + if (ctx->hdr_pos[MBOX_HDR_STATUS] != (size_t)-1) { + status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_STATUS], + mbox_status_flags); + } } static void mbox_sync_update_xstatus(struct mbox_sync_mail_context *ctx) { + if (ctx->hdr_pos[MBOX_HDR_X_STATUS] != (size_t)-1) { + status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_X_STATUS], + mbox_xstatus_flags); + } } static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx) { + // FIXME } void mbox_sync_update_header(struct mbox_sync_mail_context *ctx, - struct mail_index_sync_rec *update) + buffer_t *syncs_buf) { + const struct mail_index_sync_rec *sync; + size_t size, i; uint8_t old_flags; keywords_mask_t old_keywords; - if (update != NULL) { - old_flags = ctx->mail->flags; - memcpy(old_keywords, ctx->mail->keywords, sizeof(old_keywords)); + sync = buffer_get_data(syncs_buf, &size); + size /= sizeof(*sync); + + if (size != 0) { + old_flags = ctx->mail.flags; + memcpy(old_keywords, ctx->mail.keywords, sizeof(old_keywords)); - mail_index_sync_flags_apply(update, &ctx->mail->flags, - ctx->mail->keywords); + for (i = 0; i < size; i++) { + mail_index_sync_flags_apply(&sync[i], &ctx->mail.flags, + ctx->mail.keywords); + } if ((old_flags & STATUS_FLAGS_MASK) != - (ctx->mail->flags & STATUS_FLAGS_MASK)) + (ctx->mail.flags & STATUS_FLAGS_MASK)) mbox_sync_update_status(ctx); if ((old_flags & XSTATUS_FLAGS_MASK) != - (ctx->mail->flags & XSTATUS_FLAGS_MASK)) + (ctx->mail.flags & XSTATUS_FLAGS_MASK)) mbox_sync_update_xstatus(ctx); - if (memcmp(old_keywords, ctx->mail->keywords, - sizeof(old_keywords)) != 0) + if (memcmp(old_keywords, ctx->mail.keywords, + INDEX_KEYWORDS_BYTE_COUNT) != 0) mbox_sync_update_xkeywords(ctx); } mbox_sync_add_missing_headers(ctx); } + +void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx, + const struct mbox_sync_mail *mail) +{ + if ((ctx->mail.flags & STATUS_FLAGS_MASK) != + (mail->flags & STATUS_FLAGS_MASK)) { + ctx->mail.flags = (ctx->mail.flags & ~STATUS_FLAGS_MASK) | + (mail->flags & STATUS_FLAGS_MASK); + mbox_sync_update_status(ctx); + } + if ((ctx->mail.flags & XSTATUS_FLAGS_MASK) != + (mail->flags & XSTATUS_FLAGS_MASK)) { + ctx->mail.flags = (ctx->mail.flags & ~XSTATUS_FLAGS_MASK) | + (mail->flags & XSTATUS_FLAGS_MASK); + mbox_sync_update_xstatus(ctx); + } + if (memcmp(ctx->mail.keywords, mail->keywords, + INDEX_KEYWORDS_BYTE_COUNT) != 0) { + memcpy(ctx->mail.keywords, mail->keywords, + INDEX_KEYWORDS_BYTE_COUNT); + mbox_sync_update_xkeywords(ctx); + } + + i_assert(ctx->mail.uid == 0 || ctx->mail.uid == mail->uid); + ctx->mail.uid = mail->uid; + mbox_sync_add_missing_headers(ctx); +} diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c index cd52c26ab0..3e1239366a 100644 --- a/src/lib-storage/index/mbox/mbox-sync.c +++ b/src/lib-storage/index/mbox/mbox-sync.c @@ -73,7 +73,7 @@ static int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx, if (file_set_size(sync_ctx->fd, size) < 0) return -1; - if (mail->space_offset == 0) { + if (mail->space <= 0) { /* no X-Keywords header - place it at the end. */ grow_size += 13; @@ -91,11 +91,11 @@ static int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx, /* FIXME: can this break anything? X-Keywords text might have been already included in space calculation. now we have more.. */ - mail->space_offset = offset; + mail->offset = offset; mail->space += grow_size; } else { - offset = mail->space_offset; - if (mbox_move(sync_ctx, mail->space_offset + grow_size, + offset = mail->offset; + if (mbox_move(sync_ctx, offset + grow_size, offset, (uoff_t)-1) < 0) return -1; } @@ -117,11 +117,63 @@ static int mbox_sync_grow_file(struct mbox_sync_context *sync_ctx, return 0; } +static void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t seq) +{ + struct mail_index_sync_rec *sync; + size_t size, src, dest; + + sync = buffer_get_modifyable_data(syncs_buf, &size); + size /= sizeof(*sync); + + for (src = dest = 0; src < size; src++) { + if (sync[src].seq2 >= seq) { + if (src != dest) + sync[dest] = sync[src]; + dest++; + } + } + + buffer_set_used_size(syncs_buf, dest * sizeof(*sync)); +} + +static int +mbox_sync_next_mail(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, uint32_t seq) +{ + uoff_t from_offset; + + memset(mail_ctx, 0, sizeof(*mail_ctx)); + mail_ctx->sync_ctx = sync_ctx; + mail_ctx->seq = seq; + mail_ctx->header = sync_ctx->header; + + from_offset = istream_raw_mbox_get_start_offset(sync_ctx->input); + mail_ctx->mail.offset = + istream_raw_mbox_get_header_offset(sync_ctx->input); + + mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx); + if (sync_ctx->input->v_offset == from_offset) { + /* this was the last mail */ + return 0; + } + + mail_ctx->mail.body_size = + istream_raw_mbox_get_body_size(sync_ctx->input, + mail_ctx->content_length); + + /* save the offset permanently with recent flag state */ + from_offset <<= 1; + if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0) + from_offset |= 1; + buffer_append(sync_ctx->ibox->mbox_data_buf, &from_offset, + sizeof(from_offset)); + return 1; +} + int mbox_sync(struct index_mailbox *ibox, int last_commit) { struct mbox_sync_context sync_ctx; struct mbox_sync_mail_context mail_ctx; - struct mbox_sync_mail mail; struct mail_index_sync_ctx *index_sync_ctx; struct mail_index_sync_rec sync_rec; struct mail_index_view *sync_view; @@ -131,9 +183,8 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit) struct istream *input; uint32_t seq, need_space_seq, idx_seq, messages_count; off_t space_diff; - uoff_t from_offset, offset; - buffer_t *mails; - string_t *header; + uoff_t offset, extra_space; + buffer_t *mails, *syncs; size_t size; struct stat st; int readonly, ret = 0; @@ -166,64 +217,62 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit) buffer_set_used_size(ibox->mbox_data_buf, 0); } - readonly = TRUE; // FIXME - + readonly = FALSE; // FIXME // FIXME: lock the file memset(&sync_ctx, 0, sizeof(sync_ctx)); + sync_ctx.ibox = ibox; sync_ctx.file_input = ibox->mbox_file_stream; sync_ctx.input = ibox->mbox_stream; sync_ctx.fd = ibox->mbox_fd; sync_ctx.hdr = hdr; + sync_ctx.header = str_new(default_pool, 4096); input = sync_ctx.input; istream_raw_mbox_seek(input, 0); - header = str_new(default_pool, 4096); - mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1); + syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1); + memset(&sync_rec, 0, sizeof(sync_rec)); messages_count = mail_index_view_get_message_count(sync_view); space_diff = 0; need_space_seq = 0; idx_seq = 0; rec = NULL; for (seq = 1; !input->eof; seq++) { - if (sync_rec.seq2 < seq) { - // FIXME: we may need more than one.. + ret = 1; + + /* get all sync records related to this message */ + mbox_sync_buffer_delete_old(syncs, seq); + while (sync_rec.seq2 <= seq && ret > 0) { + if (sync_rec.seq2 != 0) { + buffer_append(syncs, &sync_rec, + sizeof(sync_rec)); + } ret = mail_index_sync_next(index_sync_ctx, &sync_rec); - if (ret < 0) - break; } + if (ret < 0) + break; - from_offset = input->v_offset; - - memset(&mail, 0, sizeof(mail)); - memset(&mail_ctx, 0, sizeof(mail_ctx)); - mail_ctx.sync_ctx = &sync_ctx; - mail_ctx.mail = &mail; - mail_ctx.seq = seq; - mail_ctx.header = header; - - mbox_sync_parse_next_mail(input, &mail_ctx); - if (input->v_offset == from_offset) { - /* this was the last mail */ + ret = mbox_sync_next_mail(&sync_ctx, &mail_ctx, seq); + if (ret <= 0) break; - } - mail.body_size = - istream_raw_mbox_get_size(input, - mail_ctx.content_length); - buffer_append(mails, &mail, sizeof(mail)); + if ((mail_ctx.need_rewrite || + buffer_get_used_size(syncs) != 0) && !readonly) { + mbox_sync_update_header(&mail_ctx, syncs); + if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0) + return -1; - /* save the offset permanently with recent flag state */ - from_offset <<= 1; - if ((mail.flags & MBOX_NONRECENT) == 0) - from_offset |= 1; - buffer_append(ibox->mbox_data_buf, - &from_offset, sizeof(from_offset)); + if (ret == 0 && need_space_seq == 0) { + /* first mail with no space to write it */ + need_space_seq = seq; + space_diff = 0; + } + } /* update index */ do { - if (rec != NULL && rec->uid >= mail.uid) + if (rec != NULL && rec->uid >= mail_ctx.mail.uid) break; if (idx_seq >= messages_count) { @@ -239,12 +288,13 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit) if (ret < 0) break; - if (rec != NULL && rec->uid != mail.uid) { + if (rec != NULL && rec->uid != mail_ctx.mail.uid) { /* new UID in the middle of the mailbox - shouldn't happen */ mail_storage_set_critical(ibox->box.storage, "mbox sync: UID inserted in the middle " - "of mailbox (%u > %u)", rec->uid, mail.uid); + "of mailbox (%u > %u)", + rec->uid, mail_ctx.mail.uid); mail_index_mark_corrupted(ibox->index); ret = -1; break; @@ -253,40 +303,34 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit) if (rec != NULL) { /* see if flags changed */ if ((rec->flags & MAIL_FLAGS_MASK) != - (mail.flags & MAIL_FLAGS_MASK) || - memcmp(rec->keywords, mail.keywords, + (mail_ctx.mail.flags & MAIL_FLAGS_MASK) || + memcmp(rec->keywords, mail_ctx.mail.keywords, INDEX_KEYWORDS_BYTE_COUNT) != 0) { uint8_t new_flags = (rec->flags & ~MAIL_FLAGS_MASK) | - (mail.flags & MAIL_FLAGS_MASK); + (mail_ctx.mail.flags & MAIL_FLAGS_MASK); mail_index_update_flags(t, idx_seq, MODIFY_REPLACE, new_flags, - mail.keywords); + mail_ctx.mail.keywords); } rec = NULL; } else { /* new message */ - mail_index_append(t, mail.uid, &idx_seq); + mail_index_append(t, mail_ctx.mail.uid, &idx_seq); mail_index_update_flags(t, idx_seq, MODIFY_REPLACE, - mail.flags & MAIL_FLAGS_MASK, - mail.keywords); + mail_ctx.mail.flags & MAIL_FLAGS_MASK, + mail_ctx.mail.keywords); } - if (mail_ctx.need_rewrite && !readonly) { - mbox_sync_update_header(&mail_ctx, NULL); - if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0) - break; - } else { - ret = 1; - } + istream_raw_mbox_next(input, mail_ctx.mail.body_size); + offset = input->v_offset; + + if (need_space_seq != 0) { + buffer_append(mails, &mail_ctx.mail, + sizeof(mail_ctx.mail)); - if (ret == 0 && need_space_seq == 0) { - /* didn't have space to write it */ - need_space_seq = seq; - space_diff = mail.space; - } else if (need_space_seq != 0) { - space_diff += mail.space; + space_diff += mail_ctx.mail.space; if (space_diff >= 0) { /* we have enough space now */ if (mbox_sync_rewrite(&sync_ctx, mails, @@ -295,25 +339,38 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit) ret = -1; break; } + + /* mail_ctx may contain wrong data after + rewrite, so make sure we don't try to access + it */ + memset(&mail_ctx, 0, sizeof(mail_ctx)); + i_stream_seek(input, input->v_offset); need_space_seq = 0; } } - - istream_raw_mbox_next(input, mail.body_size); } if (need_space_seq != 0) { i_assert(space_diff < 0); - if (mbox_sync_grow_file(&sync_ctx, &mail, mail_ctx.body_offset, - -space_diff) < 0) + extra_space = MBOX_HEADER_EXTRA_SPACE * + ((seq-1) - need_space_seq); + if (mbox_sync_grow_file(&sync_ctx, &mail_ctx.mail, + mail_ctx.body_offset, + -space_diff + extra_space) < 0) ret = -1; else if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq, - seq-1, space_diff) < 0) + seq-1, extra_space) < 0) ret = -1; } + if (sync_ctx.base_uid_last+1 != sync_ctx.next_uid) { + // FIXME: rewrite X-IMAPbase header + } + + /* only syncs left should be just appends which weren't synced yet. + we'll just ignore them, as we've overwritten those above. */ while ((ret = mail_index_sync_next(index_sync_ctx, &sync_rec)) > 0) { - // FIXME: should be just appends + i_assert(sync_rec.type == MAIL_INDEX_SYNC_TYPE_APPEND); } if (fstat(ibox->mbox_fd, &st) < 0) { @@ -350,7 +407,9 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit) ibox->mbox_data = buffer_get_data(ibox->mbox_data_buf, &size); ibox->mbox_data_count = size / sizeof(*ibox->mbox_data); - str_free(header); + str_free(sync_ctx.header); + buffer_free(mails); + buffer_free(syncs); return ret < 0 ? -1 : 0; }