time_t received_time, next_received_time;
char *sender, *next_sender;
- uoff_t from_offset, hdr_offset, next_from_offset, body_size;
+ uoff_t from_offset, hdr_offset, body_offset, mail_size;
struct istream *input;
};
struct raw_mbox_istream *rstream = (struct raw_mbox_istream *)stream;
const unsigned char *buf;
const char *fromp;
- char *sender;
+ char *sender, eoh_char;
time_t received_time;
size_t i, pos;
ssize_t ret;
buf = i_stream_get_data(rstream->input, &pos);
} while (ret > 0 && pos <= 6);
- if (pos == 1 && buf[0] == '\n') {
+ if (pos == 0 || (pos == 1 && buf[0] == '\n')) {
/* EOF */
stream->pos = 0;
stream->istream.eof = TRUE;
+ rstream->mail_size = stream->istream.v_offset -
+ rstream->hdr_offset;
return -1;
}
mbox_from_parse(buf+6, pos-6,
&received_time, &sender) == 0) {
rstream->next_received_time = received_time;
+ rstream->mail_size = stream->istream.v_offset -
+ rstream->hdr_offset;
i_free(rstream->next_sender);
rstream->next_sender = sender;
}
} else if (ret == -1) {
/* last few bytes, can't contain From-line */
+ if (buf[pos-1] == '\n') {
+ /* last LF doesn't belong to last message */
+ pos--;
+ }
+
ret = pos <= stream->pos ? -1 :
(ssize_t) (pos - stream->pos);
+ rstream->mail_size = stream->istream.v_offset + pos -
+ rstream->hdr_offset;
+
stream->buffer = buf;
stream->pos = pos;
stream->istream.eof = ret == -1;
/* See if we have From-line here - note that it works right only
because all characters are different in mbox_from. */
+ eoh_char = rstream->body_offset == (uoff_t)-1 ? '\n' : '\0';
for (i = 0, fromp = mbox_from; i < pos; i++) {
+ if (buf[i] == eoh_char && i > 0 && buf[i-1] == '\n') {
+ rstream->body_offset = stream->istream.v_offset + i + 1;
+ eoh_char = '\0';
+ }
if (buf[i] == *fromp) {
if (*++fromp == '\0') {
/* potential From-line - stop here */
rstream = p_new(pool, struct raw_mbox_istream, 1);
rstream->input = input;
- rstream->body_size = (uoff_t)-1;
+ rstream->body_offset = (uoff_t)-1;
+ rstream->mail_size = (uoff_t)-1;
+ rstream->received_time = (time_t)-1;
+ rstream->next_received_time = (time_t)-1;
rstream->istream.iostream.close = _close;
rstream->istream.iostream.destroy = _destroy;
return TRUE;
}
-uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size)
+uoff_t istream_raw_mbox_get_start_offset(struct istream *stream)
+{
+ struct raw_mbox_istream *rstream =
+ (struct raw_mbox_istream *)stream->real_stream;
+
+ return rstream->from_offset;
+}
+
+uoff_t istream_raw_mbox_get_header_offset(struct istream *stream)
+{
+ struct raw_mbox_istream *rstream =
+ (struct raw_mbox_istream *)stream->real_stream;
+
+ if (rstream->hdr_offset == rstream->from_offset)
+ (void)_read(&rstream->istream);
+
+ return rstream->hdr_offset;
+}
+
+uoff_t istream_raw_mbox_get_body_size(struct istream *stream, uoff_t body_size)
{
struct raw_mbox_istream *rstream =
(struct raw_mbox_istream *)stream->real_stream;
const unsigned char *data;
size_t size;
- if (rstream->body_size != (uoff_t)-1)
- return rstream->body_size;
+ if (rstream->mail_size != (uoff_t)-1) {
+ return rstream->mail_size -
+ (rstream->body_offset - rstream->hdr_offset);
+ }
if (body_size != (uoff_t)-1) {
- i_stream_seek(rstream->input, rstream->from_offset + body_size);
+ i_assert(rstream->body_offset != (uoff_t)-1);
+ i_stream_seek(rstream->input, rstream->body_offset + body_size);
if (istream_raw_mbox_is_valid_from(rstream) > 0) {
- rstream->body_size = body_size;
+ rstream->mail_size = body_size +
+ (rstream->body_offset - rstream->hdr_offset);
return body_size;
}
}
- old_offset = stream->v_offset;
-
/* have to read through the message body */
while (i_stream_read_data(stream, &data, &size, 0) > 0)
i_stream_skip(stream, size);
- rstream->body_size = stream->v_offset - old_offset;
+ i_assert(rstream->mail_size != (uoff_t)-1);
i_stream_seek(stream, old_offset);
- return rstream->body_size;
+ return rstream->mail_size -
+ (rstream->body_offset - rstream->hdr_offset);
}
time_t istream_raw_mbox_get_received_time(struct istream *stream)
struct raw_mbox_istream *rstream =
(struct raw_mbox_istream *)stream->real_stream;
- (void)_read(&rstream->istream);
+ if (rstream->received_time == (time_t)-1)
+ (void)_read(&rstream->istream);
return rstream->received_time;
}
struct raw_mbox_istream *rstream =
(struct raw_mbox_istream *)stream->real_stream;
- (void)_read(&rstream->istream);
+ if (rstream->sender == NULL)
+ (void)_read(&rstream->istream);
return rstream->sender == NULL ? "" : rstream->sender;
}
struct raw_mbox_istream *rstream =
(struct raw_mbox_istream *)stream->real_stream;
- body_size = istream_raw_mbox_get_size(stream, body_size);
- rstream->body_size = (uoff_t)-1;
+ body_size = istream_raw_mbox_get_body_size(stream, body_size);
+ rstream->mail_size = (uoff_t)-1;
rstream->received_time = rstream->next_received_time;
rstream->next_received_time = (time_t)-1;
rstream->sender = rstream->next_sender;
rstream->next_sender = NULL;
- rstream->from_offset = stream->v_offset + body_size;
+ rstream->from_offset = rstream->body_offset + body_size;
rstream->hdr_offset = rstream->from_offset;
+ rstream->body_offset = (uoff_t)-1;
/* don't clear stream->eof if we don't have to */
if (stream->v_offset != rstream->from_offset)
struct raw_mbox_istream *rstream =
(struct raw_mbox_istream *)stream->real_stream;
- if (offset == rstream->next_from_offset) {
+ if (rstream->mail_size != (uoff_t)-1 &&
+ rstream->hdr_offset + rstream->mail_size == offset) {
istream_raw_mbox_next(stream, (uoff_t)-1);
return;
}
/* back to beginning of current message */
offset = rstream->hdr_offset;
} else {
- rstream->body_size = (uoff_t)-1;
+ rstream->body_offset = (uoff_t)-1;
+ rstream->mail_size = (uoff_t)-1;
rstream->received_time = (time_t)-1;
rstream->next_received_time = (time_t)-1;
rstream->sender = NULL;
i_free(rstream->next_sender);
rstream->next_sender = NULL;
+
+ rstream->from_offset = offset;
+ rstream->hdr_offset = offset;
}
- rstream->from_offset = rstream->hdr_offset = offset;
i_stream_seek(stream, offset);
i_stream_seek(rstream->input, offset);
}
you'll have to call istream_raw_mbox_next() to get to next message. */
struct istream *i_stream_create_raw_mbox(pool_t pool, struct istream *input);
-/* Return number of bytes in this message after current offset.
- If body_size isn't (uoff_t)-1, we'll use it as potentially valid body size
- to avoid actually reading through the whole message. */
-uoff_t istream_raw_mbox_get_size(struct istream *stream, uoff_t body_size);
+/* Return offset to beginning of the "\nFrom"-line. */
+uoff_t istream_raw_mbox_get_start_offset(struct istream *stream);
+/* Return offset to beginning of the headers. */
+uoff_t istream_raw_mbox_get_header_offset(struct istream *stream);
+
+/* Return the number of bytes in the body of this message. If body_size isn't
+ (uoff_t)-1, we'll use it as potentially valid body size to avoid actually
+ reading through the whole message. */
+uoff_t istream_raw_mbox_get_body_size(struct istream *stream, uoff_t body_size);
/* Return received time of current message, or (time_t)-1 if the timestamp is
broken. */
size_t i;
for (i = 0; i < hdr->full_value_len; i++) {
- ctx->mail->flags |=
+ ctx->mail.flags |=
mbox_flag_find(flags_list, hdr->full_value[i]);
}
}
const char *str;
char *end;
size_t pos;
+ uint32_t uid_validity, uid_last;
- if (ctx->seq != 1 || ctx->base_uid_validity != 0) {
+ if (ctx->seq != 1 || ctx->seen_imapbase) {
/* Valid only in first message */
return FALSE;
}
/* <uid validity> <last uid> */
t_push();
str = t_strndup(hdr->full_value, hdr->full_value_len);
- ctx->base_uid_validity = strtoul(str, &end, 10);
- ctx->base_uid_last = strtoul(end, &end, 10);
+ uid_validity = strtoul(str, &end, 10);
+ uid_last = strtoul(end, &end, 10);
pos = end - str;
t_pop();
while (pos < hdr->full_value_len && IS_LWSP_LF(hdr->full_value[pos]))
pos++;
- if (ctx->base_uid_validity == 0) {
+ if (uid_validity == 0) {
/* broken */
return FALSE;
}
// FIXME: save keywords
+ if (ctx->sync_ctx->base_uid_validity == 0) {
+ ctx->sync_ctx->base_uid_validity = uid_validity;
+ ctx->sync_ctx->base_uid_last = uid_last;
+ ctx->sync_ctx->next_uid = uid_last+1;
+ }
+
ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
+ ctx->seen_imapbase = TRUE;
return TRUE;
}
{
size_t i, space = 0;
- for (i = hdr->full_value_len; i > 0; i++) {
+ for (i = hdr->full_value_len; i > 0; i--) {
if (!IS_LWSP_LF(hdr->full_value[i-1]))
break;
space++;
}
- if (space > ctx->mail->space) {
- ctx->mail->space_offset = hdr->full_value_offset + i;
- ctx->mail->space = space;
+ if (space > ctx->mail.space) {
+ ctx->mail.offset = ctx->hdr_offset +
+ hdr->full_value_offset + i;
+ ctx->mail.space = space;
}
// FIXME: parse them
uint32_t value = 0;
size_t i, space_pos, extra_space = 0;
- if (ctx->mail->uid != 0) {
+ if (ctx->mail.uid != 0) {
/* duplicate */
return FALSE;
}
/* broken - UIDs must be growing */
return FALSE;
}
+ ctx->sync_ctx->prev_msg_uid = value;
ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
- ctx->mail->uid = value;
- if (ctx->mail->space == 0) {
+ ctx->mail.uid = value;
+ if (extra_space != 0 && ctx->mail.space == 0) {
/* set it only if X-Keywords hasn't been seen. spaces in X-UID
should be removed when writing X-Keywords. */
- ctx->mail->space_offset = hdr->full_value_offset + space_pos;
- ctx->mail->space = extra_space;
+ ctx->mail.offset = ctx->hdr_offset +
+ hdr->full_value_offset + space_pos;
+ ctx->mail.space = extra_space;
}
return TRUE;
}
size_t line_start_pos;
int i;
- ctx->hdr_offset = input->v_offset;
- ctx->mail->space_offset = input->v_offset;
+ ctx->hdr_offset = ctx->mail.offset;
ctx->header_first_change = (size_t)-1;
ctx->header_last_change = (size_t)-1;
break;
}
+ if (!hdr->continued) {
+ line_start_pos = str_len(ctx->header);
+ str_append(ctx->header, hdr->name);
+ str_append(ctx->header, ": ");
+ }
+
func = header_func_find(hdr->name);
if (func != NULL) {
if (hdr->continues)
else if (!func->func(ctx, hdr)) {
/* this header is broken, remove it */
ctx->need_rewrite = TRUE;
- if (hdr->continued) {
- str_truncate(ctx->header,
- line_start_pos);
- }
+ str_truncate(ctx->header, line_start_pos);
if (ctx->header_first_change == (size_t)-1) {
ctx->header_first_change =
- str_len(ctx->header);
+ line_start_pos;
}
continue;
}
}
- if (!hdr->continued) {
- line_start_pos = str_len(ctx->header);
- str_append(ctx->header, hdr->name);
- str_append(ctx->header, ": ");
- }
buffer_append(ctx->header, hdr->full_value,
hdr->full_value_len);
if (!hdr->no_newline)
}
message_parse_header_deinit(hdr_ctx);
- if (ctx->seq == 1 && ctx->base_uid_validity == 0) {
+ if (ctx->seq == 1 && ctx->sync_ctx->base_uid_validity == 0) {
/* missing X-IMAPbase */
ctx->need_rewrite = TRUE;
}
+ if (ctx->mail.uid == 0) {
+ /* missing X-UID */
+ ctx->need_rewrite = TRUE;
+ }
ctx->body_offset = input->v_offset;
}
uint8_t flags;
keywords_mask_t keywords;
- uoff_t space_offset; /* if space is negative, points to beginning */
+ uoff_t offset; /* if space <= 0, points to beginning */
off_t space;
uoff_t body_size;
};
struct mbox_sync_mail_context {
struct mbox_sync_context *sync_ctx;
- struct mbox_sync_mail *mail;
+ struct mbox_sync_mail mail;
uint32_t seq;
uoff_t hdr_offset, body_offset;
size_t header_first_change, header_last_change;
string_t *header;
- uint32_t base_uid_validity, base_uid_last;
uoff_t content_length;
size_t hdr_pos[MBOX_HDR_COUNT];
unsigned int have_eoh:1;
unsigned int need_rewrite:1;
+ unsigned int seen_imapbase:1;
};
struct mbox_sync_context {
const struct mail_index_header *hdr;
+ buffer_t *header;
+ uint32_t base_uid_validity, base_uid_last;
uint32_t prev_msg_uid, next_uid;
};
void mbox_sync_parse_next_mail(struct istream *input,
struct mbox_sync_mail_context *ctx);
void mbox_sync_update_header(struct mbox_sync_mail_context *ctx,
- struct mail_index_sync_rec *update);
+ buffer_t *syncs_buf);
+void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
+ const struct mbox_sync_mail *mail);
int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx);
int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
uint32_t first_seq, uint32_t last_seq, off_t extra_space);
#include "str.h"
#include "write-full.h"
#include "message-parser.h"
+#include "mbox-storage.h"
#include "mbox-sync-private.h"
#include "istream-raw-mbox.h"
struct ostream *output;
off_t ret;
+ istream_raw_mbox_flush(sync_ctx->input);
+
output = o_stream_create_file(sync_ctx->fd, default_pool, 4096, FALSE);
i_stream_seek(sync_ctx->file_input, source);
o_stream_seek(output, dest);
- istream_raw_mbox_flush(sync_ctx->file_input);
-
if (size == (uoff_t)-1) {
input = sync_ctx->file_input;
return o_stream_send_istream(output, input) < 0 ? -1 : 0;
p = buffer_get_space_unsafe(ctx->header, pos, size);
memset(p, ' ', size);
+ ctx->mail.offset = ctx->hdr_offset + pos;
+ ctx->mail.space += size;
+
if (ctx->header_first_change > pos)
ctx->header_first_change = pos;
ctx->header_last_change = (size_t)-1;
size_t data_size, end, nonspace;
/* find the end of the lwsp */
- nonspace = pos;
+ nonspace = pos-1;
data = str_data(ctx->header);
data_size = str_len(ctx->header);
for (end = pos; end < data_size; end++) {
*size -= end-nonspace;
} else {
str_delete(ctx->header, nonspace, *size);
+ end -= *size;
*size = 0;
+
+ if (ctx->mail.space < end-nonspace) {
+ ctx->mail.space = end-nonspace;
+ ctx->mail.offset = ctx->hdr_offset + nonspace;
+ }
}
}
enum header_position pos;
int i;
+ ctx->header_last_change = (size_t)-1;
+
+ ctx->mail.space = 0;
+ ctx->mail.offset = ctx->hdr_offset;
+
for (i = 0; i < 3 && size > 0; i++) {
pos = space_positions[i];
if (ctx->hdr_pos[pos] != (size_t)-1) {
+ if (ctx->header_first_change > ctx->hdr_pos[pos])
+ ctx->header_first_change = ctx->hdr_pos[pos];
mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos],
&size);
}
new_hdr_size = str_len(ctx->header);
/* do we have enough space? */
- if (new_hdr_size < old_hdr_size) {
+ if (new_hdr_size < old_hdr_size)
mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
- ctx->mail->space += old_hdr_size - new_hdr_size;
- } else if (new_hdr_size > old_hdr_size) {
+ else if (new_hdr_size > old_hdr_size) {
size_t needed = new_hdr_size - old_hdr_size;
- if (ctx->mail->space < needed) {
- ctx->mail->space -= needed;
+ if (ctx->mail.space < needed)
return 0;
- }
- ctx->mail->space -= needed;
mbox_sync_headers_remove_space(ctx, needed);
}
data = str_data(ctx->header);
new_hdr_size = str_len(ctx->header);
if (pwrite_full(ctx->sync_ctx->fd, data + ctx->header_first_change,
- new_hdr_size,
+ new_hdr_size - ctx->header_first_change,
ctx->hdr_offset + ctx->header_first_change) < 0) {
// FIXME: error handling
return -1;
return 1;
}
+static void mbox_sync_fix_from_offset(struct mbox_sync_context *sync_ctx,
+ uint32_t idx, off_t diff)
+{
+ uoff_t *offset_p;
+
+ offset_p = buffer_get_space_unsafe(sync_ctx->ibox->mbox_data_buf,
+ idx * sizeof(*offset_p),
+ sizeof(*offset_p));
+ *offset_p = (*offset_p & 1) | (((*offset_p >> 1) + diff) << 1);
+}
+
+static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx,
+ struct mbox_sync_mail *mails, uint32_t idx,
+ uint32_t extra_per_mail,
+ uoff_t *end_offset)
+{
+ struct mbox_sync_mail_context mail_ctx;
+ uoff_t offset;
+
+ i_stream_seek(sync_ctx->file_input, mails[idx].offset);
+
+ memset(&mail_ctx, 0, sizeof(mail_ctx));
+ mail_ctx.sync_ctx = sync_ctx;
+ mail_ctx.seq = idx+1;
+ mail_ctx.header = sync_ctx->header;
+
+ mail_ctx.mail.offset = mails[idx].offset;
+ mail_ctx.mail.body_size = mails[idx].body_size;
+
+ mbox_sync_parse_next_mail(sync_ctx->file_input, &mail_ctx);
+ mbox_sync_update_header_from(&mail_ctx, &mails[idx]);
+
+ i_assert(mail_ctx.mail.space == mails[idx].space);
+
+ /* we're moving next message - update it's from_offset */
+ mbox_sync_fix_from_offset(sync_ctx, idx+1, mails[idx+1].space);
+
+ if (mail_ctx.mail.space <= 0) {
+ mail_ctx.mail.space = 0;
+ mbox_sync_headers_add_space(&mail_ctx, extra_per_mail);
+ } else if (mail_ctx.mail.space <= extra_per_mail) {
+ mbox_sync_headers_add_space(&mail_ctx, extra_per_mail -
+ mail_ctx.mail.space);
+ } else {
+ mbox_sync_headers_remove_space(&mail_ctx, mail_ctx.mail.space -
+ extra_per_mail);
+ }
+
+ /* now we have to move it. first move the body of the message,
+ then write the header and leave the extra space to beginning of
+ headers. */
+ offset = sync_ctx->file_input->v_offset;
+ if (mbox_move(sync_ctx, offset + mails[idx+1].space, offset,
+ *end_offset - offset - mails[idx+1].space)) {
+ // FIXME: error handling
+ return -1;
+ }
+
+ *end_offset = offset + mails[idx+1].space - str_len(mail_ctx.header);
+
+ if (pwrite_full(sync_ctx->fd, str_data(mail_ctx.header),
+ str_len(mail_ctx.header), *end_offset) < 0) {
+ // FIXME: error handling
+ return -1;
+ }
+
+ mails[idx].space += mails[idx+1].space - extra_per_mail;
+ return 0;
+}
+
int mbox_sync_rewrite(struct mbox_sync_context *sync_ctx, buffer_t *mails_buf,
uint32_t first_seq, uint32_t last_seq, off_t extra_space)
{
struct mbox_sync_mail *mails;
size_t size;
- uint32_t first_idx, last_idx, extra_per_mail;
-
- first_idx = first_seq-1;
- last_idx = last_seq-1;
+ uoff_t offset, end_offset;
+ uint32_t idx, extra_per_mail;
+ int ret = 0;
mails = buffer_get_modifyable_data(mails_buf, &size);
size /= sizeof(*mails);
/* FIXME: see if we can be faster by going back a few mails
- (update first_seq and last_seq) */
- /*while (mails[last_idx].space > 0) {
- }*/
+ (update first_seq and last_seq). */
+
+ extra_per_mail = (extra_space / (last_seq - first_seq + 1));
+
+ mails[last_seq-1].space -= extra_per_mail;
+ i_assert(mails[last_seq-1].space > 0);
+ end_offset = mails[last_seq-1].offset + mails[last_seq-1].space;
-#if 0
/* start moving backwards */
- extra_per_mail = (extra_space / (last_seq - first_seq + 1)) + 1;
- space_diff = 0;
- while (last_seq > first_seq) {
- dest = mails[last_seq].space_offset + mails[last_seq].space
+ while (--last_seq >= first_seq) {
+ idx = last_seq-1;
+ if (mails[idx].space <= 0) {
+ /* offset points to beginning of headers. read the
+ header again, update it and give enough space to
+ it */
+ if (mbox_sync_read_and_move(sync_ctx, mails, idx,
+ extra_per_mail,
+ &end_offset) < 0) {
+ ret = -1;
+ break;
+ }
+ } else {
+ /* X-Keywords: xx [offset] \n
+ ...
+ X-Keywords: xx [end_offset] \n
+
+ move data forward so mails before us gets the extra
+ space (ie. we temporarily get more space to us) */
+ offset = mails[idx].offset + mails[idx].space;
+ if (mbox_move(sync_ctx, offset + mails[idx+1].space,
+ offset, end_offset - offset)) {
+ // FIXME: error handling
+ ret = -1;
+ break;
+ }
+
+ mbox_sync_fix_from_offset(sync_ctx, idx+1,
+ mails[idx+1].space);
+
+ mails[idx].space += mails[idx+1].space - extra_per_mail;
+ i_assert(mails[idx].space > 0);
+ end_offset = mails[idx].offset + mails[idx].space;
+ }
}
-#endif
+
+ istream_raw_mbox_flush(sync_ctx->input);
+ return ret;
}
#include "mbox-sync-private.h"
static void status_flags_append(struct mbox_sync_mail_context *ctx,
- struct mbox_flag_type *flags_list)
+ const struct mbox_flag_type *flags_list)
{
int i;
for (i = 0; flags_list[i].chr != 0; i++) {
- if ((ctx->mail->flags & flags_list[i].flag) != 0)
+ if ((ctx->mail.flags & flags_list[i].flag) != 0)
str_append_c(ctx->header, flags_list[i].chr);
}
}
+
+static void status_flags_replace(struct mbox_sync_mail_context *ctx, size_t pos,
+ const struct mbox_flag_type *flags_list)
+{
+ unsigned char *data;
+ size_t size;
+ int i, need, have;
+
+ /* how many bytes do we need? */
+ for (i = 0, need = 0; flags_list[i].chr != 0; i++) {
+ if ((ctx->mail.flags & flags_list[i].flag) != 0)
+ need++;
+ }
+
+ /* how many bytes do we have now? */
+ data = buffer_get_modifyable_data(ctx->header, &size);
+ for (have = 0; pos < size; pos++) {
+ if (data[pos] == '\n')
+ break;
+
+ /* see if this is unknown flag for us */
+ for (i = 0; flags_list[i].chr != 0; i++) {
+ if (flags_list[i].chr == data[pos])
+ break;
+ }
+
+ if (flags_list[i].chr == 0)
+ have++;
+ else {
+ /* save this one */
+ data[pos-have] = data[pos];
+ }
+ }
+ pos -= have;
+
+ if (need < have)
+ str_delete(ctx->header, pos, have-need);
+ else if (need > have) {
+ buffer_copy(ctx->header, pos + (have-need),
+ ctx->header, pos, (size_t)-1);
+ }
+
+ /* @UNSAFE */
+ data = buffer_get_space_unsafe(ctx->header, pos, need);
+ for (i = 0, need = 0; flags_list[i].chr != 0; i++) {
+ if ((ctx->mail.flags & flags_list[i].flag) != 0)
+ *data++ = flags_list[i].chr;
+ }
+}
+
static void keywords_append(struct mbox_sync_mail_context *ctx,
keywords_mask_t keywords)
{
old_hdr_size = ctx->body_offset - ctx->hdr_offset;
new_hdr_size = str_len(ctx->header) + ctx->have_eoh;
- if (ctx->seq == 1 && ctx->base_uid_validity == 0) {
+ if (ctx->seq == 1 && ctx->sync_ctx->base_uid_validity == 0) {
ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
str_printfa(ctx->header, "X-IMAPbase: %u %u",
ctx->sync_ctx->hdr->uid_validity,
str_append_c(ctx->header, '\n');
}
- if (ctx->mail->uid == 0) {
+ if (ctx->hdr_pos[MBOX_HDR_X_UID] == (size_t)-1) {
+ if (ctx->mail.uid == 0)
+ ctx->mail.uid = ctx->sync_ctx->next_uid++;
ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
- str_printfa(ctx->header, "X-UID: %u\n",
- ctx->sync_ctx->next_uid++);
+ str_printfa(ctx->header, "X-UID: %u\n", ctx->mail.uid);
}
+ i_assert(ctx->mail.uid != 0);
if (ctx->hdr_pos[MBOX_HDR_STATUS] == (size_t)-1 &&
- (ctx->mail->flags & STATUS_FLAGS_MASK) != 0) {
+ (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) {
ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header);
str_append(ctx->header, "Status: ");
status_flags_append(ctx, mbox_status_flags);
}
if (ctx->hdr_pos[MBOX_HDR_X_STATUS] == (size_t)-1 &&
- (ctx->mail->flags & XSTATUS_FLAGS_MASK) != 0) {
+ (ctx->mail.flags & XSTATUS_FLAGS_MASK) != 0) {
ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header);
str_append(ctx->header, "X-Status: ");
status_flags_append(ctx, mbox_xstatus_flags);
have_keywords = FALSE;
for (i = 0; i < INDEX_KEYWORDS_BYTE_COUNT; i++) {
- if (ctx->mail->keywords[i] != 0) {
+ if (ctx->mail.keywords[i] != 0) {
have_keywords = TRUE;
break;
}
if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == (size_t)-1 && have_keywords) {
ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
str_append(ctx->header, "X-Keywords: ");
- keywords_append(ctx, ctx->mail->keywords);
+ keywords_append(ctx, ctx->mail.keywords);
str_append_c(ctx->header, '\n');
}
if (ctx->content_length == (uoff_t)-1) {
str_printfa(ctx->header, "Content-Length: %"PRIuUOFF_T"\n",
- ctx->mail->body_size);
+ ctx->mail.body_size);
}
if (str_len(ctx->header) != new_hdr_size) {
if (ctx->header_first_change == (size_t)-1)
ctx->header_first_change = new_hdr_size;
ctx->header_last_change = (size_t)-1;
- ctx->mail->space -= str_len(ctx->header) -
+ ctx->mail.space -= str_len(ctx->header) -
(new_hdr_size - ctx->have_eoh);
+ if (ctx->mail.space > 0) {
+ /* we should rewrite this header, so offset
+ must be broken if it's used anymore. */
+ ctx->mail.offset = (uoff_t)-1;
+ } else {
+ /* we don't have enough space for this header, change
+ offset to point back to beginning of headers */
+ ctx->mail.offset = ctx->hdr_offset;
+ }
new_hdr_size = str_len(ctx->header) + ctx->have_eoh;
}
static void mbox_sync_update_status(struct mbox_sync_mail_context *ctx)
{
+ if (ctx->hdr_pos[MBOX_HDR_STATUS] != (size_t)-1) {
+ status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_STATUS],
+ mbox_status_flags);
+ }
}
static void mbox_sync_update_xstatus(struct mbox_sync_mail_context *ctx)
{
+ if (ctx->hdr_pos[MBOX_HDR_X_STATUS] != (size_t)-1) {
+ status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_X_STATUS],
+ mbox_xstatus_flags);
+ }
}
static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx)
{
+ // FIXME
}
void mbox_sync_update_header(struct mbox_sync_mail_context *ctx,
- struct mail_index_sync_rec *update)
+ buffer_t *syncs_buf)
{
+ const struct mail_index_sync_rec *sync;
+ size_t size, i;
uint8_t old_flags;
keywords_mask_t old_keywords;
- if (update != NULL) {
- old_flags = ctx->mail->flags;
- memcpy(old_keywords, ctx->mail->keywords, sizeof(old_keywords));
+ sync = buffer_get_data(syncs_buf, &size);
+ size /= sizeof(*sync);
+
+ if (size != 0) {
+ old_flags = ctx->mail.flags;
+ memcpy(old_keywords, ctx->mail.keywords, sizeof(old_keywords));
- mail_index_sync_flags_apply(update, &ctx->mail->flags,
- ctx->mail->keywords);
+ for (i = 0; i < size; i++) {
+ mail_index_sync_flags_apply(&sync[i], &ctx->mail.flags,
+ ctx->mail.keywords);
+ }
if ((old_flags & STATUS_FLAGS_MASK) !=
- (ctx->mail->flags & STATUS_FLAGS_MASK))
+ (ctx->mail.flags & STATUS_FLAGS_MASK))
mbox_sync_update_status(ctx);
if ((old_flags & XSTATUS_FLAGS_MASK) !=
- (ctx->mail->flags & XSTATUS_FLAGS_MASK))
+ (ctx->mail.flags & XSTATUS_FLAGS_MASK))
mbox_sync_update_xstatus(ctx);
- if (memcmp(old_keywords, ctx->mail->keywords,
- sizeof(old_keywords)) != 0)
+ if (memcmp(old_keywords, ctx->mail.keywords,
+ INDEX_KEYWORDS_BYTE_COUNT) != 0)
mbox_sync_update_xkeywords(ctx);
}
mbox_sync_add_missing_headers(ctx);
}
+
+void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
+ const struct mbox_sync_mail *mail)
+{
+ if ((ctx->mail.flags & STATUS_FLAGS_MASK) !=
+ (mail->flags & STATUS_FLAGS_MASK)) {
+ ctx->mail.flags = (ctx->mail.flags & ~STATUS_FLAGS_MASK) |
+ (mail->flags & STATUS_FLAGS_MASK);
+ mbox_sync_update_status(ctx);
+ }
+ if ((ctx->mail.flags & XSTATUS_FLAGS_MASK) !=
+ (mail->flags & XSTATUS_FLAGS_MASK)) {
+ ctx->mail.flags = (ctx->mail.flags & ~XSTATUS_FLAGS_MASK) |
+ (mail->flags & XSTATUS_FLAGS_MASK);
+ mbox_sync_update_xstatus(ctx);
+ }
+ if (memcmp(ctx->mail.keywords, mail->keywords,
+ INDEX_KEYWORDS_BYTE_COUNT) != 0) {
+ memcpy(ctx->mail.keywords, mail->keywords,
+ INDEX_KEYWORDS_BYTE_COUNT);
+ mbox_sync_update_xkeywords(ctx);
+ }
+
+ i_assert(ctx->mail.uid == 0 || ctx->mail.uid == mail->uid);
+ ctx->mail.uid = mail->uid;
+ mbox_sync_add_missing_headers(ctx);
+}
if (file_set_size(sync_ctx->fd, size) < 0)
return -1;
- if (mail->space_offset == 0) {
+ if (mail->space <= 0) {
/* no X-Keywords header - place it at the end. */
grow_size += 13;
/* FIXME: can this break anything? X-Keywords text might
have been already included in space calculation. now we
have more.. */
- mail->space_offset = offset;
+ mail->offset = offset;
mail->space += grow_size;
} else {
- offset = mail->space_offset;
- if (mbox_move(sync_ctx, mail->space_offset + grow_size,
+ offset = mail->offset;
+ if (mbox_move(sync_ctx, offset + grow_size,
offset, (uoff_t)-1) < 0)
return -1;
}
return 0;
}
+static void mbox_sync_buffer_delete_old(buffer_t *syncs_buf, uint32_t seq)
+{
+ struct mail_index_sync_rec *sync;
+ size_t size, src, dest;
+
+ sync = buffer_get_modifyable_data(syncs_buf, &size);
+ size /= sizeof(*sync);
+
+ for (src = dest = 0; src < size; src++) {
+ if (sync[src].seq2 >= seq) {
+ if (src != dest)
+ sync[dest] = sync[src];
+ dest++;
+ }
+ }
+
+ buffer_set_used_size(syncs_buf, dest * sizeof(*sync));
+}
+
+static int
+mbox_sync_next_mail(struct mbox_sync_context *sync_ctx,
+ struct mbox_sync_mail_context *mail_ctx, uint32_t seq)
+{
+ uoff_t from_offset;
+
+ memset(mail_ctx, 0, sizeof(*mail_ctx));
+ mail_ctx->sync_ctx = sync_ctx;
+ mail_ctx->seq = seq;
+ mail_ctx->header = sync_ctx->header;
+
+ from_offset = istream_raw_mbox_get_start_offset(sync_ctx->input);
+ mail_ctx->mail.offset =
+ istream_raw_mbox_get_header_offset(sync_ctx->input);
+
+ mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx);
+ if (sync_ctx->input->v_offset == from_offset) {
+ /* this was the last mail */
+ return 0;
+ }
+
+ mail_ctx->mail.body_size =
+ istream_raw_mbox_get_body_size(sync_ctx->input,
+ mail_ctx->content_length);
+
+ /* save the offset permanently with recent flag state */
+ from_offset <<= 1;
+ if ((mail_ctx->mail.flags & MBOX_NONRECENT) == 0)
+ from_offset |= 1;
+ buffer_append(sync_ctx->ibox->mbox_data_buf, &from_offset,
+ sizeof(from_offset));
+ return 1;
+}
+
int mbox_sync(struct index_mailbox *ibox, int last_commit)
{
struct mbox_sync_context sync_ctx;
struct mbox_sync_mail_context mail_ctx;
- struct mbox_sync_mail mail;
struct mail_index_sync_ctx *index_sync_ctx;
struct mail_index_sync_rec sync_rec;
struct mail_index_view *sync_view;
struct istream *input;
uint32_t seq, need_space_seq, idx_seq, messages_count;
off_t space_diff;
- uoff_t from_offset, offset;
- buffer_t *mails;
- string_t *header;
+ uoff_t offset, extra_space;
+ buffer_t *mails, *syncs;
size_t size;
struct stat st;
int readonly, ret = 0;
buffer_set_used_size(ibox->mbox_data_buf, 0);
}
- readonly = TRUE; // FIXME
-
+ readonly = FALSE; // FIXME
// FIXME: lock the file
memset(&sync_ctx, 0, sizeof(sync_ctx));
+ sync_ctx.ibox = ibox;
sync_ctx.file_input = ibox->mbox_file_stream;
sync_ctx.input = ibox->mbox_stream;
sync_ctx.fd = ibox->mbox_fd;
sync_ctx.hdr = hdr;
+ sync_ctx.header = str_new(default_pool, 4096);
input = sync_ctx.input;
istream_raw_mbox_seek(input, 0);
- header = str_new(default_pool, 4096);
-
mails = buffer_create_dynamic(default_pool, 4096, (size_t)-1);
+ syncs = buffer_create_dynamic(default_pool, 256, (size_t)-1);
+
memset(&sync_rec, 0, sizeof(sync_rec));
messages_count = mail_index_view_get_message_count(sync_view);
space_diff = 0; need_space_seq = 0; idx_seq = 0; rec = NULL;
for (seq = 1; !input->eof; seq++) {
- if (sync_rec.seq2 < seq) {
- // FIXME: we may need more than one..
+ ret = 1;
+
+ /* get all sync records related to this message */
+ mbox_sync_buffer_delete_old(syncs, seq);
+ while (sync_rec.seq2 <= seq && ret > 0) {
+ if (sync_rec.seq2 != 0) {
+ buffer_append(syncs, &sync_rec,
+ sizeof(sync_rec));
+ }
ret = mail_index_sync_next(index_sync_ctx, &sync_rec);
- if (ret < 0)
- break;
}
+ if (ret < 0)
+ break;
- from_offset = input->v_offset;
-
- memset(&mail, 0, sizeof(mail));
- memset(&mail_ctx, 0, sizeof(mail_ctx));
- mail_ctx.sync_ctx = &sync_ctx;
- mail_ctx.mail = &mail;
- mail_ctx.seq = seq;
- mail_ctx.header = header;
-
- mbox_sync_parse_next_mail(input, &mail_ctx);
- if (input->v_offset == from_offset) {
- /* this was the last mail */
+ ret = mbox_sync_next_mail(&sync_ctx, &mail_ctx, seq);
+ if (ret <= 0)
break;
- }
- mail.body_size =
- istream_raw_mbox_get_size(input,
- mail_ctx.content_length);
- buffer_append(mails, &mail, sizeof(mail));
+ if ((mail_ctx.need_rewrite ||
+ buffer_get_used_size(syncs) != 0) && !readonly) {
+ mbox_sync_update_header(&mail_ctx, syncs);
+ if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0)
+ return -1;
- /* save the offset permanently with recent flag state */
- from_offset <<= 1;
- if ((mail.flags & MBOX_NONRECENT) == 0)
- from_offset |= 1;
- buffer_append(ibox->mbox_data_buf,
- &from_offset, sizeof(from_offset));
+ if (ret == 0 && need_space_seq == 0) {
+ /* first mail with no space to write it */
+ need_space_seq = seq;
+ space_diff = 0;
+ }
+ }
/* update index */
do {
- if (rec != NULL && rec->uid >= mail.uid)
+ if (rec != NULL && rec->uid >= mail_ctx.mail.uid)
break;
if (idx_seq >= messages_count) {
if (ret < 0)
break;
- if (rec != NULL && rec->uid != mail.uid) {
+ if (rec != NULL && rec->uid != mail_ctx.mail.uid) {
/* new UID in the middle of the mailbox -
shouldn't happen */
mail_storage_set_critical(ibox->box.storage,
"mbox sync: UID inserted in the middle "
- "of mailbox (%u > %u)", rec->uid, mail.uid);
+ "of mailbox (%u > %u)",
+ rec->uid, mail_ctx.mail.uid);
mail_index_mark_corrupted(ibox->index);
ret = -1;
break;
if (rec != NULL) {
/* see if flags changed */
if ((rec->flags & MAIL_FLAGS_MASK) !=
- (mail.flags & MAIL_FLAGS_MASK) ||
- memcmp(rec->keywords, mail.keywords,
+ (mail_ctx.mail.flags & MAIL_FLAGS_MASK) ||
+ memcmp(rec->keywords, mail_ctx.mail.keywords,
INDEX_KEYWORDS_BYTE_COUNT) != 0) {
uint8_t new_flags =
(rec->flags & ~MAIL_FLAGS_MASK) |
- (mail.flags & MAIL_FLAGS_MASK);
+ (mail_ctx.mail.flags & MAIL_FLAGS_MASK);
mail_index_update_flags(t, idx_seq,
MODIFY_REPLACE,
new_flags,
- mail.keywords);
+ mail_ctx.mail.keywords);
}
rec = NULL;
} else {
/* new message */
- mail_index_append(t, mail.uid, &idx_seq);
+ mail_index_append(t, mail_ctx.mail.uid, &idx_seq);
mail_index_update_flags(t, idx_seq, MODIFY_REPLACE,
- mail.flags & MAIL_FLAGS_MASK,
- mail.keywords);
+ mail_ctx.mail.flags & MAIL_FLAGS_MASK,
+ mail_ctx.mail.keywords);
}
- if (mail_ctx.need_rewrite && !readonly) {
- mbox_sync_update_header(&mail_ctx, NULL);
- if ((ret = mbox_sync_try_rewrite(&mail_ctx)) < 0)
- break;
- } else {
- ret = 1;
- }
+ istream_raw_mbox_next(input, mail_ctx.mail.body_size);
+ offset = input->v_offset;
+
+ if (need_space_seq != 0) {
+ buffer_append(mails, &mail_ctx.mail,
+ sizeof(mail_ctx.mail));
- if (ret == 0 && need_space_seq == 0) {
- /* didn't have space to write it */
- need_space_seq = seq;
- space_diff = mail.space;
- } else if (need_space_seq != 0) {
- space_diff += mail.space;
+ space_diff += mail_ctx.mail.space;
if (space_diff >= 0) {
/* we have enough space now */
if (mbox_sync_rewrite(&sync_ctx, mails,
ret = -1;
break;
}
+
+ /* mail_ctx may contain wrong data after
+ rewrite, so make sure we don't try to access
+ it */
+ memset(&mail_ctx, 0, sizeof(mail_ctx));
+ i_stream_seek(input, input->v_offset);
need_space_seq = 0;
}
}
-
- istream_raw_mbox_next(input, mail.body_size);
}
if (need_space_seq != 0) {
i_assert(space_diff < 0);
- if (mbox_sync_grow_file(&sync_ctx, &mail, mail_ctx.body_offset,
- -space_diff) < 0)
+ extra_space = MBOX_HEADER_EXTRA_SPACE *
+ ((seq-1) - need_space_seq);
+ if (mbox_sync_grow_file(&sync_ctx, &mail_ctx.mail,
+ mail_ctx.body_offset,
+ -space_diff + extra_space) < 0)
ret = -1;
else if (mbox_sync_rewrite(&sync_ctx, mails, need_space_seq,
- seq-1, space_diff) < 0)
+ seq-1, extra_space) < 0)
ret = -1;
}
+ if (sync_ctx.base_uid_last+1 != sync_ctx.next_uid) {
+ // FIXME: rewrite X-IMAPbase header
+ }
+
+ /* only syncs left should be just appends which weren't synced yet.
+ we'll just ignore them, as we've overwritten those above. */
while ((ret = mail_index_sync_next(index_sync_ctx, &sync_rec)) > 0) {
- // FIXME: should be just appends
+ i_assert(sync_rec.type == MAIL_INDEX_SYNC_TYPE_APPEND);
}
if (fstat(ibox->mbox_fd, &st) < 0) {
ibox->mbox_data = buffer_get_data(ibox->mbox_data_buf, &size);
ibox->mbox_data_count = size / sizeof(*ibox->mbox_data);
- str_free(header);
+ str_free(sync_ctx.header);
+ buffer_free(mails);
+ buffer_free(syncs);
return ret < 0 ? -1 : 0;
}
projects / thirdparty / dovecot / core.git / commitdiff
