struct message_header_line *hdr);
};
+static void parse_trailing_whitespace(struct mbox_sync_mail_context *ctx,
+ struct message_header_line *hdr)
+{
+ size_t i, space = 0;
+
+ /* the value may contain newlines. we can't count whitespace before
+ and after it as a single contiguous whitespace block, as that may
+ get us into situation where removing whitespace goes eg.
+ " \n \n" -> " \n\n" which would then be treated as end of headers.
+
+ that could probably be avoided by being careful, but as newlines
+ should never be there (we don't generate them), it's not worth the
+ trouble. */
+
+ for (i = hdr->full_value_len; i > 0; i--) {
+ if (!IS_LWSP(hdr->full_value[i-1]))
+ break;
+ space++;
+ }
+
+ if (space > ctx->mail.space) {
+ ctx->mail.offset = hdr->full_value_offset + i;
+ ctx->mail.space = space;
+ }
+}
+
static enum mail_flags mbox_flag_find(struct mbox_flag_type *flags, char chr)
{
int i;
// FIXME: save keywords
+ parse_trailing_whitespace(ctx, hdr);
return TRUE;
}
if (!parse_x_imap_base(ctx, hdr))
return FALSE;
- /* this is the UW-IMAP style "FOLDER INTERNAL DATA" message. skip it. */
+ /* this is the c-client style "FOLDER INTERNAL DATA" message.
+ skip it. */
ctx->pseudo = TRUE;
return TRUE;
}
static int parse_x_keywords(struct mbox_sync_mail_context *ctx,
struct message_header_line *hdr)
{
- size_t i, space = 0;
-
- for (i = hdr->full_value_len; i > 0; i--) {
- if (!IS_LWSP_LF(hdr->full_value[i-1]))
- break;
- space++;
- }
-
- if (space > ctx->mail.space) {
- ctx->mail.offset = hdr->full_value_offset + i;
- ctx->mail.space = space;
- }
-
// FIXME: parse them
ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
+ parse_trailing_whitespace(ctx, hdr);
return TRUE;
}
struct message_header_line *hdr)
{
uint32_t value = 0;
- size_t i, space_pos, extra_space = 0;
+ size_t i;
if (ctx->mail.uid != 0) {
/* duplicate */
value = value*10 + (hdr->full_value[i] - '0');
}
- space_pos = i;
- for (; i < hdr->full_value_len; i++) {
- if (!IS_LWSP_LF(hdr->full_value[i])) {
- /* broken value */
- return FALSE;
- }
- extra_space++;
- }
-
if (value >= ctx->sync_ctx->next_uid) {
/* next_uid broken - fix it */
ctx->sync_ctx->next_uid = value+1;
/* broken - UIDs must be growing */
return FALSE;
}
- ctx->sync_ctx->prev_msg_uid = value;
-
- ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
ctx->mail.uid = value;
- if (extra_space != 0 && ctx->mail.space == 0) {
- /* set it only if X-Keywords hasn't been seen. spaces in X-UID
- should be removed when writing X-Keywords. */
- ctx->mail.offset = hdr->full_value_offset + space_pos;
- ctx->mail.space = extra_space;
+ ctx->sync_ctx->prev_msg_uid = value;
+
+ if (ctx->sync_ctx->dest_first_mail && !ctx->seen_imapbase) {
+ /* everything was good, except we can't have X-UID before
+ X-IMAPbase header (to keep c-client compatibility). keep
+ the UID, but when we're rewriting this makes sure the
+ X-UID is appended after X-IMAPbase. */
+ return FALSE;
}
+
+ ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
+ parse_trailing_whitespace(ctx, hdr);
return TRUE;
}
str_truncate(ctx->header, 0);
line_start_pos = 0;
- hdr_ctx = message_parse_header_init(input, NULL);
+ hdr_ctx = message_parse_header_init(input, NULL, FALSE);
while ((hdr = message_parse_header_next(hdr_ctx)) != NULL) {
if (hdr->eoh) {
ctx->have_eoh = TRUE;
str_append(ctx->header, ": ");
}
+ if (ctx->header_first_change == (size_t)-1 &&
+ hdr->full_value_offset != str_len(ctx->header)) {
+ /* whitespaces around ':' are non-standard. either
+ there's whitespace before ':' or none after.
+ if we're going to rewrite this message, we can't
+ do it partially from here after as offsets won't
+ match. this shouldn't happen pretty much ever, so
+ don't try to optimize this - just rewrite the whole
+ thing. */
+ ctx->no_partial_rewrite = TRUE;
+ }
+
func = header_func_find(hdr->name);
if (func != NULL) {
if (hdr->continues)
message_parse_header_deinit(hdr_ctx);
if ((ctx->seq == 1 && sync_ctx->base_uid_validity == 0) ||
- (ctx->seq > 1 && sync_ctx->first_uid == 0)) {
+ (ctx->seq > 1 && sync_ctx->dest_first_mail)) {
/* missing X-IMAPbase */
ctx->need_rewrite = TRUE;
}
keywords_mask_t keywords;
uoff_t from_offset;
- uoff_t offset; /* if space <= 0, points to beginning */
- off_t space;
uoff_t body_size;
+
+ /* following variables have a bit overloaded functionality:
+
+ a) space <= 0 : offset points to beginning of headers. space is the
+ amount of space missing that is required to be able to rewrite
+ the headers
+ b) space > 0 : offset points to beginning of whitespace that can
+ be removed. space is the amount of data that can be removed from
+ there. note that the message may contain more whitespace
+ elsewhere. */
+ uoff_t offset;
+ off_t space;
};
struct mbox_sync_mail_context {
unsigned int have_eoh:1;
unsigned int need_rewrite:1;
+ unsigned int no_partial_rewrite:1;
unsigned int seen_imapbase:1;
unsigned int pseudo:1;
unsigned int updated:1;
buffer_t *mails, *syncs;
struct mail_index_sync_rec sync_rec;
- uint32_t prev_msg_uid, next_uid, first_uid;
+ uint32_t prev_msg_uid, next_uid;
uint32_t seq, idx_seq, need_space_seq;
off_t expunged_space, space_diff;
+
+ unsigned int dest_first_mail:1;
};
int mbox_sync(struct index_mailbox *ibox, int last_commit, int lock);
struct ostream *output;
off_t ret;
+ i_assert(size < OFF_T_MAX);
+
if (size == 0 || source == dest)
return 0;
static void mbox_sync_headers_add_space(struct mbox_sync_mail_context *ctx,
size_t size)
{
- size_t data_size, pos;
+ size_t data_size, pos, start_pos;
const unsigned char *data;
void *p;
/* Append at the end of X-Keywords header,
or X-UID if it doesn't exist */
- pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != (size_t)-1 ?
+ start_pos = ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] != (size_t)-1 ?
ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] :
ctx->hdr_pos[MBOX_HDR_X_UID];
- data = buffer_get_data(ctx->header, &data_size);
- while (pos < data_size && data[pos] != '\n')
- pos++;
+ data = str_data(ctx->header);
+ data_size = str_len(ctx->header);
+ for (pos = start_pos; pos < data_size; pos++) {
+ if (data[pos] == '\n') {
+ /* possibly continues in next line */
+ if (pos+1 == data_size || !IS_LWSP(data[pos+1]))
+ break;
+ start_pos = pos+1;
+ } else if (!IS_LWSP(data[pos])) {
+ start_pos = pos+1;
+ }
+ }
+
+ /* pos points to end of headers now, and start_pos to beginning of
+ whitespace. */
buffer_copy(ctx->header, pos + size,
ctx->header, pos, (size_t)-1);
p = buffer_get_space_unsafe(ctx->header, pos, size);
memset(p, ' ', size);
- ctx->mail.offset = ctx->hdr_offset + pos;
- if (ctx->mail.space < 0)
- ctx->mail.space = size;
- else
- ctx->mail.space += size;
+ ctx->mail.offset = ctx->hdr_offset + start_pos;
+ ctx->mail.space = (pos - start_pos) + size;
if (ctx->header_first_change > pos)
ctx->header_first_change = pos;
}
static void mbox_sync_header_remove_space(struct mbox_sync_mail_context *ctx,
- size_t pos, size_t *size)
+ size_t start_pos, size_t *size)
{
const unsigned char *data;
- size_t data_size, end, nonspace;
+ size_t data_size, pos, last_line_pos;
- /* find the end of the lwsp */
- nonspace = pos-1;
+ /* find the end of the LWSP */
data = str_data(ctx->header);
data_size = str_len(ctx->header);
- for (end = pos; end < data_size; end++) {
- if (data[end] == '\n') {
- if (end+1 == data_size || !IS_LWSP(data[end+1]))
+
+ for (pos = last_line_pos = start_pos; pos < data_size; pos++) {
+ if (data[pos] == '\n') {
+ /* possibly continues in next line */
+ if (pos+1 == data_size || !IS_LWSP(data[pos+1])) {
+ data_size = pos;
break;
- } else {
- if (!IS_LWSP(data[end]))
- nonspace = end;
+ }
+ last_line_pos = pos+1;
+ } else if (!IS_LWSP(data[pos])) {
+ start_pos = last_line_pos = pos+1;
}
}
+ if (start_pos == data_size)
+ return;
+
/* and remove what we can */
- nonspace++;
- if (end-nonspace < *size) {
- str_delete(ctx->header, nonspace, end-nonspace);
- *size -= end-nonspace;
- } else {
- str_delete(ctx->header, nonspace, *size);
- end -= *size;
- *size = 0;
+ if (ctx->header_first_change > start_pos)
+ ctx->header_first_change = start_pos;
+ ctx->header_last_change = (size_t)-1;
- if (ctx->mail.space < end-nonspace) {
- ctx->mail.space = end-nonspace;
- ctx->mail.offset = ctx->hdr_offset + nonspace;
- }
+ if (data_size - start_pos <= *size) {
+ /* remove it all */
+ str_delete(ctx->header, start_pos, data_size - start_pos);
+ *size -= data_size - start_pos;
+ return;
+ }
+
+ /* we have more space than needed. since we're removing from
+ the beginning of header instead of end, we don't have to
+ worry about multiline-headers. */
+ str_delete(ctx->header, start_pos, *size);
+ last_line_pos = last_line_pos <= *size ?
+ start_pos : last_line_pos - *size;
+
+ data_size -= *size;
+ *size = 0;
+
+ if (ctx->mail.space < data_size - last_line_pos) {
+ ctx->mail.space = data_size - last_line_pos;
+ ctx->mail.offset = ctx->hdr_offset + last_line_pos;
}
}
size_t size)
{
static enum header_position space_positions[] = {
- MBOX_HDR_X_KEYWORDS,
MBOX_HDR_X_UID,
+ MBOX_HDR_X_KEYWORDS,
MBOX_HDR_X_IMAPBASE
};
enum header_position pos;
int i;
- ctx->header_last_change = (size_t)-1;
-
ctx->mail.space = 0;
ctx->mail.offset = ctx->hdr_offset;
for (i = 0; i < 3 && size > 0; i++) {
pos = space_positions[i];
if (ctx->hdr_pos[pos] != (size_t)-1) {
- if (ctx->header_first_change > ctx->hdr_pos[pos])
- ctx->header_first_change = ctx->hdr_pos[pos];
mbox_sync_header_remove_space(ctx, ctx->hdr_pos[pos],
&size);
}
}
-
- i_assert(size == 0);
}
int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff)
{
size_t old_hdr_size, new_hdr_size;
- const unsigned char *data;
i_assert(ctx->sync_ctx->ibox->mbox_lock_type == F_WRLCK);
if (new_hdr_size < old_hdr_size) {
mbox_sync_headers_add_space(ctx, old_hdr_size - new_hdr_size);
} else if (new_hdr_size > old_hdr_size) {
- size_t needed = new_hdr_size - old_hdr_size;
-
- if (ctx->mail.space >= 0)
- mbox_sync_headers_remove_space(ctx, needed);
- else if (move_diff < 0 && needed <= -move_diff) {
+ /* try removing the space where we can */
+ mbox_sync_headers_remove_space(ctx,
+ new_hdr_size - old_hdr_size);
+ new_hdr_size = str_len(ctx->header);
+
+ if (new_hdr_size <= old_hdr_size) {
+ /* good, we removed enough. */
+ i_assert(new_hdr_size == old_hdr_size);
+ } else if (move_diff < 0 &&
+ new_hdr_size - old_hdr_size <= -move_diff) {
/* moving backwards - we can use the extra space from
it, just update expunged_space accordingly */
- i_assert(ctx->sync_ctx->expunged_space >= needed);
- ctx->sync_ctx->expunged_space -= needed;
+ i_assert(ctx->mail.space == 0);
+ i_assert(ctx->sync_ctx->expunged_space >=
+ new_hdr_size - old_hdr_size);
+ ctx->sync_ctx->expunged_space -=
+ new_hdr_size - old_hdr_size;
} else {
+ /* couldn't get enough space */
+ i_assert(ctx->mail.space == 0);
+ ctx->mail.space =
+ -(ssize_t)(new_hdr_size - old_hdr_size);
return 0;
}
}
+ i_assert(ctx->mail.space >= 0);
+
if (ctx->header_first_change == (size_t)-1 && move_diff == 0) {
/* no changes actually. we get here if index sync record told
us to do something that was already there */
return 1;
}
- if (move_diff != 0) {
- /* we're moving the header, forget about partial write
- optimizations */
+ if (move_diff != 0 || ctx->no_partial_rewrite) {
+ /* forget about partial write optimizations */
ctx->header_first_change = 0;
ctx->header_last_change = 0;
}
- /* FIXME: last_change should rather just tell if we want to truncate
- to beginning of extra whitespace */
if (ctx->header_last_change != (size_t)-1 &&
ctx->header_last_change != 0)
str_truncate(ctx->header, ctx->header_last_change);
- data = str_data(ctx->header);
- new_hdr_size = str_len(ctx->header);
- if (pwrite_full(ctx->sync_ctx->fd, data + ctx->header_first_change,
- new_hdr_size - ctx->header_first_change,
+ if (pwrite_full(ctx->sync_ctx->fd,
+ str_data(ctx->header) + ctx->header_first_change,
+ str_len(ctx->header) - ctx->header_first_change,
ctx->hdr_offset + move_diff +
ctx->header_first_change) < 0) {
mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()");
so we have to fool it. */
old_prev_msg_uid = sync_ctx->prev_msg_uid;
sync_ctx->prev_msg_uid = mails[idx].uid-1;
+ sync_ctx->dest_first_mail = seq == 1;
mbox_sync_parse_next_mail(sync_ctx->input, &mail_ctx, TRUE);
if (mails[idx].space != 0)
str_append_c(mail_ctx.header, '\n');
}
- i_assert(mail_ctx.mail.space == mails[idx].space);
sync_ctx->prev_msg_uid = old_prev_msg_uid;
+ sync_ctx->dest_first_mail = FALSE;
+
+ mail_ctx.mail.space =
+ -(ssize_t)(str_len(mail_ctx.header) -
+ (mail_ctx.body_offset - mail_ctx.hdr_offset));
+ i_assert(mail_ctx.mail.space == mails[idx].space);
if (mail_ctx.mail.space <= 0)
mbox_sync_headers_add_space(&mail_ctx, extra_per_mail);
so we have to fool it. */
old_prev_msg_uid = sync_ctx->prev_msg_uid;
sync_ctx->prev_msg_uid = mails[idx].uid-1;
+ sync_ctx->dest_first_mail = seq == 1;
mbox_sync_parse_next_mail(sync_ctx->input, &mail_ctx, TRUE);
mbox_sync_update_header_from(&mail_ctx, &mails[idx]);
sync_ctx->prev_msg_uid = old_prev_msg_uid;
+ sync_ctx->dest_first_mail = FALSE;
mbox_sync_headers_add_space(&mail_ctx,end_offset - start_offset);
i_assert(mails[idx].space >= 0);
end_offset = mails[idx].offset + mails[idx].space;
- i_assert(mails[0].space < 0 || mails[0].uid == 0);
- start_offset = mails[0].offset;
-
/* after expunge the next mail must have been missing space, or we
would have moved it backwards already */
- i_assert(mails[0].uid != 0 || mails[1].space < 0);
+ i_assert(mails[0].space < 0 || mails[0].uid == 0);
+ start_offset = mails[0].offset;
/* start moving backwards */
do {
static void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx,
size_t pos, size_t need, size_t have)
{
+ ssize_t diff = (ssize_t)need - (ssize_t)have;
int i;
- if (need == have) {
+ if (diff == 0) {
if (ctx->header_last_change < pos + have ||
ctx->header_last_change == (size_t)-1)
ctx->header_last_change = pos + have;
} else {
+ /* FIXME: if (diff < ctx->space && pos < ctx->offset) then
+ move the data only up to space offset and give/take the
+ space from there. update header_last_change accordingly. */
ctx->header_last_change = (size_t)-1;
for (i = 0; i < MBOX_HDR_COUNT; i++) {
if (ctx->hdr_pos[i] > pos &&
ctx->hdr_pos[i] != (size_t)-1)
- ctx->hdr_pos[i] += need - have;
+ ctx->hdr_pos[i] += diff;
}
- if (need < have) {
- str_delete(ctx->header, pos, have-need);
- ctx->mail.space += have - need;
- } else {
+ if (diff < 0)
+ str_delete(ctx->header, pos, -diff);
+ else {
ctx->header_last_change = (size_t)-1;
- buffer_copy(ctx->header, pos + (need-have),
+ buffer_copy(ctx->header, pos + diff,
ctx->header, pos, (size_t)-1);
- ctx->mail.space -= need - have;
}
}
}
str_append_c(ctx->header, '\n');
}
- if (ctx->mail.uid == ctx->sync_ctx->first_uid &&
+ if (ctx->sync_ctx->dest_first_mail &&
ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1) {
if (ctx->sync_ctx->base_uid_validity == 0) {
ctx->sync_ctx->base_uid_validity =
if (ctx->header_first_change == (size_t)-1)
ctx->header_first_change = new_hdr_size;
ctx->header_last_change = (size_t)-1;
- ctx->mail.space -= str_len(ctx->header) - new_hdr_size;
- if (ctx->mail.space > 0) {
- /* we should rewrite this header, so offset
- must be broken if it's used anymore. */
- ctx->mail.offset = (uoff_t)-1;
- } else {
- /* we don't have enough space for this header, change
- offset to point back to beginning of headers */
- ctx->mail.offset = ctx->hdr_offset;
- }
}
if (ctx->have_eoh)
const char *p, *hdr;
size_t pos;
- if (ctx->mail.uid != ctx->sync_ctx->first_uid ||
+ if (ctx->sync_ctx->dest_first_mail ||
ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == (size_t)-1 ||
ctx->sync_ctx->update_base_uid_last == 0 ||
ctx->sync_ctx->update_base_uid_last < ctx->sync_ctx->base_uid_last)
mail_ctx->mail.offset =
istream_raw_mbox_get_header_offset(sync_ctx->input);
- if (mail_ctx->seq > 1 && sync_ctx->first_uid == mail_ctx->mail.uid) {
+ if (mail_ctx->seq > 1 && sync_ctx->dest_first_mail) {
/* First message was expunged and this is the next one.
Skip \n header */
mail_ctx->from_offset++;
mail_ctx->mail.body_size;
mail_ctx->mail.body_size = 0;
- if (mail_ctx->sync_ctx->seq == 1) {
+ if (mail_ctx->sync_ctx->dest_first_mail) {
/* expunging first message, fix space to contain next
message's \n header too since it will be removed. */
mail_ctx->mail.space++;
off_t move_diff;
int ret;
- if (sync_ctx->first_uid == 0)
- sync_ctx->first_uid = mail_ctx->mail.uid;
-
if (sync_ctx->ibox->mbox_readonly)
return 0;
/* set to -1, since they're always increased later */
sync_ctx->seq = sync_ctx->idx_seq = seq-1;
+ sync_ctx->dest_first_mail = sync_ctx->seq == 0;
if (istream_raw_mbox_seek(sync_ctx->input, offset) < 0) {
mail_storage_set_critical(sync_ctx->ibox->box.storage,
"Cached message offset %s is invalid for mbox file %s",
"Mailbox isn't a valid mbox file");
return -1;
}
+ sync_ctx->dest_first_mail = TRUE;
} else {
/* we sync only what we need to. jump to first record that
needs updating */
const struct mail_index_sync_rec *sync_rec;
+ size_t size;
- if (buffer_get_used_size(sync_ctx->syncs) == 0) {
+ if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
+ sync_ctx->sync_rec.uid1 == 0) {
if (mbox_sync_read_index_syncs(sync_ctx, 1,
&expunged) < 0)
return -1;
- if (buffer_get_used_size(sync_ctx->syncs) == 0) {
+ if (buffer_get_used_size(sync_ctx->syncs) == 0 &&
+ sync_ctx->sync_rec.uid1 == 0) {
/* nothing to do */
return 0;
}
}
- sync_rec = buffer_get_data(sync_ctx->syncs, NULL);
+ sync_rec = buffer_get_data(sync_ctx->syncs, &size);
+ if (size == 0)
+ sync_rec = &sync_ctx->sync_rec;
if (mbox_sync_seek_to_uid(sync_ctx, sync_rec->uid1) < 0)
return -1;
-
- if (sync_ctx->seq > 0) {
- if (mail_index_lookup_uid(sync_ctx->sync_view, 1,
- &sync_ctx->first_uid) < 0)
- return -1;
- }
}
while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) {
if (mbox_sync_read_index_syncs(sync_ctx, uid, &expunged) < 0)
return -1;
- if (!expunged)
+ if (!expunged) {
ret = mbox_sync_handle_header(mail_ctx);
- else {
+ sync_ctx->dest_first_mail = FALSE;
+ } else {
mail_ctx->mail.uid = 0;
ret = mbox_sync_handle_expunge(mail_ctx);
}
sync_ctx->base_uid_last = 0;
sync_ctx->next_uid = 1;
- sync_ctx->prev_msg_uid = sync_ctx->first_uid = 0;
+ sync_ctx->prev_msg_uid = 0;
sync_ctx->seq = sync_ctx->idx_seq = 0;
}
projects / thirdparty / dovecot / core.git / commitdiff
