From 345212e8f61ebf14ff4f80df26df9e655eb5121e Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Sun, 20 Jun 2004 06:25:32 +0300 Subject: [PATCH] mailbox_save() and mailbox_copy() functions can now return the saved mail so it can be immediately queried. Implemented UIDPLUS extension using it. Maildir implementation missing, so it crashes with it for now.. APPEND with mbox now doesn't require resyncing the mailbox since it updates indexes directly. --HG-- branch : HEAD --- configure.in | 2 +- src/imap/cmd-append.c | 38 ++- src/imap/cmd-close.c | 2 +- src/imap/cmd-copy.c | 61 +++- src/imap/cmd-expunge.c | 36 ++- src/imap/cmd-select.c | 4 +- src/imap/cmd-status.c | 11 +- src/imap/commands-util.c | 39 +++ src/imap/commands-util.h | 11 + src/imap/commands.c | 1 + src/imap/commands.h | 1 + src/imap/imap-expunge.c | 4 +- src/imap/imap-expunge.h | 2 +- src/lib-index/mail-index-transaction.c | 58 ++-- src/lib-index/mail-index.h | 4 + src/lib-storage/index/index-mail.c | 2 + src/lib-storage/index/index-search.c | 6 +- src/lib-storage/index/maildir/maildir-copy.c | 6 +- src/lib-storage/index/maildir/maildir-save.c | 7 +- .../index/maildir/maildir-storage.h | 6 +- src/lib-storage/index/mbox/mbox-lock.c | 1 + src/lib-storage/index/mbox/mbox-mail.c | 15 +- src/lib-storage/index/mbox/mbox-save.c | 270 ++++++++++++------ src/lib-storage/index/mbox/mbox-storage.h | 4 +- src/lib-storage/index/mbox/mbox-sync-parse.c | 10 +- .../index/mbox/mbox-sync-private.h | 5 +- .../index/mbox/mbox-sync-rewrite.c | 15 +- src/lib-storage/index/mbox/mbox-sync-update.c | 11 +- src/lib-storage/index/mbox/mbox-sync.c | 64 +++-- src/lib-storage/index/mbox/mbox-transaction.c | 2 +- src/lib-storage/mail-save.c | 13 +- src/lib-storage/mail-save.h | 6 +- src/lib-storage/mail-storage-private.h | 6 +- src/lib-storage/mail-storage.c | 10 +- src/lib-storage/mail-storage.h | 16 +- 35 files changed, 525 insertions(+), 224 deletions(-) diff --git a/configure.in b/configure.in index 9b50263c46..59517fbb4c 100644 --- a/configure.in +++ b/configure.in @@ -1206,7 +1206,7 @@ dnl ** dnl ** capabilities dnl ** -capability="IMAP4rev1 SORT THREAD=REFERENCES MULTIAPPEND UNSELECT LITERAL+ IDLE CHILDREN LISTEXT NAMESPACE" +capability="IMAP4rev1 SORT THREAD=REFERENCES MULTIAPPEND UNSELECT LITERAL+ IDLE CHILDREN LISTEXT NAMESPACE UIDPLUS" AC_DEFINE_UNQUOTED(CAPABILITY_STRING, "$capability", IMAP capabilities) CFLAGS="$CFLAGS $EXTRA_CFLAGS" diff --git a/src/imap/cmd-append.c b/src/imap/cmd-append.c index 7ce6cb2fff..5b0ad4bc0f 100644 --- a/src/imap/cmd-append.c +++ b/src/imap/cmd-append.c @@ -4,6 +4,7 @@ #include "ioloop.h" #include "istream.h" #include "ostream.h" +#include "str.h" #include "commands.h" #include "imap-parser.h" #include "imap-date.h" @@ -55,9 +56,12 @@ int cmd_append(struct client *client) struct mailbox_keywords old_flags; struct mail_full_flags flags; struct istream *input; + struct mail *mail; time_t internal_date; const char *mailbox, *internal_date_str; uoff_t msg_size; + string_t *reply; + struct msgset_generator_context msgset_ctx; unsigned int count; int ret, failed, timezone_offset, nonsync; @@ -72,13 +76,19 @@ int cmd_append(struct client *client) if (storage == NULL) return TRUE; - box = mailbox_open(storage, mailbox, MAILBOX_OPEN_FAST); - if (box == NULL) { - client_send_storage_error(client, storage); - return TRUE; + if (client->mailbox != NULL && + mailbox_name_equals(mailbox_get_name(client->mailbox), mailbox)) + box = client->mailbox; + else { + box = mailbox_open(storage, mailbox, MAILBOX_OPEN_FAST); + if (box == NULL) { + client_send_storage_error(client, storage); + return TRUE; + } } - if (mailbox_get_status(box, STATUS_KEYWORDS, &status) < 0) { + if (mailbox_get_status(box, STATUS_KEYWORDS | STATUS_UIDVALIDITY, + &status) < 0) { client_send_storage_error(client, storage); mailbox_close(box); return TRUE; @@ -93,6 +103,10 @@ int cmd_append(struct client *client) /* if error occurs, the CRLF is already read. */ client->input_skip_line = FALSE; + reply = str_new(default_pool, 256); + str_printfa(reply, "OK [APPENDUID %u ", status.uidvalidity); + + msgset_generator_init(&msgset_ctx, reply); count = 0; failed = TRUE; save_parser = imap_parser_create(client->input, client->output, @@ -180,13 +194,15 @@ int cmd_append(struct client *client) client->input->v_offset, msg_size); if (mailbox_save(t, &flags, internal_date, timezone_offset, - NULL, input) < 0) { + NULL, input, &mail) < 0) { i_stream_unref(input); client_send_storage_error(client, storage); break; } i_stream_unref(input); + msgset_generator_next(&msgset_ctx, mail->uid); + if (client->input->closed) break; @@ -194,6 +210,8 @@ int cmd_append(struct client *client) } imap_parser_destroy(save_parser); + msgset_generator_finish(&msgset_ctx); + if (failed) mailbox_transaction_rollback(t); else { @@ -203,11 +221,15 @@ int cmd_append(struct client *client) } } - mailbox_close(box); + if (box != client->mailbox) + mailbox_close(box); if (!failed) { client_sync_full(client); - client_send_tagline(client, "OK Append completed."); + str_append(reply, "] Append completed."); + client_send_tagline(client, str_c(reply)); } + str_free(reply); + return TRUE; } diff --git a/src/imap/cmd-close.c b/src/imap/cmd-close.c index db7a4b132a..25671e76be 100644 --- a/src/imap/cmd-close.c +++ b/src/imap/cmd-close.c @@ -15,7 +15,7 @@ int cmd_close(struct client *client) client->mailbox = NULL; if (!mailbox_is_readonly(mailbox)) { - if (!imap_expunge(mailbox)) + if (!imap_expunge(mailbox, NULL)) client_send_untagged_storage_error(client, storage); } diff --git a/src/imap/cmd-copy.c b/src/imap/cmd-copy.c index ac947780dd..8bd51d7d84 100644 --- a/src/imap/cmd-copy.c +++ b/src/imap/cmd-copy.c @@ -1,16 +1,20 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "common.h" +#include "str.h" #include "commands.h" #include "imap-search.h" static int fetch_and_copy(struct mailbox_transaction_context *t, struct mailbox *srcbox, - struct mail_search_arg *search_args) + struct mail_search_arg *search_args, + string_t *reply) { struct mail_search_context *search_ctx; struct mailbox_transaction_context *src_trans; - struct mail *mail; + struct mail *mail, *dest_mail; + struct msgset_generator_context srcset_ctx, destset_ctx; + string_t *dest_str; int ret; src_trans = mailbox_transaction_begin(srcbox, FALSE); @@ -22,16 +26,35 @@ static int fetch_and_copy(struct mailbox_transaction_context *t, return -1; } + dest_str = t_str_new(128); + msgset_generator_init(&srcset_ctx, reply); + msgset_generator_init(&destset_ctx, dest_str); + ret = 1; while ((mail = mailbox_search_next(search_ctx)) != NULL) { if (mail->expunged) { ret = 0; break; } - if (mailbox_copy(t, mail) < 0) { + if (mailbox_copy(t, mail, &dest_mail) < 0) { ret = -1; break; } + + msgset_generator_next(&srcset_ctx, mail->uid); + msgset_generator_next(&destset_ctx, dest_mail->uid); + + } + + msgset_generator_finish(&srcset_ctx); + msgset_generator_finish(&destset_ctx); + + if (str_len(dest_str) == 0) + str_truncate(reply, 0); + else { + str_append_c(reply, ' '); + str_append_str(reply, dest_str); + str_append(reply, "] Copy completed."); } if (mailbox_search_deinit(search_ctx) < 0) @@ -49,7 +72,9 @@ int cmd_copy(struct client *client) struct mailbox *destbox; struct mailbox_transaction_context *t; struct mail_search_arg *search_arg; + struct mailbox_status status; const char *messageset, *mailbox; + string_t *reply; int ret; /* */ @@ -74,14 +99,28 @@ int cmd_copy(struct client *client) if (storage == NULL) return TRUE; - destbox = mailbox_open(storage, mailbox, MAILBOX_OPEN_FAST); - if (destbox == NULL) { + if (mailbox_name_equals(mailbox_get_name(client->mailbox), mailbox)) + destbox = client->mailbox; + else { + destbox = mailbox_open(storage, mailbox, MAILBOX_OPEN_FAST); + if (destbox == NULL) { + client_send_storage_error(client, storage); + return TRUE; + } + } + + if (mailbox_get_status(destbox, STATUS_UIDVALIDITY, &status) < 0) { client_send_storage_error(client, storage); + if (destbox != client->mailbox) + mailbox_close(destbox); return TRUE; } + reply = str_new(default_pool, 512); + str_printfa(reply, "OK [COPYUID %u ", status.uidvalidity); + t = mailbox_transaction_begin(destbox, FALSE); - ret = fetch_and_copy(t, client->mailbox, search_arg); + ret = fetch_and_copy(t, client->mailbox, search_arg, reply); if (ret <= 0) mailbox_transaction_rollback(t); @@ -102,9 +141,15 @@ int cmd_copy(struct client *client) client_sync_full(client); else client_sync_full_fast(client); - client_send_tagline(client, "OK Copy completed."); + if (str_len(reply) > 0) + client_send_tagline(client, str_c(reply)); + else { + client_send_tagline(client, + "OK Copy completed, no messages found."); + } } - mailbox_close(destbox); + if (destbox != client->mailbox) + mailbox_close(destbox); return TRUE; } diff --git a/src/imap/cmd-expunge.c b/src/imap/cmd-expunge.c index 2d9a086fb0..b4b6b385f0 100644 --- a/src/imap/cmd-expunge.c +++ b/src/imap/cmd-expunge.c @@ -2,14 +2,48 @@ #include "common.h" #include "commands.h" +#include "imap-search.h" #include "imap-expunge.h" +int cmd_uid_expunge(struct client *client) +{ + struct imap_arg *args; + struct mail_search_arg *search_arg; + const char *uidset; + + if (!client_read_args(client, 1, 0, &args)) + return FALSE; + + if (!client_verify_open_mailbox(client)) + return TRUE; + + uidset = imap_arg_string(&args[0]); + if (uidset == NULL) { + client_send_command_error(client, "Invalid arguments."); + return TRUE; + } + + search_arg = imap_search_get_arg(client, uidset, TRUE); + if (search_arg == NULL) + return TRUE; + + if (imap_expunge(client->mailbox, search_arg)) { + client_sync_full(client); + client_send_tagline(client, "OK Expunge completed."); + } else { + client_send_storage_error(client, + mailbox_get_storage(client->mailbox)); + } + + return TRUE; +} + int cmd_expunge(struct client *client) { if (!client_verify_open_mailbox(client)) return TRUE; - if (imap_expunge(client->mailbox)) { + if (imap_expunge(client->mailbox, NULL)) { client_sync_full(client); client_send_tagline(client, "OK Expunge completed."); } else { diff --git a/src/imap/cmd-select.c b/src/imap/cmd-select.c index a29a212140..080133e62e 100644 --- a/src/imap/cmd-select.c +++ b/src/imap/cmd-select.c @@ -27,8 +27,8 @@ int _cmd_select_full(struct client *client, int readonly) if (storage == NULL) return TRUE; - box = mailbox_open(storage, mailbox, - readonly ? MAILBOX_OPEN_READONLY : 0); + box = mailbox_open(storage, mailbox, !readonly ? 0 : + (MAILBOX_OPEN_READONLY | MAILBOX_OPEN_KEEP_RECENT)); if (box == NULL) { client_send_storage_error(client, storage); return TRUE; diff --git a/src/imap/cmd-status.c b/src/imap/cmd-status.c index febe2d3456..61fd3ea8cf 100644 --- a/src/imap/cmd-status.c +++ b/src/imap/cmd-status.c @@ -43,14 +43,6 @@ get_status_items(struct client *client, struct imap_arg *args) return items; } -static int mailbox_name_equals(const char *box1, const char *box2) -{ - if (strcmp(box1, box2) == 0) - return TRUE; - - return strcasecmp(box1, "INBOX") == 0 && strcasecmp(box2, "INBOX") == 0; -} - static int get_mailbox_status(struct client *client, struct mail_storage *storage, const char *mailbox, enum mailbox_status_items items, @@ -66,7 +58,8 @@ static int get_mailbox_status(struct client *client, } else { /* open the mailbox */ box = mailbox_open(storage, mailbox, MAILBOX_OPEN_FAST | - MAILBOX_OPEN_READONLY); + MAILBOX_OPEN_READONLY | + MAILBOX_OPEN_KEEP_RECENT); if (box == NULL) return FALSE; } diff --git a/src/imap/commands-util.c b/src/imap/commands-util.c index f5760a528f..93ee9b5add 100644 --- a/src/imap/commands-util.c +++ b/src/imap/commands-util.c @@ -339,3 +339,42 @@ void client_save_keywords(struct mailbox_keywords *dest, for (i = 0; i < keywords_count; i++) dest->keywords[i] = p_strdup(dest->pool, keywords[i]); } + +int mailbox_name_equals(const char *box1, const char *box2) +{ + if (strcmp(box1, box2) == 0) + return TRUE; + + return strcasecmp(box1, "INBOX") == 0 && strcasecmp(box2, "INBOX") == 0; +} + +void msgset_generator_init(struct msgset_generator_context *ctx, string_t *str) +{ + memset(ctx, 0, sizeof(*ctx)); + ctx->str = str; + ctx->last_uid = (uint32_t)-1; +} + +void msgset_generator_next(struct msgset_generator_context *ctx, uint32_t uid) +{ + if (uid != ctx->last_uid+1) { + if (ctx->first_uid == 0) + ; + else if (ctx->first_uid == ctx->last_uid) + str_printfa(ctx->str, "%u,", ctx->first_uid); + else { + str_printfa(ctx->str, "%u:%u,", + ctx->first_uid, ctx->last_uid); + } + ctx->first_uid = uid; + } + ctx->last_uid = uid; +} + +void msgset_generator_finish(struct msgset_generator_context *ctx) +{ + if (ctx->first_uid == ctx->last_uid) + str_printfa(ctx->str, "%u", ctx->first_uid); + else + str_printfa(ctx->str, "%u:%u", ctx->first_uid, ctx->last_uid); +} diff --git a/src/imap/commands-util.h b/src/imap/commands-util.h index 2df173625a..4cb8f75571 100644 --- a/src/imap/commands-util.h +++ b/src/imap/commands-util.h @@ -1,6 +1,11 @@ #ifndef __COMMANDS_UTIL_H #define __COMMANDS_UTIL_H +struct msgset_generator_context { + string_t *str; + uint32_t first_uid, last_uid; +}; + struct mail_full_flags; /* Finds mail storage for given mailbox from namespaces. If not found, @@ -54,4 +59,10 @@ void client_send_mailbox_flags(struct client *client, struct mailbox *box, void client_save_keywords(struct mailbox_keywords *dest, const char *keywords[], unsigned int keywords_count); +int mailbox_name_equals(const char *box1, const char *box2); + +void msgset_generator_init(struct msgset_generator_context *ctx, string_t *str); +void msgset_generator_next(struct msgset_generator_context *ctx, uint32_t uid); +void msgset_generator_finish(struct msgset_generator_context *ctx); + #endif diff --git a/src/imap/commands.c b/src/imap/commands.c index 16db0bbd8b..1e8912158f 100644 --- a/src/imap/commands.c +++ b/src/imap/commands.c @@ -46,6 +46,7 @@ const struct command imap_ext_commands[] = { { "NAMESPACE", cmd_namespace }, { "SORT", cmd_sort }, { "THREAD", cmd_thread }, + { "UID EXPUNGE", cmd_uid_expunge }, { "UID SORT", cmd_sort }, { "UID THREAD", cmd_thread }, { "UNSELECT", cmd_unselect } diff --git a/src/imap/commands.h b/src/imap/commands.h index 951393d8a6..f4af5002d5 100644 --- a/src/imap/commands.h +++ b/src/imap/commands.h @@ -66,6 +66,7 @@ int cmd_idle(struct client *client); int cmd_namespace(struct client *client); int cmd_sort(struct client *client); int cmd_thread(struct client *client); +int cmd_uid_expunge(struct client *client); int cmd_unselect(struct client *client); /* private: */ diff --git a/src/imap/imap-expunge.c b/src/imap/imap-expunge.c index 30052238d7..7b05d23e80 100644 --- a/src/imap/imap-expunge.c +++ b/src/imap/imap-expunge.c @@ -5,7 +5,7 @@ #include "mail-search.h" #include "imap-expunge.h" -int imap_expunge(struct mailbox *box) +int imap_expunge(struct mailbox *box, struct mail_search_arg *next_search_arg) { struct mail_search_context *ctx; struct mailbox_transaction_context *t; @@ -15,6 +15,7 @@ int imap_expunge(struct mailbox *box) memset(&search_arg, 0, sizeof(search_arg)); search_arg.type = SEARCH_DELETED; + search_arg.next = next_search_arg; t = mailbox_transaction_begin(box, FALSE); ctx = mailbox_search_init(t, NULL, &search_arg, NULL, 0, NULL); @@ -41,4 +42,3 @@ int imap_expunge(struct mailbox *box) return !failed; } - diff --git a/src/imap/imap-expunge.h b/src/imap/imap-expunge.h index 039d9022b0..8468e9d323 100644 --- a/src/imap/imap-expunge.h +++ b/src/imap/imap-expunge.h @@ -1,6 +1,6 @@ #ifndef __IMAP_EXPUNGE_H #define __IMAP_EXPUNGE_H -int imap_expunge(struct mailbox *box); +int imap_expunge(struct mailbox *box, struct mail_search_arg *next_search_arg); #endif diff --git a/src/lib-index/mail-index-transaction.c b/src/lib-index/mail-index-transaction.c index 1a0a3a4b21..0c627cf9fd 100644 --- a/src/lib-index/mail-index-transaction.c +++ b/src/lib-index/mail-index-transaction.c @@ -161,6 +161,18 @@ void mail_index_append(struct mail_index_transaction *t, uint32_t uid, rec->uid = uid; } +static struct mail_index_record * +mail_index_lookup_append_rec(struct mail_index_transaction *t, uint32_t seq) +{ + size_t pos; + + i_assert(seq >= t->first_new_seq && seq <= t->last_new_seq); + + pos = (seq - t->first_new_seq) * t->view->index->record_size; + return buffer_get_space_unsafe(t->appends, pos, + t->view->index->record_size); +} + void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq) { struct mail_transaction_expunge exp, *data; @@ -288,15 +300,10 @@ void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq, enum mail_flags flags, keywords_mask_t keywords) { struct mail_index_record *rec; - size_t pos; if (t->first_new_seq != 0 && seq >= t->first_new_seq) { /* just appended message, modify it directly */ - i_assert(seq > 0 && seq <= t->last_new_seq); - - pos = (seq - t->first_new_seq) * t->view->index->record_size; - rec = buffer_get_space_unsafe(t->appends, pos, - t->view->index->record_size); + rec = mail_index_lookup_append_rec(t, seq); mail_index_record_modify_flags(rec, modify_type, flags, keywords); return; @@ -467,33 +474,15 @@ static void mail_index_update_seq_buffer(buffer_t **buffer, uint32_t seq, memcpy(seq_p+1, record, record_size); } -static void mail_index_update_record(struct mail_index_transaction *t, - uint32_t seq, size_t offset, - const void *record, size_t record_size) -{ - struct mail_index *index = t->view->index; - struct mail_index_record *rec; - size_t pos; - - i_assert(seq > 0 && seq <= t->last_new_seq); - - pos = (seq - t->first_new_seq) * index->record_size; - rec = buffer_get_space_unsafe(t->appends, pos, - index->record_size); - - memcpy(PTR_OFFSET(rec, offset), record, record_size); -} - void mail_index_update_cache(struct mail_index_transaction *t, uint32_t seq, uint32_t offset) { + struct mail_index_record *rec; + if (t->first_new_seq != 0 && seq >= t->first_new_seq) { /* just appended message, modify it directly */ - size_t rec_offset; - - rec_offset = offsetof(struct mail_index_record, cache_offset); - mail_index_update_record(t, seq, rec_offset, - &offset, sizeof(offset)); + rec = mail_index_lookup_append_rec(t, seq); + rec->cache_offset = offset; } else { mail_index_update_seq_buffer(&t->cache_updates, seq, &offset, sizeof(offset)); @@ -505,15 +494,16 @@ void mail_index_update_extra_rec(struct mail_index_transaction *t, const void *data) { struct mail_index *index = t->view->index; + struct mail_index_record *rec; i_assert(data_id < index->extra_records_count); if (t->first_new_seq != 0 && seq >= t->first_new_seq) { /* just appended message, modify it directly */ /* FIXME: do data_id mapping conversion */ - mail_index_update_record(t, seq, - index->extra_records[data_id].offset, data, - index->extra_records[data_id].size); + rec = mail_index_lookup_append_rec(t, seq); + memcpy(PTR_OFFSET(rec, index->extra_records[data_id].offset), + data, index->extra_records[data_id].size); } else { mail_index_update_seq_buffer(&t->extra_rec_updates[data_id], seq, data, index->extra_records[data_id].size); @@ -532,3 +522,9 @@ void mail_index_update_header(struct mail_index_transaction *t, for (; size > 0; size--) t->hdr_mask[offset++] = 1; } + +const struct mail_index_record * +mail_index_lookup_append(struct mail_index_transaction *t, uint32_t seq) +{ + return mail_index_lookup_append_rec(t, seq); +} diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h index 0783dea8ea..6df0b31290 100644 --- a/src/lib-index/mail-index.h +++ b/src/lib-index/mail-index.h @@ -277,6 +277,10 @@ void mail_index_update_header(struct mail_index_transaction *t, void mail_index_update_extra_rec(struct mail_index_transaction *t, uint32_t seq, uint32_t data_id, const void *data); +/* Returns given appended message, with all updates that have been done + to it since the append. */ +const struct mail_index_record * +mail_index_lookup_append(struct mail_index_transaction *t, uint32_t seq); /* Returns the last error code. */ enum mail_index_error mail_index_get_last_error(struct mail_index *index); diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c index 89adc97fe6..4a7ed28d14 100644 --- a/src/lib-storage/index/index-mail.c +++ b/src/lib-storage/index/index-mail.c @@ -546,6 +546,8 @@ int index_mail_next(struct index_mail *mail, cache_flags = (data->cached_fields & MAIL_CACHE_INDEX_FLAGS) == 0 ? 0 : mail_cache_get_record_flags(mail->ibox->cache_view, seq); + mail->mail.seq = seq; + mail->mail.uid = rec->uid; mail->mail.has_nuls = (cache_flags & MAIL_INDEX_FLAG_HAS_NULS) != 0; mail->mail.has_no_nuls = (cache_flags & MAIL_INDEX_FLAG_HAS_NO_NULS) != 0; diff --git a/src/lib-storage/index/index-search.c b/src/lib-storage/index/index-search.c index 946d34c364..8c938f12d5 100644 --- a/src/lib-storage/index/index-search.c +++ b/src/lib-storage/index/index-search.c @@ -821,11 +821,7 @@ struct mail *index_storage_search_next(struct mail_search_context *_ctx) return NULL; } - ctx->imail.data.rec = rec; - ctx->mail->seq = ctx->seq1++; - ctx->mail->uid = rec == NULL ? 0 : rec->uid; - - ret = index_mail_next(&ctx->imail, rec, ctx->mail->seq, TRUE); + ret = index_mail_next(&ctx->imail, rec, ctx->seq1++, TRUE); if (ret < 0) break; diff --git a/src/lib-storage/index/maildir/maildir-copy.c b/src/lib-storage/index/maildir/maildir-copy.c index 01c19646c5..e669fe324a 100644 --- a/src/lib-storage/index/maildir/maildir-copy.c +++ b/src/lib-storage/index/maildir/maildir-copy.c @@ -120,7 +120,8 @@ void maildir_copy_rollback(struct maildir_copy_context *ctx) pool_unref(ctx->pool); } -int maildir_copy(struct mailbox_transaction_context *_t, struct mail *mail) +int maildir_copy(struct mailbox_transaction_context *_t, struct mail *mail, + struct mail **dest_mail_r) { struct maildir_transaction_context *t = (struct maildir_transaction_context *)_t; @@ -132,6 +133,7 @@ int maildir_copy(struct mailbox_transaction_context *_t, struct mail *mail) ctx = t->copy_ctx; if (ctx->hardlink && mail->box->storage == ctx->ibox->box.storage) { + // FIXME: handle dest_mail_r t_push(); ret = maildir_copy_hardlink(mail, ctx); t_pop(); @@ -144,5 +146,5 @@ int maildir_copy(struct mailbox_transaction_context *_t, struct mail *mail) /* non-fatal hardlinking failure, try the slow way */ } - return mail_storage_copy(_t, mail); + return mail_storage_copy(_t, mail, dest_mail_r); } diff --git a/src/lib-storage/index/maildir/maildir-save.c b/src/lib-storage/index/maildir/maildir-save.c index 0d8502e0d4..7fa7fe35b2 100644 --- a/src/lib-storage/index/maildir/maildir-save.c +++ b/src/lib-storage/index/maildir/maildir-save.c @@ -33,7 +33,7 @@ maildir_read_into_tmp(struct index_mailbox *ibox, const char *dir, { const char *path, *fname; struct ostream *output; - int fd; + int fd, crlf; fd = maildir_create_tmp(ibox, dir, ibox->mail_create_mode, &path); if (fd == -1) @@ -46,8 +46,9 @@ maildir_read_into_tmp(struct index_mailbox *ibox, const char *dir, output = o_stream_create_file(fd, pool_datastack_create(), 4096, FALSE); o_stream_set_blocking(output, 60000, NULL, NULL); + crlf = getenv("MAIL_SAVE_CRLF") != NULL; if (mail_storage_save(ibox->box.storage, path, input, output, - getenv("MAIL_SAVE_CRLF") != NULL, NULL, NULL) < 0) + crlf, crlf, NULL, NULL) < 0) fname = NULL; o_stream_unref(output); @@ -121,7 +122,7 @@ int maildir_save(struct mailbox_transaction_context *_t, const struct mail_full_flags *flags, time_t received_date, int timezone_offset __attr_unused__, const char *from_envelope __attr_unused__, - struct istream *data) + struct istream *data, struct mail **mail_r) { struct maildir_transaction_context *t = (struct maildir_transaction_context *)_t; diff --git a/src/lib-storage/index/maildir/maildir-storage.h b/src/lib-storage/index/maildir/maildir-storage.h index 7502f45859..53f53b7c32 100644 --- a/src/lib-storage/index/maildir/maildir-storage.h +++ b/src/lib-storage/index/maildir/maildir-storage.h @@ -49,11 +49,13 @@ void maildir_transaction_rollback(struct mailbox_transaction_context *t); int maildir_save(struct mailbox_transaction_context *t, const struct mail_full_flags *flags, time_t received_date, int timezone_offset, - const char *from_envelope, struct istream *data); + const char *from_envelope, struct istream *data, + struct mail **mail_r); int maildir_save_commit(struct maildir_save_context *ctx); void maildir_save_rollback(struct maildir_save_context *ctx); -int maildir_copy(struct mailbox_transaction_context *t, struct mail *mail); +int maildir_copy(struct mailbox_transaction_context *t, struct mail *mail, + struct mail **dest_mail_r); int maildir_copy_commit(struct maildir_copy_context *ctx); void maildir_copy_rollback(struct maildir_copy_context *ctx); diff --git a/src/lib-storage/index/mbox/mbox-lock.c b/src/lib-storage/index/mbox/mbox-lock.c index 515406c6e4..cc75d67170 100644 --- a/src/lib-storage/index/mbox/mbox-lock.c +++ b/src/lib-storage/index/mbox/mbox-lock.c @@ -331,6 +331,7 @@ int mbox_lock(struct index_mailbox *ibox, int lock_type, if (ibox->mbox_lock_type == lock_type) { ibox->mbox_locks++; + *lock_id_r = ibox->mbox_lock_id; return 1; } diff --git a/src/lib-storage/index/mbox/mbox-mail.c b/src/lib-storage/index/mbox/mbox-mail.c index 0d1180dfe6..c5f424c46e 100644 --- a/src/lib-storage/index/mbox/mbox-mail.c +++ b/src/lib-storage/index/mbox/mbox-mail.c @@ -19,7 +19,7 @@ static int mbox_mail_seek(struct index_mail *mail) uint64_t offset; if (ibox->mbox_lock_type == F_UNLCK) { - if (mbox_sync(ibox, FALSE, TRUE) < 0) + if (mbox_sync(ibox, FALSE, FALSE, TRUE) < 0) return -1; i_assert(ibox->mbox_lock_type != F_UNLCK); @@ -74,12 +74,15 @@ static time_t mbox_mail_get_received_date(struct mail *_mail) return (time_t)-1; data->received_date = istream_raw_mbox_get_received_time(mail->ibox->mbox_stream); - - if (data->received_date != (time_t)-1) { - index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, - &data->received_date, - sizeof(data->received_date)); + if (data->received_date == (time_t)-1) { + /* it's broken and conflicts with our "not found" + return value. change it. */ + data->received_date = 0; } + + index_mail_cache_add(mail, MAIL_CACHE_RECEIVED_DATE, + &data->received_date, + sizeof(data->received_date)); return data->received_date; } diff --git a/src/lib-storage/index/mbox/mbox-save.c b/src/lib-storage/index/mbox/mbox-save.c index 5e26c69776..a3b136ae30 100644 --- a/src/lib-storage/index/mbox/mbox-save.c +++ b/src/lib-storage/index/mbox/mbox-save.c @@ -1,6 +1,7 @@ /* Copyright (C) 2002 Timo Sirainen */ #include "lib.h" +#include "ioloop.h" #include "hostpid.h" #include "ostream.h" #include "str.h" @@ -9,8 +10,10 @@ #include "mbox-file.h" #include "mbox-from.h" #include "mbox-lock.h" +#include "mbox-sync-private.h" #include "mail-save.h" +#include #include #include #include @@ -19,12 +22,20 @@ struct mbox_save_context { struct index_mailbox *ibox; + struct mail_index_transaction *trans; uoff_t append_offset; + string_t *headers; + size_t space_end_idx; + uint32_t next_uid; + struct ostream *output; - uoff_t sync_offset, content_length_offset, eoh_offset; + uoff_t extra_hdr_offset, eoh_offset; + + struct index_mail mail; const struct mail_full_flags *flags; + unsigned int synced:1; }; static char my_hostdomain[256] = ""; @@ -111,89 +122,50 @@ static int write_from_line(struct mbox_save_context *ctx, time_t received_date, return 0; } -static const char *get_system_flags(enum mail_flags flags) +static int mbox_write_content_length(struct mbox_save_context *ctx) { - string_t *str; - - if (flags == 0) - return ""; + uoff_t end_offset; + const char *str; + size_t len; + int ret = 0; - str = t_str_new(32); - if (flags & MAIL_SEEN) - str_append(str, "Status: R\n"); + end_offset = ctx->output->offset; - if (flags & (MAIL_ANSWERED|MAIL_DRAFT|MAIL_FLAGGED|MAIL_DELETED)) { - str_append(str, "X-Status: "); + /* write Content-Length headers */ + t_push(); + str = t_strdup_printf("\nContent-Length: %s", + dec2str(end_offset - (ctx->eoh_offset + 1))); + len = strlen(str); - if ((flags & MAIL_ANSWERED) != 0) - str_append_c(str, 'A'); - if ((flags & MAIL_DELETED) != 0) - str_append_c(str, 'D'); - if ((flags & MAIL_FLAGGED) != 0) - str_append_c(str, 'F'); - if ((flags & MAIL_DRAFT) != 0) - str_append_c(str, 'T'); - str_append_c(str, '\n'); + if (o_stream_seek(ctx->output, ctx->extra_hdr_offset + + ctx->space_end_idx - len) < 0) { + mbox_set_syscall_error(ctx->ibox, "o_stream_seek()"); + ret = -1; + } else if (o_stream_send(ctx->output, str, len) < 0) { + write_error(ctx); + ret = -1; + } else { + if (o_stream_seek(ctx->output, end_offset) < 0) { + mbox_set_syscall_error(ctx->ibox, "o_stream_seek()"); + ret = -1; + } } - return str_c(str); -} - -static const char *get_keywords(const struct mail_full_flags *flags) -{ - string_t *str; - unsigned int i; - - if (flags->keywords_count == 0) - return ""; - - str = t_str_new(256); - for (i = 0; i < flags->keywords_count; i++) { - if (str_len(str) > 0) - str_append_c(str, ' '); - str_append(str, flags->keywords[i]); - } - return str_c(str); + t_pop(); + return ret; } static int save_header_callback(const char *name, write_func_t *write_func, void *context) { - static const char *content_length = "Content-Length: "; struct mbox_save_context *ctx = context; - const char *str; - char *buf; - size_t space; if (name == NULL) { - /* write system flags */ - str = get_system_flags(ctx->flags->flags); - if (write_func(ctx->output, str, strlen(str)) < 0) - return -1; - - /* write beginning of content-length header */ - if (write_func(ctx->output, content_length, - strlen(content_length)) < 0) { - write_error(ctx); + /* write our extra headers */ + ctx->extra_hdr_offset = ctx->output->offset; + if (write_func(ctx->output, str_data(ctx->headers), + str_len(ctx->headers)) < 0) return -1; - } - ctx->content_length_offset = ctx->output->offset; - - /* calculate how much space keywords and content-length - value needs, then write that amount of spaces. */ - space = strlen(get_keywords(ctx->flags)); - space += sizeof("X-Keywords: "); - space += MBOX_HEADER_EXTRA_SPACE + MAX_INT_STRLEN + 1; - - /* @UNSAFE */ - buf = t_malloc(space); - memset(buf, ' ', space-1); - buf[space-1] = '\n'; - - if (write_func(ctx->output, buf, space) < 0) { - write_error(ctx); - return -1; - } ctx->eoh_offset = ctx->output->offset; return 1; } @@ -217,6 +189,8 @@ static int save_header_callback(const char *name, write_func_t *write_func, return 0; if (strcasecmp(name, "X-Keywords") == 0) return 0; + if (strcasecmp(name, "X-IMAP") == 0) + return 0; if (strcasecmp(name, "X-IMAPbase") == 0) return 0; break; @@ -225,51 +199,95 @@ static int save_header_callback(const char *name, write_func_t *write_func, return 1; } -static int mbox_fix_header(struct mbox_save_context *ctx) +static int mbox_save_init_sync(struct mbox_transaction_context *t) { - uoff_t old_offset; - const char *str; - int crlf = getenv("MAIL_SAVE_CRLF") != NULL; + struct mbox_save_context *ctx = t->save_ctx; + const struct mail_index_header *hdr; - old_offset = ctx->output->offset; - if (o_stream_seek(ctx->output, ctx->content_length_offset) < 0) - return mbox_set_syscall_error(ctx->ibox, "o_stream_seek()"); + if (mail_index_get_header(ctx->ibox->view, &hdr) < 0) { + mail_storage_set_index_error(ctx->ibox); + return -1; + } + ctx->next_uid = hdr->next_uid; + ctx->synced = TRUE; + t->mbox_modified = TRUE; - /* write value for Content-Length */ - str = dec2str(old_offset - (ctx->eoh_offset + 1 + crlf)); - if (o_stream_send_str(ctx->output, str) < 0) - return write_error(ctx); + index_mail_init(&t->ictx, &ctx->mail, 0, NULL); + return 0; +} - /* [CR]LF X-Keywords: */ - str = crlf ? "\r\nX-Keywords:" : "\nX-Keywords:"; - if (o_stream_send_str(ctx->output, str) < 0) - return write_error(ctx); +static void status_flags_append(string_t *str, enum mail_flags flags, + const struct mbox_flag_type *flags_list) +{ + int i; - /* write keywords into X-Keywords */ - str = get_keywords(ctx->flags); - if (o_stream_send_str(ctx->output, str) < 0) - return write_error(ctx); + for (i = 0; flags_list[i].chr != 0; i++) { + if ((flags & flags_list[i].flag) != 0) + str_append_c(str, flags_list[i].chr); + } +} - if (o_stream_seek(ctx->output, old_offset) < 0) - return mbox_set_syscall_error(ctx->ibox, "o_stream_seek()"); - return 0; +static void mbox_save_append_flag_headers(string_t *str, enum mail_flags flags) +{ + if ((flags & STATUS_FLAGS_MASK) != 0) { + str_append(str, "Status: "); + status_flags_append(str, flags, mbox_status_flags); + str_append_c(str, '\n'); + } + + if ((flags & XSTATUS_FLAGS_MASK) != 0) { + str_append(str, "X-Status: "); + status_flags_append(str, flags, mbox_xstatus_flags); + str_append_c(str, '\n'); + } +} + +static void mbox_save_append_keyword_headers(struct mbox_save_context *ctx, + const char *const *keywords, + unsigned int count) +{ + unsigned char space[MBOX_HEADER_EXTRA_SPACE+1 + + sizeof("Content-Length: \n")-1 + MAX_INT_STRLEN]; + unsigned int i; + + str_append(ctx->headers, "X-Keywords:"); + for (i = 0; i < count; i++) { + str_append_c(ctx->headers, ' '); + str_append(ctx->headers, keywords[i]); + } + + memset(space, ' ', sizeof(space)); + str_append_n(ctx->headers, space, sizeof(space)); + ctx->space_end_idx = str_len(ctx->headers); + str_append_c(ctx->headers, '\n'); } int mbox_save(struct mailbox_transaction_context *_t, const struct mail_full_flags *flags, time_t received_date, int timezone_offset __attr_unused__, - const char *from_envelope, struct istream *data) + const char *from_envelope, struct istream *data, + struct mail **mail_r) { struct mbox_transaction_context *t = (struct mbox_transaction_context *)_t; struct index_mailbox *ibox = t->ictx.ibox; struct mbox_save_context *ctx = t->save_ctx; + enum mail_flags save_flags; + keywords_mask_t keywords; + uint64_t offset; + uint32_t seq = 0; int ret; + /* FIXME: we could write timezone_offset to From-line.. */ + if (received_date == (time_t)-1) + received_date = ioloop_time; + if (ctx == NULL) { ctx = t->save_ctx = i_new(struct mbox_save_context, 1); ctx->ibox = ibox; + ctx->trans = t->ictx.trans; ctx->append_offset = (uoff_t)-1; + ctx->headers = str_new(default_pool, 512); } ctx->flags = flags; @@ -284,6 +302,17 @@ int mbox_save(struct mailbox_transaction_context *_t, return -1; } + if (mail_r == NULL) { + /* assign UIDs only if mbox doesn't require syncing */ + ret = mbox_sync_has_changed(ibox); + if (ret < 0) + return -1; + if (ret == 0) { + if (mbox_save_init_sync(t) < 0) + return -1; + } + } + if (mbox_seek_to_end(ctx, &ctx->append_offset) < 0) return -1; @@ -292,27 +321,72 @@ int mbox_save(struct mailbox_transaction_context *_t, o_stream_set_blocking(ctx->output, 60000, NULL, NULL); } + if (!ctx->synced && mail_r != NULL) { + /* we'll need to assign UID for the mail immediately. */ + if (mbox_sync(ibox, FALSE, FALSE, FALSE) < 0) + return -1; + if (mbox_save_init_sync(t) < 0) + return -1; + } + + save_flags = (flags->flags & ~MAIL_RECENT) | MAIL_RECENT; + str_truncate(ctx->headers, 0); + if (ctx->synced) { + str_printfa(ctx->headers, "X-UID: %u\n", ctx->next_uid); + if (!ibox->keep_recent) + save_flags &= ~MAIL_RECENT; + + memset(keywords, 0, INDEX_KEYWORDS_BYTE_COUNT); + // FIXME: set keywords + mail_index_append(ctx->trans, ctx->next_uid, &seq); + mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, + save_flags, keywords); + + offset = ctx->output->offset - 1; + mail_index_update_extra_rec(ctx->trans, seq, + ibox->mbox_extra_idx, &offset); + ctx->next_uid++; + } + mbox_save_append_flag_headers(ctx->headers, + save_flags ^ MBOX_NONRECENT); + mbox_save_append_keyword_headers(ctx, flags->keywords, + flags->keywords_count); + i_assert(ibox->mbox_lock_type == F_WRLCK); t_push(); if (write_from_line(ctx, received_date, from_envelope) < 0 || mail_storage_save(ibox->box.storage, ibox->path, data, ctx->output, - getenv("MAIL_SAVE_CRLF") != NULL, + FALSE, getenv("MAIL_SAVE_CRLF") != NULL, save_header_callback, ctx) < 0 || - mbox_fix_header(ctx) < 0 || + mbox_write_content_length(ctx) < 0 || mbox_append_lf(ctx) < 0) { ret = -1; } else { ret = 0; } t_pop(); + + if (mail_r != NULL) { + const struct mail_index_record *rec; + + rec = mail_index_lookup_append(ctx->trans, seq); + if (index_mail_next(&ctx->mail, rec, seq, FALSE) <= 0) + return -1; + *mail_r = &ctx->mail.mail; + } + return ret; } static void mbox_save_deinit(struct mbox_save_context *ctx) { + if (ctx->mail.pool != NULL) + index_mail_deinit(&ctx->mail); + if (ctx->output != NULL) o_stream_unref(ctx->output); + str_free(ctx->headers); i_free(ctx); } @@ -320,7 +394,13 @@ int mbox_save_commit(struct mbox_save_context *ctx) { int ret = 0; - if (ctx->ibox->mbox_fd != -1) { + if (ctx->synced) { + mail_index_update_header(ctx->trans, + offsetof(struct mail_index_header, next_uid), + &ctx->next_uid, sizeof(ctx->next_uid)); + } + + if (!ctx->synced && ctx->ibox->mbox_fd != -1) { if (fdatasync(ctx->ibox->mbox_fd) < 0) { mbox_set_syscall_error(ctx->ibox, "fsync()"); ret = -1; diff --git a/src/lib-storage/index/mbox/mbox-storage.h b/src/lib-storage/index/mbox/mbox-storage.h index 73e5e81636..a5cddcca52 100644 --- a/src/lib-storage/index/mbox/mbox-storage.h +++ b/src/lib-storage/index/mbox/mbox-storage.h @@ -14,6 +14,7 @@ struct mbox_transaction_context { struct mbox_save_context *save_ctx; unsigned int mbox_lock_id; + unsigned int mbox_modified:1; }; extern struct mail mbox_mail; @@ -36,7 +37,8 @@ int mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags); int mbox_save(struct mailbox_transaction_context *t, const struct mail_full_flags *flags, time_t received_date, int timezone_offset, - const char *from_envelope, struct istream *data); + const char *from_envelope, struct istream *data, + struct mail **mail_r); int mbox_save_commit(struct mbox_save_context *ctx); void mbox_save_rollback(struct mbox_save_context *ctx); diff --git a/src/lib-storage/index/mbox/mbox-sync-parse.c b/src/lib-storage/index/mbox/mbox-sync-parse.c index 67c6b42c24..779ec2b17d 100644 --- a/src/lib-storage/index/mbox/mbox-sync-parse.c +++ b/src/lib-storage/index/mbox/mbox-sync-parse.c @@ -131,10 +131,16 @@ static int parse_x_imap_base(struct mbox_sync_mail_context *ctx, if (ctx->sync_ctx->base_uid_validity == 0) { ctx->sync_ctx->base_uid_validity = uid_validity; ctx->sync_ctx->base_uid_last = uid_last; - ctx->sync_ctx->next_uid = uid_last+1; + if (ctx->sync_ctx->next_uid-1 <= uid_last) + ctx->sync_ctx->next_uid = uid_last+1; + else { + ctx->sync_ctx->update_base_uid_last = + ctx->sync_ctx->next_uid - 1; + ctx->need_rewrite = TRUE; + } } - if (ctx->sync_ctx->prev_msg_uid >= ctx->sync_ctx->next_uid) { + if (ctx->sync_ctx->next_uid <= ctx->sync_ctx->prev_msg_uid) { /* broken, update */ ctx->sync_ctx->next_uid = ctx->sync_ctx->prev_msg_uid+1; } diff --git a/src/lib-storage/index/mbox/mbox-sync-private.h b/src/lib-storage/index/mbox/mbox-sync-private.h index 84b242e195..1728efb1f5 100644 --- a/src/lib-storage/index/mbox/mbox-sync-private.h +++ b/src/lib-storage/index/mbox/mbox-sync-private.h @@ -97,7 +97,10 @@ struct mbox_sync_context { unsigned int seen_first_mail:1; }; -int mbox_sync(struct index_mailbox *ibox, int last_commit, int lock); +int mbox_sync(struct index_mailbox *ibox, int last_commit, + int sync_header, int lock); +int mbox_sync_has_changed(struct index_mailbox *ibox); + void mbox_sync_parse_next_mail(struct istream *input, struct mbox_sync_mail_context *ctx, int rewriting); diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c index 4afe068964..2b27d87759 100644 --- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c @@ -262,6 +262,13 @@ int mbox_sync_try_rewrite(struct mbox_sync_mail_context *ctx, off_t move_diff) mbox_set_syscall_error(ctx->sync_ctx->ibox, "pwrite_full()"); return -1; } + + if (ctx->sync_ctx->dest_first_mail) { + ctx->sync_ctx->base_uid_last = + ctx->sync_ctx->update_base_uid_last; + ctx->sync_ctx->update_base_uid_last = 0; + } + istream_raw_mbox_flush(ctx->sync_ctx->input); return 1; } @@ -292,7 +299,7 @@ static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx, so we have to fool it. */ old_prev_msg_uid = sync_ctx->prev_msg_uid; sync_ctx->prev_msg_uid = mails[idx].uid-1; - sync_ctx->dest_first_mail = seq == 1; + sync_ctx->dest_first_mail = mails[idx].from_offset == 0; mbox_sync_parse_next_mail(sync_ctx->input, &mail_ctx, TRUE); if (mails[idx].space != 0) @@ -337,6 +344,12 @@ static int mbox_sync_read_and_move(struct mbox_sync_context *sync_ctx, return -1; } + if (mails[idx].from_offset == 0) { + sync_ctx->base_uid_last = + sync_ctx->update_base_uid_last; + sync_ctx->update_base_uid_last = 0; + } + return 0; } diff --git a/src/lib-storage/index/mbox/mbox-sync-update.c b/src/lib-storage/index/mbox/mbox-sync-update.c index 139c87db57..89a569a204 100644 --- a/src/lib-storage/index/mbox/mbox-sync-update.c +++ b/src/lib-storage/index/mbox/mbox-sync-update.c @@ -3,6 +3,7 @@ #include "buffer.h" #include "str.h" #include "message-parser.h" +#include "index-storage.h" #include "mbox-sync-private.h" static void status_flags_append(struct mbox_sync_mail_context *ctx, @@ -140,7 +141,8 @@ static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx) if (ctx->hdr_pos[MBOX_HDR_STATUS] == (size_t)-1 && (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) { - ctx->mail.flags |= MBOX_NONRECENT; + if (!ctx->sync_ctx->ibox->keep_recent) + ctx->mail.flags |= MBOX_NONRECENT; ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header); str_append(ctx->header, "Status: "); status_flags_append(ctx, mbox_status_flags); @@ -278,7 +280,8 @@ void mbox_sync_update_header(struct mbox_sync_mail_context *ctx, INDEX_KEYWORDS_BYTE_COUNT) != 0) mbox_sync_update_xkeywords(ctx); } else { - if ((ctx->mail.flags & MBOX_NONRECENT) == 0) { + if ((ctx->mail.flags & MBOX_NONRECENT) == 0 && + !ctx->sync_ctx->ibox->keep_recent) { ctx->mail.flags |= MBOX_NONRECENT; mbox_sync_update_status(ctx); } @@ -296,7 +299,9 @@ void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx, (mail->flags & STATUS_FLAGS_MASK) || (ctx->mail.flags & MBOX_NONRECENT) == 0) { ctx->mail.flags = (ctx->mail.flags & ~STATUS_FLAGS_MASK) | - (mail->flags & STATUS_FLAGS_MASK) | MBOX_NONRECENT; + (mail->flags & STATUS_FLAGS_MASK); + if (!ctx->sync_ctx->ibox->keep_recent) + ctx->mail.flags |= MBOX_NONRECENT; mbox_sync_update_status(ctx); } if ((ctx->mail.flags & XSTATUS_FLAGS_MASK) != diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c index 670e016443..0d4872da39 100644 --- a/src/lib-storage/index/mbox/mbox-sync.c +++ b/src/lib-storage/index/mbox/mbox-sync.c @@ -258,7 +258,8 @@ static int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx, mbox_sync_buffer_delete_old(sync_ctx->syncs, uid); while (uid >= sync_rec->uid1) { - if (sync_rec->uid1 != 0) { + if (sync_rec->uid1 != 0 && + sync_rec->type != MAIL_INDEX_SYNC_TYPE_APPEND) { i_assert(uid <= sync_rec->uid2); buffer_append(sync_ctx->syncs, sync_rec, sizeof(*sync_rec)); @@ -277,6 +278,14 @@ static int mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx, memset(sync_rec, 0, sizeof(*sync_rec)); break; } + + if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND) { + if (sync_rec->uid2 >= sync_ctx->next_uid) { + sync_ctx->next_uid = sync_rec->uid2 + 1; + sync_ctx->update_base_uid_last = sync_rec->uid2; + } + memset(sync_rec, 0, sizeof(*sync_rec)); + } } if (!*sync_expunge_r) @@ -555,7 +564,9 @@ static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx) return -1; } } else if (mail_ctx->need_rewrite || - buffer_get_used_size(sync_ctx->syncs) != 0) { + buffer_get_used_size(sync_ctx->syncs) != 0 || + (mail_ctx->seq == 1 && + sync_ctx->update_base_uid_last != 0)) { if ((ret = mbox_sync_check_excl_lock(sync_ctx)) < 0) return ret; @@ -923,22 +934,26 @@ static void mbox_sync_restart(struct mbox_sync_context *sync_ctx) sync_ctx->seen_first_mail = FALSE; } -static int mbox_sync_do(struct mbox_sync_context *sync_ctx) +static int mbox_sync_do(struct mbox_sync_context *sync_ctx, int sync_header) { struct mbox_sync_mail_context mail_ctx; struct stat st; uint32_t min_msg_count; int ret; - if (fstat(sync_ctx->fd, &st) < 0) { - mbox_set_syscall_error(sync_ctx->ibox, "stat()"); - return -1; - } + if (sync_header) + min_msg_count = 1; + else { + if (fstat(sync_ctx->fd, &st) < 0) { + mbox_set_syscall_error(sync_ctx->ibox, "stat()"); + return -1; + } - min_msg_count = - (uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp && - (uint64_t)st.st_size == sync_ctx->hdr->sync_size ? - 0 : (uint32_t)-1; + min_msg_count = + (uint32_t)st.st_mtime == sync_ctx->hdr->sync_stamp && + (uint64_t)st.st_size == sync_ctx->hdr->sync_size ? + 0 : (uint32_t)-1; + } mbox_sync_restart(sync_ctx); if ((ret = mbox_sync_loop(sync_ctx, &mail_ctx, min_msg_count)) == -1) @@ -975,7 +990,7 @@ static int mbox_sync_do(struct mbox_sync_context *sync_ctx) return 0; } -static int mbox_sync_has_changed(struct index_mailbox *ibox) +int mbox_sync_has_changed(struct index_mailbox *ibox) { const struct mail_index_header *hdr; struct stat st; @@ -1020,7 +1035,8 @@ static int mbox_sync_update_imap_base(struct mbox_sync_context *sync_ctx) return 0; } -int mbox_sync(struct index_mailbox *ibox, int last_commit, int lock) +int mbox_sync(struct index_mailbox *ibox, int last_commit, + int sync_header, int lock) { struct mail_index_sync_ctx *index_sync_ctx; struct mail_index_view *sync_view; @@ -1035,11 +1051,16 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit, int lock) return -1; } - if ((ret = mbox_sync_has_changed(ibox)) < 0) { - if (lock) - (void)mbox_unlock(ibox, lock_id); - return -1; + if (sync_header) + ret = 0; + else { + if ((ret = mbox_sync_has_changed(ibox)) < 0) { + if (lock) + (void)mbox_unlock(ibox, lock_id); + return -1; + } } + if (ret == 0 && !last_commit) return 0; @@ -1084,7 +1105,7 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit, int lock) if (mbox_sync_lock(&sync_ctx, lock_type) < 0) ret = -1; - else if (mbox_sync_do(&sync_ctx) < 0) + else if (mbox_sync_do(&sync_ctx, sync_header) < 0) ret = -1; if (ret < 0) @@ -1103,8 +1124,9 @@ int mbox_sync(struct index_mailbox *ibox, int last_commit, int lock) ret = -1; } - if (sync_ctx.base_uid_last != sync_ctx.next_uid-1 && ret == 0 && - !ibox->mbox_readonly) { + if (sync_ctx.seen_first_mail && + sync_ctx.base_uid_last != sync_ctx.next_uid-1 && + ret == 0 && !ibox->mbox_readonly) { /* rewrite X-IMAPbase header. do it after mail_index_sync_end() so previous transactions have been committed. */ /* FIXME: ugly .. */ @@ -1155,7 +1177,7 @@ int mbox_storage_sync(struct mailbox *box, enum mailbox_sync_flags flags) ibox->sync_last_check + MAILBOX_FULL_SYNC_INTERVAL <= ioloop_time) { ibox->sync_last_check = ioloop_time; - if (mbox_sync(ibox, FALSE, FALSE) < 0) + if (mbox_sync(ibox, FALSE, FALSE, FALSE) < 0) return -1; } diff --git a/src/lib-storage/index/mbox/mbox-transaction.c b/src/lib-storage/index/mbox/mbox-transaction.c index 9fdb2a0bca..eaff99995a 100644 --- a/src/lib-storage/index/mbox/mbox-transaction.c +++ b/src/lib-storage/index/mbox/mbox-transaction.c @@ -37,7 +37,7 @@ int mbox_transaction_commit(struct mailbox_transaction_context *_t) } if (ret == 0) { - if (mbox_sync(ibox, TRUE, FALSE) < 0) + if (mbox_sync(ibox, TRUE, t->mbox_modified, FALSE) < 0) ret = -1; } diff --git a/src/lib-storage/mail-save.c b/src/lib-storage/mail-save.c index e8ae86090d..69e85f65a6 100644 --- a/src/lib-storage/mail-save.c +++ b/src/lib-storage/mail-save.c @@ -130,7 +130,8 @@ static int save_headers(struct istream *input, struct ostream *output, } int mail_storage_save(struct mail_storage *storage, const char *path, - struct istream *input, struct ostream *output, int crlf, + struct istream *input, struct ostream *output, + int crlf_hdr, int crlf_body, header_callback_t *header_callback, void *context) { write_func_t *write_func; @@ -139,14 +140,15 @@ int mail_storage_save(struct mail_storage *storage, const char *path, ssize_t ret; int failed; - write_func = crlf ? write_with_crlf : write_with_lf; - if (header_callback != NULL) { + write_func = crlf_hdr ? write_with_crlf : write_with_lf; if (save_headers(input, output, header_callback, context, write_func) < 0) return -1; } + write_func = crlf_body ? write_with_crlf : write_with_lf; + failed = FALSE; for (;;) { data = i_stream_get_data(input, &size); @@ -188,7 +190,8 @@ int mail_storage_save(struct mail_storage *storage, const char *path, return failed ? -1 : 0; } -int mail_storage_copy(struct mailbox_transaction_context *t, struct mail *mail) +int mail_storage_copy(struct mailbox_transaction_context *t, struct mail *mail, + struct mail **dest_mail_r) { struct istream *input; @@ -199,5 +202,5 @@ int mail_storage_copy(struct mailbox_transaction_context *t, struct mail *mail) return mailbox_save(t, mail->get_flags(mail), mail->get_received_date(mail), 0, mail->get_special(mail, MAIL_FETCH_FROM_ENVELOPE), - input); + input, dest_mail_r); } diff --git a/src/lib-storage/mail-save.h b/src/lib-storage/mail-save.h index ddfcce00d1..b971d5fb52 100644 --- a/src/lib-storage/mail-save.h +++ b/src/lib-storage/mail-save.h @@ -8,9 +8,11 @@ typedef int header_callback_t(const char *name, write_func_t *write_func, void *context); int mail_storage_save(struct mail_storage *storage, const char *path, - struct istream *input, struct ostream *output, int crlf, + struct istream *input, struct ostream *output, + int crlf_hdr, int crlf_body, header_callback_t *header_callback, void *context); -int mail_storage_copy(struct mailbox_transaction_context *t, struct mail *mail); +int mail_storage_copy(struct mailbox_transaction_context *t, struct mail *mail, + struct mail **dest_mail_r); #endif diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h index 64e9ef53be..8f270085ea 100644 --- a/src/lib-storage/mail-storage-private.h +++ b/src/lib-storage/mail-storage-private.h @@ -96,8 +96,10 @@ struct mailbox { int (*save)(struct mailbox_transaction_context *t, const struct mail_full_flags *flags, time_t received_date, int timezone_offset, - const char *from_envelope, struct istream *data); - int (*copy)(struct mailbox_transaction_context *t, struct mail *mail); + const char *from_envelope, struct istream *data, + struct mail **dest_mail_r); + int (*copy)(struct mailbox_transaction_context *t, struct mail *mail, + struct mail **dest_mail_r); int (*is_inconsistent)(struct mailbox *box); }; diff --git a/src/lib-storage/mail-storage.c b/src/lib-storage/mail-storage.c index c6dd947728..e90da3cd03 100644 --- a/src/lib-storage/mail-storage.c +++ b/src/lib-storage/mail-storage.c @@ -436,15 +436,17 @@ void mailbox_transaction_rollback(struct mailbox_transaction_context *t) int mailbox_save(struct mailbox_transaction_context *t, const struct mail_full_flags *flags, time_t received_date, int timezone_offset, - const char *from_envelope, struct istream *data) + const char *from_envelope, struct istream *data, + struct mail **mail_r) { return t->box->save(t, flags, received_date, timezone_offset, - from_envelope, data); + from_envelope, data, mail_r); } -int mailbox_copy(struct mailbox_transaction_context *t, struct mail *mail) +int mailbox_copy(struct mailbox_transaction_context *t, struct mail *mail, + struct mail **dest_mail_r) { - return t->box->copy(t, mail); + return t->box->copy(t, mail, dest_mail_r); } int mailbox_is_inconsistent(struct mailbox *box) diff --git a/src/lib-storage/mail-storage.h b/src/lib-storage/mail-storage.h index 66c018fb3f..aa7b10c59c 100644 --- a/src/lib-storage/mail-storage.h +++ b/src/lib-storage/mail-storage.h @@ -330,13 +330,21 @@ struct mail *mailbox_search_next(struct mail_search_context *ctx); /* Save a mail into mailbox. timezone_offset specifies the timezone in minutes in which received_date was originally given with. To use - current time, set received_date to (time_t)-1. */ + current time, set received_date to (time_t)-1. + + If mail_r is non-NULL, the saved message can be accessed using it. + Note that setting it non-NULL may require mailbox syncing, so don't give + give it unless you need it. */ int mailbox_save(struct mailbox_transaction_context *t, const struct mail_full_flags *flags, time_t received_date, int timezone_offset, - const char *from_envelope, struct istream *data); -/* Copy given message. */ -int mailbox_copy(struct mailbox_transaction_context *t, struct mail *mail); + const char *from_envelope, struct istream *data, + struct mail **mail_r); +/* Copy given message. If dest_mail_r is non-NULL, the copied message can be + accessed using it. Note that setting it non-NULL may require mailbox + syncing, so don't give give it unless you need it. */ +int mailbox_copy(struct mailbox_transaction_context *t, struct mail *mail, + struct mail **dest_mail_r); /* Returns TRUE if mailbox is now in inconsistent state, meaning that the message IDs etc. may have changed - only way to recover this -- 2.47.3