From: Timo Sirainen Date: Thu, 7 Aug 2008 19:33:52 +0000 (-0400) Subject: Added the concept of Global UIDs that are preserved across copies. X-Git-Tag: 1.2.alpha1~84 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b19a1420da0618a10edf67c2cfd13c8c8633057a;p=thirdparty%2Fdovecot%2Fcore.git Added the concept of Global UIDs that are preserved across copies. They can be fetched, searched and specified for the save API. dbox format supports them correctly, but with maildir the base filename is used so maildir_copy_preserve_filename=yes is required to preserve GUIDs when copying. --HG-- branch : HEAD --- diff --git a/src/imap/imap-fetch.c b/src/imap/imap-fetch.c index c3975de39f..a1d204b35d 100644 --- a/src/imap/imap-fetch.c +++ b/src/imap/imap-fetch.c @@ -22,7 +22,7 @@ #define ENVELOPE_NIL_REPLY \ "(NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL)" -#define IMAP_FETCH_HANDLER_COUNT 9 +#define IMAP_FETCH_HANDLER_COUNT 10 extern const struct imap_fetch_handler imap_fetch_default_handlers[IMAP_FETCH_HANDLER_COUNT]; static buffer_t *fetch_handlers = NULL; @@ -779,6 +779,27 @@ fetch_uid_init(struct imap_fetch_context *ctx ATTR_UNUSED, const char *name, return TRUE; } +static int fetch_guid(struct imap_fetch_context *ctx, struct mail *mail, + void *context ATTR_UNUSED) +{ + const char *value; + + if (mail_get_special(mail, MAIL_FETCH_GUID, &value) < 0) + return -1; + + str_append(ctx->cur_str, "X-GUID "); + imap_quote_append_string(ctx->cur_str, value, FALSE); + return 1; +} + +static bool +fetch_guid_init(struct imap_fetch_context *ctx ATTR_UNUSED, const char *name, + const struct imap_arg **args ATTR_UNUSED) +{ + imap_fetch_add_handler(ctx, TRUE, FALSE, name, "", fetch_guid, NULL); + return TRUE; +} + static int fetch_x_mailbox(struct imap_fetch_context *ctx, struct mail *mail, void *context ATTR_UNUSED) { @@ -811,5 +832,6 @@ imap_fetch_default_handlers[IMAP_FETCH_HANDLER_COUNT] = { { "MODSEQ", fetch_modseq_init }, { "RFC822", fetch_rfc822_init }, { "UID", fetch_uid_init }, + { "X-GUID", fetch_guid_init }, { "X-MAILBOX", fetch_x_mailbox_init } }; diff --git a/src/lib-storage/index/cydir/cydir-save.c b/src/lib-storage/index/cydir/cydir-save.c index 7deec9f25d..b74b9168b1 100644 --- a/src/lib-storage/index/cydir/cydir-save.c +++ b/src/lib-storage/index/cydir/cydir-save.c @@ -216,6 +216,7 @@ int cydir_save_finish(struct mail_save_context *_ctx) _ctx->received_date, !ctx->failed); i_stream_unref(&ctx->input); + index_save_context_free(_ctx); return ctx->failed ? -1 : 0; } diff --git a/src/lib-storage/index/dbox/dbox-file-maildir.c b/src/lib-storage/index/dbox/dbox-file-maildir.c index 5872e1bc49..da64b507e2 100644 --- a/src/lib-storage/index/dbox/dbox-file-maildir.c +++ b/src/lib-storage/index/dbox/dbox-file-maildir.c @@ -40,13 +40,18 @@ const char *dbox_file_maildir_metadata_get(struct dbox_file *file, { struct stat st; uoff_t size; - const char *value = NULL; + const char *p, *value = NULL; switch (key) { case DBOX_METADATA_FLAGS: case DBOX_METADATA_KEYWORDS: value = dbox_file_maildir_get_flags(file, key); break; + case DBOX_METADATA_GUID: + p = strchr(file->fname, MAILDIR_INFO_SEP); + value = p == NULL ? file->fname : + t_strdup_until(file->fname, p); + break; case DBOX_METADATA_RECEIVED_TIME: case DBOX_METADATA_SAVE_TIME: if (file->fd != -1) { diff --git a/src/lib-storage/index/dbox/dbox-file.h b/src/lib-storage/index/dbox/dbox-file.h index 58ac4e7c0f..50afe1ed32 100644 --- a/src/lib-storage/index/dbox/dbox-file.h +++ b/src/lib-storage/index/dbox/dbox-file.h @@ -52,6 +52,9 @@ enum dbox_metadata_key { /* Message flags in dbox_metadata_flags order. '0' = not set, anything else = set. Unknown flags should be preserved. */ DBOX_METADATA_FLAGS = 'F', + /* Globally unique identifier for the message. Preserved when + copying. */ + DBOX_METADATA_GUID = 'G', /* Space separated list of keywords */ DBOX_METADATA_KEYWORDS = 'K', /* POP3 UIDL overriding the default format */ diff --git a/src/lib-storage/index/dbox/dbox-mail.c b/src/lib-storage/index/dbox/dbox-mail.c index 2fe27eb495..7daa838316 100644 --- a/src/lib-storage/index/dbox/dbox-mail.c +++ b/src/lib-storage/index/dbox/dbox-mail.c @@ -178,39 +178,52 @@ static int dbox_mail_get_physical_size(struct mail *_mail, uoff_t *size_r) } static int -dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, - const char **value_r) +dbox_get_cached_metadata(struct dbox_mail *mail, enum dbox_metadata_key key, + enum index_cache_field cache_field, + const char **value_r) { - struct dbox_mail *mail = (struct dbox_mail *)_mail; struct index_mail *imail = &mail->imail; - const unsigned int pop3_uidl_cache_field = - imail->ibox->cache_fields[MAIL_CACHE_POP3_UIDL].idx; + const struct mail_cache_field *cache_fields = imail->ibox->cache_fields; struct dbox_file *file; const char *value; string_t *str; - switch (field) { - case MAIL_FETCH_UIDL_BACKEND: - /* keep the UIDL in cache file, otherwise POP3 would open all - mail files and read the metadata */ - str = str_new(imail->data_pool, 64); - if (mail_cache_lookup_field(imail->trans->cache_view, str, - _mail->seq, - pop3_uidl_cache_field) > 0) { - *value_r = str_c(str); - return 0; - } + str = str_new(imail->data_pool, 64); + if (mail_cache_lookup_field(imail->trans->cache_view, str, + imail->mail.mail.seq, + cache_fields[cache_field].idx) > 0) { + *value_r = str_c(str); + return 0; + } - if (dbox_mail_metadata_seek(mail, &file) < 0) - return -1; + if (dbox_mail_metadata_seek(mail, &file) < 0) + return -1; - value = dbox_file_metadata_get(file, DBOX_METADATA_POP3_UIDL); - if (value == NULL) - value = ""; - index_mail_cache_add_idx(imail, pop3_uidl_cache_field, - value, strlen(value)+1); - *value_r = value; - return 0; + value = dbox_file_metadata_get(file, key); + if (value == NULL) + value = ""; + index_mail_cache_add_idx(imail, cache_fields[cache_field].idx, + value, strlen(value)+1); + *value_r = value; + return 0; +} + +static int +dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, + const char **value_r) +{ + struct dbox_mail *mail = (struct dbox_mail *)_mail; + + /* keep the UIDL in cache file, otherwise POP3 would open all + mail files and read the metadata. same for GUIDs if they're + used. */ + switch (field) { + case MAIL_FETCH_UIDL_BACKEND: + return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, + MAIL_CACHE_POP3_UIDL, value_r); + case MAIL_FETCH_GUID: + return dbox_get_cached_metadata(mail, DBOX_METADATA_GUID, + MAIL_CACHE_GUID, value_r); default: break; } diff --git a/src/lib-storage/index/dbox/dbox-save.c b/src/lib-storage/index/dbox/dbox-save.c index b3e8c13b3d..d6d2ac77ea 100644 --- a/src/lib-storage/index/dbox/dbox-save.c +++ b/src/lib-storage/index/dbox/dbox-save.c @@ -178,6 +178,7 @@ static void dbox_save_write_metadata(struct dbox_save_context *ctx) { struct dbox_metadata_header metadata_hdr; char space[DBOX_EXTRA_SPACE]; + const char *guid; string_t *str; uoff_t vsize; @@ -197,6 +198,10 @@ static void dbox_save_write_metadata(struct dbox_save_context *ctx) str_printfa(str, "%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE, (unsigned long long)vsize); + guid = ctx->ctx.guid != NULL ? ctx->ctx.guid : + mail_generate_guid_string(); + str_printfa(str, "%c%s\n", DBOX_METADATA_GUID, guid); + /* flags */ str_append_c(str, DBOX_METADATA_FLAGS); dbox_mail_metadata_flags_append(str, ctx->ctx.flags); @@ -238,7 +243,7 @@ static int dbox_save_mail_write_header(struct dbox_save_mail *mail) return 0; } -int dbox_save_finish(struct mail_save_context *_ctx) +static int dbox_save_finish_write(struct mail_save_context *_ctx) { struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; struct mail_storage *storage = &ctx->mbox->storage->storage; @@ -290,6 +295,15 @@ int dbox_save_finish(struct mail_save_context *_ctx) } } +int dbox_save_finish(struct mail_save_context *ctx) +{ + int ret; + + ret = dbox_save_finish_write(ctx); + index_save_context_free(ctx); + return ret; +} + void dbox_save_cancel(struct mail_save_context *_ctx) { struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c index 25b572233b..022b3cdea2 100644 --- a/src/lib-storage/index/index-mail.c +++ b/src/lib-storage/index/index-mail.c @@ -33,6 +33,7 @@ struct mail_cache_field global_cache_fields[MAIL_INDEX_CACHE_FIELD_COUNT] = { { "imap.bodystructure", 0, MAIL_CACHE_FIELD_STRING, 0, 0 }, { "imap.envelope", 0, MAIL_CACHE_FIELD_STRING, 0, 0 }, { "pop3.uidl", 0, MAIL_CACHE_FIELD_STRING, 0, 0 }, + { "guid", 0, MAIL_CACHE_FIELD_STRING, 0, 0 }, { "mime.parts", 0, MAIL_CACHE_FIELD_VARIABLE_SIZE, 0, 0 } }; @@ -1013,6 +1014,7 @@ int index_mail_get_special(struct mail *_mail, case MAIL_FETCH_UIDL_FILE_NAME: case MAIL_FETCH_UIDL_BACKEND: case MAIL_FETCH_SEARCH_SCORE: + case MAIL_FETCH_GUID: *value_r = ""; return 0; case MAIL_FETCH_HEADER_MD5: diff --git a/src/lib-storage/index/index-mail.h b/src/lib-storage/index/index-mail.h index 014e3d10f9..141782376b 100644 --- a/src/lib-storage/index/index-mail.h +++ b/src/lib-storage/index/index-mail.h @@ -19,6 +19,7 @@ enum index_cache_field { MAIL_CACHE_IMAP_BODYSTRUCTURE, MAIL_CACHE_IMAP_ENVELOPE, MAIL_CACHE_POP3_UIDL, + MAIL_CACHE_GUID, MAIL_CACHE_MESSAGE_PARTS, MAIL_INDEX_CACHE_FIELD_COUNT diff --git a/src/lib-storage/index/index-search.c b/src/lib-storage/index/index-search.c index 8953a8134f..41531b7a1b 100644 --- a/src/lib-storage/index/index-search.c +++ b/src/lib-storage/index/index-search.c @@ -267,6 +267,10 @@ static int search_arg_match_cached(struct index_search_context *ctx, else return virtual_size > arg->value.size; + case SEARCH_GUID: + if (mail_get_special(ctx->mail, MAIL_FETCH_GUID, &str) < 0) + return -1; + return strcmp(str, arg->value.str) == 0; case SEARCH_MAILBOX: if (mail_get_special(ctx->mail, MAIL_FETCH_MAILBOX_NAME, &str) < 0) @@ -1131,6 +1135,7 @@ static bool search_arg_is_static(struct mail_search_arg *arg) case SEARCH_TEXT: case SEARCH_BODY_FAST: case SEARCH_TEXT_FAST: + case SEARCH_GUID: case SEARCH_MAILBOX: return TRUE; } diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c index 72f9aeeb67..969d51bfca 100644 --- a/src/lib-storage/index/index-storage.c +++ b/src/lib-storage/index/index-storage.c @@ -621,3 +621,9 @@ void index_keywords_free(struct mail_keywords *keywords) { mail_index_keywords_free(&keywords); } + +void index_save_context_free(struct mail_save_context *ctx) +{ + i_free_and_null(ctx->from_envelope); + i_free_and_null(ctx->guid); +} diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h index e7335efa8c..3b7bc7938a 100644 --- a/src/lib-storage/index/index-storage.h +++ b/src/lib-storage/index/index-storage.h @@ -189,6 +189,7 @@ int index_transaction_commit(struct mailbox_transaction_context *t, uint32_t *first_saved_uid_r, uint32_t *last_saved_uid_r); void index_transaction_rollback(struct mailbox_transaction_context *t); +void index_save_context_free(struct mail_save_context *ctx); bool index_keyword_array_cmp(const ARRAY_TYPE(keyword_indexes) *k1, const ARRAY_TYPE(keyword_indexes) *k2); diff --git a/src/lib-storage/index/maildir/maildir-mail.c b/src/lib-storage/index/maildir/maildir-mail.c index 9eba59062d..996c084189 100644 --- a/src/lib-storage/index/maildir/maildir-mail.c +++ b/src/lib-storage/index/maildir/maildir-mail.c @@ -414,7 +414,9 @@ maildir_mail_get_special(struct mail *_mail, enum mail_fetch_field field, struct maildir_mailbox *mbox = (struct maildir_mailbox *)mail->ibox; const char *path, *fname, *end, *uidl; - if (field == MAIL_FETCH_UIDL_FILE_NAME) { + switch (field) { + case MAIL_FETCH_UIDL_FILE_NAME: + case MAIL_FETCH_GUID: if (_mail->uid != 0) { if (!maildir_mail_get_fname(mbox, _mail, &fname)) return -1; @@ -427,14 +429,14 @@ maildir_mail_get_special(struct mail *_mail, enum mail_fetch_field field, end = strchr(fname, MAILDIR_INFO_SEP); *value_r = end == NULL ? fname : t_strdup_until(fname, end); return 0; - } else if (field == MAIL_FETCH_UIDL_BACKEND) { + case MAIL_FETCH_UIDL_BACKEND: uidl = maildir_uidlist_lookup_ext(mbox->uidlist, _mail->uid, MAILDIR_UIDLIST_REC_EXT_POP3_UIDL); *value_r = uidl != NULL ? uidl : ""; return 0; + default: + return index_mail_get_special(_mail, field, value_r); } - - return index_mail_get_special(_mail, field, value_r); } static int maildir_mail_get_stream(struct mail *_mail, diff --git a/src/lib-storage/index/maildir/maildir-save.c b/src/lib-storage/index/maildir/maildir-save.c index 8f18a0971c..00f829e73d 100644 --- a/src/lib-storage/index/maildir/maildir-save.c +++ b/src/lib-storage/index/maildir/maildir-save.c @@ -543,6 +543,7 @@ int maildir_save_finish(struct mail_save_context *ctx) T_BEGIN { ret = maildir_save_finish_real(ctx); } T_END; + index_save_context_free(ctx); return ret; } diff --git a/src/lib-storage/index/mbox/mbox-save.c b/src/lib-storage/index/mbox/mbox-save.c index 758883defa..3f55bd4a91 100644 --- a/src/lib-storage/index/mbox/mbox-save.c +++ b/src/lib-storage/index/mbox/mbox-save.c @@ -662,7 +662,7 @@ int mbox_save_finish(struct mail_save_context *_ctx) if (ctx->output != NULL) { /* make sure everything is written */ if (o_stream_flush(ctx->output) < 0) - return write_error(ctx); + write_error(ctx); } ctx->finished = TRUE; @@ -690,6 +690,7 @@ int mbox_save_finish(struct mail_save_context *_ctx) ctx->mail_offset = (uoff_t)-1; } + index_save_context_free(_ctx); return ctx->failed ? -1 : 0; } diff --git a/src/lib-storage/mail-copy.c b/src/lib-storage/mail-copy.c index ceb524ca7c..5a3f55f0dd 100644 --- a/src/lib-storage/mail-copy.c +++ b/src/lib-storage/mail-copy.c @@ -11,7 +11,7 @@ int mail_storage_copy(struct mailbox_transaction_context *t, struct mail *mail, { struct mail_save_context *ctx; struct istream *input; - const char *from_envelope; + const char *from_envelope, *guid; time_t received_date; if (mail_get_stream(mail, NULL, NULL, &input) < 0) @@ -21,14 +21,16 @@ int mail_storage_copy(struct mailbox_transaction_context *t, struct mail *mail, if (mail_get_special(mail, MAIL_FETCH_FROM_ENVELOPE, &from_envelope) < 0) return -1; - - if (*from_envelope == '\0') - from_envelope = NULL; + if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) + return -1; ctx = mailbox_save_alloc(t); mailbox_save_set_flags(ctx, flags, keywords); mailbox_save_set_received_date(ctx, received_date, 0); - mailbox_save_set_from_envelope(ctx, from_envelope); + if (*from_envelope != '\0') + mailbox_save_set_from_envelope(ctx, from_envelope); + if (*guid != '\0') + mailbox_save_set_guid(ctx, guid); mailbox_save_set_dest_mail(ctx, dest_mail); if (mailbox_save_begin(&ctx, input) < 0) diff --git a/src/lib-storage/mail-search-build.c b/src/lib-storage/mail-search-build.c index 6c59b53204..204325b598 100644 --- a/src/lib-storage/mail-search-build.c +++ b/src/lib-storage/mail-search-build.c @@ -599,6 +599,9 @@ static bool search_arg_build(struct search_build_data *data, return ARG_NEW_SINGLE(SEARCH_ALL); } return ARG_NEW_STR(SEARCH_TEXT_FAST); + } else if (strcmp(str, "X-GUID") == 0) { + /* */ + return ARG_NEW_STR(SEARCH_GUID); } else if (strcmp(str, "X-MAILBOX") == 0) { /* */ return ARG_NEW_STR(SEARCH_MAILBOX); diff --git a/src/lib-storage/mail-search.h b/src/lib-storage/mail-search.h index c0934b020d..3c16ff6de1 100644 --- a/src/lib-storage/mail-search.h +++ b/src/lib-storage/mail-search.h @@ -44,6 +44,7 @@ enum mail_search_arg_type { /* extensions */ SEARCH_MODSEQ, SEARCH_INTHREAD, + SEARCH_GUID, SEARCH_MAILBOX }; diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h index 22f8b858d9..70f43ad26a 100644 --- a/src/lib-storage/mail-storage-private.h +++ b/src/lib-storage/mail-storage-private.h @@ -323,7 +323,7 @@ struct mail_save_context { time_t received_date; int received_tz_offset; - char *from_envelope; + char *guid, *from_envelope; }; struct mailbox_sync_context { @@ -356,6 +356,7 @@ void mail_storage_set_critical(struct mail_storage *storage, void mail_storage_set_internal_error(struct mail_storage *storage); bool mail_storage_set_error_from_errno(struct mail_storage *storage); +const char *mail_generate_guid_string(void); void mail_set_expunged(struct mail *mail); void mailbox_set_deleted(struct mailbox *box); diff --git a/src/lib-storage/mail-storage.c b/src/lib-storage/mail-storage.c index 747d773756..936ef6395d 100644 --- a/src/lib-storage/mail-storage.c +++ b/src/lib-storage/mail-storage.c @@ -816,6 +816,14 @@ void mailbox_save_set_from_envelope(struct mail_save_context *ctx, ctx->from_envelope = i_strdup(envelope); } +void mailbox_save_set_guid(struct mail_save_context *ctx, const char *guid) +{ + i_assert(guid == NULL || *guid != '\0'); + + i_free(ctx->guid); + ctx->guid = i_strdup(guid); +} + void mailbox_save_set_dest_mail(struct mail_save_context *ctx, struct mail *mail) { @@ -852,7 +860,6 @@ int mailbox_save_finish(struct mail_save_context **_ctx) struct mail_save_context *ctx = *_ctx; *_ctx = NULL; - i_free(ctx->from_envelope); return ctx->transaction->box->v.save_finish(ctx); } @@ -861,7 +868,6 @@ void mailbox_save_cancel(struct mail_save_context **_ctx) struct mail_save_context *ctx = *_ctx; *_ctx = NULL; - i_free(ctx->from_envelope); ctx->transaction->box->v.save_cancel(ctx); } diff --git a/src/lib-storage/mail-storage.h b/src/lib-storage/mail-storage.h index 9e9933a419..80dd30ef53 100644 --- a/src/lib-storage/mail-storage.h +++ b/src/lib-storage/mail-storage.h @@ -128,7 +128,8 @@ enum mail_fetch_field { MAIL_FETCH_UIDL_FILE_NAME = 0x00020000, MAIL_FETCH_UIDL_BACKEND = 0x00040000, MAIL_FETCH_MAILBOX_NAME = 0x00080000, - MAIL_FETCH_SEARCH_SCORE = 0x00100000 + MAIL_FETCH_SEARCH_SCORE = 0x00100000, + MAIL_FETCH_GUID = 0x00200000 }; enum mailbox_transaction_flags { @@ -466,6 +467,10 @@ void mailbox_save_set_received_date(struct mail_save_context *ctx, specify the address in From_-line. */ void mailbox_save_set_from_envelope(struct mail_save_context *ctx, const char *envelope); +/* Set globally unique ID for the saved mail. A new GUID is generated by + default. This function should usually be called only when copying an + existing mail (or restoring a mail from backup). */ +void mailbox_save_set_guid(struct mail_save_context *ctx, const char *guid); /* If dest_mail is set, the saved message can be accessed using it. Note that setting it may require mailbox syncing, so don't set it unless you need it. Also you shouldn't try to access it before mailbox_save_finish() is diff --git a/src/lib-storage/mail.c b/src/lib-storage/mail.c index a88a626cc5..c6f9b80f24 100644 --- a/src/lib-storage/mail.c +++ b/src/lib-storage/mail.c @@ -2,8 +2,11 @@ #include "lib.h" #include "ioloop.h" +#include "hostpid.h" #include "mail-storage-private.h" +#include + struct mail *mail_alloc(struct mailbox_transaction_context *t, enum mail_fetch_field wanted_fields, struct mailbox_header_lookup_ctx *wanted_headers) @@ -200,3 +203,26 @@ void mail_set_cache_corrupted(struct mail *mail, enum mail_fetch_field field) p->v.set_cache_corrupted(mail, field); } + +const char *mail_generate_guid_string(void) +{ + static struct timespec ts = { 0, 0 }; + static unsigned int pid = 0; + + /* we'll use the current time in nanoseconds as the initial 64bit + counter. */ + if (ts.tv_sec == 0) { + if (clock_gettime(CLOCK_REALTIME, &ts) < 0) + i_fatal("clock_gettime() failed: %m"); + pid = getpid(); + } else if ((uint32_t)ts.tv_nsec < (uint32_t)-1) { + ts.tv_nsec++; + } else { + ts.tv_sec++; + ts.tv_nsec = 0; + } + return t_strdup_printf("%04x%04lx%04x%s", + (unsigned int)ts.tv_nsec, + (unsigned long)ts.tv_sec, + pid, my_hostname); +}