From 300baf826ba39003b861e7716c35bae372e6f23e Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Sat, 11 Oct 2008 13:26:46 +0300 Subject: [PATCH] Maildir/dbox: Try harder to assign unique UIDVALIDITY values to mailboxes. --HG-- branch : HEAD --- src/lib-storage/Makefile.am | 6 +- src/lib-storage/index/dbox/dbox-index.c | 17 +- src/lib-storage/index/dbox/dbox-storage.h | 1 + .../index/maildir/maildir-storage.c | 11 + .../index/maildir/maildir-storage.h | 2 + .../index/maildir/maildir-sync-index.c | 4 +- .../index/maildir/maildir-uidlist.c | 10 +- src/lib-storage/mailbox-uidvalidity.c | 196 ++++++++++++++++++ src/lib-storage/mailbox-uidvalidity.h | 6 + 9 files changed, 243 insertions(+), 10 deletions(-) create mode 100644 src/lib-storage/mailbox-uidvalidity.c create mode 100644 src/lib-storage/mailbox-uidvalidity.h diff --git a/src/lib-storage/Makefile.am b/src/lib-storage/Makefile.am index 2eaf088530..6516485185 100644 --- a/src/lib-storage/Makefile.am +++ b/src/lib-storage/Makefile.am @@ -19,7 +19,8 @@ libstorage_a_SOURCES = \ mail-user.c \ mailbox-list.c \ mailbox-search-result.c \ - mailbox-tree.c + mailbox-tree.c \ + mailbox-uidvalidity.c headers = \ mail-copy.h \ @@ -34,7 +35,8 @@ headers = \ mailbox-list.h \ mailbox-list-private.h \ mailbox-search-result-private.h \ - mailbox-tree.h + mailbox-tree.h \ + mailbox-uidvalidity.h if INSTALL_HEADERS pkginc_libdir=$(pkgincludedir)/src/lib-storage diff --git a/src/lib-storage/index/dbox/dbox-index.c b/src/lib-storage/index/dbox/dbox-index.c index 8812e88267..512a0db677 100644 --- a/src/lib-storage/index/dbox/dbox-index.c +++ b/src/lib-storage/index/dbox/dbox-index.c @@ -9,6 +9,7 @@ #include "write-full.h" #include "nfs-workarounds.h" #include "safe-mkstemp.h" +#include "mailbox-uidvalidity.h" #include "dbox-storage.h" #include "dbox-file.h" #include "dbox-index.h" @@ -151,15 +152,27 @@ dbox_index_set_corrupted(struct dbox_index *index, const char *reason) return -1; } +static uint32_t dbox_get_uidvalidity_next(struct mail_storage *storage) +{ + const char *path; + + path = mailbox_list_get_path(storage->list, NULL, + MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"DBOX_UIDVALIDITY_FILE_NAME, NULL); + return mailbox_uidvalidity_next(path); +} + static void dbox_index_header_init(struct dbox_index *index, struct dbox_index_file_header *hdr) { if (index->uid_validity == 0) { + struct index_mailbox *ibox = &index->mbox->ibox; const struct mail_index_header *idx_hdr; - idx_hdr = mail_index_get_header(index->mbox->ibox.view); + idx_hdr = mail_index_get_header(ibox->view); index->uid_validity = idx_hdr->uid_validity != 0 ? - idx_hdr->uid_validity : (uint32_t)ioloop_time; + idx_hdr->uid_validity : + dbox_get_uidvalidity_next(ibox->box.storage); } memset(hdr, ' ', sizeof(*hdr)); diff --git a/src/lib-storage/index/dbox/dbox-storage.h b/src/lib-storage/index/dbox/dbox-storage.h index 838ff54e0d..dafcefdb03 100644 --- a/src/lib-storage/index/dbox/dbox-storage.h +++ b/src/lib-storage/index/dbox/dbox-storage.h @@ -6,6 +6,7 @@ #define DBOX_STORAGE_NAME "dbox" #define DBOX_SUBSCRIPTION_FILE_NAME ".dbox-subscriptions" +#define DBOX_UIDVALIDITY_FILE_NAME ".dbox-uidvalidity" #define DBOX_INDEX_PREFIX "dovecot.index" #define DBOX_MAILDIR_NAME "dbox-Mails" diff --git a/src/lib-storage/index/maildir/maildir-storage.c b/src/lib-storage/index/maildir/maildir-storage.c index d558336be2..8e1de79b2f 100644 --- a/src/lib-storage/index/maildir/maildir-storage.c +++ b/src/lib-storage/index/maildir/maildir-storage.c @@ -8,6 +8,7 @@ #include "mkdir-parents.h" #include "unlink-directory.h" #include "unlink-old-files.h" +#include "mailbox-uidvalidity.h" #include "maildir-storage.h" #include "maildir-uidlist.h" #include "maildir-keywords.h" @@ -998,6 +999,16 @@ maildirplusplus_iter_is_mailbox(struct mailbox_list_iterate_context *ctx, return ret; } +uint32_t maildir_get_uidvalidity_next(struct mail_storage *storage) +{ + const char *path; + + path = mailbox_list_get_path(storage->list, NULL, + MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"MAILDIR_UIDVALIDITY_FNAME, NULL); + return mailbox_uidvalidity_next(path); +} + static void maildir_class_init(void) { maildir_transaction_class_init(); diff --git a/src/lib-storage/index/maildir/maildir-storage.h b/src/lib-storage/index/maildir/maildir-storage.h index 8be7ec5811..ed1ecd5e5f 100644 --- a/src/lib-storage/index/maildir/maildir-storage.h +++ b/src/lib-storage/index/maildir/maildir-storage.h @@ -5,6 +5,7 @@ #define MAILDIR_SUBSCRIPTION_FILE_NAME "subscriptions" #define MAILDIR_INDEX_PREFIX "dovecot.index" #define MAILDIR_UNLINK_DIRNAME "DOVECOT-TRASHED" +#define MAILDIR_UIDVALIDITY_FNAME "dovecot-uidvalidity" /* "base,S=123:2," means: [ [..]] 2 */ @@ -122,6 +123,7 @@ int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid, #endif bool maildir_set_deleted(struct maildir_mailbox *mbox); +uint32_t maildir_get_uidvalidity_next(struct mail_storage *storage); void maildir_transaction_class_init(void); void maildir_transaction_class_deinit(void); diff --git a/src/lib-storage/index/maildir/maildir-sync-index.c b/src/lib-storage/index/maildir/maildir-sync-index.c index 66797f3a1a..79b8762568 100644 --- a/src/lib-storage/index/maildir/maildir-sync-index.c +++ b/src/lib-storage/index/maildir/maildir-sync-index.c @@ -465,8 +465,8 @@ int maildir_sync_index(struct maildir_index_sync_context *ctx, mbox->maildir_hdr.cur_mtime = time(NULL); if (uid_validity == 0) { - uid_validity = hdr->uid_validity != 0 ? - hdr->uid_validity : (uint32_t)ioloop_time; + uid_validity = hdr->uid_validity != 0 ? hdr->uid_validity : + maildir_get_uidvalidity_next(mbox->ibox.box.storage); maildir_uidlist_set_uid_validity(mbox->uidlist, uid_validity); } maildir_uidlist_set_next_uid(mbox->uidlist, hdr_next_uid, FALSE); diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c index f9e9def33a..7dda2a4e83 100644 --- a/src/lib-storage/index/maildir/maildir-uidlist.c +++ b/src/lib-storage/index/maildir/maildir-uidlist.c @@ -1189,6 +1189,7 @@ static bool maildir_uidlist_want_recreate(struct maildir_uidlist_sync_ctx *ctx) static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx) { struct maildir_uidlist *uidlist = ctx->uidlist; + struct mail_storage *storage = uidlist->ibox->box.storage; struct stat st; uoff_t file_size; @@ -1198,7 +1199,8 @@ static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx) hdr = mail_index_get_header(uidlist->ibox->view); uidlist->uid_validity = hdr->uid_validity != 0 ? - hdr->uid_validity : (uint32_t)ioloop_time; + hdr->uid_validity : + maildir_get_uidvalidity_next(storage); } @@ -1211,7 +1213,7 @@ static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx) uidlist->fd = nfs_safe_open(uidlist->path, O_RDWR); if (uidlist->fd == -1) { - mail_storage_set_critical(uidlist->ibox->box.storage, + mail_storage_set_critical(storage, "open(%s) failed: %m", uidlist->path); return -1; } @@ -1220,7 +1222,7 @@ static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx) i_assert(ctx->first_unwritten_pos != (unsigned int)-1); if (lseek(uidlist->fd, 0, SEEK_END) < 0) { - mail_storage_set_critical(uidlist->ibox->box.storage, + mail_storage_set_critical(storage, "lseek(%s) failed: %m", uidlist->path); return -1; } @@ -1230,7 +1232,7 @@ static int maildir_uidlist_sync_update(struct maildir_uidlist_sync_ctx *ctx) return -1; if (fstat(uidlist->fd, &st) < 0) { - mail_storage_set_critical(uidlist->ibox->box.storage, + mail_storage_set_critical(storage, "fstat(%s) failed: %m", uidlist->path); return -1; } diff --git a/src/lib-storage/mailbox-uidvalidity.c b/src/lib-storage/mailbox-uidvalidity.c new file mode 100644 index 0000000000..bc10f3cca1 --- /dev/null +++ b/src/lib-storage/mailbox-uidvalidity.c @@ -0,0 +1,196 @@ +/* Copyright (c) 2008 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "str.h" +#include "read-full.h" +#include "write-full.h" +#include "mailbox-uidvalidity.h" + +#include +#include +#include +#include +#include + +#define RETRY_COUNT 10 + +static uint32_t mailbox_uidvalidity_next_fallback(void) +{ + static uint32_t uid_validity = 0; + + /* we failed to use the uidvalidity file. don't fail the mailbox + creation because of it though, most of the time it's safe enough + to use the current time as the uidvalidity value. */ + if (uid_validity < ioloop_time) + uid_validity = ioloop_time; + else + uid_validity++; + return uid_validity; +} + +static void mailbox_uidvalidity_write(const char *path, uint32_t uid_validity) +{ + char buf[8+1]; + int fd; + + fd = open(path, O_RDWR | O_CREAT, 0666); + if (fd == -1) { + i_error("open(%s) failed: %m", path); + return; + } + i_snprintf(buf, sizeof(buf), "%08x", uid_validity); + if (pwrite_full(fd, buf, strlen(buf), 0) < 0) + i_error("write(%s) failed: %m", path); + if (close(fd) < 0) + i_error("close(%s) failed: %m", path); +} + +static int mailbox_uidvalidity_rename(const char *path, uint32_t *uid_validity) +{ + string_t *src, *dest; + unsigned int i, prefix_len; + int ret; + + src = t_str_new(256); + str_append(src, path); + dest = t_str_new(256); + str_append(dest, path); + prefix_len = str_len(src); + + for (i = 0; i < RETRY_COUNT; i++) { + str_printfa(src, ".%08x", *uid_validity); + *uid_validity += 1; + str_printfa(dest, ".%08x", *uid_validity); + + if ((ret = rename(str_c(src), str_c(dest))) == 0 || + errno != ENOENT) + break; + + /* possibly a race condition. try the next value. */ + str_truncate(src, prefix_len); + str_truncate(dest, prefix_len); + } + if (ret < 0) + i_error("rename(%s, %s) failed: %m", str_c(src), str_c(dest)); + return ret; +} + +static uint32_t mailbox_uidvalidity_next_rescan(const char *path) +{ + DIR *d; + struct dirent *dp; + const char *fname, *dir, *prefix, *tmp; + char *endp; + unsigned int i, prefix_len; + uint32_t cur_value, min_value, max_value; + int fd; + + fname = strrchr(path, '/'); + if (fname == NULL) { + dir = "."; + fname = path; + } else { + dir = t_strdup_until(path, fname); + fname++; + } + + d = opendir(dir); + if (d == NULL) { + i_error("opendir(%s) failed: %m", dir); + return mailbox_uidvalidity_next_fallback(); + } + prefix = t_strconcat(fname, ".", NULL); + prefix_len = strlen(prefix); + + /* just in case there happens to be multiple matching uidvalidity + files, track the min/max values. use the max value and delete the + min value file. */ + max_value = 0; min_value = (uint32_t)-1; + while ((dp = readdir(d)) != NULL) { + if (strncmp(dp->d_name, prefix, prefix_len) == 0) { + cur_value = strtoul(dp->d_name + prefix_len, &endp, 16); + if (*endp == '\0') { + if (min_value > cur_value) + min_value = cur_value; + if (max_value < cur_value) + max_value = cur_value; + } + } + } + if (closedir(d) < 0) + i_error("closedir(%s) failed: %m", dir); + + if (max_value == 0) { + /* no uidvalidity files. create one. */ + for (i = 0; i < RETRY_COUNT; i++) { + cur_value = mailbox_uidvalidity_next_fallback(); + tmp = t_strdup_printf("%s.%08x", path, cur_value); + fd = open(tmp, O_RDWR | O_CREAT | O_EXCL, 0666); + if (fd != -1 || errno != EEXIST) + break; + /* already exists. although it's quite unlikely we'll + hit this race condition. more likely we'll create + a duplicate file.. */ + } + if (fd == -1) { + i_error("creat(%s) failed: %m", tmp); + return cur_value; + } + (void)close(fd); + mailbox_uidvalidity_write(path, cur_value); + return cur_value; + } + if (min_value != max_value) { + /* duplicate uidvalidity files, delete the oldest */ + tmp = t_strdup_printf("%s.%08x", path, min_value); + if (unlink(tmp) < 0 && errno != ENOENT) + i_error("unlink(%s) failed: %m", tmp); + } + + cur_value = max_value; + if (mailbox_uidvalidity_rename(path, &cur_value) < 0) + return mailbox_uidvalidity_next_fallback(); + mailbox_uidvalidity_write(path, cur_value); + return cur_value; +} + +uint32_t mailbox_uidvalidity_next(const char *path) +{ + char buf[8+1], *endp; + uint32_t cur_value; + int fd, ret; + + fd = open(path, O_RDWR); + if (fd == -1) { + if (errno != ENOENT) + i_error("open(%s) failed: %m", path); + return mailbox_uidvalidity_next_rescan(path); + } + ret = read_full(fd, buf, sizeof(buf)-1); + if (ret < 0) { + i_error("read(%s) failed: %m", path); + (void)close(fd); + return mailbox_uidvalidity_next_rescan(path); + } + buf[sizeof(buf)-1] = 0; + cur_value = strtoul(buf, &endp, 16); + if (ret == 0 || endp != buf+sizeof(buf)-1) { + /* broken value */ + (void)close(fd); + return mailbox_uidvalidity_next_rescan(path); + } + + /* we now have the current uidvalidity value that's hopefully correct */ + if (mailbox_uidvalidity_rename(path, &cur_value) < 0) + return mailbox_uidvalidity_next_rescan(path); + + /* fast path succeeded. write the current value to the main + uidvalidity file. */ + i_snprintf(buf, sizeof(buf), "%08x", cur_value); + if (pwrite_full(fd, buf, strlen(buf), 0) < 0) + i_error("write(%s) failed: %m", path); + if (close(fd) < 0) + i_error("close(%s) failed: %m", path); + return cur_value; +} diff --git a/src/lib-storage/mailbox-uidvalidity.h b/src/lib-storage/mailbox-uidvalidity.h new file mode 100644 index 0000000000..735de5375c --- /dev/null +++ b/src/lib-storage/mailbox-uidvalidity.h @@ -0,0 +1,6 @@ +#ifndef MAILBOX_UIDVALIDITY_H +#define MAILBOX_UIDVALIDITY_H + +uint32_t mailbox_uidvalidity_next(const char *path); + +#endif -- 2.47.3