From: Timo Sirainen Date: Tue, 17 Feb 2009 21:24:39 +0000 (-0500) Subject: dbox: Cleaned up code to get ready for global storage. X-Git-Tag: 2.0.alpha1~1038^2~86 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=cae1db7ea032acc2afbd5414891bc4f970df5859;p=thirdparty%2Fdovecot%2Fcore.git dbox: Cleaned up code to get ready for global storage. --HG-- branch : HEAD --- diff --git a/src/lib-storage/index/dbox/Makefile.am b/src/lib-storage/index/dbox/Makefile.am index 6707b20bf7..e695532cd9 100644 --- a/src/lib-storage/index/dbox/Makefile.am +++ b/src/lib-storage/index/dbox/Makefile.am @@ -11,8 +11,8 @@ AM_CPPFLAGS = \ libstorage_dbox_a_SOURCES = \ dbox-file.c \ dbox-file-maildir.c \ - dbox-index.c \ dbox-mail.c \ + dbox-map.c \ dbox-save.c \ dbox-sync.c \ dbox-sync-file.c \ @@ -23,7 +23,7 @@ libstorage_dbox_a_SOURCES = \ headers = \ dbox-file.h \ dbox-file-maildir.h \ - dbox-index.h \ + dbox-map.h \ dbox-storage.h \ dbox-sync.h diff --git a/src/lib-storage/index/dbox/dbox-file-maildir.c b/src/lib-storage/index/dbox/dbox-file-maildir.c index 59a6cd25e1..29fdd01df1 100644 --- a/src/lib-storage/index/dbox/dbox-file-maildir.c +++ b/src/lib-storage/index/dbox/dbox-file-maildir.c @@ -12,17 +12,6 @@ #include -static const char * -dbox_maildir_file_get_ext(struct dbox_file *file, - enum maildir_uidlist_rec_ext_key key) -{ - uint32_t uid; - - uid = file->file_id & ~DBOX_FILE_ID_FLAG_UID; - return maildir_uidlist_lookup_ext(file->mbox->maildir_uidlist, - uid, key); -} - const char *dbox_file_maildir_metadata_get(struct dbox_file *file, enum dbox_metadata_key key) { @@ -59,8 +48,9 @@ const char *dbox_file_maildir_metadata_get(struct dbox_file *file, if (!maildir_filename_get_size(file->fname, MAILDIR_EXTRA_VIRTUAL_SIZE, &size)) { - value = dbox_maildir_file_get_ext(file, - MAILDIR_UIDLIST_REC_EXT_VSIZE); + value = maildir_uidlist_lookup_ext( + file->single_mbox->maildir_uidlist, + file->uid, MAILDIR_UIDLIST_REC_EXT_VSIZE); if (value == NULL) break; size = strtoull(value, NULL, 10); @@ -68,8 +58,9 @@ const char *dbox_file_maildir_metadata_get(struct dbox_file *file, value = t_strdup_printf("%llx", (unsigned long long)size); break; case DBOX_METADATA_POP3_UIDL: - value = dbox_maildir_file_get_ext(file, - MAILDIR_UIDLIST_REC_EXT_POP3_UIDL); + value = maildir_uidlist_lookup_ext( + file->single_mbox->maildir_uidlist, + file->uid, MAILDIR_UIDLIST_REC_EXT_POP3_UIDL); if (value != NULL && *value == '\0') { /* special case: use base filename */ p = strchr(file->fname, MAILDIR_INFO_SEP); @@ -79,9 +70,9 @@ const char *dbox_file_maildir_metadata_get(struct dbox_file *file, value = t_strdup_until(file->fname, p); } break; - case DBOX_METADATA_OLD_EXPUNGED: - case DBOX_METADATA_OLD_FLAGS: - case DBOX_METADATA_OLD_KEYWORDS: + case DBOX_METADATA_OLDV1_EXPUNGED: + case DBOX_METADATA_OLDV1_FLAGS: + case DBOX_METADATA_OLDV1_KEYWORDS: case DBOX_METADATA_EXT_REF: case DBOX_METADATA_SPACE: break; diff --git a/src/lib-storage/index/dbox/dbox-file.c b/src/lib-storage/index/dbox/dbox-file.c index d3024969c0..e04855db3b 100644 --- a/src/lib-storage/index/dbox/dbox-file.c +++ b/src/lib-storage/index/dbox/dbox-file.c @@ -9,7 +9,6 @@ #include "ostream.h" #include "mkdir-parents.h" #include "fdatasync-path.h" -#include "write-full.h" #include "str.h" #include "maildir/maildir-uidlist.h" #include "dbox-storage.h" @@ -48,7 +47,7 @@ static char *dbox_generate_tmp_filename(void) void dbox_file_set_syscall_error(struct dbox_file *file, const char *function) { - mail_storage_set_critical(file->mbox->ibox.box.storage, + mail_storage_set_critical(&file->storage->storage, "%s(%s) failed: %m", function, dbox_file_get_path(file)); } @@ -56,25 +55,25 @@ void dbox_file_set_syscall_error(struct dbox_file *file, const char *function) static void dbox_file_set_corrupted(struct dbox_file *file, const char *reason) { - mail_storage_set_critical(file->mbox->ibox.box.storage, + mail_storage_set_critical(&file->storage->storage, "%s corrupted: %s", dbox_file_get_path(file), reason); } static struct dbox_file * -dbox_find_and_move_open_file(struct dbox_mailbox *mbox, unsigned int file_id) +dbox_find_and_move_open_file(struct dbox_storage *storage, uint32_t file_id) { struct dbox_file *const *files, *file; unsigned int i, count; - files = array_get(&mbox->open_files, &count); + files = array_get(&storage->open_files, &count); for (i = 0; i < count; i++) { if (files[i]->file_id == file_id) { /* move to last in the array */ file = files[i]; - array_delete(&mbox->open_files, i, 1); - array_append(&mbox->open_files, &file, 1); + array_delete(&storage->open_files, i, 1); + array_append(&storage->open_files, &file, 1); return file; } } @@ -93,33 +92,33 @@ static void dbox_file_free(struct dbox_file *file) i_free(file); } -void dbox_files_free(struct dbox_mailbox *mbox) +void dbox_files_free(struct dbox_storage *storage) { struct dbox_file *const *files; unsigned int i, count; - files = array_get(&mbox->open_files, &count); + files = array_get(&storage->open_files, &count); for (i = 0; i < count; i++) dbox_file_free(files[i]); - array_clear(&mbox->open_files); + array_clear(&storage->open_files); } static void -dbox_close_open_files(struct dbox_mailbox *mbox, unsigned int close_count) +dbox_close_open_files(struct dbox_storage *storage, unsigned int close_count) { struct dbox_file *const *files; unsigned int i, count; - files = array_get(&mbox->open_files, &count); + files = array_get(&storage->open_files, &count); for (i = 0; i < count;) { if (files[i]->refcount == 0) { dbox_file_free(files[i]); - array_delete(&mbox->open_files, i, 1); + array_delete(&storage->open_files, i, 1); if (--close_count == 0) break; - files = array_get(&mbox->open_files, &count); + files = array_get(&storage->open_files, &count); } else { i++; } @@ -137,99 +136,132 @@ dbox_maildir_uid_get_fname(struct dbox_mailbox *mbox, uint32_t uid, } static char * -dbox_file_id_get_fname(struct dbox_mailbox *mbox, unsigned int file_id, - bool *maildir_file_r) +dbox_file_uid_get_fname(struct dbox_mailbox *mbox, uint32_t uid, + bool *maildir_file_r) { const char *fname; - uint32_t uid; - - *maildir_file_r = FALSE; - if ((file_id & DBOX_FILE_ID_FLAG_UID) != 0) { - uid = file_id & ~DBOX_FILE_ID_FLAG_UID; - if (uid <= mbox->highest_maildir_uid && - dbox_maildir_uid_get_fname(mbox, uid, &fname)) { - *maildir_file_r = TRUE; - return i_strdup(fname); - } else { - return i_strdup_printf(DBOX_MAIL_FILE_UID_FORMAT, uid); - } + + if (uid <= mbox->highest_maildir_uid && + dbox_maildir_uid_get_fname(mbox, uid, &fname)) { + *maildir_file_r = TRUE; + return i_strdup(fname); + } else { + *maildir_file_r = FALSE; + return i_strdup_printf(DBOX_MAIL_FILE_UID_FORMAT, uid); } +} + +const char *dbox_file_get_primary_path(struct dbox_file *file) +{ + const char *dir; - return i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, file_id); + dir = file->single_mbox != NULL ? file->single_mbox->path : + file->storage->storage_dir; + return t_strdup_printf("%s/%s", dir, file->fname); +} + +const char *dbox_file_get_alt_path(struct dbox_file *file) +{ + const char *dir; + + dir = file->single_mbox != NULL ? file->single_mbox->alt_path : + file->storage->alt_storage_dir; + return t_strdup_printf("%s/%s", dir, file->fname); } struct dbox_file * -dbox_file_init(struct dbox_mailbox *mbox, unsigned int file_id) +dbox_file_init_single(struct dbox_mailbox *mbox, uint32_t uid) { struct dbox_file *file; - unsigned int count; bool maildir; + file = i_new(struct dbox_file, 1); + file->refcount = 1; + file->storage = mbox->storage; + file->single_mbox = mbox; + file->fd = -1; + if (uid != 0) { + file->uid = uid; + file->fname = dbox_file_uid_get_fname(mbox, uid, &maildir); + file->maildir_file = maildir; + } else { + file->fname = dbox_generate_tmp_filename(); + } + file->current_path = i_strdup_printf("%s/%s", mbox->path, file->fname); + return file; +} + +struct dbox_file * +dbox_file_init_multi(struct dbox_storage *storage, uint32_t file_id) +{ + struct dbox_file *file; + unsigned int count; + file = file_id == 0 ? NULL : - dbox_find_and_move_open_file(mbox, file_id); + dbox_find_and_move_open_file(storage, file_id); if (file != NULL) { file->refcount++; return file; } - count = array_count(&mbox->open_files); - if (count > mbox->max_open_files) - dbox_close_open_files(mbox, count - mbox->max_open_files); + count = array_count(&storage->open_files); + if (count > storage->max_open_files) + dbox_close_open_files(storage, count - storage->max_open_files); file = i_new(struct dbox_file, 1); file->refcount = 1; - file->mbox = mbox; - if (file_id != 0) { - file->file_id = file_id; - file->fname = dbox_file_id_get_fname(mbox, file_id, &maildir); - file->maildir_file = maildir; - } else { - file->fname = dbox_generate_tmp_filename(); - } - if (file->maildir_file || file_id == 0) { - /* newly created files and maildir files always exist in the - primary path */ - file->current_path = - i_strdup_printf("%s/%s", mbox->path, file->fname); - } + file->storage = storage; file->fd = -1; + file->fname = file_id == 0 ? dbox_generate_tmp_filename() : + i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, file_id); + file->current_path = + i_strdup_printf("%s/%s", storage->storage_dir, file->fname); if (file_id != 0) - array_append(&file->mbox->open_files, &file, 1); + array_append(&storage->open_files, &file, 1); return file; } -int dbox_file_assign_id(struct dbox_file *file, unsigned int file_id) +int dbox_file_assign_id(struct dbox_file *file, uint32_t id) { - struct dbox_mailbox *mbox = file->mbox; const char *old_path; char *new_fname, *new_path; bool maildir; - i_assert(file->file_id == 0); - i_assert(file_id != 0); - - if (!file->maildir_file) { - old_path = dbox_file_get_path(file); - new_fname = dbox_file_id_get_fname(mbox, file_id, &maildir); - new_path = i_strdup_printf("%s/%s", mbox->path, new_fname); + i_assert(!file->maildir_file); + i_assert(file->uid == 0 && file->file_id == 0); + i_assert(id != 0); - if (rename(old_path, new_path) < 0) { - mail_storage_set_critical(mbox->ibox.box.storage, - "rename(%s, %s) failed: %m", - old_path, new_path); - i_free(new_fname); - i_free(new_path); - return -1; - } - i_free(file->fname); - i_free(file->current_path); - file->fname = new_fname; - file->current_path = new_path; + old_path = dbox_file_get_path(file); + if (file->single_mbox != NULL) { + new_fname = dbox_file_uid_get_fname(file->single_mbox, + id, &maildir); + new_path = i_strdup_printf("%s/%s", file->single_mbox->path, + new_fname); + } else { + new_fname = i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, id); + new_path = i_strdup_printf("%s/%s", file->storage->storage_dir, + new_fname); } + if (rename(old_path, new_path) < 0) { + mail_storage_set_critical(&file->storage->storage, + "rename(%s, %s) failed: %m", + old_path, new_path); + i_free(new_fname); + i_free(new_path); + return -1; + } + i_free(file->fname); + i_free(file->current_path); + file->fname = new_fname; + file->current_path = new_path; - file->file_id = file_id; - array_append(&mbox->open_files, &file, 1); + if (file->single_mbox != NULL) + file->uid = id; + else { + file->file_id = id; + array_append(&file->storage->open_files, &file, 1); + } return 0; } @@ -249,8 +281,8 @@ void dbox_file_unref(struct dbox_file **_file) file->metadata_read_offset = 0; if (file->file_id != 0) { - files = array_get(&file->mbox->open_files, &count); - if (!file->deleted && count <= file->mbox->max_open_files) { + files = array_get(&file->storage->open_files, &count); + if (!file->deleted && count <= file->storage->max_open_files) { /* we can leave this file open for now */ return; } @@ -260,7 +292,7 @@ void dbox_file_unref(struct dbox_file **_file) break; } i_assert(i != count); - array_delete(&file->mbox->open_files, i, 1); + array_delete(&file->storage->open_files, i, 1); } dbox_file_free(file); @@ -296,28 +328,28 @@ bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size) return FALSE; } - if (file->append_offset < file->mbox->rotate_min_size || + if (file->append_offset < file->storage->rotate_min_size || file->append_offset == file->file_header_size) return TRUE; - if (file->append_offset + mail_size >= file->mbox->rotate_size) + if (file->append_offset + mail_size >= file->storage->rotate_size) return FALSE; - return file->create_time >= day_begin_stamp(file->mbox->rotate_days); + return file->create_time >= day_begin_stamp(file->storage->rotate_days); } static int dbox_file_parse_header(struct dbox_file *file, const char *line) { const char *const *tmp, *value; - unsigned int pos; + unsigned int pos, version; enum dbox_header_key key; - if (*line - '0' != DBOX_VERSION || line[1] != ' ') { + version = *line - '0'; + if (line[1] != ' ' || (version != 1 && version != DBOX_VERSION)) { dbox_file_set_corrupted(file, "Invalid dbox version"); return -1; } line += 2; pos = 2; - file->append_offset = 0; file->msg_header_size = 0; for (tmp = t_strsplit(line, " "); *tmp != NULL; tmp++) { @@ -325,10 +357,7 @@ static int dbox_file_parse_header(struct dbox_file *file, const char *line) value = *tmp + 1; switch (key) { - case DBOX_HEADER_APPEND_OFFSET: - file->append_offset_header_pos = pos + 1; - file->append_offset = *value == 'X' ? 0 : - strtoull(value, NULL, 16); + case DBOX_HEADER_OLDV1_APPEND_OFFSET: break; case DBOX_HEADER_MSG_HEADER_SIZE: file->msg_header_size = strtoul(value, NULL, 16); @@ -375,29 +404,23 @@ static int dbox_file_open_fd(struct dbox_file *file) { const char *path; bool alt = FALSE; - int i; /* try the primary path first */ - path = t_strdup_printf("%s/%s", file->mbox->path, file->fname); - for (i = 0;; i++) { - file->fd = open(path, O_RDWR); - if (file->fd != -1) - break; - + path = dbox_file_get_primary_path(file); + while ((file->fd = open(path, O_RDWR)) == -1) { if (errno != ENOENT) { - mail_storage_set_critical(file->mbox->ibox.box.storage, + mail_storage_set_critical(&file->storage->storage, "open(%s) failed: %m", path); return -1; } - if (file->mbox->alt_path == NULL || i == 1) { - /* file doesn't exist */ + if (file->storage->alt_storage_dir == NULL || alt) { + /* not found */ return 0; } /* try the alternative path */ - path = t_strdup_printf("%s/%s", file->mbox->alt_path, - file->fname); + path = dbox_file_get_alt_path(file); alt = TRUE; } i_free(file->current_path); @@ -432,17 +455,24 @@ static int dbox_file_open(struct dbox_file *file, bool read_header, dbox_file_read_header(file); } -int dbox_create_fd(struct dbox_mailbox *mbox, const char *path) +static int dbox_create_fd(struct dbox_storage *storage, const char *path) { mode_t old_mask; int fd; - old_mask = umask(0777 & ~mbox->ibox.box.file_create_mode); + old_mask = umask(0777 & ~storage->create_mode); fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0777); umask(old_mask); if (fd == -1) { - mail_storage_set_critical(mbox->ibox.box.storage, + mail_storage_set_critical(&storage->storage, "open(%s, O_CREAT) failed: %m", path); + } else if (storage->create_gid != (gid_t)-1) { + if (fchown(fd, (uid_t)-1, storage->create_gid) < 0) { + mail_storage_set_critical(&storage->storage, + "fchown(%s, -1, %ld) failed: %m", + path, (long)storage->create_gid); + /* continue anyway */ + } } return fd; } @@ -450,35 +480,24 @@ int dbox_create_fd(struct dbox_mailbox *mbox, const char *path) static int dbox_file_create(struct dbox_file *file) { string_t *hdr; - const char *hdrsize; i_assert(file->fd == -1); - if (file->current_path == NULL) { - file->current_path = - i_strdup_printf("%s/%s", file->mbox->path, file->fname); - } - file->fd = dbox_create_fd(file->mbox, file->current_path); + file->fd = dbox_create_fd(file->storage, file->current_path); if (file->fd == -1) return -1; file->output = o_stream_create_fd_file(file->fd, 0, FALSE); hdr = t_str_new(128); - str_printfa(hdr, "%u %c%x %c%x %c", DBOX_VERSION, + str_printfa(hdr, "%u %c%x %c%x\n", DBOX_VERSION, DBOX_HEADER_MSG_HEADER_SIZE, (unsigned int)sizeof(struct dbox_message_header), - DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time, - DBOX_HEADER_APPEND_OFFSET); - file->append_offset_header_pos = str_len(hdr); - str_printfa(hdr, "%08x\n", 0); + DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time); file->file_header_size = str_len(hdr); file->msg_header_size = sizeof(struct dbox_message_header); file->append_offset = str_len(hdr); - hdrsize = t_strdup_printf("%08x", (unsigned int)file->append_offset); - buffer_write(hdr, file->append_offset_header_pos, hdrsize, 8); - if (o_stream_send(file->output, str_data(hdr), str_len(hdr)) < 0) { dbox_file_set_syscall_error(file, "write"); return -1; @@ -493,7 +512,7 @@ int dbox_file_open_or_create(struct dbox_file *file, bool read_header, *deleted_r = FALSE; - if (file->file_id == 0) { + if (file->file_id == 0 && file->uid == 0) { T_BEGIN { ret = dbox_file_create(file) < 0 ? -1 : 1; } T_END; @@ -516,8 +535,8 @@ int dbox_file_open_if_needed(struct dbox_file *file) ret = dbox_file_open_fd(file); } T_END; if (ret == 0) { - path = t_strdup_printf("%s/%s", file->mbox->path, file->fname); - mail_storage_set_critical(file->mbox->ibox.box.storage, + path = dbox_file_get_primary_path(file); + mail_storage_set_critical(&file->storage->storage, "open(%s) failed: %m", path); } return ret <= 0 ? -1 : 0; @@ -538,31 +557,27 @@ void dbox_file_close(struct dbox_file *file) const char *dbox_file_get_path(struct dbox_file *file) { - i_assert(file->current_path != NULL); - return file->current_path; } static int -dbox_file_get_maildir_data(struct dbox_file *file, uint32_t *uid_r, - uoff_t *physical_size_r) +dbox_file_get_maildir_data(struct dbox_file *file, uoff_t *physical_size_r) { struct stat st; - i_assert((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0); + i_assert(file->uid != 0); if (fstat(file->fd, &st) < 0) { dbox_file_set_syscall_error(file, "fstat"); return -1; } - *uid_r = file->file_id & ~DBOX_FILE_ID_FLAG_UID; *physical_size_r = st.st_size; return 1; } -static int dbox_file_read_mail_header(struct dbox_file *file, uint32_t *uid_r, - uoff_t *physical_size_r) +static int +dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r) { struct dbox_message_header hdr; const unsigned char *data; @@ -570,7 +585,7 @@ static int dbox_file_read_mail_header(struct dbox_file *file, uint32_t *uid_r, int ret; if (file->maildir_file) - return dbox_file_get_maildir_data(file, uid_r, physical_size_r); + return dbox_file_get_maildir_data(file, physical_size_r); ret = i_stream_read_data(file->input, &data, &size, file->msg_header_size - 1); @@ -591,17 +606,13 @@ static int dbox_file_read_mail_header(struct dbox_file *file, uint32_t *uid_r, return 0; } - /* Ignore the UID header with UID files */ - *uid_r = (file->file_id & DBOX_FILE_ID_FLAG_UID) != 0 ? - (file->file_id & ~DBOX_FILE_ID_FLAG_UID) : - hex2dec(hdr.uid_hex, sizeof(hdr.uid_hex)); *physical_size_r = hex2dec(hdr.message_size_hex, sizeof(hdr.message_size_hex)); return 1; } int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, - uint32_t *uid_r, uoff_t *physical_size_r, + uoff_t *physical_size_r, struct istream **stream_r, bool *expunged_r) { int ret; @@ -617,11 +628,10 @@ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, if (offset == 0) offset = file->file_header_size; - if (offset != file->cur_offset || file->cur_uid == 0) { + if (offset != file->cur_offset || file->cur_physical_size == 0) { file->cur_offset = offset; i_stream_seek(file->input, offset); - ret = dbox_file_read_mail_header(file, &file->cur_uid, - &file->cur_physical_size); + ret = dbox_file_read_mail_header(file, &file->cur_physical_size); if (ret <= 0) return ret; } @@ -630,14 +640,13 @@ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, *stream_r = i_stream_create_limit(file->input, file->cur_physical_size); } - *uid_r = file->cur_uid; *physical_size_r = file->cur_physical_size; return 1; } static int dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset, - uint32_t *uid_r, uoff_t *physical_size_r) + uoff_t *physical_size_r) { const char *line; int ret; @@ -656,42 +665,37 @@ dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset, (void)i_stream_read(file->input); if (!i_stream_have_bytes_left(file->input)) { - *uid_r = 0; *physical_size_r = 0; return 1; } - return dbox_file_read_mail_header(file, uid_r, physical_size_r); + return dbox_file_read_mail_header(file, physical_size_r); } int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset, - uint32_t *uid_r, uoff_t *physical_size_r) + uoff_t *physical_size_r) { - uint32_t uid; uoff_t size; bool first = *offset == 0; bool deleted; int ret; - ret = dbox_file_get_mail_stream(file, *offset, &uid, &size, NULL, + ret = dbox_file_get_mail_stream(file, *offset, &size, NULL, &deleted); if (ret <= 0) return ret; if (deleted) { - *uid_r = 0; *physical_size_r = 0; return 1; } if (first) { - *uid_r = uid; *physical_size_r = size; return 1; } i_stream_skip(file->input, size); - return dbox_file_seek_next_at_metadata(file, offset, uid_r, - physical_size_r); + return dbox_file_seek_next_at_metadata(file, offset, physical_size_r); } static int dbox_file_seek_append_pos(struct dbox_file *file, uoff_t mail_size) @@ -896,10 +900,9 @@ int dbox_file_metadata_seek_mail_offset(struct dbox_file *file, uoff_t offset, bool *expunged_r) { uoff_t physical_size, metadata_offset; - uint32_t uid; int ret; - ret = dbox_file_get_mail_stream(file, offset, &uid, &physical_size, + ret = dbox_file_get_mail_stream(file, offset, &physical_size, NULL, expunged_r); if (ret <= 0 || *expunged_r) return ret; @@ -950,36 +953,6 @@ int dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output) return 0; } -bool dbox_file_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view, - uint32_t seq, uint32_t *file_id_r, uoff_t *offset_r) -{ - const struct dbox_mail_index_record *dbox_rec; - const void *data; - uint32_t uid; - bool expunged; - - mail_index_lookup_ext(view, seq, mbox->dbox_ext_id, &data, &expunged); - if (expunged) - return FALSE; - dbox_rec = data; - - if (dbox_rec == NULL || dbox_rec->file_id == 0) { - mail_index_lookup_uid(view, seq, &uid); - if ((uid & DBOX_FILE_ID_FLAG_UID) != 0) { - /* something's broken, we can't handle this high UIDs */ - mail_storage_set_critical(mbox->ibox.box.storage, - "found too high uid=%u", uid); - return FALSE; - } - *file_id_r = DBOX_FILE_ID_FLAG_UID | uid; - *offset_r = 0; - } else { - *file_id_r = dbox_rec->file_id; - *offset_r = dbox_rec->offset; - } - return TRUE; -} - int dbox_file_move(struct dbox_file *file, bool alt_path) { struct ostream *output; @@ -998,7 +971,8 @@ int dbox_file_move(struct dbox_file *file, bool alt_path) return 0; } - dest_dir = alt_path ? file->mbox->alt_path : file->mbox->path; + dest_dir = !alt_path ? dbox_file_get_primary_path(file) : + dbox_file_get_alt_path(file); temp_path = t_strdup_printf("%s/%s", dest_dir, dbox_generate_tmp_filename()); @@ -1036,7 +1010,8 @@ int dbox_file_move(struct dbox_file *file, bool alt_path) } o_stream_unref(&output); - if (!file->mbox->ibox.fsync_disable && ret == 0) { + if ((file->storage->storage.flags & + MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0 && ret == 0) { if (fsync(out_fd) < 0) { i_error("fsync(%s) failed: %m", temp_path); ret = -1; @@ -1060,7 +1035,8 @@ int dbox_file_move(struct dbox_file *file, bool alt_path) (void)unlink(temp_path); return -1; } - if (!file->mbox->ibox.fsync_disable) { + if ((file->storage->storage.flags & + MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0) { if (fdatasync_path(dest_dir) < 0) { i_error("fdatasync(%s) failed: %m", dest_dir); (void)unlink(dest_path); @@ -1088,13 +1064,12 @@ int dbox_file_move(struct dbox_file *file, bool alt_path) } void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, - uint32_t uid, uoff_t message_size) + uoff_t message_size) { memset(dbox_msg_hdr, ' ', sizeof(*dbox_msg_hdr)); memcpy(dbox_msg_hdr->magic_pre, DBOX_MAGIC_PRE, sizeof(dbox_msg_hdr->magic_pre)); dbox_msg_hdr->type = DBOX_MESSAGE_TYPE_NORMAL; - dec2hex(dbox_msg_hdr->uid_hex, uid, sizeof(dbox_msg_hdr->uid_hex)); dec2hex(dbox_msg_hdr->message_size_hex, message_size, sizeof(dbox_msg_hdr->message_size_hex)); dbox_msg_hdr->save_lf = '\n'; diff --git a/src/lib-storage/index/dbox/dbox-file.h b/src/lib-storage/index/dbox/dbox-file.h index 5f63b95940..adc18acf39 100644 --- a/src/lib-storage/index/dbox/dbox-file.h +++ b/src/lib-storage/index/dbox/dbox-file.h @@ -16,19 +16,16 @@ There should be no duplicates for the current metadata, but future extensions may need them so they should be preserved. */ -#define DBOX_VERSION 1 +#define DBOX_VERSION 2 #define DBOX_MAGIC_PRE "\001\002" #define DBOX_MAGIC_POST "\n\001\003\n" -/* If file_id has this flag set, the file is a single file with file_id=UID. */ -#define DBOX_FILE_ID_FLAG_UID 0x80000000 - enum dbox_header_key { /* Offset for appending next message. In %08x format so it can be updated without moving data in header. If messages have been expunged and file must not be appended anymore, the value is filled with 'X'. */ - DBOX_HEADER_APPEND_OFFSET = 'A', + DBOX_HEADER_OLDV1_APPEND_OFFSET = 'A', /* Must be sizeof(struct dbox_message_header) when appending (hex) */ DBOX_HEADER_MSG_HEADER_SIZE = 'M', /* Creation UNIX timestamp (hex) */ @@ -47,9 +44,9 @@ enum dbox_metadata_flags { enum dbox_metadata_key { /* metadata used by old Dovecot versions */ - DBOX_METADATA_OLD_EXPUNGED = 'E', - DBOX_METADATA_OLD_FLAGS = 'F', - DBOX_METADATA_OLD_KEYWORDS = 'K', + DBOX_METADATA_OLDV1_EXPUNGED = 'E', + DBOX_METADATA_OLDV1_FLAGS = 'F', + DBOX_METADATA_OLDV1_KEYWORDS = 'K', /* Globally unique identifier for the message. Preserved when copying. */ @@ -83,7 +80,7 @@ struct dbox_message_header { unsigned char magic_pre[2]; unsigned char type; unsigned char space1; - unsigned char uid_hex[8]; + unsigned char oldv1_uid_hex[8]; unsigned char space2; unsigned char message_size_hex[16]; /* */ @@ -95,13 +92,16 @@ struct dbox_metadata_header { }; struct dbox_file { - struct dbox_mailbox *mbox; + struct dbox_storage *storage; + /* set only for single-msg-per-file */ + struct dbox_mailbox *single_mbox; + int refcount; - unsigned int file_id; + /* uid is for single-msg-per-file, file_id for multi-msgs-per-file */ + uint32_t uid, file_id; unsigned int file_header_size; unsigned int msg_header_size; - unsigned int append_offset_header_pos; unsigned int append_count; uint32_t last_append_uid; @@ -111,7 +111,6 @@ struct dbox_file { uoff_t output_stream_offset; uoff_t cur_offset; - uint32_t cur_uid; uoff_t cur_physical_size; char *fname; @@ -136,18 +135,21 @@ extern enum mail_flags dbox_mail_flags_map[DBOX_METADATA_FLAGS_COUNT]; extern char dbox_mail_flag_chars[DBOX_METADATA_FLAGS_COUNT]; struct dbox_file * -dbox_file_init(struct dbox_mailbox *mbox, unsigned int file_id); +dbox_file_init_single(struct dbox_mailbox *mbox, uint32_t uid); +struct dbox_file * +dbox_file_init_multi(struct dbox_storage *storage, uint32_t file_id); void dbox_file_unref(struct dbox_file **file); /* Free all currently opened files. */ -void dbox_files_free(struct dbox_mailbox *mbox); +void dbox_files_free(struct dbox_storage *storage); -/* Assign a newly created file (file_id=0) a new id. */ -int dbox_file_assign_id(struct dbox_file *file, unsigned int file_id); +/* Assign a newly created file a new id. For single files assign UID, + for multi files assign map UID. */ +int dbox_file_assign_id(struct dbox_file *file, uint32_t id); -/* Open the file if file_id is not 0, otherwise create it. Returns 1 if ok, - 0 if read_header=TRUE and opened file was broken, -1 if error. If file is - deleted, deleted_r=TRUE and 1 is returned. */ +/* Open the file if uid or file_id is not 0, otherwise create it. Returns 1 if + ok, 0 if read_header=TRUE and opened file was broken, -1 if error. If file + is deleted, deleted_r=TRUE and 1 is returned. */ int dbox_file_open_or_create(struct dbox_file *file, bool read_header, bool *deleted_r); /* Open the file's fd if it's currently closed. Assumes that the file exists. */ @@ -159,17 +161,17 @@ void dbox_file_close(struct dbox_file *file); call this function for a non-opened file. */ const char *dbox_file_get_path(struct dbox_file *file); -/* Seek to given offset in file and return the message's input stream, UID +/* Seek to given offset in file and return the message's input stream and physical size. Returns 1 if ok, 0 if file/offset is corrupted, -1 if I/O error. */ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, - uint32_t *uid_r, uoff_t *physical_size_r, + uoff_t *physical_size_r, struct istream **stream_r, bool *expunged_r); /* Seek to next message after given offset, or to first message if offset=0. - If there are no more messages, uid_r is set to 0. Returns 1 if ok, 0 if - file/offset is corrupted, -1 if I/O error. */ + If there are no more messages, physical_size_r is set to 0. Returns 1 if ok, + 0 if file/offset is corrupted, -1 if I/O error. */ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset, - uint32_t *uid_r, uoff_t *physical_size_r); + uoff_t *physical_size_r); /* Returns TRUE if mail_size bytes can be appended to the file. */ bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size); @@ -203,18 +205,15 @@ const char *dbox_file_metadata_get(struct dbox_file *file, /* Write all metadata to output stream. Returns 0 if ok, -1 if I/O error. */ int dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output); -/* Get file/offset for wanted message. Returns TRUE if found. */ -bool dbox_file_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view, - uint32_t seq, uint32_t *file_id_r, uoff_t *offset_r); - /* Move the file to alt path or back. */ int dbox_file_move(struct dbox_file *file, bool alt_path); -/* Fill dbox_message_header with given uid/size. */ +/* Fill dbox_message_header with given size. */ void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, - uint32_t uid, uoff_t message_size); + uoff_t message_size); -int dbox_create_fd(struct dbox_mailbox *mbox, const char *path); +const char *dbox_file_get_primary_path(struct dbox_file *file); +const char *dbox_file_get_alt_path(struct dbox_file *file); void dbox_file_set_syscall_error(struct dbox_file *file, const char *function); #endif diff --git a/src/lib-storage/index/dbox/dbox-index.h b/src/lib-storage/index/dbox/dbox-index.h deleted file mode 100644 index 721f834f9f..0000000000 --- a/src/lib-storage/index/dbox/dbox-index.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef DBOX_INDEX_H -#define DBOX_INDEX_H - -struct dbox_file; -struct dbox_index_append_context; - -struct dbox_index *dbox_index_init(struct dbox_mailbox *mbox); -void dbox_index_deinit(struct dbox_index **index); - -struct dbox_index_append_context * -dbox_index_append_begin(struct dbox_index *index); -/* Request file for saving a new message with given size. If an existing file - can be used, the record is locked and updated in index. Returns 0 if ok, - -1 if error. */ -int dbox_index_append_next(struct dbox_index_append_context *ctx, - uoff_t mail_size, - struct dbox_file **file_r, - struct ostream **output_r); -/* Assign file_ids to all appended files. */ -int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx); -/* Returns 0 if ok, -1 if error. */ -int dbox_index_append_commit(struct dbox_index_append_context **ctx); -void dbox_index_append_rollback(struct dbox_index_append_context **ctx); - -#endif diff --git a/src/lib-storage/index/dbox/dbox-mail.c b/src/lib-storage/index/dbox/dbox-mail.c index 29d4b21deb..73d15f7b15 100644 --- a/src/lib-storage/index/dbox/dbox-mail.c +++ b/src/lib-storage/index/dbox/dbox-mail.c @@ -6,6 +6,7 @@ #include "str.h" #include "index-mail.h" #include "dbox-storage.h" +#include "dbox-map.h" #include "dbox-file.h" #include @@ -43,20 +44,38 @@ static void dbox_mail_close(struct mail *_mail) index_mail_close(_mail); } -static int dbox_mail_lookup(struct dbox_mail *mail, - uoff_t *offset_r, struct dbox_file **file_r) +uint32_t dbox_mail_lookup(struct dbox_mailbox *mbox, + struct mail_index_view *view, uint32_t seq) +{ + const struct dbox_mail_index_record *dbox_rec; + const void *data; + bool expunged; + + mail_index_lookup_ext(view, seq, mbox->dbox_ext_id, &data, &expunged); + dbox_rec = data; + return dbox_rec == NULL ? 0 : dbox_rec->map_uid; +} + +static int dbox_mail_open(struct dbox_mail *mail, + uoff_t *offset_r, struct dbox_file **file_r) { struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox; - unsigned int file_id; + struct mail *_mail = &mail->imail.mail.mail; + uint32_t map_uid, file_id; if (mail->open_file == NULL) { - if (!dbox_file_lookup(mbox, mbox->ibox.view, - mail->imail.mail.mail.seq, - &file_id, &mail->offset)) { - mail_set_expunged(&mail->imail.mail.mail); + map_uid = dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq); + if (map_uid == 0) { + mail->open_file = + dbox_file_init_single(mbox, _mail->uid); + } else if (!dbox_map_lookup(mbox->storage->map_index, map_uid, + &file_id, &mail->offset)) { + mail_set_expunged(_mail); return -1; + } else { + mail->open_file = + dbox_file_init_multi(mbox->storage, file_id); } - mail->open_file = dbox_file_init(mbox, file_id); } *file_r = mail->open_file; @@ -71,7 +90,7 @@ dbox_mail_metadata_seek(struct dbox_mail *mail, struct dbox_file **file_r) bool expunged; int ret; - if (dbox_mail_lookup(mail, &offset, file_r) < 0) + if (dbox_mail_open(mail, &offset, file_r) < 0) return -1; ret = dbox_file_metadata_seek_mail_offset(*file_r, offset, &expunged); @@ -239,15 +258,14 @@ dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, struct index_mail_data *data = &mail->imail.data; struct istream *input; uoff_t offset, size; - uint32_t uid = 0; bool expunged; int ret; if (data->stream == NULL) { - if (dbox_mail_lookup(mail, &offset, &mail->open_file) < 0) + if (dbox_mail_open(mail, &offset, &mail->open_file) < 0) return -1; - ret = dbox_file_get_mail_stream(mail->open_file, offset, &uid, + ret = dbox_file_get_mail_stream(mail->open_file, offset, &size, &input, &expunged); if (ret < 0) return -1; @@ -255,13 +273,13 @@ dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, mail_set_expunged(_mail); return -1; } - if (ret == 0 || uid != _mail->uid) { + if (ret == 0) { /* FIXME: broken file/offset */ if (ret > 0) i_stream_unref(&input); mail_storage_set_critical(_mail->box->storage, - "broken pointer to dbox file %s (uid %u vs %u)", - mail->open_file->current_path, uid, _mail->uid); + "broken pointer to dbox file %s", + mail->open_file->current_path); return -1; } data->physical_size = size; diff --git a/src/lib-storage/index/dbox/dbox-index.c b/src/lib-storage/index/dbox/dbox-map.c similarity index 59% rename from src/lib-storage/index/dbox/dbox-index.c rename to src/lib-storage/index/dbox/dbox-map.c index 8790ee8dbe..b24c47b1a2 100644 --- a/src/lib-storage/index/dbox/dbox-index.c +++ b/src/lib-storage/index/dbox/dbox-map.c @@ -4,14 +4,16 @@ #include "array.h" #include "dbox-storage.h" #include "dbox-file.h" -#include "dbox-index.h" +#include "dbox-map.h" -struct dbox_index { - struct dbox_mailbox *mbox; +struct dbox_map { + struct dbox_storage *storage; + struct mail_index *index; }; -struct dbox_index_append_context { - struct dbox_index *index; +struct dbox_map_append_context { + struct dbox_mailbox *mbox; + ARRAY_DEFINE(files, struct dbox_file *); uoff_t output_offset; @@ -21,40 +23,47 @@ struct dbox_index_append_context { unsigned int locked_header:1; }; -struct dbox_index *dbox_index_init(struct dbox_mailbox *mbox) +struct dbox_map *dbox_map_init(struct dbox_storage *storage) { - struct dbox_index *index; + struct dbox_map *map; - index = i_new(struct dbox_index, 1); - index->mbox = mbox; - return index; + map = i_new(struct dbox_map, 1); + map->storage = storage; + map->index = mail_index_alloc(storage->storage_dir, + DBOX_GLOBAL_INDEX_PREFIX); + return map; } -void dbox_index_deinit(struct dbox_index **_index) +void dbox_map_deinit(struct dbox_map **_map) { - struct dbox_index *index = *_index; + struct dbox_map *map = *_map; - *_index = NULL; + *_map = NULL; - i_free(index); + mail_index_free(&map->index); + i_free(map); } -struct dbox_index_append_context * -dbox_index_append_begin(struct dbox_index *index) +bool dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r) { - struct dbox_index_append_context *ctx; + return FALSE; +} - ctx = i_new(struct dbox_index_append_context, 1); - ctx->index = index; +struct dbox_map_append_context * +dbox_map_append_begin(struct dbox_mailbox *mbox) +{ + struct dbox_map_append_context *ctx; + + ctx = i_new(struct dbox_map_append_context, 1); + ctx->mbox = mbox; ctx->first_new_file_id = (uint32_t)-1; i_array_init(&ctx->files, 64); return ctx; } -int dbox_index_append_next(struct dbox_index_append_context *ctx, - uoff_t mail_size, - struct dbox_file **file_r, - struct ostream **output_r) +int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size, + struct dbox_file **file_r, struct ostream **output_r) { struct dbox_file *const *files, *file = NULL; unsigned int i, count; @@ -74,7 +83,7 @@ int dbox_index_append_next(struct dbox_index_append_context *ctx, if (file == NULL) { /* create a new file */ - file = dbox_file_init(ctx->index->mbox, 0); + file = dbox_file_init_single(ctx->mbox, 0); if ((ret = dbox_file_get_append_stream(file, mail_size, output_r)) <= 0) { i_assert(ret < 0); @@ -89,26 +98,23 @@ int dbox_index_append_next(struct dbox_index_append_context *ctx, return 0; } -static int dbox_index_append_commit_new(struct dbox_index_append_context *ctx, - struct dbox_file *file) +static int dbox_map_append_commit_new(struct dbox_map_append_context *ctx, + struct dbox_file *file) { - unsigned int file_id; - i_assert(!file->maildir_file); i_assert(file->append_count > 0); - if (file->append_count == 1 && !dbox_file_can_append(file, 0)) { + if (file->single_mbox != NULL) { /* single UID message file */ i_assert(file->last_append_uid != 0); - file_id = file->last_append_uid | DBOX_FILE_ID_FLAG_UID; - return dbox_file_assign_id(file, file_id); + return dbox_file_assign_id(file, file->last_append_uid); } /* FIXME */ return -1; } -int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx) +int dbox_map_append_assign_file_ids(struct dbox_map_append_context *ctx) { struct dbox_file *const *files, *file; unsigned int i, count; @@ -118,8 +124,8 @@ int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx) for (i = 0; i < count; i++) { file = files[i]; - if (file->file_id == 0) { - if (dbox_index_append_commit_new(ctx, file) < 0) { + if (file->uid == 0 && file->file_id == 0) { + if (dbox_map_append_commit_new(ctx, file) < 0) { ret = -1; break; } @@ -132,9 +138,9 @@ int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx) return ret; } -int dbox_index_append_commit(struct dbox_index_append_context **_ctx) +int dbox_map_append_commit(struct dbox_map_append_context **_ctx) { - struct dbox_index_append_context *ctx = *_ctx; + struct dbox_map_append_context *ctx = *_ctx; struct dbox_file **files; unsigned int i, count; @@ -149,9 +155,9 @@ int dbox_index_append_commit(struct dbox_index_append_context **_ctx) return 0; } -void dbox_index_append_rollback(struct dbox_index_append_context **_ctx) +void dbox_map_append_rollback(struct dbox_map_append_context **_ctx) { - struct dbox_index_append_context *ctx = *_ctx; + struct dbox_map_append_context *ctx = *_ctx; struct dbox_file *const *files, *file; unsigned int i, count; diff --git a/src/lib-storage/index/dbox/dbox-map.h b/src/lib-storage/index/dbox/dbox-map.h new file mode 100644 index 0000000000..d1e67fcf2e --- /dev/null +++ b/src/lib-storage/index/dbox/dbox-map.h @@ -0,0 +1,27 @@ +#ifndef DBOX_MAP_H +#define DBOX_MAP_H + +struct dbox_storage; +struct dbox_file; +struct dbox_map_append_context; + +struct dbox_map *dbox_map_init(struct dbox_storage *storage); +void dbox_map_deinit(struct dbox_map **map); + +bool dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r); + +struct dbox_map_append_context * +dbox_map_append_begin(struct dbox_mailbox *mbox); +/* Request file for saving a new message with given size. If an existing file + can be used, the record is locked and updated in index. Returns 0 if ok, + -1 if error. */ +int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size, + struct dbox_file **file_r, struct ostream **output_r); +/* Assign file_ids to all appended files. */ +int dbox_map_append_assign_file_ids(struct dbox_map_append_context *ctx); +/* Returns 0 if ok, -1 if error. */ +int dbox_map_append_commit(struct dbox_map_append_context **ctx); +void dbox_map_append_rollback(struct dbox_map_append_context **ctx); + +#endif diff --git a/src/lib-storage/index/dbox/dbox-save.c b/src/lib-storage/index/dbox/dbox-save.c index d8063290a9..579c5e99d5 100644 --- a/src/lib-storage/index/dbox/dbox-save.c +++ b/src/lib-storage/index/dbox/dbox-save.c @@ -11,7 +11,7 @@ #include "write-full.h" #include "index-mail.h" #include "dbox-storage.h" -#include "dbox-index.h" +#include "dbox-map.h" #include "dbox-file.h" #include "dbox-sync.h" @@ -30,7 +30,7 @@ struct dbox_save_context { struct dbox_mailbox *mbox; struct mail_index_transaction *trans; - struct dbox_index_append_context *append_ctx; + struct dbox_map_append_context *append_ctx; struct dbox_sync_context *sync_ctx; /* updated for each appended mail: */ @@ -65,7 +65,7 @@ dbox_save_alloc(struct mailbox_transaction_context *_t) ctx->ctx.transaction = &t->ictx.mailbox_ctx; ctx->mbox = mbox; ctx->trans = t->ictx.trans; - ctx->append_ctx = dbox_index_append_begin(mbox->dbox_index); + ctx->append_ctx = dbox_map_append_begin(mbox); i_array_init(&ctx->mails, 32); return &ctx->ctx; } @@ -84,8 +84,8 @@ int dbox_save_begin(struct mail_save_context *_ctx, struct istream *input) st = i_stream_stat(input, TRUE); mail_size = st == NULL || st->st_size == -1 ? 0 : st->st_size; - if (dbox_index_append_next(ctx->append_ctx, mail_size, - &ctx->cur_file, &ctx->cur_output) < 0) { + if (dbox_map_append_next(ctx->append_ctx, mail_size, + &ctx->cur_file, &ctx->cur_output) < 0) { ctx->failed = TRUE; return -1; } @@ -199,15 +199,15 @@ static int dbox_save_mail_write_header(struct dbox_save_mail *mail) i_assert(mail->file->msg_header_size == sizeof(dbox_msg_hdr)); - mail->file->last_append_uid = mail->uid; - dbox_msg_header_fill(&dbox_msg_hdr, mail->uid, mail->message_size); - + dbox_msg_header_fill(&dbox_msg_hdr, mail->message_size); if (pwrite_full(mail->file->fd, &dbox_msg_hdr, sizeof(dbox_msg_hdr), mail->append_offset) < 0) { dbox_file_set_syscall_error(mail->file, "write"); return -1; } - if (!mail->file->mbox->ibox.fsync_disable) { + // FIXME: don't fsync here. especially with multi files + if ((mail->file->storage->storage.flags & + MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0) { if (fdatasync(mail->file->fd) < 0) { dbox_file_set_syscall_error(mail->file, "fdatasync"); return -1; @@ -258,12 +258,10 @@ static int dbox_save_finish_write(struct mail_save_context *_ctx) dbox_file_finish_append(save_mail->file); save_mail->message_size = offset - save_mail->append_offset - save_mail->file->msg_header_size; + dbox_save_mail_write_header(save_mail); - if (save_mail->file->append_count == 1 && - !dbox_file_can_append(save_mail->file, 0)) { - dbox_save_mail_write_header(save_mail); + if (save_mail->file->single_mbox != NULL) dbox_file_close(save_mail->file); - } return 0; } } @@ -285,118 +283,32 @@ void dbox_save_cancel(struct mail_save_context *_ctx) (void)dbox_save_finish(_ctx); } -static int -dbox_save_file_write_append_offset(struct dbox_file *file, uoff_t append_offset) -{ - char buf[8+1]; - - i_assert(append_offset <= (uint32_t)-1); - - i_snprintf(buf, sizeof(buf), "%08x", (unsigned int)append_offset); - if (pwrite_full(file->fd, buf, sizeof(buf)-1, - file->append_offset_header_pos) < 0) { - dbox_file_set_syscall_error(file, "pwrite"); - return -1; - } - return 0; -} - -static int dbox_save_file_commit_header(struct dbox_save_mail *mail) -{ - uoff_t append_offset; - - append_offset = dbox_file_get_next_append_offset(mail->file); - return dbox_save_file_write_append_offset(mail->file, append_offset); -} - -static void dbox_save_file_uncommit_header(struct dbox_save_mail *mail) -{ - if (mail->file->file_id == 0) { - /* temporary file, we'll just unlink it later */ - return; - } - (void)dbox_save_file_write_append_offset(mail->file, - mail->append_offset); -} - -static int dbox_save_mail_file_cmp(const void *p1, const void *p2) -{ - const struct dbox_save_mail *m1 = p1, *m2 = p2; - int ret; - - ret = strcmp(m1->file->fname, m2->file->fname); - if (ret == 0) { - /* the oldest sequence is first. this is needed for uncommit - to work right. */ - ret = (int)m1->seq - (int)m2->seq; - } - return ret; -} - static int dbox_save_commit(struct dbox_save_context *ctx, uint32_t first_uid) { struct dbox_mail_index_record rec; struct dbox_save_mail *mails; unsigned int i, count; - int ret = 0; /* assign UIDs to mails */ mails = array_get_modifiable(&ctx->mails, &count); - for (i = 0; i < count; i++) - mails[i].uid = first_uid++; - - /* update headers */ - qsort(mails, count, sizeof(*mails), dbox_save_mail_file_cmp); for (i = 0; i < count; i++) { + mails[i].uid = first_uid++; mails[i].file->last_append_uid = mails[i].uid; - if (mails[i].file->append_count == 1 && - !dbox_file_can_append(mails[i].file, 0)) { - /* UID file - there's no need to write it to the - header */ - continue; - } - - if (dbox_file_open_if_needed(mails[i].file) < 0 || - dbox_save_mail_write_header(&mails[i]) < 0) { - ret = -1; - break; - } - - /* write file header only once after all mails headers - have been written */ - if (i+1 == count || mails[i].file != mails[i+1].file) { - if (dbox_save_file_commit_header(&mails[i]) < 0) { - ret = -1; - break; - } - dbox_file_close(mails[i].file); - } } - if (ret < 0) { - /* have to uncommit all written changes */ - for (; i > 0; i--) { - if (i > 1 && mails[i-2].file == mails[i-1].file) - continue; - dbox_save_file_uncommit_header(&mails[i-1]); - } - return -1; - } - - /* set file_id / offsets to records */ - if (dbox_index_append_assign_file_ids(ctx->append_ctx) < 0) + if (dbox_map_append_assign_file_ids(ctx->append_ctx) < 0) return -1; memset(&rec, 0, sizeof(rec)); +#if 0 //FIXME for (i = 0; i < count; i++) { - rec.file_id = mails[i].file->file_id; - rec.offset = mails[i].append_offset; - - if ((rec.file_id & DBOX_FILE_ID_FLAG_UID) == 0) { + rec.map_uid = mails[i].map_uid; + if (rec.map_uid != 0) { mail_index_update_ext(ctx->trans, mails[i].seq, ctx->mbox->dbox_ext_id, &rec, NULL); } } +#endif return 0; } @@ -429,7 +341,7 @@ int dbox_transaction_save_commit_pre(struct dbox_save_context *ctx) *t->ictx.first_saved_uid = uid; *t->ictx.last_saved_uid = next_uid - 1; - dbox_index_append_commit(&ctx->append_ctx); + dbox_map_append_commit(&ctx->append_ctx); if (ctx->mail != NULL) mail_free(&ctx->mail); @@ -456,7 +368,7 @@ void dbox_transaction_save_rollback(struct dbox_save_context *ctx) if (!ctx->finished) dbox_save_cancel(&ctx->ctx); if (ctx->append_ctx != NULL) - dbox_index_append_rollback(&ctx->append_ctx); + dbox_map_append_rollback(&ctx->append_ctx); if (ctx->sync_ctx != NULL) (void)dbox_sync_finish(&ctx->sync_ctx, FALSE); diff --git a/src/lib-storage/index/dbox/dbox-storage.c b/src/lib-storage/index/dbox/dbox-storage.c index eea14ea46c..183785186e 100644 --- a/src/lib-storage/index/dbox/dbox-storage.c +++ b/src/lib-storage/index/dbox/dbox-storage.c @@ -10,7 +10,7 @@ #include "mail-copy.h" #include "maildir/maildir-uidlist.h" #include "dbox-sync.h" -#include "dbox-index.h" +#include "dbox-map.h" #include "dbox-file.h" #include "dbox-storage.h" @@ -93,7 +93,7 @@ static int dbox_create(struct mail_storage *_storage, const char *data, struct dbox_storage *storage = (struct dbox_storage *)_storage; struct mailbox_list_settings list_set; struct stat st; - const char *layout, *alt_dir; + const char *layout, *alt_dir, *dir, *value; if (dbox_get_list_settings(&list_set, data, _storage, &layout, &alt_dir, error_r) < 0) @@ -142,15 +142,62 @@ static int dbox_create(struct mail_storage *_storage, const char *data, _storage->list->v.rename_mailbox = dbox_list_rename_mailbox; _storage->list->v.rename_mailbox_pre = dbox_list_rename_mailbox_pre; + value = getenv("DBOX_ROTATE_SIZE"); + if (value != NULL) + storage->rotate_size = (uoff_t)strtoul(value, NULL, 10) * 1024; + else + storage->rotate_size = DBOX_DEFAULT_ROTATE_SIZE; + value = getenv("DBOX_ROTATE_MIN_SIZE"); + if (value != NULL) + storage->rotate_min_size = (uoff_t)strtoul(value, NULL, 10) * 1024; + else + storage->rotate_min_size = DBOX_DEFAULT_ROTATE_MIN_SIZE; + if (storage->rotate_min_size > storage->rotate_size) + storage->rotate_min_size = storage->rotate_size; + value = getenv("DBOX_ROTATE_DAYS"); + if (value != NULL) + storage->rotate_days = (unsigned int)strtoul(value, NULL, 10); + else + storage->rotate_days = DBOX_DEFAULT_ROTATE_DAYS; + + value = getenv("DBOX_MAX_OPEN_FILES"); + if (value != NULL) + storage->max_open_files = (unsigned int)strtoul(value, NULL, 10); + else + storage->max_open_files = DBOX_DEFAULT_MAX_OPEN_FILES; + MODULE_CONTEXT_SET_FULL(_storage->list, dbox_mailbox_list_module, storage, &storage->list_module_ctx); /* finish list init after we've overridden vfuncs */ mailbox_list_init(_storage->list, _storage->ns, &list_set, mail_storage_get_list_flags(_storage->flags)); + + dir = mailbox_list_get_path(storage->storage.list, NULL, + MAILBOX_LIST_PATH_TYPE_DIR); + storage->storage_dir = p_strconcat(_storage->pool, dir, + "/"DBOX_GLOBAL_DIR_NAME, NULL); + storage->alt_storage_dir = p_strconcat(_storage->pool, alt_dir, + "/"DBOX_GLOBAL_DIR_NAME, NULL); + i_array_init(&storage->open_files, I_MIN(storage->max_open_files, 128)); + + storage->map_index = dbox_map_init(storage); + mailbox_list_get_permissions(storage->storage.list, + &storage->create_mode, + &storage->create_gid); return 0; } +static void dbox_destroy(struct mail_storage *_storage) +{ + struct dbox_storage *storage = (struct dbox_storage *)_storage; + + dbox_files_free(storage); + dbox_map_deinit(&storage->map_index); + array_free(&storage->open_files); + index_storage_destroy(_storage); +} + static int create_dbox(struct mail_storage *storage, const char *path) { mode_t mode; @@ -195,7 +242,7 @@ dbox_open(struct dbox_storage *storage, const char *name, struct mail_storage *_storage = &storage->storage; struct dbox_mailbox *mbox; struct mail_index *index; - const char *path, *value; + const char *path; pool_t pool; path = mailbox_list_get_path(_storage->list, name, @@ -216,32 +263,6 @@ dbox_open(struct dbox_storage *storage, const char *name, mbox->alt_path = p_strdup(pool, dbox_get_alt_path(storage, path)); mbox->storage = storage; - value = getenv("DBOX_ROTATE_SIZE"); - if (value != NULL) - mbox->rotate_size = (uoff_t)strtoul(value, NULL, 10) * 1024; - else - mbox->rotate_size = DBOX_DEFAULT_ROTATE_SIZE; - mbox->rotate_size = 0; /* FIXME: currently anything else doesn't work */ - value = getenv("DBOX_ROTATE_MIN_SIZE"); - if (value != NULL) - mbox->rotate_min_size = (uoff_t)strtoul(value, NULL, 10) * 1024; - else - mbox->rotate_min_size = DBOX_DEFAULT_ROTATE_MIN_SIZE; - if (mbox->rotate_min_size > mbox->rotate_size) - mbox->rotate_min_size = mbox->rotate_size; - value = getenv("DBOX_ROTATE_DAYS"); - if (value != NULL) - mbox->rotate_days = (unsigned int)strtoul(value, NULL, 10); - else - mbox->rotate_days = DBOX_DEFAULT_ROTATE_DAYS; - - value = getenv("DBOX_MAX_OPEN_FILES"); - if (value != NULL) - mbox->max_open_files = (unsigned int)strtoul(value, NULL, 10); - else - mbox->max_open_files = DBOX_DEFAULT_MAX_OPEN_FILES; - i_array_init(&mbox->open_files, I_MIN(mbox->max_open_files, 128)); - mbox->dbox_ext_id = mail_index_ext_register(index, "dbox", 0, sizeof(struct dbox_mail_index_record), @@ -249,7 +270,6 @@ dbox_open(struct dbox_storage *storage, const char *name, mbox->dbox_hdr_ext_id = mail_index_ext_register(index, "dbox-hdr", sizeof(struct dbox_index_header), 0, 0); - mbox->dbox_index = dbox_index_init(mbox); index_storage_mailbox_init(&mbox->ibox, name, flags, FALSE); mbox->maildir_uidlist = maildir_uidlist_init_readonly(&mbox->ibox); @@ -322,10 +342,6 @@ static int dbox_storage_mailbox_close(struct mailbox *box) struct dbox_mailbox *mbox = (struct dbox_mailbox *)box; maildir_uidlist_deinit(&mbox->maildir_uidlist); - dbox_index_deinit(&mbox->dbox_index); - dbox_files_free(mbox); - array_free(&mbox->open_files); - return index_storage_mailbox_close(box); } @@ -679,7 +695,7 @@ struct mail_storage dbox_storage = { dbox_class_deinit, dbox_alloc, dbox_create, - index_storage_destroy, + dbox_destroy, NULL, dbox_mailbox_open, dbox_mailbox_create diff --git a/src/lib-storage/index/dbox/dbox-storage.h b/src/lib-storage/index/dbox/dbox-storage.h index d5d20b57e6..7b729c5dd9 100644 --- a/src/lib-storage/index/dbox/dbox-storage.h +++ b/src/lib-storage/index/dbox/dbox-storage.h @@ -10,6 +10,8 @@ #define DBOX_INDEX_PREFIX "dovecot.index" #define DBOX_MAILDIR_NAME "dbox-Mails" +#define DBOX_GLOBAL_INDEX_PREFIX "dovecot.map.index" +#define DBOX_GLOBAL_DIR_NAME "storage" #define DBOX_MAIL_FILE_MULTI_PREFIX "m." #define DBOX_MAIL_FILE_UID_PREFIX "u." #define DBOX_MAIL_FILE_MULTI_FORMAT DBOX_MAIL_FILE_MULTI_PREFIX"%u" @@ -37,12 +39,25 @@ struct dbox_index_header { struct dbox_storage { struct mail_storage storage; union mailbox_list_module_context list_module_ctx; + + /* root path for alt directory */ const char *alt_dir; + /* paths for storage directories */ + const char *storage_dir, *alt_storage_dir; + struct dbox_map *map_index; + + /* mode/gid to use for new dbox storage files */ + mode_t create_mode; + gid_t create_gid; + + uoff_t rotate_size, rotate_min_size; + unsigned int rotate_days; + unsigned int max_open_files; + ARRAY_DEFINE(open_files, struct dbox_file *); }; struct dbox_mail_index_record { - uint32_t file_id; - uint32_t offset; + uint32_t map_uid; }; struct dbox_mailbox { @@ -52,15 +67,8 @@ struct dbox_mailbox { struct maildir_uidlist *maildir_uidlist; uint32_t highest_maildir_uid; - struct dbox_index *dbox_index; uint32_t dbox_ext_id, dbox_hdr_ext_id; - uoff_t rotate_size, rotate_min_size; - unsigned int rotate_days; - - ARRAY_DEFINE(open_files, struct dbox_file *); - unsigned int max_open_files; - const char *path, *alt_path; }; @@ -82,6 +90,10 @@ dbox_mail_alloc(struct mailbox_transaction_context *t, enum mail_fetch_field wanted_fields, struct mailbox_header_lookup_ctx *wanted_headers); +/* Get map_uid for wanted message. */ +uint32_t dbox_mail_lookup(struct dbox_mailbox *mbox, + struct mail_index_view *view, uint32_t seq); + struct mail_save_context * dbox_save_alloc(struct mailbox_transaction_context *_t); int dbox_save_begin(struct mail_save_context *ctx, struct istream *input); diff --git a/src/lib-storage/index/dbox/dbox-sync-file.c b/src/lib-storage/index/dbox/dbox-sync-file.c index 4654bd30a2..1f28a3d5a2 100644 --- a/src/lib-storage/index/dbox/dbox-sync-file.c +++ b/src/lib-storage/index/dbox/dbox-sync-file.c @@ -11,27 +11,26 @@ static int dbox_sync_file_unlink(struct dbox_file *file) { - const char *path; - int i = 0; + const char *path, *primary_path; + bool alt = FALSE; - path = t_strdup_printf("%s/%s", file->mbox->path, file->fname); + path = primary_path = dbox_file_get_primary_path(file); while (unlink(path) < 0) { if (errno != ENOENT) { - mail_storage_set_critical(file->mbox->ibox.box.storage, + mail_storage_set_critical(&file->storage->storage, "unlink(%s) failed: %m", path); return -1; } - if (file->mbox->alt_path == NULL || i == 1) { + if (file->storage->alt_storage_dir == NULL || alt) { /* not found */ i_warning("dbox: File unexpectedly lost: %s/%s", - file->mbox->path, file->fname); + primary_path, file->fname); return 0; } /* try the alternative path */ - path = t_strdup_printf("%s/%s", file->mbox->alt_path, - file->fname); - i++; + path = dbox_file_get_alt_path(file); + alt = TRUE; } return 1; } @@ -40,6 +39,7 @@ static int dbox_sync_file_expunge(struct dbox_sync_context *ctx, struct dbox_file *file, const struct dbox_sync_file_entry *entry) { +#if 0 //FIXME const struct seq_range *expunges; struct dbox_file *out_file = NULL; struct istream *input; @@ -62,10 +62,10 @@ dbox_sync_file_expunge(struct dbox_sync_context *ctx, struct dbox_file *file, offset = first_offset; for (i = 0;;) { - if ((ret = dbox_file_seek_next(file, &offset, &uid, + if ((ret = dbox_file_seek_next(file, &offset, &physical_size)) <= 0) break; - if (uid == 0) { + if (physical_size == 0) { /* EOF */ break; } @@ -142,6 +142,8 @@ dbox_sync_file_expunge(struct dbox_sync_context *ctx, struct dbox_file *file, if (out_file != NULL) dbox_file_unref(&out_file); return ret; +#endif + return -1; } #if 0 @@ -162,7 +164,6 @@ dbox_sync_file_split(struct dbox_sync_context *ctx, struct dbox_file *in_file, struct dbox_message_header dbox_msg_hdr; struct dbox_mail_index_record rec; const char *out_path, *value; - uint32_t uid; uoff_t size, append_offset; unsigned int i; int ret; @@ -171,7 +172,7 @@ dbox_sync_file_split(struct dbox_sync_context *ctx, struct dbox_file *in_file, /* FIXME: for now we handle only maildir file conversion */ i_assert(in_file->maildir_file); - ret = dbox_file_get_mail_stream(in_file, offset, &uid, + ret = dbox_file_get_mail_stream(in_file, offset, &size, &input, &expunged); if (ret <= 0) return ret; @@ -188,7 +189,7 @@ dbox_sync_file_split(struct dbox_sync_context *ctx, struct dbox_file *in_file, return -1; } append_offset = output->offset; - dbox_msg_header_fill(&dbox_msg_hdr, uid, size); + dbox_msg_header_fill(&dbox_msg_hdr, size); /* set static metadata */ for (i = 0; i < N_ELEMENTS(maildir_metadata_keys); i++) { @@ -280,9 +281,10 @@ int dbox_sync_file(struct dbox_sync_context *ctx, bool deleted; int ret; - file = dbox_file_init(ctx->mbox, entry->file_id); - if ((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0 && - array_is_created(&entry->expunges)) { + file = entry->file_id != 0 ? + dbox_file_init_multi(ctx->mbox->storage, entry->file_id) : + dbox_file_init_single(ctx->mbox, entry->uid); + if (entry->uid != 0 && array_is_created(&entry->expunges)) { /* fast path to expunging the whole file */ if ((ret = dbox_sync_file_unlink(file)) == 0) { /* file was lost, delete it */ diff --git a/src/lib-storage/index/dbox/dbox-sync-rebuild.c b/src/lib-storage/index/dbox/dbox-sync-rebuild.c index 5551310e11..640720909b 100644 --- a/src/lib-storage/index/dbox/dbox-sync-rebuild.c +++ b/src/lib-storage/index/dbox/dbox-sync-rebuild.c @@ -117,7 +117,6 @@ static void dbox_sync_index_copy_from_maildir(struct dbox_sync_rebuild_context *ctx, struct dbox_file *file, uint32_t seq) { - struct dbox_mailbox *mbox = file->mbox; ARRAY_TYPE(keyword_indexes) keyword_indexes; struct mail_keywords *keywords; enum mail_flags flags; @@ -127,7 +126,7 @@ dbox_sync_index_copy_from_maildir(struct dbox_sync_rebuild_context *ctx, file->fname, &flags, &keyword_indexes); mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags); - keywords = mail_index_keywords_create_from_indexes(mbox->ibox.index, + keywords = mail_index_keywords_create_from_indexes(ctx->mbox->ibox.index, &keyword_indexes); mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, keywords); mail_index_keywords_free(&keywords); @@ -153,36 +152,31 @@ dbox_sync_index_metadata(struct dbox_sync_rebuild_context *ctx, static int dbox_sync_index_file_next(struct dbox_sync_rebuild_context *ctx, struct dbox_file *file, uoff_t *offset) { - uint32_t seq, uid; + uint32_t seq; uoff_t physical_size; const char *path; bool expunged; int ret; path = dbox_file_get_path(file); - ret = dbox_file_seek_next(file, offset, &uid, &physical_size); + ret = dbox_file_seek_next(file, offset, &physical_size); if (ret <= 0) { if (ret < 0) return -1; - if (uid == 0 && (file->file_id & DBOX_FILE_ID_FLAG_UID) != 0) { +#if 0 //FIXME: needed? + if (physical_size == 0 && file->uid != 0) { /* EOF */ return 0; } +#endif i_warning("%s: Ignoring broken file (header)", path); return 0; } - if ((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0 && - uid != (file->file_id & ~DBOX_FILE_ID_FLAG_UID)) { - i_warning("%s: Header contains wrong UID %u", path, uid); - return 0; - } if (file->maildir_file) { - i_assert(uid != 0); - file->append_count = 1; - file->last_append_uid = uid; + file->last_append_uid = file->uid; } ret = dbox_file_metadata_seek_mail_offset(file, *offset, &expunged); @@ -193,8 +187,9 @@ static int dbox_sync_index_file_next(struct dbox_sync_rebuild_context *ctx, return 0; } if (!expunged) { - mail_index_append(ctx->trans, uid, &seq); - dbox_sync_index_metadata(ctx, file, seq, uid); + /* FIXME: file->uid doesn't work for multi files */ + mail_index_append(ctx->trans, file->uid, &seq); + dbox_sync_index_metadata(ctx, file, seq, file->uid); } return 1; } @@ -220,7 +215,7 @@ dbox_sync_index_uid_file(struct dbox_sync_rebuild_context *ctx, if (ctx->highest_uid < uid) ctx->highest_uid = uid; - file = dbox_file_init(ctx->mbox, uid | DBOX_FILE_ID_FLAG_UID); + file = dbox_file_init_single(ctx->mbox, uid); file->current_path = i_strdup_printf("%s/%s", dir, fname); ret = dbox_sync_index_file_next(ctx, file, &offset) < 0 ? -1 : 0; @@ -366,7 +361,7 @@ static int dbox_sync_maildir_finish(struct dbox_sync_rebuild_context *ctx) iter = maildir_uidlist_iter_init(mbox->maildir_uidlist); while (maildir_uidlist_iter_next(iter, &uid, &flags, &fname)) { - file = dbox_file_init(mbox, uid | DBOX_FILE_ID_FLAG_UID); + file = dbox_file_init_single(mbox, uid); file->current_path = i_strdup_printf("%s/%s", ctx->mbox->path, fname); diff --git a/src/lib-storage/index/dbox/dbox-sync.c b/src/lib-storage/index/dbox/dbox-sync.c index 9243aa1540..7c5b6f8662 100644 --- a/src/lib-storage/index/dbox/dbox-sync.c +++ b/src/lib-storage/index/dbox/dbox-sync.c @@ -6,39 +6,68 @@ #include "str.h" #include "hash.h" #include "dbox-storage.h" +#include "dbox-map.h" #include "dbox-file.h" #include "dbox-sync.h" #define DBOX_REBUILD_COUNT 3 +static unsigned int dbox_sync_file_entry_hash(const void *p) +{ + const struct dbox_sync_file_entry *entry = p; + + if (entry->file_id != 0) + return entry->file_id | 0x80000000; + else + return entry->uid; +} + +static int dbox_sync_file_entry_cmp(const void *p1, const void *p2) +{ + const struct dbox_sync_file_entry *entry1 = p1, *entry2 = p2; + + /* this is only for hashing, don't bother ever returning 1. */ + if (entry1->file_id != entry2->file_id) + return -1; + if (entry1->uid != entry2->uid) + return -1; + return 0; +} + static int dbox_sync_add_seq(struct dbox_sync_context *ctx, const struct mail_index_sync_rec *sync_rec, uint32_t seq) { - struct dbox_sync_file_entry *entry; - uint32_t file_id; + struct dbox_sync_file_entry *entry, lookup_entry; + uint32_t map_uid; uoff_t offset; - bool uid_file; i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE || sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); - if (!dbox_file_lookup(ctx->mbox, ctx->sync_view, seq, - &file_id, &offset)) - return -1; + memset(&lookup_entry, 0, sizeof(lookup_entry)); + map_uid = dbox_mail_lookup(ctx->mbox, ctx->sync_view, seq); + if (map_uid == 0) + mail_index_lookup_uid(ctx->sync_view, seq, &lookup_entry.uid); + else { + if (!dbox_map_lookup(ctx->mbox->storage->map_index, + map_uid, &lookup_entry.file_id, &offset)) { + // FIXME: now what? + return -1; + } + } - entry = hash_table_lookup(ctx->syncs, POINTER_CAST(file_id)); + entry = hash_table_lookup(ctx->syncs, &lookup_entry); if (entry == NULL) { entry = p_new(ctx->pool, struct dbox_sync_file_entry, 1); - entry->file_id = file_id; - hash_table_insert(ctx->syncs, POINTER_CAST(file_id), entry); + *entry = lookup_entry; + hash_table_insert(ctx->syncs, entry, entry); } - uid_file = (file_id & DBOX_FILE_ID_FLAG_UID) != 0; if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) { if (!array_is_created(&entry->expunges)) { p_array_init(&entry->expunges, ctx->pool, - uid_file ? 1 : 3); + lookup_entry.uid != 0 ? 1 : 3); } seq_range_array_add(&entry->expunges, 0, seq); } else { @@ -104,9 +133,11 @@ static int dbox_sync_index(struct dbox_sync_context *ctx) seq1, seq2); } - /* read all changes and sort them to file_id order */ + /* read all changes and group changes to same file_id together */ ctx->pool = pool_alloconly_create("dbox sync pool", 1024*32); - ctx->syncs = hash_table_create(default_pool, ctx->pool, 0, NULL, NULL); + ctx->syncs = hash_table_create(default_pool, ctx->pool, 0, + dbox_sync_file_entry_hash, + dbox_sync_file_entry_cmp); for (;;) { if (!mail_index_sync_next(ctx->index_sync_ctx, &sync_rec)) diff --git a/src/lib-storage/index/dbox/dbox-sync.h b/src/lib-storage/index/dbox/dbox-sync.h index f1d9c6cddb..0d68d59c8b 100644 --- a/src/lib-storage/index/dbox/dbox-sync.h +++ b/src/lib-storage/index/dbox/dbox-sync.h @@ -4,7 +4,7 @@ struct mailbox; struct dbox_sync_file_entry { - uint32_t file_id; + uint32_t uid, file_id; unsigned int move_from_alt:1; unsigned int move_to_alt:1;