From: Timo Sirainen Date: Wed, 25 Feb 2009 00:45:07 +0000 (-0500) Subject: dbox: Initial support for saving multiple messages per file. X-Git-Tag: 2.0.alpha1~1038^2~79 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=3c52b294054995d62bd1851c5c42b975fd8c22c9;p=thirdparty%2Fdovecot%2Fcore.git dbox: Initial support for saving multiple messages per file. --HG-- branch : HEAD --- diff --git a/src/lib-storage/index/dbox/dbox-file-maildir.c b/src/lib-storage/index/dbox/dbox-file-maildir.c index 77ef4dfc5e..2a3be03d06 100644 --- a/src/lib-storage/index/dbox/dbox-file-maildir.c +++ b/src/lib-storage/index/dbox/dbox-file-maildir.c @@ -27,14 +27,14 @@ const char *dbox_file_maildir_metadata_get(struct dbox_file *file, case DBOX_METADATA_SAVE_TIME: if (file->fd != -1) { if (fstat(file->fd, &st) < 0) { - dbox_file_set_syscall_error(file, "fstat"); + dbox_file_set_syscall_error(file, "fstat()"); return NULL; } } else { - if (stat(dbox_file_get_path(file), &st) < 0) { + if (stat(file->current_path, &st) < 0) { if (errno == ENOENT) return NULL; - dbox_file_set_syscall_error(file, "stat"); + dbox_file_set_syscall_error(file, "stat()"); return NULL; } } diff --git a/src/lib-storage/index/dbox/dbox-file.c b/src/lib-storage/index/dbox/dbox-file.c index cdb2f237cc..d4170d0a85 100644 --- a/src/lib-storage/index/dbox/dbox-file.c +++ b/src/lib-storage/index/dbox/dbox-file.c @@ -7,6 +7,7 @@ #include "hostpid.h" #include "istream.h" #include "ostream.h" +#include "file-lock.h" #include "mkdir-parents.h" #include "fdatasync-path.h" #include "str.h" @@ -17,6 +18,7 @@ #include #include #include +#include #include static int dbox_file_metadata_skip_header(struct dbox_file *file); @@ -35,19 +37,18 @@ static char *dbox_generate_tmp_filename(void) void dbox_file_set_syscall_error(struct dbox_file *file, const char *function) { mail_storage_set_critical(&file->storage->storage, - "%s(%s) failed: %m", function, - dbox_file_get_path(file)); + "%s failed for file %s: %m", + function, file->current_path); } -static void -dbox_file_set_corrupted(struct dbox_file *file, const char *reason) +static void dbox_file_set_corrupted(struct dbox_file *file, const char *reason) { mail_storage_set_critical(&file->storage->storage, - "%s corrupted: %s", dbox_file_get_path(file), - reason); + "Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s", + file->current_path, + file->input == NULL ? 0 : file->input->v_offset, reason); } - static struct dbox_file * dbox_find_and_move_open_file(struct dbox_storage *storage, uint32_t file_id) { @@ -188,6 +189,7 @@ dbox_file_init_multi(struct dbox_storage *storage, uint32_t file_id) file = i_new(struct dbox_file, 1); file->refcount = 1; file->storage = storage; + file->file_id = file_id; file->fd = -1; file->fname = file_id == 0 ? dbox_generate_tmp_filename() : i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, file_id); @@ -209,7 +211,7 @@ int dbox_file_assign_id(struct dbox_file *file, uint32_t id) i_assert(file->uid == 0 && file->file_id == 0); i_assert(id != 0); - old_path = dbox_file_get_path(file); + old_path = file->current_path; if (file->single_mbox != NULL) { new_fname = dbox_file_uid_get_fname(file->single_mbox, id, &maildir); @@ -245,7 +247,7 @@ int dbox_file_assign_id(struct dbox_file *file, uint32_t id) void dbox_file_unref(struct dbox_file **_file) { struct dbox_file *file = *_file; - struct dbox_file *const *files; + struct dbox_file *const *files, *oldest_file; unsigned int i, count; *_file = NULL; @@ -264,63 +266,32 @@ void dbox_file_unref(struct dbox_file **_file) return; } + /* close the oldest file with refcount=0 */ for (i = 0; i < count; i++) { - if (files[i] == file) + if (files[i]->refcount == 0) break; } - i_assert(i != count); + oldest_file = files[i]; array_delete(&file->storage->open_files, i, 1); + if (oldest_file != file) { + dbox_file_free(oldest_file); + return; + } + /* have to close ourself */ } dbox_file_free(file); } -static time_t day_begin_stamp(unsigned int days) -{ - struct tm tm; - time_t stamp; - - if (days == 0) - return 0; - - /* get beginning of today */ - tm = *localtime(&ioloop_time); - tm.tm_hour = 0; - tm.tm_min = 0; - tm.tm_sec = 0; - stamp = mktime(&tm); - if (stamp == (time_t)-1) - i_panic("mktime(today) failed"); - - return stamp - (3600*24 * (days-1)); -} - -bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size) -{ - if (file->nonappendable) - return FALSE; - - if (file->append_offset == 0) { - /* messages have been expunged */ - return FALSE; - } - - if (file->append_offset < file->storage->rotate_min_size || - file->append_offset == file->file_header_size) - return TRUE; - if (file->append_offset + mail_size >= file->storage->rotate_size) - return FALSE; - return file->create_time >= day_begin_stamp(file->storage->rotate_days); -} - static int dbox_file_parse_header(struct dbox_file *file, const char *line) { const char *const *tmp, *value; - unsigned int pos, version; + unsigned int pos; enum dbox_header_key key; - version = *line - '0'; - if (line[1] != ' ' || (version != 1 && version != DBOX_VERSION)) { + file->file_version = *line - '0'; + if (!i_isdigit(line[0]) || line[1] != ' ' || + (file->file_version != 1 && file->file_version != DBOX_VERSION)) { dbox_file_set_corrupted(file, "Invalid dbox version"); return -1; } @@ -350,9 +321,6 @@ static int dbox_file_parse_header(struct dbox_file *file, const char *line) dbox_file_set_corrupted(file, "Missing message header size"); return -1; } - - if (!file->nonappendable) - file->nonappendable = !dbox_file_can_append(file, 0); return 0; } @@ -364,10 +332,13 @@ static int dbox_file_read_header(struct dbox_file *file) i_stream_seek(file->input, 0); line = i_stream_read_next_line(file->input); if (line == NULL) { - if (file->input->stream_errno == 0) + if (file->input->stream_errno == 0) { + dbox_file_set_corrupted(file, + "EOF while reading file header"); return 0; + } - dbox_file_set_syscall_error(file, "read"); + dbox_file_set_syscall_error(file, "read()"); return -1; } file->file_header_size = file->input->v_offset; @@ -406,8 +377,7 @@ static int dbox_file_open_fd(struct dbox_file *file) return 1; } -static int dbox_file_open(struct dbox_file *file, bool read_header, - bool *deleted_r) +static int dbox_file_open(struct dbox_file *file, bool *deleted_r) { int ret; @@ -428,7 +398,7 @@ static int dbox_file_open(struct dbox_file *file, bool read_header, } file->input = i_stream_create_fd(file->fd, MAIL_READ_BLOCK_SIZE, FALSE); - return !read_header || file->maildir_file ? 1 : + return file->maildir_file ? 1 : dbox_file_read_header(file); } @@ -473,17 +443,15 @@ static int dbox_file_create(struct dbox_file *file) file->file_header_size = str_len(hdr); file->msg_header_size = sizeof(struct dbox_message_header); - file->append_offset = str_len(hdr); if (o_stream_send(file->output, str_data(hdr), str_len(hdr)) < 0) { - dbox_file_set_syscall_error(file, "write"); + dbox_file_set_syscall_error(file, "write()"); return -1; } return 0; } -int dbox_file_open_or_create(struct dbox_file *file, bool read_header, - bool *deleted_r) +int dbox_file_open_or_create(struct dbox_file *file, bool *deleted_r) { int ret; @@ -497,7 +465,7 @@ int dbox_file_open_or_create(struct dbox_file *file, bool read_header, } else if (file->input != NULL) return 1; else - return dbox_file_open(file, read_header, deleted_r); + return dbox_file_open(file, deleted_r); } int dbox_file_open_if_needed(struct dbox_file *file) @@ -521,20 +489,32 @@ int dbox_file_open_if_needed(struct dbox_file *file) void dbox_file_close(struct dbox_file *file) { + i_assert(file->lock == NULL); + if (file->input != NULL) i_stream_unref(&file->input); if (file->output != NULL) o_stream_unref(&file->output); if (file->fd != -1) { if (close(file->fd) < 0) - dbox_file_set_syscall_error(file, "close"); + dbox_file_set_syscall_error(file, "close()"); file->fd = -1; } } -const char *dbox_file_get_path(struct dbox_file *file) +int dbox_file_try_lock(struct dbox_file *file) +{ + i_assert(file->fd != -1); + + return file_try_lock(file->fd, file->current_path, F_WRLCK, + FILE_LOCK_METHOD_FCNTL, &file->lock); + +} + +void dbox_file_unlock(struct dbox_file *file) { - return file->current_path; + if (file->lock != NULL) + file_unlock(&file->lock); } static int @@ -548,7 +528,7 @@ dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r) if (file->maildir_file) { if (fstat(file->fd, &st) < 0) { - dbox_file_set_syscall_error(file, "fstat"); + dbox_file_set_syscall_error(file, "fstat()"); return -1; } *physical_size_r = st.st_size; @@ -559,18 +539,25 @@ dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r) file->msg_header_size - 1); if (ret <= 0) { if (file->input->stream_errno == 0) { - /* EOF, broken offset */ + /* EOF, broken offset or file truncated */ + dbox_file_set_corrupted(file, t_strdup_printf( + "EOF reading msg header " + "(got %"PRIuSIZE_T"/%u bytes)", + size, file->msg_header_size)); return 0; } - dbox_file_set_syscall_error(file, "read"); + dbox_file_set_syscall_error(file, "read()"); return -1; } - if (data[file->msg_header_size-1] != '\n') + if (data[file->msg_header_size-1] != '\n') { + dbox_file_set_corrupted(file, "msg header doesn't end with LF"); return 0; + } memcpy(&hdr, data, I_MIN(sizeof(hdr), file->msg_header_size)); if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) { /* probably broken offset */ + dbox_file_set_corrupted(file, "bad magic value"); return 0; } @@ -588,7 +575,7 @@ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, *expunged_r = FALSE; if (file->input == NULL) { - if ((ret = dbox_file_open(file, TRUE, expunged_r)) <= 0 || + if ((ret = dbox_file_open(file, expunged_r)) <= 0 || *expunged_r) return ret; } @@ -603,8 +590,8 @@ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, if (ret <= 0) return ret; } + i_stream_seek(file->input, offset + file->msg_header_size); if (stream_r != NULL) { - i_stream_seek(file->input, offset + file->msg_header_size); *stream_r = i_stream_create_limit(file->input, file->cur_physical_size); } @@ -641,20 +628,22 @@ dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset, } int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset, - uoff_t *physical_size_r) + uoff_t *physical_size_r, bool *last_r) { uoff_t size; bool first = *offset == 0; bool deleted; int ret; + *last_r = FALSE; + ret = dbox_file_get_mail_stream(file, *offset, &size, NULL, &deleted); if (ret <= 0) return ret; if (deleted) { - *physical_size_r = 0; + *last_r = TRUE; return 1; } if (first) { @@ -666,51 +655,63 @@ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset, return dbox_file_seek_next_at_metadata(file, offset, physical_size_r); } -static int dbox_file_seek_append_pos(struct dbox_file *file, uoff_t mail_size) +static int +dbox_file_seek_append_pos(struct dbox_file *file, uoff_t last_msg_offset) { + uoff_t offset, size; + bool last; int ret; if ((ret = dbox_file_read_header(file)) <= 0) return ret; - if (file->append_offset == 0 || - file->msg_header_size != sizeof(struct dbox_message_header) || - !dbox_file_can_append(file, mail_size)) { - /* can't append */ + if (file->file_version != DBOX_VERSION || + file->msg_header_size != sizeof(struct dbox_message_header)) { + /* created by an incompatible version, can't append */ return 0; } - file->output = o_stream_create_fd_file(file->fd, (uoff_t)-2, FALSE); - o_stream_seek(file->output, file->append_offset); + offset = last_msg_offset; + ret = dbox_file_seek_next(file, &offset, &size, &last); + if (ret <= 0) + return ret; + if (!last) { + /* not end of file? previous write probably crashed. */ + if (ftruncate(file->fd, offset) < 0) { + dbox_file_set_syscall_error(file, "ftruncate()"); + return -1; + } + i_stream_sync(file->input); + } + + file->output = o_stream_create_fd_file(file->fd, 0, FALSE); + o_stream_seek(file->output, offset); return 1; } -static int -dbox_file_get_append_stream_int(struct dbox_file *file, uoff_t mail_size, +int dbox_file_get_append_stream(struct dbox_file *file, uoff_t last_msg_offset, struct ostream **stream_r) { bool deleted; int ret; if (file->fd == -1) { + /* creating a new file */ i_assert(file->output == NULL); - if ((ret = dbox_file_open_or_create(file, FALSE, - &deleted)) <= 0 || deleted) - return ret; - } - if (file->output == NULL) { - ret = dbox_file_seek_append_pos(file, mail_size); + ret = dbox_file_open_or_create(file, &deleted); if (ret <= 0) return ret; - } else { - if (!dbox_file_can_append(file, mail_size)) + if (deleted) return 0; } - if (file->output->offset > (uint32_t)-1) { - /* we use 32bit offsets to messages */ - return 0; + if (file->output == NULL) { + i_assert(file->lock != NULL || last_msg_offset == 0); + + ret = dbox_file_seek_append_pos(file, last_msg_offset); + if (ret <= 0) + return ret; } o_stream_ref(file->output); @@ -718,51 +719,27 @@ dbox_file_get_append_stream_int(struct dbox_file *file, uoff_t mail_size, return 1; } -int dbox_file_get_append_stream(struct dbox_file *file, uoff_t mail_size, - struct ostream **stream_r) -{ - int ret; - - if (file->append_count == 0) { - if (file->nonappendable) - return 0; - } else { - if (!dbox_file_can_append(file, mail_size)) - return 0; - } - - ret = dbox_file_get_append_stream_int(file, mail_size, stream_r); - if (ret == 0) - file->nonappendable = TRUE; - return ret; -} - uoff_t dbox_file_get_next_append_offset(struct dbox_file *file) { - i_assert(file->output_stream_offset != 0); - i_assert(file->output == NULL || - file->output_stream_offset == file->output->offset); + i_assert(file->output != NULL); - return file->output_stream_offset; + return file->output->offset; } void dbox_file_cancel_append(struct dbox_file *file, uoff_t append_offset) { - if (ftruncate(file->fd, append_offset) < 0) { - dbox_file_set_syscall_error(file, "ftruncate"); - file->append_offset = 0; - file->nonappendable = TRUE; - } + if (ftruncate(file->fd, append_offset) < 0) + dbox_file_set_syscall_error(file, "ftruncate()"); + if (file->input != NULL) + i_stream_sync(file->input); o_stream_seek(file->output, append_offset); - file->output_stream_offset = append_offset; } void dbox_file_finish_append(struct dbox_file *file) { - file->output_stream_offset = file->output->offset; - file->append_offset = file->output->offset; - file->append_count++; + if (file->input != NULL) + i_stream_sync(file->input); } static uoff_t @@ -793,7 +770,7 @@ static int dbox_file_metadata_skip_header(struct dbox_file *file) /* EOF, broken offset */ return 0; } - dbox_file_set_syscall_error(file, "read"); + dbox_file_set_syscall_error(file, "read()"); return -1; } memcpy(&metadata_hdr, data, sizeof(metadata_hdr)); @@ -830,15 +807,12 @@ int dbox_file_metadata_seek(struct dbox_file *file, uoff_t metadata_offset, i_assert(!file->maildir_file); /* previous check should catch this */ if (file->input == NULL) { - if ((ret = dbox_file_open(file, TRUE, &deleted)) <= 0) + if ((ret = dbox_file_open(file, &deleted)) <= 0) return ret; if (deleted) { *expunged_r = TRUE; return 1; } - } else { - /* make sure to flush any cached data */ - i_stream_sync(file->input); } i_stream_seek(file->input, metadata_offset); @@ -949,13 +923,15 @@ int dbox_file_move(struct dbox_file *file, bool alt_path) out_fd = open(temp_path, O_WRONLY | O_CREAT | O_TRUNC, 0600); if (out_fd == -1 && errno == ENOENT) { if (mkdir_parents(dest_dir, 0700) < 0 && errno != EEXIST) { - i_error("mkdir_parents(%s) failed: %m", dest_dir); + mail_storage_set_critical(&file->storage->storage, + "mkdir_parents(%s) failed: %m", dest_dir); return -1; } out_fd = open(temp_path, O_WRONLY | O_CREAT | O_TRUNC, 0600); } if (out_fd == -1) { - i_error("open(%s, O_CREAT) failed: %m", temp_path); + mail_storage_set_critical(&file->storage->storage, + "open(%s, O_CREAT) failed: %m", temp_path); return -1; } output = o_stream_create_fd_file(out_fd, 0, FALSE); @@ -965,14 +941,16 @@ int dbox_file_move(struct dbox_file *file, bool alt_path) ret = o_stream_flush(output); if (output->stream_errno != 0) { errno = output->stream_errno; - i_error("write(%s) failed: %m", temp_path); + mail_storage_set_critical(&file->storage->storage, + "write(%s) failed: %m", temp_path); ret = -1; } else if (file->input->stream_errno != 0) { errno = file->input->stream_errno; - i_error("read(%s) failed: %m", file->current_path); + dbox_file_set_syscall_error(file, "ftruncate()"); ret = -1; } else if (ret < 0) { - i_error("o_stream_send_istream(%s, %s) " + mail_storage_set_critical(&file->storage->storage, + "o_stream_send_istream(%s, %s) " "failed with unknown error", temp_path, file->current_path); } @@ -981,12 +959,14 @@ int dbox_file_move(struct dbox_file *file, bool alt_path) if ((file->storage->storage.flags & MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0 && ret == 0) { if (fsync(out_fd) < 0) { - i_error("fsync(%s) failed: %m", temp_path); + mail_storage_set_critical(&file->storage->storage, + "fsync(%s) failed: %m", temp_path); ret = -1; } } if (close(out_fd) < 0) { - i_error("close(%s) failed: %m", temp_path); + mail_storage_set_critical(&file->storage->storage, + "close(%s) failed: %m", temp_path); ret = -1; } if (ret < 0) { @@ -999,20 +979,22 @@ int dbox_file_move(struct dbox_file *file, bool alt_path) its contents should be the same (except for maybe older metadata) */ dest_path = t_strdup_printf("%s/%s", dest_dir, file->fname); if (rename(temp_path, dest_path) < 0) { - i_error("rename(%s, %s) failed: %m", temp_path, dest_path); + mail_storage_set_critical(&file->storage->storage, + "rename(%s, %s) failed: %m", temp_path, dest_path); (void)unlink(temp_path); return -1; } if ((file->storage->storage.flags & MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0) { if (fdatasync_path(dest_dir) < 0) { - i_error("fdatasync(%s) failed: %m", dest_dir); + mail_storage_set_critical(&file->storage->storage, + "fdatasync(%s) failed: %m", dest_dir); (void)unlink(dest_path); return -1; } } if (unlink(file->current_path) < 0) { - i_error("unlink(%s) failed: %m", file->current_path); + dbox_file_set_syscall_error(file, "unlink()"); if (errno == EACCES) { /* configuration problem? revert the write */ (void)unlink(dest_path); @@ -1024,8 +1006,9 @@ int dbox_file_move(struct dbox_file *file, bool alt_path) /* file was successfully moved - reopen it */ dbox_file_close(file); - if (dbox_file_open(file, TRUE, &deleted) <= 0) { - i_error("dbox_file_move(%s): reopening file failed", dest_path); + if (dbox_file_open(file, &deleted) <= 0) { + mail_storage_set_critical(&file->storage->storage, + "dbox_file_move(%s): reopening file failed", dest_path); return -1; } return 0; diff --git a/src/lib-storage/index/dbox/dbox-file.h b/src/lib-storage/index/dbox/dbox-file.h index 1a52061a8f..f02e68bd0b 100644 --- a/src/lib-storage/index/dbox/dbox-file.h +++ b/src/lib-storage/index/dbox/dbox-file.h @@ -86,16 +86,11 @@ struct dbox_file { /* uid is for single-msg-per-file, file_id for multi-msgs-per-file */ uint32_t uid, file_id; + time_t create_time; + unsigned int file_version; unsigned int file_header_size; unsigned int msg_header_size; - unsigned int append_count; - uint32_t last_append_uid; - - uoff_t append_offset; - time_t create_time; - uoff_t output_stream_offset; - uoff_t cur_offset; uoff_t cur_physical_size; @@ -105,6 +100,7 @@ struct dbox_file { int fd; struct istream *input; struct ostream *output; + struct file_lock *lock; /* Metadata for the currently seeked metadata block. */ pool_t metadata_pool; @@ -113,7 +109,6 @@ struct dbox_file { unsigned int alt_path:1; unsigned int maildir_file:1; - unsigned int nonappendable:1; unsigned int deleted:1; }; @@ -131,18 +126,18 @@ void dbox_files_free(struct dbox_storage *storage); int dbox_file_assign_id(struct dbox_file *file, uint32_t id); /* Open the file if uid or file_id is not 0, otherwise create it. Returns 1 if - ok, 0 if read_header=TRUE and opened file was broken, -1 if error. If file - is deleted, deleted_r=TRUE and 1 is returned. */ -int dbox_file_open_or_create(struct dbox_file *file, bool read_header, - bool *deleted_r); + ok, 0 if file header is corrupted, -1 if error. If file is deleted, + deleted_r=TRUE and 1 is returned. */ +int dbox_file_open_or_create(struct dbox_file *file, bool *deleted_r); /* Open the file's fd if it's currently closed. Assumes that the file exists. */ int dbox_file_open_if_needed(struct dbox_file *file); /* Close the file handle from the file, but don't free it. */ void dbox_file_close(struct dbox_file *file); -/* Returns the current fulle path for an opened/created file. It's an error to - call this function for a non-opened file. */ -const char *dbox_file_get_path(struct dbox_file *file); +/* Try to lock the dbox file. Returns 1 if ok, 0 if already locked by someone + else, -1 if error. */ +int dbox_file_try_lock(struct dbox_file *file); +void dbox_file_unlock(struct dbox_file *file); /* Seek to given offset in file and return the message's input stream and physical size. Returns 1 if ok, 0 if file/offset is corrupted, @@ -151,17 +146,18 @@ int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset, uoff_t *physical_size_r, struct istream **stream_r, bool *expunged_r); /* Seek to next message after given offset, or to first message if offset=0. - If there are no more messages, physical_size_r is set to 0. Returns 1 if ok, + If there are no more messages, last_r is set to TRUE. Returns 1 if ok, 0 if file/offset is corrupted, -1 if I/O error. */ int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset, - uoff_t *physical_size_r); + uoff_t *physical_size_r, bool *last_r); /* Returns TRUE if mail_size bytes can be appended to the file. */ bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size); -/* Get output stream for appending a new message. Returns 1 if ok, 0 if - file can't be appended to (limits reached, expunges, corrupted) or - -1 if error. If 0 is returned, index is also updated. */ -int dbox_file_get_append_stream(struct dbox_file *file, uoff_t mail_size, +/* Get output stream for appending a new message. last_msg_offset points to + the beginning of the last message in the file, or 0 for new files. Returns + 1 if ok, 0 if file can't be appended to (old file version or corruption) + or -1 if error. */ +int dbox_file_get_append_stream(struct dbox_file *file, uoff_t last_msg_offset, struct ostream **stream_r); /* Returns the next offset for append a message. dbox_file_get_append_stream() must have been called for this file already at least once. */ diff --git a/src/lib-storage/index/dbox/dbox-mail.c b/src/lib-storage/index/dbox/dbox-mail.c index 73d15f7b15..0edcc912fa 100644 --- a/src/lib-storage/index/dbox/dbox-mail.c +++ b/src/lib-storage/index/dbox/dbox-mail.c @@ -62,17 +62,22 @@ static int dbox_mail_open(struct dbox_mail *mail, struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox; struct mail *_mail = &mail->imail.mail.mail; uint32_t map_uid, file_id; + int ret; if (mail->open_file == NULL) { map_uid = dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq); if (map_uid == 0) { mail->open_file = dbox_file_init_single(mbox, _mail->uid); - } else if (!dbox_map_lookup(mbox->storage->map_index, map_uid, - &file_id, &mail->offset)) { - mail_set_expunged(_mail); - return -1; } else { + ret = dbox_map_lookup(mbox->storage->map, map_uid, + &file_id, &mail->offset); + if (ret <= 0) { + // FIXME: ret=0 case - should we resync? + if (ret == 0) + mail_set_expunged(_mail); + return -1; + } mail->open_file = dbox_file_init_multi(mbox->storage, file_id); } @@ -86,6 +91,7 @@ static int dbox_mail_open(struct dbox_mail *mail, static int dbox_mail_metadata_seek(struct dbox_mail *mail, struct dbox_file **file_r) { + struct mail *_mail = &mail->imail.mail.mail; uoff_t offset; bool expunged; int ret; @@ -98,10 +104,14 @@ dbox_mail_metadata_seek(struct dbox_mail *mail, struct dbox_file **file_r) if (ret < 0) return -1; /* FIXME */ + mail_storage_set_critical(mail->imail.ibox->storage, + "Broken metadata in dbox file %s offset %"PRIuUOFF_T + " (uid=%u)", + (*file_r)->current_path, offset, _mail->uid); return -1; } if (expunged) { - mail_set_expunged(&mail->imail.mail.mail); + mail_set_expunged(_mail); return -1; } return 0; @@ -278,8 +288,10 @@ dbox_mail_get_stream(struct mail *_mail, struct message_size *hdr_size, if (ret > 0) i_stream_unref(&input); mail_storage_set_critical(_mail->box->storage, - "broken pointer to dbox file %s", - mail->open_file->current_path); + "broken pointer to dbox file %s " + "offset %"PRIuUOFF_T" (uid=%u)", + mail->open_file->current_path, + offset, _mail->uid); return -1; } data->physical_size = size; diff --git a/src/lib-storage/index/dbox/dbox-map.c b/src/lib-storage/index/dbox/dbox-map.c index b24c47b1a2..d520b3ddc7 100644 --- a/src/lib-storage/index/dbox/dbox-map.c +++ b/src/lib-storage/index/dbox/dbox-map.c @@ -2,25 +2,49 @@ #include "lib.h" #include "array.h" +#include "ostream.h" #include "dbox-storage.h" #include "dbox-file.h" #include "dbox-map.h" +#define MAX_BACKWARDS_LOOKUPS 10 + +struct dbox_mail_index_map_header { + uint32_t highest_file_id; +}; + +struct dbox_mail_index_map_record { + uint32_t file_id; + uint32_t offset; +}; + struct dbox_map { struct dbox_storage *storage; struct mail_index *index; + struct mail_index_view *view; + + uint32_t map_ext_id, ref_ext_id; +}; + +struct dbox_map_append { + struct dbox_file *file; + uoff_t offset; }; struct dbox_map_append_context { struct dbox_mailbox *mbox; + struct dbox_map *map; ARRAY_DEFINE(files, struct dbox_file *); + ARRAY_DEFINE(appends, struct dbox_map_append); - uoff_t output_offset; - unsigned int new_record_idx; uint32_t first_new_file_id; + uint32_t orig_msg_count; - unsigned int locked_header:1; + unsigned int files_nonappendable_count; + uint32_t retry_seq_min, retry_seq_max; + + unsigned int failed:1; }; struct dbox_map *dbox_map_init(struct dbox_storage *storage) @@ -31,6 +55,12 @@ struct dbox_map *dbox_map_init(struct dbox_storage *storage) map->storage = storage; map->index = mail_index_alloc(storage->storage_dir, DBOX_GLOBAL_INDEX_PREFIX); + map->map_ext_id = mail_index_ext_register(map->index, "map", + sizeof(struct dbox_mail_index_map_header), + sizeof(struct dbox_mail_index_map_record), + sizeof(uint32_t)); + map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0, + sizeof(uint16_t), sizeof(uint16_t)); return map; } @@ -40,105 +70,536 @@ void dbox_map_deinit(struct dbox_map **_map) *_map = NULL; + if (map->view != NULL) + mail_index_view_close(&map->view); mail_index_free(&map->index); i_free(map); } -bool dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, - uint32_t *file_id_r, uoff_t *offset_r) +static int dbox_map_open(struct dbox_map *map, bool create) { - return FALSE; + struct mail_storage *storage = &map->storage->storage; + enum mail_index_open_flags open_flags; + int ret; + + if (map->view != NULL) { + /* already opened */ + return 1; + } + + open_flags = index_storage_get_index_open_flags(storage); + if (create) + open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE; + + ret = mail_index_open(map->index, open_flags, storage->lock_method); + if (ret <= 0) { + mail_storage_set_internal_error(storage); + mail_index_reset_error(map->index); + return ret; + } + + map->view = mail_index_view_open(map->index); + return 1; +} + +static int dbox_map_refresh(struct dbox_map *map) +{ + struct mail_index_view_sync_ctx *ctx; + bool delayed_expunges; + + ctx = mail_index_view_sync_begin(map->view, + MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT); + if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) { + mail_storage_set_internal_error(&map->storage->storage); + mail_index_reset_error(map->index); + return -1; + } + return 0; +} + +static int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq, + uint32_t *file_id_r, uoff_t *offset_r) +{ + const struct dbox_mail_index_map_record *rec; + const void *data; + bool expunged; + + mail_index_lookup_ext(map->view, seq, map->map_ext_id, + &data, &expunged); + rec = data; + + if (rec == NULL || rec->file_id == 0) { + /* corrupted */ + mail_storage_set_critical(&map->storage->storage, + "dbox map %s corrupted: file_id=0 for seq=%u", + map->index->filepath, seq); + return -1; + } + + *file_id_r = rec->file_id; + *offset_r = rec->offset; + return 0; +} + +int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r) +{ + uint32_t seq; + int ret; + + if ((ret = dbox_map_open(map, FALSE)) <= 0) { + /* map doesn't exist or is broken */ + return ret; + } + + if (!mail_index_lookup_seq(map->view, map_uid, &seq)) { + /* not found - try again after a refresh */ + if (dbox_map_refresh(map) < 0) + return -1; + if (!mail_index_lookup_seq(map->view, map_uid, &seq)) + return 0; + } + + if (dbox_map_lookup_seq(map, seq, file_id_r, offset_r) < 0) + return 0; + return 1; } struct dbox_map_append_context * dbox_map_append_begin(struct dbox_mailbox *mbox) { struct dbox_map_append_context *ctx; + int ret; ctx = i_new(struct dbox_map_append_context, 1); + ctx->map = mbox->storage->map; ctx->mbox = mbox; ctx->first_new_file_id = (uint32_t)-1; i_array_init(&ctx->files, 64); + i_array_init(&ctx->appends, 128); + + if ((ret = dbox_map_open(ctx->map, TRUE)) <= 0) { + i_assert(ret != 0); + ctx->failed = TRUE; + } + /* refresh the map so we can try appending to the latest files */ + (void)dbox_map_refresh(ctx->map); return ctx; } -int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size, - struct dbox_file **file_r, struct ostream **output_r) +static time_t day_begin_stamp(unsigned int days) { - struct dbox_file *const *files, *file = NULL; - unsigned int i, count; + struct tm tm; + time_t stamp; + + if (days == 0) + return 0; + + /* get beginning of today */ + tm = *localtime(&ioloop_time); + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + stamp = mktime(&tm); + if (stamp == (time_t)-1) + i_panic("mktime(today) failed"); + + return stamp - (3600*24 * (days-1)); +} + +static bool +dbox_map_file_try_append(struct dbox_map_append_context *ctx, + uint32_t file_id, time_t stamp, + uoff_t mail_size, uoff_t last_msg_offset, + struct dbox_file **file_r, struct ostream **output_r, + bool *retry_later_r) +{ + struct dbox_map *map = ctx->map; + const struct mail_index_header *hdr; + struct dbox_file *file; + struct stat st; + uint32_t seq, tmp_file_id; + uoff_t tmp_offset; + bool deleted, file_too_old = FALSE; int ret; - /* first try to use files already used in this append */ + *file_r = NULL; + *retry_later_r = FALSE; + + file = dbox_file_init_multi(map->storage, file_id); + if (dbox_file_open_or_create(file, &deleted) <= 0) + return TRUE; + if (deleted) + return TRUE; + + if (fstat(file->fd, &st) < 0) { + mail_storage_set_critical(&map->storage->storage, + "fstat(%s) failed: %m", file->current_path); + } else if (I_MIN(st.st_mtime, st.st_ctime) < stamp) + file_too_old = TRUE; + else if (st.st_size + mail_size > map->storage->rotate_size) { + /* file too large */ + } else if ((ret = dbox_file_try_lock(file)) <= 0) { + /* locking failed */ + *retry_later_r = ret == 0; + } else if (dbox_map_refresh(map) == 0) { + /* now that the file is locked and map is refreshed, make sure + we still have the last msg's offset. */ + hdr = mail_index_get_header(map->view); + seq = ctx->orig_msg_count + 1; + for (; seq <= hdr->messages_count; seq++) { + if (dbox_map_lookup_seq(map, seq, + &tmp_file_id, &tmp_offset) < 0) + break; + if (tmp_file_id == file->file_id) { + i_assert(last_msg_offset < tmp_offset); + last_msg_offset = tmp_offset; + } + } + + if (seq > hdr->messages_count && + dbox_file_get_append_stream(file, last_msg_offset, + output_r) > 0) { + /* success */ + *file_r = file; + return TRUE; + } + } + + /* failure */ + dbox_file_unlock(file); + dbox_file_unref(&file); + return !file_too_old; +} + +static bool +dbox_map_is_appending(struct dbox_map_append_context *ctx, uint32_t file_id) +{ + struct dbox_file *const *files; + unsigned int i, count; + + /* there shouldn't be many files open, don't bother with anything + faster. */ files = array_get(&ctx->files, &count); for (i = 0; i < count; i++) { - if (dbox_file_get_append_stream(files[i], mail_size, + if (files[i]->file_id == file_id) + return TRUE; + } + return FALSE; +} + +static int +dbox_map_find_appendable_file(struct dbox_map_append_context *ctx, + uoff_t mail_size, struct dbox_file **file_r, + struct ostream **output_r, bool *existing_r) +{ + struct dbox_map *map = ctx->map; + bool fsync_disable = (map->storage->storage.flags & + MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0; + struct dbox_file *const *files; + const struct mail_index_header *hdr; + unsigned int i, count, backwards_lookup_count; + uint32_t seq, file_id, min_seen_file_id; + uoff_t offset, append_offset; + time_t stamp; + bool retry_later, updated_retry_seq_min; + + *existing_r = FALSE; + + if (mail_size >= map->storage->rotate_size) + return 0; + + /* first try to use files already used in this append */ + files = array_get(&ctx->files, &count); + for (i = count; i > ctx->files_nonappendable_count; i--) { + append_offset = dbox_file_get_next_append_offset(files[i-1]); + if (append_offset + mail_size <= map->storage->rotate_size && + dbox_file_get_append_stream(files[i-1], append_offset, output_r) > 0) { - *file_r = files[i]; - return 0; + *file_r = files[i-1]; + *existing_r = TRUE; + return 1; } + /* can't append to this file anymore */ + if (files[i-1]->fd != -1) { + /* avoid wasting fds by closing the file */ + if (!fsync_disable) { + if (fdatasync(files[i-1]->fd) < 0) { + dbox_file_set_syscall_error(files[i-1], + "fdatasync()"); + return -1; + } + } + dbox_file_unlock(files[i-1]); + dbox_file_close(files[i-1]); + } + } + ctx->files_nonappendable_count = count; + + /* try to find an existing appendable file */ + stamp = day_begin_stamp(map->storage->rotate_days); + hdr = mail_index_get_header(map->view); + min_seen_file_id = (uint32_t)-1; + + ctx->orig_msg_count = hdr->messages_count; + updated_retry_seq_min = FALSE; + backwards_lookup_count = 0; + for (seq = hdr->messages_count; seq > 0; seq--) { + if (seq == ctx->retry_seq_max) { + /* skip over files that we know won't match */ + seq = ctx->retry_seq_min + 1; + continue; + } + if (dbox_map_lookup_seq(map, seq, &file_id, &offset) < 0) + return -1; + if (file_id >= min_seen_file_id) + continue; + min_seen_file_id = file_id; + + /* first lookup: this doesn't really tell us if we're going + to exceed the rotate size, it just potentially reduces + how many files we have to open. */ + if (offset + mail_size >= map->storage->rotate_size) + continue; + + if (dbox_map_is_appending(ctx, file_id)) { + /* already checked this */ + continue; + } + + if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) { + /* we've wasted enough time here */ + break; + } + + if (!dbox_map_file_try_append(ctx, file_id, stamp, + mail_size, offset, + file_r, output_r, &retry_later)) { + /* file is too old. the rest of the files are too. */ + break; + } + if (*file_r != NULL) { + if (seq > ctx->retry_seq_min) + ctx->retry_seq_min = seq; + return 1; + } + if (retry_later) { + ctx->retry_seq_min = seq; + updated_retry_seq_min = TRUE; + } + } + ctx->retry_seq_max = ctx->orig_msg_count; + if (!updated_retry_seq_min) { + /* there are no files that can be appended to */ + ctx->retry_seq_min = 0; } + return 0; +} - /* FIXME: try to find an existing appendable file */ +int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size, + struct dbox_file **file_r, struct ostream **output_r) +{ + struct dbox_file *file = NULL; + struct dbox_map_append *append; + bool existing; + int ret; + + if (ctx->failed) + return -1; - if (file == NULL) { + ret = dbox_map_find_appendable_file(ctx, mail_size, &file, + output_r, &existing); + if (ret < 0) + return -1; + + if (ret == 0) { /* create a new file */ - file = dbox_file_init_single(ctx->mbox, 0); - if ((ret = dbox_file_get_append_stream(file, mail_size, - output_r)) <= 0) { + file = ctx->map->storage->rotate_size == 0 ? + dbox_file_init_single(ctx->mbox, 0) : + dbox_file_init_multi(ctx->map->storage, 0); + ret = dbox_file_get_append_stream(file, 0, output_r); + if (ret <= 0) { i_assert(ret < 0); - (void)unlink(dbox_file_get_path(file)); + (void)unlink(file->current_path); dbox_file_unref(&file); return -1; } } + if (file->single_mbox == NULL) { + append = array_append_space(&ctx->appends); + append->file = file; + append->offset = (*output_r)->offset; + } + if (!existing) + array_append(&ctx->files, &file, 1); + *file_r = file; - array_append(&ctx->files, &file, 1); return 0; } -static int dbox_map_append_commit_new(struct dbox_map_append_context *ctx, - struct dbox_file *file) +static int +dbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view, + uint32_t *file_id_r) { - i_assert(!file->maildir_file); - i_assert(file->append_count > 0); - - if (file->single_mbox != NULL) { - /* single UID message file */ - i_assert(file->last_append_uid != 0); - return dbox_file_assign_id(file, file->last_append_uid); + const struct dbox_mail_index_map_header *hdr; + const void *data; + size_t data_size; + + mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size); + if (data_size != sizeof(*hdr)) { + if (data_size != 0) { + mail_storage_set_critical(&map->storage->storage, + "dbox map %s corrupted: hdr size=%u", + map->index->filepath, data_size); + return -1; + } + /* first file */ + *file_id_r = 1; + } else { + hdr = data; + *file_id_r = hdr->highest_file_id + 1; } - - /* FIXME */ - return -1; + return 0; } -int dbox_map_append_assign_file_ids(struct dbox_map_append_context *ctx) +int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx, + uint32_t *first_map_uid_r, + uint32_t *last_map_uid_r) { - struct dbox_file *const *files, *file; + bool fsync_disable = (ctx->map->storage->storage.flags & + MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0; + struct dbox_file *const *files; + const struct dbox_map_append *appends; + struct mail_index_sync_ctx *sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + const struct mail_index_header *hdr; + struct dbox_mail_index_map_record rec; unsigned int i, count; + uint32_t seq, first_uid, next_uid, first_file_id, file_id; + uint16_t ref16; int ret = 0; + if (array_count(&ctx->appends) == 0) { + *first_map_uid_r = 0; + *last_map_uid_r = 0; + return 0; + } + + ret = mail_index_sync_begin(ctx->map->index, &sync_ctx, &sync_view, + &trans, 0); + if (ret <= 0) { + i_assert(ret != 0); + mail_storage_set_internal_error(&ctx->map->storage->storage); + mail_index_reset_error(ctx->map->index); + return -1; + } + + if (dbox_map_get_next_file_id(ctx->map, sync_view, &file_id) < 0) { + mail_index_sync_rollback(&sync_ctx); + return -1; + } + + /* assign file_ids for newly created multi-files */ files = array_get(&ctx->files, &count); + first_file_id = file_id; for (i = 0; i < count; i++) { - file = files[i]; - - if (file->uid == 0 && file->file_id == 0) { - if (dbox_map_append_commit_new(ctx, file) < 0) { + if (files[i]->single_mbox != NULL) + continue; + + if (!fsync_disable && files[i]->single_mbox == NULL && + files[i]->fd != -1) { + if (fdatasync(files[i]->fd) < 0) { + dbox_file_set_syscall_error(files[i], + "fdatasync()"); ret = -1; break; } } + + if (files[i]->file_id != 0) + continue; + + if (dbox_file_assign_id(files[i], file_id++) < 0) { + ret = -1; + break; + } } if (ret < 0) { /* FIXME: we have to rollback the changes we made */ + mail_index_sync_rollback(&sync_ctx); + return -1; + } + + /* update the highest used file_id */ + if (first_file_id != file_id) { + file_id--; + mail_index_update_header_ext(trans, ctx->map->map_ext_id, + 0, &file_id, sizeof(file_id)); + } + + /* append map records to index */ + memset(&rec, 0, sizeof(rec)); + ref16 = 1; + appends = array_get(&ctx->appends, &count); + for (i = 0; i < count; i++) { + rec.file_id = appends[i].file->file_id; + rec.offset = appends[i].offset; + + mail_index_append(trans, 0, &seq); + mail_index_update_ext(trans, seq, ctx->map->map_ext_id, + &rec, NULL); + mail_index_update_ext(trans, seq, ctx->map->ref_ext_id, + &ref16, NULL); + } + + /* assign map UIDs for appended records */ + hdr = mail_index_get_header(sync_view); + first_uid = hdr->next_uid; + mail_index_append_assign_uids(trans, first_uid, &next_uid); + i_assert(next_uid - first_uid == count); + + if (hdr->uid_validity == 0) { + /* we don't really care about uidvalidity, but it can't be 0 */ + uint32_t uid_validity = ioloop_time; + mail_index_update_header(trans, + offsetof(struct mail_index_header, uid_validity), + &uid_validity, sizeof(uid_validity), TRUE); } + + if (mail_index_sync_commit(&sync_ctx) < 0) { + mail_storage_set_internal_error(&ctx->map->storage->storage); + mail_index_reset_error(ctx->map->index); + return -1; + } + + *first_map_uid_r = first_uid; + *last_map_uid_r = next_uid - 1; return ret; } -int dbox_map_append_commit(struct dbox_map_append_context **_ctx) +int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx, + uint32_t first_uid, uint32_t last_uid) +{ + struct dbox_file *const *files; + unsigned int i, count; + uint32_t next_uid = first_uid; + + files = array_get(&ctx->files, &count); + for (i = 0; i < count; i++) { + if (files[i]->single_mbox == NULL) + continue; + + if (dbox_file_assign_id(files[i], next_uid++) < 0) + return -1; + } + i_assert(next_uid == last_uid + 1); + return 0; +} + +void dbox_map_append_commit(struct dbox_map_append_context **_ctx) { struct dbox_map_append_context *ctx = *_ctx; struct dbox_file **files; @@ -147,17 +608,20 @@ int dbox_map_append_commit(struct dbox_map_append_context **_ctx) *_ctx = NULL; files = array_get_modifiable(&ctx->files, &count); - for (i = 0; i < count; i++) + for (i = 0; i < count; i++) { + dbox_file_unlock(files[i]); dbox_file_unref(&files[i]); + } + array_free(&ctx->appends); array_free(&ctx->files); i_free(ctx); - return 0; } void dbox_map_append_rollback(struct dbox_map_append_context **_ctx) { struct dbox_map_append_context *ctx = *_ctx; + struct mail_storage *storage = &ctx->map->storage->storage; struct dbox_file *const *files, *file; unsigned int i, count; @@ -170,13 +634,15 @@ void dbox_map_append_rollback(struct dbox_map_append_context **_ctx) if (file->file_id != 0) { /* FIXME: truncate? */ } else { - if (unlink(dbox_file_get_path(file)) < 0) { - i_error("unlink(%s) failed: %m", - dbox_file_get_path(file)); + if (unlink(file->current_path) < 0) { + mail_storage_set_critical(storage, + "unlink(%s) failed: %m", + file->current_path); } } dbox_file_unref(&file); } + array_free(&ctx->appends); array_free(&ctx->files); i_free(ctx); } diff --git a/src/lib-storage/index/dbox/dbox-map.h b/src/lib-storage/index/dbox/dbox-map.h index d1e67fcf2e..81f7b57889 100644 --- a/src/lib-storage/index/dbox/dbox-map.h +++ b/src/lib-storage/index/dbox/dbox-map.h @@ -8,8 +8,8 @@ struct dbox_map_append_context; struct dbox_map *dbox_map_init(struct dbox_storage *storage); void dbox_map_deinit(struct dbox_map **map); -bool dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, - uint32_t *file_id_r, uoff_t *offset_r); +int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid, + uint32_t *file_id_r, uoff_t *offset_r); struct dbox_map_append_context * dbox_map_append_begin(struct dbox_mailbox *mbox); @@ -18,10 +18,15 @@ dbox_map_append_begin(struct dbox_mailbox *mbox); -1 if error. */ int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size, struct dbox_file **file_r, struct ostream **output_r); -/* Assign file_ids to all appended files. */ -int dbox_map_append_assign_file_ids(struct dbox_map_append_context *ctx); +/* Assign map UIDs to all appended msgs to multi-files. */ +int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx, + uint32_t *first_map_uid_r, + uint32_t *last_map_uid_r); +/* Assign UIDs to all created single-files. */ +int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx, + uint32_t first_uid, uint32_t last_uid); /* Returns 0 if ok, -1 if error. */ -int dbox_map_append_commit(struct dbox_map_append_context **ctx); +void dbox_map_append_commit(struct dbox_map_append_context **ctx); void dbox_map_append_rollback(struct dbox_map_append_context **ctx); #endif diff --git a/src/lib-storage/index/dbox/dbox-save.c b/src/lib-storage/index/dbox/dbox-save.c index 579c5e99d5..de8dd492de 100644 --- a/src/lib-storage/index/dbox/dbox-save.c +++ b/src/lib-storage/index/dbox/dbox-save.c @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "fdatasync-path.h" +#include "hex-binary.h" #include "hex-dec.h" #include "str.h" #include "istream.h" @@ -19,7 +20,7 @@ struct dbox_save_mail { struct dbox_file *file; - uint32_t seq, uid; + uint32_t seq; uint32_t append_offset; uoff_t message_size; }; @@ -37,12 +38,12 @@ struct dbox_save_context { uint32_t seq; struct istream *input; struct mail *mail; - string_t *cur_keywords; struct dbox_file *cur_file; struct ostream *cur_output; ARRAY_DEFINE(mails, struct dbox_save_mail); + unsigned int single_count; unsigned int failed:1; unsigned int finished:1; @@ -124,7 +125,7 @@ int dbox_save_begin(struct mail_save_context *_ctx, struct istream *input) sizeof(dbox_msg_hdr)) < 0) { mail_storage_set_critical(_ctx->transaction->box->storage, "o_stream_send(%s) failed: %m", - dbox_file_get_path(ctx->cur_file)); + ctx->cur_file->current_path); ctx->failed = TRUE; } @@ -137,18 +138,16 @@ int dbox_save_continue(struct mail_save_context *_ctx) { struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; struct mail_storage *storage = &ctx->mbox->storage->storage; - const char *cur_path; if (ctx->failed) return -1; - cur_path = dbox_file_get_path(ctx->cur_file); do { if (o_stream_send_istream(ctx->cur_output, ctx->input) < 0) { if (!mail_storage_set_error_from_errno(storage)) { mail_storage_set_critical(storage, "o_stream_send_istream(%s) failed: %m", - cur_path); + ctx->cur_file->current_path); } ctx->failed = TRUE; return -1; @@ -165,8 +164,10 @@ int dbox_save_continue(struct mail_save_context *_ctx) static void dbox_save_write_metadata(struct dbox_save_context *ctx) { struct dbox_metadata_header metadata_hdr; + uint8_t guid_128[16]; const char *guid; string_t *str; + buffer_t *guid_buf; uoff_t vsize; memset(&metadata_hdr, 0, sizeof(metadata_hdr)); @@ -175,7 +176,6 @@ static void dbox_save_write_metadata(struct dbox_save_context *ctx) o_stream_send(ctx->cur_output, &metadata_hdr, sizeof(metadata_hdr)); str = t_str_new(256); - /* write first fields that don't change */ str_printfa(str, "%c%lx\n", DBOX_METADATA_RECEIVED_TIME, (unsigned long)ctx->ctx.received_date); str_printfa(str, "%c%lx\n", DBOX_METADATA_SAVE_TIME, @@ -185,11 +185,35 @@ static void dbox_save_write_metadata(struct dbox_save_context *ctx) str_printfa(str, "%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE, (unsigned long long)vsize); - guid = ctx->ctx.guid != NULL ? ctx->ctx.guid : - mail_generate_guid_string(); + /* we can use user-given GUID if + a) we're not saving to a multi-file, + b) it's 128 bit hex-encoded */ + guid = ctx->ctx.guid; + if (ctx->ctx.guid != NULL && ctx->cur_file->single_mbox == NULL) { + guid_buf = buffer_create_dynamic(pool_datastack_create(), + sizeof(guid_128)); + if (strlen(guid) != sizeof(guid_128)*2 || + hex_to_binary(guid, guid_buf) < 0 || + guid_buf->used != sizeof(guid_128)) + guid = NULL; + else + memcpy(guid_128, guid_buf->data, sizeof(guid_128)); + } + + if (guid == NULL) { + mail_generate_guid_128(guid_128); + guid = binary_to_hex(guid_128, sizeof(guid_128)); + } + if (ctx->cur_file->single_mbox == NULL) { + /* multi-file: save the 128bit GUID to index so if the map + index gets corrupted we can still find the message */ + mail_index_update_ext(ctx->trans, ctx->seq, + ctx->mbox->guid_ext_id, + guid_128, NULL); + } str_printfa(str, "%c%s\n", DBOX_METADATA_GUID, guid); - str_append_c(str, '\n'); + str_append_c(str, '\n'); o_stream_send(ctx->cur_output, str_data(str), str_len(str)); } @@ -202,14 +226,15 @@ static int dbox_save_mail_write_header(struct dbox_save_mail *mail) dbox_msg_header_fill(&dbox_msg_hdr, mail->message_size); if (pwrite_full(mail->file->fd, &dbox_msg_hdr, sizeof(dbox_msg_hdr), mail->append_offset) < 0) { - dbox_file_set_syscall_error(mail->file, "write"); + dbox_file_set_syscall_error(mail->file, "write()"); return -1; } - // FIXME: don't fsync here. especially with multi files + /* we're done writing to single-files now, so fsync them here. */ if ((mail->file->storage->storage.flags & - MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0) { + MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0 && + mail->file->single_mbox != NULL) { if (fdatasync(mail->file->fd) < 0) { - dbox_file_set_syscall_error(mail->file, "fdatasync"); + dbox_file_set_syscall_error(mail->file, "fdatasync()"); return -1; } } @@ -221,7 +246,7 @@ static int dbox_save_finish_write(struct mail_save_context *_ctx) struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx; struct mail_storage *storage = &ctx->mbox->storage->storage; struct dbox_save_mail *save_mail; - uoff_t offset = 0; + uoff_t metadata_offset = 0; unsigned int count; ctx->finished = TRUE; @@ -232,14 +257,12 @@ static int dbox_save_finish_write(struct mail_save_context *_ctx) _ctx->received_date, !ctx->failed); if (!ctx->failed) T_BEGIN { - const char *cur_path; - - cur_path = dbox_file_get_path(ctx->cur_file); - offset = ctx->cur_output->offset; + metadata_offset = ctx->cur_output->offset; dbox_save_write_metadata(ctx); if (o_stream_flush(ctx->cur_output) < 0) { mail_storage_set_critical(storage, - "o_stream_flush(%s) failed: %m", cur_path); + "o_stream_flush(%s) failed: %m", + ctx->cur_file->current_path); ctx->failed = TRUE; } } T_END; @@ -249,21 +272,26 @@ static int dbox_save_finish_write(struct mail_save_context *_ctx) count = array_count(&ctx->mails); save_mail = array_idx_modifiable(&ctx->mails, count - 1); + if (!ctx->failed) { + dbox_file_finish_append(save_mail->file); + save_mail->message_size = metadata_offset - + save_mail->append_offset - + save_mail->file->msg_header_size; + if (dbox_save_mail_write_header(save_mail) < 0) + ctx->failed = TRUE; + } if (ctx->failed) { dbox_file_cancel_append(save_mail->file, save_mail->append_offset); array_delete(&ctx->mails, count - 1, 1); return -1; - } else { - dbox_file_finish_append(save_mail->file); - save_mail->message_size = offset - save_mail->append_offset - - save_mail->file->msg_header_size; - dbox_save_mail_write_header(save_mail); + } - if (save_mail->file->single_mbox != NULL) - dbox_file_close(save_mail->file); - return 0; + if (save_mail->file->single_mbox != NULL) { + dbox_file_close(save_mail->file); + ctx->single_count++; } + return 0; } int dbox_save_finish(struct mail_save_context *ctx) @@ -283,68 +311,72 @@ void dbox_save_cancel(struct mail_save_context *_ctx) (void)dbox_save_finish(_ctx); } -static int dbox_save_commit(struct dbox_save_context *ctx, uint32_t first_uid) -{ - struct dbox_mail_index_record rec; - struct dbox_save_mail *mails; - unsigned int i, count; - - /* assign UIDs to mails */ - mails = array_get_modifiable(&ctx->mails, &count); - for (i = 0; i < count; i++) { - mails[i].uid = first_uid++; - mails[i].file->last_append_uid = mails[i].uid; - } - if (dbox_map_append_assign_file_ids(ctx->append_ctx) < 0) - return -1; - - memset(&rec, 0, sizeof(rec)); -#if 0 //FIXME - for (i = 0; i < count; i++) { - rec.map_uid = mails[i].map_uid; - if (rec.map_uid != 0) { - mail_index_update_ext(ctx->trans, mails[i].seq, - ctx->mbox->dbox_ext_id, - &rec, NULL); - } - } -#endif - return 0; -} - int dbox_transaction_save_commit_pre(struct dbox_save_context *ctx) { struct dbox_transaction_context *t = (struct dbox_transaction_context *)ctx->ctx.transaction; const struct mail_index_header *hdr; - uint32_t uid, next_uid; + uint32_t uid, first_map_uid, last_map_uid, next_uid; i_assert(ctx->finished); + /* get map UIDs for messages saved to multi-files */ + if (dbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid, + &last_map_uid) < 0) { + dbox_transaction_save_rollback(ctx); + return -1; + } + + /* lock the mailbox */ if (dbox_sync_begin(ctx->mbox, TRUE, &ctx->sync_ctx) < 0) { - ctx->failed = TRUE; dbox_transaction_save_rollback(ctx); return -1; } + /* assign UIDs for new messages */ hdr = mail_index_get_header(ctx->sync_ctx->sync_view); uid = hdr->next_uid; mail_index_append_assign_uids(ctx->trans, uid, &next_uid); - if (dbox_save_commit(ctx, uid) < 0) { - ctx->failed = TRUE; - dbox_transaction_save_rollback(ctx); - return -1; + /* if we saved any single-files, rename the files to contain UIDs */ + if (ctx->single_count > 0) { + uint32_t last_uid = uid + ctx->single_count - 1; + + if (dbox_map_append_assign_uids(ctx->append_ctx, uid, + last_uid) < 0) { + dbox_transaction_save_rollback(ctx); + return -1; + } } - *t->ictx.saved_uid_validity = hdr->uid_validity; - *t->ictx.first_saved_uid = uid; - *t->ictx.last_saved_uid = next_uid - 1; + /* add map_uids for all messages saved to multi-files */ + if (first_map_uid != 0) { + struct dbox_mail_index_record rec; + const struct dbox_save_mail *mails; + unsigned int i, count; + uint32_t next_map_uid = first_map_uid; - dbox_map_append_commit(&ctx->append_ctx); + memset(&rec, 0, sizeof(rec)); + mails = array_get(&ctx->mails, &count); + for (i = 0; i < count; i++) { + if (mails[i].file->single_mbox != NULL) + continue; + rec.map_uid = next_map_uid++; + mail_index_update_ext(ctx->trans, mails[i].seq, + ctx->mbox->dbox_ext_id, + &rec, NULL); + } + i_assert(next_map_uid == last_map_uid + 1); + } + + dbox_map_append_commit(&ctx->append_ctx); if (ctx->mail != NULL) mail_free(&ctx->mail); + + *t->ictx.saved_uid_validity = hdr->uid_validity; + *t->ictx.first_saved_uid = uid; + *t->ictx.last_saved_uid = next_uid - 1; return 0; } @@ -375,8 +407,6 @@ void dbox_transaction_save_rollback(struct dbox_save_context *ctx) if (ctx->mail != NULL) mail_free(&ctx->mail); - if (ctx->cur_keywords != NULL) - str_free(&ctx->cur_keywords); array_free(&ctx->mails); i_free(ctx); } diff --git a/src/lib-storage/index/dbox/dbox-storage.c b/src/lib-storage/index/dbox/dbox-storage.c index 183785186e..193e6f40b1 100644 --- a/src/lib-storage/index/dbox/dbox-storage.c +++ b/src/lib-storage/index/dbox/dbox-storage.c @@ -133,15 +133,6 @@ static int dbox_create(struct mail_storage *_storage, const char *data, return -1; } - if (mailbox_list_alloc(layout, &_storage->list, error_r) < 0) - return -1; - storage->list_module_ctx.super = _storage->list->v; - storage->alt_dir = p_strdup(_storage->pool, alt_dir); - _storage->list->v.iter_is_mailbox = dbox_list_iter_is_mailbox; - _storage->list->v.delete_mailbox = dbox_list_delete_mailbox; - _storage->list->v.rename_mailbox = dbox_list_rename_mailbox; - _storage->list->v.rename_mailbox_pre = dbox_list_rename_mailbox_pre; - value = getenv("DBOX_ROTATE_SIZE"); if (value != NULL) storage->rotate_size = (uoff_t)strtoul(value, NULL, 10) * 1024; @@ -159,13 +150,32 @@ static int dbox_create(struct mail_storage *_storage, const char *data, storage->rotate_days = (unsigned int)strtoul(value, NULL, 10); else storage->rotate_days = DBOX_DEFAULT_ROTATE_DAYS; - value = getenv("DBOX_MAX_OPEN_FILES"); if (value != NULL) storage->max_open_files = (unsigned int)strtoul(value, NULL, 10); else storage->max_open_files = DBOX_DEFAULT_MAX_OPEN_FILES; + if (storage->max_open_files <= 1) { + /* we store file offsets in a 32bit integer. */ + *error_r = "dbox_max_open_files must be at least 2"; + return -1; + } + if (storage->rotate_size > (uint32_t)-1) { + /* we store file offsets in a 32bit integer. */ + *error_r = "dbox_rotate_size must be less than 4 GB"; + return -1; + } + + if (mailbox_list_alloc(layout, &_storage->list, error_r) < 0) + return -1; + storage->list_module_ctx.super = _storage->list->v; + storage->alt_dir = p_strdup(_storage->pool, alt_dir); + _storage->list->v.iter_is_mailbox = dbox_list_iter_is_mailbox; + _storage->list->v.delete_mailbox = dbox_list_delete_mailbox; + _storage->list->v.rename_mailbox = dbox_list_rename_mailbox; + _storage->list->v.rename_mailbox_pre = dbox_list_rename_mailbox_pre; + MODULE_CONTEXT_SET_FULL(_storage->list, dbox_mailbox_list_module, storage, &storage->list_module_ctx); @@ -181,7 +191,7 @@ static int dbox_create(struct mail_storage *_storage, const char *data, "/"DBOX_GLOBAL_DIR_NAME, NULL); i_array_init(&storage->open_files, I_MIN(storage->max_open_files, 128)); - storage->map_index = dbox_map_init(storage); + storage->map = dbox_map_init(storage); mailbox_list_get_permissions(storage->storage.list, &storage->create_mode, &storage->create_gid); @@ -193,7 +203,7 @@ static void dbox_destroy(struct mail_storage *_storage) struct dbox_storage *storage = (struct dbox_storage *)_storage; dbox_files_free(storage); - dbox_map_deinit(&storage->map_index); + dbox_map_deinit(&storage->map); array_free(&storage->open_files); index_storage_destroy(_storage); } @@ -270,6 +280,8 @@ dbox_open(struct dbox_storage *storage, const char *name, mbox->dbox_hdr_ext_id = mail_index_ext_register(index, "dbox-hdr", sizeof(struct dbox_index_header), 0, 0); + mbox->guid_ext_id = + mail_index_ext_register(index, "guid", 0, 16, 1); index_storage_mailbox_init(&mbox->ibox, name, flags, FALSE); mbox->maildir_uidlist = maildir_uidlist_init_readonly(&mbox->ibox); diff --git a/src/lib-storage/index/dbox/dbox-storage.h b/src/lib-storage/index/dbox/dbox-storage.h index 7b729c5dd9..ddfb89d507 100644 --- a/src/lib-storage/index/dbox/dbox-storage.h +++ b/src/lib-storage/index/dbox/dbox-storage.h @@ -44,7 +44,7 @@ struct dbox_storage { const char *alt_dir; /* paths for storage directories */ const char *storage_dir, *alt_storage_dir; - struct dbox_map *map_index; + struct dbox_map *map; /* mode/gid to use for new dbox storage files */ mode_t create_mode; @@ -67,7 +67,7 @@ struct dbox_mailbox { struct maildir_uidlist *maildir_uidlist; uint32_t highest_maildir_uid; - uint32_t dbox_ext_id, dbox_hdr_ext_id; + uint32_t dbox_ext_id, dbox_hdr_ext_id, guid_ext_id; const char *path, *alt_path; }; diff --git a/src/lib-storage/index/dbox/dbox-sync-file.c b/src/lib-storage/index/dbox/dbox-sync-file.c index 1f28a3d5a2..b785851ab1 100644 --- a/src/lib-storage/index/dbox/dbox-sync-file.c +++ b/src/lib-storage/index/dbox/dbox-sync-file.c @@ -134,7 +134,7 @@ dbox_sync_file_expunge(struct dbox_sync_context *ctx, struct dbox_file *file, ret = dbox_sync_file_unlink(file); } else { if (ftruncate(file->fd, first_offset) < 0) { - dbox_file_set_syscall_error(file, "ftruncate"); + dbox_file_set_syscall_error(file, "ftruncate()"); ret = -1; } } @@ -292,7 +292,7 @@ int dbox_sync_file(struct dbox_sync_context *ctx, ret = 1; } } else { - ret = dbox_file_open_or_create(file, TRUE, &deleted); + ret = dbox_file_open_or_create(file, &deleted); if (ret > 0 && !deleted) { dbox_sync_file_move_if_needed(file, entry); if (array_is_created(&entry->expunges)) diff --git a/src/lib-storage/index/dbox/dbox-sync-rebuild.c b/src/lib-storage/index/dbox/dbox-sync-rebuild.c index 640720909b..25c9b16802 100644 --- a/src/lib-storage/index/dbox/dbox-sync-rebuild.c +++ b/src/lib-storage/index/dbox/dbox-sync-rebuild.c @@ -154,36 +154,25 @@ static int dbox_sync_index_file_next(struct dbox_sync_rebuild_context *ctx, { uint32_t seq; uoff_t physical_size; - const char *path; - bool expunged; + bool expunged, last; int ret; - path = dbox_file_get_path(file); - ret = dbox_file_seek_next(file, offset, &physical_size); + ret = dbox_file_seek_next(file, offset, &physical_size, &last); if (ret <= 0) { if (ret < 0) return -1; -#if 0 //FIXME: needed? - if (physical_size == 0 && file->uid != 0) { - /* EOF */ - return 0; - } -#endif - - i_warning("%s: Ignoring broken file (header)", path); + i_warning("%s: Ignoring broken file (header)", + file->current_path); return 0; } - if (file->maildir_file) { - file->append_count = 1; - file->last_append_uid = file->uid; - } ret = dbox_file_metadata_seek_mail_offset(file, *offset, &expunged); if (ret <= 0) { if (ret < 0) return -1; - i_warning("%s: Ignoring broken file (metadata)", path); + i_warning("%s: Ignoring broken file (metadata)", + file->current_path); return 0; } if (!expunged) { @@ -340,6 +329,9 @@ static int dbox_sync_maildir_finish(struct dbox_sync_rebuild_context *ctx) uoff_t offset; int ret = 0; + if (ctx->maildir_sync_ctx == NULL) + return 0; + /* we'll need the uidlist to contain the latest filenames. since there's no easy way to figure out if they changed, just recreate the uidlist always. */ diff --git a/src/lib-storage/index/dbox/dbox-sync.c b/src/lib-storage/index/dbox/dbox-sync.c index 7c5b6f8662..6971348d09 100644 --- a/src/lib-storage/index/dbox/dbox-sync.c +++ b/src/lib-storage/index/dbox/dbox-sync.c @@ -41,6 +41,7 @@ static int dbox_sync_add_seq(struct dbox_sync_context *ctx, struct dbox_sync_file_entry *entry, lookup_entry; uint32_t map_uid; uoff_t offset; + int ret; i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE || sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS); @@ -50,9 +51,10 @@ static int dbox_sync_add_seq(struct dbox_sync_context *ctx, if (map_uid == 0) mail_index_lookup_uid(ctx->sync_view, seq, &lookup_entry.uid); else { - if (!dbox_map_lookup(ctx->mbox->storage->map_index, - map_uid, &lookup_entry.file_id, &offset)) { - // FIXME: now what? + ret = dbox_map_lookup(ctx->mbox->storage->map, map_uid, + &lookup_entry.file_id, &offset); + if (ret <= 0) { + // FIXME: ret=0 case - should we resync? return -1; } } diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c index d6e4aa90f5..407ce3f334 100644 --- a/src/lib-storage/index/index-storage.c +++ b/src/lib-storage/index/index-storage.c @@ -391,28 +391,37 @@ void index_storage_lock_notify_reset(struct index_mailbox *ibox) ibox->last_notify_type = MAILBOX_LOCK_NOTIFY_NONE; } +enum mail_index_open_flags +index_storage_get_index_open_flags(struct mail_storage *storage) +{ + enum mail_index_open_flags flags = 0; + +#ifndef MMAP_CONFLICTS_WRITE + if ((storage->flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0) +#endif + flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE; + if ((storage->flags & MAIL_STORAGE_FLAG_DOTLOCK_USE_EXCL) != 0) + flags |= MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL; + if ((storage->flags & MAIL_STORAGE_FLAG_NFS_FLUSH_INDEX) != 0) + flags |= MAIL_INDEX_OPEN_FLAG_NFS_FLUSH; + if ((storage->flags & MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0) + flags |= MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE; + return flags; +} + void index_storage_mailbox_open(struct index_mailbox *ibox) { struct mail_storage *storage = ibox->storage; - enum mail_index_open_flags index_flags = 0; + enum mail_index_open_flags index_flags; int ret; i_assert(!ibox->box.opened); + index_flags = index_storage_get_index_open_flags(storage); if (!ibox->move_to_memory) index_flags |= MAIL_INDEX_OPEN_FLAG_CREATE; -#ifndef MMAP_CONFLICTS_WRITE - if ((storage->flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0) -#endif - index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE; - if ((storage->flags & MAIL_STORAGE_FLAG_DOTLOCK_USE_EXCL) != 0) - index_flags |= MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL; - if ((storage->flags & MAIL_STORAGE_FLAG_NFS_FLUSH_INDEX) != 0) - index_flags |= MAIL_INDEX_OPEN_FLAG_NFS_FLUSH; - if ((storage->flags & MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0) { - index_flags |= MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE; + if ((index_flags & MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE) != 0) ibox->fsync_disable = TRUE; - } ret = mail_index_open(ibox->index, index_flags, storage->lock_method); if (ret <= 0 || ibox->move_to_memory) { diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h index bde4f9fee1..f6af4da3ef 100644 --- a/src/lib-storage/index/index-storage.h +++ b/src/lib-storage/index/index-storage.h @@ -99,6 +99,8 @@ void index_storage_unref(struct mail_index *index); void index_storage_destroy_unrefed(void); void index_storage_destroy(struct mail_storage *storage ATTR_UNUSED); +enum mail_index_open_flags +index_storage_get_index_open_flags(struct mail_storage *storage); void index_storage_mailbox_init(struct index_mailbox *ibox, const char *name, enum mailbox_open_flags flags, bool move_to_memory); diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h index 3901346f97..0286a85726 100644 --- a/src/lib-storage/mail-storage-private.h +++ b/src/lib-storage/mail-storage-private.h @@ -374,6 +374,7 @@ void mail_storage_set_internal_error(struct mail_storage *storage); bool mail_storage_set_error_from_errno(struct mail_storage *storage); const char *mail_generate_guid_string(void); +void mail_generate_guid_128(uint8_t guid[16]); void mail_set_expunged(struct mail *mail); void mailbox_set_deleted(struct mailbox *box); diff --git a/src/lib-storage/mail.c b/src/lib-storage/mail.c index df97355a8b..ce4769f7e0 100644 --- a/src/lib-storage/mail.c +++ b/src/lib-storage/mail.c @@ -226,3 +226,43 @@ const char *mail_generate_guid_string(void) (unsigned long)ts.tv_sec, pid, my_hostname); } + +void mail_generate_guid_128(uint8_t guid[16]) +{ + static struct timespec ts = { 0, 0 }; + static uint8_t guid_static[8]; + uint32_t pid, host_crc; + + /* we'll use the current time in nanoseconds as the initial 64bit + counter. */ + if (ts.tv_sec == 0) { + if (clock_gettime(CLOCK_REALTIME, &ts) < 0) + i_fatal("clock_gettime() failed: %m"); + pid = getpid(); + host_crc = crc32_str(my_hostname); + + guid_static[0] = (pid & 0x000000ff); + guid_static[1] = (pid & 0x0000ff00) >> 8; + guid_static[2] = (pid & 0x00ff0000) >> 16; + guid_static[3] = (pid & 0xff000000) >> 24; + guid_static[4] = (host_crc & 0x000000ff); + guid_static[5] = (host_crc & 0x0000ff00) >> 8; + guid_static[6] = (host_crc & 0x00ff0000) >> 16; + guid_static[7] = (host_crc & 0xff000000) >> 24; + } else if ((uint32_t)ts.tv_nsec < (uint32_t)-1) { + ts.tv_nsec++; + } else { + ts.tv_sec++; + ts.tv_nsec = 0; + } + + guid[0] = (ts.tv_nsec & 0x000000ff); + guid[1] = (ts.tv_nsec & 0x0000ff00) >> 8; + guid[2] = (ts.tv_nsec & 0x00ff0000) >> 16; + guid[3] = (ts.tv_nsec & 0xff000000) >> 24; + guid[4] = (ts.tv_sec & 0x000000ff); + guid[5] = (ts.tv_sec & 0x0000ff00) >> 8; + guid[6] = (ts.tv_sec & 0x00ff0000) >> 16; + guid[7] = (ts.tv_sec & 0xff000000) >> 24; + memcpy(guid + 8, guid_static, 8); +}