case DBOX_METADATA_SAVE_TIME:
if (file->fd != -1) {
if (fstat(file->fd, &st) < 0) {
- dbox_file_set_syscall_error(file, "fstat");
+ dbox_file_set_syscall_error(file, "fstat()");
return NULL;
}
} else {
- if (stat(dbox_file_get_path(file), &st) < 0) {
+ if (stat(file->current_path, &st) < 0) {
if (errno == ENOENT)
return NULL;
- dbox_file_set_syscall_error(file, "stat");
+ dbox_file_set_syscall_error(file, "stat()");
return NULL;
}
}
#include "hostpid.h"
#include "istream.h"
#include "ostream.h"
+#include "file-lock.h"
#include "mkdir-parents.h"
#include "fdatasync-path.h"
#include "str.h"
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
+#include <ctype.h>
#include <fcntl.h>
static int dbox_file_metadata_skip_header(struct dbox_file *file);
void dbox_file_set_syscall_error(struct dbox_file *file, const char *function)
{
mail_storage_set_critical(&file->storage->storage,
- "%s(%s) failed: %m", function,
- dbox_file_get_path(file));
+ "%s failed for file %s: %m",
+ function, file->current_path);
}
-static void
-dbox_file_set_corrupted(struct dbox_file *file, const char *reason)
+static void dbox_file_set_corrupted(struct dbox_file *file, const char *reason)
{
mail_storage_set_critical(&file->storage->storage,
- "%s corrupted: %s", dbox_file_get_path(file),
- reason);
+ "Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s",
+ file->current_path,
+ file->input == NULL ? 0 : file->input->v_offset, reason);
}
-
static struct dbox_file *
dbox_find_and_move_open_file(struct dbox_storage *storage, uint32_t file_id)
{
file = i_new(struct dbox_file, 1);
file->refcount = 1;
file->storage = storage;
+ file->file_id = file_id;
file->fd = -1;
file->fname = file_id == 0 ? dbox_generate_tmp_filename() :
i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, file_id);
i_assert(file->uid == 0 && file->file_id == 0);
i_assert(id != 0);
- old_path = dbox_file_get_path(file);
+ old_path = file->current_path;
if (file->single_mbox != NULL) {
new_fname = dbox_file_uid_get_fname(file->single_mbox,
id, &maildir);
void dbox_file_unref(struct dbox_file **_file)
{
struct dbox_file *file = *_file;
- struct dbox_file *const *files;
+ struct dbox_file *const *files, *oldest_file;
unsigned int i, count;
*_file = NULL;
return;
}
+ /* close the oldest file with refcount=0 */
for (i = 0; i < count; i++) {
- if (files[i] == file)
+ if (files[i]->refcount == 0)
break;
}
- i_assert(i != count);
+ oldest_file = files[i];
array_delete(&file->storage->open_files, i, 1);
+ if (oldest_file != file) {
+ dbox_file_free(oldest_file);
+ return;
+ }
+ /* have to close ourself */
}
dbox_file_free(file);
}
-static time_t day_begin_stamp(unsigned int days)
-{
- struct tm tm;
- time_t stamp;
-
- if (days == 0)
- return 0;
-
- /* get beginning of today */
- tm = *localtime(&ioloop_time);
- tm.tm_hour = 0;
- tm.tm_min = 0;
- tm.tm_sec = 0;
- stamp = mktime(&tm);
- if (stamp == (time_t)-1)
- i_panic("mktime(today) failed");
-
- return stamp - (3600*24 * (days-1));
-}
-
-bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size)
-{
- if (file->nonappendable)
- return FALSE;
-
- if (file->append_offset == 0) {
- /* messages have been expunged */
- return FALSE;
- }
-
- if (file->append_offset < file->storage->rotate_min_size ||
- file->append_offset == file->file_header_size)
- return TRUE;
- if (file->append_offset + mail_size >= file->storage->rotate_size)
- return FALSE;
- return file->create_time >= day_begin_stamp(file->storage->rotate_days);
-}
-
static int dbox_file_parse_header(struct dbox_file *file, const char *line)
{
const char *const *tmp, *value;
- unsigned int pos, version;
+ unsigned int pos;
enum dbox_header_key key;
- version = *line - '0';
- if (line[1] != ' ' || (version != 1 && version != DBOX_VERSION)) {
+ file->file_version = *line - '0';
+ if (!i_isdigit(line[0]) || line[1] != ' ' ||
+ (file->file_version != 1 && file->file_version != DBOX_VERSION)) {
dbox_file_set_corrupted(file, "Invalid dbox version");
return -1;
}
dbox_file_set_corrupted(file, "Missing message header size");
return -1;
}
-
- if (!file->nonappendable)
- file->nonappendable = !dbox_file_can_append(file, 0);
return 0;
}
i_stream_seek(file->input, 0);
line = i_stream_read_next_line(file->input);
if (line == NULL) {
- if (file->input->stream_errno == 0)
+ if (file->input->stream_errno == 0) {
+ dbox_file_set_corrupted(file,
+ "EOF while reading file header");
return 0;
+ }
- dbox_file_set_syscall_error(file, "read");
+ dbox_file_set_syscall_error(file, "read()");
return -1;
}
file->file_header_size = file->input->v_offset;
return 1;
}
-static int dbox_file_open(struct dbox_file *file, bool read_header,
- bool *deleted_r)
+static int dbox_file_open(struct dbox_file *file, bool *deleted_r)
{
int ret;
}
file->input = i_stream_create_fd(file->fd, MAIL_READ_BLOCK_SIZE, FALSE);
- return !read_header || file->maildir_file ? 1 :
+ return file->maildir_file ? 1 :
dbox_file_read_header(file);
}
file->file_header_size = str_len(hdr);
file->msg_header_size = sizeof(struct dbox_message_header);
- file->append_offset = str_len(hdr);
if (o_stream_send(file->output, str_data(hdr), str_len(hdr)) < 0) {
- dbox_file_set_syscall_error(file, "write");
+ dbox_file_set_syscall_error(file, "write()");
return -1;
}
return 0;
}
-int dbox_file_open_or_create(struct dbox_file *file, bool read_header,
- bool *deleted_r)
+int dbox_file_open_or_create(struct dbox_file *file, bool *deleted_r)
{
int ret;
} else if (file->input != NULL)
return 1;
else
- return dbox_file_open(file, read_header, deleted_r);
+ return dbox_file_open(file, deleted_r);
}
int dbox_file_open_if_needed(struct dbox_file *file)
void dbox_file_close(struct dbox_file *file)
{
+ i_assert(file->lock == NULL);
+
if (file->input != NULL)
i_stream_unref(&file->input);
if (file->output != NULL)
o_stream_unref(&file->output);
if (file->fd != -1) {
if (close(file->fd) < 0)
- dbox_file_set_syscall_error(file, "close");
+ dbox_file_set_syscall_error(file, "close()");
file->fd = -1;
}
}
-const char *dbox_file_get_path(struct dbox_file *file)
+int dbox_file_try_lock(struct dbox_file *file)
+{
+ i_assert(file->fd != -1);
+
+ return file_try_lock(file->fd, file->current_path, F_WRLCK,
+ FILE_LOCK_METHOD_FCNTL, &file->lock);
+
+}
+
+void dbox_file_unlock(struct dbox_file *file)
{
- return file->current_path;
+ if (file->lock != NULL)
+ file_unlock(&file->lock);
}
static int
if (file->maildir_file) {
if (fstat(file->fd, &st) < 0) {
- dbox_file_set_syscall_error(file, "fstat");
+ dbox_file_set_syscall_error(file, "fstat()");
return -1;
}
*physical_size_r = st.st_size;
file->msg_header_size - 1);
if (ret <= 0) {
if (file->input->stream_errno == 0) {
- /* EOF, broken offset */
+ /* EOF, broken offset or file truncated */
+ dbox_file_set_corrupted(file, t_strdup_printf(
+ "EOF reading msg header "
+ "(got %"PRIuSIZE_T"/%u bytes)",
+ size, file->msg_header_size));
return 0;
}
- dbox_file_set_syscall_error(file, "read");
+ dbox_file_set_syscall_error(file, "read()");
return -1;
}
- if (data[file->msg_header_size-1] != '\n')
+ if (data[file->msg_header_size-1] != '\n') {
+ dbox_file_set_corrupted(file, "msg header doesn't end with LF");
return 0;
+ }
memcpy(&hdr, data, I_MIN(sizeof(hdr), file->msg_header_size));
if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) {
/* probably broken offset */
+ dbox_file_set_corrupted(file, "bad magic value");
return 0;
}
*expunged_r = FALSE;
if (file->input == NULL) {
- if ((ret = dbox_file_open(file, TRUE, expunged_r)) <= 0 ||
+ if ((ret = dbox_file_open(file, expunged_r)) <= 0 ||
*expunged_r)
return ret;
}
if (ret <= 0)
return ret;
}
+ i_stream_seek(file->input, offset + file->msg_header_size);
if (stream_r != NULL) {
- i_stream_seek(file->input, offset + file->msg_header_size);
*stream_r = i_stream_create_limit(file->input,
file->cur_physical_size);
}
}
int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset,
- uoff_t *physical_size_r)
+ uoff_t *physical_size_r, bool *last_r)
{
uoff_t size;
bool first = *offset == 0;
bool deleted;
int ret;
+ *last_r = FALSE;
+
ret = dbox_file_get_mail_stream(file, *offset, &size, NULL,
&deleted);
if (ret <= 0)
return ret;
if (deleted) {
- *physical_size_r = 0;
+ *last_r = TRUE;
return 1;
}
if (first) {
return dbox_file_seek_next_at_metadata(file, offset, physical_size_r);
}
-static int dbox_file_seek_append_pos(struct dbox_file *file, uoff_t mail_size)
+static int
+dbox_file_seek_append_pos(struct dbox_file *file, uoff_t last_msg_offset)
{
+ uoff_t offset, size;
+ bool last;
int ret;
if ((ret = dbox_file_read_header(file)) <= 0)
return ret;
- if (file->append_offset == 0 ||
- file->msg_header_size != sizeof(struct dbox_message_header) ||
- !dbox_file_can_append(file, mail_size)) {
- /* can't append */
+ if (file->file_version != DBOX_VERSION ||
+ file->msg_header_size != sizeof(struct dbox_message_header)) {
+ /* created by an incompatible version, can't append */
return 0;
}
- file->output = o_stream_create_fd_file(file->fd, (uoff_t)-2, FALSE);
- o_stream_seek(file->output, file->append_offset);
+ offset = last_msg_offset;
+ ret = dbox_file_seek_next(file, &offset, &size, &last);
+ if (ret <= 0)
+ return ret;
+ if (!last) {
+ /* not end of file? previous write probably crashed. */
+ if (ftruncate(file->fd, offset) < 0) {
+ dbox_file_set_syscall_error(file, "ftruncate()");
+ return -1;
+ }
+ i_stream_sync(file->input);
+ }
+
+ file->output = o_stream_create_fd_file(file->fd, 0, FALSE);
+ o_stream_seek(file->output, offset);
return 1;
}
-static int
-dbox_file_get_append_stream_int(struct dbox_file *file, uoff_t mail_size,
+int dbox_file_get_append_stream(struct dbox_file *file, uoff_t last_msg_offset,
struct ostream **stream_r)
{
bool deleted;
int ret;
if (file->fd == -1) {
+ /* creating a new file */
i_assert(file->output == NULL);
- if ((ret = dbox_file_open_or_create(file, FALSE,
- &deleted)) <= 0 || deleted)
- return ret;
- }
- if (file->output == NULL) {
- ret = dbox_file_seek_append_pos(file, mail_size);
+ ret = dbox_file_open_or_create(file, &deleted);
if (ret <= 0)
return ret;
- } else {
- if (!dbox_file_can_append(file, mail_size))
+ if (deleted)
return 0;
}
- if (file->output->offset > (uint32_t)-1) {
- /* we use 32bit offsets to messages */
- return 0;
+ if (file->output == NULL) {
+ i_assert(file->lock != NULL || last_msg_offset == 0);
+
+ ret = dbox_file_seek_append_pos(file, last_msg_offset);
+ if (ret <= 0)
+ return ret;
}
o_stream_ref(file->output);
return 1;
}
-int dbox_file_get_append_stream(struct dbox_file *file, uoff_t mail_size,
- struct ostream **stream_r)
-{
- int ret;
-
- if (file->append_count == 0) {
- if (file->nonappendable)
- return 0;
- } else {
- if (!dbox_file_can_append(file, mail_size))
- return 0;
- }
-
- ret = dbox_file_get_append_stream_int(file, mail_size, stream_r);
- if (ret == 0)
- file->nonappendable = TRUE;
- return ret;
-}
-
uoff_t dbox_file_get_next_append_offset(struct dbox_file *file)
{
- i_assert(file->output_stream_offset != 0);
- i_assert(file->output == NULL ||
- file->output_stream_offset == file->output->offset);
+ i_assert(file->output != NULL);
- return file->output_stream_offset;
+ return file->output->offset;
}
void dbox_file_cancel_append(struct dbox_file *file, uoff_t append_offset)
{
- if (ftruncate(file->fd, append_offset) < 0) {
- dbox_file_set_syscall_error(file, "ftruncate");
- file->append_offset = 0;
- file->nonappendable = TRUE;
- }
+ if (ftruncate(file->fd, append_offset) < 0)
+ dbox_file_set_syscall_error(file, "ftruncate()");
+ if (file->input != NULL)
+ i_stream_sync(file->input);
o_stream_seek(file->output, append_offset);
- file->output_stream_offset = append_offset;
}
void dbox_file_finish_append(struct dbox_file *file)
{
- file->output_stream_offset = file->output->offset;
- file->append_offset = file->output->offset;
- file->append_count++;
+ if (file->input != NULL)
+ i_stream_sync(file->input);
}
static uoff_t
/* EOF, broken offset */
return 0;
}
- dbox_file_set_syscall_error(file, "read");
+ dbox_file_set_syscall_error(file, "read()");
return -1;
}
memcpy(&metadata_hdr, data, sizeof(metadata_hdr));
i_assert(!file->maildir_file); /* previous check should catch this */
if (file->input == NULL) {
- if ((ret = dbox_file_open(file, TRUE, &deleted)) <= 0)
+ if ((ret = dbox_file_open(file, &deleted)) <= 0)
return ret;
if (deleted) {
*expunged_r = TRUE;
return 1;
}
- } else {
- /* make sure to flush any cached data */
- i_stream_sync(file->input);
}
i_stream_seek(file->input, metadata_offset);
out_fd = open(temp_path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
if (out_fd == -1 && errno == ENOENT) {
if (mkdir_parents(dest_dir, 0700) < 0 && errno != EEXIST) {
- i_error("mkdir_parents(%s) failed: %m", dest_dir);
+ mail_storage_set_critical(&file->storage->storage,
+ "mkdir_parents(%s) failed: %m", dest_dir);
return -1;
}
out_fd = open(temp_path, O_WRONLY | O_CREAT | O_TRUNC, 0600);
}
if (out_fd == -1) {
- i_error("open(%s, O_CREAT) failed: %m", temp_path);
+ mail_storage_set_critical(&file->storage->storage,
+ "open(%s, O_CREAT) failed: %m", temp_path);
return -1;
}
output = o_stream_create_fd_file(out_fd, 0, FALSE);
ret = o_stream_flush(output);
if (output->stream_errno != 0) {
errno = output->stream_errno;
- i_error("write(%s) failed: %m", temp_path);
+ mail_storage_set_critical(&file->storage->storage,
+ "write(%s) failed: %m", temp_path);
ret = -1;
} else if (file->input->stream_errno != 0) {
errno = file->input->stream_errno;
- i_error("read(%s) failed: %m", file->current_path);
+ dbox_file_set_syscall_error(file, "ftruncate()");
ret = -1;
} else if (ret < 0) {
- i_error("o_stream_send_istream(%s, %s) "
+ mail_storage_set_critical(&file->storage->storage,
+ "o_stream_send_istream(%s, %s) "
"failed with unknown error",
temp_path, file->current_path);
}
if ((file->storage->storage.flags &
MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0 && ret == 0) {
if (fsync(out_fd) < 0) {
- i_error("fsync(%s) failed: %m", temp_path);
+ mail_storage_set_critical(&file->storage->storage,
+ "fsync(%s) failed: %m", temp_path);
ret = -1;
}
}
if (close(out_fd) < 0) {
- i_error("close(%s) failed: %m", temp_path);
+ mail_storage_set_critical(&file->storage->storage,
+ "close(%s) failed: %m", temp_path);
ret = -1;
}
if (ret < 0) {
its contents should be the same (except for maybe older metadata) */
dest_path = t_strdup_printf("%s/%s", dest_dir, file->fname);
if (rename(temp_path, dest_path) < 0) {
- i_error("rename(%s, %s) failed: %m", temp_path, dest_path);
+ mail_storage_set_critical(&file->storage->storage,
+ "rename(%s, %s) failed: %m", temp_path, dest_path);
(void)unlink(temp_path);
return -1;
}
if ((file->storage->storage.flags &
MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0) {
if (fdatasync_path(dest_dir) < 0) {
- i_error("fdatasync(%s) failed: %m", dest_dir);
+ mail_storage_set_critical(&file->storage->storage,
+ "fdatasync(%s) failed: %m", dest_dir);
(void)unlink(dest_path);
return -1;
}
}
if (unlink(file->current_path) < 0) {
- i_error("unlink(%s) failed: %m", file->current_path);
+ dbox_file_set_syscall_error(file, "unlink()");
if (errno == EACCES) {
/* configuration problem? revert the write */
(void)unlink(dest_path);
/* file was successfully moved - reopen it */
dbox_file_close(file);
- if (dbox_file_open(file, TRUE, &deleted) <= 0) {
- i_error("dbox_file_move(%s): reopening file failed", dest_path);
+ if (dbox_file_open(file, &deleted) <= 0) {
+ mail_storage_set_critical(&file->storage->storage,
+ "dbox_file_move(%s): reopening file failed", dest_path);
return -1;
}
return 0;
/* uid is for single-msg-per-file, file_id for multi-msgs-per-file */
uint32_t uid, file_id;
+ time_t create_time;
+ unsigned int file_version;
unsigned int file_header_size;
unsigned int msg_header_size;
- unsigned int append_count;
- uint32_t last_append_uid;
-
- uoff_t append_offset;
- time_t create_time;
- uoff_t output_stream_offset;
-
uoff_t cur_offset;
uoff_t cur_physical_size;
int fd;
struct istream *input;
struct ostream *output;
+ struct file_lock *lock;
/* Metadata for the currently seeked metadata block. */
pool_t metadata_pool;
unsigned int alt_path:1;
unsigned int maildir_file:1;
- unsigned int nonappendable:1;
unsigned int deleted:1;
};
int dbox_file_assign_id(struct dbox_file *file, uint32_t id);
/* Open the file if uid or file_id is not 0, otherwise create it. Returns 1 if
- ok, 0 if read_header=TRUE and opened file was broken, -1 if error. If file
- is deleted, deleted_r=TRUE and 1 is returned. */
-int dbox_file_open_or_create(struct dbox_file *file, bool read_header,
- bool *deleted_r);
+ ok, 0 if file header is corrupted, -1 if error. If file is deleted,
+ deleted_r=TRUE and 1 is returned. */
+int dbox_file_open_or_create(struct dbox_file *file, bool *deleted_r);
/* Open the file's fd if it's currently closed. Assumes that the file exists. */
int dbox_file_open_if_needed(struct dbox_file *file);
/* Close the file handle from the file, but don't free it. */
void dbox_file_close(struct dbox_file *file);
-/* Returns the current fulle path for an opened/created file. It's an error to
- call this function for a non-opened file. */
-const char *dbox_file_get_path(struct dbox_file *file);
+/* Try to lock the dbox file. Returns 1 if ok, 0 if already locked by someone
+ else, -1 if error. */
+int dbox_file_try_lock(struct dbox_file *file);
+void dbox_file_unlock(struct dbox_file *file);
/* Seek to given offset in file and return the message's input stream
and physical size. Returns 1 if ok, 0 if file/offset is corrupted,
uoff_t *physical_size_r,
struct istream **stream_r, bool *expunged_r);
/* Seek to next message after given offset, or to first message if offset=0.
- If there are no more messages, physical_size_r is set to 0. Returns 1 if ok,
+ If there are no more messages, last_r is set to TRUE. Returns 1 if ok,
0 if file/offset is corrupted, -1 if I/O error. */
int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset,
- uoff_t *physical_size_r);
+ uoff_t *physical_size_r, bool *last_r);
/* Returns TRUE if mail_size bytes can be appended to the file. */
bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size);
-/* Get output stream for appending a new message. Returns 1 if ok, 0 if
- file can't be appended to (limits reached, expunges, corrupted) or
- -1 if error. If 0 is returned, index is also updated. */
-int dbox_file_get_append_stream(struct dbox_file *file, uoff_t mail_size,
+/* Get output stream for appending a new message. last_msg_offset points to
+ the beginning of the last message in the file, or 0 for new files. Returns
+ 1 if ok, 0 if file can't be appended to (old file version or corruption)
+ or -1 if error. */
+int dbox_file_get_append_stream(struct dbox_file *file, uoff_t last_msg_offset,
struct ostream **stream_r);
/* Returns the next offset for append a message. dbox_file_get_append_stream()
must have been called for this file already at least once. */
struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox;
struct mail *_mail = &mail->imail.mail.mail;
uint32_t map_uid, file_id;
+ int ret;
if (mail->open_file == NULL) {
map_uid = dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq);
if (map_uid == 0) {
mail->open_file =
dbox_file_init_single(mbox, _mail->uid);
- } else if (!dbox_map_lookup(mbox->storage->map_index, map_uid,
- &file_id, &mail->offset)) {
- mail_set_expunged(_mail);
- return -1;
} else {
+ ret = dbox_map_lookup(mbox->storage->map, map_uid,
+ &file_id, &mail->offset);
+ if (ret <= 0) {
+ // FIXME: ret=0 case - should we resync?
+ if (ret == 0)
+ mail_set_expunged(_mail);
+ return -1;
+ }
mail->open_file =
dbox_file_init_multi(mbox->storage, file_id);
}
static int
dbox_mail_metadata_seek(struct dbox_mail *mail, struct dbox_file **file_r)
{
+ struct mail *_mail = &mail->imail.mail.mail;
uoff_t offset;
bool expunged;
int ret;
if (ret < 0)
return -1;
/* FIXME */
+ mail_storage_set_critical(mail->imail.ibox->storage,
+ "Broken metadata in dbox file %s offset %"PRIuUOFF_T
+ " (uid=%u)",
+ (*file_r)->current_path, offset, _mail->uid);
return -1;
}
if (expunged) {
- mail_set_expunged(&mail->imail.mail.mail);
+ mail_set_expunged(_mail);
return -1;
}
return 0;
if (ret > 0)
i_stream_unref(&input);
mail_storage_set_critical(_mail->box->storage,
- "broken pointer to dbox file %s",
- mail->open_file->current_path);
+ "broken pointer to dbox file %s "
+ "offset %"PRIuUOFF_T" (uid=%u)",
+ mail->open_file->current_path,
+ offset, _mail->uid);
return -1;
}
data->physical_size = size;
#include "lib.h"
#include "array.h"
+#include "ostream.h"
#include "dbox-storage.h"
#include "dbox-file.h"
#include "dbox-map.h"
+#define MAX_BACKWARDS_LOOKUPS 10
+
+struct dbox_mail_index_map_header {
+ uint32_t highest_file_id;
+};
+
+struct dbox_mail_index_map_record {
+ uint32_t file_id;
+ uint32_t offset;
+};
+
struct dbox_map {
struct dbox_storage *storage;
struct mail_index *index;
+ struct mail_index_view *view;
+
+ uint32_t map_ext_id, ref_ext_id;
+};
+
+struct dbox_map_append {
+ struct dbox_file *file;
+ uoff_t offset;
};
struct dbox_map_append_context {
struct dbox_mailbox *mbox;
+ struct dbox_map *map;
ARRAY_DEFINE(files, struct dbox_file *);
+ ARRAY_DEFINE(appends, struct dbox_map_append);
- uoff_t output_offset;
- unsigned int new_record_idx;
uint32_t first_new_file_id;
+ uint32_t orig_msg_count;
- unsigned int locked_header:1;
+ unsigned int files_nonappendable_count;
+ uint32_t retry_seq_min, retry_seq_max;
+
+ unsigned int failed:1;
};
struct dbox_map *dbox_map_init(struct dbox_storage *storage)
map->storage = storage;
map->index = mail_index_alloc(storage->storage_dir,
DBOX_GLOBAL_INDEX_PREFIX);
+ map->map_ext_id = mail_index_ext_register(map->index, "map",
+ sizeof(struct dbox_mail_index_map_header),
+ sizeof(struct dbox_mail_index_map_record),
+ sizeof(uint32_t));
+ map->ref_ext_id = mail_index_ext_register(map->index, "ref", 0,
+ sizeof(uint16_t), sizeof(uint16_t));
return map;
}
*_map = NULL;
+ if (map->view != NULL)
+ mail_index_view_close(&map->view);
mail_index_free(&map->index);
i_free(map);
}
-bool dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
- uint32_t *file_id_r, uoff_t *offset_r)
+static int dbox_map_open(struct dbox_map *map, bool create)
{
- return FALSE;
+ struct mail_storage *storage = &map->storage->storage;
+ enum mail_index_open_flags open_flags;
+ int ret;
+
+ if (map->view != NULL) {
+ /* already opened */
+ return 1;
+ }
+
+ open_flags = index_storage_get_index_open_flags(storage);
+ if (create)
+ open_flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
+
+ ret = mail_index_open(map->index, open_flags, storage->lock_method);
+ if (ret <= 0) {
+ mail_storage_set_internal_error(storage);
+ mail_index_reset_error(map->index);
+ return ret;
+ }
+
+ map->view = mail_index_view_open(map->index);
+ return 1;
+}
+
+static int dbox_map_refresh(struct dbox_map *map)
+{
+ struct mail_index_view_sync_ctx *ctx;
+ bool delayed_expunges;
+
+ ctx = mail_index_view_sync_begin(map->view,
+ MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT);
+ if (mail_index_view_sync_commit(&ctx, &delayed_expunges) < 0) {
+ mail_storage_set_internal_error(&map->storage->storage);
+ mail_index_reset_error(map->index);
+ return -1;
+ }
+ return 0;
+}
+
+static int dbox_map_lookup_seq(struct dbox_map *map, uint32_t seq,
+ uint32_t *file_id_r, uoff_t *offset_r)
+{
+ const struct dbox_mail_index_map_record *rec;
+ const void *data;
+ bool expunged;
+
+ mail_index_lookup_ext(map->view, seq, map->map_ext_id,
+ &data, &expunged);
+ rec = data;
+
+ if (rec == NULL || rec->file_id == 0) {
+ /* corrupted */
+ mail_storage_set_critical(&map->storage->storage,
+ "dbox map %s corrupted: file_id=0 for seq=%u",
+ map->index->filepath, seq);
+ return -1;
+ }
+
+ *file_id_r = rec->file_id;
+ *offset_r = rec->offset;
+ return 0;
+}
+
+int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
+ uint32_t *file_id_r, uoff_t *offset_r)
+{
+ uint32_t seq;
+ int ret;
+
+ if ((ret = dbox_map_open(map, FALSE)) <= 0) {
+ /* map doesn't exist or is broken */
+ return ret;
+ }
+
+ if (!mail_index_lookup_seq(map->view, map_uid, &seq)) {
+ /* not found - try again after a refresh */
+ if (dbox_map_refresh(map) < 0)
+ return -1;
+ if (!mail_index_lookup_seq(map->view, map_uid, &seq))
+ return 0;
+ }
+
+ if (dbox_map_lookup_seq(map, seq, file_id_r, offset_r) < 0)
+ return 0;
+ return 1;
}
struct dbox_map_append_context *
dbox_map_append_begin(struct dbox_mailbox *mbox)
{
struct dbox_map_append_context *ctx;
+ int ret;
ctx = i_new(struct dbox_map_append_context, 1);
+ ctx->map = mbox->storage->map;
ctx->mbox = mbox;
ctx->first_new_file_id = (uint32_t)-1;
i_array_init(&ctx->files, 64);
+ i_array_init(&ctx->appends, 128);
+
+ if ((ret = dbox_map_open(ctx->map, TRUE)) <= 0) {
+ i_assert(ret != 0);
+ ctx->failed = TRUE;
+ }
+ /* refresh the map so we can try appending to the latest files */
+ (void)dbox_map_refresh(ctx->map);
return ctx;
}
-int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
- struct dbox_file **file_r, struct ostream **output_r)
+static time_t day_begin_stamp(unsigned int days)
{
- struct dbox_file *const *files, *file = NULL;
- unsigned int i, count;
+ struct tm tm;
+ time_t stamp;
+
+ if (days == 0)
+ return 0;
+
+ /* get beginning of today */
+ tm = *localtime(&ioloop_time);
+ tm.tm_hour = 0;
+ tm.tm_min = 0;
+ tm.tm_sec = 0;
+ stamp = mktime(&tm);
+ if (stamp == (time_t)-1)
+ i_panic("mktime(today) failed");
+
+ return stamp - (3600*24 * (days-1));
+}
+
+static bool
+dbox_map_file_try_append(struct dbox_map_append_context *ctx,
+ uint32_t file_id, time_t stamp,
+ uoff_t mail_size, uoff_t last_msg_offset,
+ struct dbox_file **file_r, struct ostream **output_r,
+ bool *retry_later_r)
+{
+ struct dbox_map *map = ctx->map;
+ const struct mail_index_header *hdr;
+ struct dbox_file *file;
+ struct stat st;
+ uint32_t seq, tmp_file_id;
+ uoff_t tmp_offset;
+ bool deleted, file_too_old = FALSE;
int ret;
- /* first try to use files already used in this append */
+ *file_r = NULL;
+ *retry_later_r = FALSE;
+
+ file = dbox_file_init_multi(map->storage, file_id);
+ if (dbox_file_open_or_create(file, &deleted) <= 0)
+ return TRUE;
+ if (deleted)
+ return TRUE;
+
+ if (fstat(file->fd, &st) < 0) {
+ mail_storage_set_critical(&map->storage->storage,
+ "fstat(%s) failed: %m", file->current_path);
+ } else if (I_MIN(st.st_mtime, st.st_ctime) < stamp)
+ file_too_old = TRUE;
+ else if (st.st_size + mail_size > map->storage->rotate_size) {
+ /* file too large */
+ } else if ((ret = dbox_file_try_lock(file)) <= 0) {
+ /* locking failed */
+ *retry_later_r = ret == 0;
+ } else if (dbox_map_refresh(map) == 0) {
+ /* now that the file is locked and map is refreshed, make sure
+ we still have the last msg's offset. */
+ hdr = mail_index_get_header(map->view);
+ seq = ctx->orig_msg_count + 1;
+ for (; seq <= hdr->messages_count; seq++) {
+ if (dbox_map_lookup_seq(map, seq,
+ &tmp_file_id, &tmp_offset) < 0)
+ break;
+ if (tmp_file_id == file->file_id) {
+ i_assert(last_msg_offset < tmp_offset);
+ last_msg_offset = tmp_offset;
+ }
+ }
+
+ if (seq > hdr->messages_count &&
+ dbox_file_get_append_stream(file, last_msg_offset,
+ output_r) > 0) {
+ /* success */
+ *file_r = file;
+ return TRUE;
+ }
+ }
+
+ /* failure */
+ dbox_file_unlock(file);
+ dbox_file_unref(&file);
+ return !file_too_old;
+}
+
+static bool
+dbox_map_is_appending(struct dbox_map_append_context *ctx, uint32_t file_id)
+{
+ struct dbox_file *const *files;
+ unsigned int i, count;
+
+ /* there shouldn't be many files open, don't bother with anything
+ faster. */
files = array_get(&ctx->files, &count);
for (i = 0; i < count; i++) {
- if (dbox_file_get_append_stream(files[i], mail_size,
+ if (files[i]->file_id == file_id)
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static int
+dbox_map_find_appendable_file(struct dbox_map_append_context *ctx,
+ uoff_t mail_size, struct dbox_file **file_r,
+ struct ostream **output_r, bool *existing_r)
+{
+ struct dbox_map *map = ctx->map;
+ bool fsync_disable = (map->storage->storage.flags &
+ MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0;
+ struct dbox_file *const *files;
+ const struct mail_index_header *hdr;
+ unsigned int i, count, backwards_lookup_count;
+ uint32_t seq, file_id, min_seen_file_id;
+ uoff_t offset, append_offset;
+ time_t stamp;
+ bool retry_later, updated_retry_seq_min;
+
+ *existing_r = FALSE;
+
+ if (mail_size >= map->storage->rotate_size)
+ return 0;
+
+ /* first try to use files already used in this append */
+ files = array_get(&ctx->files, &count);
+ for (i = count; i > ctx->files_nonappendable_count; i--) {
+ append_offset = dbox_file_get_next_append_offset(files[i-1]);
+ if (append_offset + mail_size <= map->storage->rotate_size &&
+ dbox_file_get_append_stream(files[i-1], append_offset,
output_r) > 0) {
- *file_r = files[i];
- return 0;
+ *file_r = files[i-1];
+ *existing_r = TRUE;
+ return 1;
}
+ /* can't append to this file anymore */
+ if (files[i-1]->fd != -1) {
+ /* avoid wasting fds by closing the file */
+ if (!fsync_disable) {
+ if (fdatasync(files[i-1]->fd) < 0) {
+ dbox_file_set_syscall_error(files[i-1],
+ "fdatasync()");
+ return -1;
+ }
+ }
+ dbox_file_unlock(files[i-1]);
+ dbox_file_close(files[i-1]);
+ }
+ }
+ ctx->files_nonappendable_count = count;
+
+ /* try to find an existing appendable file */
+ stamp = day_begin_stamp(map->storage->rotate_days);
+ hdr = mail_index_get_header(map->view);
+ min_seen_file_id = (uint32_t)-1;
+
+ ctx->orig_msg_count = hdr->messages_count;
+ updated_retry_seq_min = FALSE;
+ backwards_lookup_count = 0;
+ for (seq = hdr->messages_count; seq > 0; seq--) {
+ if (seq == ctx->retry_seq_max) {
+ /* skip over files that we know won't match */
+ seq = ctx->retry_seq_min + 1;
+ continue;
+ }
+ if (dbox_map_lookup_seq(map, seq, &file_id, &offset) < 0)
+ return -1;
+ if (file_id >= min_seen_file_id)
+ continue;
+ min_seen_file_id = file_id;
+
+ /* first lookup: this doesn't really tell us if we're going
+ to exceed the rotate size, it just potentially reduces
+ how many files we have to open. */
+ if (offset + mail_size >= map->storage->rotate_size)
+ continue;
+
+ if (dbox_map_is_appending(ctx, file_id)) {
+ /* already checked this */
+ continue;
+ }
+
+ if (++backwards_lookup_count > MAX_BACKWARDS_LOOKUPS) {
+ /* we've wasted enough time here */
+ break;
+ }
+
+ if (!dbox_map_file_try_append(ctx, file_id, stamp,
+ mail_size, offset,
+ file_r, output_r, &retry_later)) {
+ /* file is too old. the rest of the files are too. */
+ break;
+ }
+ if (*file_r != NULL) {
+ if (seq > ctx->retry_seq_min)
+ ctx->retry_seq_min = seq;
+ return 1;
+ }
+ if (retry_later) {
+ ctx->retry_seq_min = seq;
+ updated_retry_seq_min = TRUE;
+ }
+ }
+ ctx->retry_seq_max = ctx->orig_msg_count;
+ if (!updated_retry_seq_min) {
+ /* there are no files that can be appended to */
+ ctx->retry_seq_min = 0;
}
+ return 0;
+}
- /* FIXME: try to find an existing appendable file */
+int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
+ struct dbox_file **file_r, struct ostream **output_r)
+{
+ struct dbox_file *file = NULL;
+ struct dbox_map_append *append;
+ bool existing;
+ int ret;
+
+ if (ctx->failed)
+ return -1;
- if (file == NULL) {
+ ret = dbox_map_find_appendable_file(ctx, mail_size, &file,
+ output_r, &existing);
+ if (ret < 0)
+ return -1;
+
+ if (ret == 0) {
/* create a new file */
- file = dbox_file_init_single(ctx->mbox, 0);
- if ((ret = dbox_file_get_append_stream(file, mail_size,
- output_r)) <= 0) {
+ file = ctx->map->storage->rotate_size == 0 ?
+ dbox_file_init_single(ctx->mbox, 0) :
+ dbox_file_init_multi(ctx->map->storage, 0);
+ ret = dbox_file_get_append_stream(file, 0, output_r);
+ if (ret <= 0) {
i_assert(ret < 0);
- (void)unlink(dbox_file_get_path(file));
+ (void)unlink(file->current_path);
dbox_file_unref(&file);
return -1;
}
}
+ if (file->single_mbox == NULL) {
+ append = array_append_space(&ctx->appends);
+ append->file = file;
+ append->offset = (*output_r)->offset;
+ }
+ if (!existing)
+ array_append(&ctx->files, &file, 1);
+
*file_r = file;
- array_append(&ctx->files, &file, 1);
return 0;
}
-static int dbox_map_append_commit_new(struct dbox_map_append_context *ctx,
- struct dbox_file *file)
+static int
+dbox_map_get_next_file_id(struct dbox_map *map, struct mail_index_view *view,
+ uint32_t *file_id_r)
{
- i_assert(!file->maildir_file);
- i_assert(file->append_count > 0);
-
- if (file->single_mbox != NULL) {
- /* single UID message file */
- i_assert(file->last_append_uid != 0);
- return dbox_file_assign_id(file, file->last_append_uid);
+ const struct dbox_mail_index_map_header *hdr;
+ const void *data;
+ size_t data_size;
+
+ mail_index_get_header_ext(view, map->map_ext_id, &data, &data_size);
+ if (data_size != sizeof(*hdr)) {
+ if (data_size != 0) {
+ mail_storage_set_critical(&map->storage->storage,
+ "dbox map %s corrupted: hdr size=%u",
+ map->index->filepath, data_size);
+ return -1;
+ }
+ /* first file */
+ *file_id_r = 1;
+ } else {
+ hdr = data;
+ *file_id_r = hdr->highest_file_id + 1;
}
-
- /* FIXME */
- return -1;
+ return 0;
}
-int dbox_map_append_assign_file_ids(struct dbox_map_append_context *ctx)
+int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx,
+ uint32_t *first_map_uid_r,
+ uint32_t *last_map_uid_r)
{
- struct dbox_file *const *files, *file;
+ bool fsync_disable = (ctx->map->storage->storage.flags &
+ MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0;
+ struct dbox_file *const *files;
+ const struct dbox_map_append *appends;
+ struct mail_index_sync_ctx *sync_ctx;
+ struct mail_index_view *sync_view;
+ struct mail_index_transaction *trans;
+ const struct mail_index_header *hdr;
+ struct dbox_mail_index_map_record rec;
unsigned int i, count;
+ uint32_t seq, first_uid, next_uid, first_file_id, file_id;
+ uint16_t ref16;
int ret = 0;
+ if (array_count(&ctx->appends) == 0) {
+ *first_map_uid_r = 0;
+ *last_map_uid_r = 0;
+ return 0;
+ }
+
+ ret = mail_index_sync_begin(ctx->map->index, &sync_ctx, &sync_view,
+ &trans, 0);
+ if (ret <= 0) {
+ i_assert(ret != 0);
+ mail_storage_set_internal_error(&ctx->map->storage->storage);
+ mail_index_reset_error(ctx->map->index);
+ return -1;
+ }
+
+ if (dbox_map_get_next_file_id(ctx->map, sync_view, &file_id) < 0) {
+ mail_index_sync_rollback(&sync_ctx);
+ return -1;
+ }
+
+ /* assign file_ids for newly created multi-files */
files = array_get(&ctx->files, &count);
+ first_file_id = file_id;
for (i = 0; i < count; i++) {
- file = files[i];
-
- if (file->uid == 0 && file->file_id == 0) {
- if (dbox_map_append_commit_new(ctx, file) < 0) {
+ if (files[i]->single_mbox != NULL)
+ continue;
+
+ if (!fsync_disable && files[i]->single_mbox == NULL &&
+ files[i]->fd != -1) {
+ if (fdatasync(files[i]->fd) < 0) {
+ dbox_file_set_syscall_error(files[i],
+ "fdatasync()");
ret = -1;
break;
}
}
+
+ if (files[i]->file_id != 0)
+ continue;
+
+ if (dbox_file_assign_id(files[i], file_id++) < 0) {
+ ret = -1;
+ break;
+ }
}
if (ret < 0) {
/* FIXME: we have to rollback the changes we made */
+ mail_index_sync_rollback(&sync_ctx);
+ return -1;
+ }
+
+ /* update the highest used file_id */
+ if (first_file_id != file_id) {
+ file_id--;
+ mail_index_update_header_ext(trans, ctx->map->map_ext_id,
+ 0, &file_id, sizeof(file_id));
+ }
+
+ /* append map records to index */
+ memset(&rec, 0, sizeof(rec));
+ ref16 = 1;
+ appends = array_get(&ctx->appends, &count);
+ for (i = 0; i < count; i++) {
+ rec.file_id = appends[i].file->file_id;
+ rec.offset = appends[i].offset;
+
+ mail_index_append(trans, 0, &seq);
+ mail_index_update_ext(trans, seq, ctx->map->map_ext_id,
+ &rec, NULL);
+ mail_index_update_ext(trans, seq, ctx->map->ref_ext_id,
+ &ref16, NULL);
+ }
+
+ /* assign map UIDs for appended records */
+ hdr = mail_index_get_header(sync_view);
+ first_uid = hdr->next_uid;
+ mail_index_append_assign_uids(trans, first_uid, &next_uid);
+ i_assert(next_uid - first_uid == count);
+
+ if (hdr->uid_validity == 0) {
+ /* we don't really care about uidvalidity, but it can't be 0 */
+ uint32_t uid_validity = ioloop_time;
+ mail_index_update_header(trans,
+ offsetof(struct mail_index_header, uid_validity),
+ &uid_validity, sizeof(uid_validity), TRUE);
}
+
+ if (mail_index_sync_commit(&sync_ctx) < 0) {
+ mail_storage_set_internal_error(&ctx->map->storage->storage);
+ mail_index_reset_error(ctx->map->index);
+ return -1;
+ }
+
+ *first_map_uid_r = first_uid;
+ *last_map_uid_r = next_uid - 1;
return ret;
}
-int dbox_map_append_commit(struct dbox_map_append_context **_ctx)
+int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx,
+ uint32_t first_uid, uint32_t last_uid)
+{
+ struct dbox_file *const *files;
+ unsigned int i, count;
+ uint32_t next_uid = first_uid;
+
+ files = array_get(&ctx->files, &count);
+ for (i = 0; i < count; i++) {
+ if (files[i]->single_mbox == NULL)
+ continue;
+
+ if (dbox_file_assign_id(files[i], next_uid++) < 0)
+ return -1;
+ }
+ i_assert(next_uid == last_uid + 1);
+ return 0;
+}
+
+void dbox_map_append_commit(struct dbox_map_append_context **_ctx)
{
struct dbox_map_append_context *ctx = *_ctx;
struct dbox_file **files;
*_ctx = NULL;
files = array_get_modifiable(&ctx->files, &count);
- for (i = 0; i < count; i++)
+ for (i = 0; i < count; i++) {
+ dbox_file_unlock(files[i]);
dbox_file_unref(&files[i]);
+ }
+ array_free(&ctx->appends);
array_free(&ctx->files);
i_free(ctx);
- return 0;
}
void dbox_map_append_rollback(struct dbox_map_append_context **_ctx)
{
struct dbox_map_append_context *ctx = *_ctx;
+ struct mail_storage *storage = &ctx->map->storage->storage;
struct dbox_file *const *files, *file;
unsigned int i, count;
if (file->file_id != 0) {
/* FIXME: truncate? */
} else {
- if (unlink(dbox_file_get_path(file)) < 0) {
- i_error("unlink(%s) failed: %m",
- dbox_file_get_path(file));
+ if (unlink(file->current_path) < 0) {
+ mail_storage_set_critical(storage,
+ "unlink(%s) failed: %m",
+ file->current_path);
}
}
dbox_file_unref(&file);
}
+ array_free(&ctx->appends);
array_free(&ctx->files);
i_free(ctx);
}
struct dbox_map *dbox_map_init(struct dbox_storage *storage);
void dbox_map_deinit(struct dbox_map **map);
-bool dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
- uint32_t *file_id_r, uoff_t *offset_r);
+int dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
+ uint32_t *file_id_r, uoff_t *offset_r);
struct dbox_map_append_context *
dbox_map_append_begin(struct dbox_mailbox *mbox);
-1 if error. */
int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
struct dbox_file **file_r, struct ostream **output_r);
-/* Assign file_ids to all appended files. */
-int dbox_map_append_assign_file_ids(struct dbox_map_append_context *ctx);
+/* Assign map UIDs to all appended msgs to multi-files. */
+int dbox_map_append_assign_map_uids(struct dbox_map_append_context *ctx,
+ uint32_t *first_map_uid_r,
+ uint32_t *last_map_uid_r);
+/* Assign UIDs to all created single-files. */
+int dbox_map_append_assign_uids(struct dbox_map_append_context *ctx,
+ uint32_t first_uid, uint32_t last_uid);
/* Returns 0 if ok, -1 if error. */
-int dbox_map_append_commit(struct dbox_map_append_context **ctx);
+void dbox_map_append_commit(struct dbox_map_append_context **ctx);
void dbox_map_append_rollback(struct dbox_map_append_context **ctx);
#endif
#include "lib.h"
#include "array.h"
#include "fdatasync-path.h"
+#include "hex-binary.h"
#include "hex-dec.h"
#include "str.h"
#include "istream.h"
struct dbox_save_mail {
struct dbox_file *file;
- uint32_t seq, uid;
+ uint32_t seq;
uint32_t append_offset;
uoff_t message_size;
};
uint32_t seq;
struct istream *input;
struct mail *mail;
- string_t *cur_keywords;
struct dbox_file *cur_file;
struct ostream *cur_output;
ARRAY_DEFINE(mails, struct dbox_save_mail);
+ unsigned int single_count;
unsigned int failed:1;
unsigned int finished:1;
sizeof(dbox_msg_hdr)) < 0) {
mail_storage_set_critical(_ctx->transaction->box->storage,
"o_stream_send(%s) failed: %m",
- dbox_file_get_path(ctx->cur_file));
+ ctx->cur_file->current_path);
ctx->failed = TRUE;
}
{
struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;
struct mail_storage *storage = &ctx->mbox->storage->storage;
- const char *cur_path;
if (ctx->failed)
return -1;
- cur_path = dbox_file_get_path(ctx->cur_file);
do {
if (o_stream_send_istream(ctx->cur_output, ctx->input) < 0) {
if (!mail_storage_set_error_from_errno(storage)) {
mail_storage_set_critical(storage,
"o_stream_send_istream(%s) failed: %m",
- cur_path);
+ ctx->cur_file->current_path);
}
ctx->failed = TRUE;
return -1;
static void dbox_save_write_metadata(struct dbox_save_context *ctx)
{
struct dbox_metadata_header metadata_hdr;
+ uint8_t guid_128[16];
const char *guid;
string_t *str;
+ buffer_t *guid_buf;
uoff_t vsize;
memset(&metadata_hdr, 0, sizeof(metadata_hdr));
o_stream_send(ctx->cur_output, &metadata_hdr, sizeof(metadata_hdr));
str = t_str_new(256);
- /* write first fields that don't change */
str_printfa(str, "%c%lx\n", DBOX_METADATA_RECEIVED_TIME,
(unsigned long)ctx->ctx.received_date);
str_printfa(str, "%c%lx\n", DBOX_METADATA_SAVE_TIME,
str_printfa(str, "%c%llx\n", DBOX_METADATA_VIRTUAL_SIZE,
(unsigned long long)vsize);
- guid = ctx->ctx.guid != NULL ? ctx->ctx.guid :
- mail_generate_guid_string();
+ /* we can use user-given GUID if
+ a) we're not saving to a multi-file,
+ b) it's 128 bit hex-encoded */
+ guid = ctx->ctx.guid;
+ if (ctx->ctx.guid != NULL && ctx->cur_file->single_mbox == NULL) {
+ guid_buf = buffer_create_dynamic(pool_datastack_create(),
+ sizeof(guid_128));
+ if (strlen(guid) != sizeof(guid_128)*2 ||
+ hex_to_binary(guid, guid_buf) < 0 ||
+ guid_buf->used != sizeof(guid_128))
+ guid = NULL;
+ else
+ memcpy(guid_128, guid_buf->data, sizeof(guid_128));
+ }
+
+ if (guid == NULL) {
+ mail_generate_guid_128(guid_128);
+ guid = binary_to_hex(guid_128, sizeof(guid_128));
+ }
+ if (ctx->cur_file->single_mbox == NULL) {
+ /* multi-file: save the 128bit GUID to index so if the map
+ index gets corrupted we can still find the message */
+ mail_index_update_ext(ctx->trans, ctx->seq,
+ ctx->mbox->guid_ext_id,
+ guid_128, NULL);
+ }
str_printfa(str, "%c%s\n", DBOX_METADATA_GUID, guid);
- str_append_c(str, '\n');
+ str_append_c(str, '\n');
o_stream_send(ctx->cur_output, str_data(str), str_len(str));
}
dbox_msg_header_fill(&dbox_msg_hdr, mail->message_size);
if (pwrite_full(mail->file->fd, &dbox_msg_hdr,
sizeof(dbox_msg_hdr), mail->append_offset) < 0) {
- dbox_file_set_syscall_error(mail->file, "write");
+ dbox_file_set_syscall_error(mail->file, "write()");
return -1;
}
- // FIXME: don't fsync here. especially with multi files
+ /* we're done writing to single-files now, so fsync them here. */
if ((mail->file->storage->storage.flags &
- MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0) {
+ MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0 &&
+ mail->file->single_mbox != NULL) {
if (fdatasync(mail->file->fd) < 0) {
- dbox_file_set_syscall_error(mail->file, "fdatasync");
+ dbox_file_set_syscall_error(mail->file, "fdatasync()");
return -1;
}
}
struct dbox_save_context *ctx = (struct dbox_save_context *)_ctx;
struct mail_storage *storage = &ctx->mbox->storage->storage;
struct dbox_save_mail *save_mail;
- uoff_t offset = 0;
+ uoff_t metadata_offset = 0;
unsigned int count;
ctx->finished = TRUE;
_ctx->received_date, !ctx->failed);
if (!ctx->failed) T_BEGIN {
- const char *cur_path;
-
- cur_path = dbox_file_get_path(ctx->cur_file);
- offset = ctx->cur_output->offset;
+ metadata_offset = ctx->cur_output->offset;
dbox_save_write_metadata(ctx);
if (o_stream_flush(ctx->cur_output) < 0) {
mail_storage_set_critical(storage,
- "o_stream_flush(%s) failed: %m", cur_path);
+ "o_stream_flush(%s) failed: %m",
+ ctx->cur_file->current_path);
ctx->failed = TRUE;
}
} T_END;
count = array_count(&ctx->mails);
save_mail = array_idx_modifiable(&ctx->mails, count - 1);
+ if (!ctx->failed) {
+ dbox_file_finish_append(save_mail->file);
+ save_mail->message_size = metadata_offset -
+ save_mail->append_offset -
+ save_mail->file->msg_header_size;
+ if (dbox_save_mail_write_header(save_mail) < 0)
+ ctx->failed = TRUE;
+ }
if (ctx->failed) {
dbox_file_cancel_append(save_mail->file,
save_mail->append_offset);
array_delete(&ctx->mails, count - 1, 1);
return -1;
- } else {
- dbox_file_finish_append(save_mail->file);
- save_mail->message_size = offset - save_mail->append_offset -
- save_mail->file->msg_header_size;
- dbox_save_mail_write_header(save_mail);
+ }
- if (save_mail->file->single_mbox != NULL)
- dbox_file_close(save_mail->file);
- return 0;
+ if (save_mail->file->single_mbox != NULL) {
+ dbox_file_close(save_mail->file);
+ ctx->single_count++;
}
+ return 0;
}
int dbox_save_finish(struct mail_save_context *ctx)
(void)dbox_save_finish(_ctx);
}
-static int dbox_save_commit(struct dbox_save_context *ctx, uint32_t first_uid)
-{
- struct dbox_mail_index_record rec;
- struct dbox_save_mail *mails;
- unsigned int i, count;
-
- /* assign UIDs to mails */
- mails = array_get_modifiable(&ctx->mails, &count);
- for (i = 0; i < count; i++) {
- mails[i].uid = first_uid++;
- mails[i].file->last_append_uid = mails[i].uid;
- }
- if (dbox_map_append_assign_file_ids(ctx->append_ctx) < 0)
- return -1;
-
- memset(&rec, 0, sizeof(rec));
-#if 0 //FIXME
- for (i = 0; i < count; i++) {
- rec.map_uid = mails[i].map_uid;
- if (rec.map_uid != 0) {
- mail_index_update_ext(ctx->trans, mails[i].seq,
- ctx->mbox->dbox_ext_id,
- &rec, NULL);
- }
- }
-#endif
- return 0;
-}
-
int dbox_transaction_save_commit_pre(struct dbox_save_context *ctx)
{
struct dbox_transaction_context *t =
(struct dbox_transaction_context *)ctx->ctx.transaction;
const struct mail_index_header *hdr;
- uint32_t uid, next_uid;
+ uint32_t uid, first_map_uid, last_map_uid, next_uid;
i_assert(ctx->finished);
+ /* get map UIDs for messages saved to multi-files */
+ if (dbox_map_append_assign_map_uids(ctx->append_ctx, &first_map_uid,
+ &last_map_uid) < 0) {
+ dbox_transaction_save_rollback(ctx);
+ return -1;
+ }
+
+ /* lock the mailbox */
if (dbox_sync_begin(ctx->mbox, TRUE, &ctx->sync_ctx) < 0) {
- ctx->failed = TRUE;
dbox_transaction_save_rollback(ctx);
return -1;
}
+ /* assign UIDs for new messages */
hdr = mail_index_get_header(ctx->sync_ctx->sync_view);
uid = hdr->next_uid;
mail_index_append_assign_uids(ctx->trans, uid, &next_uid);
- if (dbox_save_commit(ctx, uid) < 0) {
- ctx->failed = TRUE;
- dbox_transaction_save_rollback(ctx);
- return -1;
+ /* if we saved any single-files, rename the files to contain UIDs */
+ if (ctx->single_count > 0) {
+ uint32_t last_uid = uid + ctx->single_count - 1;
+
+ if (dbox_map_append_assign_uids(ctx->append_ctx, uid,
+ last_uid) < 0) {
+ dbox_transaction_save_rollback(ctx);
+ return -1;
+ }
}
- *t->ictx.saved_uid_validity = hdr->uid_validity;
- *t->ictx.first_saved_uid = uid;
- *t->ictx.last_saved_uid = next_uid - 1;
+ /* add map_uids for all messages saved to multi-files */
+ if (first_map_uid != 0) {
+ struct dbox_mail_index_record rec;
+ const struct dbox_save_mail *mails;
+ unsigned int i, count;
+ uint32_t next_map_uid = first_map_uid;
- dbox_map_append_commit(&ctx->append_ctx);
+ memset(&rec, 0, sizeof(rec));
+ mails = array_get(&ctx->mails, &count);
+ for (i = 0; i < count; i++) {
+ if (mails[i].file->single_mbox != NULL)
+ continue;
+ rec.map_uid = next_map_uid++;
+ mail_index_update_ext(ctx->trans, mails[i].seq,
+ ctx->mbox->dbox_ext_id,
+ &rec, NULL);
+ }
+ i_assert(next_map_uid == last_map_uid + 1);
+ }
+
+ dbox_map_append_commit(&ctx->append_ctx);
if (ctx->mail != NULL)
mail_free(&ctx->mail);
+
+ *t->ictx.saved_uid_validity = hdr->uid_validity;
+ *t->ictx.first_saved_uid = uid;
+ *t->ictx.last_saved_uid = next_uid - 1;
return 0;
}
if (ctx->mail != NULL)
mail_free(&ctx->mail);
- if (ctx->cur_keywords != NULL)
- str_free(&ctx->cur_keywords);
array_free(&ctx->mails);
i_free(ctx);
}
return -1;
}
- if (mailbox_list_alloc(layout, &_storage->list, error_r) < 0)
- return -1;
- storage->list_module_ctx.super = _storage->list->v;
- storage->alt_dir = p_strdup(_storage->pool, alt_dir);
- _storage->list->v.iter_is_mailbox = dbox_list_iter_is_mailbox;
- _storage->list->v.delete_mailbox = dbox_list_delete_mailbox;
- _storage->list->v.rename_mailbox = dbox_list_rename_mailbox;
- _storage->list->v.rename_mailbox_pre = dbox_list_rename_mailbox_pre;
-
value = getenv("DBOX_ROTATE_SIZE");
if (value != NULL)
storage->rotate_size = (uoff_t)strtoul(value, NULL, 10) * 1024;
storage->rotate_days = (unsigned int)strtoul(value, NULL, 10);
else
storage->rotate_days = DBOX_DEFAULT_ROTATE_DAYS;
-
value = getenv("DBOX_MAX_OPEN_FILES");
if (value != NULL)
storage->max_open_files = (unsigned int)strtoul(value, NULL, 10);
else
storage->max_open_files = DBOX_DEFAULT_MAX_OPEN_FILES;
+ if (storage->max_open_files <= 1) {
+ /* we store file offsets in a 32bit integer. */
+ *error_r = "dbox_max_open_files must be at least 2";
+ return -1;
+ }
+ if (storage->rotate_size > (uint32_t)-1) {
+ /* we store file offsets in a 32bit integer. */
+ *error_r = "dbox_rotate_size must be less than 4 GB";
+ return -1;
+ }
+
+ if (mailbox_list_alloc(layout, &_storage->list, error_r) < 0)
+ return -1;
+ storage->list_module_ctx.super = _storage->list->v;
+ storage->alt_dir = p_strdup(_storage->pool, alt_dir);
+ _storage->list->v.iter_is_mailbox = dbox_list_iter_is_mailbox;
+ _storage->list->v.delete_mailbox = dbox_list_delete_mailbox;
+ _storage->list->v.rename_mailbox = dbox_list_rename_mailbox;
+ _storage->list->v.rename_mailbox_pre = dbox_list_rename_mailbox_pre;
+
MODULE_CONTEXT_SET_FULL(_storage->list, dbox_mailbox_list_module,
storage, &storage->list_module_ctx);
"/"DBOX_GLOBAL_DIR_NAME, NULL);
i_array_init(&storage->open_files, I_MIN(storage->max_open_files, 128));
- storage->map_index = dbox_map_init(storage);
+ storage->map = dbox_map_init(storage);
mailbox_list_get_permissions(storage->storage.list,
&storage->create_mode,
&storage->create_gid);
struct dbox_storage *storage = (struct dbox_storage *)_storage;
dbox_files_free(storage);
- dbox_map_deinit(&storage->map_index);
+ dbox_map_deinit(&storage->map);
array_free(&storage->open_files);
index_storage_destroy(_storage);
}
mbox->dbox_hdr_ext_id =
mail_index_ext_register(index, "dbox-hdr",
sizeof(struct dbox_index_header), 0, 0);
+ mbox->guid_ext_id =
+ mail_index_ext_register(index, "guid", 0, 16, 1);
index_storage_mailbox_init(&mbox->ibox, name, flags, FALSE);
mbox->maildir_uidlist = maildir_uidlist_init_readonly(&mbox->ibox);
const char *alt_dir;
/* paths for storage directories */
const char *storage_dir, *alt_storage_dir;
- struct dbox_map *map_index;
+ struct dbox_map *map;
/* mode/gid to use for new dbox storage files */
mode_t create_mode;
struct maildir_uidlist *maildir_uidlist;
uint32_t highest_maildir_uid;
- uint32_t dbox_ext_id, dbox_hdr_ext_id;
+ uint32_t dbox_ext_id, dbox_hdr_ext_id, guid_ext_id;
const char *path, *alt_path;
};
ret = dbox_sync_file_unlink(file);
} else {
if (ftruncate(file->fd, first_offset) < 0) {
- dbox_file_set_syscall_error(file, "ftruncate");
+ dbox_file_set_syscall_error(file, "ftruncate()");
ret = -1;
}
}
ret = 1;
}
} else {
- ret = dbox_file_open_or_create(file, TRUE, &deleted);
+ ret = dbox_file_open_or_create(file, &deleted);
if (ret > 0 && !deleted) {
dbox_sync_file_move_if_needed(file, entry);
if (array_is_created(&entry->expunges))
{
uint32_t seq;
uoff_t physical_size;
- const char *path;
- bool expunged;
+ bool expunged, last;
int ret;
- path = dbox_file_get_path(file);
- ret = dbox_file_seek_next(file, offset, &physical_size);
+ ret = dbox_file_seek_next(file, offset, &physical_size, &last);
if (ret <= 0) {
if (ret < 0)
return -1;
-#if 0 //FIXME: needed?
- if (physical_size == 0 && file->uid != 0) {
- /* EOF */
- return 0;
- }
-#endif
-
- i_warning("%s: Ignoring broken file (header)", path);
+ i_warning("%s: Ignoring broken file (header)",
+ file->current_path);
return 0;
}
- if (file->maildir_file) {
- file->append_count = 1;
- file->last_append_uid = file->uid;
- }
ret = dbox_file_metadata_seek_mail_offset(file, *offset, &expunged);
if (ret <= 0) {
if (ret < 0)
return -1;
- i_warning("%s: Ignoring broken file (metadata)", path);
+ i_warning("%s: Ignoring broken file (metadata)",
+ file->current_path);
return 0;
}
if (!expunged) {
uoff_t offset;
int ret = 0;
+ if (ctx->maildir_sync_ctx == NULL)
+ return 0;
+
/* we'll need the uidlist to contain the latest filenames.
since there's no easy way to figure out if they changed, just
recreate the uidlist always. */
struct dbox_sync_file_entry *entry, lookup_entry;
uint32_t map_uid;
uoff_t offset;
+ int ret;
i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE ||
sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
if (map_uid == 0)
mail_index_lookup_uid(ctx->sync_view, seq, &lookup_entry.uid);
else {
- if (!dbox_map_lookup(ctx->mbox->storage->map_index,
- map_uid, &lookup_entry.file_id, &offset)) {
- // FIXME: now what?
+ ret = dbox_map_lookup(ctx->mbox->storage->map, map_uid,
+ &lookup_entry.file_id, &offset);
+ if (ret <= 0) {
+ // FIXME: ret=0 case - should we resync?
return -1;
}
}
ibox->last_notify_type = MAILBOX_LOCK_NOTIFY_NONE;
}
+enum mail_index_open_flags
+index_storage_get_index_open_flags(struct mail_storage *storage)
+{
+ enum mail_index_open_flags flags = 0;
+
+#ifndef MMAP_CONFLICTS_WRITE
+ if ((storage->flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0)
+#endif
+ flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE;
+ if ((storage->flags & MAIL_STORAGE_FLAG_DOTLOCK_USE_EXCL) != 0)
+ flags |= MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL;
+ if ((storage->flags & MAIL_STORAGE_FLAG_NFS_FLUSH_INDEX) != 0)
+ flags |= MAIL_INDEX_OPEN_FLAG_NFS_FLUSH;
+ if ((storage->flags & MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0)
+ flags |= MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE;
+ return flags;
+}
+
void index_storage_mailbox_open(struct index_mailbox *ibox)
{
struct mail_storage *storage = ibox->storage;
- enum mail_index_open_flags index_flags = 0;
+ enum mail_index_open_flags index_flags;
int ret;
i_assert(!ibox->box.opened);
+ index_flags = index_storage_get_index_open_flags(storage);
if (!ibox->move_to_memory)
index_flags |= MAIL_INDEX_OPEN_FLAG_CREATE;
-#ifndef MMAP_CONFLICTS_WRITE
- if ((storage->flags & MAIL_STORAGE_FLAG_MMAP_DISABLE) != 0)
-#endif
- index_flags |= MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE;
- if ((storage->flags & MAIL_STORAGE_FLAG_DOTLOCK_USE_EXCL) != 0)
- index_flags |= MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL;
- if ((storage->flags & MAIL_STORAGE_FLAG_NFS_FLUSH_INDEX) != 0)
- index_flags |= MAIL_INDEX_OPEN_FLAG_NFS_FLUSH;
- if ((storage->flags & MAIL_STORAGE_FLAG_FSYNC_DISABLE) != 0) {
- index_flags |= MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE;
+ if ((index_flags & MAIL_INDEX_OPEN_FLAG_FSYNC_DISABLE) != 0)
ibox->fsync_disable = TRUE;
- }
ret = mail_index_open(ibox->index, index_flags, storage->lock_method);
if (ret <= 0 || ibox->move_to_memory) {
void index_storage_destroy_unrefed(void);
void index_storage_destroy(struct mail_storage *storage ATTR_UNUSED);
+enum mail_index_open_flags
+index_storage_get_index_open_flags(struct mail_storage *storage);
void index_storage_mailbox_init(struct index_mailbox *ibox, const char *name,
enum mailbox_open_flags flags,
bool move_to_memory);
bool mail_storage_set_error_from_errno(struct mail_storage *storage);
const char *mail_generate_guid_string(void);
+void mail_generate_guid_128(uint8_t guid[16]);
void mail_set_expunged(struct mail *mail);
void mailbox_set_deleted(struct mailbox *box);
(unsigned long)ts.tv_sec,
pid, my_hostname);
}
+
+void mail_generate_guid_128(uint8_t guid[16])
+{
+ static struct timespec ts = { 0, 0 };
+ static uint8_t guid_static[8];
+ uint32_t pid, host_crc;
+
+ /* we'll use the current time in nanoseconds as the initial 64bit
+ counter. */
+ if (ts.tv_sec == 0) {
+ if (clock_gettime(CLOCK_REALTIME, &ts) < 0)
+ i_fatal("clock_gettime() failed: %m");
+ pid = getpid();
+ host_crc = crc32_str(my_hostname);
+
+ guid_static[0] = (pid & 0x000000ff);
+ guid_static[1] = (pid & 0x0000ff00) >> 8;
+ guid_static[2] = (pid & 0x00ff0000) >> 16;
+ guid_static[3] = (pid & 0xff000000) >> 24;
+ guid_static[4] = (host_crc & 0x000000ff);
+ guid_static[5] = (host_crc & 0x0000ff00) >> 8;
+ guid_static[6] = (host_crc & 0x00ff0000) >> 16;
+ guid_static[7] = (host_crc & 0xff000000) >> 24;
+ } else if ((uint32_t)ts.tv_nsec < (uint32_t)-1) {
+ ts.tv_nsec++;
+ } else {
+ ts.tv_sec++;
+ ts.tv_nsec = 0;
+ }
+
+ guid[0] = (ts.tv_nsec & 0x000000ff);
+ guid[1] = (ts.tv_nsec & 0x0000ff00) >> 8;
+ guid[2] = (ts.tv_nsec & 0x00ff0000) >> 16;
+ guid[3] = (ts.tv_nsec & 0xff000000) >> 24;
+ guid[4] = (ts.tv_sec & 0x000000ff);
+ guid[5] = (ts.tv_sec & 0x0000ff00) >> 8;
+ guid[6] = (ts.tv_sec & 0x00ff0000) >> 16;
+ guid[7] = (ts.tv_sec & 0xff000000) >> 24;
+ memcpy(guid + 8, guid_static, 8);
+}
projects / thirdparty / dovecot / core.git / commitdiff
