libstorage_dbox_a_SOURCES = \
dbox-file.c \
dbox-file-maildir.c \
- dbox-index.c \
dbox-mail.c \
+ dbox-map.c \
dbox-save.c \
dbox-sync.c \
dbox-sync-file.c \
headers = \
dbox-file.h \
dbox-file-maildir.h \
- dbox-index.h \
+ dbox-map.h \
dbox-storage.h \
dbox-sync.h
#include <stdlib.h>
-static const char *
-dbox_maildir_file_get_ext(struct dbox_file *file,
- enum maildir_uidlist_rec_ext_key key)
-{
- uint32_t uid;
-
- uid = file->file_id & ~DBOX_FILE_ID_FLAG_UID;
- return maildir_uidlist_lookup_ext(file->mbox->maildir_uidlist,
- uid, key);
-}
-
const char *dbox_file_maildir_metadata_get(struct dbox_file *file,
enum dbox_metadata_key key)
{
if (!maildir_filename_get_size(file->fname,
MAILDIR_EXTRA_VIRTUAL_SIZE,
&size)) {
- value = dbox_maildir_file_get_ext(file,
- MAILDIR_UIDLIST_REC_EXT_VSIZE);
+ value = maildir_uidlist_lookup_ext(
+ file->single_mbox->maildir_uidlist,
+ file->uid, MAILDIR_UIDLIST_REC_EXT_VSIZE);
if (value == NULL)
break;
size = strtoull(value, NULL, 10);
value = t_strdup_printf("%llx", (unsigned long long)size);
break;
case DBOX_METADATA_POP3_UIDL:
- value = dbox_maildir_file_get_ext(file,
- MAILDIR_UIDLIST_REC_EXT_POP3_UIDL);
+ value = maildir_uidlist_lookup_ext(
+ file->single_mbox->maildir_uidlist,
+ file->uid, MAILDIR_UIDLIST_REC_EXT_POP3_UIDL);
if (value != NULL && *value == '\0') {
/* special case: use base filename */
p = strchr(file->fname, MAILDIR_INFO_SEP);
value = t_strdup_until(file->fname, p);
}
break;
- case DBOX_METADATA_OLD_EXPUNGED:
- case DBOX_METADATA_OLD_FLAGS:
- case DBOX_METADATA_OLD_KEYWORDS:
+ case DBOX_METADATA_OLDV1_EXPUNGED:
+ case DBOX_METADATA_OLDV1_FLAGS:
+ case DBOX_METADATA_OLDV1_KEYWORDS:
case DBOX_METADATA_EXT_REF:
case DBOX_METADATA_SPACE:
break;
#include "ostream.h"
#include "mkdir-parents.h"
#include "fdatasync-path.h"
-#include "write-full.h"
#include "str.h"
#include "maildir/maildir-uidlist.h"
#include "dbox-storage.h"
void dbox_file_set_syscall_error(struct dbox_file *file, const char *function)
{
- mail_storage_set_critical(file->mbox->ibox.box.storage,
+ mail_storage_set_critical(&file->storage->storage,
"%s(%s) failed: %m", function,
dbox_file_get_path(file));
}
static void
dbox_file_set_corrupted(struct dbox_file *file, const char *reason)
{
- mail_storage_set_critical(file->mbox->ibox.box.storage,
+ mail_storage_set_critical(&file->storage->storage,
"%s corrupted: %s", dbox_file_get_path(file),
reason);
}
static struct dbox_file *
-dbox_find_and_move_open_file(struct dbox_mailbox *mbox, unsigned int file_id)
+dbox_find_and_move_open_file(struct dbox_storage *storage, uint32_t file_id)
{
struct dbox_file *const *files, *file;
unsigned int i, count;
- files = array_get(&mbox->open_files, &count);
+ files = array_get(&storage->open_files, &count);
for (i = 0; i < count; i++) {
if (files[i]->file_id == file_id) {
/* move to last in the array */
file = files[i];
- array_delete(&mbox->open_files, i, 1);
- array_append(&mbox->open_files, &file, 1);
+ array_delete(&storage->open_files, i, 1);
+ array_append(&storage->open_files, &file, 1);
return file;
}
}
i_free(file);
}
-void dbox_files_free(struct dbox_mailbox *mbox)
+void dbox_files_free(struct dbox_storage *storage)
{
struct dbox_file *const *files;
unsigned int i, count;
- files = array_get(&mbox->open_files, &count);
+ files = array_get(&storage->open_files, &count);
for (i = 0; i < count; i++)
dbox_file_free(files[i]);
- array_clear(&mbox->open_files);
+ array_clear(&storage->open_files);
}
static void
-dbox_close_open_files(struct dbox_mailbox *mbox, unsigned int close_count)
+dbox_close_open_files(struct dbox_storage *storage, unsigned int close_count)
{
struct dbox_file *const *files;
unsigned int i, count;
- files = array_get(&mbox->open_files, &count);
+ files = array_get(&storage->open_files, &count);
for (i = 0; i < count;) {
if (files[i]->refcount == 0) {
dbox_file_free(files[i]);
- array_delete(&mbox->open_files, i, 1);
+ array_delete(&storage->open_files, i, 1);
if (--close_count == 0)
break;
- files = array_get(&mbox->open_files, &count);
+ files = array_get(&storage->open_files, &count);
} else {
i++;
}
}
static char *
-dbox_file_id_get_fname(struct dbox_mailbox *mbox, unsigned int file_id,
- bool *maildir_file_r)
+dbox_file_uid_get_fname(struct dbox_mailbox *mbox, uint32_t uid,
+ bool *maildir_file_r)
{
const char *fname;
- uint32_t uid;
-
- *maildir_file_r = FALSE;
- if ((file_id & DBOX_FILE_ID_FLAG_UID) != 0) {
- uid = file_id & ~DBOX_FILE_ID_FLAG_UID;
- if (uid <= mbox->highest_maildir_uid &&
- dbox_maildir_uid_get_fname(mbox, uid, &fname)) {
- *maildir_file_r = TRUE;
- return i_strdup(fname);
- } else {
- return i_strdup_printf(DBOX_MAIL_FILE_UID_FORMAT, uid);
- }
+
+ if (uid <= mbox->highest_maildir_uid &&
+ dbox_maildir_uid_get_fname(mbox, uid, &fname)) {
+ *maildir_file_r = TRUE;
+ return i_strdup(fname);
+ } else {
+ *maildir_file_r = FALSE;
+ return i_strdup_printf(DBOX_MAIL_FILE_UID_FORMAT, uid);
}
+}
+
+const char *dbox_file_get_primary_path(struct dbox_file *file)
+{
+ const char *dir;
- return i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, file_id);
+ dir = file->single_mbox != NULL ? file->single_mbox->path :
+ file->storage->storage_dir;
+ return t_strdup_printf("%s/%s", dir, file->fname);
+}
+
+const char *dbox_file_get_alt_path(struct dbox_file *file)
+{
+ const char *dir;
+
+ dir = file->single_mbox != NULL ? file->single_mbox->alt_path :
+ file->storage->alt_storage_dir;
+ return t_strdup_printf("%s/%s", dir, file->fname);
}
struct dbox_file *
-dbox_file_init(struct dbox_mailbox *mbox, unsigned int file_id)
+dbox_file_init_single(struct dbox_mailbox *mbox, uint32_t uid)
{
struct dbox_file *file;
- unsigned int count;
bool maildir;
+ file = i_new(struct dbox_file, 1);
+ file->refcount = 1;
+ file->storage = mbox->storage;
+ file->single_mbox = mbox;
+ file->fd = -1;
+ if (uid != 0) {
+ file->uid = uid;
+ file->fname = dbox_file_uid_get_fname(mbox, uid, &maildir);
+ file->maildir_file = maildir;
+ } else {
+ file->fname = dbox_generate_tmp_filename();
+ }
+ file->current_path = i_strdup_printf("%s/%s", mbox->path, file->fname);
+ return file;
+}
+
+struct dbox_file *
+dbox_file_init_multi(struct dbox_storage *storage, uint32_t file_id)
+{
+ struct dbox_file *file;
+ unsigned int count;
+
file = file_id == 0 ? NULL :
- dbox_find_and_move_open_file(mbox, file_id);
+ dbox_find_and_move_open_file(storage, file_id);
if (file != NULL) {
file->refcount++;
return file;
}
- count = array_count(&mbox->open_files);
- if (count > mbox->max_open_files)
- dbox_close_open_files(mbox, count - mbox->max_open_files);
+ count = array_count(&storage->open_files);
+ if (count > storage->max_open_files)
+ dbox_close_open_files(storage, count - storage->max_open_files);
file = i_new(struct dbox_file, 1);
file->refcount = 1;
- file->mbox = mbox;
- if (file_id != 0) {
- file->file_id = file_id;
- file->fname = dbox_file_id_get_fname(mbox, file_id, &maildir);
- file->maildir_file = maildir;
- } else {
- file->fname = dbox_generate_tmp_filename();
- }
- if (file->maildir_file || file_id == 0) {
- /* newly created files and maildir files always exist in the
- primary path */
- file->current_path =
- i_strdup_printf("%s/%s", mbox->path, file->fname);
- }
+ file->storage = storage;
file->fd = -1;
+ file->fname = file_id == 0 ? dbox_generate_tmp_filename() :
+ i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, file_id);
+ file->current_path =
+ i_strdup_printf("%s/%s", storage->storage_dir, file->fname);
if (file_id != 0)
- array_append(&file->mbox->open_files, &file, 1);
+ array_append(&storage->open_files, &file, 1);
return file;
}
-int dbox_file_assign_id(struct dbox_file *file, unsigned int file_id)
+int dbox_file_assign_id(struct dbox_file *file, uint32_t id)
{
- struct dbox_mailbox *mbox = file->mbox;
const char *old_path;
char *new_fname, *new_path;
bool maildir;
- i_assert(file->file_id == 0);
- i_assert(file_id != 0);
-
- if (!file->maildir_file) {
- old_path = dbox_file_get_path(file);
- new_fname = dbox_file_id_get_fname(mbox, file_id, &maildir);
- new_path = i_strdup_printf("%s/%s", mbox->path, new_fname);
+ i_assert(!file->maildir_file);
+ i_assert(file->uid == 0 && file->file_id == 0);
+ i_assert(id != 0);
- if (rename(old_path, new_path) < 0) {
- mail_storage_set_critical(mbox->ibox.box.storage,
- "rename(%s, %s) failed: %m",
- old_path, new_path);
- i_free(new_fname);
- i_free(new_path);
- return -1;
- }
- i_free(file->fname);
- i_free(file->current_path);
- file->fname = new_fname;
- file->current_path = new_path;
+ old_path = dbox_file_get_path(file);
+ if (file->single_mbox != NULL) {
+ new_fname = dbox_file_uid_get_fname(file->single_mbox,
+ id, &maildir);
+ new_path = i_strdup_printf("%s/%s", file->single_mbox->path,
+ new_fname);
+ } else {
+ new_fname = i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, id);
+ new_path = i_strdup_printf("%s/%s", file->storage->storage_dir,
+ new_fname);
}
+ if (rename(old_path, new_path) < 0) {
+ mail_storage_set_critical(&file->storage->storage,
+ "rename(%s, %s) failed: %m",
+ old_path, new_path);
+ i_free(new_fname);
+ i_free(new_path);
+ return -1;
+ }
+ i_free(file->fname);
+ i_free(file->current_path);
+ file->fname = new_fname;
+ file->current_path = new_path;
- file->file_id = file_id;
- array_append(&mbox->open_files, &file, 1);
+ if (file->single_mbox != NULL)
+ file->uid = id;
+ else {
+ file->file_id = id;
+ array_append(&file->storage->open_files, &file, 1);
+ }
return 0;
}
file->metadata_read_offset = 0;
if (file->file_id != 0) {
- files = array_get(&file->mbox->open_files, &count);
- if (!file->deleted && count <= file->mbox->max_open_files) {
+ files = array_get(&file->storage->open_files, &count);
+ if (!file->deleted && count <= file->storage->max_open_files) {
/* we can leave this file open for now */
return;
}
break;
}
i_assert(i != count);
- array_delete(&file->mbox->open_files, i, 1);
+ array_delete(&file->storage->open_files, i, 1);
}
dbox_file_free(file);
return FALSE;
}
- if (file->append_offset < file->mbox->rotate_min_size ||
+ if (file->append_offset < file->storage->rotate_min_size ||
file->append_offset == file->file_header_size)
return TRUE;
- if (file->append_offset + mail_size >= file->mbox->rotate_size)
+ if (file->append_offset + mail_size >= file->storage->rotate_size)
return FALSE;
- return file->create_time >= day_begin_stamp(file->mbox->rotate_days);
+ return file->create_time >= day_begin_stamp(file->storage->rotate_days);
}
static int dbox_file_parse_header(struct dbox_file *file, const char *line)
{
const char *const *tmp, *value;
- unsigned int pos;
+ unsigned int pos, version;
enum dbox_header_key key;
- if (*line - '0' != DBOX_VERSION || line[1] != ' ') {
+ version = *line - '0';
+ if (line[1] != ' ' || (version != 1 && version != DBOX_VERSION)) {
dbox_file_set_corrupted(file, "Invalid dbox version");
return -1;
}
line += 2;
pos = 2;
- file->append_offset = 0;
file->msg_header_size = 0;
for (tmp = t_strsplit(line, " "); *tmp != NULL; tmp++) {
value = *tmp + 1;
switch (key) {
- case DBOX_HEADER_APPEND_OFFSET:
- file->append_offset_header_pos = pos + 1;
- file->append_offset = *value == 'X' ? 0 :
- strtoull(value, NULL, 16);
+ case DBOX_HEADER_OLDV1_APPEND_OFFSET:
break;
case DBOX_HEADER_MSG_HEADER_SIZE:
file->msg_header_size = strtoul(value, NULL, 16);
{
const char *path;
bool alt = FALSE;
- int i;
/* try the primary path first */
- path = t_strdup_printf("%s/%s", file->mbox->path, file->fname);
- for (i = 0;; i++) {
- file->fd = open(path, O_RDWR);
- if (file->fd != -1)
- break;
-
+ path = dbox_file_get_primary_path(file);
+ while ((file->fd = open(path, O_RDWR)) == -1) {
if (errno != ENOENT) {
- mail_storage_set_critical(file->mbox->ibox.box.storage,
+ mail_storage_set_critical(&file->storage->storage,
"open(%s) failed: %m", path);
return -1;
}
- if (file->mbox->alt_path == NULL || i == 1) {
- /* file doesn't exist */
+ if (file->storage->alt_storage_dir == NULL || alt) {
+ /* not found */
return 0;
}
/* try the alternative path */
- path = t_strdup_printf("%s/%s", file->mbox->alt_path,
- file->fname);
+ path = dbox_file_get_alt_path(file);
alt = TRUE;
}
i_free(file->current_path);
dbox_file_read_header(file);
}
-int dbox_create_fd(struct dbox_mailbox *mbox, const char *path)
+static int dbox_create_fd(struct dbox_storage *storage, const char *path)
{
mode_t old_mask;
int fd;
- old_mask = umask(0777 & ~mbox->ibox.box.file_create_mode);
+ old_mask = umask(0777 & ~storage->create_mode);
fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0777);
umask(old_mask);
if (fd == -1) {
- mail_storage_set_critical(mbox->ibox.box.storage,
+ mail_storage_set_critical(&storage->storage,
"open(%s, O_CREAT) failed: %m", path);
+ } else if (storage->create_gid != (gid_t)-1) {
+ if (fchown(fd, (uid_t)-1, storage->create_gid) < 0) {
+ mail_storage_set_critical(&storage->storage,
+ "fchown(%s, -1, %ld) failed: %m",
+ path, (long)storage->create_gid);
+ /* continue anyway */
+ }
}
return fd;
}
static int dbox_file_create(struct dbox_file *file)
{
string_t *hdr;
- const char *hdrsize;
i_assert(file->fd == -1);
- if (file->current_path == NULL) {
- file->current_path =
- i_strdup_printf("%s/%s", file->mbox->path, file->fname);
- }
- file->fd = dbox_create_fd(file->mbox, file->current_path);
+ file->fd = dbox_create_fd(file->storage, file->current_path);
if (file->fd == -1)
return -1;
file->output = o_stream_create_fd_file(file->fd, 0, FALSE);
hdr = t_str_new(128);
- str_printfa(hdr, "%u %c%x %c%x %c", DBOX_VERSION,
+ str_printfa(hdr, "%u %c%x %c%x\n", DBOX_VERSION,
DBOX_HEADER_MSG_HEADER_SIZE,
(unsigned int)sizeof(struct dbox_message_header),
- DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time,
- DBOX_HEADER_APPEND_OFFSET);
- file->append_offset_header_pos = str_len(hdr);
- str_printfa(hdr, "%08x\n", 0);
+ DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time);
file->file_header_size = str_len(hdr);
file->msg_header_size = sizeof(struct dbox_message_header);
file->append_offset = str_len(hdr);
- hdrsize = t_strdup_printf("%08x", (unsigned int)file->append_offset);
- buffer_write(hdr, file->append_offset_header_pos, hdrsize, 8);
-
if (o_stream_send(file->output, str_data(hdr), str_len(hdr)) < 0) {
dbox_file_set_syscall_error(file, "write");
return -1;
*deleted_r = FALSE;
- if (file->file_id == 0) {
+ if (file->file_id == 0 && file->uid == 0) {
T_BEGIN {
ret = dbox_file_create(file) < 0 ? -1 : 1;
} T_END;
ret = dbox_file_open_fd(file);
} T_END;
if (ret == 0) {
- path = t_strdup_printf("%s/%s", file->mbox->path, file->fname);
- mail_storage_set_critical(file->mbox->ibox.box.storage,
+ path = dbox_file_get_primary_path(file);
+ mail_storage_set_critical(&file->storage->storage,
"open(%s) failed: %m", path);
}
return ret <= 0 ? -1 : 0;
const char *dbox_file_get_path(struct dbox_file *file)
{
- i_assert(file->current_path != NULL);
-
return file->current_path;
}
static int
-dbox_file_get_maildir_data(struct dbox_file *file, uint32_t *uid_r,
- uoff_t *physical_size_r)
+dbox_file_get_maildir_data(struct dbox_file *file, uoff_t *physical_size_r)
{
struct stat st;
- i_assert((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0);
+ i_assert(file->uid != 0);
if (fstat(file->fd, &st) < 0) {
dbox_file_set_syscall_error(file, "fstat");
return -1;
}
- *uid_r = file->file_id & ~DBOX_FILE_ID_FLAG_UID;
*physical_size_r = st.st_size;
return 1;
}
-static int dbox_file_read_mail_header(struct dbox_file *file, uint32_t *uid_r,
- uoff_t *physical_size_r)
+static int
+dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r)
{
struct dbox_message_header hdr;
const unsigned char *data;
int ret;
if (file->maildir_file)
- return dbox_file_get_maildir_data(file, uid_r, physical_size_r);
+ return dbox_file_get_maildir_data(file, physical_size_r);
ret = i_stream_read_data(file->input, &data, &size,
file->msg_header_size - 1);
return 0;
}
- /* Ignore the UID header with UID files */
- *uid_r = (file->file_id & DBOX_FILE_ID_FLAG_UID) != 0 ?
- (file->file_id & ~DBOX_FILE_ID_FLAG_UID) :
- hex2dec(hdr.uid_hex, sizeof(hdr.uid_hex));
*physical_size_r = hex2dec(hdr.message_size_hex,
sizeof(hdr.message_size_hex));
return 1;
}
int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset,
- uint32_t *uid_r, uoff_t *physical_size_r,
+ uoff_t *physical_size_r,
struct istream **stream_r, bool *expunged_r)
{
int ret;
if (offset == 0)
offset = file->file_header_size;
- if (offset != file->cur_offset || file->cur_uid == 0) {
+ if (offset != file->cur_offset || file->cur_physical_size == 0) {
file->cur_offset = offset;
i_stream_seek(file->input, offset);
- ret = dbox_file_read_mail_header(file, &file->cur_uid,
- &file->cur_physical_size);
+ ret = dbox_file_read_mail_header(file, &file->cur_physical_size);
if (ret <= 0)
return ret;
}
*stream_r = i_stream_create_limit(file->input,
file->cur_physical_size);
}
- *uid_r = file->cur_uid;
*physical_size_r = file->cur_physical_size;
return 1;
}
static int
dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset,
- uint32_t *uid_r, uoff_t *physical_size_r)
+ uoff_t *physical_size_r)
{
const char *line;
int ret;
(void)i_stream_read(file->input);
if (!i_stream_have_bytes_left(file->input)) {
- *uid_r = 0;
*physical_size_r = 0;
return 1;
}
- return dbox_file_read_mail_header(file, uid_r, physical_size_r);
+ return dbox_file_read_mail_header(file, physical_size_r);
}
int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset,
- uint32_t *uid_r, uoff_t *physical_size_r)
+ uoff_t *physical_size_r)
{
- uint32_t uid;
uoff_t size;
bool first = *offset == 0;
bool deleted;
int ret;
- ret = dbox_file_get_mail_stream(file, *offset, &uid, &size, NULL,
+ ret = dbox_file_get_mail_stream(file, *offset, &size, NULL,
&deleted);
if (ret <= 0)
return ret;
if (deleted) {
- *uid_r = 0;
*physical_size_r = 0;
return 1;
}
if (first) {
- *uid_r = uid;
*physical_size_r = size;
return 1;
}
i_stream_skip(file->input, size);
- return dbox_file_seek_next_at_metadata(file, offset, uid_r,
- physical_size_r);
+ return dbox_file_seek_next_at_metadata(file, offset, physical_size_r);
}
static int dbox_file_seek_append_pos(struct dbox_file *file, uoff_t mail_size)
bool *expunged_r)
{
uoff_t physical_size, metadata_offset;
- uint32_t uid;
int ret;
- ret = dbox_file_get_mail_stream(file, offset, &uid, &physical_size,
+ ret = dbox_file_get_mail_stream(file, offset, &physical_size,
NULL, expunged_r);
if (ret <= 0 || *expunged_r)
return ret;
return 0;
}
-bool dbox_file_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view,
- uint32_t seq, uint32_t *file_id_r, uoff_t *offset_r)
-{
- const struct dbox_mail_index_record *dbox_rec;
- const void *data;
- uint32_t uid;
- bool expunged;
-
- mail_index_lookup_ext(view, seq, mbox->dbox_ext_id, &data, &expunged);
- if (expunged)
- return FALSE;
- dbox_rec = data;
-
- if (dbox_rec == NULL || dbox_rec->file_id == 0) {
- mail_index_lookup_uid(view, seq, &uid);
- if ((uid & DBOX_FILE_ID_FLAG_UID) != 0) {
- /* something's broken, we can't handle this high UIDs */
- mail_storage_set_critical(mbox->ibox.box.storage,
- "found too high uid=%u", uid);
- return FALSE;
- }
- *file_id_r = DBOX_FILE_ID_FLAG_UID | uid;
- *offset_r = 0;
- } else {
- *file_id_r = dbox_rec->file_id;
- *offset_r = dbox_rec->offset;
- }
- return TRUE;
-}
-
int dbox_file_move(struct dbox_file *file, bool alt_path)
{
struct ostream *output;
return 0;
}
- dest_dir = alt_path ? file->mbox->alt_path : file->mbox->path;
+ dest_dir = !alt_path ? dbox_file_get_primary_path(file) :
+ dbox_file_get_alt_path(file);
temp_path = t_strdup_printf("%s/%s", dest_dir,
dbox_generate_tmp_filename());
}
o_stream_unref(&output);
- if (!file->mbox->ibox.fsync_disable && ret == 0) {
+ if ((file->storage->storage.flags &
+ MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0 && ret == 0) {
if (fsync(out_fd) < 0) {
i_error("fsync(%s) failed: %m", temp_path);
ret = -1;
(void)unlink(temp_path);
return -1;
}
- if (!file->mbox->ibox.fsync_disable) {
+ if ((file->storage->storage.flags &
+ MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0) {
if (fdatasync_path(dest_dir) < 0) {
i_error("fdatasync(%s) failed: %m", dest_dir);
(void)unlink(dest_path);
}
void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr,
- uint32_t uid, uoff_t message_size)
+ uoff_t message_size)
{
memset(dbox_msg_hdr, ' ', sizeof(*dbox_msg_hdr));
memcpy(dbox_msg_hdr->magic_pre, DBOX_MAGIC_PRE,
sizeof(dbox_msg_hdr->magic_pre));
dbox_msg_hdr->type = DBOX_MESSAGE_TYPE_NORMAL;
- dec2hex(dbox_msg_hdr->uid_hex, uid, sizeof(dbox_msg_hdr->uid_hex));
dec2hex(dbox_msg_hdr->message_size_hex, message_size,
sizeof(dbox_msg_hdr->message_size_hex));
dbox_msg_hdr->save_lf = '\n';
There should be no duplicates for the current metadata, but future
extensions may need them so they should be preserved.
*/
-#define DBOX_VERSION 1
+#define DBOX_VERSION 2
#define DBOX_MAGIC_PRE "\001\002"
#define DBOX_MAGIC_POST "\n\001\003\n"
-/* If file_id has this flag set, the file is a single file with file_id=UID. */
-#define DBOX_FILE_ID_FLAG_UID 0x80000000
-
enum dbox_header_key {
/* Offset for appending next message. In %08x format so it can be
updated without moving data in header. If messages have been
expunged and file must not be appended anymore, the value is filled
with 'X'. */
- DBOX_HEADER_APPEND_OFFSET = 'A',
+ DBOX_HEADER_OLDV1_APPEND_OFFSET = 'A',
/* Must be sizeof(struct dbox_message_header) when appending (hex) */
DBOX_HEADER_MSG_HEADER_SIZE = 'M',
/* Creation UNIX timestamp (hex) */
enum dbox_metadata_key {
/* metadata used by old Dovecot versions */
- DBOX_METADATA_OLD_EXPUNGED = 'E',
- DBOX_METADATA_OLD_FLAGS = 'F',
- DBOX_METADATA_OLD_KEYWORDS = 'K',
+ DBOX_METADATA_OLDV1_EXPUNGED = 'E',
+ DBOX_METADATA_OLDV1_FLAGS = 'F',
+ DBOX_METADATA_OLDV1_KEYWORDS = 'K',
/* Globally unique identifier for the message. Preserved when
copying. */
unsigned char magic_pre[2];
unsigned char type;
unsigned char space1;
- unsigned char uid_hex[8];
+ unsigned char oldv1_uid_hex[8];
unsigned char space2;
unsigned char message_size_hex[16];
/* <space reserved for future extensions, LF is always last> */
};
struct dbox_file {
- struct dbox_mailbox *mbox;
+ struct dbox_storage *storage;
+ /* set only for single-msg-per-file */
+ struct dbox_mailbox *single_mbox;
+
int refcount;
- unsigned int file_id;
+ /* uid is for single-msg-per-file, file_id for multi-msgs-per-file */
+ uint32_t uid, file_id;
unsigned int file_header_size;
unsigned int msg_header_size;
- unsigned int append_offset_header_pos;
unsigned int append_count;
uint32_t last_append_uid;
uoff_t output_stream_offset;
uoff_t cur_offset;
- uint32_t cur_uid;
uoff_t cur_physical_size;
char *fname;
extern char dbox_mail_flag_chars[DBOX_METADATA_FLAGS_COUNT];
struct dbox_file *
-dbox_file_init(struct dbox_mailbox *mbox, unsigned int file_id);
+dbox_file_init_single(struct dbox_mailbox *mbox, uint32_t uid);
+struct dbox_file *
+dbox_file_init_multi(struct dbox_storage *storage, uint32_t file_id);
void dbox_file_unref(struct dbox_file **file);
/* Free all currently opened files. */
-void dbox_files_free(struct dbox_mailbox *mbox);
+void dbox_files_free(struct dbox_storage *storage);
-/* Assign a newly created file (file_id=0) a new id. */
-int dbox_file_assign_id(struct dbox_file *file, unsigned int file_id);
+/* Assign a newly created file a new id. For single files assign UID,
+ for multi files assign map UID. */
+int dbox_file_assign_id(struct dbox_file *file, uint32_t id);
-/* Open the file if file_id is not 0, otherwise create it. Returns 1 if ok,
- 0 if read_header=TRUE and opened file was broken, -1 if error. If file is
- deleted, deleted_r=TRUE and 1 is returned. */
+/* Open the file if uid or file_id is not 0, otherwise create it. Returns 1 if
+ ok, 0 if read_header=TRUE and opened file was broken, -1 if error. If file
+ is deleted, deleted_r=TRUE and 1 is returned. */
int dbox_file_open_or_create(struct dbox_file *file, bool read_header,
bool *deleted_r);
/* Open the file's fd if it's currently closed. Assumes that the file exists. */
call this function for a non-opened file. */
const char *dbox_file_get_path(struct dbox_file *file);
-/* Seek to given offset in file and return the message's input stream, UID
+/* Seek to given offset in file and return the message's input stream
and physical size. Returns 1 if ok, 0 if file/offset is corrupted,
-1 if I/O error. */
int dbox_file_get_mail_stream(struct dbox_file *file, uoff_t offset,
- uint32_t *uid_r, uoff_t *physical_size_r,
+ uoff_t *physical_size_r,
struct istream **stream_r, bool *expunged_r);
/* Seek to next message after given offset, or to first message if offset=0.
- If there are no more messages, uid_r is set to 0. Returns 1 if ok, 0 if
- file/offset is corrupted, -1 if I/O error. */
+ If there are no more messages, physical_size_r is set to 0. Returns 1 if ok,
+ 0 if file/offset is corrupted, -1 if I/O error. */
int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset,
- uint32_t *uid_r, uoff_t *physical_size_r);
+ uoff_t *physical_size_r);
/* Returns TRUE if mail_size bytes can be appended to the file. */
bool dbox_file_can_append(struct dbox_file *file, uoff_t mail_size);
/* Write all metadata to output stream. Returns 0 if ok, -1 if I/O error. */
int dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output);
-/* Get file/offset for wanted message. Returns TRUE if found. */
-bool dbox_file_lookup(struct dbox_mailbox *mbox, struct mail_index_view *view,
- uint32_t seq, uint32_t *file_id_r, uoff_t *offset_r);
-
/* Move the file to alt path or back. */
int dbox_file_move(struct dbox_file *file, bool alt_path);
-/* Fill dbox_message_header with given uid/size. */
+/* Fill dbox_message_header with given size. */
void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr,
- uint32_t uid, uoff_t message_size);
+ uoff_t message_size);
-int dbox_create_fd(struct dbox_mailbox *mbox, const char *path);
+const char *dbox_file_get_primary_path(struct dbox_file *file);
+const char *dbox_file_get_alt_path(struct dbox_file *file);
void dbox_file_set_syscall_error(struct dbox_file *file, const char *function);
#endif
+++ /dev/null
-#ifndef DBOX_INDEX_H
-#define DBOX_INDEX_H
-
-struct dbox_file;
-struct dbox_index_append_context;
-
-struct dbox_index *dbox_index_init(struct dbox_mailbox *mbox);
-void dbox_index_deinit(struct dbox_index **index);
-
-struct dbox_index_append_context *
-dbox_index_append_begin(struct dbox_index *index);
-/* Request file for saving a new message with given size. If an existing file
- can be used, the record is locked and updated in index. Returns 0 if ok,
- -1 if error. */
-int dbox_index_append_next(struct dbox_index_append_context *ctx,
- uoff_t mail_size,
- struct dbox_file **file_r,
- struct ostream **output_r);
-/* Assign file_ids to all appended files. */
-int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx);
-/* Returns 0 if ok, -1 if error. */
-int dbox_index_append_commit(struct dbox_index_append_context **ctx);
-void dbox_index_append_rollback(struct dbox_index_append_context **ctx);
-
-#endif
#include "str.h"
#include "index-mail.h"
#include "dbox-storage.h"
+#include "dbox-map.h"
#include "dbox-file.h"
#include <stdlib.h>
index_mail_close(_mail);
}
-static int dbox_mail_lookup(struct dbox_mail *mail,
- uoff_t *offset_r, struct dbox_file **file_r)
+uint32_t dbox_mail_lookup(struct dbox_mailbox *mbox,
+ struct mail_index_view *view, uint32_t seq)
+{
+ const struct dbox_mail_index_record *dbox_rec;
+ const void *data;
+ bool expunged;
+
+ mail_index_lookup_ext(view, seq, mbox->dbox_ext_id, &data, &expunged);
+ dbox_rec = data;
+ return dbox_rec == NULL ? 0 : dbox_rec->map_uid;
+}
+
+static int dbox_mail_open(struct dbox_mail *mail,
+ uoff_t *offset_r, struct dbox_file **file_r)
{
struct dbox_mailbox *mbox = (struct dbox_mailbox *)mail->imail.ibox;
- unsigned int file_id;
+ struct mail *_mail = &mail->imail.mail.mail;
+ uint32_t map_uid, file_id;
if (mail->open_file == NULL) {
- if (!dbox_file_lookup(mbox, mbox->ibox.view,
- mail->imail.mail.mail.seq,
- &file_id, &mail->offset)) {
- mail_set_expunged(&mail->imail.mail.mail);
+ map_uid = dbox_mail_lookup(mbox, mbox->ibox.view, _mail->seq);
+ if (map_uid == 0) {
+ mail->open_file =
+ dbox_file_init_single(mbox, _mail->uid);
+ } else if (!dbox_map_lookup(mbox->storage->map_index, map_uid,
+ &file_id, &mail->offset)) {
+ mail_set_expunged(_mail);
return -1;
+ } else {
+ mail->open_file =
+ dbox_file_init_multi(mbox->storage, file_id);
}
- mail->open_file = dbox_file_init(mbox, file_id);
}
*file_r = mail->open_file;
bool expunged;
int ret;
- if (dbox_mail_lookup(mail, &offset, file_r) < 0)
+ if (dbox_mail_open(mail, &offset, file_r) < 0)
return -1;
ret = dbox_file_metadata_seek_mail_offset(*file_r, offset, &expunged);
struct index_mail_data *data = &mail->imail.data;
struct istream *input;
uoff_t offset, size;
- uint32_t uid = 0;
bool expunged;
int ret;
if (data->stream == NULL) {
- if (dbox_mail_lookup(mail, &offset, &mail->open_file) < 0)
+ if (dbox_mail_open(mail, &offset, &mail->open_file) < 0)
return -1;
- ret = dbox_file_get_mail_stream(mail->open_file, offset, &uid,
+ ret = dbox_file_get_mail_stream(mail->open_file, offset,
&size, &input, &expunged);
if (ret < 0)
return -1;
mail_set_expunged(_mail);
return -1;
}
- if (ret == 0 || uid != _mail->uid) {
+ if (ret == 0) {
/* FIXME: broken file/offset */
if (ret > 0)
i_stream_unref(&input);
mail_storage_set_critical(_mail->box->storage,
- "broken pointer to dbox file %s (uid %u vs %u)",
- mail->open_file->current_path, uid, _mail->uid);
+ "broken pointer to dbox file %s",
+ mail->open_file->current_path);
return -1;
}
data->physical_size = size;
#include "array.h"
#include "dbox-storage.h"
#include "dbox-file.h"
-#include "dbox-index.h"
+#include "dbox-map.h"
-struct dbox_index {
- struct dbox_mailbox *mbox;
+struct dbox_map {
+ struct dbox_storage *storage;
+ struct mail_index *index;
};
-struct dbox_index_append_context {
- struct dbox_index *index;
+struct dbox_map_append_context {
+ struct dbox_mailbox *mbox;
+
ARRAY_DEFINE(files, struct dbox_file *);
uoff_t output_offset;
unsigned int locked_header:1;
};
-struct dbox_index *dbox_index_init(struct dbox_mailbox *mbox)
+struct dbox_map *dbox_map_init(struct dbox_storage *storage)
{
- struct dbox_index *index;
+ struct dbox_map *map;
- index = i_new(struct dbox_index, 1);
- index->mbox = mbox;
- return index;
+ map = i_new(struct dbox_map, 1);
+ map->storage = storage;
+ map->index = mail_index_alloc(storage->storage_dir,
+ DBOX_GLOBAL_INDEX_PREFIX);
+ return map;
}
-void dbox_index_deinit(struct dbox_index **_index)
+void dbox_map_deinit(struct dbox_map **_map)
{
- struct dbox_index *index = *_index;
+ struct dbox_map *map = *_map;
- *_index = NULL;
+ *_map = NULL;
- i_free(index);
+ mail_index_free(&map->index);
+ i_free(map);
}
-struct dbox_index_append_context *
-dbox_index_append_begin(struct dbox_index *index)
+bool dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
+ uint32_t *file_id_r, uoff_t *offset_r)
{
- struct dbox_index_append_context *ctx;
+ return FALSE;
+}
- ctx = i_new(struct dbox_index_append_context, 1);
- ctx->index = index;
+struct dbox_map_append_context *
+dbox_map_append_begin(struct dbox_mailbox *mbox)
+{
+ struct dbox_map_append_context *ctx;
+
+ ctx = i_new(struct dbox_map_append_context, 1);
+ ctx->mbox = mbox;
ctx->first_new_file_id = (uint32_t)-1;
i_array_init(&ctx->files, 64);
return ctx;
}
-int dbox_index_append_next(struct dbox_index_append_context *ctx,
- uoff_t mail_size,
- struct dbox_file **file_r,
- struct ostream **output_r)
+int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
+ struct dbox_file **file_r, struct ostream **output_r)
{
struct dbox_file *const *files, *file = NULL;
unsigned int i, count;
if (file == NULL) {
/* create a new file */
- file = dbox_file_init(ctx->index->mbox, 0);
+ file = dbox_file_init_single(ctx->mbox, 0);
if ((ret = dbox_file_get_append_stream(file, mail_size,
output_r)) <= 0) {
i_assert(ret < 0);
return 0;
}
-static int dbox_index_append_commit_new(struct dbox_index_append_context *ctx,
- struct dbox_file *file)
+static int dbox_map_append_commit_new(struct dbox_map_append_context *ctx,
+ struct dbox_file *file)
{
- unsigned int file_id;
-
i_assert(!file->maildir_file);
i_assert(file->append_count > 0);
- if (file->append_count == 1 && !dbox_file_can_append(file, 0)) {
+ if (file->single_mbox != NULL) {
/* single UID message file */
i_assert(file->last_append_uid != 0);
- file_id = file->last_append_uid | DBOX_FILE_ID_FLAG_UID;
- return dbox_file_assign_id(file, file_id);
+ return dbox_file_assign_id(file, file->last_append_uid);
}
/* FIXME */
return -1;
}
-int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx)
+int dbox_map_append_assign_file_ids(struct dbox_map_append_context *ctx)
{
struct dbox_file *const *files, *file;
unsigned int i, count;
for (i = 0; i < count; i++) {
file = files[i];
- if (file->file_id == 0) {
- if (dbox_index_append_commit_new(ctx, file) < 0) {
+ if (file->uid == 0 && file->file_id == 0) {
+ if (dbox_map_append_commit_new(ctx, file) < 0) {
ret = -1;
break;
}
return ret;
}
-int dbox_index_append_commit(struct dbox_index_append_context **_ctx)
+int dbox_map_append_commit(struct dbox_map_append_context **_ctx)
{
- struct dbox_index_append_context *ctx = *_ctx;
+ struct dbox_map_append_context *ctx = *_ctx;
struct dbox_file **files;
unsigned int i, count;
return 0;
}
-void dbox_index_append_rollback(struct dbox_index_append_context **_ctx)
+void dbox_map_append_rollback(struct dbox_map_append_context **_ctx)
{
- struct dbox_index_append_context *ctx = *_ctx;
+ struct dbox_map_append_context *ctx = *_ctx;
struct dbox_file *const *files, *file;
unsigned int i, count;
--- /dev/null
+#ifndef DBOX_MAP_H
+#define DBOX_MAP_H
+
+struct dbox_storage;
+struct dbox_file;
+struct dbox_map_append_context;
+
+struct dbox_map *dbox_map_init(struct dbox_storage *storage);
+void dbox_map_deinit(struct dbox_map **map);
+
+bool dbox_map_lookup(struct dbox_map *map, uint32_t map_uid,
+ uint32_t *file_id_r, uoff_t *offset_r);
+
+struct dbox_map_append_context *
+dbox_map_append_begin(struct dbox_mailbox *mbox);
+/* Request file for saving a new message with given size. If an existing file
+ can be used, the record is locked and updated in index. Returns 0 if ok,
+ -1 if error. */
+int dbox_map_append_next(struct dbox_map_append_context *ctx, uoff_t mail_size,
+ struct dbox_file **file_r, struct ostream **output_r);
+/* Assign file_ids to all appended files. */
+int dbox_map_append_assign_file_ids(struct dbox_map_append_context *ctx);
+/* Returns 0 if ok, -1 if error. */
+int dbox_map_append_commit(struct dbox_map_append_context **ctx);
+void dbox_map_append_rollback(struct dbox_map_append_context **ctx);
+
+#endif
#include "write-full.h"
#include "index-mail.h"
#include "dbox-storage.h"
-#include "dbox-index.h"
+#include "dbox-map.h"
#include "dbox-file.h"
#include "dbox-sync.h"
struct dbox_mailbox *mbox;
struct mail_index_transaction *trans;
- struct dbox_index_append_context *append_ctx;
+ struct dbox_map_append_context *append_ctx;
struct dbox_sync_context *sync_ctx;
/* updated for each appended mail: */
ctx->ctx.transaction = &t->ictx.mailbox_ctx;
ctx->mbox = mbox;
ctx->trans = t->ictx.trans;
- ctx->append_ctx = dbox_index_append_begin(mbox->dbox_index);
+ ctx->append_ctx = dbox_map_append_begin(mbox);
i_array_init(&ctx->mails, 32);
return &ctx->ctx;
}
st = i_stream_stat(input, TRUE);
mail_size = st == NULL || st->st_size == -1 ? 0 : st->st_size;
- if (dbox_index_append_next(ctx->append_ctx, mail_size,
- &ctx->cur_file, &ctx->cur_output) < 0) {
+ if (dbox_map_append_next(ctx->append_ctx, mail_size,
+ &ctx->cur_file, &ctx->cur_output) < 0) {
ctx->failed = TRUE;
return -1;
}
i_assert(mail->file->msg_header_size == sizeof(dbox_msg_hdr));
- mail->file->last_append_uid = mail->uid;
- dbox_msg_header_fill(&dbox_msg_hdr, mail->uid, mail->message_size);
-
+ dbox_msg_header_fill(&dbox_msg_hdr, mail->message_size);
if (pwrite_full(mail->file->fd, &dbox_msg_hdr,
sizeof(dbox_msg_hdr), mail->append_offset) < 0) {
dbox_file_set_syscall_error(mail->file, "write");
return -1;
}
- if (!mail->file->mbox->ibox.fsync_disable) {
+ // FIXME: don't fsync here. especially with multi files
+ if ((mail->file->storage->storage.flags &
+ MAIL_STORAGE_FLAG_FSYNC_DISABLE) == 0) {
if (fdatasync(mail->file->fd) < 0) {
dbox_file_set_syscall_error(mail->file, "fdatasync");
return -1;
dbox_file_finish_append(save_mail->file);
save_mail->message_size = offset - save_mail->append_offset -
save_mail->file->msg_header_size;
+ dbox_save_mail_write_header(save_mail);
- if (save_mail->file->append_count == 1 &&
- !dbox_file_can_append(save_mail->file, 0)) {
- dbox_save_mail_write_header(save_mail);
+ if (save_mail->file->single_mbox != NULL)
dbox_file_close(save_mail->file);
- }
return 0;
}
}
(void)dbox_save_finish(_ctx);
}
-static int
-dbox_save_file_write_append_offset(struct dbox_file *file, uoff_t append_offset)
-{
- char buf[8+1];
-
- i_assert(append_offset <= (uint32_t)-1);
-
- i_snprintf(buf, sizeof(buf), "%08x", (unsigned int)append_offset);
- if (pwrite_full(file->fd, buf, sizeof(buf)-1,
- file->append_offset_header_pos) < 0) {
- dbox_file_set_syscall_error(file, "pwrite");
- return -1;
- }
- return 0;
-}
-
-static int dbox_save_file_commit_header(struct dbox_save_mail *mail)
-{
- uoff_t append_offset;
-
- append_offset = dbox_file_get_next_append_offset(mail->file);
- return dbox_save_file_write_append_offset(mail->file, append_offset);
-}
-
-static void dbox_save_file_uncommit_header(struct dbox_save_mail *mail)
-{
- if (mail->file->file_id == 0) {
- /* temporary file, we'll just unlink it later */
- return;
- }
- (void)dbox_save_file_write_append_offset(mail->file,
- mail->append_offset);
-}
-
-static int dbox_save_mail_file_cmp(const void *p1, const void *p2)
-{
- const struct dbox_save_mail *m1 = p1, *m2 = p2;
- int ret;
-
- ret = strcmp(m1->file->fname, m2->file->fname);
- if (ret == 0) {
- /* the oldest sequence is first. this is needed for uncommit
- to work right. */
- ret = (int)m1->seq - (int)m2->seq;
- }
- return ret;
-}
-
static int dbox_save_commit(struct dbox_save_context *ctx, uint32_t first_uid)
{
struct dbox_mail_index_record rec;
struct dbox_save_mail *mails;
unsigned int i, count;
- int ret = 0;
/* assign UIDs to mails */
mails = array_get_modifiable(&ctx->mails, &count);
- for (i = 0; i < count; i++)
- mails[i].uid = first_uid++;
-
- /* update headers */
- qsort(mails, count, sizeof(*mails), dbox_save_mail_file_cmp);
for (i = 0; i < count; i++) {
+ mails[i].uid = first_uid++;
mails[i].file->last_append_uid = mails[i].uid;
- if (mails[i].file->append_count == 1 &&
- !dbox_file_can_append(mails[i].file, 0)) {
- /* UID file - there's no need to write it to the
- header */
- continue;
- }
-
- if (dbox_file_open_if_needed(mails[i].file) < 0 ||
- dbox_save_mail_write_header(&mails[i]) < 0) {
- ret = -1;
- break;
- }
-
- /* write file header only once after all mails headers
- have been written */
- if (i+1 == count || mails[i].file != mails[i+1].file) {
- if (dbox_save_file_commit_header(&mails[i]) < 0) {
- ret = -1;
- break;
- }
- dbox_file_close(mails[i].file);
- }
}
- if (ret < 0) {
- /* have to uncommit all written changes */
- for (; i > 0; i--) {
- if (i > 1 && mails[i-2].file == mails[i-1].file)
- continue;
- dbox_save_file_uncommit_header(&mails[i-1]);
- }
- return -1;
- }
-
- /* set file_id / offsets to records */
- if (dbox_index_append_assign_file_ids(ctx->append_ctx) < 0)
+ if (dbox_map_append_assign_file_ids(ctx->append_ctx) < 0)
return -1;
memset(&rec, 0, sizeof(rec));
+#if 0 //FIXME
for (i = 0; i < count; i++) {
- rec.file_id = mails[i].file->file_id;
- rec.offset = mails[i].append_offset;
-
- if ((rec.file_id & DBOX_FILE_ID_FLAG_UID) == 0) {
+ rec.map_uid = mails[i].map_uid;
+ if (rec.map_uid != 0) {
mail_index_update_ext(ctx->trans, mails[i].seq,
ctx->mbox->dbox_ext_id,
&rec, NULL);
}
}
+#endif
return 0;
}
*t->ictx.first_saved_uid = uid;
*t->ictx.last_saved_uid = next_uid - 1;
- dbox_index_append_commit(&ctx->append_ctx);
+ dbox_map_append_commit(&ctx->append_ctx);
if (ctx->mail != NULL)
mail_free(&ctx->mail);
if (!ctx->finished)
dbox_save_cancel(&ctx->ctx);
if (ctx->append_ctx != NULL)
- dbox_index_append_rollback(&ctx->append_ctx);
+ dbox_map_append_rollback(&ctx->append_ctx);
if (ctx->sync_ctx != NULL)
(void)dbox_sync_finish(&ctx->sync_ctx, FALSE);
#include "mail-copy.h"
#include "maildir/maildir-uidlist.h"
#include "dbox-sync.h"
-#include "dbox-index.h"
+#include "dbox-map.h"
#include "dbox-file.h"
#include "dbox-storage.h"
struct dbox_storage *storage = (struct dbox_storage *)_storage;
struct mailbox_list_settings list_set;
struct stat st;
- const char *layout, *alt_dir;
+ const char *layout, *alt_dir, *dir, *value;
if (dbox_get_list_settings(&list_set, data, _storage,
&layout, &alt_dir, error_r) < 0)
_storage->list->v.rename_mailbox = dbox_list_rename_mailbox;
_storage->list->v.rename_mailbox_pre = dbox_list_rename_mailbox_pre;
+ value = getenv("DBOX_ROTATE_SIZE");
+ if (value != NULL)
+ storage->rotate_size = (uoff_t)strtoul(value, NULL, 10) * 1024;
+ else
+ storage->rotate_size = DBOX_DEFAULT_ROTATE_SIZE;
+ value = getenv("DBOX_ROTATE_MIN_SIZE");
+ if (value != NULL)
+ storage->rotate_min_size = (uoff_t)strtoul(value, NULL, 10) * 1024;
+ else
+ storage->rotate_min_size = DBOX_DEFAULT_ROTATE_MIN_SIZE;
+ if (storage->rotate_min_size > storage->rotate_size)
+ storage->rotate_min_size = storage->rotate_size;
+ value = getenv("DBOX_ROTATE_DAYS");
+ if (value != NULL)
+ storage->rotate_days = (unsigned int)strtoul(value, NULL, 10);
+ else
+ storage->rotate_days = DBOX_DEFAULT_ROTATE_DAYS;
+
+ value = getenv("DBOX_MAX_OPEN_FILES");
+ if (value != NULL)
+ storage->max_open_files = (unsigned int)strtoul(value, NULL, 10);
+ else
+ storage->max_open_files = DBOX_DEFAULT_MAX_OPEN_FILES;
+
MODULE_CONTEXT_SET_FULL(_storage->list, dbox_mailbox_list_module,
storage, &storage->list_module_ctx);
/* finish list init after we've overridden vfuncs */
mailbox_list_init(_storage->list, _storage->ns, &list_set,
mail_storage_get_list_flags(_storage->flags));
+
+ dir = mailbox_list_get_path(storage->storage.list, NULL,
+ MAILBOX_LIST_PATH_TYPE_DIR);
+ storage->storage_dir = p_strconcat(_storage->pool, dir,
+ "/"DBOX_GLOBAL_DIR_NAME, NULL);
+ storage->alt_storage_dir = p_strconcat(_storage->pool, alt_dir,
+ "/"DBOX_GLOBAL_DIR_NAME, NULL);
+ i_array_init(&storage->open_files, I_MIN(storage->max_open_files, 128));
+
+ storage->map_index = dbox_map_init(storage);
+ mailbox_list_get_permissions(storage->storage.list,
+ &storage->create_mode,
+ &storage->create_gid);
return 0;
}
+static void dbox_destroy(struct mail_storage *_storage)
+{
+ struct dbox_storage *storage = (struct dbox_storage *)_storage;
+
+ dbox_files_free(storage);
+ dbox_map_deinit(&storage->map_index);
+ array_free(&storage->open_files);
+ index_storage_destroy(_storage);
+}
+
static int create_dbox(struct mail_storage *storage, const char *path)
{
mode_t mode;
struct mail_storage *_storage = &storage->storage;
struct dbox_mailbox *mbox;
struct mail_index *index;
- const char *path, *value;
+ const char *path;
pool_t pool;
path = mailbox_list_get_path(_storage->list, name,
mbox->alt_path = p_strdup(pool, dbox_get_alt_path(storage, path));
mbox->storage = storage;
- value = getenv("DBOX_ROTATE_SIZE");
- if (value != NULL)
- mbox->rotate_size = (uoff_t)strtoul(value, NULL, 10) * 1024;
- else
- mbox->rotate_size = DBOX_DEFAULT_ROTATE_SIZE;
- mbox->rotate_size = 0; /* FIXME: currently anything else doesn't work */
- value = getenv("DBOX_ROTATE_MIN_SIZE");
- if (value != NULL)
- mbox->rotate_min_size = (uoff_t)strtoul(value, NULL, 10) * 1024;
- else
- mbox->rotate_min_size = DBOX_DEFAULT_ROTATE_MIN_SIZE;
- if (mbox->rotate_min_size > mbox->rotate_size)
- mbox->rotate_min_size = mbox->rotate_size;
- value = getenv("DBOX_ROTATE_DAYS");
- if (value != NULL)
- mbox->rotate_days = (unsigned int)strtoul(value, NULL, 10);
- else
- mbox->rotate_days = DBOX_DEFAULT_ROTATE_DAYS;
-
- value = getenv("DBOX_MAX_OPEN_FILES");
- if (value != NULL)
- mbox->max_open_files = (unsigned int)strtoul(value, NULL, 10);
- else
- mbox->max_open_files = DBOX_DEFAULT_MAX_OPEN_FILES;
- i_array_init(&mbox->open_files, I_MIN(mbox->max_open_files, 128));
-
mbox->dbox_ext_id =
mail_index_ext_register(index, "dbox", 0,
sizeof(struct dbox_mail_index_record),
mbox->dbox_hdr_ext_id =
mail_index_ext_register(index, "dbox-hdr",
sizeof(struct dbox_index_header), 0, 0);
- mbox->dbox_index = dbox_index_init(mbox);
index_storage_mailbox_init(&mbox->ibox, name, flags, FALSE);
mbox->maildir_uidlist = maildir_uidlist_init_readonly(&mbox->ibox);
struct dbox_mailbox *mbox = (struct dbox_mailbox *)box;
maildir_uidlist_deinit(&mbox->maildir_uidlist);
- dbox_index_deinit(&mbox->dbox_index);
- dbox_files_free(mbox);
- array_free(&mbox->open_files);
-
return index_storage_mailbox_close(box);
}
dbox_class_deinit,
dbox_alloc,
dbox_create,
- index_storage_destroy,
+ dbox_destroy,
NULL,
dbox_mailbox_open,
dbox_mailbox_create
#define DBOX_INDEX_PREFIX "dovecot.index"
#define DBOX_MAILDIR_NAME "dbox-Mails"
+#define DBOX_GLOBAL_INDEX_PREFIX "dovecot.map.index"
+#define DBOX_GLOBAL_DIR_NAME "storage"
#define DBOX_MAIL_FILE_MULTI_PREFIX "m."
#define DBOX_MAIL_FILE_UID_PREFIX "u."
#define DBOX_MAIL_FILE_MULTI_FORMAT DBOX_MAIL_FILE_MULTI_PREFIX"%u"
struct dbox_storage {
struct mail_storage storage;
union mailbox_list_module_context list_module_ctx;
+
+ /* root path for alt directory */
const char *alt_dir;
+ /* paths for storage directories */
+ const char *storage_dir, *alt_storage_dir;
+ struct dbox_map *map_index;
+
+ /* mode/gid to use for new dbox storage files */
+ mode_t create_mode;
+ gid_t create_gid;
+
+ uoff_t rotate_size, rotate_min_size;
+ unsigned int rotate_days;
+ unsigned int max_open_files;
+ ARRAY_DEFINE(open_files, struct dbox_file *);
};
struct dbox_mail_index_record {
- uint32_t file_id;
- uint32_t offset;
+ uint32_t map_uid;
};
struct dbox_mailbox {
struct maildir_uidlist *maildir_uidlist;
uint32_t highest_maildir_uid;
- struct dbox_index *dbox_index;
uint32_t dbox_ext_id, dbox_hdr_ext_id;
- uoff_t rotate_size, rotate_min_size;
- unsigned int rotate_days;
-
- ARRAY_DEFINE(open_files, struct dbox_file *);
- unsigned int max_open_files;
-
const char *path, *alt_path;
};
enum mail_fetch_field wanted_fields,
struct mailbox_header_lookup_ctx *wanted_headers);
+/* Get map_uid for wanted message. */
+uint32_t dbox_mail_lookup(struct dbox_mailbox *mbox,
+ struct mail_index_view *view, uint32_t seq);
+
struct mail_save_context *
dbox_save_alloc(struct mailbox_transaction_context *_t);
int dbox_save_begin(struct mail_save_context *ctx, struct istream *input);
static int dbox_sync_file_unlink(struct dbox_file *file)
{
- const char *path;
- int i = 0;
+ const char *path, *primary_path;
+ bool alt = FALSE;
- path = t_strdup_printf("%s/%s", file->mbox->path, file->fname);
+ path = primary_path = dbox_file_get_primary_path(file);
while (unlink(path) < 0) {
if (errno != ENOENT) {
- mail_storage_set_critical(file->mbox->ibox.box.storage,
+ mail_storage_set_critical(&file->storage->storage,
"unlink(%s) failed: %m", path);
return -1;
}
- if (file->mbox->alt_path == NULL || i == 1) {
+ if (file->storage->alt_storage_dir == NULL || alt) {
/* not found */
i_warning("dbox: File unexpectedly lost: %s/%s",
- file->mbox->path, file->fname);
+ primary_path, file->fname);
return 0;
}
/* try the alternative path */
- path = t_strdup_printf("%s/%s", file->mbox->alt_path,
- file->fname);
- i++;
+ path = dbox_file_get_alt_path(file);
+ alt = TRUE;
}
return 1;
}
dbox_sync_file_expunge(struct dbox_sync_context *ctx, struct dbox_file *file,
const struct dbox_sync_file_entry *entry)
{
+#if 0 //FIXME
const struct seq_range *expunges;
struct dbox_file *out_file = NULL;
struct istream *input;
offset = first_offset;
for (i = 0;;) {
- if ((ret = dbox_file_seek_next(file, &offset, &uid,
+ if ((ret = dbox_file_seek_next(file, &offset,
&physical_size)) <= 0)
break;
- if (uid == 0) {
+ if (physical_size == 0) {
/* EOF */
break;
}
if (out_file != NULL)
dbox_file_unref(&out_file);
return ret;
+#endif
+ return -1;
}
#if 0
struct dbox_message_header dbox_msg_hdr;
struct dbox_mail_index_record rec;
const char *out_path, *value;
- uint32_t uid;
uoff_t size, append_offset;
unsigned int i;
int ret;
/* FIXME: for now we handle only maildir file conversion */
i_assert(in_file->maildir_file);
- ret = dbox_file_get_mail_stream(in_file, offset, &uid,
+ ret = dbox_file_get_mail_stream(in_file, offset,
&size, &input, &expunged);
if (ret <= 0)
return ret;
return -1;
}
append_offset = output->offset;
- dbox_msg_header_fill(&dbox_msg_hdr, uid, size);
+ dbox_msg_header_fill(&dbox_msg_hdr, size);
/* set static metadata */
for (i = 0; i < N_ELEMENTS(maildir_metadata_keys); i++) {
bool deleted;
int ret;
- file = dbox_file_init(ctx->mbox, entry->file_id);
- if ((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0 &&
- array_is_created(&entry->expunges)) {
+ file = entry->file_id != 0 ?
+ dbox_file_init_multi(ctx->mbox->storage, entry->file_id) :
+ dbox_file_init_single(ctx->mbox, entry->uid);
+ if (entry->uid != 0 && array_is_created(&entry->expunges)) {
/* fast path to expunging the whole file */
if ((ret = dbox_sync_file_unlink(file)) == 0) {
/* file was lost, delete it */
dbox_sync_index_copy_from_maildir(struct dbox_sync_rebuild_context *ctx,
struct dbox_file *file, uint32_t seq)
{
- struct dbox_mailbox *mbox = file->mbox;
ARRAY_TYPE(keyword_indexes) keyword_indexes;
struct mail_keywords *keywords;
enum mail_flags flags;
file->fname, &flags, &keyword_indexes);
mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags);
- keywords = mail_index_keywords_create_from_indexes(mbox->ibox.index,
+ keywords = mail_index_keywords_create_from_indexes(ctx->mbox->ibox.index,
&keyword_indexes);
mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, keywords);
mail_index_keywords_free(&keywords);
static int dbox_sync_index_file_next(struct dbox_sync_rebuild_context *ctx,
struct dbox_file *file, uoff_t *offset)
{
- uint32_t seq, uid;
+ uint32_t seq;
uoff_t physical_size;
const char *path;
bool expunged;
int ret;
path = dbox_file_get_path(file);
- ret = dbox_file_seek_next(file, offset, &uid, &physical_size);
+ ret = dbox_file_seek_next(file, offset, &physical_size);
if (ret <= 0) {
if (ret < 0)
return -1;
- if (uid == 0 && (file->file_id & DBOX_FILE_ID_FLAG_UID) != 0) {
+#if 0 //FIXME: needed?
+ if (physical_size == 0 && file->uid != 0) {
/* EOF */
return 0;
}
+#endif
i_warning("%s: Ignoring broken file (header)", path);
return 0;
}
- if ((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0 &&
- uid != (file->file_id & ~DBOX_FILE_ID_FLAG_UID)) {
- i_warning("%s: Header contains wrong UID %u", path, uid);
- return 0;
- }
if (file->maildir_file) {
- i_assert(uid != 0);
-
file->append_count = 1;
- file->last_append_uid = uid;
+ file->last_append_uid = file->uid;
}
ret = dbox_file_metadata_seek_mail_offset(file, *offset, &expunged);
return 0;
}
if (!expunged) {
- mail_index_append(ctx->trans, uid, &seq);
- dbox_sync_index_metadata(ctx, file, seq, uid);
+ /* FIXME: file->uid doesn't work for multi files */
+ mail_index_append(ctx->trans, file->uid, &seq);
+ dbox_sync_index_metadata(ctx, file, seq, file->uid);
}
return 1;
}
if (ctx->highest_uid < uid)
ctx->highest_uid = uid;
- file = dbox_file_init(ctx->mbox, uid | DBOX_FILE_ID_FLAG_UID);
+ file = dbox_file_init_single(ctx->mbox, uid);
file->current_path = i_strdup_printf("%s/%s", dir, fname);
ret = dbox_sync_index_file_next(ctx, file, &offset) < 0 ? -1 : 0;
iter = maildir_uidlist_iter_init(mbox->maildir_uidlist);
while (maildir_uidlist_iter_next(iter, &uid, &flags, &fname)) {
- file = dbox_file_init(mbox, uid | DBOX_FILE_ID_FLAG_UID);
+ file = dbox_file_init_single(mbox, uid);
file->current_path =
i_strdup_printf("%s/%s", ctx->mbox->path, fname);
#include "str.h"
#include "hash.h"
#include "dbox-storage.h"
+#include "dbox-map.h"
#include "dbox-file.h"
#include "dbox-sync.h"
#define DBOX_REBUILD_COUNT 3
+static unsigned int dbox_sync_file_entry_hash(const void *p)
+{
+ const struct dbox_sync_file_entry *entry = p;
+
+ if (entry->file_id != 0)
+ return entry->file_id | 0x80000000;
+ else
+ return entry->uid;
+}
+
+static int dbox_sync_file_entry_cmp(const void *p1, const void *p2)
+{
+ const struct dbox_sync_file_entry *entry1 = p1, *entry2 = p2;
+
+ /* this is only for hashing, don't bother ever returning 1. */
+ if (entry1->file_id != entry2->file_id)
+ return -1;
+ if (entry1->uid != entry2->uid)
+ return -1;
+ return 0;
+}
+
static int dbox_sync_add_seq(struct dbox_sync_context *ctx,
const struct mail_index_sync_rec *sync_rec,
uint32_t seq)
{
- struct dbox_sync_file_entry *entry;
- uint32_t file_id;
+ struct dbox_sync_file_entry *entry, lookup_entry;
+ uint32_t map_uid;
uoff_t offset;
- bool uid_file;
i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE ||
sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
- if (!dbox_file_lookup(ctx->mbox, ctx->sync_view, seq,
- &file_id, &offset))
- return -1;
+ memset(&lookup_entry, 0, sizeof(lookup_entry));
+ map_uid = dbox_mail_lookup(ctx->mbox, ctx->sync_view, seq);
+ if (map_uid == 0)
+ mail_index_lookup_uid(ctx->sync_view, seq, &lookup_entry.uid);
+ else {
+ if (!dbox_map_lookup(ctx->mbox->storage->map_index,
+ map_uid, &lookup_entry.file_id, &offset)) {
+ // FIXME: now what?
+ return -1;
+ }
+ }
- entry = hash_table_lookup(ctx->syncs, POINTER_CAST(file_id));
+ entry = hash_table_lookup(ctx->syncs, &lookup_entry);
if (entry == NULL) {
entry = p_new(ctx->pool, struct dbox_sync_file_entry, 1);
- entry->file_id = file_id;
- hash_table_insert(ctx->syncs, POINTER_CAST(file_id), entry);
+ *entry = lookup_entry;
+ hash_table_insert(ctx->syncs, entry, entry);
}
- uid_file = (file_id & DBOX_FILE_ID_FLAG_UID) != 0;
if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
if (!array_is_created(&entry->expunges)) {
p_array_init(&entry->expunges, ctx->pool,
- uid_file ? 1 : 3);
+ lookup_entry.uid != 0 ? 1 : 3);
}
seq_range_array_add(&entry->expunges, 0, seq);
} else {
seq1, seq2);
}
- /* read all changes and sort them to file_id order */
+ /* read all changes and group changes to same file_id together */
ctx->pool = pool_alloconly_create("dbox sync pool", 1024*32);
- ctx->syncs = hash_table_create(default_pool, ctx->pool, 0, NULL, NULL);
+ ctx->syncs = hash_table_create(default_pool, ctx->pool, 0,
+ dbox_sync_file_entry_hash,
+ dbox_sync_file_entry_cmp);
for (;;) {
if (!mail_index_sync_next(ctx->index_sync_ctx, &sync_rec))
struct mailbox;
struct dbox_sync_file_entry {
- uint32_t file_id;
+ uint32_t uid, file_id;
unsigned int move_from_alt:1;
unsigned int move_to_alt:1;
projects / thirdparty / dovecot / core.git / commitdiff
