- Maildir code now uses dovecot-uidlist directly instead of dbox.index.
- Flags and keywords are no longer written to metadata.
- Removed metadata modifying code entirely.
- dbox.index is no longer read or written. The code will be removed soon.
--HG--
branch : HEAD
#include "lib.h"
#include "array.h"
#include "str.h"
+#include "maildir/maildir-storage.h"
+#include "maildir/maildir-uidlist.h"
+#include "maildir/maildir-filename.h"
#include "dbox-storage.h"
-#include "../maildir/maildir-storage.h"
-#include "../maildir/maildir-filename.h"
-#include "dbox-index.h"
#include "dbox-file.h"
#include "dbox-file-maildir.h"
#include <stdlib.h>
static const char *
-dbox_file_maildir_get_flags(struct dbox_file *file, enum dbox_metadata_key key)
+dbox_maildir_file_get_ext(struct dbox_file *file,
+ enum maildir_uidlist_rec_ext_key key)
{
- ARRAY_TYPE(keyword_indexes) keyword_indexes;
- struct mail_keywords *keywords;
- enum mail_flags flags;
- string_t *str;
+ uint32_t uid;
- if (file->mbox->maildir_sync_keywords == NULL)
- return NULL;
-
- t_array_init(&keyword_indexes, 32);
- maildir_filename_get_flags(file->mbox->maildir_sync_keywords,
- file->fname, &flags, &keyword_indexes);
- str = t_str_new(64);
- if (key == DBOX_METADATA_FLAGS)
- dbox_mail_metadata_flags_append(str, flags);
- else {
- keywords = mail_index_keywords_create_from_indexes(
- file->mbox->ibox.index, &keyword_indexes);
- dbox_mail_metadata_keywords_append(file->mbox, str, keywords);
- mail_index_keywords_free(&keywords);
- }
- return str_c(str);
-}
-
-static const char *
-dbox_file_maildir_get_old_metadata(struct dbox_file *file, char key)
-{
- struct dbox_index_record *rec;
- const char *p, *end;
-
- rec = dbox_index_record_lookup(file->mbox->dbox_index, file->file_id);
- if (rec == NULL)
- return NULL;
-
- for (p = strchr(rec->data, ' '); *p != '\0'; p++) {
- if (*p == ' ') {
- if (p[1] == key) {
- end = strchr(p+2, ' ');
- return t_strdup_until(p+2, end);
- }
- if (p[1] == ':')
- break;
- }
- }
- return NULL;
+ uid = file->file_id & ~DBOX_FILE_ID_FLAG_UID;
+ return maildir_uidlist_lookup_ext(file->mbox->maildir_uidlist,
+ uid, key);
}
const char *dbox_file_maildir_metadata_get(struct dbox_file *file,
const char *p, *value = NULL;
switch (key) {
- case DBOX_METADATA_FLAGS:
- case DBOX_METADATA_KEYWORDS:
- value = dbox_file_maildir_get_flags(file, key);
- break;
case DBOX_METADATA_GUID:
p = strchr(file->fname, MAILDIR_INFO_SEP);
value = p == NULL ? file->fname :
if (!maildir_filename_get_size(file->fname,
MAILDIR_EXTRA_VIRTUAL_SIZE,
&size)) {
- value = dbox_file_maildir_get_old_metadata(file, 'W');
+ value = dbox_maildir_file_get_ext(file,
+ MAILDIR_UIDLIST_REC_EXT_VSIZE);
if (value == NULL)
break;
size = strtoull(value, NULL, 10);
value = t_strdup_printf("%llx", (unsigned long long)size);
break;
case DBOX_METADATA_POP3_UIDL:
- value = dbox_file_maildir_get_old_metadata(file, 'P');
+ value = dbox_maildir_file_get_ext(file,
+ MAILDIR_UIDLIST_REC_EXT_POP3_UIDL);
if (value != NULL && *value == '\0') {
/* special case: use base filename */
p = strchr(file->fname, MAILDIR_INFO_SEP);
value = t_strdup_until(file->fname, p);
}
break;
- case DBOX_METADATA_EXPUNGED:
+ case DBOX_METADATA_OLD_EXPUNGED:
+ case DBOX_METADATA_OLD_FLAGS:
+ case DBOX_METADATA_OLD_KEYWORDS:
case DBOX_METADATA_EXT_REF:
case DBOX_METADATA_SPACE:
break;
}
- if (value != NULL)
- dbox_file_metadata_set(file, key, value);
return value;
}
#include "fdatasync-path.h"
#include "write-full.h"
#include "str.h"
+#include "maildir/maildir-uidlist.h"
#include "dbox-storage.h"
-#include "dbox-index.h"
#include "dbox-file.h"
#include "dbox-file-maildir.h"
}
}
+static bool
+dbox_maildir_uid_get_fname(struct dbox_mailbox *mbox, uint32_t uid,
+ const char **fname_r)
+{
+ enum maildir_uidlist_rec_flag flags;
+
+ *fname_r = maildir_uidlist_lookup(mbox->maildir_uidlist, uid, &flags);
+ return *fname_r != NULL;
+}
+
static char *
dbox_file_id_get_fname(struct dbox_mailbox *mbox, unsigned int file_id,
bool *maildir_file_r)
{
- struct dbox_index_record *rec;
- const char *p;
+ const char *fname;
+ uint32_t uid;
*maildir_file_r = FALSE;
if ((file_id & DBOX_FILE_ID_FLAG_UID) != 0) {
- file_id &= ~DBOX_FILE_ID_FLAG_UID;
- return i_strdup_printf(DBOX_MAIL_FILE_UID_FORMAT, file_id);
- }
-
- rec = dbox_index_record_lookup(mbox->dbox_index, file_id);
- if (rec != NULL && rec->status == DBOX_INDEX_FILE_STATUS_MAILDIR) {
- /* data contains <uid> [<fields>] :<filename> */
- *maildir_file_r = TRUE;
- p = strstr(rec->data, " :");
- i_assert(p != NULL);
- return i_strdup_printf("%s", p + 2);
+ uid = file_id & ~DBOX_FILE_ID_FLAG_UID;
+ if (uid <= mbox->highest_maildir_uid &&
+ dbox_maildir_uid_get_fname(mbox, uid, &fname)) {
+ *maildir_file_r = TRUE;
+ return i_strdup(fname);
+ } else {
+ return i_strdup_printf(DBOX_MAIL_FILE_UID_FORMAT, uid);
+ }
}
return i_strdup_printf(DBOX_MAIL_FILE_MULTI_FORMAT, file_id);
return file;
}
-struct dbox_file *
-dbox_file_init_new_maildir(struct dbox_mailbox *mbox, const char *fname)
-{
- struct dbox_file *file;
-
- file = dbox_file_init(mbox, 0);
- file->maildir_file = TRUE;
- file->fname = i_strdup(fname);
- return file;
-}
-
int dbox_file_assign_id(struct dbox_file *file, unsigned int file_id)
{
struct dbox_mailbox *mbox = file->mbox;
dbox_file_get_maildir_data(struct dbox_file *file, uint32_t *uid_r,
uoff_t *physical_size_r)
{
- struct dbox_index_record *rec;
struct stat st;
+#if 0 //FIXME
+ uint32_t uid;
+
+ if ((file->file_id & DBOX_FILE_ID_FLAG_UID) == 0) {
+ i_assert(file->file_id == 0);
+ if (maildir_uidlist_get_uid(file->mbox->maildir_uidlist,
+ file->fname, &uid))
+ file->file_id = uid | DBOX_FILE_ID_FLAG_UID;
+ }
+#else
+ i_assert((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0);
+#endif
+
if (fstat(file->fd, &st) < 0) {
dbox_file_set_syscall_error(file, "fstat");
return -1;
}
- rec = dbox_index_record_lookup(file->mbox->dbox_index, file->file_id);
- if (rec == NULL) {
- /* should happen only when we're rebuilding the index */
- *uid_r = 0;
- } else {
- i_assert(rec->status == DBOX_INDEX_FILE_STATUS_MAILDIR);
- *uid_r = strtoul(rec->data, NULL, 10);
- }
+ *uid_r = file->file_id & ~DBOX_FILE_ID_FLAG_UID;
*physical_size_r = st.st_size;
return 1;
}
*expunged_r = FALSE;
- if (file->metadata_pool != NULL) {
- if (array_is_created(&file->metadata_changes))
- array_free(&file->metadata_changes);
+ if (file->metadata_pool != NULL)
p_clear(file->metadata_pool);
- } else {
+ else {
file->metadata_pool =
pool_alloconly_create("dbox metadata", 1024);
}
if (*line == DBOX_METADATA_SPACE || *line == '\0') {
/* end of metadata */
- file->metadata_space_pos =
- prev_offset - metadata_data_offset;
*expunged_r = FALSE;
break;
}
array_append(&file->metadata, &line, 1);
}
file->metadata_read_offset = metadata_offset;
- file->metadata_len = file->input->v_offset - metadata_data_offset;
- if (*expunged_r)
- file->metadata_space_pos = file->metadata_len;
return 1;
}
return NULL;
}
-void dbox_file_metadata_set(struct dbox_file *file, enum dbox_metadata_key key,
- const char *value)
-{
- const char **changes, *data;
- unsigned int i, count;
-
- data = file->maildir_file ? NULL : dbox_file_metadata_get(file, key);
- if (data != NULL && strcmp(data, value) == 0) {
- /* value didn't change */
- return;
- }
-
- data = p_strdup_printf(file->metadata_pool, "%c%s", (char)key, value);
-
- if (!array_is_created(&file->metadata_changes))
- p_array_init(&file->metadata_changes, file->metadata_pool, 16);
- else {
- /* see if we have already changed this metadata */
- changes = array_get_modifiable(&file->metadata_changes, &count);
- for (i = 0; i < count; i++) {
- if (*changes[i] == (char)key) {
- changes[i] = data;
- return;
- }
- }
- }
-
- array_append(&file->metadata_changes, &data, 1);
-}
-
-static int dbox_file_metadata_is_at_eof(struct dbox_file *file)
-{
- uoff_t size;
- uint32_t uid;
- uoff_t offset;
- int ret;
-
- if ((file->file_id & DBOX_FILE_ID_FLAG_UID) != 0)
- return 1;
-
- offset = file->metadata_read_offset;
- ret = dbox_file_seek_next_at_metadata(file, &offset, &uid, &size);
- return ret <= 0 ? ret : uid == 0;
-}
-
-static int dbox_file_write_empty_block(struct dbox_file *file, uoff_t offset,
- unsigned int len)
-{
- char space[256];
-
- i_assert(len > 0);
-
- len--;
- memset(space, DBOX_METADATA_SPACE, I_MIN(sizeof(space), len));
- while (len >= sizeof(space)) {
- if (pwrite_full(file->fd, space, sizeof(space), offset) < 0) {
- dbox_file_set_syscall_error(file, "pwrite");
- return -1;
- }
- }
- /* @UNSAFE: last block ends with LF */
- space[len++] = '\n';
- if (pwrite_full(file->fd, space, len, offset) < 0) {
- dbox_file_set_syscall_error(file, "pwrite");
- return -1;
- }
- file->metadata_len += len;
- return 1;
-}
-
-static int dbox_file_grow_metadata(struct dbox_file *file, unsigned int len)
-{
- enum dbox_index_file_lock_status lock_status;
- uoff_t offset;
- int ret;
-
- ret = dbox_index_try_lock_file(file->mbox->dbox_index, file->file_id,
- &lock_status);
- if (ret <= 0 || (ret = dbox_file_metadata_is_at_eof(file)) <= 0)
- return ret;
-
- offset = file->metadata_read_offset +
- sizeof(struct dbox_metadata_header) + file->metadata_len;
- i_stream_seek(file->input, offset);
- (void)i_stream_read(file->input);
- if (!i_stream_have_bytes_left(file->input)) {
- len = len - file->metadata_len + DBOX_EXTRA_SPACE;
- ret = dbox_file_write_empty_block(file, offset, len);
- } else {
- i_error("%s: Metadata changed unexpectedly",
- dbox_file_get_path(file));
- ret = -1;
- }
-
- dbox_index_unlock_file(file->mbox->dbox_index, file->file_id);
- return ret;
-}
-
-static int dbox_file_metadata_write_real(struct dbox_file *file)
-{
- const char *const *metadata, *const *changes;
- unsigned int i, j, count, changes_count, space_needed, skip_pos;
- char space[DBOX_EXTRA_SPACE];
- string_t *str;
- uoff_t offset;
- int ret;
-
- if (!array_is_created(&file->metadata_changes)) {
- /* nothing to write */
- return 1;
- }
- if (file->maildir_file)
- return 0;
-
- offset = file->metadata_read_offset +
- sizeof(struct dbox_metadata_header);
- metadata = array_get(&file->metadata, &count);
- changes = array_get(&file->metadata_changes, &changes_count);
-
- /* skip as many metadata fields from beginning as we can */
- for (i = skip_pos = 0; i < count; i++) {
- for (j = 0; j < changes_count; j++) {
- if (*changes[j] == *metadata[i])
- break;
- }
- if (j != changes_count)
- break;
- skip_pos += strlen(metadata[i]) + 1;
- }
-
- str = t_str_new(512);
- /* overwrite existing metadata fields */
- for (; i < count; i++) {
- for (j = 0; j < changes_count; j++) {
- if (*changes[j] == *metadata[i])
- break;
- }
- if (j != changes_count) {
- str_append(str, changes[j]);
- str_append_c(str, '\n');
- } else {
- str_append(str, metadata[i]);
- str_append_c(str, '\n');
- }
- }
- /* add new metadata */
- for (j = 0; j < changes_count; j++) {
- for (i = 0; i < count; i++) {
- if (*changes[j] == *metadata[i])
- break;
- }
- if (i == count) {
- str_append(str, changes[j]);
- str_append_c(str, '\n');
- }
- }
- if (skip_pos + str_len(str) >= file->metadata_len) {
- if ((ret = dbox_file_grow_metadata(file, skip_pos +
- str_len(str))) <= 0)
- return ret;
- }
-
- memset(space, DBOX_METADATA_SPACE, sizeof(space));
- while (skip_pos + str_len(str) < file->metadata_space_pos) {
- space_needed = file->metadata_space_pos -
- (skip_pos + str_len(str));
- str_append_n(str, space, I_MIN(sizeof(space), space_needed));
- }
- i_assert(skip_pos + str_len(str) <= file->metadata_len);
-
- if (file->metadata_space_pos < skip_pos + str_len(str)) {
- /* metadata was grown, update space position */
- file->metadata_space_pos = skip_pos + str_len(str);
- }
-
- ret = pwrite_full(file->fd, str_data(str), str_len(str),
- offset + skip_pos);
- if (ret < 0) {
- dbox_file_set_syscall_error(file, "pwrite");
- return -1;
- }
- return 1;
-}
-
-int dbox_file_metadata_write(struct dbox_file *file)
-{
- int ret;
-
- T_BEGIN {
- ret = dbox_file_metadata_write_real(file);
- } T_END;
- return ret;
-}
-
int dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output)
{
struct dbox_metadata_header metadata_hdr;
- char space[DBOX_EXTRA_SPACE];
- const char *const *metadata, *const *changes;
- unsigned int i, j, count, changes_count;
+ const char *const *metadata;
+ unsigned int i, count;
memset(&metadata_hdr, 0, sizeof(metadata_hdr));
memcpy(metadata_hdr.magic_post, DBOX_MAGIC_POST,
return -1;
metadata = array_get(&file->metadata, &count);
- if (!array_is_created(&file->metadata_changes)) {
- for (i = 0; i < count; i++) {
- if (o_stream_send_str(output, metadata[i]) < 0 ||
- o_stream_send(output, "\n", 1) < 0)
- return -1;
- }
- } else {
- changes = array_get(&file->metadata_changes, &changes_count);
- /* write unmodified metadata */
- for (i = 0; i < count; i++) {
- for (j = 0; j < changes_count; j++) {
- if (*changes[j] == *metadata[i])
- break;
- }
- if (j == changes_count) {
- if (o_stream_send_str(output, metadata[i]) < 0)
- return -1;
- if (o_stream_send(output, "\n", 1) < 0)
- return -1;
- }
- }
- /* write modified metadata */
- for (i = 0; i < changes_count; i++) {
- if (o_stream_send_str(output, changes[i]) < 0 ||
- o_stream_send(output, "\n", 1) < 0)
- return -1;
- }
+ for (i = 0; i < count; i++) {
+ if (o_stream_send_str(output, metadata[i]) < 0 ||
+ o_stream_send(output, "\n", 1) < 0)
+ return -1;
}
- memset(space, ' ', sizeof(space));
- if (o_stream_send(output, space, sizeof(space)) < 0 ||
- o_stream_send(output, "\n", 1) < 0)
+ if (o_stream_send(output, "\n", 1) < 0)
return -1;
return 0;
}
mail_index_lookup_uid(view, seq, &uid);
if ((uid & DBOX_FILE_ID_FLAG_UID) != 0) {
/* something's broken, we can't handle this high UIDs */
+ mail_storage_set_critical(mbox->ibox.box.storage,
+ "found too high uid=%u", uid);
return FALSE;
}
*file_id_r = DBOX_FILE_ID_FLAG_UID | uid;
return 0;
}
-void dbox_mail_metadata_flags_append(string_t *str, enum mail_flags flags)
-{
- unsigned int i;
-
- for (i = 0; i < DBOX_METADATA_FLAGS_COUNT; i++) {
- if ((flags & dbox_mail_flags_map[i]) != 0)
- str_append_c(str, dbox_mail_flag_chars[i]);
- else
- str_append_c(str, '0');
- }
-}
-
-void dbox_mail_metadata_keywords_append(struct dbox_mailbox *mbox,
- string_t *str,
- const struct mail_keywords *keywords)
-{
- const ARRAY_TYPE(keywords) *keyword_names_list;
- const char *const *keyword_names;
- unsigned int i, keyword_names_count;
-
- if (keywords == NULL || keywords->count == 0)
- return;
-
- keyword_names_list = mail_index_get_keywords(mbox->ibox.index);
- keyword_names = array_get(keyword_names_list, &keyword_names_count);
-
- for (i = 0; i < keywords->count; i++) {
- i_assert(keywords->idx[i] < keyword_names_count);
-
- str_append(str, keyword_names[keywords->idx[i]]);
- str_append_c(str, ' ');
- }
- str_truncate(str, str_len(str)-1);
-}
-
void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr,
uint32_t uid, uoff_t message_size)
{
#define DBOX_MAGIC_PRE "\001\002"
#define DBOX_MAGIC_POST "\n\001\003\n"
-#define DBOX_EXTRA_SPACE 64
/* If file_id has this flag set, the file is a single file with file_id=UID. */
#define DBOX_FILE_ID_FLAG_UID 0x80000000
};
enum dbox_metadata_key {
- /* Message is marked as expunged. '0' = no, '1' = yes */
- DBOX_METADATA_EXPUNGED = 'E',
- /* Message flags in dbox_metadata_flags order. '0' = not set, anything
- else = set. Unknown flags should be preserved. */
- DBOX_METADATA_FLAGS = 'F',
+ /* metadata used by old Dovecot versions */
+ DBOX_METADATA_OLD_EXPUNGED = 'E',
+ DBOX_METADATA_OLD_FLAGS = 'F',
+ DBOX_METADATA_OLD_KEYWORDS = 'K',
+
/* Globally unique identifier for the message. Preserved when
copying. */
DBOX_METADATA_GUID = 'G',
- /* Space separated list of keywords */
- DBOX_METADATA_KEYWORDS = 'K',
/* POP3 UIDL overriding the default format */
DBOX_METADATA_POP3_UIDL = 'P',
/* Received UNIX timestamp in hex */
unsigned int append_offset_header_pos;
unsigned int append_count;
- uint32_t last_append_uid, maildir_append_seq;
+ uint32_t last_append_uid;
uoff_t append_offset;
time_t create_time;
/* Metadata for the currently seeked metadata block. */
pool_t metadata_pool;
ARRAY_DEFINE(metadata, const char *);
- ARRAY_DEFINE(metadata_changes, const char *);
uoff_t metadata_read_offset;
- unsigned int metadata_space_pos;
- /* Includes the trailing LF that shouldn't be used */
- unsigned int metadata_len;
unsigned int alt_path:1;
unsigned int maildir_file:1;
struct dbox_file *
dbox_file_init(struct dbox_mailbox *mbox, unsigned int file_id);
-struct dbox_file *
-dbox_file_init_new_maildir(struct dbox_mailbox *mbox, const char *fname);
void dbox_file_unref(struct dbox_file **file);
/* Free all currently opened files. */
/* Return wanted metadata value, or NULL if not found. */
const char *dbox_file_metadata_get(struct dbox_file *file,
enum dbox_metadata_key key);
-/* Add key=value metadata update (not written yet, not visible to _get()).
- The changes are reset by dbox_file_metadata_seek() call. */
-void dbox_file_metadata_set(struct dbox_file *file, enum dbox_metadata_key key,
- const char *value);
-/* Write all metadata updates to disk. Returns 1 if ok, 0 if metadata doesn't
- fit to its reserved space and message isn't last in file, -1 if I/O error. */
-int dbox_file_metadata_write(struct dbox_file *file);
/* Write all metadata to output stream. Returns 0 if ok, -1 if I/O error. */
int dbox_file_metadata_write_to(struct dbox_file *file, struct ostream *output);
/* Move the file to alt path or back. */
int dbox_file_move(struct dbox_file *file, bool alt_path);
-/* Append flags as metadata value to given string */
-void dbox_mail_metadata_flags_append(string_t *str, enum mail_flags flags);
-/* Append keywords as metadata value to given string */
-void dbox_mail_metadata_keywords_append(struct dbox_mailbox *mbox,
- string_t *str,
- const struct mail_keywords *keywords);
-
/* Fill dbox_message_header with given uid/size. */
void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr,
uint32_t uid, uoff_t message_size);
return 0;
}
-int dbox_index_get_uid_validity(struct dbox_index *index,
- uint32_t *uid_validity_r)
-{
- if (index->fd == -1) {
- if (dbox_index_refresh(index) < 0)
- return -1;
- }
- *uid_validity_r = index->uid_validity;
- return 0;
-}
-
static int dbox_index_record_cmp(const void *key, const void *data)
{
const unsigned int *file_id = key;
rec->locked = FALSE;
}
-int dbox_index_try_lock_recreate(struct dbox_index *index)
-{
- int i, ret;
-
- if (index->fd == -1) {
- if (dbox_index_refresh(index) < 0)
- return 1;
- }
-
- for (i = 0; i < DBOX_INDEX_LOCK_RETRY_COUNT; i++) {
- /* lock the whole file */
- ret = dbox_index_lock_range(index, F_SETLK, F_WRLCK, 0, 0);
- if (ret <= 0)
- return ret;
- if ((ret = dbox_index_refresh(index)) <= 0)
- return ret < 0 ? -1 : 1;
- }
-
- i_warning("dbox index keeps getting recreated: %s", index->path);
- return 0;
-}
-
static int dbox_index_lock_header(struct dbox_index *index)
{
int i, ret;
return 0;
}
-void dbox_index_append_file(struct dbox_index_append_context *ctx,
- struct dbox_file *file)
-{
- file->refcount++;
- array_append(&ctx->files, &file, 1);
-}
-
static const char *dbox_file_maildir_get_index_data(struct dbox_file *file)
{
- const char *pop3_uidl = NULL, *const *changes;
- unsigned int i, count;
-
- if (array_is_created(&file->metadata_changes))
- changes = array_get(&file->metadata_changes, &count);
- else {
- changes = NULL;
- count = 0;
- }
- for (i = 0; i < count; i++) {
- if (*changes[i] == DBOX_METADATA_POP3_UIDL) {
- pop3_uidl = changes[i] + 1;
- break;
- }
- }
-
- if (pop3_uidl == NULL) {
- return t_strdup_printf("%u :%s", file->last_append_uid,
- file->fname);
- } else {
- return t_strdup_printf("%u P%s :%s", file->last_append_uid,
- pop3_uidl, file->fname);
- }
+ return t_strdup_printf("%u :%s", file->last_append_uid,
+ file->fname);
}
static int dbox_index_append_commit_new(struct dbox_index_append_context *ctx,
struct dbox_index *dbox_index_init(struct dbox_mailbox *mbox);
void dbox_index_deinit(struct dbox_index **index);
-/* Get the current UIDVALIDITY. Returns 0 if ok, -1 if I/O error. */
-int dbox_index_get_uid_validity(struct dbox_index *index,
- uint32_t *uid_validity_r);
-
struct dbox_index_record *
dbox_index_record_lookup(struct dbox_index *index, unsigned int file_id);
enum dbox_index_file_lock_status *lock_status_r);
void dbox_index_unlock_file(struct dbox_index *index, unsigned int file_id);
-/* Try to lock index file for recreating. Returns 1 if ok, 0 if file already
- contains locks, -1 if error. */
-int dbox_index_try_lock_recreate(struct dbox_index *index);
-
struct dbox_index_append_context *
dbox_index_append_begin(struct dbox_index *index);
/* Request file for saving a new message with given size. If an existing file
uoff_t mail_size,
struct dbox_file **file_r,
struct ostream **output_r);
-void dbox_index_append_file(struct dbox_index_append_context *ctx,
- struct dbox_file *file);
/* Assign file_ids to all appended files. */
int dbox_index_append_assign_file_ids(struct dbox_index_append_context *ctx);
/* Returns 0 if ok, -1 if error. */
int dbox_index_append_commit(struct dbox_index_append_context **ctx);
void dbox_index_append_rollback(struct dbox_index_append_context **ctx);
-/* Mark */
-void dbox_index_mark_expunges(struct dbox_index *index, unsigned int file_id);
-void dbox_index_mark_dirty(struct dbox_index *index, unsigned int file_id);
-
#endif
unsigned int finished:1;
};
-static void dbox_save_keywords(struct dbox_save_context *ctx,
- struct mail_keywords *keywords)
-{
- if (ctx->cur_keywords == NULL)
- ctx->cur_keywords = str_new(default_pool, 128);
- else
- str_truncate(ctx->cur_keywords, 0);
- dbox_mail_metadata_keywords_append(ctx->mbox, ctx->cur_keywords,
- keywords);
-}
-
struct mail_save_context *
dbox_save_alloc(struct mailbox_transaction_context *_t)
{
if (_ctx->received_date == (time_t)-1)
_ctx->received_date = ioloop_time;
- dbox_save_keywords(ctx, _ctx->keywords);
return ctx->failed ? -1 : 0;
}
static void dbox_save_write_metadata(struct dbox_save_context *ctx)
{
struct dbox_metadata_header metadata_hdr;
- char space[DBOX_EXTRA_SPACE];
const char *guid;
string_t *str;
uoff_t vsize;
guid = ctx->ctx.guid != NULL ? ctx->ctx.guid :
mail_generate_guid_string();
str_printfa(str, "%c%s\n", DBOX_METADATA_GUID, guid);
-
- /* flags */
- str_append_c(str, DBOX_METADATA_FLAGS);
- dbox_mail_metadata_flags_append(str, ctx->ctx.flags);
str_append_c(str, '\n');
- /* keywords */
- if (ctx->cur_keywords != NULL && str_len(ctx->cur_keywords) > 0) {
- str_append_c(str, DBOX_METADATA_KEYWORDS);
- str_append_str(str, ctx->cur_keywords);
- str_append_c(str, '\n');
- }
-
o_stream_send(ctx->cur_output, str_data(str), str_len(str));
- memset(space, ' ', sizeof(space));
- o_stream_send(ctx->cur_output, space, sizeof(space));
- o_stream_send(ctx->cur_output, "\n", 1);
}
static int dbox_save_mail_write_header(struct dbox_save_mail *mail)
i_assert(ctx->finished);
- if (dbox_sync_begin(ctx->mbox, &ctx->sync_ctx, FALSE, TRUE) < 0) {
+ if (dbox_sync_begin(ctx->mbox, TRUE, &ctx->sync_ctx) < 0) {
ctx->failed = TRUE;
dbox_transaction_save_rollback(ctx);
return -1;
#include "unlink-old-files.h"
#include "index-mail.h"
#include "mail-copy.h"
+#include "maildir/maildir-uidlist.h"
#include "dbox-sync.h"
#include "dbox-index.h"
#include "dbox-file.h"
mbox->path = p_strdup(pool, path);
mbox->alt_path = p_strdup(pool, dbox_get_alt_path(storage, path));
mbox->storage = storage;
- mbox->last_interactive_change = ioloop_time;
value = getenv("DBOX_ROTATE_SIZE");
if (value != NULL)
mbox->dbox_index = dbox_index_init(mbox);
index_storage_mailbox_init(&mbox->ibox, name, flags, FALSE);
+ mbox->maildir_uidlist = maildir_uidlist_init_readonly(&mbox->ibox);
return &mbox->ibox.box;
}
static int dbox_storage_mailbox_close(struct mailbox *box)
{
struct dbox_mailbox *mbox = (struct dbox_mailbox *)box;
- int ret = 0;
-
- if (box->opened) {
- /* see if we want to flush dirty flags */
- ret = dbox_sync(mbox, TRUE);
- }
+ maildir_uidlist_deinit(&mbox->maildir_uidlist);
dbox_index_deinit(&mbox->dbox_index);
dbox_files_free(mbox);
array_free(&mbox->open_files);
- return index_storage_mailbox_close(box) < 0 ? -1 : ret;
+ return index_storage_mailbox_close(box);
}
static int dbox_mailbox_create(struct mail_storage *_storage,
#define DBOX_INDEX_FLAG_ALT MAIL_INDEX_MAIL_FLAG_BACKEND
struct dbox_index_header {
- uint32_t last_dirty_flush_stamp;
+ uint32_t unused; /* for backwards compatibility */
+ uint32_t highest_maildir_uid;
};
struct dbox_storage {
struct index_mailbox ibox;
struct dbox_storage *storage;
+ struct maildir_uidlist *maildir_uidlist;
+ uint32_t highest_maildir_uid;
+
struct dbox_index *dbox_index;
uint32_t dbox_ext_id, dbox_hdr_ext_id;
- /* timestamp when the mailbox was last modified interactively */
- time_t last_interactive_change;
- /* set while rebuilding indexes with converted maildir files */
- struct maildir_keywords_sync_ctx *maildir_sync_keywords;
uoff_t rotate_size, rotate_min_size;
unsigned int rotate_days;
static int dbox_sync_file_unlink(struct dbox_file *file)
{
const char *path;
- int i;
+ int i = 0;
path = t_strdup_printf("%s/%s", file->mbox->path, file->fname);
- for (i = 0;; i++) {
- if (unlink(path) == 0)
- break;
-
+ while (unlink(path) < 0) {
if (errno != ENOENT) {
mail_storage_set_critical(file->mbox->ibox.box.storage,
"unlink(%s) failed: %m", path);
/* not found */
i_warning("dbox: File unexpectedly lost: %s/%s",
file->mbox->path, file->fname);
- break;
+ return 0;
}
/* try the alternative path */
path = t_strdup_printf("%s/%s", file->mbox->alt_path,
file->fname);
+ i++;
}
- return 0;
-}
-
-static void
-dbox_sync_update_metadata(struct dbox_sync_context *ctx, struct dbox_file *file,
- const struct dbox_sync_file_entry *entry,
- uint32_t seq)
-{
- const struct mail_index_record *rec;
- ARRAY_TYPE(keyword_indexes) keyword_indexes;
- struct mail_keywords *keywords;
- string_t *value;
- const char *old_value;
-
- value = t_str_new(256);
-
- /* flags */
- rec = mail_index_lookup(ctx->sync_view, seq);
- dbox_mail_metadata_flags_append(value, rec->flags);
- dbox_file_metadata_set(file, DBOX_METADATA_FLAGS, str_c(value));
-
- /* keywords */
- t_array_init(&keyword_indexes, 32);
- mail_index_lookup_keywords(ctx->sync_view, seq, &keyword_indexes);
- old_value = dbox_file_metadata_get(file, DBOX_METADATA_KEYWORDS);
- if (array_count(&keyword_indexes) > 0 ||
- (old_value != NULL && *old_value != '\0' &&
- array_count(&keyword_indexes) == 0)) {
- str_truncate(value, 0);
- keywords = mail_index_keywords_create_from_indexes(
- ctx->mbox->ibox.index, &keyword_indexes);
- dbox_mail_metadata_keywords_append(ctx->mbox, value, keywords);
- mail_index_keywords_free(&keywords);
-
- dbox_file_metadata_set(file, DBOX_METADATA_KEYWORDS,
- str_c(value));
- }
-
- /* expunge state */
- if (entry != NULL &&
- array_is_created(&entry->expunges) &&
- seq_range_exists(&entry->expunges, seq)) {
- dbox_file_metadata_set(file, DBOX_METADATA_EXPUNGED, "1");
- mail_index_expunge(ctx->trans, seq);
- }
+ return 1;
}
static int
/* write metadata */
(void)dbox_file_metadata_seek_mail_offset(file, offset,
&expunged);
- T_BEGIN {
- dbox_sync_update_metadata(ctx, file, entry, seq);
- } T_END;
if ((ret = dbox_file_metadata_write_to(file, output)) < 0)
break;
return ret;
}
+#if 0
static int
dbox_sync_file_split(struct dbox_sync_context *ctx, struct dbox_file *in_file,
uoff_t offset, uint32_t seq)
}
append_offset = output->offset;
dbox_msg_header_fill(&dbox_msg_hdr, uid, size);
- T_BEGIN {
- dbox_sync_update_metadata(ctx, out_file, NULL, seq);
- } T_END;
/* set static metadata */
for (i = 0; i < N_ELEMENTS(maildir_metadata_keys); i++) {
}
return ret < 0 ? -1 : 1;
}
-
-static int
-dbox_sync_file_changes(struct dbox_sync_context *ctx, struct dbox_file *file,
- const struct dbox_sync_file_entry *entry, uint32_t seq)
-{
- uint32_t file_id;
- uoff_t offset;
- bool expunged;
- int ret;
-
- if (!dbox_file_lookup(ctx->mbox, ctx->sync_view, seq,
- &file_id, &offset))
- return 0;
- i_assert(file_id == file->file_id);
-
- ret = dbox_file_metadata_seek_mail_offset(file, offset, &expunged);
- if (ret <= 0)
- return ret;
- if (expunged) {
- mail_index_expunge(ctx->trans, seq);
- return 1;
- }
-
- T_BEGIN {
- dbox_sync_update_metadata(ctx, file, entry, seq);
- } T_END;
- ret = dbox_file_metadata_write(file);
- if (ret <= 0) {
- return ret < 0 ? -1 :
- dbox_sync_file_split(ctx, file, offset, seq);
- }
-
- mail_index_update_flags(ctx->trans, seq, MODIFY_REMOVE,
- (enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY);
- return 1;
-}
-
-static int
-dbox_sync_file_int(struct dbox_sync_context *ctx, struct dbox_file *file,
- const struct dbox_sync_file_entry *entry, bool full_expunge)
-{
- const struct seq_range *seqs;
- unsigned int i, count;
- uint32_t seq, first_expunge_seq;
- int ret;
-
- if (array_is_created(&entry->expunges) && full_expunge) {
- seqs = array_idx(&entry->expunges, 0);
- first_expunge_seq = seqs->seq1;
- } else {
- first_expunge_seq = (uint32_t)-1;
- }
-
- if (array_is_created(&entry->changes)) {
- seqs = array_get(&entry->changes, &count);
- } else {
- seqs = NULL;
- count = 0;
- }
- for (i = 0; i < count; ) {
- for (seq = seqs[i].seq1; seq <= seqs[i].seq2; seq++) {
- if (seq >= first_expunge_seq)
- return dbox_sync_file_expunge(ctx, file, entry);
-
- ret = dbox_sync_file_changes(ctx, file, entry, seq);
- if (ret <= 0)
- return ret;
- }
- i++;
- }
- if (first_expunge_seq != (uint32_t)-1)
- return dbox_sync_file_expunge(ctx, file, entry);
- return 1;
-}
+#endif
static void
-dbox_sync_file_move_if_needed(struct dbox_sync_context *ctx,
- struct dbox_file *file,
+dbox_sync_file_move_if_needed(struct dbox_file *file,
const struct dbox_sync_file_entry *entry)
{
- const struct seq_range *seq;
- const struct mail_index_record *rec;
- bool new_alt_path;
-
- if (!array_is_created(&entry->changes))
+ if (!entry->move_to_alt && !entry->move_from_alt)
return;
- /* check if we want to move the file to alt path or back.
- FIXME: change this check somehow when a file may contain
- multiple messages. */
- seq = array_idx(&entry->changes, 0);
- rec = mail_index_lookup(ctx->sync_view, seq[0].seq1);
- new_alt_path = (rec->flags & DBOX_INDEX_FLAG_ALT) != 0;
- if (new_alt_path != file->alt_path) {
+ if (entry->move_to_alt != file->alt_path) {
/* move the file. if it fails, nothing broke so
don't worry about it. */
- (void)dbox_file_move(file, new_alt_path);
+ (void)dbox_file_move(file, !file->alt_path);
}
}
status == DBOX_INDEX_FILE_STATUS_MAILDIR) &&
array_is_created(&entry->expunges)) {
/* fast path to expunging the whole file */
- if (dbox_sync_file_unlink(file) < 0)
- ret = -1;
- else {
+ if ((ret = dbox_sync_file_unlink(file)) == 0) {
+ /* file was lost, delete it */
dbox_sync_mark_single_file_expunged(ctx, entry);
ret = 1;
}
} else {
ret = dbox_file_open_or_create(file, TRUE, &deleted);
if (ret > 0 && !deleted) {
- dbox_sync_file_move_if_needed(ctx, file, entry);
- ret = dbox_sync_file_int(ctx, file, entry, locked);
+ dbox_sync_file_move_if_needed(file, entry);
+ if (array_is_created(&entry->expunges) && locked)
+ ret = dbox_sync_file_expunge(ctx, file, entry);
}
}
dbox_file_unref(&file);
/* Copyright (c) 2007-2009 Dovecot authors, see the included COPYING file */
#include "lib.h"
+#include "ioloop.h"
#include "array.h"
#include "dbox-storage.h"
-#include "../maildir/maildir-uidlist.h"
-#include "../maildir/maildir-keywords.h"
-#include "dbox-index.h"
+#include "maildir/maildir-uidlist.h"
+#include "maildir/maildir-keywords.h"
+#include "maildir/maildir-filename.h"
#include "dbox-file.h"
#include "dbox-sync.h"
struct dbox_sync_rebuild_context {
struct dbox_mailbox *mbox;
- struct dbox_index_append_context *append_ctx;
struct mail_index_view *view;
struct mail_index_transaction *trans;
uint32_t cache_ext_id;
uint32_t cache_reset_id;
- struct maildir_uidlist *maildir_uidlist;
+ struct maildir_uidlist_sync_ctx *maildir_sync_ctx;
struct maildir_keywords *mk;
+ struct maildir_keywords_sync_ctx *maildir_sync_keywords;
- ARRAY_DEFINE(maildir_new_files, char *);
- uint32_t maildir_new_uid;
+ uint32_t highest_uid;
unsigned int cache_used:1;
};
-static int dbox_sync_set_uidvalidity(struct dbox_sync_rebuild_context *ctx)
+static void dbox_sync_set_uidvalidity(struct dbox_sync_rebuild_context *ctx)
{
+
uint32_t uid_validity;
- if (dbox_index_get_uid_validity(ctx->mbox->dbox_index,
- &uid_validity) < 0)
- return -1;
+ /* if uidvalidity is set in the old index, use it */
+ uid_validity = mail_index_get_header(ctx->view)->uid_validity;
+ if (uid_validity == 0)
+ uid_validity = ioloop_time;
mail_index_update_header(ctx->trans,
offsetof(struct mail_index_header, uid_validity),
&uid_validity, sizeof(uid_validity), TRUE);
- return 0;
}
static void
dbox_sync_index_copy_cache(ctx, old_seq, new_seq);
}
+static void
+dbox_sync_index_copy_from_maildir(struct dbox_sync_rebuild_context *ctx,
+ struct dbox_file *file, uint32_t seq)
+{
+ struct dbox_mailbox *mbox = file->mbox;
+ ARRAY_TYPE(keyword_indexes) keyword_indexes;
+ struct mail_keywords *keywords;
+ enum mail_flags flags;
+
+ t_array_init(&keyword_indexes, 32);
+ maildir_filename_get_flags(ctx->maildir_sync_keywords,
+ file->fname, &flags, &keyword_indexes);
+ mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags);
+
+ keywords = mail_index_keywords_create_from_indexes(mbox->ibox.index,
+ &keyword_indexes);
+ mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE, keywords);
+ mail_index_keywords_free(&keywords);
+}
+
static void
dbox_sync_index_metadata(struct dbox_sync_rebuild_context *ctx,
struct dbox_file *file, uint32_t seq, uint32_t uid)
{
- const char *value;
- struct mail_keywords *keywords;
- enum mail_flags flags = 0;
uint32_t old_seq;
- unsigned int i;
if (mail_index_lookup_seq(ctx->view, uid, &old_seq)) {
/* the message exists in the old index.
copy the metadata from it. */
dbox_sync_index_copy_from_old(ctx, old_seq, seq);
- return;
+ } else if (file->maildir_file) {
+ /* we're probably doing initial sync after migration from
+ maildir. preserve the old flags. */
+ dbox_sync_index_copy_from_maildir(ctx, file, seq);
}
-
- value = dbox_file_metadata_get(file, DBOX_METADATA_FLAGS);
- if (value != NULL) {
- for (i = 0; value[i] != '\0'; i++) {
- if (value[i] != '0' && i < DBOX_METADATA_FLAGS_COUNT)
- flags |= dbox_mail_flags_map[i];
- }
- mail_index_update_flags(ctx->trans, seq, MODIFY_REPLACE, flags);
- }
-
- value = dbox_file_metadata_get(file, DBOX_METADATA_KEYWORDS);
- if (value != NULL) T_BEGIN {
- keywords = mail_index_keywords_create(ctx->mbox->ibox.index,
- t_strsplit_spaces(value, " "));
- mail_index_update_keywords(ctx->trans, seq, MODIFY_REPLACE,
- keywords);
- mail_index_keywords_free(&keywords);
- } T_END;
}
static int dbox_sync_index_file_next(struct dbox_sync_rebuild_context *ctx,
{
uint32_t seq, uid;
uoff_t physical_size;
- const char *path, *value;
+ const char *path;
bool expunged;
int ret;
return 0;
}
if (file->maildir_file) {
- i_assert(uid == 0);
- if (!maildir_uidlist_get_uid(ctx->maildir_uidlist, file->fname,
- &uid)) {
- if (ctx->maildir_new_uid == 0) {
- /* not in uidlist, give it an uid later */
- char *fname = i_strdup(file->fname);
- array_append(&ctx->maildir_new_files,
- &fname, 1);
- return 0;
- }
- uid = ctx->maildir_new_uid++;
- }
+ i_assert(uid != 0);
+
file->append_count = 1;
file->last_append_uid = uid;
}
i_warning("%s: Ignoring broken file (metadata)", path);
return 0;
}
- if (file->maildir_file) {
- /* preserve POP3 UIDL */
- value = maildir_uidlist_lookup_ext(ctx->maildir_uidlist, uid,
- MAILDIR_UIDLIST_REC_EXT_POP3_UIDL);
- if (value != NULL) {
- dbox_file_metadata_set(file, DBOX_METADATA_POP3_UIDL,
- value);
- }
- }
if (!expunged) {
mail_index_append(ctx->trans, uid, &seq);
- file->maildir_append_seq = seq;
dbox_sync_index_metadata(ctx, file, seq, uid);
}
return 1;
return 0;
}
+ if (ctx->highest_uid < uid)
+ ctx->highest_uid = uid;
+
file = dbox_file_init(ctx->mbox, uid | DBOX_FILE_ID_FLAG_UID);
file->current_path = i_strdup_printf("%s/%s", dir, fname);
dbox_sync_index_maildir_file(struct dbox_sync_rebuild_context *ctx,
const char *fname)
{
- struct dbox_file *file;
- uoff_t offset = 0;
int ret;
- if (ctx->mbox->maildir_sync_keywords == NULL) {
- ctx->maildir_uidlist =
- maildir_uidlist_init_readonly(&ctx->mbox->ibox);
+ if (ctx->maildir_sync_ctx == NULL) {
+ i_assert(ctx->mk == NULL);
+
ctx->mk = maildir_keywords_init_readonly(&ctx->mbox->ibox.box);
- ctx->mbox->maildir_sync_keywords =
+ ctx->maildir_sync_keywords =
maildir_keywords_sync_init(ctx->mk,
ctx->mbox->ibox.index);
- if (maildir_uidlist_refresh(ctx->maildir_uidlist) < 0)
+ ret = maildir_uidlist_sync_init(ctx->mbox->maildir_uidlist,
+ MAILDIR_UIDLIST_SYNC_NOLOCK,
+ &ctx->maildir_sync_ctx);
+ if (ret <= 0) {
+ i_assert(ret < 0);
return -1;
+ }
}
- file = dbox_file_init_new_maildir(ctx->mbox, fname);
- if ((ret = dbox_sync_index_file_next(ctx, file, &offset)) > 0) {
- dbox_index_append_file(ctx->append_ctx, file);
- /* appending referenced the file, so make sure it gets closed
- so we don't have too many open files. */
- dbox_file_close(file);
+ /* sync all maildir files first and let maildir uidlist code assign
+ UIDs for unseen files. */
+ ret = maildir_uidlist_sync_next(ctx->maildir_sync_ctx, fname, 0);
+ if (ret == 0) {
+ i_warning("%s: Ignoring duplicate maildir file: %s",
+ ctx->mbox->path, fname);
}
- dbox_file_unref(&file);
- return ret < 0 ? -1 : 0;
+ return ret;
}
static int
"opendir(%s) failed: %m", path);
return -1;
}
- for (;;) {
+ do {
errno = 0;
if ((d = readdir(dir)) == NULL)
break;
ret = dbox_sync_index_file(ctx, path, d->d_name,
primary);
} T_END;
- }
+ } while (ret >= 0);
if (errno != 0) {
mail_storage_set_critical(storage,
"readdir(%s) failed: %m", path);
return ret;
}
-static int dbox_sync_new_maildir(struct dbox_sync_rebuild_context *ctx)
+static int dbox_sync_maildir_finish(struct dbox_sync_rebuild_context *ctx)
{
+ struct dbox_mailbox *mbox = ctx->mbox;
+ struct maildir_uidlist_iter_ctx *iter;
struct mail_index_view *trans_view;
- char *const *fnames;
- unsigned int i, count;
+ struct dbox_file *file;
+ const char *fname;
+ enum maildir_uidlist_rec_flag flags;
+ uint32_t uid, next_uid;
+ uoff_t offset;
int ret = 0;
- fnames = array_get(&ctx->maildir_new_files, &count);
- if (count == 0)
- return 0;
+ /* we'll need the uidlist to contain the latest filenames.
+ since there's no easy way to figure out if they changed, just
+ recreate the uidlist always. */
+ maildir_uidlist_sync_recreate(ctx->maildir_sync_ctx);
+ /* update the maildir uidlist's next_uid if we have seen higher
+ dbox UIDs */
trans_view = mail_index_transaction_open_updated_view(ctx->trans);
- ctx->maildir_new_uid = mail_index_get_header(trans_view)->next_uid;
+ next_uid = mail_index_get_header(trans_view)->next_uid;
mail_index_view_close(&trans_view);
+ maildir_uidlist_set_next_uid(mbox->maildir_uidlist, next_uid, FALSE);
+ maildir_uidlist_set_next_uid(mbox->maildir_uidlist,
+ ctx->highest_uid + 1, FALSE);
+ /* assign UIDs for new maildir mails before iterating */
+ maildir_uidlist_sync_finish(ctx->maildir_sync_ctx);
+
+ mbox->highest_maildir_uid =
+ maildir_uidlist_get_next_uid(mbox->maildir_uidlist);
+
+ iter = maildir_uidlist_iter_init(mbox->maildir_uidlist);
+ while (maildir_uidlist_iter_next(iter, &uid, &flags, &fname)) {
+ file = dbox_file_init(mbox, uid | DBOX_FILE_ID_FLAG_UID);
+ file->current_path =
+ i_strdup_printf("%s/%s", ctx->mbox->path, fname);
+
+ offset = 0;
+ ret = dbox_sync_index_file_next(ctx, file, &offset);
+ dbox_file_unref(&file);
+ if (ret < 0)
+ break;
+ }
+ maildir_uidlist_iter_deinit(&iter);
+ return ret < 0 ? -1 : 0;
+}
- for (i = 0; i < count && ret == 0; i++) {
- T_BEGIN {
- ret = dbox_sync_index_maildir_file(ctx, fnames[i]);
- } T_END;
+static void dbox_sync_update_header(struct dbox_sync_rebuild_context *ctx)
+{
+ const struct dbox_index_header *hdr;
+ struct dbox_index_header new_hdr;
+ const void *data;
+ size_t data_size;
+
+ mail_index_get_header_ext(ctx->mbox->ibox.view,
+ ctx->mbox->dbox_hdr_ext_id,
+ &data, &data_size);
+ hdr = data;
+ if (data_size == sizeof(*hdr)) {
+ if (hdr->highest_maildir_uid >= ctx->mbox->highest_maildir_uid) {
+ /* nothing to change */
+ return;
+ }
+ new_hdr = *hdr;
+ } else {
+ memset(&new_hdr, 0, sizeof(new_hdr));
}
- return ret;
+ new_hdr.highest_maildir_uid = ctx->mbox->highest_maildir_uid;
+ mail_index_update_header_ext(ctx->trans, ctx->mbox->dbox_hdr_ext_id, 0,
+ &new_hdr, sizeof(new_hdr));
}
static int dbox_sync_index_rebuild_ctx(struct dbox_sync_rebuild_context *ctx)
{
- if (dbox_sync_set_uidvalidity(ctx) < 0)
- return -1;
-
+ dbox_sync_set_uidvalidity(ctx);
if (dbox_sync_index_rebuild_dir(ctx, ctx->mbox->path, TRUE) < 0)
return -1;
return -1;
}
- /* finally give UIDs to newly seen maildir files */
- return dbox_sync_new_maildir(ctx);
-}
-
-static void dbox_sync_update_maildir_ids(struct dbox_sync_rebuild_context *ctx)
-{
- struct dbox_mail_index_record rec;
- struct dbox_file *const *files;
- unsigned int i, count;
-
- memset(&rec, 0, sizeof(rec));
- files = array_get(&ctx->mbox->open_files, &count);
- for (i = 0; i < count; i++) {
- if (!files[i]->maildir_file)
- continue;
-
- i_assert(files[i]->file_id != 0);
- rec.file_id = files[i]->file_id;
- mail_index_update_ext(ctx->trans, files[i]->maildir_append_seq,
- ctx->mbox->dbox_ext_id, &rec, NULL);
- }
+ if (dbox_sync_maildir_finish(ctx) < 0)
+ return -1;
+ dbox_sync_update_header(ctx);
+ return 0;
}
int dbox_sync_index_rebuild(struct dbox_mailbox *mbox)
struct dbox_sync_rebuild_context ctx;
uint32_t seq;
uoff_t offset;
- char **fnames;
- unsigned int i, count;
int ret;
memset(&ctx, 0, sizeof(ctx));
ctx.mbox = mbox;
- ctx.append_ctx = dbox_index_append_begin(mbox->dbox_index);
ctx.view = mail_index_view_open(mbox->ibox.index);
ctx.trans = mail_index_transaction_begin(ctx.view,
MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
- i_array_init(&ctx.maildir_new_files, 8);
mail_index_reset(ctx.trans);
index_mailbox_reset_uidvalidity(&mbox->ibox);
mail_index_ext_lookup(mbox->ibox.index, "cache", &ctx.cache_ext_id);
if ((ret = dbox_sync_index_rebuild_ctx(&ctx)) < 0)
mail_index_transaction_rollback(&ctx.trans);
- else {
- ret = dbox_index_append_assign_file_ids(ctx.append_ctx);
- if (ret == 0) {
- dbox_sync_update_maildir_ids(&ctx);
- ret = mail_index_transaction_commit(&ctx.trans,
- &seq, &offset);
- }
- }
- mail_index_view_close(&ctx.view);
-
- fnames = array_get_modifiable(&ctx.maildir_new_files, &count);
- for (i = 0; i < count; i++)
- i_free(fnames[i]);
- array_free(&ctx.maildir_new_files);
-
- if (ret == 0)
- ret = dbox_index_append_commit(&ctx.append_ctx);
else
- dbox_index_append_rollback(&ctx.append_ctx);
+ ret = mail_index_transaction_commit(&ctx.trans, &seq, &offset);
+ mail_index_view_close(&ctx.view);
- if (mbox->maildir_sync_keywords != NULL)
- maildir_keywords_sync_deinit(&mbox->maildir_sync_keywords);
+ if (ctx.maildir_sync_ctx != NULL) {
+ if (maildir_uidlist_sync_deinit(&ctx.maildir_sync_ctx) < 0)
+ ret = -1;
+ }
+ if (ctx.maildir_sync_keywords != NULL)
+ maildir_keywords_sync_deinit(&ctx.maildir_sync_keywords);
if (ctx.mk != NULL)
maildir_keywords_deinit(&ctx.mk);
- if (ctx.maildir_uidlist != NULL)
- maildir_uidlist_deinit(&ctx.maildir_uidlist);
return ret;
}
#include "str.h"
#include "hash.h"
#include "dbox-storage.h"
-#include "dbox-index.h"
#include "dbox-file.h"
#include "dbox-sync.h"
-#define DBOX_FLUSH_SECS_INTERACTIVE (4*60*60)
-#define DBOX_FLUSH_SECS_CLOSE (4*60*60)
-#define DBOX_FLUSH_SECS_IMMEDIATE (24*60*60)
-
#define DBOX_REBUILD_COUNT 3
static int dbox_sync_add_seq(struct dbox_sync_context *ctx,
struct dbox_sync_file_entry *entry;
uint32_t file_id;
uoff_t offset;
- bool uid_file, add;
+ bool uid_file;
+
+ i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE ||
+ sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
if (!dbox_file_lookup(ctx->mbox, ctx->sync_view, seq,
&file_id, &offset))
entry = hash_table_lookup(ctx->syncs, POINTER_CAST(file_id));
if (entry == NULL) {
- if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE ||
- ctx->flush_dirty_flags) {
- /* expunges / flushing dirty flags */
- add = TRUE;
- } else if (sync_rec->type != MAIL_INDEX_SYNC_TYPE_FLAGS) {
- /* keywords, not flushing dirty flags */
- add = FALSE;
- } else {
- /* add if we're moving from/to alternative storage
- and we actually have an alt directory specified */
- add = ((sync_rec->add_flags | sync_rec->remove_flags) &
- DBOX_INDEX_FLAG_ALT) != 0 &&
- ctx->mbox->alt_path != NULL;
- }
-
- if (!add) {
- mail_index_update_flags(ctx->trans, seq, MODIFY_ADD,
- (enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY);
- return 0;
- }
-
entry = p_new(ctx->pool, struct dbox_sync_file_entry, 1);
entry->file_id = file_id;
hash_table_insert(ctx->syncs, POINTER_CAST(file_id), entry);
if (!array_is_created(&entry->expunges)) {
p_array_init(&entry->expunges, ctx->pool,
uid_file ? 1 : 3);
- seq_range_array_add(&ctx->expunge_files, 0, file_id);
}
seq_range_array_add(&entry->expunges, 0, seq);
} else {
- if (!array_is_created(&entry->changes)) {
- p_array_init(&entry->changes, ctx->pool,
- uid_file ? 1 : 8);
- }
- seq_range_array_add(&entry->changes, 0, seq);
+ if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) != 0)
+ entry->move_to_alt = TRUE;
+ else
+ entry->move_from_alt = TRUE;
}
return 0;
}
{
uint32_t seq, seq1, seq2;
- if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_APPEND) {
- /* don't care about appends */
+ if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_EXPUNGE) {
+ /* we're interested */
+ } else if (sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS) {
+ /* we care only about alt flag changes */
+ if ((sync_rec->add_flags & DBOX_INDEX_FLAG_ALT) == 0 &&
+ (sync_rec->remove_flags & DBOX_INDEX_FLAG_ALT) == 0)
+ return 0;
+ } else {
+ /* not interested */
return 0;
}
- /* we assume that anything else than appends are interactive changes */
- ctx->mbox->last_interactive_change = ioloop_time;
-
if (!mail_index_lookup_seq_range(ctx->sync_view,
sync_rec->uid1, sync_rec->uid2,
&seq1, &seq2)) {
return 0;
}
-static int
-dbox_sync_lock_expunge_file(struct dbox_sync_context *ctx, unsigned int file_id)
-{
- struct dbox_index_record *rec;
- enum dbox_index_file_lock_status lock_status;
- int ret;
-
- ret = dbox_index_try_lock_file(ctx->mbox->dbox_index, file_id,
- &lock_status);
- if (ret < 0)
- return -1;
-
- rec = dbox_index_record_lookup(ctx->mbox->dbox_index, file_id);
- switch (lock_status) {
- case DBOX_INDEX_FILE_LOCKED:
- seq_range_array_add(&ctx->locked_files, 0, file_id);
- rec->status = DBOX_INDEX_FILE_STATUS_NONAPPENDABLE;
- break;
- case DBOX_INDEX_FILE_LOCK_NOT_NEEDED:
- case DBOX_INDEX_FILE_LOCK_UNLINKED:
- i_assert(rec == NULL ||
- rec->status != DBOX_INDEX_FILE_STATUS_APPENDABLE);
- break;
- case DBOX_INDEX_FILE_LOCK_TRY_AGAIN:
- rec->expunges = TRUE;
- break;
- }
- return 0;
-}
-
-static int dbox_sync_lock_expunge_files(struct dbox_sync_context *ctx)
-{
- const struct seq_range *range;
- unsigned int i, count, id;
-
- range = array_get(&ctx->expunge_files, &count);
- for (i = 0; i < count; i++) {
- for (id = range[i].seq1; id <= range[i].seq2; id++) {
- if (dbox_sync_lock_expunge_file(ctx, id) < 0)
- return -1;
- }
- }
- return 0;
-}
-
-static void dbox_sync_unlock_files(struct dbox_sync_context *ctx)
-{
- const struct seq_range *range;
- unsigned int i, count, id;
-
- range = array_get(&ctx->locked_files, &count);
- for (i = 0; i < count; i++) {
- for (id = range[i].seq1; id <= range[i].seq2; id++)
- dbox_index_unlock_file(ctx->mbox->dbox_index, id);
- }
-}
-
-static void dbox_sync_update_header(struct dbox_sync_context *ctx)
-{
- struct dbox_index_header hdr;
- const void *data;
- size_t data_size;
-
- if (!ctx->flush_dirty_flags) {
- /* write the header if it doesn't exist yet */
- mail_index_get_header_ext(ctx->mbox->ibox.view,
- ctx->mbox->dbox_hdr_ext_id,
- &data, &data_size);
- if (data_size != 0)
- return;
- }
-
- hdr.last_dirty_flush_stamp = ioloop_time;
- mail_index_update_header_ext(ctx->trans, ctx->mbox->dbox_hdr_ext_id, 0,
- &hdr.last_dirty_flush_stamp,
- sizeof(hdr.last_dirty_flush_stamp));
-}
-
static int dbox_sync_index(struct dbox_sync_context *ctx)
{
struct mailbox *box = &ctx->mbox->ibox.box;
/* read all changes and sort them to file_id order */
ctx->pool = pool_alloconly_create("dbox sync pool", 1024*32);
ctx->syncs = hash_table_create(default_pool, ctx->pool, 0, NULL, NULL);
- i_array_init(&ctx->expunge_files, 32);
- i_array_init(&ctx->locked_files, 32);
for (;;) {
if (!mail_index_sync_next(ctx->index_sync_ctx, &sync_rec))
break;
}
}
- if (ret > 0) {
- if (dbox_sync_lock_expunge_files(ctx) < 0)
- ret = -1;
- }
- array_free(&ctx->expunge_files);
if (ret > 0) {
/* now sync each file separately */
hash_table_iterate_deinit(&iter);
}
- if (ret > 0)
- dbox_sync_update_header(ctx);
-
if (box->v.sync_notify != NULL)
box->v.sync_notify(box, 0, 0);
- dbox_sync_unlock_files(ctx);
- array_free(&ctx->locked_files);
hash_table_destroy(&ctx->syncs);
pool_unref(&ctx->pool);
return ret;
}
-static int dbox_sync_want_flush_dirty(struct dbox_mailbox *mbox,
- bool close_flush_dirty_flags)
+static int dbox_refresh_header(struct dbox_mailbox *mbox)
{
const struct dbox_index_header *hdr;
const void *data;
size_t data_size;
- if (mbox->last_interactive_change <
- ioloop_time - DBOX_FLUSH_SECS_INTERACTIVE)
- return 1;
-
mail_index_get_header_ext(mbox->ibox.view, mbox->dbox_hdr_ext_id,
&data, &data_size);
if (data_size != sizeof(*hdr)) {
- /* data_size=0 means it's never been synced as dbox */
- if (data_size != 0) {
+ /* data_size=0 means it's never been synced as dbox.
+ data_size=4 is for backwards compatibility */
+ if (data_size != 0 && data_size != 4) {
i_warning("dbox %s: Invalid dbox header size",
mbox->path);
}
}
hdr = data;
- if (!close_flush_dirty_flags) {
- if ((time_t)hdr->last_dirty_flush_stamp <
- ioloop_time - DBOX_FLUSH_SECS_IMMEDIATE)
- return 1;
- } else {
- if ((time_t)hdr->last_dirty_flush_stamp <
- ioloop_time - DBOX_FLUSH_SECS_CLOSE)
- return 1;
- }
+ mbox->highest_maildir_uid = hdr->highest_maildir_uid;
return 0;
}
-int dbox_sync_begin(struct dbox_mailbox *mbox,
- struct dbox_sync_context **ctx_r,
- bool close_flush_dirty_flags, bool force)
+int dbox_sync_begin(struct dbox_mailbox *mbox, bool force,
+ struct dbox_sync_context **ctx_r)
{
struct mail_storage *storage = mbox->ibox.box.storage;
struct dbox_sync_context *ctx;
enum mail_index_sync_flags sync_flags = 0;
unsigned int i;
int ret;
- bool rebuild = FALSE;
-
- ret = dbox_sync_want_flush_dirty(mbox, close_flush_dirty_flags);
- if (ret > 0)
- sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY;
- else if (ret < 0)
- rebuild = TRUE;
- else {
- if (close_flush_dirty_flags) {
- /* no need to sync */
- *ctx_r = NULL;
- return 0;
- }
- }
+ bool rebuild;
+
+ rebuild = dbox_refresh_header(mbox) < 0;
ctx = i_new(struct dbox_sync_context, 1);
ctx->mbox = mbox;
- ctx->flush_dirty_flags =
- (sync_flags & MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY) != 0;
if (!mbox->ibox.keep_recent)
sync_flags |= MAIL_INDEX_SYNC_FLAG_DROP_RECENT;
return ret;
}
- if (rebuild && dbox_sync_want_flush_dirty(mbox, FALSE) >= 0) {
+ if (rebuild && dbox_refresh_header(mbox) < 0) {
/* another process rebuilt it already */
rebuild = FALSE;
}
return 0;
}
-int dbox_sync(struct dbox_mailbox *mbox, bool close_flush_dirty_flags)
+int dbox_sync(struct dbox_mailbox *mbox)
{
struct dbox_sync_context *sync_ctx;
- if (dbox_sync_begin(mbox, &sync_ctx,
- close_flush_dirty_flags, FALSE) < 0)
+ if (dbox_sync_begin(mbox, FALSE, &sync_ctx) < 0)
return -1;
if (sync_ctx == NULL)
index_storage_mailbox_open(&mbox->ibox);
if (index_mailbox_want_full_sync(&mbox->ibox, flags))
- ret = dbox_sync(mbox, FALSE);
+ ret = dbox_sync(mbox);
return index_mailbox_sync_init(box, flags, ret < 0);
}
struct dbox_sync_file_entry {
uint32_t file_id;
- ARRAY_TYPE(seq_range) changes;
+ unsigned int move_from_alt:1;
+ unsigned int move_to_alt:1;
ARRAY_TYPE(seq_range) expunges;
};
pool_t pool;
struct hash_table *syncs; /* struct dbox_sync_file_entry */
- ARRAY_TYPE(seq_range) expunge_files;
- ARRAY_TYPE(seq_range) locked_files;
-
- unsigned int flush_dirty_flags:1;
};
-int dbox_sync_begin(struct dbox_mailbox *mbox,
- struct dbox_sync_context **ctx_r,
- bool close_flush_dirty_flags, bool force);
+int dbox_sync_begin(struct dbox_mailbox *mbox, bool force,
+ struct dbox_sync_context **ctx_r);
int dbox_sync_finish(struct dbox_sync_context **ctx, bool success);
-int dbox_sync(struct dbox_mailbox *mbox, bool close_flush_dirty_flags);
+int dbox_sync(struct dbox_mailbox *mbox);
int dbox_sync_file(struct dbox_sync_context *ctx,
const struct dbox_sync_file_entry *entry);
#define UIDLIST_IS_LOCKED(uidlist) \
((uidlist)->lock_count > 0)
+#define UIDLIST_ALLOW_WRITING(uidlist) \
+ (UIDLIST_IS_LOCKED(uidlist) || (uidlist)->mbox == NULL)
struct maildir_uidlist_rec {
uint32_t uid;
*sync_ctx_r = ctx = i_new(struct maildir_uidlist_sync_ctx, 1);
ctx->uidlist = uidlist;
ctx->sync_flags = sync_flags;
- ctx->partial = !locked ||
+ ctx->partial = (!locked && ctx->uidlist->mbox != NULL) ||
(sync_flags & MAILDIR_UIDLIST_SYNC_PARTIAL) != 0;
ctx->locked = locked;
ctx->first_unwritten_pos = (unsigned int)-1;
{
struct maildir_uidlist *uidlist = ctx->uidlist;
struct maildir_uidlist_rec *rec, *old_rec;
- const char *p;
+ const char *p, *dir;
if (ctx->failed)
return -1;
for (p = filename; *p != '\0'; p++) {
if (*p == 13 || *p == 10) {
+ struct mailbox *box = &uidlist->ibox->box;
+
+ dir = mailbox_list_get_path(box->storage->list,
+ box->name,
+ MAILBOX_LIST_PATH_TYPE_MAILBOX);
i_warning("Maildir %s: Ignoring a file with #0x%x: %s",
- uidlist->mbox->path, *p, filename);
+ dir, *p, filename);
return 1;
}
}
MAILDIR_UIDLIST_REC_FLAG_MOVED);
} else {
old_rec = hash_table_lookup(uidlist->files, filename);
- i_assert(old_rec != NULL || UIDLIST_IS_LOCKED(uidlist));
+ i_assert(old_rec != NULL || UIDLIST_ALLOW_WRITING(uidlist));
rec = p_new(ctx->record_pool, struct maildir_uidlist_rec, 1);
struct maildir_uidlist_rec **recs;
unsigned int dest, count;
- i_assert(UIDLIST_IS_LOCKED(ctx->uidlist));
+ i_assert(UIDLIST_ALLOW_WRITING(ctx->uidlist));
i_assert(ctx->first_nouid_pos != (unsigned int)-1);
if (ctx->first_unwritten_pos == (unsigned int)-1)
}
}
+void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx)
+{
+ ctx->uidlist->recreate = TRUE;
+}
+
void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx)
{
if (!ctx->partial) {
ctx->finished = TRUE;
ctx->uidlist->initial_sync = TRUE;
- i_assert(ctx->locked || !ctx->changed);
+ /* mbox=NULL means we're coming from dbox rebuilding code.
+ the dbox is already locked, so allow uidlist recreation */
+ i_assert(ctx->locked || !ctx->changed || ctx->uidlist->mbox == NULL);
if ((ctx->changed || ctx->uidlist->recreate ||
maildir_uidlist_want_compress(ctx)) &&
- !ctx->failed && ctx->locked) T_BEGIN {
- if (maildir_uidlist_sync_update(ctx) < 0)
- ctx->failed = TRUE;
- } T_END;
+ !ctx->failed && (ctx->locked || ctx->uidlist->mbox == NULL)) {
+ T_BEGIN {
+ if (maildir_uidlist_sync_update(ctx) < 0)
+ ctx->failed = TRUE;
+ } T_END;
+ }
}
int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **_ctx)
const char *
maildir_uidlist_sync_get_full_filename(struct maildir_uidlist_sync_ctx *ctx,
const char *filename);
+void maildir_uidlist_sync_recreate(struct maildir_uidlist_sync_ctx *ctx);
void maildir_uidlist_sync_finish(struct maildir_uidlist_sync_ctx *ctx);
int maildir_uidlist_sync_deinit(struct maildir_uidlist_sync_ctx **ctx);
projects / thirdparty / dovecot / core.git / commitdiff
