unsigned int sync_appends:1;
};
-int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx);
+int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
+ uint32_t sync_stamp, uint64_t sync_size);
void mail_index_header_update_counts(struct mail_index_header *hdr,
uint8_t old_flags, uint8_t new_flags);
return 0;
}
-int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx)
+int mail_index_sync_update_index(struct mail_index_sync_ctx *sync_ctx,
+ uint32_t sync_stamp, uint64_t sync_size)
{
struct mail_index *index = sync_ctx->index;
struct mail_index_map *map = index->map;
uint32_t count, file_seq, src_idx, dest_idx;
uoff_t file_offset;
unsigned int lock_id;
- int ret;
+ int ret, changed;
/* rewind */
sync_ctx->update_idx = sync_ctx->expunge_idx = 0;
sync_ctx->sync_appends =
buffer_get_used_size(sync_ctx->appends_buf) != 0;
- if (!mail_index_sync_have_more(sync_ctx)) {
+ changed = mail_index_sync_have_more(sync_ctx);
+
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.index = index;
+ ctx.hdr = *index->hdr;
+ ctx.log_view = sync_ctx->view->log_view;
+
+ /* see if we need to update sync headers */
+ if (ctx.hdr.sync_stamp != sync_stamp && sync_stamp != 0) {
+ ctx.hdr.sync_stamp = sync_stamp;
+ changed = TRUE;
+ }
+ if (ctx.hdr.sync_size != sync_size && sync_size != 0) {
+ ctx.hdr.sync_size = sync_size;
+ changed = TRUE;
+ }
+
+ if (!changed) {
/* nothing to sync */
return 0;
}
if (MAIL_INDEX_MAP_IS_IN_MEMORY(map))
map->write_to_disk = TRUE;
- memset(&ctx, 0, sizeof(ctx));
- ctx.index = index;
- ctx.hdr = *index->hdr;
- ctx.log_view = sync_ctx->view->log_view;
-
src_idx = dest_idx = 0;
append_count = 0; appends = NULL;
while (mail_index_sync_next(sync_ctx, &rec) > 0) {
index->hdr->log_file_offset,
seq, offset,
MAIL_TRANSACTION_TYPE_MASK) < 0) {
- mail_index_sync_end(ctx);
+ mail_index_sync_end(ctx, 0, 0);
return -1;
}
ctx->appends_buf = buffer_create_dynamic(default_pool,
1024, (size_t)-1);
if (mail_index_sync_read_and_sort(ctx, FALSE) < 0) {
- mail_index_sync_end(ctx);
+ mail_index_sync_end(ctx, 0, 0);
return -1;
}
ctx->sync_appends;
}
-int mail_index_sync_end(struct mail_index_sync_ctx *ctx)
+int mail_index_sync_end(struct mail_index_sync_ctx *ctx,
+ uint32_t sync_stamp, uint64_t sync_size)
{
const struct mail_index_header *hdr;
uint32_t seq;
if (ret == 0) {
mail_index_sync_read_and_sort(ctx, TRUE);
- if (mail_index_sync_update_index(ctx) < 0)
+ if (mail_index_sync_update_index(ctx, sync_stamp,
+ sync_size) < 0)
ret = -1;
}
};
enum mail_index_header_flag {
- /* Corrupted-flag should be set while anything dangerous is done to
- index file, such as when expunging messages. Once the operation
- is finished, the corrupted-flag is removed. If this flag is noticed
- unexpectedly, the index must be assumed to be corrupted and must
- not be used. */
- MAIL_INDEX_HDR_FLAG_CORRUPTED = 0x0001,
- MAIL_INDEX_HDR_FLAG_COMPRESS_CACHE = 0x0002
+ /* Index file is corrupted, reopen or recreate it. */
+ MAIL_INDEX_HDR_FLAG_CORRUPTED = 0x0001
};
enum mail_index_record_flag {
struct mail_index_sync_rec *sync_rec);
/* Returns 1 if there's more to sync, 0 if not. */
int mail_index_sync_have_more(struct mail_index_sync_ctx *ctx);
-/* End synchronization by unlocking the index and closing the view. */
-int mail_index_sync_end(struct mail_index_sync_ctx *ctx);
+/* End synchronization by unlocking the index and closing the view.
+ sync_stamp/sync_size in header is updated to given values. */
+int mail_index_sync_end(struct mail_index_sync_ctx *ctx,
+ uint32_t sync_stamp, uint64_t sync_size);
/* Mark index file corrupted. Invalidates all views. */
void mail_index_mark_corrupted(struct mail_index *index);
unsigned int mail_read_mmaped:1;
unsigned int maildir_keep_new:1;
+ unsigned int last_cur_dirty:1;
};
struct index_transaction_context {
struct maildir_sync_context {
struct index_mailbox *ibox;
const char *new_dir, *cur_dir;
+ int partial;
struct maildir_uidlist_sync_ctx *uidlist_sync_ctx;
};
break;
}
}
- if (mail_index_sync_end(sync_ctx) < 0)
+ if (mail_index_sync_end(sync_ctx, 0, 0) < 0)
ret = -1;
}
static int maildir_sync_quick_check(struct maildir_sync_context *ctx,
int *new_changed_r, int *cur_changed_r)
{
+ const struct mail_index_header *hdr;
struct index_mailbox *ibox = ctx->ibox;
struct stat st;
time_t new_mtime, cur_mtime;
}
cur_mtime = st.st_mtime;
+ if (ibox->last_cur_mtime == 0) {
+ /* first sync in this session, get cur stamp from index */
+ if (mail_index_get_header(ibox->view, &hdr) == 0)
+ ibox->last_cur_mtime = hdr->sync_stamp;
+ }
+
if (new_mtime != ibox->last_new_mtime ||
new_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS) {
*new_changed_r = TRUE;
ibox->last_new_mtime = new_mtime;
}
+
if (cur_mtime != ibox->last_cur_mtime ||
- (cur_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS &&
+ (ibox->last_cur_dirty &&
ioloop_time - ibox->last_sync > MAILDIR_SYNC_SECS)) {
/* cur/ changed, or delayed cur/ check */
*cur_changed_r = TRUE;
ibox->last_cur_mtime = cur_mtime;
}
ibox->last_sync = ioloop_time;
+ ibox->last_cur_dirty = cur_mtime >= ibox->last_sync - MAILDIR_SYNC_SECS;
return 0;
}
struct mail_index_view *view;
const struct mail_index_header *hdr;
const struct mail_index_record *rec;
- uint32_t seq, uid, uflags;
+ uint32_t seq, uid;
+ enum maildir_uidlist_rec_flag uflags;
const char *filename;
enum mail_flags flags;
custom_flags_mask_t custom_flags;
+ uint32_t sync_stamp;
int ret;
if (mail_index_sync_begin(ibox->index, &sync_ctx, &view,
__again:
seq++;
+ if ((uflags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) != 0) {
+ /* partial syncing */
+ continue;
+ }
+
if (seq > hdr->messages_count) {
mail_index_append(trans, uid, &seq);
mail_index_update_flags(trans, seq, MODIFY_REPLACE,
}
maildir_uidlist_iter_deinit(iter);
+ if (!ctx->partial) {
+ /* expunge the rest */
+ for (seq++; seq <= hdr->messages_count; seq++)
+ mail_index_expunge(trans, seq);
+ }
+
if (ret < 0)
mail_index_transaction_rollback(trans);
else {
break;
}
}
- if (mail_index_sync_end(sync_ctx) < 0)
+
+ sync_stamp = ibox->last_cur_dirty ? 0 : ibox->last_cur_mtime;
+ if (mail_index_sync_end(sync_ctx, sync_stamp, 0) < 0)
ret = -1;
if (ret == 0) {
if (!new_changed && !cur_changed)
return 0;
- // FIXME: don't sync cur/ directory if not needed
- ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist);
+ ctx->partial = !cur_changed;
+ ctx->uidlist_sync_ctx =
+ maildir_uidlist_sync_init(ctx->ibox->uidlist, ctx->partial);
if (maildir_scan_dir(ctx, TRUE) < 0)
return -1;
- if (maildir_scan_dir(ctx, FALSE) < 0)
- return -1;
+ if (cur_changed) {
+ if (maildir_scan_dir(ctx, FALSE) < 0)
+ return -1;
+ }
ret = maildir_uidlist_sync_deinit(ctx->uidlist_sync_ctx);
ctx->uidlist_sync_ctx = NULL;
{
int ret;
- ctx->uidlist_sync_ctx = maildir_uidlist_sync_init(ctx->ibox->uidlist);
+ ctx->uidlist_sync_ctx =
+ maildir_uidlist_sync_init(ctx->ibox->uidlist, FALSE);
if (maildir_scan_dir(ctx, TRUE) < 0)
return -1;
struct hash_table *files;
buffer_t *new_record_buf;
+ unsigned int partial_new_pos;
+
+ unsigned int partial:1;
unsigned int new_files:1;
unsigned int synced:1;
unsigned int failed:1;
}
rec = buffer_append_space_unsafe(uidlist->record_buf, sizeof(*rec));
+ memset(rec, 0, sizeof(*rec));
rec->uid = uid;
rec->flags = flags;
rec->filename = p_strdup(uidlist->filename_pool, line);
unsigned int idx, left_idx, right_idx;
size_t size;
- i_assert(uidlist->last_mtime != 0);
+ if (uidlist->last_mtime == 0) {
+ /* first time we need to read uidlist */
+ if (maildir_uidlist_update(uidlist) < 0)
+ return NULL;
+ }
rec = buffer_get_data(uidlist->record_buf, &size);
size /= sizeof(*rec);
rec = buffer_get_data(uidlist->record_buf, &size);
size /= sizeof(*rec);
- maildir_uidlist_lookup(uidlist, uidlist->first_recent_uid, &idx);
+ maildir_uidlist_lookup_rec(uidlist, uidlist->first_recent_uid, &idx);
for (count = 0; idx < size; idx++) {
if ((rec[idx].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0)
count++;
struct maildir_uidlist_iter_ctx *iter;
struct utimbuf ut;
string_t *str;
- uint32_t uid, flags;
+ uint32_t uid;
+ enum maildir_uidlist_rec_flag flags;
const char *filename, *flags_str;
int ret = 0;
return ret;
}
+static void maildir_uidlist_mark_all(struct maildir_uidlist *uidlist,
+ int nonsynced)
+{
+ struct maildir_uidlist_rec *rec;
+ size_t i, size;
+
+ rec = buffer_get_modifyable_data(uidlist->record_buf, &size);
+ size /= sizeof(*rec);
+
+ if (nonsynced) {
+ for (i = 0; i < size; i++)
+ rec[i].flags |= MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
+ } else {
+ for (i = 0; i < size; i++)
+ rec[i].flags &= ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
+ }
+}
+
struct maildir_uidlist_sync_ctx *
-maildir_uidlist_sync_init(struct maildir_uidlist *uidlist)
+maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, int partial)
{
struct maildir_uidlist_sync_ctx *ctx;
ctx = i_new(struct maildir_uidlist_sync_ctx, 1);
ctx->uidlist = uidlist;
+ ctx->partial = partial;
+
+ if (partial) {
+ /* initially mark all nonsynced */
+ maildir_uidlist_mark_all(uidlist, TRUE);
+ return ctx;
+ }
+
ctx->filename_pool =
pool_alloconly_create("maildir_uidlist_sync", 16384);
ctx->new_record_buf =
buffer_create_dynamic(default_pool, 512, (size_t)-1);
ctx->files = hash_create(default_pool, ctx->filename_pool, 4096,
maildir_hash, maildir_cmp);
+ return ctx;
+}
- if (uidlist->last_mtime == 0) {
- /* uidlist not read yet, do it */
- if (maildir_uidlist_update(uidlist) < 0)
+static int maildir_uidlist_sync_uidlist(struct maildir_uidlist_sync_ctx *ctx)
+{
+ int ret;
+
+ if (ctx->uidlist->last_mtime == 0) {
+ /* first time reading the uidlist,
+ no locking yet */
+ if (maildir_uidlist_update(ctx->uidlist) < 0) {
ctx->failed = TRUE;
+ return -1;
+ }
+ return 0;
}
- return ctx;
+
+ /* lock and update uidlist to see if it's just been added */
+ ret = maildir_uidlist_try_lock(ctx->uidlist);
+ if (ret <= 0) {
+ if (ret == 0)
+ return 1; // FIXME: does it work right?
+ ctx->failed = TRUE;
+ return -1;
+ }
+ if (maildir_uidlist_update(ctx->uidlist) < 0) {
+ ctx->failed = TRUE;
+ return -1;
+ }
+
+ ctx->synced = TRUE;
+ return 1;
+}
+
+static int
+maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx,
+ const char *filename,
+ enum maildir_uidlist_rec_flag flags)
+{
+ struct maildir_uidlist *uidlist = ctx->uidlist;
+ struct maildir_uidlist_rec *rec;
+ int ret;
+
+ /* we'll update uidlist directly */
+ rec = hash_lookup(uidlist->files, filename);
+ if (rec == NULL && !ctx->synced) {
+ ret = maildir_uidlist_sync_uidlist(ctx);
+ if (ret < 0)
+ return -1;
+ if (ret == 0) {
+ return maildir_uidlist_sync_next_partial(ctx, filename,
+ flags);
+ }
+ rec = hash_lookup(uidlist->files, filename);
+ }
+
+ if (rec == NULL) {
+ ctx->new_files = TRUE;
+ ctx->partial_new_pos =
+ buffer_get_used_size(uidlist->record_buf) /
+ sizeof(*rec);
+ rec = buffer_append_space_unsafe(uidlist->record_buf,
+ sizeof(*rec));
+ memset(rec, 0, sizeof(*rec));
+ }
+
+ rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
+ rec->filename = p_strdup(uidlist->filename_pool, filename);
+ hash_insert(uidlist->files, rec->filename, rec);
+ return 1;
}
int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
enum maildir_uidlist_rec_flag flags)
{
struct maildir_uidlist_rec *rec;
- char *fname;
int ret;
if (ctx->failed)
return -1;
+ if (ctx->partial)
+ return maildir_uidlist_sync_next_partial(ctx, filename, flags);
+
rec = hash_lookup(ctx->files, filename);
if (rec != NULL) {
if ((rec->flags & (MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
} else {
rec = hash_lookup(ctx->uidlist->files, filename);
if (rec == NULL && !ctx->synced) {
- /* lock and update uidlist to see if it's just
- been added */
- ret = maildir_uidlist_try_lock(ctx->uidlist);
- if (ret <= 0) {
- if (ret == 0)
- return 1; // FIXME: does it work right?
- ctx->failed = TRUE;
- return -1;
- }
- if (maildir_uidlist_update(ctx->uidlist) < 0) {
- ctx->failed = TRUE;
+ ret = maildir_uidlist_sync_uidlist(ctx);
+ if (ret < 0)
return -1;
+ if (ret == 0) {
+ return maildir_uidlist_sync_next(ctx, filename,
+ flags);
}
-
- ctx->synced = TRUE;
rec = hash_lookup(ctx->uidlist->files, filename);
}
}
rec->flags |= flags;
-
- fname = p_strdup(ctx->filename_pool, filename);
- if (rec->filename == NULL)
- rec->filename = fname;
- hash_insert(ctx->files, fname, rec);
+ rec->filename = p_strdup(ctx->filename_pool, filename);
+ hash_insert(ctx->files, rec->filename, rec);
return 1;
}
return t1 < t2 ? -1 : t1 > t2 ? 1 : 0;
}
+static void maildir_uidlist_assign_uids(struct maildir_uidlist *uidlist,
+ unsigned int first_new_pos)
+{
+ struct maildir_uidlist_rec *rec;
+ unsigned int dest;
+ size_t size;
+
+ rec = buffer_get_modifyable_data(uidlist->record_buf, &size);
+ size /= sizeof(*rec);
+
+ /* sort new files and assign UIDs for them */
+ qsort(rec + first_new_pos, size - first_new_pos,
+ sizeof(*rec), maildir_time_cmp);
+ for (dest = first_new_pos; dest < size; dest++) {
+ i_assert(rec[dest].uid == 0);
+ rec[dest].uid = uidlist->next_uid++;
+ rec[dest].flags &= ~MAILDIR_UIDLIST_REC_FLAG_MOVED;
+
+ if ((rec[dest].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0) {
+ maildir_uidlist_mark_recent(uidlist,
+ rec[dest].uid);
+ }
+ }
+}
+
static void maildir_uidlist_swap(struct maildir_uidlist_sync_ctx *ctx)
{
struct maildir_uidlist *uidlist = ctx->uidlist;
buffer_append_buf(uidlist->record_buf, ctx->new_record_buf,
0, (size_t)-1);
-
- rec = buffer_get_modifyable_data(uidlist->record_buf, &size);
- size /= sizeof(*rec);
-
- /* sort new files and assign UIDs for them */
- qsort(rec + dest, size - dest, sizeof(*rec), maildir_time_cmp);
- for (; dest < size; dest++) {
- rec[dest].uid = uidlist->next_uid++;
- rec[dest].flags &= ~MAILDIR_UIDLIST_REC_FLAG_MOVED;
-
- if ((rec[dest].flags & MAILDIR_UIDLIST_REC_FLAG_RECENT) != 0) {
- maildir_uidlist_mark_recent(ctx->uidlist,
- rec[dest].uid);
- }
- }
+ maildir_uidlist_assign_uids(uidlist, dest);
hash_destroy(uidlist->files);
uidlist->files = ctx->files;
if (ctx->failed)
ret = -1;
else {
- maildir_uidlist_swap(ctx);
+ if (!ctx->partial)
+ maildir_uidlist_swap(ctx);
+ else {
+ if (ctx->new_files) {
+ maildir_uidlist_assign_uids(ctx->uidlist,
+ ctx->partial_new_pos);
+ }
+ maildir_uidlist_mark_all(ctx->uidlist, FALSE);
+ }
+
if (!ctx->new_files)
ret = 0;
else
hash_destroy(ctx->files);
if (ctx->filename_pool != NULL)
pool_unref(ctx->filename_pool);
- buffer_free(ctx->new_record_buf);
+ if (ctx->new_record_buf != NULL)
+ buffer_free(ctx->new_record_buf);
i_free(ctx);
return ret;
}
}
int maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx,
- uint32_t *uid_r, uint32_t *flags_r,
+ uint32_t *uid_r,
+ enum maildir_uidlist_rec_flag *flags_r,
const char **filename_r)
{
if (ctx->next == ctx->end)
enum maildir_uidlist_rec_flag {
MAILDIR_UIDLIST_REC_FLAG_NEW_DIR = 0x01,
MAILDIR_UIDLIST_REC_FLAG_MOVED = 0x02,
- MAILDIR_UIDLIST_REC_FLAG_RECENT = 0x04
+ MAILDIR_UIDLIST_REC_FLAG_RECENT = 0x04,
+ MAILDIR_UIDLIST_REC_FLAG_NONSYNCED = 0x08
};
int maildir_uidlist_try_lock(struct maildir_uidlist *uidlist);
/* Sync uidlist with what's actually on maildir. */
struct maildir_uidlist_sync_ctx *
-maildir_uidlist_sync_init(struct maildir_uidlist *uidlist);
+maildir_uidlist_sync_init(struct maildir_uidlist *uidlist, int partial);
int maildir_uidlist_sync_next(struct maildir_uidlist_sync_ctx *ctx,
const char *filename,
enum maildir_uidlist_rec_flag flags);
struct maildir_uidlist_iter_ctx *
maildir_uidlist_iter_init(struct maildir_uidlist *uidlist);
int maildir_uidlist_iter_next(struct maildir_uidlist_iter_ctx *ctx,
- uint32_t *uid_r, uint32_t *flags_r,
+ uint32_t *uid_r,
+ enum maildir_uidlist_rec_flag *flags_r,
const char **filename_r);
void maildir_uidlist_iter_deinit(struct maildir_uidlist_iter_ctx *ctx);
projects / thirdparty / dovecot / core.git / commitdiff
