From: Timo Sirainen Date: Mon, 28 Apr 2014 17:14:03 +0000 (+0300) Subject: dsync: If incremental dsync finds that its state is stale, retry (later) with full... X-Git-Tag: 2.2.13.rc1~80 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=40440c0fee87be994ba7eb60fc3512a9355708aa;p=thirdparty%2Fdovecot%2Fcore.git dsync: If incremental dsync finds that its state is stale, retry (later) with full sync. This works only when the master itself notices such changes locally. Remote changes aren't noticed, because the master doesn't send the state to remote dsync and the master is also the one that decides which mailboxes are synced. The fix to this probably means sending the state string to remote dsync and have it check if the state is still valid (= protocol change). --- diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c index 055dacf6a4..f34ace7ad1 100644 --- a/src/doveadm/dsync/dsync-brain-mailbox.c +++ b/src/doveadm/dsync/dsync-brain-mailbox.c @@ -196,9 +196,11 @@ dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, const struct dsync_mailbox *remote_dsync_box) { + struct mailbox_status status; enum dsync_mailbox_exporter_flags exporter_flags = 0; uint32_t last_common_uid, highest_wanted_uid; uint64_t last_common_modseq, last_common_pvt_modseq; + int ret; i_assert(brain->log_scan == NULL); i_assert(brain->box_exporter == NULL); @@ -208,18 +210,33 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq; highest_wanted_uid = last_common_uid == 0 ? (uint32_t)-1 : last_common_uid; - if (dsync_transaction_log_scan_init(brain->box->view, - brain->box->view_pvt, - highest_wanted_uid, - last_common_modseq, - last_common_pvt_modseq, - &brain->log_scan) < 0) { + ret = dsync_transaction_log_scan_init(brain->box->view, + brain->box->view_pvt, + highest_wanted_uid, + last_common_modseq, + last_common_pvt_modseq, + &brain->log_scan); + if (ret < 0) { i_error("Failed to read transaction log for mailbox %s", mailbox_get_vname(brain->box)); brain->failed = TRUE; return -1; } + if (last_common_uid != 0) { + mailbox_get_open_status(brain->box, STATUS_UIDNEXT | + STATUS_HIGHESTMODSEQ | + STATUS_HIGHESTPVTMODSEQ, &status); + if (status.uidnext < last_common_uid || + status.highest_modseq < last_common_modseq || + status.highest_pvt_modseq < last_common_pvt_modseq) { + /* last_common_* is higher than our current ones. + incremental sync state is stale, we need to do + a full resync */ + ret = 0; + } + } + if (!brain->mail_requests) exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS; if (remote_dsync_box->have_save_guids && @@ -232,13 +249,25 @@ int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, last_common_uid, exporter_flags); dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box); - return 0; + if (ret == 0) { + i_warning("Failed to do incremental sync for mailbox %s, " + "retry with a full sync", + mailbox_get_vname(brain->box)); + brain->changes_during_sync = TRUE; + brain->require_full_resync = TRUE; + return 0; + } + return 1; } void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain) { i_assert(brain->box != NULL); + if (brain->require_full_resync) { + brain->mailbox_state.last_uidvalidity = 0; + brain->require_full_resync = FALSE; + } array_append(&brain->remote_mailbox_states, &brain->mailbox_state, 1); if (brain->box_exporter != NULL) { const char *error; @@ -544,13 +573,14 @@ dsync_cache_fields_update(const struct dsync_mailbox *local_box, } } -void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, +bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain, struct mailbox *box, const struct dsync_mailbox *local_box, const struct dsync_mailbox *remote_box) { struct mailbox_update update; const struct dsync_mailbox_state *state; + bool ret = TRUE; memset(&update, 0, sizeof(update)); @@ -568,7 +598,7 @@ void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, session, because the other side already started sending mailbox changes, but not for all mails. */ dsync_mailbox_state_remove(brain, local_box->mailbox_guid); - // FIXME: handle this properly + ret = FALSE; } } @@ -577,7 +607,7 @@ void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, if (update.uid_validity == 0 && update.cache_updates == NULL) { /* no changes */ - return; + return ret; } if (mailbox_update(box, &update) < 0) { @@ -586,6 +616,7 @@ void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, mailbox_get_last_error(box, NULL)); brain->failed = TRUE; } + return ret; } static void @@ -614,6 +645,7 @@ bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain) struct mailbox *box; const char *error; int ret; + bool resync; i_assert(!brain->master_brain); i_assert(brain->box == NULL); diff --git a/src/doveadm/dsync/dsync-brain-mails.c b/src/doveadm/dsync/dsync-brain-mails.c index b2ea9a0c37..5de9ee0d79 100644 --- a/src/doveadm/dsync/dsync-brain-mails.c +++ b/src/doveadm/dsync/dsync-brain-mails.c @@ -22,6 +22,7 @@ static bool dsync_brain_master_sync_recv_mailbox(struct dsync_brain *brain) { const struct dsync_mailbox *dsync_box; enum dsync_ibc_recv_ret ret; + bool resync; i_assert(brain->master_brain); @@ -42,19 +43,26 @@ static bool dsync_brain_master_sync_recv_mailbox(struct dsync_brain *brain) if (dsync_box->mailbox_lost) { /* remote lost the mailbox. it's probably already deleted, but verify it on next sync just to be sure */ + brain->changes_during_sync = TRUE; + brain->require_full_resync = TRUE; dsync_brain_sync_mailbox_deinit(brain); return TRUE; } - dsync_brain_mailbox_update_pre(brain, brain->box, - &brain->local_dsync_box, dsync_box); + resync = !dsync_brain_mailbox_update_pre(brain, brain->box, + &brain->local_dsync_box, + dsync_box); if (!dsync_boxes_need_sync(brain, &brain->local_dsync_box, dsync_box)) { /* no fields appear to have changed, skip this mailbox */ dsync_brain_sync_mailbox_deinit(brain); return TRUE; } - if (dsync_brain_sync_mailbox_open(brain, dsync_box) < 0) + if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0) return TRUE; + if (ret == 0 || resync) { + brain->changes_during_sync = TRUE; + brain->require_full_resync = TRUE; + } dsync_brain_sync_init_box_states(brain); return TRUE; } diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h index 3d90785b4d..f8c445f569 100644 --- a/src/doveadm/dsync/dsync-brain-private.h +++ b/src/doveadm/dsync/dsync-brain-private.h @@ -99,6 +99,7 @@ struct dsync_brain { unsigned int no_mail_sync:1; unsigned int no_backup_overwrite:1; unsigned int changes_during_sync:1; + unsigned int require_full_resync:1; unsigned int verbose_proctitle:1; unsigned int failed:1; }; @@ -116,7 +117,7 @@ int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain, void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain); int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, struct mailbox **box_r, const char **error_r); -void dsync_brain_mailbox_update_pre(struct dsync_brain *brain, +bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain, struct mailbox *box, const struct dsync_mailbox *local_box, const struct dsync_mailbox *remote_box); diff --git a/src/doveadm/dsync/dsync-transaction-log-scan.c b/src/doveadm/dsync/dsync-transaction-log-scan.c index 3d9ee6cd9b..bf766afd1d 100644 --- a/src/doveadm/dsync/dsync-transaction-log-scan.c +++ b/src/doveadm/dsync/dsync-transaction-log-scan.c @@ -399,7 +399,13 @@ dsync_log_set(struct dsync_transaction_log_scan *ctx, view->index->filepath, log_seq, end_seq); ret = -1; } - return ret < 0 ? -1 : 0; + if (ret < 0) + return -1; + if (modseq != 0) { + /* we didn't see all the changes that we wanted to */ + return 0; + } + return 1; } static int @@ -412,9 +418,10 @@ dsync_log_scan(struct dsync_transaction_log_scan *ctx, uint32_t file_seq, max_seq; uoff_t file_offset, max_offset; uint64_t cur_modseq; + int ret; log_view = mail_transaction_log_view_open(view->index->log); - if (dsync_log_set(ctx, view, pvt_scan, log_view, modseq) < 0) { + if ((ret = dsync_log_set(ctx, view, pvt_scan, log_view, modseq)) < 0) { mail_transaction_log_view_close(&log_view); return -1; } @@ -475,7 +482,7 @@ dsync_log_scan(struct dsync_transaction_log_scan *ctx, ctx->last_log_offset = file_offset; } mail_transaction_log_view_close(&log_view); - return 0; + return ret; } static int @@ -503,6 +510,7 @@ int dsync_transaction_log_scan_init(struct mail_index_view *view, { struct dsync_transaction_log_scan *ctx; pool_t pool; + int ret, ret2; pool = pool_alloconly_create(MEMPOOL_GROWING"dsync transaction log scan", 10240); @@ -515,15 +523,17 @@ int dsync_transaction_log_scan_init(struct mail_index_view *view, ctx->view = view; ctx->highest_wanted_uid = highest_wanted_uid; - if (dsync_log_scan(ctx, view, modseq, FALSE) < 0) + if ((ret = dsync_log_scan(ctx, view, modseq, FALSE)) < 0) return -1; if (pvt_view != NULL) { - if (dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE) < 0) + if ((ret2 = dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE)) < 0) return -1; + if (ret2 == 0) + ret = 0; } *scan_r = ctx; - return 0; + return ret; } HASH_TABLE_TYPE(dsync_uid_mail_change)