From: Timo Sirainen Date: Thu, 1 Jul 2010 16:02:20 +0000 (+0100) Subject: dsync: Added backup command, which syncs source to destination, discarding any change... X-Git-Tag: 2.0.rc1~42 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=360ee950ceacc749413300eb41306d5e593a2c07;p=thirdparty%2Fdovecot%2Fcore.git dsync: Added backup command, which syncs source to destination, discarding any changes in dest. It doesn't work perfectly in all situations. Especially if destination had saved/expunged mails in INBOX, dsync can't resolve it. For non-INBOXes it deletes the mailbox and fails a bit later, so that the next dsync can do a full resync for the mailbox. --HG-- branch : HEAD --- diff --git a/src/dsync/dsync-brain-msgs.c b/src/dsync/dsync-brain-msgs.c index b47bca2ef2..61247e55a2 100644 --- a/src/dsync/dsync-brain-msgs.c +++ b/src/dsync/dsync-brain-msgs.c @@ -77,7 +77,8 @@ static int dsync_brain_msg_iter_next(struct dsync_brain_msg_iter *iter) int ret = 1; if (iter->msg.guid == NULL) { - ret = dsync_worker_msg_iter_next(iter->iter, &iter->mailbox_idx, + ret = dsync_worker_msg_iter_next(iter->iter, + &iter->mailbox_idx, &iter->msg); if (ret > 0) dsync_brain_guid_add(iter); @@ -90,10 +91,32 @@ static int dsync_brain_msg_iter_next(struct dsync_brain_msg_iter *iter) return ret; } +static int +dsync_brain_msg_iter_skip_mailbox(struct dsync_brain_mailbox_sync *sync) +{ + int ret; + + while ((ret = dsync_brain_msg_iter_next(sync->src_msg_iter)) > 0) ; + if (ret == 0) + return 0; + + while ((ret = dsync_brain_msg_iter_next(sync->dest_msg_iter)) > 0) ; + if (ret == 0) + return 0; + + sync->skip_mailbox = FALSE; + return -1; +} + static int dsync_brain_msg_iter_next_pair(struct dsync_brain_mailbox_sync *sync) { int ret; + if (sync->skip_mailbox) { + if (dsync_brain_msg_iter_skip_mailbox(sync) == 0) + return 0; + } + if ((ret = dsync_brain_msg_iter_next(sync->src_msg_iter)) <= 0) return ret; if ((ret = dsync_brain_msg_iter_next(sync->dest_msg_iter)) <= 0) @@ -129,6 +152,18 @@ dsync_brain_msg_sync_conflict(struct dsync_brain_msg_iter *conflict_iter, brain_box = array_idx_modifiable(&save_iter->sync->mailboxes, save_iter->mailbox_idx); + + if (save_iter->sync->brain->backup) { + i_warning("Destination mailbox %s has been modified, " + "need to recreate it before we can continue syncing", + brain_box->box.name); + dsync_worker_delete_mailbox(save_iter->sync->brain->dest_worker, + &brain_box->box); + save_iter->sync->brain->unexpected_changes = TRUE; + save_iter->sync->skip_mailbox = TRUE; + return; + } + new_uid = brain_box->box.uid_next++; conflict = array_append_space(&conflict_iter->uid_conflicts); @@ -185,7 +220,7 @@ static void dsync_brain_msg_sync_existing(struct dsync_brain_mailbox_sync *sync, int ret; ret = dsync_message_flag_importance_cmp(src_msg, dest_msg); - if (ret < 0) + if (ret < 0 || (sync->brain->backup && ret > 0)) dsync_worker_msg_update_metadata(sync->dest_worker, src_msg); else if (ret > 0) dsync_worker_msg_update_metadata(sync->src_worker, dest_msg); @@ -230,7 +265,7 @@ static int dsync_brain_msg_sync_pair(struct dsync_brain_mailbox_sync *sync) /* message has been expunged from dest. */ if (src_expunged) { /* expunged from source already */ - } else if (sync->uid_conflict) { + } else if (sync->uid_conflict || sync->brain->backup) { /* update uid src, copy to dest */ dsync_brain_msg_sync_conflict(sync->src_msg_iter, sync->dest_msg_iter, @@ -246,7 +281,7 @@ static int dsync_brain_msg_sync_pair(struct dsync_brain_mailbox_sync *sync) /* message has been expunged from src. */ if (dest_expunged) { /* expunged from dest already */ - } else if (sync->uid_conflict) { + } else if (sync->uid_conflict && !sync->brain->backup) { /* update uid in dest, copy to src */ dsync_brain_msg_sync_conflict(sync->dest_msg_iter, sync->src_msg_iter, @@ -281,7 +316,13 @@ static int dsync_brain_msg_sync_pair(struct dsync_brain_mailbox_sync *sync) } } else if (dest_expunged) { /* message expunged from destination */ - if (!src_expunged) { + if (src_expunged) { + /* expunged from source already */ + } else if (sync->brain->backup) { + dsync_brain_msg_sync_conflict(sync->src_msg_iter, + sync->dest_msg_iter, + src_msg); + } else { dsync_worker_msg_expunge(sync->src_worker, src_msg->uid); } @@ -329,9 +370,11 @@ dsync_brain_msg_sync_mailbox_more(struct dsync_brain_mailbox_sync *sync) /* finished syncing messages in this mailbox that exist in both source and destination. if there are messages left, we can't reliably know if they should be expunged, so just copy them to the other side. */ - if (!dsync_brain_msg_sync_mailbox_end(sync->dest_msg_iter, - sync->src_msg_iter)) - return FALSE; + if (!sync->brain->backup) { + if (!dsync_brain_msg_sync_mailbox_end(sync->dest_msg_iter, + sync->src_msg_iter)) + return FALSE; + } if (!dsync_brain_msg_sync_mailbox_end(sync->src_msg_iter, sync->dest_msg_iter)) return FALSE; diff --git a/src/dsync/dsync-brain-private.h b/src/dsync/dsync-brain-private.h index 561b7aed8c..617db0d261 100644 --- a/src/dsync/dsync-brain-private.h +++ b/src/dsync/dsync-brain-private.h @@ -104,6 +104,7 @@ struct dsync_brain_mailbox_sync { struct dsync_brain_msg_iter *dest_msg_iter; unsigned int uid_conflict:1; + unsigned int skip_mailbox:1; }; struct dsync_brain { @@ -124,6 +125,8 @@ struct dsync_brain { unsigned int failed:1; unsigned int verbose:1; + unsigned int backup:1; + unsigned int unexpected_changes:1; }; void dsync_brain_fail(struct dsync_brain *brain); diff --git a/src/dsync/dsync-brain.c b/src/dsync/dsync-brain.c index b3c37b2a78..f453208da8 100644 --- a/src/dsync/dsync-brain.c +++ b/src/dsync/dsync-brain.c @@ -25,6 +25,7 @@ dsync_brain_init(struct dsync_worker *src_worker, brain->mailbox = i_strdup(mailbox); brain->flags = flags; brain->verbose = (flags & DSYNC_BRAIN_FLAG_VERBOSE) != 0; + brain->backup = (flags & DSYNC_BRAIN_FLAG_BACKUP) != 0; return brain; } @@ -235,20 +236,68 @@ dsync_brain_subs_list_deinit(struct dsync_brain_subs_list **_list) pool_unref(&list->pool); } +enum dsync_brain_mailbox_action { + DSYNC_BRAIN_MAILBOX_ACTION_NONE, + DSYNC_BRAIN_MAILBOX_ACTION_CREATE, + DSYNC_BRAIN_MAILBOX_ACTION_DELETE +}; + +static void +dsync_brain_mailbox_action(struct dsync_brain *brain, + enum dsync_brain_mailbox_action action, + struct dsync_worker *action_worker, + struct dsync_mailbox *action_box) +{ + struct dsync_mailbox new_box; + + if (brain->backup && action_worker == brain->src_worker) { + /* backup mode: switch actions */ + action_worker = brain->dest_worker; + switch (action) { + case DSYNC_BRAIN_MAILBOX_ACTION_NONE: + break; + case DSYNC_BRAIN_MAILBOX_ACTION_CREATE: + action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; + break; + case DSYNC_BRAIN_MAILBOX_ACTION_DELETE: + action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; + break; + } + } + + switch (action) { + case DSYNC_BRAIN_MAILBOX_ACTION_NONE: + break; + case DSYNC_BRAIN_MAILBOX_ACTION_CREATE: + new_box = *action_box; + new_box.uid_next = 0; + new_box.highest_modseq = 0; + dsync_worker_create_mailbox(action_worker, &new_box); + break; + case DSYNC_BRAIN_MAILBOX_ACTION_DELETE: + if (!dsync_mailbox_is_noselect(action_box)) + dsync_worker_delete_mailbox(action_worker, action_box); + else + dsync_worker_delete_dir(action_worker, action_box); + break; + } +} + static void dsync_brain_sync_mailboxes(struct dsync_brain *brain) { - struct dsync_mailbox *const *src_boxes, *const *dest_boxes, new_box; + struct dsync_mailbox *const *src_boxes, *const *dest_boxes, *action_box; unsigned int src, dest, src_count, dest_count; + enum dsync_brain_mailbox_action action; + struct dsync_worker *action_worker; bool src_deleted, dest_deleted; int ret; - memset(&new_box, 0, sizeof(new_box)); - /* create/delete missing mailboxes. the mailboxes are sorted by GUID, so we can do this quickly. */ src_boxes = array_get(&brain->src_mailbox_list->mailboxes, &src_count); dest_boxes = array_get(&brain->dest_mailbox_list->mailboxes, &dest_count); for (src = dest = 0; src < src_count && dest < dest_count; ) { + action = DSYNC_BRAIN_MAILBOX_ACTION_NONE; src_deleted = (src_boxes[src]->flags & DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0; dest_deleted = (dest_boxes[dest]->flags & @@ -258,74 +307,73 @@ static void dsync_brain_sync_mailboxes(struct dsync_brain *brain) if (ret < 0) { /* exists only in source */ if (!src_deleted) { - new_box = *src_boxes[src]; - new_box.uid_next = 0; - new_box.highest_modseq = 0; - dsync_worker_create_mailbox(brain->dest_worker, - &new_box); + action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; + action_worker = brain->dest_worker; + action_box = src_boxes[src]; } src++; } else if (ret > 0) { /* exists only in dest */ if (!dest_deleted) { - new_box = *dest_boxes[dest]; - new_box.uid_next = 0; - new_box.highest_modseq = 0; - dsync_worker_create_mailbox(brain->src_worker, - &new_box); + action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; + action_worker = brain->src_worker; + action_box = dest_boxes[dest]; } dest++; } else if (src_deleted) { /* delete from dest too */ if (!dest_deleted) { - dsync_worker_delete_mailbox(brain->dest_worker, - src_boxes[src]); + action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; + action_worker = brain->dest_worker; + action_box = dest_boxes[dest]; } src++; dest++; } else if (dest_deleted) { /* delete from src too */ - dsync_worker_delete_mailbox(brain->src_worker, - dest_boxes[dest]); + action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; + action_worker = brain->src_worker; + action_box = src_boxes[src]; src++; dest++; } else { src++; dest++; } + dsync_brain_mailbox_action(brain, action, + action_worker, action_box); } for (; src < src_count; src++) { if ((src_boxes[src]->flags & DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0) continue; - new_box = *src_boxes[src]; - new_box.uid_next = 0; - new_box.highest_modseq = 0; - dsync_worker_create_mailbox(brain->dest_worker, &new_box); + dsync_brain_mailbox_action(brain, + DSYNC_BRAIN_MAILBOX_ACTION_CREATE, + brain->dest_worker, src_boxes[src]); } for (; dest < dest_count; dest++) { if ((dest_boxes[dest]->flags & DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0) continue; - new_box = *dest_boxes[dest]; - new_box.uid_next = 0; - new_box.highest_modseq = 0; - dsync_worker_create_mailbox(brain->src_worker, &new_box); + dsync_brain_mailbox_action(brain, + DSYNC_BRAIN_MAILBOX_ACTION_CREATE, + brain->src_worker, dest_boxes[dest]); } } static void dsync_brain_sync_dirs(struct dsync_brain *brain) { - struct dsync_mailbox *const *src_boxes, *const *dest_boxes, new_box; + struct dsync_mailbox *const *src_boxes, *const *dest_boxes, *action_box; unsigned int src, dest, src_count, dest_count; + enum dsync_brain_mailbox_action action; + struct dsync_worker *action_worker; bool src_deleted, dest_deleted; int ret; - memset(&new_box, 0, sizeof(new_box)); - /* create/delete missing directories. */ src_boxes = array_get(&brain->src_mailbox_list->dirs, &src_count); dest_boxes = array_get(&brain->dest_mailbox_list->dirs, &dest_count); for (src = dest = 0; src < src_count && dest < dest_count; ) { + action = DSYNC_BRAIN_MAILBOX_ACTION_NONE; src_deleted = (src_boxes[src]->flags & DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0; dest_deleted = (dest_boxes[dest]->flags & @@ -336,50 +384,57 @@ static void dsync_brain_sync_dirs(struct dsync_brain *brain) if (ret < 0) { /* exists only in source */ if (!src_deleted) { - new_box = *src_boxes[src]; - dsync_worker_create_mailbox(brain->dest_worker, - &new_box); + action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; + action_worker = brain->dest_worker; + action_box = src_boxes[src]; } src++; } else if (ret > 0) { /* exists only in dest */ if (!dest_deleted) { - new_box = *dest_boxes[dest]; - dsync_worker_create_mailbox(brain->src_worker, - &new_box); + action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE; + action_worker = brain->src_worker; + action_box = dest_boxes[dest]; } dest++; } else if (src_deleted) { /* delete from dest too */ if (!dest_deleted) { - dsync_worker_delete_dir(brain->dest_worker, - dest_boxes[dest]); + action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; + action_worker = brain->dest_worker; + action_box = dest_boxes[dest]; } src++; dest++; } else if (dest_deleted) { /* delete from src too */ - dsync_worker_delete_dir(brain->src_worker, - src_boxes[src]); + action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE; + action_worker = brain->src_worker; + action_box = src_boxes[src]; src++; dest++; } else { src++; dest++; } + i_assert(dsync_mailbox_is_noselect(action_box)); + dsync_brain_mailbox_action(brain, action, + action_worker, action_box); } for (; src < src_count; src++) { if ((src_boxes[src]->flags & DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0) continue; - new_box = *src_boxes[src]; - dsync_worker_create_mailbox(brain->dest_worker, &new_box); + dsync_brain_mailbox_action(brain, + DSYNC_BRAIN_MAILBOX_ACTION_CREATE, + brain->dest_worker, src_boxes[src]); } for (; dest < dest_count; dest++) { if ((dest_boxes[dest]->flags & DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0) continue; - new_box = *dest_boxes[dest]; - dsync_worker_create_mailbox(brain->src_worker, &new_box); + dsync_brain_mailbox_action(brain, + DSYNC_BRAIN_MAILBOX_ACTION_CREATE, + brain->src_worker, dest_boxes[dest]); } } @@ -410,8 +465,11 @@ dsync_brain_is_unsubscribed(struct dsync_brain_subs_list *list, static void dsync_brain_sync_subscriptions(struct dsync_brain *brain) { const struct dsync_worker_subscription *src_subs, *dest_subs; + const struct dsync_worker_subscription *action_subs; + struct dsync_worker *action_worker; unsigned int src, dest, src_count, dest_count; time_t last_change; + bool subscribe; int ret; /* subscriptions are sorted by name. */ @@ -435,33 +493,40 @@ static void dsync_brain_sync_subscriptions(struct dsync_brain *brain) if (ret < 0) { /* subscribed only in source */ + action_subs = &src_subs[src]; if (dsync_brain_is_unsubscribed(brain->dest_subs_list, &src_subs[src], &last_change)) { - dsync_worker_set_subscribed(brain->src_worker, - src_subs[src].vname, - last_change, FALSE); + action_worker = brain->src_worker; + subscribe = FALSE; } else { - dsync_worker_set_subscribed(brain->dest_worker, - src_subs[src].vname, - last_change, TRUE); + action_worker = brain->dest_worker; + subscribe = TRUE; } src++; } else { /* subscribed only in dest */ + action_subs = &dest_subs[dest]; if (dsync_brain_is_unsubscribed(brain->src_subs_list, &dest_subs[dest], &last_change)) { - dsync_worker_set_subscribed(brain->dest_worker, - dest_subs[dest].vname, - last_change, FALSE); + action_worker = brain->dest_worker; + subscribe = FALSE; } else { - dsync_worker_set_subscribed(brain->src_worker, - dest_subs[dest].vname, - last_change, TRUE); + action_worker = brain->src_worker; + subscribe = TRUE; } dest++; } + + if (brain->backup && action_worker == brain->src_worker) { + /* backup mode: switch action */ + action_worker = brain->dest_worker; + subscribe = !subscribe; + last_change = ioloop_time; + } + dsync_worker_set_subscribed(action_worker, action_subs->vname, + last_change, subscribe); } } @@ -612,7 +677,8 @@ static void dsync_brain_sync_rename_mailbox(struct dsync_brain *brain, const struct dsync_brain_mailbox *mailbox) { - if (mailbox->src->last_change > mailbox->dest->last_change) { + if (mailbox->src->last_change > mailbox->dest->last_change || + brain->backup) { dsync_worker_rename_mailbox(brain->dest_worker, &mailbox->box.mailbox_guid, mailbox->src); @@ -629,7 +695,10 @@ dsync_brain_sync_update_mailboxes(struct dsync_brain *brain) const struct dsync_brain_mailbox *mailbox; array_foreach(&brain->mailbox_sync->mailboxes, mailbox) { - dsync_worker_update_mailbox(brain->src_worker, &mailbox->box); + if (!brain->backup) { + dsync_worker_update_mailbox(brain->src_worker, + &mailbox->box); + } dsync_worker_update_mailbox(brain->dest_worker, &mailbox->box); if (mailbox->src != NULL && mailbox->dest != NULL && @@ -735,6 +804,7 @@ void dsync_brain_sync_all(struct dsync_brain *brain) bool dsync_brain_has_unexpected_changes(struct dsync_brain *brain) { - return dsync_worker_has_unexpected_changes(brain->src_worker) || + return brain->unexpected_changes || + dsync_worker_has_unexpected_changes(brain->src_worker) || dsync_worker_has_unexpected_changes(brain->dest_worker); } diff --git a/src/dsync/dsync-brain.h b/src/dsync/dsync-brain.h index 0acbeff43e..0c6319b27c 100644 --- a/src/dsync/dsync-brain.h +++ b/src/dsync/dsync-brain.h @@ -3,7 +3,10 @@ enum dsync_brain_flags { DSYNC_BRAIN_FLAG_FULL_SYNC = 0x01, - DSYNC_BRAIN_FLAG_VERBOSE = 0x02 + DSYNC_BRAIN_FLAG_VERBOSE = 0x02, + /* Run in backup mode. All changes from src are forced into dest, + discarding any potential changes in dest. */ + DSYNC_BRAIN_FLAG_BACKUP = 0x04 }; struct dsync_worker; diff --git a/src/dsync/dsync.c b/src/dsync/dsync.c index 4fe18c5dc1..2b5bf37acc 100644 --- a/src/dsync/dsync.c +++ b/src/dsync/dsync.c @@ -195,10 +195,13 @@ int main(int argc, char *argv[]) cmd_name = argv[optind++]; if (strcmp(cmd_name, "mirror") == 0 || - strcmp(cmd_name, "convert") == 0) { + strcmp(cmd_name, "convert") == 0 || + strcmp(cmd_name, "backup") == 0) { if (optind == argc) usage(); + if (strcmp(cmd_name, "backup") == 0) + brain_flags |= DSYNC_BRAIN_FLAG_BACKUP; if (!mirror_get_remote_cmd(argv+optind, &remote_cmd_args)) { if (optind+1 != argc) usage();