dsync: Added backup command, which syncs source to destination, discarding any change...

It doesn't work perfectly in all situations. Especially if destination had
saved/expunged mails in INBOX, dsync can't resolve it. For non-INBOXes it
deletes the mailbox and fails a bit later, so that the next dsync can do a
full resync for the mailbox.

--HG--
branch : HEAD

diff --git a/src/dsync/dsync-brain-msgs.c b/src/dsync/dsync-brain-msgs.c
index b47bca2ef26adc21f6fbd28de66e3302bf1d0240..61247e55a264c3e54a7df8ebacada899e2f9d698 100644 (file)
--- a/src/dsync/dsync-brain-msgs.c
+++ b/src/dsync/dsync-brain-msgs.c
@@ -77,7 +77,8 @@ static int dsync_brain_msg_iter_next(struct dsync_brain_msg_iter *iter)
        int ret = 1;
 
        if (iter->msg.guid == NULL) {
-               ret = dsync_worker_msg_iter_next(iter->iter, &iter->mailbox_idx,
+               ret = dsync_worker_msg_iter_next(iter->iter,
+                                                &iter->mailbox_idx,
                                                 &iter->msg);
                if (ret > 0)
                        dsync_brain_guid_add(iter);
@@ -90,10 +91,32 @@ static int dsync_brain_msg_iter_next(struct dsync_brain_msg_iter *iter)
        return ret;
 }
 
+static int
+dsync_brain_msg_iter_skip_mailbox(struct dsync_brain_mailbox_sync *sync)
+{
+       int ret;
+
+       while ((ret = dsync_brain_msg_iter_next(sync->src_msg_iter)) > 0) ;
+       if (ret == 0)
+               return 0;
+
+       while ((ret = dsync_brain_msg_iter_next(sync->dest_msg_iter)) > 0) ;
+       if (ret == 0)
+               return 0;
+
+       sync->skip_mailbox = FALSE;
+       return -1;
+}
+
 static int dsync_brain_msg_iter_next_pair(struct dsync_brain_mailbox_sync *sync)
 {
        int ret;
 
+       if (sync->skip_mailbox) {
+               if (dsync_brain_msg_iter_skip_mailbox(sync) == 0)
+                       return 0;
+       }
+
        if ((ret = dsync_brain_msg_iter_next(sync->src_msg_iter)) <= 0)
                return ret;
        if ((ret = dsync_brain_msg_iter_next(sync->dest_msg_iter)) <= 0)
@@ -129,6 +152,18 @@ dsync_brain_msg_sync_conflict(struct dsync_brain_msg_iter *conflict_iter,
 
        brain_box = array_idx_modifiable(&save_iter->sync->mailboxes,
                                         save_iter->mailbox_idx);
+
+       if (save_iter->sync->brain->backup) {
+               i_warning("Destination mailbox %s has been modified, "
+                         "need to recreate it before we can continue syncing",
+                         brain_box->box.name);
+               dsync_worker_delete_mailbox(save_iter->sync->brain->dest_worker,
+                                           &brain_box->box);
+               save_iter->sync->brain->unexpected_changes = TRUE;
+               save_iter->sync->skip_mailbox = TRUE;
+               return;
+       }
+
        new_uid = brain_box->box.uid_next++;
 
        conflict = array_append_space(&conflict_iter->uid_conflicts);
@@ -185,7 +220,7 @@ static void dsync_brain_msg_sync_existing(struct dsync_brain_mailbox_sync *sync,
        int ret;
 
        ret = dsync_message_flag_importance_cmp(src_msg, dest_msg);
-       if (ret < 0)
+       if (ret < 0 || (sync->brain->backup && ret > 0))
                dsync_worker_msg_update_metadata(sync->dest_worker, src_msg);
        else if (ret > 0)
                dsync_worker_msg_update_metadata(sync->src_worker, dest_msg);
@@ -230,7 +265,7 @@ static int dsync_brain_msg_sync_pair(struct dsync_brain_mailbox_sync *sync)
                /* message has been expunged from dest. */
                if (src_expunged) {
                        /* expunged from source already */
-               } else if (sync->uid_conflict) {
+               } else if (sync->uid_conflict || sync->brain->backup) {
                        /* update uid src, copy to dest */
                        dsync_brain_msg_sync_conflict(sync->src_msg_iter,
                                                      sync->dest_msg_iter,
@@ -246,7 +281,7 @@ static int dsync_brain_msg_sync_pair(struct dsync_brain_mailbox_sync *sync)
                /* message has been expunged from src. */
                if (dest_expunged) {
                        /* expunged from dest already */
-               } else if (sync->uid_conflict) {
+               } else if (sync->uid_conflict && !sync->brain->backup) {
                        /* update uid in dest, copy to src */
                        dsync_brain_msg_sync_conflict(sync->dest_msg_iter,
                                                      sync->src_msg_iter,
@@ -281,7 +316,13 @@ static int dsync_brain_msg_sync_pair(struct dsync_brain_mailbox_sync *sync)
                }
        } else if (dest_expunged) {
                /* message expunged from destination */
-               if (!src_expunged) {
+               if (src_expunged) {
+                       /* expunged from source already */
+               } else if (sync->brain->backup) {
+                       dsync_brain_msg_sync_conflict(sync->src_msg_iter,
+                                                     sync->dest_msg_iter,
+                                                     src_msg);
+               } else {
                        dsync_worker_msg_expunge(sync->src_worker,
                                                 src_msg->uid);
                }
@@ -329,9 +370,11 @@ dsync_brain_msg_sync_mailbox_more(struct dsync_brain_mailbox_sync *sync)
        /* finished syncing messages in this mailbox that exist in both source
           and destination. if there are messages left, we can't reliably know
           if they should be expunged, so just copy them to the other side. */
-       if (!dsync_brain_msg_sync_mailbox_end(sync->dest_msg_iter,
-                                             sync->src_msg_iter))
-               return FALSE;
+       if (!sync->brain->backup) {
+               if (!dsync_brain_msg_sync_mailbox_end(sync->dest_msg_iter,
+                                                     sync->src_msg_iter))
+                       return FALSE;
+       }
        if (!dsync_brain_msg_sync_mailbox_end(sync->src_msg_iter,
                                              sync->dest_msg_iter))
                return FALSE;

diff --git a/src/dsync/dsync-brain-private.h b/src/dsync/dsync-brain-private.h
index 561b7aed8ce8ed948f9637f17e7f0b04a9c0baba..617db0d2612942587177c019bb77ce6963c9fcf9 100644 (file)
--- a/src/dsync/dsync-brain-private.h
+++ b/src/dsync/dsync-brain-private.h
@@ -104,6 +104,7 @@ struct dsync_brain_mailbox_sync {
        struct dsync_brain_msg_iter *dest_msg_iter;
 
        unsigned int uid_conflict:1;
+       unsigned int skip_mailbox:1;
 };
 
 struct dsync_brain {
@@ -124,6 +125,8 @@ struct dsync_brain {
 
        unsigned int failed:1;
        unsigned int verbose:1;
+       unsigned int backup:1;
+       unsigned int unexpected_changes:1;
 };
 
 void dsync_brain_fail(struct dsync_brain *brain);

diff --git a/src/dsync/dsync-brain.c b/src/dsync/dsync-brain.c
index b3c37b2a78ea454fa90e85d6f28cda2e979f2b5b..f453208da8c92da8d1979e9ba92cbb655ed8b043 100644 (file)
--- a/src/dsync/dsync-brain.c
+++ b/src/dsync/dsync-brain.c
@@ -25,6 +25,7 @@ dsync_brain_init(struct dsync_worker *src_worker,
        brain->mailbox = i_strdup(mailbox);
        brain->flags = flags;
        brain->verbose = (flags & DSYNC_BRAIN_FLAG_VERBOSE) != 0;
+       brain->backup = (flags & DSYNC_BRAIN_FLAG_BACKUP) != 0;
        return brain;
 }
 
@@ -235,20 +236,68 @@ dsync_brain_subs_list_deinit(struct dsync_brain_subs_list **_list)
        pool_unref(&list->pool);
 }
 
+enum dsync_brain_mailbox_action {
+       DSYNC_BRAIN_MAILBOX_ACTION_NONE,
+       DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+       DSYNC_BRAIN_MAILBOX_ACTION_DELETE
+};
+
+static void
+dsync_brain_mailbox_action(struct dsync_brain *brain,
+                          enum dsync_brain_mailbox_action action,
+                          struct dsync_worker *action_worker,
+                          struct dsync_mailbox *action_box)
+{
+       struct dsync_mailbox new_box;
+
+       if (brain->backup && action_worker == brain->src_worker) {
+               /* backup mode: switch actions */
+               action_worker = brain->dest_worker;
+               switch (action) {
+               case DSYNC_BRAIN_MAILBOX_ACTION_NONE:
+                       break;
+               case DSYNC_BRAIN_MAILBOX_ACTION_CREATE:
+                       action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+                       break;
+               case DSYNC_BRAIN_MAILBOX_ACTION_DELETE:
+                       action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+                       break;
+               }
+       }
+
+       switch (action) {
+       case DSYNC_BRAIN_MAILBOX_ACTION_NONE:
+               break;
+       case DSYNC_BRAIN_MAILBOX_ACTION_CREATE:
+               new_box = *action_box;
+               new_box.uid_next = 0;
+               new_box.highest_modseq = 0;
+               dsync_worker_create_mailbox(action_worker, &new_box);
+               break;
+       case DSYNC_BRAIN_MAILBOX_ACTION_DELETE:
+               if (!dsync_mailbox_is_noselect(action_box))
+                       dsync_worker_delete_mailbox(action_worker, action_box);
+               else
+                       dsync_worker_delete_dir(action_worker, action_box);
+               break;
+       }
+}
+
 static void dsync_brain_sync_mailboxes(struct dsync_brain *brain)
 {
-       struct dsync_mailbox *const *src_boxes, *const *dest_boxes, new_box;
+       struct dsync_mailbox *const *src_boxes, *const *dest_boxes, *action_box;
        unsigned int src, dest, src_count, dest_count;
+       enum dsync_brain_mailbox_action action;
+       struct dsync_worker *action_worker;
        bool src_deleted, dest_deleted;
        int ret;
 
-       memset(&new_box, 0, sizeof(new_box));
-
        /* create/delete missing mailboxes. the mailboxes are sorted by
           GUID, so we can do this quickly. */
        src_boxes = array_get(&brain->src_mailbox_list->mailboxes, &src_count);
        dest_boxes = array_get(&brain->dest_mailbox_list->mailboxes, &dest_count);
        for (src = dest = 0; src < src_count && dest < dest_count; ) {
+               action = DSYNC_BRAIN_MAILBOX_ACTION_NONE;
                src_deleted = (src_boxes[src]->flags &
                               DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0;
                dest_deleted = (dest_boxes[dest]->flags &
@@ -258,74 +307,73 @@ static void dsync_brain_sync_mailboxes(struct dsync_brain *brain)
                if (ret < 0) {
                        /* exists only in source */
                        if (!src_deleted) {
-                               new_box = *src_boxes[src];
-                               new_box.uid_next = 0;
-                               new_box.highest_modseq = 0;
-                               dsync_worker_create_mailbox(brain->dest_worker,
-                                                           &new_box);
+                               action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+                               action_worker = brain->dest_worker;
+                               action_box = src_boxes[src];
                        }
                        src++;
                } else if (ret > 0) {
                        /* exists only in dest */
                        if (!dest_deleted) {
-                               new_box = *dest_boxes[dest];
-                               new_box.uid_next = 0;
-                               new_box.highest_modseq = 0;
-                               dsync_worker_create_mailbox(brain->src_worker,
-                                                           &new_box);
+                               action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+                               action_worker = brain->src_worker;
+                               action_box = dest_boxes[dest];
                        }
                        dest++;
                } else if (src_deleted) {
                        /* delete from dest too */
                        if (!dest_deleted) {
-                               dsync_worker_delete_mailbox(brain->dest_worker,
-                                                           src_boxes[src]);
+                               action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+                               action_worker = brain->dest_worker;
+                               action_box = dest_boxes[dest];
                        }
                        src++; dest++;
                } else if (dest_deleted) {
                        /* delete from src too */
-                       dsync_worker_delete_mailbox(brain->src_worker,
-                                                   dest_boxes[dest]);
+                       action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+                       action_worker = brain->src_worker;
+                       action_box = src_boxes[src];
                        src++; dest++;
                } else {
                        src++; dest++;
                }
+               dsync_brain_mailbox_action(brain, action,
+                                          action_worker, action_box);
        }
        for (; src < src_count; src++) {
                if ((src_boxes[src]->flags &
                     DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0)
                        continue;
 
-               new_box = *src_boxes[src];
-               new_box.uid_next = 0;
-               new_box.highest_modseq = 0;
-               dsync_worker_create_mailbox(brain->dest_worker, &new_box);
+               dsync_brain_mailbox_action(brain,
+                       DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+                       brain->dest_worker, src_boxes[src]);
        }
        for (; dest < dest_count; dest++) {
                if ((dest_boxes[dest]->flags &
                     DSYNC_MAILBOX_FLAG_DELETED_MAILBOX) != 0)
                        continue;
 
-               new_box = *dest_boxes[dest];
-               new_box.uid_next = 0;
-               new_box.highest_modseq = 0;
-               dsync_worker_create_mailbox(brain->src_worker, &new_box);
+               dsync_brain_mailbox_action(brain,
+                       DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+                       brain->src_worker, dest_boxes[dest]);
        }
 }
 
 static void dsync_brain_sync_dirs(struct dsync_brain *brain)
 {
-       struct dsync_mailbox *const *src_boxes, *const *dest_boxes, new_box;
+       struct dsync_mailbox *const *src_boxes, *const *dest_boxes, *action_box;
        unsigned int src, dest, src_count, dest_count;
+       enum dsync_brain_mailbox_action action;
+       struct dsync_worker *action_worker;
        bool src_deleted, dest_deleted;
        int ret;
 
-       memset(&new_box, 0, sizeof(new_box));
-
        /* create/delete missing directories. */
        src_boxes = array_get(&brain->src_mailbox_list->dirs, &src_count);
        dest_boxes = array_get(&brain->dest_mailbox_list->dirs, &dest_count);
        for (src = dest = 0; src < src_count && dest < dest_count; ) {
+               action = DSYNC_BRAIN_MAILBOX_ACTION_NONE;
                src_deleted = (src_boxes[src]->flags &
                               DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0;
                dest_deleted = (dest_boxes[dest]->flags &
@@ -336,50 +384,57 @@ static void dsync_brain_sync_dirs(struct dsync_brain *brain)
                if (ret < 0) {
                        /* exists only in source */
                        if (!src_deleted) {
-                               new_box = *src_boxes[src];
-                               dsync_worker_create_mailbox(brain->dest_worker,
-                                                           &new_box);
+                               action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+                               action_worker = brain->dest_worker;
+                               action_box = src_boxes[src];
                        }
                        src++;
                } else if (ret > 0) {
                        /* exists only in dest */
                        if (!dest_deleted) {
-                               new_box = *dest_boxes[dest];
-                               dsync_worker_create_mailbox(brain->src_worker,
-                                                           &new_box);
+                               action = DSYNC_BRAIN_MAILBOX_ACTION_CREATE;
+                               action_worker = brain->src_worker;
+                               action_box = dest_boxes[dest];
                        }
                        dest++;
                } else if (src_deleted) {
                        /* delete from dest too */
                        if (!dest_deleted) {
-                               dsync_worker_delete_dir(brain->dest_worker,
-                                                       dest_boxes[dest]);
+                               action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+                               action_worker = brain->dest_worker;
+                               action_box = dest_boxes[dest];
                        }
                        src++; dest++;
                } else if (dest_deleted) {
                        /* delete from src too */
-                       dsync_worker_delete_dir(brain->src_worker,
-                                               src_boxes[src]);
+                       action = DSYNC_BRAIN_MAILBOX_ACTION_DELETE;
+                       action_worker = brain->src_worker;
+                       action_box = src_boxes[src];
                        src++; dest++;
                } else {
                        src++; dest++;
                }
+               i_assert(dsync_mailbox_is_noselect(action_box));
+               dsync_brain_mailbox_action(brain, action,
+                                          action_worker, action_box);
        }
        for (; src < src_count; src++) {
                if ((src_boxes[src]->flags &
                     DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0)
                        continue;
 
-               new_box = *src_boxes[src];
-               dsync_worker_create_mailbox(brain->dest_worker, &new_box);
+               dsync_brain_mailbox_action(brain,
+                       DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+                       brain->dest_worker, src_boxes[src]);
        }
        for (; dest < dest_count; dest++) {
                if ((dest_boxes[dest]->flags &
                     DSYNC_MAILBOX_FLAG_DELETED_DIR) != 0)
                        continue;
 
-               new_box = *dest_boxes[dest];
-               dsync_worker_create_mailbox(brain->src_worker, &new_box);
+               dsync_brain_mailbox_action(brain,
+                       DSYNC_BRAIN_MAILBOX_ACTION_CREATE,
+                       brain->src_worker, dest_boxes[dest]);
        }
 }
 
@@ -410,8 +465,11 @@ dsync_brain_is_unsubscribed(struct dsync_brain_subs_list *list,
 static void dsync_brain_sync_subscriptions(struct dsync_brain *brain)
 {
        const struct dsync_worker_subscription *src_subs, *dest_subs;
+       const struct dsync_worker_subscription *action_subs;
+       struct dsync_worker *action_worker;
        unsigned int src, dest, src_count, dest_count;
        time_t last_change;
+       bool subscribe;
        int ret;
 
        /* subscriptions are sorted by name. */
@@ -435,33 +493,40 @@ static void dsync_brain_sync_subscriptions(struct dsync_brain *brain)
 
                if (ret < 0) {
                        /* subscribed only in source */
+                       action_subs = &src_subs[src];
                        if (dsync_brain_is_unsubscribed(brain->dest_subs_list,
                                                        &src_subs[src],
                                                        &last_change)) {
-                               dsync_worker_set_subscribed(brain->src_worker,
-                                                           src_subs[src].vname,
-                                                           last_change, FALSE);
+                               action_worker = brain->src_worker;
+                               subscribe = FALSE;
                        } else {
-                               dsync_worker_set_subscribed(brain->dest_worker,
-                                                           src_subs[src].vname,
-                                                           last_change, TRUE);
+                               action_worker = brain->dest_worker;
+                               subscribe = TRUE;
                        }
                        src++;
                } else {
                        /* subscribed only in dest */
+                       action_subs = &dest_subs[dest];
                        if (dsync_brain_is_unsubscribed(brain->src_subs_list,
                                                        &dest_subs[dest],
                                                        &last_change)) {
-                               dsync_worker_set_subscribed(brain->dest_worker,
-                                                           dest_subs[dest].vname,
-                                                           last_change, FALSE);
+                               action_worker = brain->dest_worker;
+                               subscribe = FALSE;
                        } else {
-                               dsync_worker_set_subscribed(brain->src_worker,
-                                                           dest_subs[dest].vname,
-                                                           last_change, TRUE);
+                               action_worker = brain->src_worker;
+                               subscribe = TRUE;
                        }
                        dest++;
                }
+
+               if (brain->backup && action_worker == brain->src_worker) {
+                       /* backup mode: switch action */
+                       action_worker = brain->dest_worker;
+                       subscribe = !subscribe;
+                       last_change = ioloop_time;
+               }
+               dsync_worker_set_subscribed(action_worker, action_subs->vname,
+                                           last_change, subscribe);
        }
 }
 
@@ -612,7 +677,8 @@ static void
 dsync_brain_sync_rename_mailbox(struct dsync_brain *brain,
                                const struct dsync_brain_mailbox *mailbox)
 {
-       if (mailbox->src->last_change > mailbox->dest->last_change) {
+       if (mailbox->src->last_change > mailbox->dest->last_change ||
+           brain->backup) {
                dsync_worker_rename_mailbox(brain->dest_worker,
                                            &mailbox->box.mailbox_guid,
                                            mailbox->src);
@@ -629,7 +695,10 @@ dsync_brain_sync_update_mailboxes(struct dsync_brain *brain)
        const struct dsync_brain_mailbox *mailbox;
 
        array_foreach(&brain->mailbox_sync->mailboxes, mailbox) {
-               dsync_worker_update_mailbox(brain->src_worker, &mailbox->box);
+               if (!brain->backup) {
+                       dsync_worker_update_mailbox(brain->src_worker,
+                                                   &mailbox->box);
+               }
                dsync_worker_update_mailbox(brain->dest_worker, &mailbox->box);
 
                if (mailbox->src != NULL && mailbox->dest != NULL &&
@@ -735,6 +804,7 @@ void dsync_brain_sync_all(struct dsync_brain *brain)
 
 bool dsync_brain_has_unexpected_changes(struct dsync_brain *brain)
 {
-       return dsync_worker_has_unexpected_changes(brain->src_worker) ||
+       return brain->unexpected_changes ||
+               dsync_worker_has_unexpected_changes(brain->src_worker) ||
                dsync_worker_has_unexpected_changes(brain->dest_worker);
 }

diff --git a/src/dsync/dsync-brain.h b/src/dsync/dsync-brain.h
index 0acbeff43e0d88459c115f012127f701746edfb3..0c6319b27cc89cc72c161a6244b5db89965b5d6f 100644 (file)
--- a/src/dsync/dsync-brain.h
+++ b/src/dsync/dsync-brain.h
@@ -3,7 +3,10 @@
 
 enum dsync_brain_flags {
        DSYNC_BRAIN_FLAG_FULL_SYNC      = 0x01,
-       DSYNC_BRAIN_FLAG_VERBOSE        = 0x02
+       DSYNC_BRAIN_FLAG_VERBOSE        = 0x02,
+       /* Run in backup mode. All changes from src are forced into dest,
+          discarding any potential changes in dest. */
+       DSYNC_BRAIN_FLAG_BACKUP         = 0x04
 };
 
 struct dsync_worker;

diff --git a/src/dsync/dsync.c b/src/dsync/dsync.c
index 4fe18c5dc1c5b1afabd9aaddbb191c0fadb08f93..2b5bf37accb04db65077832ba9263692e930d61e 100644 (file)
--- a/src/dsync/dsync.c
+++ b/src/dsync/dsync.c
@@ -195,10 +195,13 @@ int main(int argc, char *argv[])
        cmd_name = argv[optind++];
 
        if (strcmp(cmd_name, "mirror") == 0 ||
-           strcmp(cmd_name, "convert") == 0) {
+           strcmp(cmd_name, "convert") == 0 ||
+           strcmp(cmd_name, "backup") == 0) {
                if (optind == argc)
                        usage();
 
+               if (strcmp(cmd_name, "backup") == 0)
+                       brain_flags |= DSYNC_BRAIN_FLAG_BACKUP;
                if (!mirror_get_remote_cmd(argv+optind, &remote_cmd_args)) {
                        if (optind+1 != argc)
                                usage();