imapc: Support retrying some IMAP commands if we get disconnected.

diff --git a/src/lib-imap-client/imapc-client-private.h b/src/lib-imap-client/imapc-client-private.h
index b427872a55376a8cbec54dc8209a8359c171b3a8..70c54b2a0c5bb42fe96289308acb617599189c7c 100644 (file)
--- a/src/lib-imap-client/imapc-client-private.h
+++ b/src/lib-imap-client/imapc-client-private.h
@@ -35,6 +35,7 @@ struct imapc_client_mailbox {
        unsigned int pending_box_command_count;
 
        bool reconnect_ok;
+       bool reconnecting;
 };
 
 void imapc_client_ref(struct imapc_client *client);

diff --git a/src/lib-imap-client/imapc-client.c b/src/lib-imap-client/imapc-client.c
index 83da9c7f4e472dbb8754c5640c1526244877d233..0a99ffa5bf80620e68fadebde3a1d7220e756199 100644 (file)
--- a/src/lib-imap-client/imapc-client.c
+++ b/src/lib-imap-client/imapc-client.c
@@ -258,16 +258,27 @@ imapc_client_reconnect_cb(const struct imapc_command_reply *reply,
 {
        struct imapc_client_mailbox *box = context;
 
+       i_assert(box->reconnecting);
+       box->reconnecting = FALSE;
+
        if (reply->state == IMAPC_COMMAND_STATE_OK) {
                /* reopen the mailbox */
                box->reopen_callback(box->reopen_context);
+       } else {
+               imapc_connection_abort_commands(box->conn);
        }
 }
 
 void imapc_client_mailbox_reconnect(struct imapc_client_mailbox *box)
 {
+       bool reconnect = box->reopen_callback != NULL && box->reconnect_ok;
+
+       if (reconnect) {
+               i_assert(!box->reconnecting);
+               box->reconnecting = TRUE;
+       }
        imapc_connection_disconnect(box->conn);
-       if (box->reopen_callback != NULL && box->reconnect_ok) {
+       if (reconnect) {
                imapc_connection_connect(box->conn,
                                         imapc_client_reconnect_cb, box);
        }
@@ -282,6 +293,12 @@ void imapc_client_mailbox_close(struct imapc_client_mailbox **_box)
        /* cancel any pending commands */
        imapc_connection_unselect(box);
 
+       if (box->reconnecting) {
+               /* need to abort the reconnection so it won't try to access
+                  the box */
+               imapc_connection_disconnect(box->conn);
+       }
+
        /* set this only after unselect, which may cancel some commands that
           reference this box */
        *_box = NULL;

diff --git a/src/lib-imap-client/imapc-connection.c b/src/lib-imap-client/imapc-connection.c
index 9f28c89cf618e07de1a1db96fe675459e4a182e4..39b72b3a4bc104197cdbfa2cdc2a16b6c693f164 100644 (file)
--- a/src/lib-imap-client/imapc-connection.c
+++ b/src/lib-imap-client/imapc-connection.c
@@ -63,6 +63,7 @@ struct imapc_command {
        /* Waiting for '+' literal reply before we can continue */
        unsigned int wait_for_literal:1;
 };
+ARRAY_DEFINE_TYPE(imapc_command, struct imapc_command *);
 
 struct imapc_connection_literal {
        char *temp_path;
@@ -103,9 +104,9 @@ struct imapc_connection {
        void *login_context;
 
        /* commands pending in queue to be sent */
-       ARRAY_DEFINE(cmd_send_queue, struct imapc_command *);
+       ARRAY_TYPE(imapc_command) cmd_send_queue;
        /* commands that have been sent, waiting for their tagged reply */
-       ARRAY_DEFINE(cmd_wait_list, struct imapc_command *);
+       ARRAY_TYPE(imapc_command) cmd_wait_list;
 
        unsigned int ips_count, prev_connect_idx;
        struct ip_addr *ips;
@@ -209,31 +210,72 @@ static const char *imapc_command_get_readable(struct imapc_command *cmd)
 }
 
 static void
-imapc_connection_abort_pending_commands(struct imapc_connection *conn,
-                                       const struct imapc_command_reply *reply)
+imapc_connection_abort_commands_array(ARRAY_TYPE(imapc_command) *cmd_array,
+                                     ARRAY_TYPE(imapc_command) *dest_array,
+                                     bool keep_retriable)
 {
        struct imapc_command *const *cmdp, *cmd;
+       unsigned int i;
 
-       while (array_count(&conn->cmd_wait_list) > 0) {
-               cmdp = array_idx(&conn->cmd_wait_list, 0);
+       for (i = 0; i < array_count(cmd_array); ) {
+               cmdp = array_idx(cmd_array, i);
                cmd = *cmdp;
-               array_delete(&conn->cmd_wait_list, 0, 1);
 
-               if (cmd->callback != NULL)
-                       cmd->callback(reply, cmd->context);
-               imapc_command_free(cmd);
+               if (keep_retriable &&
+                   (cmd->flags & IMAPC_COMMAND_FLAG_RETRIABLE) != 0) {
+                       cmd->send_pos = 0;
+                       cmd->wait_for_literal = 0;
+                       i++;
+               } else {
+                       array_delete(cmd_array, i, 1);
+                       array_append(dest_array, &cmd, 1);
+               }
+       }
+}
+
+static void
+imapc_connection_abort_commands_full(struct imapc_connection *conn,
+                                    bool keep_retriable)
+{
+       struct imapc_command *const *cmdp, *cmd;
+       ARRAY_TYPE(imapc_command) tmp_array;
+       struct imapc_command_reply reply;
+
+       t_array_init(&tmp_array, 8);
+       imapc_connection_abort_commands_array(&conn->cmd_wait_list,
+                                             &tmp_array, keep_retriable);
+       imapc_connection_abort_commands_array(&conn->cmd_send_queue,
+                                             &tmp_array, keep_retriable);
+
+       if (array_count(&conn->cmd_wait_list) > 0) {
+               /* need to move all the waiting commands to send queue */
+               array_append_array(&conn->cmd_wait_list,
+                                  &conn->cmd_send_queue);
+               array_clear(&conn->cmd_send_queue);
+               array_append_array(&conn->cmd_send_queue,
+                                  &conn->cmd_wait_list);
+               array_clear(&conn->cmd_wait_list);
        }
-       while (array_count(&conn->cmd_send_queue) > 0) {
-               cmdp = array_idx(&conn->cmd_send_queue, 0);
+
+       /* abort the commands. we'll do it here later so that if the
+          callback recurses us back here we don't crash */
+       memset(&reply, 0, sizeof(reply));
+       reply.state = IMAPC_COMMAND_STATE_DISCONNECTED;
+       reply.text_without_resp = reply.text_full =
+               "Disconnected from server";
+       array_foreach(&tmp_array, cmdp) {
                cmd = *cmdp;
-               array_delete(&conn->cmd_send_queue, 0, 1);
 
-               if (cmd->callback != NULL)
-                       cmd->callback(reply, cmd->context);
+               cmd->callback(&reply, cmd->context);
                imapc_command_free(cmd);
        }
 }
 
+void imapc_connection_abort_commands(struct imapc_connection *conn)
+{
+       imapc_connection_abort_commands_full(conn, FALSE);
+}
+
 static void
 imapc_login_callback(struct imapc_connection *conn,
                     const struct imapc_command_reply *reply)
@@ -262,7 +304,6 @@ static void imapc_connection_set_state(struct imapc_connection *conn,
                reply.state = IMAPC_COMMAND_STATE_DISCONNECTED;
                reply.text_without_resp = reply.text_full =
                        "Disconnected from server";
-               imapc_connection_abort_pending_commands(conn, &reply);
                imapc_login_callback(conn, &reply);
 
                conn->idling = FALSE;
@@ -272,9 +313,6 @@ static void imapc_connection_set_state(struct imapc_connection *conn,
                conn->selecting_box = NULL;
                conn->selected_box = NULL;
                break;
-       case IMAPC_CONNECTION_STATE_DONE:
-               imapc_command_send_more(conn);
-               break;
        default:
                break;
        }
@@ -306,6 +344,9 @@ imapc_connection_literal_reset(struct imapc_connection_literal *literal)
 
 void imapc_connection_disconnect(struct imapc_connection *conn)
 {
+       bool reconnecting = conn->selected_box != NULL &&
+               conn->selected_box->reconnecting;
+
        if (conn->fd == -1)
                return;
 
@@ -327,6 +368,13 @@ void imapc_connection_disconnect(struct imapc_connection *conn)
        net_disconnect(conn->fd);
        conn->fd = -1;
 
+       imapc_connection_abort_commands_full(conn, reconnecting);
+       imapc_connection_set_state(conn, IMAPC_CONNECTION_STATE_DISCONNECTED);
+}
+
+static void imapc_connection_set_disconnected(struct imapc_connection *conn)
+{
+       imapc_connection_abort_commands(conn);
        imapc_connection_set_state(conn, IMAPC_CONNECTION_STATE_DISCONNECTED);
 }
 
@@ -794,6 +842,7 @@ static int imapc_connection_input_untagged(struct imapc_connection *conn)
 {
        const struct imap_arg *imap_args;
        const char *name, *value;
+       struct imap_parser *parser;
        struct imapc_untagged_reply reply;
        int ret;
 
@@ -854,7 +903,12 @@ static int imapc_connection_input_untagged(struct imapc_connection *conn)
                reply.untagged_box_context =
                        conn->selected_box->untagged_box_context;
        }
+
+       /* the callback may disconnect and destroy the parser */
+       parser = conn->parser;
+       imap_parser_ref(parser);
        conn->client->untagged_callback(&reply, conn->client->untagged_context);
+       imap_parser_unref(&parser);
        imapc_connection_input_reset(conn);
        return 1;
 }
@@ -893,8 +947,7 @@ static void
 imapc_command_reply_free(struct imapc_command *cmd,
                         const struct imapc_command_reply *reply)
 {
-       if (cmd->callback != NULL)
-               cmd->callback(reply, cmd->context);
+       cmd->callback(reply, cmd->context);
        imapc_command_free(cmd);
 }
 
@@ -904,6 +957,7 @@ static int imapc_connection_input_tagged(struct imapc_connection *conn)
        unsigned int i, count;
        char *line, *linep;
        const char *p;
+       struct imap_parser *parser;
        struct imapc_command_reply reply;
 
        line = i_stream_next_line(conn->input);
@@ -985,7 +1039,13 @@ static int imapc_connection_input_tagged(struct imapc_connection *conn)
        }
 
        imapc_connection_input_reset(conn);
+
+       parser = conn->parser;
+       imap_parser_ref(parser);
        imapc_command_reply_free(cmd, &reply);
+       imap_parser_unref(&parser);
+
+       imapc_command_send_more(conn);
        return 1;
 }
 
@@ -1215,8 +1275,7 @@ static void imapc_connection_connect_next_ip(struct imapc_connection *conn)
        ip = &conn->ips[conn->prev_connect_idx];
        fd = net_connect_ip(ip, conn->client->set.port, NULL);
        if (fd == -1) {
-               imapc_connection_set_state(conn,
-                       IMAPC_CONNECTION_STATE_DISCONNECTED);
+               imapc_connection_set_disconnected(conn);
                return;
        }
        conn->fd = fd;
@@ -1252,8 +1311,7 @@ imapc_connection_dns_callback(const struct dns_lookup_result *result,
        if (result->ret != 0) {
                i_error("imapc(%s): dns_lookup(%s) failed: %s",
                        conn->name, conn->client->set.host, result->error);
-               imapc_connection_set_state(conn,
-                       IMAPC_CONNECTION_STATE_DISCONNECTED);
+               imapc_connection_set_disconnected(conn);
                return;
        }
 
@@ -1483,10 +1541,14 @@ static void imapc_command_send_more(struct imapc_connection *conn)
                /* SELECT/EXAMINE command */
                imapc_connection_set_selecting(cmd->box);
        } else if (!imapc_client_mailbox_is_opened(cmd->box)) {
+               if (cmd->box->reconnecting) {
+                       /* wait for SELECT/EXAMINE */
+                       return;
+               }
                /* shouldn't normally happen */
                memset(&reply, 0, sizeof(reply));
                reply.text_without_resp = reply.text_full = "Mailbox not open";
-               reply.state = IMAPC_COMMAND_STATE_BAD;
+               reply.state = IMAPC_COMMAND_STATE_DISCONNECTED;
 
                array_delete(&conn->cmd_send_queue, 0, 1);
                imapc_command_reply_free(cmd, &reply);
@@ -1577,7 +1639,14 @@ static void imapc_connection_cmd_send(struct imapc_command *cmd)
                                               imapc_command_timeout, conn);
                }
        }
-       array_append(&conn->cmd_send_queue, &cmd, 1);
+       if ((cmd->flags & IMAPC_COMMAND_FLAG_SELECT) != 0 &&
+           conn->selected_box == NULL) {
+               /* reopening the mailbox. add it before other
+                  queued commands. */
+               array_insert(&conn->cmd_send_queue, 0, &cmd, 1);
+       } else {
+               array_append(&conn->cmd_send_queue, &cmd, 1);
+       }
        imapc_command_send_more(conn);
 }
 
@@ -1735,37 +1804,9 @@ imapc_connection_get_capabilities(struct imapc_connection *conn)
 void imapc_connection_unselect(struct imapc_client_mailbox *box)
 {
        struct imapc_connection *conn = box->conn;
-       struct imapc_command *const *cmdp, *cmd;
-       struct imapc_command_reply reply;
-       unsigned int i;
-
-       /* mailbox is being closed. if there are any pending commands, we must
-          finish them immediately so callbacks don't access any freed
-          contexts */
-       memset(&reply, 0, sizeof(reply));
-       reply.state = IMAPC_COMMAND_STATE_DISCONNECTED;
-       reply.text_without_resp = reply.text_full = "Closing mailbox";
 
        imapc_connection_send_idle_done(conn);
-
-       array_foreach(&conn->cmd_wait_list, cmdp) {
-               if ((*cmdp)->callback != NULL && (*cmdp)->box != NULL) {
-                       (*cmdp)->callback(&reply, (*cmdp)->context);
-                       (*cmdp)->callback = NULL;
-               }
-       }
-       for (i = 0; i < array_count(&conn->cmd_send_queue); ) {
-               cmdp = array_idx(&conn->cmd_send_queue, i);
-               cmd = *cmdp;
-               if (cmd->box == NULL)
-                       i++;
-               else {
-                       array_delete(&conn->cmd_send_queue, i, 1);
-                       if (cmd->callback != NULL)
-                               cmd->callback(&reply, cmd->context);
-                       imapc_command_free(cmd);
-               }
-       }
+       imapc_connection_abort_commands(conn);
 
        if (conn->selected_box != NULL || conn->selecting_box != NULL) {
                i_assert(conn->selected_box == box ||

diff --git a/src/lib-imap-client/imapc-connection.h b/src/lib-imap-client/imapc-connection.h
index 0b50a6cf2d9abcb02a34e0ddd73865de2665a5c0..b7cec0798e2c7e933a26d5db3e46c6220386a5da 100644 (file)
--- a/src/lib-imap-client/imapc-connection.h
+++ b/src/lib-imap-client/imapc-connection.h
@@ -25,6 +25,7 @@ void imapc_connection_connect(struct imapc_connection *conn,
                              imapc_command_callback_t *login_callback,
                              void *login_context);
 void imapc_connection_disconnect(struct imapc_connection *conn);
+void imapc_connection_abort_commands(struct imapc_connection *conn);
 void imapc_connection_ioloop_changed(struct imapc_connection *conn);
 void imapc_connection_input_pending(struct imapc_connection *conn);
 

diff --git a/src/lib-storage/index/imapc/imapc-mail-fetch.c b/src/lib-storage/index/imapc/imapc-mail-fetch.c
index acb374c532d854049f6309a779cb5ddc02cfd3db..c9adbd4a6a62dca35fa1251935735a641293a22a 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-mail-fetch.c
+++ b/src/lib-storage/index/imapc/imapc-mail-fetch.c
@@ -109,6 +109,7 @@ imapc_mail_send_fetch(struct mail *_mail, enum mail_fetch_field fields)
 
        cmd = imapc_client_mailbox_cmd(mbox->client_box,
                                       imapc_mail_prefetch_callback, mail);
+       imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
        imapc_command_send(cmd, str_c(str));
        mail->imail.data.prefetch_sent = TRUE;
        return 0;

diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c
index 658de0a87f83cbfee871914cbd05d6b46a378311..852dd561019d7a225471b781aaa0694947ea70ff 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-storage.c
+++ b/src/lib-storage/index/imapc/imapc-storage.c
@@ -380,6 +380,9 @@ static void imapc_mailbox_reopen(void *context)
        else
                imapc_command_sendf(cmd, "SELECT %s", mbox->box.name);
        mbox->storage->reopen_count++;
+
+       if (mbox->syncing)
+               imapc_sync_mailbox_reopened(mbox);
 }
 
 static void

diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h
index 1c10bc30455444c9edeb51b8cbd4261815242bf5..7046cb25453d3d393e20ed19e341b09925be8263 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-storage.h
+++ b/src/lib-storage/index/imapc/imapc-storage.h
@@ -80,6 +80,7 @@ struct imapc_mailbox {
        struct imapc_mail_cache prev_mail_cache;
 
        uint32_t prev_skipped_rseq, prev_skipped_uid;
+       struct imapc_sync_context *sync_ctx;
 
        unsigned int selecting:1;
        unsigned int syncing:1;

diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c
index 25d3e7e1ca7274b9b085a4289ccb236f3ed0478c..f2715bdd36ffd87439b6aa07b2f1977f82ec218d 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-sync.c
+++ b/src/lib-storage/index/imapc/imapc-sync.c
@@ -44,6 +44,7 @@ static void imapc_sync_cmd(struct imapc_sync_context *ctx, const char *cmd_str)
        ctx->sync_command_count++;
        cmd = imapc_client_mailbox_cmd(ctx->mbox->client_box,
                                       imapc_sync_callback, ctx);
+       imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
        imapc_command_send(cmd, cmd_str);
 }
 
@@ -365,6 +366,20 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
        }
 }
 
+void imapc_sync_mailbox_reopened(struct imapc_mailbox *mbox)
+{
+       struct imapc_sync_context *ctx = mbox->sync_ctx;
+
+       i_assert(mbox->syncing);
+
+       /* we got disconnected while syncing. need to
+          re-fetch everything */
+       mbox->sync_next_lseq = 1;
+       mbox->sync_next_rseq = 1;
+
+       imapc_sync_cmd(ctx, "UID FETCH 1:* FLAGS");
+}
+
 static int
 imapc_sync_begin(struct imapc_mailbox *mbox,
                 struct imapc_sync_context **ctx_r, bool force)
@@ -402,6 +417,7 @@ imapc_sync_begin(struct imapc_mailbox *mbox,
        mbox->min_append_uid = mail_index_get_header(ctx->sync_view)->next_uid;
 
        mbox->syncing = TRUE;
+       mbox->sync_ctx = ctx;
        if (!mbox->box.deleting)
                imapc_sync_index(ctx);
 
@@ -429,6 +445,7 @@ static int imapc_sync_finish(struct imapc_sync_context **_ctx)
                mail_index_sync_rollback(&ctx->index_sync_ctx);
        }
        ctx->mbox->syncing = FALSE;
+       ctx->mbox->sync_ctx = NULL;
 
        /* this is done simply to commit delayed expunges if there are any
           (has to be done after sync is committed) */

diff --git a/src/lib-storage/index/imapc/imapc-sync.h b/src/lib-storage/index/imapc/imapc-sync.h
index de0da627a6f67b8da77109a2237d042fa6bafe9c..327171cec944f4ffe07c830b9c814c980425756d 100644 (file)
--- a/src/lib-storage/index/imapc/imapc-sync.h
+++ b/src/lib-storage/index/imapc/imapc-sync.h
@@ -22,5 +22,6 @@ struct mailbox_sync_context *
 imapc_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags);
 int imapc_mailbox_sync_deinit(struct mailbox_sync_context *ctx,
                              struct mailbox_sync_status *status_r);
+void imapc_sync_mailbox_reopened(struct imapc_mailbox *mbox);
 
 #endif