]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
imapc: Many fixes related to syncing and error handling.
authorTimo Sirainen <tss@iki.fi>
Thu, 24 Feb 2011 13:17:47 +0000 (15:17 +0200)
committerTimo Sirainen <tss@iki.fi>
Thu, 24 Feb 2011 13:17:47 +0000 (15:17 +0200)
12 files changed:
src/lib-storage/index/imapc/imapc-client.c
src/lib-storage/index/imapc/imapc-client.h
src/lib-storage/index/imapc/imapc-connection.c
src/lib-storage/index/imapc/imapc-connection.h
src/lib-storage/index/imapc/imapc-mailbox.c
src/lib-storage/index/imapc/imapc-save.c
src/lib-storage/index/imapc/imapc-seqmap.c
src/lib-storage/index/imapc/imapc-seqmap.h
src/lib-storage/index/imapc/imapc-storage.c
src/lib-storage/index/imapc/imapc-storage.h
src/lib-storage/index/imapc/imapc-sync.c
src/lib-storage/index/imapc/imapc-sync.h

index d784bcf72d1208d3ef9257c42201149b5f7bf3cf..7374eb786f7696e37921db2dbe89ab4dc1f44e2d 100644 (file)
@@ -84,8 +84,10 @@ void imapc_client_deinit(struct imapc_client **_client)
 
        if (client->ssl_ctx != NULL)
                ssl_iostream_context_deinit(&client->ssl_ctx);
-       array_foreach_modifiable(&client->conns, connp)
+       array_foreach_modifiable(&client->conns, connp) {
                imapc_connection_deinit(&(*connp)->conn);
+               i_free(*connp);
+       }
        pool_unref(&client->pool);
 }
 
@@ -212,6 +214,12 @@ imapc_client_mailbox_open(struct imapc_client *client, const char *name,
        return box;
 }
 
+void imapc_client_mailbox_disconnect(struct imapc_client_mailbox *box)
+{
+       if (box->conn != NULL)
+               imapc_connection_disconnect(box->conn);
+}
+
 void imapc_client_mailbox_close(struct imapc_client_mailbox **_box)
 {
        struct imapc_client_mailbox *box = *_box;
@@ -226,7 +234,8 @@ void imapc_client_mailbox_close(struct imapc_client_mailbox **_box)
                }
        }
 
-       imapc_connection_unselect(box);
+       if (box->conn != NULL)
+               imapc_connection_unselect(box);
        imapc_seqmap_deinit(&box->seqmap);
        i_free(box);
 }
@@ -258,12 +267,43 @@ imapc_client_mailbox_cmd_common(struct imapc_client_mailbox *box,
        return ctx;
 }
 
+static bool
+imapc_client_mailbox_is_selected(struct imapc_client_mailbox *box,
+                                struct imapc_command_reply *reply_r)
+{
+       struct imapc_client_mailbox *selected_box;
+
+       selected_box = box->conn == NULL ? NULL :
+               imapc_connection_get_mailbox(box->conn);
+       if (selected_box == box)
+               return TRUE;
+
+       memset(reply_r, 0, sizeof(*reply_r));
+       reply_r->state = IMAPC_COMMAND_STATE_DISCONNECTED;
+       if (selected_box == NULL) {
+               reply_r->text_full = "Disconnected from server";
+       } else {
+               i_error("imapc: Selected mailbox changed unexpectedly");
+               reply_r->text_full = "Internal error";
+       }
+       reply_r->text_without_resp = reply_r->text_full;
+
+       box->conn = NULL;
+       return FALSE;
+}
+
 void imapc_client_mailbox_cmd(struct imapc_client_mailbox *box,
                              const char *cmd,
                              imapc_command_callback_t *callback,
                              void *context)
 {
        struct imapc_client_command_context *ctx;
+       struct imapc_command_reply reply;
+
+       if (!imapc_client_mailbox_is_selected(box, &reply)) {
+               callback(&reply, context);
+               return;
+       }
 
        ctx = imapc_client_mailbox_cmd_common(box, callback, context);
        imapc_connection_cmd(box->conn, cmd, imapc_client_mailbox_cmd_cb, ctx);
@@ -275,6 +315,12 @@ void imapc_client_mailbox_cmdf(struct imapc_client_mailbox *box,
 {
        struct imapc_client_command_context *ctx;
        va_list args;
+       struct imapc_command_reply reply;
+
+       if (!imapc_client_mailbox_is_selected(box, &reply)) {
+               callback(&reply, context);
+               return;
+       }
 
        ctx = imapc_client_mailbox_cmd_common(box, callback, context);
        va_start(args, cmd_fmt);
@@ -291,7 +337,17 @@ imapc_client_mailbox_get_seqmap(struct imapc_client_mailbox *box)
 
 void imapc_client_mailbox_idle(struct imapc_client_mailbox *box)
 {
-       imapc_connection_idle(box->conn);
+       struct imapc_command_reply reply;
+
+       if (imapc_client_mailbox_is_selected(box, &reply))
+               imapc_connection_idle(box->conn);
+}
+
+bool imapc_client_mailbox_is_connected(struct imapc_client_mailbox *box)
+{
+       struct imapc_command_reply reply;
+
+       return imapc_client_mailbox_is_selected(box, &reply);
 }
 
 enum imapc_capability
index 210ee9a57573920a17805d7914829e8d7274f8b2..308bba8ee671b4772689302a861919faa77efb8b 100644 (file)
@@ -123,6 +123,7 @@ imapc_client_mailbox_open(struct imapc_client *client, const char *name,
                          imapc_command_callback_t *callback, void *context,
                          void *untagged_box_context);
 void imapc_client_mailbox_close(struct imapc_client_mailbox **box);
+void imapc_client_mailbox_disconnect(struct imapc_client_mailbox *box);
 void imapc_client_mailbox_cmd(struct imapc_client_mailbox *box,
                              const char *cmd,
                              imapc_command_callback_t *callback,
@@ -135,6 +136,7 @@ struct imapc_seqmap *
 imapc_client_mailbox_get_seqmap(struct imapc_client_mailbox *box);
 
 void imapc_client_mailbox_idle(struct imapc_client_mailbox *box);
+bool imapc_client_mailbox_is_connected(struct imapc_client_mailbox *box);
 
 enum imapc_capability
 imapc_client_get_capabilities(struct imapc_client *client);
index bdff698eb9da8e5fc214b217a3b824968116aba2..be39e4effef53ea9d30c635556978bf304e3d312 100644 (file)
 
 #define IMAPC_DNS_LOOKUP_TIMEOUT_MSECS (1000*30)
 #define IMAPC_CONNECT_TIMEOUT_MSECS (1000*30)
+#define IMAPC_COMMAND_TIMEOUT_MSECS (1000*60*5)
 #define IMAPC_MAX_INLINE_LITERAL_SIZE (1024*32)
 
 enum imapc_input_state {
        IMAPC_INPUT_STATE_NONE = 0,
+       IMAPC_INPUT_STATE_PLUS,
        IMAPC_INPUT_STATE_UNTAGGED,
        IMAPC_INPUT_STATE_UNTAGGED_NUM,
-       IMAPC_INPUT_STATE_TAGGED,
-       IMAPC_INPUT_STATE_SKIPLINE
+       IMAPC_INPUT_STATE_TAGGED
 };
 
 struct imapc_command_stream {
@@ -84,7 +85,9 @@ struct imapc_connection {
        enum imapc_capability capabilities;
        char **capabilities_list;
 
+       /* commands pending in queue to be sent */
        ARRAY_DEFINE(cmd_send_queue, struct imapc_command *);
+       /* commands that have been sent, waiting for their tagged reply */
        ARRAY_DEFINE(cmd_wait_list, struct imapc_command *);
 
        unsigned int ips_count, prev_connect_idx;
@@ -99,8 +102,6 @@ struct imapc_connection {
 };
 
 static int imapc_connection_ssl_init(struct imapc_connection *conn);
-static void imapc_connection_disconnect(struct imapc_connection *conn);
-
 static void imapc_command_free(struct imapc_command *cmd);
 static void imapc_command_send_more(struct imapc_connection *conn,
                                    struct imapc_command *cmd);
@@ -147,39 +148,50 @@ void imapc_connection_ioloop_changed(struct imapc_connection *conn)
                conn->to = io_loop_move_timeout(&conn->to);
 }
 
+static void
+imapc_connection_abort_pending_commands(struct imapc_connection *conn,
+                                       const struct imapc_command_reply *reply)
+{
+       struct imapc_command *const *cmdp, *cmd;
+
+       while (array_count(&conn->cmd_wait_list) > 0) {
+               cmdp = array_idx(&conn->cmd_wait_list, 0);
+               cmd = *cmdp;
+               array_delete(&conn->cmd_wait_list, 0, 1);
+
+               if (cmd->callback != NULL)
+                       cmd->callback(reply, cmd->context);
+               imapc_command_free(cmd);
+       }
+       while (array_count(&conn->cmd_send_queue) > 0) {
+               cmdp = array_idx(&conn->cmd_send_queue, 0);
+               cmd = *cmdp;
+               array_delete(&conn->cmd_send_queue, 0, 1);
+
+               if (cmd->callback != NULL)
+                       cmd->callback(reply, cmd->context);
+               imapc_command_free(cmd);
+       }
+}
+
 static void imapc_connection_set_state(struct imapc_connection *conn,
                                       enum imapc_connection_state state)
 {
        if (state == IMAPC_CONNECTION_STATE_DISCONNECTED) {
-               /* abort all pending commands */
                struct imapc_command_reply reply;
-               struct imapc_command *const *cmdp, *cmd;
 
                memset(&reply, 0, sizeof(reply));
                reply.state = IMAPC_COMMAND_STATE_DISCONNECTED;
                reply.text_without_resp = reply.text_full =
                        "Disconnected from server";
+               imapc_connection_abort_pending_commands(conn, &reply);
 
                conn->idling = FALSE;
                conn->idle_plus_waiting = FALSE;
                conn->idle_stopping = FALSE;
 
-               while (array_count(&conn->cmd_wait_list) > 0) {
-                       cmdp = array_idx(&conn->cmd_wait_list, 0);
-                       cmd = *cmdp;
-                       array_delete(&conn->cmd_wait_list, 0, 1);
-
-                       cmd->callback(&reply, cmd->context);
-                       imapc_command_free(cmd);
-               }
-               while (array_count(&conn->cmd_send_queue) > 0) {
-                       cmdp = array_idx(&conn->cmd_send_queue, 0);
-                       cmd = *cmdp;
-                       array_delete(&conn->cmd_send_queue, 0, 1);
-
-                       cmd->callback(&reply, cmd->context);
-                       imapc_command_free(cmd);
-               }
+               conn->selecting_box = NULL;
+               conn->selected_box = NULL;
        }
        if (state == IMAPC_CONNECTION_STATE_DONE) {
                if (array_count(&conn->cmd_send_queue) > 0) {
@@ -215,7 +227,7 @@ imapc_connection_literal_reset(struct imapc_connection_literal *literal)
        literal->fd = -1;
 }
 
-static void imapc_connection_disconnect(struct imapc_connection *conn)
+void imapc_connection_disconnect(struct imapc_connection *conn)
 {
        if (conn->fd == -1)
                return;
@@ -375,10 +387,22 @@ static int
 imapc_connection_read_line(struct imapc_connection *conn,
                           const struct imap_arg **imap_args_r)
 {
+       const unsigned char *data;
+       size_t size;
        int ret;
 
        while ((ret = imapc_connection_read_line_more(conn, imap_args_r)) == 2)
                ;
+
+       if (ret > 0) {
+               data = i_stream_get_data(conn->input, &size);
+               if (size >= 2 && data[0] == '\r' && data[1] == '\n')
+                       i_stream_skip(conn->input, 2);
+               else if (size >= 1 && data[0] == '\n')
+                       i_stream_skip(conn->input, 1);
+               else
+                       i_panic("imapc: Missing LF from input line");
+       }
        return ret;
 }
 
@@ -506,29 +530,11 @@ static void imapc_connection_input_reset(struct imapc_connection *conn)
        conn->input_state = IMAPC_INPUT_STATE_NONE;
        conn->cur_tag = 0;
        conn->cur_num = 0;
-       imap_parser_reset(conn->parser);
+       if (conn->parser != NULL)
+               imap_parser_reset(conn->parser);
        imapc_connection_lfiles_free(conn);
 }
 
-static int imapc_connection_skip_line(struct imapc_connection *conn)
-{
-       const unsigned char *data;
-       size_t i, data_size;
-       int ret = 0;
-
-       data = i_stream_get_data(conn->input, &data_size);
-       for (i = 0; i < data_size; i++) {
-               if (data[i] == '\n') {
-                       imapc_connection_input_reset(conn);
-                       ret = 1;
-                       i++;
-                       break;
-               }
-       }
-       i_stream_skip(conn->input, i);
-       return ret;
-}
-
 static void imapc_connection_login_cb(const struct imapc_command_reply *reply,
                                      void *context)
 {
@@ -749,11 +755,6 @@ static int imapc_connection_input_untagged(struct imapc_connection *conn)
                        conn->selected_box->untagged_box_context;
        }
        conn->client->untagged_callback(&reply, conn->client->untagged_context);
-       if (imap_arg_atom_equals(imap_args, "EXPUNGE") &&
-           conn->selected_box != NULL) {
-               /* keep track of expunge map internally */
-               imapc_seqmap_expunge(conn->selected_box->seqmap, conn->cur_num);
-       }
        imapc_connection_input_reset(conn);
        return 1;
 }
@@ -761,23 +762,26 @@ static int imapc_connection_input_untagged(struct imapc_connection *conn)
 static int imapc_connection_input_plus(struct imapc_connection *conn)
 {
        struct imapc_command *const *cmd_p;
+       const char *line;
+
+       if ((line = i_stream_next_line(conn->input)) == NULL)
+               return 0;
 
        if (conn->idle_plus_waiting) {
                /* "+ idling" reply for IDLE command */
                conn->idle_plus_waiting = FALSE;
                conn->idling = TRUE;
-               return imapc_connection_skip_line(conn);
-       }
-
-       if (array_count(&conn->cmd_send_queue) == 0) {
-               imapc_connection_input_error(conn, "Unexpected '+'");
+       } else if (array_count(&conn->cmd_send_queue) > 0) {
+               /* reply for literal */
+               cmd_p = array_idx(&conn->cmd_send_queue, 0);
+               imapc_command_send_more(conn, *cmd_p);
+       } else {
+               imapc_connection_input_error(conn, "Unexpected '+': %s", line);
                return -1;
        }
-       cmd_p = array_idx(&conn->cmd_send_queue, 0);
-       imapc_command_send_more(conn, *cmd_p);
 
-       conn->input_state = IMAPC_INPUT_STATE_SKIPLINE;
-       return imapc_connection_skip_line(conn);
+       imapc_connection_input_reset(conn);
+       return 1;
 }
 
 static int imapc_connection_input_tagged(struct imapc_connection *conn)
@@ -806,14 +810,12 @@ static int imapc_connection_input_tagged(struct imapc_connection *conn)
                reply.state = IMAPC_COMMAND_STATE_OK;
        else if (strcasecmp(line, "no") == 0)
                reply.state = IMAPC_COMMAND_STATE_NO;
-       else if (strcasecmp(line, "bad") == 0) {
-               i_error("imapc(%s): Command failed with BAD: %u %s",
-                       conn->name, conn->cur_tag, line);
+       else if (strcasecmp(line, "bad") == 0)
                reply.state = IMAPC_COMMAND_STATE_BAD;
-       else {
+       else {
                imapc_connection_input_error(conn,
-                       "Invalid state in tagged reply: %u %s",
-                       conn->cur_tag, line);
+                       "Invalid state in tagged reply: %u %s %s",
+                       conn->cur_tag, line, reply.text_full);
                return -1;
        }
 
@@ -849,15 +851,27 @@ static int imapc_connection_input_tagged(struct imapc_connection *conn)
                        }
                }
        }
+       if (array_count(&conn->cmd_wait_list) == 0 &&
+           array_count(&conn->cmd_send_queue) == 0 &&
+           conn->state == IMAPC_CONNECTION_STATE_DONE && conn->to != NULL)
+               timeout_remove(&conn->to);
 
        if (cmd == NULL) {
                imapc_connection_input_error(conn,
-                       "Unknown tag in a reply: %u %s", conn->cur_tag, line);
+                       "Unknown tag in a reply: %u %s %s",
+                       conn->cur_tag, line, reply.text_full);
                return -1;
        }
 
+       if (reply.state == IMAPC_COMMAND_STATE_BAD) {
+               i_error("imapc(%s): Command '%s' failed with BAD: %u %s",
+                       conn->name, str_c(cmd->data), conn->cur_tag,
+                       reply.text_full);
+       }
+
        imapc_connection_input_reset(conn);
-       cmd->callback(&reply, cmd->context);
+       if (cmd->callback != NULL)
+               cmd->callback(&reply, cmd->context);
        imapc_command_free(cmd);
        return 1;
 }
@@ -876,15 +890,12 @@ static int imapc_connection_input_one(struct imapc_connection *conn)
                if (tag == NULL)
                        return 0;
 
-               if (strcmp(tag, "") == 0) {
-                       /* FIXME: why do we get here.. */
-                       conn->input_state = IMAPC_INPUT_STATE_SKIPLINE;
-                       return imapc_connection_skip_line(conn);
-               } else if (strcmp(tag, "*") == 0) {
+               if (strcmp(tag, "*") == 0) {
                        conn->input_state = IMAPC_INPUT_STATE_UNTAGGED;
                        conn->cur_num = 0;
                        ret = imapc_connection_input_untagged(conn);
                } else if (strcmp(tag, "+") == 0) {
+                       conn->input_state = IMAPC_INPUT_STATE_PLUS;
                        ret = imapc_connection_input_plus(conn);
                } else {
                        conn->input_state = IMAPC_INPUT_STATE_TAGGED;
@@ -898,6 +909,9 @@ static int imapc_connection_input_one(struct imapc_connection *conn)
                        }
                }
                break;
+       case IMAPC_INPUT_STATE_PLUS:
+               ret = imapc_connection_input_plus(conn);
+               break;
        case IMAPC_INPUT_STATE_UNTAGGED:
        case IMAPC_INPUT_STATE_UNTAGGED_NUM:
                ret = imapc_connection_input_untagged(conn);
@@ -905,9 +919,6 @@ static int imapc_connection_input_one(struct imapc_connection *conn)
        case IMAPC_INPUT_STATE_TAGGED:
                ret = imapc_connection_input_tagged(conn);
                break;
-       case IMAPC_INPUT_STATE_SKIPLINE:
-               ret = imapc_connection_skip_line(conn);
-               break;
        }
        return ret;
 }
@@ -1092,6 +1103,8 @@ void imapc_connection_connect(struct imapc_connection *conn)
        if (conn->fd != -1)
                return;
 
+       imapc_connection_input_reset(conn);
+
        if (conn->client->set.debug)
                i_debug("imapc(%s): Looking up IP address", conn->name);
 
@@ -1116,6 +1129,9 @@ void imapc_connection_input_pending(struct imapc_connection *conn)
        if (conn->input == NULL)
                return;
 
+       if (conn->to != NULL)
+               timeout_reset(conn->to);
+
        o_stream_cork(conn->output);
        while (ret > 0 && !conn->client->stop_now && conn->input != NULL) {
                T_BEGIN {
@@ -1134,7 +1150,7 @@ imapc_command_begin(imapc_command_callback_t *callback, void *context)
        struct imapc_command *cmd;
        pool_t pool;
 
-       pool = pool_alloconly_create("imapc command", 1024);
+       pool = pool_alloconly_create("imapc command", 2048);
        cmd = p_new(pool, struct imapc_command, 1);
        cmd->pool = pool;
        cmd->callback = callback;
@@ -1273,6 +1289,19 @@ static void imapc_command_send_more(struct imapc_connection *conn,
        }
 }
 
+static void imapc_command_timeout(struct imapc_connection *conn)
+{
+       struct imapc_command *const *cmds;
+       unsigned int count;
+
+       cmds = array_get(&conn->cmd_wait_list, &count);
+       i_assert(count > 0);
+
+       i_error("imapc(%s): Command '%s' timed out, disconnecting",
+               conn->name, str_c(cmds[0]->data));
+       imapc_connection_disconnect(conn);
+}
+
 static void imapc_command_send(struct imapc_connection *conn,
                               struct imapc_command *cmd)
 {
@@ -1286,6 +1315,14 @@ static void imapc_command_send(struct imapc_connection *conn,
                imapc_command_send_more(conn, cmd);
                break;
        case IMAPC_CONNECTION_STATE_DONE:
+               if (cmd->idle) {
+                       if (conn->to != NULL)
+                               timeout_remove(&conn->to);
+               } else if (conn->to == NULL) {
+                       conn->to = timeout_add(IMAPC_COMMAND_TIMEOUT_MSECS,
+                                              imapc_command_timeout, conn);
+               }
+
                array_append(&conn->cmd_send_queue, &cmd, 1);
                if (array_count(&conn->cmd_send_queue) == 1)
                        imapc_command_send_more(conn, cmd);
@@ -1436,11 +1473,47 @@ void imapc_connection_select(struct imapc_client_mailbox *box, const char *name,
 
 void imapc_connection_unselect(struct imapc_client_mailbox *box)
 {
-       i_assert(box->conn->selected_box == box ||
-                box->conn->selecting_box == box);
+       struct imapc_command *const *cmdp;
+       struct imapc_command_reply reply;
 
-       box->conn->selected_box = NULL;
-       box->conn->selecting_box = NULL;
+       /* mailbox is being closed. if there are any pending commands, we must
+          finish them immediately so callbacks don't access any freed
+          contexts */
+       memset(&reply, 0, sizeof(reply));
+       reply.state = IMAPC_COMMAND_STATE_DISCONNECTED;
+       reply.text_without_resp = reply.text_full = "Closing mailbox";
+
+       array_foreach(&box->conn->cmd_wait_list, cmdp) {
+               if ((*cmdp)->callback != NULL) {
+                       (*cmdp)->callback(&reply, (*cmdp)->context);
+                       (*cmdp)->callback = NULL;
+               }
+       }
+       array_foreach(&box->conn->cmd_send_queue, cmdp) {
+               if ((*cmdp)->callback != NULL) {
+                       (*cmdp)->callback(&reply, (*cmdp)->context);
+                       (*cmdp)->callback = NULL;
+               }
+       }
+
+       if (box->conn->selected_box == NULL &&
+           box->conn->selecting_box == NULL) {
+               i_assert(box->conn->state == IMAPC_CONNECTION_STATE_DISCONNECTED);
+       } else {
+               i_assert(box->conn->selected_box == box ||
+                        box->conn->selecting_box == box);
+
+               box->conn->selected_box = NULL;
+               box->conn->selecting_box = NULL;
+       }
+}
+
+struct imapc_client_mailbox *
+imapc_connection_get_mailbox(struct imapc_connection *conn)
+{
+       if (conn->selecting_box != NULL)
+               return conn->selecting_box;
+       return conn->selected_box;
 }
 
 static void
index 842b014ed703ed674442db1d2c01fac6e35834fb..3053613327c37eb046e0aa0d33349c72e372caf7 100644 (file)
@@ -22,6 +22,7 @@ imapc_connection_init(struct imapc_client *client);
 void imapc_connection_deinit(struct imapc_connection **conn);
 
 void imapc_connection_connect(struct imapc_connection *conn);
+void imapc_connection_disconnect(struct imapc_connection *conn);
 void imapc_connection_ioloop_changed(struct imapc_connection *conn);
 void imapc_connection_input_pending(struct imapc_connection *conn);
 
@@ -43,6 +44,9 @@ imapc_connection_get_state(struct imapc_connection *conn);
 enum imapc_capability
 imapc_connection_get_capabilities(struct imapc_connection *conn);
 
+struct imapc_client_mailbox *
+imapc_connection_get_mailbox(struct imapc_connection *conn);
+
 void imapc_connection_idle(struct imapc_connection *conn);
 
 #endif
index aced9e4c677c7f236fc57ecfda5763d1b8a6daf6..53d23f4ff090e9b37929f4408462c3f314d89057 100644 (file)
 
 #define NOTIFY_DELAY_MSECS 500
 
+static void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox,
+                                       const char *reason, ...)
+{
+       va_list va;
+
+       va_start(va, reason);
+       i_error("imapc: Mailbox '%s' state corrupted: %s",
+               mbox->box.name, t_strdup_vprintf(reason, va));
+       va_end(va);
+
+       mail_index_mark_corrupted(mbox->box.index);
+       imapc_client_mailbox_disconnect(mbox->client_box);
+}
+
 static void imapc_mailbox_init_delayed_trans(struct imapc_mailbox *mbox)
 {
        if (mbox->delayed_sync_trans != NULL)
                return;
 
+       mbox->sync_view = mail_index_view_open(mbox->box.index);
        mbox->delayed_sync_trans =
-               mail_index_transaction_begin(mbox->box.view,
+               mail_index_transaction_begin(mbox->sync_view,
                                        MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
        mbox->delayed_sync_view =
                mail_index_transaction_open_updated_view(mbox->delayed_sync_trans);
@@ -44,6 +59,7 @@ imapc_newmsgs_callback(const struct imapc_command_reply *reply,
 static void imapc_untagged_exists(const struct imapc_untagged_reply *reply,
                                  struct imapc_mailbox *mbox)
 {
+       struct mail_index_view *view = mbox->delayed_sync_view;
        uint32_t rcount = reply->num;
        const struct mail_index_header *hdr;
        struct imapc_seqmap *seqmap;
@@ -52,13 +68,16 @@ static void imapc_untagged_exists(const struct imapc_untagged_reply *reply,
        if (mbox == NULL)
                return;
 
+       if (view == NULL)
+               view = mbox->box.view;
+
        seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-       next_lseq = mail_index_view_get_messages_count(mbox->box.view) + 1;
+       next_lseq = mail_index_view_get_messages_count(view) + 1;
        next_rseq = imapc_seqmap_lseq_to_rseq(seqmap, next_lseq);
        if (next_rseq > rcount)
                return;
 
-       hdr = mail_index_get_header(mbox->box.view);
+       hdr = mail_index_get_header(view);
 
        mbox->new_msgs = TRUE;
        imapc_client_mailbox_cmdf(mbox->client_box, imapc_newmsgs_callback,
@@ -85,7 +104,7 @@ static void imapc_mailbox_idle_notify(struct imapc_mailbox *mbox)
 static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
                                 struct imapc_mailbox *mbox)
 {
-       uint32_t seq = reply->num;
+       uint32_t lseq, rseq = reply->num;
        struct imapc_seqmap *seqmap;
        const struct imap_arg *list, *flags_list;
        const char *atom;
@@ -96,7 +115,7 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
        ARRAY_TYPE(const_string) keywords = ARRAY_INIT;
        bool seen_flags = FALSE;
 
-       if (mbox == NULL || seq == 0 || !imap_arg_get_list(reply->args, &list))
+       if (mbox == NULL || rseq == 0 || !imap_arg_get_list(reply->args, &list))
                return;
 
        uid = 0; flags = 0;
@@ -126,26 +145,33 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
                        }
                }
        }
+       /* FIXME: need to do something about recent flags */
+       flags &= ~MAIL_RECENT;
 
        seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-       seq = imapc_seqmap_rseq_to_lseq(seqmap, seq);
+       lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq);
 
-       if (mbox->cur_fetch_mail != NULL && mbox->cur_fetch_mail->seq == seq) {
+       if (mbox->cur_fetch_mail != NULL && mbox->cur_fetch_mail->seq == lseq) {
                i_assert(uid == 0 || mbox->cur_fetch_mail->uid == uid);
                imapc_fetch_mail_update(mbox->cur_fetch_mail, reply, list);
        }
 
        imapc_mailbox_init_delayed_trans(mbox);
        old_count = mail_index_view_get_messages_count(mbox->delayed_sync_view);
-       if (seq > old_count) {
-               if (uid == 0)
+       if (lseq > old_count) {
+               if (uid == 0 || lseq != old_count + 1)
                        return;
-               i_assert(seq == old_count + 1);
-               mail_index_append(mbox->delayed_sync_trans, uid, &seq);
+               i_assert(lseq == old_count + 1);
+               mail_index_append(mbox->delayed_sync_trans, uid, &lseq);
+       }
+       rec = mail_index_lookup(mbox->delayed_sync_view, lseq);
+       if (rec->uid != uid && uid != 0) {
+               imapc_mailbox_set_corrupted(mbox, "Message UID changed %u -> %u",
+                                           rec->uid, uid);
+               return;
        }
-       rec = mail_index_lookup(mbox->delayed_sync_view, seq);
        if (seen_flags && rec->flags != flags) {
-               mail_index_update_flags(mbox->delayed_sync_trans, seq,
+               mail_index_update_flags(mbox->delayed_sync_trans, lseq,
                                        MODIFY_REPLACE, flags);
        }
        if (seen_flags) {
@@ -154,7 +180,7 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
                (void)array_append_space(&keywords);
                kw = mail_index_keywords_create(mbox->box.index,
                                                array_idx(&keywords, 0));
-               mail_index_update_keywords(mbox->delayed_sync_trans, seq,
+               mail_index_update_keywords(mbox->delayed_sync_trans, lseq,
                                           MODIFY_REPLACE, kw);
                mail_index_keywords_unref(&kw);
        }
@@ -166,16 +192,28 @@ static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply,
 {
        struct imapc_seqmap *seqmap;
        uint32_t lseq, rseq = reply->num;
-
+       
        if (mbox == NULL || rseq == 0)
                return;
 
+       imapc_mailbox_init_delayed_trans(mbox);
+
        seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
        lseq = imapc_seqmap_rseq_to_lseq(seqmap, rseq);
-       imapc_seqmap_expunge(seqmap, rseq);
 
-       imapc_mailbox_init_delayed_trans(mbox);
-       mail_index_expunge(mbox->delayed_sync_trans, lseq);
+       if (lseq <= mail_index_view_get_messages_count(mbox->sync_view)) {
+               /* expunging a message in index */
+               imapc_seqmap_expunge(seqmap, rseq);
+               mail_index_expunge(mbox->delayed_sync_trans, lseq);
+               i_assert(array_count(&mbox->delayed_sync_trans->expunges) > 0);
+       } else if (lseq <= mail_index_view_get_messages_count(mbox->delayed_sync_view)) {
+               /* expunging a message that was added to transaction,
+                  but not yet committed. expunging it here takes
+                  effect immediately. */
+               mail_index_expunge(mbox->delayed_sync_trans, lseq);
+       } else {
+               /* expunging a message whose UID wasn't known yet */
+       }
 
        imapc_mailbox_idle_notify(mbox);
 }
index 61f4640b853be52a91897d1ccd7868f0e8f91911..47e9b12b47b4e160637cdc378023764892b21abb 100644 (file)
@@ -172,7 +172,7 @@ static int imapc_save_append(struct imapc_save_context *ctx)
                string_t *str = t_str_new(64);
 
                str_append(str, " (");
-               imap_write_flags(str, _ctx->flags, NULL);
+               imap_write_flags(str, _ctx->flags & ~MAIL_RECENT, NULL);
                if (_ctx->keywords != NULL)
                        imapc_append_keywords(str, _ctx->keywords);
                str_append_c(str, ')');
index baa6db766a61803751a5501f05b89be6fc3dbbbd..7022080d77d9ffd5e5990501453c97d83985b25d 100644 (file)
@@ -36,6 +36,12 @@ void imapc_seqmap_reset(struct imapc_seqmap *seqmap)
        array_clear(&seqmap->expunges);
 }
 
+bool imapc_seqmap_is_reset(struct imapc_seqmap *seqmap)
+{
+       return array_count(&seqmap->queue) == 0 &&
+               array_count(&seqmap->expunges) == 0;
+}
+
 void imapc_seqmap_expunge(struct imapc_seqmap *seqmap, uint32_t rseq)
 {
        i_assert(rseq > 0);
index 7c7bd19ecbd2d6541dce4b989859a22c9f0d3631..7e8d4a49e5dcbfc1101b0ea221562a2420e92990 100644 (file)
@@ -15,6 +15,7 @@ void imapc_seqmap_deinit(struct imapc_seqmap **seqmap);
 
 /* Reset local and remote sequences to be equal. */
 void imapc_seqmap_reset(struct imapc_seqmap *seqmap);
+bool imapc_seqmap_is_reset(struct imapc_seqmap *seqmap);
 
 /* Mark given remote sequence expunged. */
 void imapc_seqmap_expunge(struct imapc_seqmap *seqmap, uint32_t rseq);
index 86d638b56831734e8baa43dd7a3f36121d7bfb64..cf183144e4c00d3783815fbda5fb5d786dd95ffd 100644 (file)
@@ -72,7 +72,7 @@ static struct mail_storage *imapc_storage_alloc(void)
        struct imapc_storage *storage;
        pool_t pool;
 
-       pool = pool_alloconly_create("imapc storage", 512+256);
+       pool = pool_alloconly_create("imapc storage", 2048);
        storage = p_new(pool, struct imapc_storage, 1);
        storage->storage = imapc_storage;
        storage->storage.pool = pool;
@@ -341,6 +341,8 @@ static void imapc_mailbox_close(struct mailbox *box)
                if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0)
                        mail_storage_set_index_error(&mbox->box);
        }
+       if (mbox->sync_view != NULL)
+               mail_index_view_close(&mbox->sync_view);
        if (mbox->to_idle_delay != NULL)
                timeout_remove(&mbox->to_idle_delay);
        if (mbox->to_idle_check != NULL)
@@ -523,6 +525,16 @@ static void imapc_notify_changes(struct mailbox *box)
        }
 }
 
+static bool imapc_is_inconsistent(struct mailbox *box)
+{
+       struct imapc_mailbox *mbox = (struct imapc_mailbox *)box;
+
+       if (mail_index_view_is_inconsistent(box->view))
+               return TRUE;
+
+       return !imapc_client_mailbox_is_connected(mbox->client_box);
+}
+
 struct mail_storage imapc_storage = {
        .name = IMAPC_STORAGE_NAME,
        .class_flags = MAIL_STORAGE_CLASS_FLAG_NO_ROOT,
@@ -577,6 +589,6 @@ struct mailbox imapc_mailbox = {
                imapc_save_finish,
                imapc_save_cancel,
                imapc_copy,
-               index_storage_is_inconsistent
+               imapc_is_inconsistent
        }
 };
index 98d298e0ed8910cd9eabe05e9747bdef028d10a2..0ded1a2bbf29ce9e82dc2a1a1afca62b845df157 100644 (file)
@@ -45,7 +45,7 @@ struct imapc_mailbox {
        struct imapc_client_mailbox *client_box;
 
        struct mail_index_transaction *delayed_sync_trans;
-       struct mail_index_view *delayed_sync_view;
+       struct mail_index_view *sync_view, *delayed_sync_view;
        struct timeout *to_idle_check, *to_idle_delay;
 
        struct mail *cur_fetch_mail;
index 18121063a5b007c3a499619e613bd5ba942d581f..c617ee6c4f52f8942bc1f98eb172a50eb3443163 100644 (file)
@@ -22,9 +22,15 @@ static void imapc_sync_callback(const struct imapc_command_reply *reply,
        else if (reply->state == IMAPC_COMMAND_STATE_NO) {
                /* maybe the message was expunged already.
                   some servers fail STOREs with NO in such situation. */
+       } else if (reply->state == IMAPC_COMMAND_STATE_DISCONNECTED) {
+               /* the disconnection is already logged, don't flood
+                  the logs unnecessarily */
+               mail_storage_set_internal_error(&ctx->mbox->storage->storage);
+               ctx->failed = TRUE;
        } else {
                mail_storage_set_critical(&ctx->mbox->storage->storage,
                        "imapc: Sync command failed: %s", reply->text_full);
+               ctx->failed = TRUE;
        }
        
        if (--ctx->sync_command_count == 0)
@@ -71,6 +77,7 @@ static void imapc_sync_index_flags(struct imapc_sync_context *ctx,
        i_assert(sync_rec->type == MAIL_INDEX_SYNC_TYPE_FLAGS);
 
        if (sync_rec->add_flags != 0) {
+               i_assert((sync_rec->add_flags & MAIL_RECENT) == 0);
                str_printfa(str, "UID STORE %u:%u +FLAGS (",
                            sync_rec->uid1, sync_rec->uid2);
                imap_write_flags(str, sync_rec->add_flags, NULL);
@@ -79,6 +86,7 @@ static void imapc_sync_index_flags(struct imapc_sync_context *ctx,
        }
 
        if (sync_rec->remove_flags != 0) {
+               i_assert((sync_rec->remove_flags & MAIL_RECENT) == 0);
                str_truncate(str, 0);
                str_printfa(str, "UID STORE %u:%u -FLAGS (",
                            sync_rec->uid1, sync_rec->uid2);
@@ -126,6 +134,7 @@ imapc_sync_index_keyword_reset(struct imapc_sync_context *ctx,
 
        for (seq = seq1; seq <= seq2; seq++) {
                rec = mail_index_lookup(ctx->sync_view, seq);
+               i_assert((rec->flags & MAIL_RECENT) == 0);
 
                str_truncate(str, 0);
                str_printfa(str, "UID STORE %u FLAGS (", rec->uid);
@@ -202,7 +211,7 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
        } T_END;
 
        imapc_sync_expunge_finish(ctx);
-       if (ctx->sync_command_count > 0)
+       while (ctx->sync_command_count > 0)
                imapc_client_run(ctx->mbox->storage->client);
        array_free(&ctx->expunged_uids);
 
@@ -238,25 +247,28 @@ imapc_sync_begin(struct imapc_mailbox *mbox,
        }
 
        i_assert(mbox->delayed_sync_trans == NULL);
-       mbox->delayed_sync_view = ctx->sync_view;
+       mbox->sync_view = ctx->sync_view;
+       mbox->delayed_sync_view =
+               mail_index_transaction_open_updated_view(ctx->trans);
        mbox->delayed_sync_trans = ctx->trans;
 
        imapc_sync_index(ctx);
 
-       mbox->delayed_sync_view = NULL;
+       mail_index_view_close(&mbox->delayed_sync_view);
        mbox->delayed_sync_trans = NULL;
+       mbox->sync_view = NULL;
 
        *ctx_r = ctx;
        return 0;
 }
 
-static int imapc_sync_finish(struct imapc_sync_context **_ctx, bool success)
+static int imapc_sync_finish(struct imapc_sync_context **_ctx)
 {
        struct imapc_sync_context *ctx = *_ctx;
-       int ret = success ? 0 : -1;
+       int ret = ctx->failed ? -1 : 0;
 
        *_ctx = NULL;
-       if (success) {
+       if (ret == 0) {
                if (mail_index_sync_commit(&ctx->index_sync_ctx) < 0) {
                        mail_storage_set_index_error(&ctx->mbox->box);
                        ret = -1;
@@ -271,12 +283,25 @@ static int imapc_sync_finish(struct imapc_sync_context **_ctx, bool success)
 static int imapc_sync(struct imapc_mailbox *mbox)
 {
        struct imapc_sync_context *sync_ctx;
+       struct imapc_seqmap *seqmap;
+
+       /* if there are any pending expunges, they're now committed. syncing
+          will return a view where they no longer exist, so reset the seqmap
+          before syncing. */
+       seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
+       imapc_seqmap_reset(seqmap);
 
        if (imapc_sync_begin(mbox, &sync_ctx, FALSE) < 0)
                return -1;
+       if (sync_ctx == NULL)
+               return 0;
+       if (imapc_sync_finish(&sync_ctx) < 0)
+               return -1;
 
-       return sync_ctx == NULL ? 0 :
-               imapc_sync_finish(&sync_ctx, TRUE);
+       /* syncing itself may have also seen new expunges, which are also now
+          committed and synced. reset the seqmap again. */
+       imapc_seqmap_reset(seqmap);
+       return 0;
 }
 
 struct mailbox_sync_context *
@@ -284,6 +309,7 @@ imapc_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
 {
        struct imapc_mailbox *mbox = (struct imapc_mailbox *)box;
        enum imapc_capability capabilities;
+       bool changes = FALSE;
        int ret = 0;
 
        if (!box->opened) {
@@ -305,26 +331,29 @@ imapc_mailbox_sync_init(struct mailbox *box, enum mailbox_sync_flags flags)
                mail_index_view_close(&mbox->delayed_sync_view);
        if (mbox->delayed_sync_trans != NULL) {
                if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) {
-                       // FIXME: mark inconsistent
                        mail_storage_set_index_error(&mbox->box);
                        ret = -1;
                }
+               changes = TRUE;
        }
+       if (mbox->sync_view != NULL)
+               mail_index_view_close(&mbox->sync_view);
 
-       if (index_mailbox_want_full_sync(&mbox->box, flags) && ret == 0)
+       if ((changes || index_mailbox_want_full_sync(&mbox->box, flags)) &&
+           ret == 0)
                ret = imapc_sync(mbox);
 
+       if (changes && ret < 0) {
+               /* we're now out of sync and can't safely continue */
+               mail_index_mark_corrupted(mbox->box.index);
+       }
        return index_mailbox_sync_init(box, flags, ret < 0);
 }
 
 int imapc_mailbox_sync_deinit(struct mailbox_sync_context *ctx,
                              struct mailbox_sync_status *status_r)
 {
-       struct index_mailbox_sync_context *ictx =
-               (struct index_mailbox_sync_context *)ctx;
        struct imapc_mailbox *mbox = (struct imapc_mailbox *)ctx->box;
-       enum mailbox_sync_flags flags = ictx->flags;
-       struct imapc_seqmap *seqmap;
        int ret;
 
        ret = index_mailbox_sync_deinit(ctx, status_r);
@@ -333,10 +362,6 @@ int imapc_mailbox_sync_deinit(struct mailbox_sync_context *ctx,
        if (mbox->client_box == NULL)
                return ret;
 
-       if ((flags & MAILBOX_SYNC_FLAG_NO_EXPUNGES) == 0 && ret == 0) {
-               seqmap = imapc_client_mailbox_get_seqmap(mbox->client_box);
-               imapc_seqmap_reset(seqmap);
-       }
        imapc_client_mailbox_idle(mbox->client_box);
        return ret;
 }
index c56bcdc10b67b25ad70ad4e4fd66d97b5306073c..d83b240a058ca8b413a4f5708de611e8efd6d886 100644 (file)
@@ -12,6 +12,8 @@ struct imapc_sync_context {
        const ARRAY_TYPE(keywords) *keywords;
        ARRAY_TYPE(seq_range) expunged_uids;
        unsigned int sync_command_count;
+
+       unsigned int failed:1;
 };
 
 struct mailbox_sync_context *