]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
Finished making full text search indexes work fast with virtual mailboxes.
authorTimo Sirainen <tss@iki.fi>
Sat, 29 Nov 2008 20:39:44 +0000 (22:39 +0200)
committerTimo Sirainen <tss@iki.fi>
Sat, 29 Nov 2008 20:39:44 +0000 (22:39 +0200)
--HG--
branch : HEAD

21 files changed:
src/lib-storage/index/cydir/cydir-storage.c
src/lib-storage/index/dbox/dbox-storage.c
src/lib-storage/index/maildir/maildir-storage.c
src/lib-storage/index/mbox/mbox-storage.c
src/lib-storage/index/raw/raw-storage.c
src/lib-storage/mail-storage-private.h
src/lib-storage/mail-storage.c
src/lib-storage/mail-storage.h
src/plugins/fts-lucene/fts-backend-lucene.c
src/plugins/fts-solr/fts-backend-solr.c
src/plugins/fts-solr/solr-connection.c
src/plugins/fts-solr/solr-connection.h
src/plugins/fts-squat/fts-backend-squat.c
src/plugins/fts/fts-api-private.h
src/plugins/fts/fts-api.c
src/plugins/fts/fts-api.h
src/plugins/fts/fts-storage.c
src/plugins/fts/fts-storage.h
src/plugins/virtual/virtual-config.c
src/plugins/virtual/virtual-storage.c
src/plugins/virtual/virtual-storage.h

index 97908749e398dacdad9e4d0eee2a9b7fbfe6e394..682a0d90e5b529616e95dc05d2766ef0d6932bbf 100644 (file)
@@ -447,6 +447,8 @@ struct mailbox cydir_mailbox = {
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
                NULL,
+               NULL,
+               NULL,
                index_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index 2670ce1c3e1f1b86ffe3e3b21e612ceaf94fc11d..132b4503a8704f00325f3a6f25e4235084a2d042 100644 (file)
@@ -718,6 +718,8 @@ struct mailbox dbox_mailbox = {
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
                NULL,
+               NULL,
+               NULL,
                dbox_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index 33f5aac40e3359e3ebfebe1fbcc515be659eb765..224f4d4686669f79cfbfdf6d71987c8c84c0a249 100644 (file)
@@ -1106,6 +1106,8 @@ struct mailbox maildir_mailbox = {
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
                NULL,
+               NULL,
+               NULL,
                index_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index 642d45f6cd92cc2dac882aea9e09452d6864cea0..0f5ed87fa143f5388d98b78e568c536a3274e15d 100644 (file)
@@ -1028,6 +1028,8 @@ struct mailbox mbox_mailbox = {
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
                NULL,
+               NULL,
+               NULL,
                index_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index e6694892e07c6eecb4c657218bfaa300dfb5bb35..d0c48fb15e8bfd13e59f55f98f1c9012b3121b7b 100644 (file)
@@ -297,6 +297,8 @@ struct mailbox raw_mailbox = {
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
                NULL,
+               NULL,
+               NULL,
                index_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index ece194da0627d63244df6c8442856f1454d78673..7d0f0d24d6a7a3701cab01bb046c2cb44cbe9e85 100644 (file)
@@ -142,6 +142,12 @@ struct mailbox_vfuncs {
                                const char *backend_mailbox,
                                uint32_t backend_uidvalidity,
                                uint32_t backend_uid, uint32_t *uid_r);
+       void (*get_virtual_backend_boxes)(struct mailbox *box,
+                                         ARRAY_TYPE(mailboxes) *mailboxes,
+                                         bool only_with_msgs);
+       void (*get_virtual_box_patterns)(struct mailbox *box,
+                                        ARRAY_TYPE(const_string) *includes,
+                                        ARRAY_TYPE(const_string) *excludes);
 
        struct mail *
                (*mail_alloc)(struct mailbox_transaction_context *t,
index a7aa44d451d735741f34cc18ad07f9da52de08b1..9fe26e6bc49e4500027eb3d10bc1c3e17998e4e8 100644 (file)
@@ -640,6 +640,29 @@ bool mailbox_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
                                      backend_uid, uid_r);
 }
 
+void mailbox_get_virtual_backend_boxes(struct mailbox *box,
+                                      ARRAY_TYPE(mailboxes) *mailboxes,
+                                      bool only_with_msgs)
+{
+       if (box->v.get_virtual_backend_boxes == NULL)
+               array_append(mailboxes, &box, 1);
+       else
+               box->v.get_virtual_backend_boxes(box, mailboxes, only_with_msgs);
+}
+
+void mailbox_get_virtual_box_patterns(struct mailbox *box,
+                                     ARRAY_TYPE(const_string) *includes,
+                                     ARRAY_TYPE(const_string) *excludes)
+{
+       if (box->v.get_virtual_box_patterns == NULL) {
+               const char *name = box->name;
+
+               array_append(includes, &name, 1);
+       } else {
+               box->v.get_virtual_box_patterns(box, includes, excludes);
+       }
+}
+
 struct mailbox_header_lookup_ctx *
 mailbox_header_lookup_init(struct mailbox *box, const char *const headers[])
 {
index 1e1ec3e2b54f588a538b09b10e776823c1e1708b..1b9f67010438410e684be168e6e49ef413b2d8d3 100644 (file)
@@ -234,6 +234,8 @@ struct mail_storage_callbacks {
 
 };
 
+ARRAY_DEFINE_TYPE(mailboxes, struct mailbox *);
+
 typedef void mailbox_notify_callback_t(struct mailbox *box, void *context);
 
 void mail_storage_init(void);
@@ -421,6 +423,17 @@ bool mailbox_get_expunged_uids(struct mailbox *box, uint64_t modseq,
 bool mailbox_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
                             uint32_t backend_uidvalidity,
                             uint32_t backend_uid, uint32_t *uid_r);
+/* If box is a virtual mailbox, return all backend mailboxes. If
+   only_with_msgs=TRUE, return only those mailboxes that have at least one
+   message existing in the virtual mailbox. */
+void mailbox_get_virtual_backend_boxes(struct mailbox *box,
+                                      ARRAY_TYPE(mailboxes) *mailboxes,
+                                      bool only_with_msgs);
+/* If mailbox is a virtual mailbox, return all mailbox list patterns that
+   are used to figure out which mailboxes belong to the virtual mailbox. */
+void mailbox_get_virtual_box_patterns(struct mailbox *box,
+                                     ARRAY_TYPE(const_string) *includes,
+                                     ARRAY_TYPE(const_string) *excludes);
 
 /* Initialize header lookup for given headers. */
 struct mailbox_header_lookup_ctx *
index 0d1bd49809828a6f25f5aad05bb5a5e4556d98e2..178f3e6beadd4b40bd641650a43fbce66b09ec1f 100644 (file)
@@ -210,6 +210,7 @@ struct fts_backend fts_backend_lucene = {
                fts_backend_lucene_init,
                fts_backend_lucene_deinit,
                fts_backend_lucene_get_last_uid,
+               NULL,
                fts_backend_lucene_build_init,
                fts_backend_lucene_build_more,
                fts_backend_lucene_build_deinit,
index 93271278a2ed7be048b947dbe2b81b513a9e8bb0..33f27393ddc4b7d9d94e6fe2f6d151cc0cf78e68 100644 (file)
@@ -3,12 +3,13 @@
 #include "lib.h"
 #include "array.h"
 #include "str.h"
+#include "strescape.h"
 #include "mail-storage-private.h"
 #include "mail-namespace.h"
 #include "solr-connection.h"
 #include "fts-solr-plugin.h"
 
-#include <curl/curl.h>
+#include <ctype.h>
 
 #define SOLR_CMDBUF_SIZE (1024*64)
 #define SOLR_MAX_ROWS 100000
@@ -27,6 +28,11 @@ struct solr_fts_backend_build_context {
        bool headers;
 };
 
+struct fts_backend_solr_get_last_uids_context {
+       pool_t pool;
+       ARRAY_TYPE(fts_backend_uid_map) *last_uids;
+};
+
 static struct solr_connection *solr_conn = NULL;
 
 static void solr_quote_str(string_t *dest, const char *str)
@@ -199,6 +205,120 @@ static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
        return 0;
 }
 
+static bool
+solr_virtual_get_last_uids(const char *mailbox, uint32_t uidvalidity,
+                          uint32_t *uid, void *context)
+{
+       struct fts_backend_solr_get_last_uids_context *ctx = context;
+       struct fts_backend_uid_map *map;
+
+       map = array_append_space(ctx->last_uids);
+       map->mailbox = p_strdup(ctx->pool, mailbox);
+       map->uidvalidity = uidvalidity;
+       map->uid = *uid;
+       return FALSE;
+}
+
+static void add_pattern_as_solr(string_t *str, const char *pattern)
+{
+       const char *p;
+
+       /* first check if there are any wildcards in the pattern */
+       for (p = pattern; *p != '\0'; p++) {
+               if (*p == '%' || *p == '*')
+                       break;
+       }
+       if (*p == '\0') {
+               /* full mailbox name */
+               str_append_c(str, '"');
+               str_append(str, str_escape(pattern));
+               str_append_c(str, '"');
+               return;
+       }
+
+       /* there are at least some wildcards. */
+       for (p = pattern; *p != '\0'; p++) {
+               if (*p == '%' || *p == '*') {
+                       if (p == pattern || (p[-1] != '%' && p[-1] != '*'))
+                               str_append_c(str, '*');
+               } else {
+                       if (!i_isalnum(*p))
+                               str_append_c(str, '\\');
+                       str_append_c(str, *p);
+               }
+       }
+}
+
+static void
+fts_backend_solr_filter_mailboxes(struct solr_connection *solr_conn,
+                                 string_t *str, struct mailbox *box)
+{
+       ARRAY_TYPE(const_string) includes_arr, excludes_arr;
+       const char *const *includes, *const *excludes;
+       unsigned int i, inc_count, exc_count;
+       string_t *fq;
+
+       t_array_init(&includes_arr, 16);
+       t_array_init(&excludes_arr, 16);
+       mailbox_get_virtual_box_patterns(box, &includes_arr, &excludes_arr);
+       includes = array_get(&includes_arr, &inc_count);
+       excludes = array_get(&excludes_arr, &exc_count);
+       i_assert(inc_count > 0);
+
+       /* First see if there are any patterns that begin with a wildcard.
+          Solr doesn't allow them, so in that case we'll need to return
+          all mailboxes. */
+       for (i = 0; i < inc_count; i++) {
+               if (*includes[i] == '*' || *includes[i] == '%')
+                       break;
+       }
+
+       fq = t_str_new(128);
+       if (i == inc_count) {
+               /* we can filter what mailboxes we want returned */
+               str_append_c(fq, '(');
+               for (i = 0; i < inc_count; i++) {
+                       if (i != 0)
+                               str_append(fq, " OR ");
+                       str_append(fq, "box:");
+                       add_pattern_as_solr(fq, includes[i]);
+               }
+               str_append_c(fq, ')');
+       }
+       for (i = 0; i < exc_count; i++) {
+               if (str_len(fq) > 0)
+                       str_append_c(fq, ' ');
+               str_append(fq, "-box:");
+               add_pattern_as_solr(fq, excludes[i]);
+       }
+       if (str_len(fq) > 0) {
+               str_append(str, "&fq=");
+               solr_connection_http_escape(solr_conn, str, str_c(fq));
+       }
+}
+
+static int
+fts_backend_solr_get_all_last_uids(struct fts_backend *backend, pool_t pool,
+                                  ARRAY_TYPE(fts_backend_uid_map) *last_uids)
+{
+       struct fts_backend_solr_get_last_uids_context ctx;
+       string_t *str;
+
+       memset(&ctx, 0, sizeof(ctx));
+       ctx.pool = pool;
+       ctx.last_uids = last_uids;
+
+       str = t_str_new(256);
+       str_printfa(str, "fl=uid,box,uidv&rows=%u&q=last_uid:TRUE%%20user:",
+                   SOLR_MAX_ROWS);
+       solr_quote_str(str, backend->box->storage->ns->user->username);
+       fts_backend_solr_filter_mailboxes(solr_conn, str, backend->box);
+
+       return solr_connection_select(solr_conn, str_c(str),
+                                     solr_virtual_get_last_uids, &ctx,
+                                     NULL, NULL);
+}
+
 static int
 fts_backend_solr_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
                            struct fts_backend_build_context **ctx_r)
@@ -210,7 +330,6 @@ fts_backend_solr_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
 
        ctx = i_new(struct solr_fts_backend_build_context, 1);
        ctx->ctx.backend = backend;
-       ctx->post = solr_connection_post_begin(solr_conn);
        ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
 
        mailbox_get_status(backend->box, STATUS_UIDVALIDITY, &status);
@@ -251,10 +370,12 @@ fts_backend_solr_build_more(struct fts_backend_build_context *_ctx,
        /* body comes first, then headers */
        if (ctx->prev_uid != uid) {
                /* uid changed */
-               if (ctx->prev_uid == 0)
+               if (ctx->post == NULL) {
+                       ctx->post = solr_connection_post_begin(solr_conn);
                        str_append(cmd, "<add>");
-               else
+               } else {
                        str_append(cmd, "</field></doc>");
+               }
                ctx->prev_uid = uid;
 
                fts_backend_solr_add_doc_prefix(ctx, uid);
@@ -292,7 +413,7 @@ fts_backed_solr_build_commit(struct solr_fts_backend_build_context *ctx)
        struct mailbox *box = ctx->ctx.backend->box;
        int ret;
 
-       if (ctx->prev_uid == 0)
+       if (ctx->post == NULL)
                return 0;
 
        str_append(ctx->cmd, "</field></doc>");
@@ -435,9 +556,9 @@ static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
           affect the score and there could be some caching benefits too. */
        str_append(str, "&fq=user:");
        solr_quote_str(str, box->storage->ns->user->username);
-
-       /* FIXME: limit what mailboxes to search with virtual storage */
-       if (!virtual) {
+       if (virtual)
+               fts_backend_solr_filter_mailboxes(solr_conn, str, box);
+       else {
                str_printfa(str, "%%20uidv:%u%%20box:", status.uidvalidity);
                solr_quote_str(str, box->name);
        }
@@ -461,6 +582,7 @@ struct fts_backend fts_backend_solr = {
                fts_backend_solr_init,
                fts_backend_solr_deinit,
                fts_backend_solr_get_last_uid,
+               fts_backend_solr_get_all_last_uids,
                fts_backend_solr_build_init,
                fts_backend_solr_build_more,
                fts_backend_solr_build_deinit,
index 3a35e7d83a8adb8c6fcd3a8463536d37c808b0cc..7ea856ede74d2a6a61722d289ba6369e32273970 100644 (file)
@@ -202,6 +202,16 @@ void solr_connection_quote_str(struct solr_connection *conn, string_t *dest,
        curl_free(encoded);
 }
 
+void solr_connection_http_escape(struct solr_connection *conn, string_t *dest,
+                                const char *str)
+{
+       char *encoded;
+
+       encoded = curl_easy_escape(conn->curl, str, 0);
+       str_append(dest, encoded);
+       curl_free(encoded);
+}
+
 static const char *attrs_get_name(const char **attrs)
 {
        for (; *attrs != NULL; attrs += 2) {
index f2ffc7f20545ccfc6577908dea58ceae5d4e9470..7d0220b2535bb513db98917920930c253ba6eb6c 100644 (file)
@@ -13,6 +13,8 @@ void solr_connection_deinit(struct solr_connection *conn);
 
 void solr_connection_quote_str(struct solr_connection *conn, string_t *dest,
                               const char *str);
+void solr_connection_http_escape(struct solr_connection *conn, string_t *dest,
+                                const char *str);
 
 int solr_connection_select(struct solr_connection *conn, const char *query,
                           solr_uid_map_callback_t *callback, void *context,
index bc3aeda0c5bc0a4bd905c4c4833588a6512be6f3..be7cf41081a0da8f8e4b7ca2dc96272577206d35 100644 (file)
@@ -247,6 +247,7 @@ struct fts_backend fts_backend_squat = {
                fts_backend_squat_init,
                fts_backend_squat_deinit,
                fts_backend_squat_get_last_uid,
+               NULL,
                fts_backend_squat_build_init,
                fts_backend_squat_build_more,
                fts_backend_squat_build_deinit,
index 974ef4b79b078bbe8fca1e0bd8c99a517e6192a8..261640ea97df65afc091285ff8aaea7a3efbf2cf 100644 (file)
@@ -8,6 +8,8 @@ struct fts_backend_vfuncs {
        void (*deinit)(struct fts_backend *backend);
 
        int (*get_last_uid)(struct fts_backend *backend, uint32_t *last_uid_r);
+       int (*get_all_last_uids)(struct fts_backend *backend, pool_t pool,
+                                ARRAY_TYPE(fts_backend_uid_map) *last_uids);
 
        int (*build_init)(struct fts_backend *backend, uint32_t *last_uid_r,
                          struct fts_backend_build_context **ctx_r);
index 814e1b53cda8ada2dd327b92e6b141a65f9d7085..450a9160cab0c645d0fd91651028387114f0db2f 100644 (file)
@@ -80,6 +80,12 @@ int fts_backend_get_last_uid(struct fts_backend *backend, uint32_t *last_uid_r)
        return backend->v.get_last_uid(backend, last_uid_r);
 }
 
+int fts_backend_get_all_last_uids(struct fts_backend *backend, pool_t pool,
+                                 ARRAY_TYPE(fts_backend_uid_map) *last_uids)
+{
+       return backend->v.get_all_last_uids(backend, pool, last_uids);
+}
+
 int fts_backend_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
                           struct fts_backend_build_context **ctx_r)
 {
index 9882ac15f255ea71236a130304e6cc25ae4a4791..8bed62e4cd308c493c866d42e5b42f80b186f9da 100644 (file)
@@ -13,6 +13,13 @@ enum fts_lookup_flags {
        FTS_LOOKUP_FLAG_INVERT  = 0x04
 };
 
+struct fts_backend_uid_map {
+       const char *mailbox;
+       uint32_t uidvalidity;
+       uint32_t uid;
+};
+ARRAY_DEFINE_TYPE(fts_backend_uid_map, struct fts_backend_uid_map);
+
 struct fts_score_map {
        uint32_t uid;
        float score;
@@ -23,8 +30,14 @@ struct fts_backend *
 fts_backend_init(const char *backend_name, struct mailbox *box);
 void fts_backend_deinit(struct fts_backend **backend);
 
-/* Get the last_uid. */
+/* Get the last_uid for the mailbox. */
 int fts_backend_get_last_uid(struct fts_backend *backend, uint32_t *last_uid_r);
+/* Get last_uids for all mailboxes that might be backend mailboxes for a
+   virtual mailbox. Depending on virtual mailbox configuration, this function
+   may also return mailboxes that don't really even match the virtual mailbox
+   patterns. The caller should filter out the list itself. */
+int fts_backend_get_all_last_uids(struct fts_backend *backend, pool_t pool,
+                                 ARRAY_TYPE(fts_backend_uid_map) *last_uids);
 
 /* Initialize adding new data to the index. last_uid_r is set to the last UID
    that exists in the index. */
index b8dd646ae9001433b569c5428ed5d23d52703eef..ffa3a9024f52c1f2a28a914deacd0c124e5bf436 100644 (file)
@@ -170,33 +170,21 @@ static int fts_build_mail(struct fts_storage_build_context *ctx, uint32_t uid)
        return ret;
 }
 
-static int fts_build_init(struct fts_search_context *fctx)
+static int fts_build_init_seq(struct fts_search_context *fctx,
+                             struct fts_backend *backend,
+                             struct mailbox_transaction_context *t,
+                             uint32_t seq1, uint32_t seq2, uint32_t last_uid)
 {
-       struct mailbox_transaction_context *t = fctx->t;
        struct mail_search_args *search_args;
-       struct fts_backend *backend = fctx->build_backend;
        struct fts_storage_build_context *ctx;
        struct fts_backend_build_context *build;
-       uint32_t last_uid, last_uid_locked, seq1, seq2;
-
-       if (fctx->fbox->virtual) {
-               /* FIXME: update all mailboxes */
-               return 0;
-       }
-
-       if (fts_backend_get_last_uid(backend, &last_uid) < 0)
-               return -1;
-
-       mailbox_get_seq_range(t->box, last_uid+1, (uint32_t)-1, &seq1, &seq2);
-       fctx->first_nonindexed_seq = seq1;
-       if (seq1 == 0) {
-               /* no new messages */
-               return 0;
-       }
+       uint32_t last_uid_locked;
 
        if (fctx->best_arg->type == SEARCH_HEADER ||
            fctx->best_arg->type == SEARCH_HEADER_COMPRESS_LWSP) {
                /* we're not updating the index just for header lookups */
+               if (seq1 < fctx->first_nonindexed_seq)
+                       fctx->first_nonindexed_seq = seq1;
                return 0;
        }
 
@@ -204,12 +192,9 @@ static int fts_build_init(struct fts_search_context *fctx)
                return -1;
        if (last_uid != last_uid_locked && last_uid_locked != (uint32_t)-1) {
                /* changed, need to get again the sequences */
-               i_assert(last_uid < last_uid_locked);
-
                last_uid = last_uid_locked;
                mailbox_get_seq_range(t->box, last_uid+1, (uint32_t)-1,
                                      &seq1, &seq2);
-               fctx->first_nonindexed_seq = seq1;
                if (seq1 == 0) {
                        /* no new messages */
                        (void)fts_backend_build_deinit(&build);
@@ -228,7 +213,176 @@ static int fts_build_init(struct fts_search_context *fctx)
        ctx->search_args = search_args;
 
        fctx->build_ctx = ctx;
-       return 0;
+       return 1;
+}
+
+static struct fts_backend *
+fts_mailbox_get_backend(struct fts_search_context *fctx,
+                       struct mailbox *box)
+{
+       struct fts_mailbox *fbox = FTS_CONTEXT(box);
+
+       if (fctx->build_backend == fctx->fbox->backend_fast)
+               return fbox->backend_fast;
+       else {
+               i_assert(fctx->build_backend == fctx->fbox->backend_substr);
+               return fbox->backend_substr;
+       }
+}
+
+static int fts_build_init_trans(struct fts_search_context *fctx,
+                               struct mailbox_transaction_context *t)
+{
+       struct fts_backend *backend;
+       uint32_t last_uid, seq1, seq2;
+       int ret;
+
+       backend = fts_mailbox_get_backend(fctx, t->box);
+       if (fts_backend_get_last_uid(backend, &last_uid) < 0)
+               return -1;
+
+       mailbox_get_seq_range(t->box, last_uid+1, (uint32_t)-1, &seq1, &seq2);
+       if (seq1 == 0) {
+               /* no new messages */
+               return 0;
+       }
+
+       ret = fts_build_init_seq(fctx, backend, t, seq1, seq2, last_uid);
+       return ret < 0 ? -1 : 0;
+}
+
+static int
+fts_build_init_box(struct fts_search_context *fctx, struct mailbox *box,
+                  uint32_t last_uid)
+{
+       struct fts_backend *backend;
+       uint32_t seq1, seq2;
+
+       mailbox_get_seq_range(box, last_uid + 1, (uint32_t)-1, &seq1, &seq2);
+       if (seq1 == 0)
+               return 0;
+
+       backend = fts_mailbox_get_backend(fctx, box);
+       fctx->virtual_ctx.trans = mailbox_transaction_begin(box, 0);
+       return fts_build_init_seq(fctx, backend, fctx->virtual_ctx.trans,
+                                 seq1, seq2, last_uid);
+}
+
+static int mailbox_name_cmp(const void *p1, const void *p2)
+{
+       struct mailbox *const *box1 = p1, *const *box2 = p2;
+
+       return strcmp((*box1)->name, (*box2)->name);
+}
+
+static int fts_backend_uid_map_mailbox_cmp(const void *p1, const void *p2)
+{
+       const struct fts_backend_uid_map *map1 = p1, *map2 = p2;
+
+       return strcmp(map1->mailbox, map2->mailbox);
+}
+
+static int fts_build_init_virtual_next(struct fts_search_context *fctx)
+{
+       struct fts_search_virtual_context *vctx = &fctx->virtual_ctx;
+       struct mailbox_status status;
+       struct mailbox *const *boxes;
+       const struct fts_backend_uid_map *last_uids;
+       unsigned int boxi, uidi, box_count, last_uid_count;
+       int ret, vret = 0;
+
+       if (vctx->pool == NULL)
+               return 0;
+
+       if (fctx->virtual_ctx.trans != NULL)
+               (void)mailbox_transaction_commit(&fctx->virtual_ctx.trans);
+
+       boxes = array_get(&vctx->mailboxes, &box_count);
+       last_uids = array_get(&vctx->last_uids, &last_uid_count);
+
+       boxi = vctx->boxi;
+       uidi = vctx->uidi;
+       while (vret == 0 && boxi < box_count && uidi < last_uid_count) {
+               ret = strcmp(boxes[boxi]->name, last_uids[uidi].mailbox);
+               if (ret == 0) {
+                       /* match. check also that uidvalidity matches. */
+                       mailbox_get_status(boxes[boxi], STATUS_UIDVALIDITY,
+                                          &status);
+                       if (status.uidvalidity != last_uids[uidi].uidvalidity) {
+                               uidi++;
+                               continue;
+                       }
+                       vret = fts_build_init_box(fctx, boxes[boxi],
+                                                 last_uids[uidi].uid);
+                       boxi++;
+                       uidi++;
+               } else if (ret > 0) {
+                       /* not part of this virtual mailbox */
+                       uidi++;
+               } else {
+                       /* no messages indexed in the mailbox */
+                       vret = fts_build_init_box(fctx, boxes[boxi], 0);
+                       boxi++;
+               }
+       }
+       while (vret == 0 && boxi < box_count) {
+               vret = fts_build_init_box(fctx, boxes[boxi], 0);
+               boxi++;
+       }
+       vctx->boxi = boxi;
+       vctx->uidi = uidi;
+       return vret;
+}
+
+static int fts_build_init_virtual(struct fts_search_context *fctx)
+{
+       struct fts_search_virtual_context *vctx = &fctx->virtual_ctx;
+       struct fts_backend_uid_map *last_uids;
+       struct mailbox **boxes;
+       unsigned int box_count, last_uid_count;
+       int ret;
+
+       vctx->pool = pool_alloconly_create("fts virtual build", 1024);
+       p_array_init(&vctx->mailboxes, vctx->pool, 64);
+       mailbox_get_virtual_backend_boxes(fctx->t->box, &vctx->mailboxes, TRUE);
+       boxes = array_get_modifiable(&vctx->mailboxes, &box_count);
+
+       if (box_count <= 0) {
+               if (box_count == 0) {
+                       /* empty virtual mailbox */
+                       return 0;
+               }
+               /* virtual mailbox is built from only a single mailbox
+                  (currently). check that directly. */
+               fctx->virtual_ctx.trans =
+                       mailbox_transaction_begin(boxes[0], 0);
+               ret = fts_build_init_trans(fctx, fctx->virtual_ctx.trans);
+               return ret;
+       }
+
+       /* virtual mailbox is built from multiple mailboxes. figure out which
+          ones need updating. */
+       p_array_init(&vctx->last_uids, vctx->pool, 64);
+       if (fts_backend_get_all_last_uids(fctx->build_backend, vctx->pool,
+                                         &vctx->last_uids) < 0) {
+               pool_unref(&vctx->pool);
+               return -1;
+       }
+       last_uids = array_get_modifiable(&vctx->last_uids, &last_uid_count);
+
+       qsort(boxes, box_count, sizeof(*boxes), mailbox_name_cmp);
+       qsort(last_uids, last_uid_count, sizeof(*last_uids),
+             fts_backend_uid_map_mailbox_cmp);
+
+       ret = fts_build_init_virtual_next(fctx);
+       return ret < 0 ? -1 : 0;
+}
+
+static int fts_build_init(struct fts_search_context *fctx)
+{
+       if (fctx->fbox->virtual)
+               return fts_build_init_virtual(fctx);
+       return fts_build_init_trans(fctx, fctx->t);
 }
 
 static int fts_build_deinit(struct fts_storage_build_context **_ctx)
@@ -360,6 +514,7 @@ fts_mailbox_search_init(struct mailbox_transaction_context *t,
        fctx->fbox = fbox;
        fctx->t = t;
        fctx->args = args;
+       fctx->first_nonindexed_seq = (uint32_t)-1;
        MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx);
 
        if (fbox->backend_substr == NULL && fbox->backend_fast == NULL)
@@ -388,7 +543,7 @@ static int fts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
                }
        }
 
-       if (fctx->build_ctx != NULL) {
+       while (fctx->build_ctx != NULL) {
                /* this command is still building the indexes */
                ret = fts_build_more(fctx->build_ctx);
                if (ret == 0) {
@@ -400,8 +555,10 @@ static int fts_mailbox_search_next_nonblock(struct mail_search_context *ctx,
                if (fts_build_deinit(&fctx->build_ctx) < 0)
                        ret = -1;
                if (ret > 0) {
-                       fctx->first_nonindexed_seq = 0;
-                       fts_search_lookup(fctx);
+                       if (fts_build_init_virtual_next(fctx) == 0) {
+                               /* all finished */
+                               fts_search_lookup(fctx);
+                       }
                }
        }
 
@@ -460,12 +617,11 @@ static bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
                if (fctx->definite_idx == def_count) {
                        if (fctx->maybe_idx == maybe_count) {
                                /* look for the non-indexed mails */
-                               if (fctx->first_nonindexed_seq == 0)
+                               if (fctx->first_nonindexed_seq == (uint32_t)-1)
                                        return FALSE;
                                fctx->seqs_set = FALSE;
                                ctx->seq = fctx->first_nonindexed_seq - 1;
-                               return fbox->module_ctx.super.
-                                       search_next_update_seq(ctx);
+                               return fts_mailbox_search_next_update_seq(ctx);
                        }
                        use_maybe = TRUE;
                } else if (fctx->maybe_idx == maybe_count) {
@@ -506,6 +662,17 @@ static bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
                fts_mailbox_search_args_definite_set(fctx);
        }
 
+       if (ctx->seq + 1 >= fctx->first_nonindexed_seq) {
+               /* this is a virtual mailbox and we're searching headers.
+                  some mailboxes had more messages indexed than others.
+                  to avoid duplicates or jumping around, ignore the rest of
+                  the search results and just go through the messages in
+                  order. */
+               fctx->seqs_set = FALSE;
+               ctx->seq = fctx->first_nonindexed_seq - 1;
+               return fts_mailbox_search_next_update_seq(ctx);
+       }
+
        return ret;
 }
 
@@ -552,6 +719,10 @@ static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
                array_free(&fctx->maybe_seqs);
        if (array_is_created(&fctx->score_map))
                array_free(&fctx->score_map);
+       if (fctx->virtual_ctx.trans != NULL)
+               (void)mailbox_transaction_commit(&fctx->virtual_ctx.trans);
+       if (fctx->virtual_ctx.pool != NULL)
+               pool_unref(&fctx->virtual_ctx.pool);
        i_free(fctx);
        return fbox->module_ctx.super.search_deinit(ctx);
 }
index f3097cf96fb1c9645313fec547eed808ba402ee4..d03b33d8cc0fcdcbaeaa5f2d4382d80ff43dcbef 100644 (file)
@@ -11,6 +11,16 @@ struct fts_mailbox {
        unsigned int backend_set:1;
 };
 
+struct fts_search_virtual_context {
+       pool_t pool;
+
+       struct mailbox_transaction_context *trans;
+       ARRAY_TYPE(mailboxes) mailboxes;
+       ARRAY_TYPE(fts_backend_uid_map) last_uids;
+
+       unsigned int boxi, uidi;
+};
+
 struct fts_search_context {
        union mail_search_module_context module_ctx;
 
@@ -27,6 +37,7 @@ struct fts_search_context {
 
        struct fts_backend *build_backend;
        struct fts_storage_build_context *build_ctx;
+       struct fts_search_virtual_context virtual_ctx;
 
        unsigned int build_initialized:1;
        unsigned int seqs_set:1;
index 777074ab0b51b3be230a977ba649ba675c2cdb35..bfd0d34e3ad0baf04533829371417097a97e64df 100644 (file)
@@ -116,10 +116,12 @@ virtual_config_parse_line(struct virtual_parse_context *ctx, const char *line,
 
        /* new mailbox. the search args are added to it later. */
        bbox = p_new(ctx->pool, struct virtual_backend_box, 1);
+       if (strcasecmp(line, "INBOX") == 0)
+               line = "INBOX";
        bbox->name = p_strdup(ctx->pool, line);
        if (strchr(bbox->name, '*') != NULL ||
            strchr(bbox->name, '%') != NULL) {
-               name = bbox->name[0] == '-' ? bbox->name +1 : bbox->name;
+               name = bbox->name[0] == '-' ? bbox->name + 1 : bbox->name;
                bbox->glob = imap_match_init(ctx->pool, name, TRUE, ctx->sep);
                bbox->ns = mail_namespace_find(user->namespaces, &line);
                ctx->have_wildcards = TRUE;
@@ -128,6 +130,26 @@ virtual_config_parse_line(struct virtual_parse_context *ctx, const char *line,
        return 0;
 }
 
+static void
+virtual_mailbox_get_list_patterns(struct virtual_parse_context *ctx)
+{
+       struct virtual_mailbox *mbox = ctx->mbox;
+       ARRAY_TYPE(const_string) *dest;
+       struct virtual_backend_box *const *bboxes;
+       unsigned int i, count;
+
+       bboxes = array_get_modifiable(&mbox->backend_boxes, &count);
+       p_array_init(&mbox->list_include_patterns, ctx->pool, count);
+       p_array_init(&mbox->list_exclude_patterns, ctx->pool, count);
+       for (i = 0; i < count; i++) {
+               if (*bboxes[i]->name == '-')
+                       dest = &mbox->list_exclude_patterns;
+               else
+                       dest = &mbox->list_include_patterns;
+               array_append(dest, &bboxes[i]->name, 1);
+       }
+}
+
 static void
 separate_wildcard_mailboxes(struct virtual_mailbox *mbox,
                            ARRAY_TYPE(virtual_backend_box) *wildcard_boxes,
@@ -290,6 +312,7 @@ int virtual_config_read(struct virtual_mailbox *mbox)
        if (ret == 0)
                ret = virtual_config_add_rule(&ctx, &error);
 
+       virtual_mailbox_get_list_patterns(&ctx);
        if (ret == 0 && ctx.have_wildcards)
                ret = virtual_config_expand_wildcards(&ctx);
 
index bba997e97eef66322fad651f0472c1db364b8a42..2445e0c44c075557a1d8df279465781f5ffabb60 100644 (file)
@@ -577,6 +577,33 @@ virtual_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
        return TRUE;
 }
 
+static void
+virtual_get_virtual_backend_boxes(struct mailbox *box,
+                                 ARRAY_TYPE(mailboxes) *mailboxes,
+                                 bool only_with_msgs)
+{
+       struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
+       struct virtual_backend_box *const *bboxes;
+       unsigned int i, count;
+
+       bboxes = array_get(&mbox->backend_boxes, &count);
+       for (i = 0; i < count; i++) {
+               if (!only_with_msgs || array_count(&bboxes[i]->uids) > 0)
+                       array_append(mailboxes, &bboxes[i]->box, 1);
+       }
+}
+
+static void
+virtual_get_virtual_box_patterns(struct mailbox *box,
+                                ARRAY_TYPE(const_string) *includes,
+                                ARRAY_TYPE(const_string) *excludes)
+{
+       struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
+
+       array_append_array(includes, &mbox->list_include_patterns);
+       array_append_array(excludes, &mbox->list_exclude_patterns);
+}
+
 static void virtual_class_init(void)
 {
        virtual_transaction_class_init();
@@ -631,6 +658,8 @@ struct mailbox virtual_mailbox = {
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
                virtual_get_virtual_uid,
+               virtual_get_virtual_backend_boxes,
+               virtual_get_virtual_box_patterns,
                virtual_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index 707aae200a24c5b1bf1a453787e5163856741fc7..2984b3d32c7bb1bcf3010f37527adf3a2c80fbcf 100644 (file)
@@ -109,6 +109,9 @@ struct virtual_mailbox {
        /* Mailboxes this virtual mailbox consists of, sorted by mailbox_id */
        ARRAY_TYPE(virtual_backend_box) backend_boxes;
 
+       ARRAY_TYPE(const_string) list_include_patterns;
+       ARRAY_TYPE(const_string) list_exclude_patterns;
+
        unsigned int uids_mapped:1;
        unsigned int sync_initialized:1;
 };