]> git.ipfire.org Git - thirdparty/dovecot/core.git/commitdiff
Initial support for getting full text search working nicely with virtual mailboxes.
authorTimo Sirainen <tss@iki.fi>
Sun, 23 Nov 2008 01:59:55 +0000 (03:59 +0200)
committerTimo Sirainen <tss@iki.fi>
Sun, 23 Nov 2008 01:59:55 +0000 (03:59 +0200)
--HG--
branch : HEAD

15 files changed:
src/lib-storage/index/cydir/cydir-storage.c
src/lib-storage/index/dbox/dbox-storage.c
src/lib-storage/index/maildir/maildir-storage.c
src/lib-storage/index/mbox/mbox-storage.c
src/lib-storage/index/raw/raw-storage.c
src/lib-storage/mail-storage-private.h
src/lib-storage/mail-storage.c
src/lib-storage/mail-storage.h
src/plugins/fts-solr/fts-backend-solr.c
src/plugins/fts-solr/solr-connection.c
src/plugins/fts-solr/solr-connection.h
src/plugins/fts/fts-storage.c
src/plugins/fts/fts-storage.h
src/plugins/virtual/virtual-storage.c
src/plugins/virtual/virtual-storage.h

index 2f7f789e834b7ed81a2ed695c5a284bc80fb5b81..97908749e398dacdad9e4d0eee2a9b7fbfe6e394 100644 (file)
@@ -446,6 +446,7 @@ struct mailbox cydir_mailbox = {
                index_storage_get_seq_range,
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
+               NULL,
                index_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index 8dca62bd82379fdf69433a2271568f8fdc99b2f2..2670ce1c3e1f1b86ffe3e3b21e612ceaf94fc11d 100644 (file)
@@ -717,6 +717,7 @@ struct mailbox dbox_mailbox = {
                index_storage_get_seq_range,
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
+               NULL,
                dbox_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index d5fcf17a8760d44ba9b44709cb78a439b713ff12..33f5aac40e3359e3ebfebe1fbcc515be659eb765 100644 (file)
@@ -1105,6 +1105,7 @@ struct mailbox maildir_mailbox = {
                index_storage_get_seq_range,
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
+               NULL,
                index_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index f5f96a7b2889d18da5122211d5fb301353c6ad02..642d45f6cd92cc2dac882aea9e09452d6864cea0 100644 (file)
@@ -1027,6 +1027,7 @@ struct mailbox mbox_mailbox = {
                index_storage_get_seq_range,
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
+               NULL,
                index_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index 31a514268f3f6b5384cc7500ce62bcbf21dc3bd9..e6694892e07c6eecb4c657218bfaa300dfb5bb35 100644 (file)
@@ -296,6 +296,7 @@ struct mailbox raw_mailbox = {
                index_storage_get_seq_range,
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
+               NULL,
                index_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index 43425ca998111ae6bd1eec0f76d5c5ddd7de10ab..ece194da0627d63244df6c8442856f1454d78673 100644 (file)
@@ -138,6 +138,10 @@ struct mailbox_vfuncs {
        bool (*get_expunged_uids)(struct mailbox *box, uint64_t modseq,
                                  const ARRAY_TYPE(seq_range) *uids,
                                  ARRAY_TYPE(seq_range) *expunged_uids);
+       bool (*get_virtual_uid)(struct mailbox *box,
+                               const char *backend_mailbox,
+                               uint32_t backend_uidvalidity,
+                               uint32_t backend_uid, uint32_t *uid_r);
 
        struct mail *
                (*mail_alloc)(struct mailbox_transaction_context *t,
index c7b2029646a7603dda181fbf66d9f72f328d7e3d..a7aa44d451d735741f34cc18ad07f9da52de08b1 100644 (file)
@@ -630,6 +630,16 @@ bool mailbox_get_expunged_uids(struct mailbox *box, uint64_t modseq,
        return box->v.get_expunged_uids(box, modseq, uids, expunged_uids);
 }
 
+bool mailbox_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
+                            uint32_t backend_uidvalidity,
+                            uint32_t backend_uid, uint32_t *uid_r)
+{
+       if (box->v.get_virtual_uid == NULL)
+               return FALSE;
+       return box->v.get_virtual_uid(box, backend_mailbox, backend_uidvalidity,
+                                     backend_uid, uid_r);
+}
+
 struct mailbox_header_lookup_ctx *
 mailbox_header_lookup_init(struct mailbox *box, const char *const headers[])
 {
index 13e8c39bf0567d683883901c23d4d573c1c802ee..1e1ec3e2b54f588a538b09b10e776823c1e1708b 100644 (file)
@@ -416,6 +416,11 @@ void mailbox_get_uid_range(struct mailbox *box,
 bool mailbox_get_expunged_uids(struct mailbox *box, uint64_t modseq,
                               const ARRAY_TYPE(seq_range) *uids,
                               ARRAY_TYPE(seq_range) *expunged_uids);
+/* If box is a virtual mailbox, look up UID for the given backend message.
+   Returns TRUE if found, FALSE if not. */
+bool mailbox_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
+                            uint32_t backend_uidvalidity,
+                            uint32_t backend_uid, uint32_t *uid_r);
 
 /* Initialize header lookup for given headers. */
 struct mailbox_header_lookup_ctx *
index 19d1d4344b291c17558e4eb0456092e248043460..b922dbeb1204bf2ef9ff202fc583da6c601a523c 100644 (file)
@@ -11,6 +11,7 @@
 #include <curl/curl.h>
 
 #define SOLR_CMDBUF_SIZE (1024*64)
+#define SOLR_MAX_ROWS 100000
 
 struct solr_fts_backend_build_context {
        struct fts_backend_build_context ctx;
@@ -97,7 +98,8 @@ static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
        solr_quote_str(str, backend->box->storage->ns->user->username);
 
        t_array_init(&uids, 1);
-       if (solr_connection_select(solr_conn, str_c(str), &uids, NULL) < 0)
+       if (solr_connection_select(solr_conn, str_c(str),
+                                  NULL, NULL, &uids, NULL) < 0)
                return -1;
 
        uidvals = array_get(&uids, &count);
@@ -256,6 +258,15 @@ static void fts_backend_solr_unlock(struct fts_backend *backend ATTR_UNUSED)
 {
 }
 
+static bool solr_virtual_uid_map(const char *mailbox, uint32_t uidvalidity,
+                                uint32_t *uid, void *context)
+{
+       struct mailbox *box = context;
+
+       return mailbox_get_virtual_uid(box, mailbox, uidvalidity,
+                                      *uid, uid);
+}
+
 static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
                                   ARRAY_TYPE(seq_range) *definite_uids,
                                   ARRAY_TYPE(seq_range) *maybe_uids,
@@ -266,12 +277,20 @@ static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
        unsigned int i, count;
        struct mailbox_status status;
        string_t *str;
+       bool virtual;
 
+       virtual = strcmp(box->storage->name, "virtual") == 0;
        mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
 
        str = t_str_new(256);
-       str_printfa(str, "fl=uid,score&rows=%u&sort=uid%%20asc&q=",
-                   status.uidnext);
+       if (!virtual) {
+               str_printfa(str, "fl=uid,score&rows=%u&sort=uid%%20asc&q=",
+                           status.uidnext);
+       } else {
+               str_printfa(str, "fl=uid,score,box,uidv&rows=%u"
+                           "&sort=box%%20asc,uid%%20asc&q=",
+                           SOLR_MAX_ROWS);
+       }
 
        /* build a lucene search query from the fields */
        fields = array_get(&ctx->fields, &count);
@@ -298,14 +317,24 @@ static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
 
        /* use a separate filter query for selecting the mailbox. it shouldn't
           affect the score and there could be some caching benefits too. */
-       str_printfa(str, "&fq=uidv:%u%%20box:", status.uidvalidity);
-       solr_quote_str(str, box->name);
-       str_append(str, "%20user:");
+       str_append(str, "&fq=user:");
        solr_quote_str(str, box->storage->ns->user->username);
 
+       /* FIXME: limit what mailboxes to search with virtual storage */
+       if (!virtual) {
+               str_printfa(str, "%%20uidv:%u%%20box:", status.uidvalidity);
+               solr_quote_str(str, box->name);
+       }
+
        array_clear(maybe_uids);
-       return solr_connection_select(solr_conn, str_c(str),
-                                     definite_uids, scores);
+       if (!virtual) {
+               return solr_connection_select(solr_conn, str_c(str), NULL, NULL,
+                                             definite_uids, scores);
+       } else {
+               return solr_connection_select(solr_conn, str_c(str),
+                                             solr_virtual_uid_map, box,
+                                             definite_uids, scores);
+       }
 }
 
 struct fts_backend fts_backend_solr = {
index c79aadbb501e5a6e1e13de97dbafdd8af9cb8e6f..ae0624099c949e8bfcf163e4974a6538e052746d 100644 (file)
@@ -22,7 +22,9 @@ enum solr_xml_response_state {
 enum solr_xml_content_state {
        SOLR_XML_CONTENT_STATE_NONE = 0,
        SOLR_XML_CONTENT_STATE_UID,
-       SOLR_XML_CONTENT_STATE_SCORE
+       SOLR_XML_CONTENT_STATE_SCORE,
+       SOLR_XML_CONTENT_STATE_MAILBOX,
+       SOLR_XML_CONTENT_STATE_UIDVALIDITY
 };
 
 struct solr_lookup_xml_context {
@@ -30,8 +32,12 @@ struct solr_lookup_xml_context {
        enum solr_xml_content_state content_state;
        int depth;
 
-       uint32_t uid;
+       uint32_t uid, uidvalidity;
        float score;
+       char *mailbox;
+
+       solr_uid_map_callback_t *callback;
+       void *context;
 
        ARRAY_TYPE(seq_range) *uids;
        ARRAY_TYPE(fts_score_map) *scores;
@@ -234,6 +240,8 @@ solr_lookup_xml_start(void *context, const char *name, const char **attrs)
                        ctx->state++;
                        ctx->uid = 0;
                        ctx->score = 0;
+                       i_free_and_null(ctx->mailbox);
+                       ctx->uidvalidity = 0;
                }
                break;
        case SOLR_XML_RESPONSE_STATE_DOC:
@@ -242,6 +250,10 @@ solr_lookup_xml_start(void *context, const char *name, const char **attrs)
                        ctx->content_state = SOLR_XML_CONTENT_STATE_UID;
                else if (strcmp(name_attr, "score") == 0)
                        ctx->content_state = SOLR_XML_CONTENT_STATE_SCORE;
+               else if (strcmp(name_attr, "box") == 0)
+                       ctx->content_state = SOLR_XML_CONTENT_STATE_MAILBOX;
+               else if (strcmp(name_attr, "uidv") == 0)
+                       ctx->content_state = SOLR_XML_CONTENT_STATE_UIDVALIDITY;
                else 
                        break;
                ctx->state++;
@@ -256,10 +268,20 @@ static void solr_lookup_add_doc(struct solr_lookup_xml_context *ctx)
        struct fts_score_map *score;
 
        if (ctx->uid == 0) {
-               i_error("fts_solr: missing uid");
+               i_error("fts_solr: Query didn't return uid");
                return;
        }
 
+       if (ctx->callback != NULL) {
+               if (ctx->mailbox == NULL) {
+                       i_error("fts_solr: Query didn't return mailbox");
+                       return;
+               }
+               if (!ctx->callback(ctx->mailbox, ctx->uidvalidity,
+                                  &ctx->uid, ctx->context))
+                       return;
+       }
+
        seq_range_array_add(ctx->uids, 0, ctx->uid);
        if (ctx->scores != NULL && ctx->score != 0) {
                score = array_append_space(ctx->scores);
@@ -283,36 +305,52 @@ static void solr_lookup_xml_end(void *context, const char *name ATTR_UNUSED)
        ctx->depth--;
 }
 
+static int uint32_parse(const char *str, int len, uint32_t *value_r)
+{
+       uint32_t value = 0;
+       int i;
+
+       for (i = 0; i < len; i++) {
+               if (str[i] < '0' || str[i] > '9')
+                       break;
+               value = value*10 + str[i]-'0';
+       }
+       if (i != len)
+               return -1;
+
+       *value_r = value;
+       return 0;
+}
+
 static void solr_lookup_xml_data(void *context, const char *str, int len)
 {
        struct solr_lookup_xml_context *ctx = context;
-       uint32_t uid;
-       int i;
 
        switch (ctx->content_state) {
        case SOLR_XML_CONTENT_STATE_NONE:
                break;
        case SOLR_XML_CONTENT_STATE_UID:
-               for (i = 0, uid = 0; i < len; i++) {
-                       if (str[i] < '0' || str[i] > '9')
-                               break;
-                       uid = uid*10 + str[i]-'0';
-               }
-               if (i != len) {
+               if (uint32_parse(str, len, &ctx->uid) < 0)
                        i_error("fts_solr: received invalid uid");
-                       break;
-               }
-               ctx->uid = uid;
                break;
        case SOLR_XML_CONTENT_STATE_SCORE:
                T_BEGIN {
                        ctx->score = strtod(t_strndup(str, len), NULL);
                } T_END;
                break;
+       case SOLR_XML_CONTENT_STATE_MAILBOX:
+               i_free(ctx->mailbox);
+               ctx->mailbox = i_strndup(str, len);
+               break;
+       case SOLR_XML_CONTENT_STATE_UIDVALIDITY:
+               if (uint32_parse(str, len, &ctx->uidvalidity) < 0)
+                       i_error("fts_solr: received invalid uidvalidity");
+               break;
        }
 }
 
 int solr_connection_select(struct solr_connection *conn, const char *query,
+                          solr_uid_map_callback_t *callback, void *context,
                           ARRAY_TYPE(seq_range) *uids,
                           ARRAY_TYPE(fts_score_map) *scores)
 {
@@ -326,6 +364,8 @@ int solr_connection_select(struct solr_connection *conn, const char *query,
        memset(&solr_lookup_context, 0, sizeof(solr_lookup_context));
        solr_lookup_context.uids = uids;
        solr_lookup_context.scores = scores;
+       solr_lookup_context.callback = callback;
+       solr_lookup_context.context = context;
 
        i_free_and_null(conn->http_failure);
        conn->xml_failed = FALSE;
index c9eda72e99617818036a6cbfcaa73be35ece7eff..f2ffc7f20545ccfc6577908dea58ceae5d4e9470 100644 (file)
@@ -4,6 +4,10 @@
 #include "seq-range-array.h"
 #include "fts-api.h"
 
+/* Returns TRUE if UID conversion was done, FALSE if uid should be skipped. */
+typedef bool solr_uid_map_callback_t(const char *mailbox, uint32_t uidvalidity,
+                                    uint32_t *uid, void *context);
+
 struct solr_connection *solr_connection_init(const char *url, bool debug);
 void solr_connection_deinit(struct solr_connection *conn);
 
@@ -11,6 +15,7 @@ void solr_connection_quote_str(struct solr_connection *conn, string_t *dest,
                               const char *str);
 
 int solr_connection_select(struct solr_connection *conn, const char *query,
+                          solr_uid_map_callback_t *callback, void *context,
                           ARRAY_TYPE(seq_range) *uids,
                           ARRAY_TYPE(fts_score_map) *scores);
 int solr_connection_post(struct solr_connection *conn, const char *cmd);
index 7c06162ca0cdaefa7f10711aceeef3ff16d01032..b8dd646ae9001433b569c5428ed5d23d52703eef 100644 (file)
@@ -179,6 +179,11 @@ static int fts_build_init(struct fts_search_context *fctx)
        struct fts_backend_build_context *build;
        uint32_t last_uid, last_uid_locked, seq1, seq2;
 
+       if (fctx->fbox->virtual) {
+               /* FIXME: update all mailboxes */
+               return 0;
+       }
+
        if (fts_backend_get_last_uid(backend, &last_uid) < 0)
                return -1;
 
@@ -504,6 +509,29 @@ static bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
        return ret;
 }
 
+static bool
+fts_mailbox_search_next_update_seq_virtual(struct mail_search_context *ctx)
+{
+       struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
+       struct fts_search_context *fctx = FTS_CONTEXT(ctx);
+
+       while (fbox->module_ctx.super.search_next_update_seq(ctx)) {
+               if (!fctx->seqs_set)
+                       return TRUE;
+
+               /* virtual mailbox searches don't return sequences sorted.
+                  just check if the suggested sequence exists. */
+               if (seq_range_exists(&fctx->definite_seqs, ctx->seq)) {
+                       fts_mailbox_search_args_definite_set(fctx);
+                       return TRUE;
+               }
+               if (seq_range_exists(&fctx->maybe_seqs, ctx->seq))
+                       return TRUE;
+               mail_search_args_reset(ctx->args->args, FALSE);
+       }
+       return FALSE;
+}
+
 static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
 {
        struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction);
@@ -729,12 +757,15 @@ static void fts_mailbox_init(struct mailbox *box, const char *env)
        struct fts_mailbox *fbox;
 
        fbox = i_new(struct fts_mailbox, 1);
+       fbox->virtual = strcmp(box->storage->name, "virtual") == 0;
        fbox->env = env;
        fbox->module_ctx.super = box->v;
        box->v.close = fts_mailbox_close;
        box->v.search_init = fts_mailbox_search_init;
        box->v.search_next_nonblock = fts_mailbox_search_next_nonblock;
-       box->v.search_next_update_seq = fts_mailbox_search_next_update_seq;
+       box->v.search_next_update_seq = fbox->virtual ?
+               fts_mailbox_search_next_update_seq_virtual :
+               fts_mailbox_search_next_update_seq;
        box->v.search_deinit = fts_mailbox_search_deinit;
        box->v.mail_alloc = fts_mail_alloc;
        box->v.transaction_begin = fts_transaction_begin;
index ef27aecd49de1649522d2fc63908677280bc0558..f3097cf96fb1c9645313fec547eed808ba402ee4 100644 (file)
@@ -7,6 +7,7 @@ struct fts_mailbox {
        struct fts_backend *backend_fast;
 
        const char *env;
+       unsigned int virtual:1;
        unsigned int backend_set:1;
 };
 
index 1c76ee9c193670b87ad89ab17f7221d5d7a38ac1..bba997e97eef66322fad651f0472c1db364b8a42 100644 (file)
@@ -277,6 +277,7 @@ virtual_open(struct virtual_storage *storage, const char *name,
 
        mbox->storage = storage;
        mbox->path = p_strdup(pool, path);
+       mbox->vseq_lookup_prev_mailbox = i_strdup("");
 
        mbox->virtual_ext_id =
                mail_index_ext_register(index, "virtual", 0,
@@ -353,6 +354,7 @@ static int virtual_storage_mailbox_close(struct mailbox *box)
                array_free(&bboxes[i]->uids);
        }
        array_free(&mbox->backend_boxes);
+       i_free(mbox->vseq_lookup_prev_mailbox);
 
        return index_storage_mailbox_close(box) < 0 ? -1 : ret;
 }
@@ -529,6 +531,52 @@ virtual_list_iter_is_mailbox(struct mailbox_list_iterate_context *ctx
        return ret;
 }
 
+static int virtual_backend_uidmap_cmp(const void *key, const void *data)
+{
+       const uint32_t *uid = key;
+       const struct virtual_backend_uidmap *map = data;
+
+       return *uid < map->real_uid ? -1 :
+               *uid > map->real_uid ? 1 : 0;
+}
+
+static bool
+virtual_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
+                       uint32_t backend_uidvalidity,
+                       uint32_t backend_uid, uint32_t *uid_r)
+{
+       struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
+       struct virtual_backend_box *bbox;
+       struct mailbox_status status;
+       const struct virtual_backend_uidmap *uids;
+       unsigned int count;
+
+       if (strcmp(mbox->vseq_lookup_prev_mailbox, backend_mailbox) == 0)
+               bbox = mbox->vseq_lookup_prev_bbox;
+       else {
+               i_free(mbox->vseq_lookup_prev_mailbox);
+               mbox->vseq_lookup_prev_mailbox = i_strdup(backend_mailbox);
+
+               bbox = virtual_backend_box_lookup_name(mbox, backend_mailbox);
+               mbox->vseq_lookup_prev_bbox = bbox;
+       }
+       if (bbox == NULL)
+               return FALSE;
+
+       mailbox_get_status(bbox->box, STATUS_UIDVALIDITY, &status);
+       if (status.uidvalidity != backend_uidvalidity)
+               return FALSE;
+
+       uids = array_get(&bbox->uids, &count);
+       uids = bsearch(&backend_uid, uids, count, sizeof(*uids),
+                      virtual_backend_uidmap_cmp);
+       if (uids == NULL)
+               return FALSE;
+
+       *uid_r = uids->virtual_uid;
+       return TRUE;
+}
+
 static void virtual_class_init(void)
 {
        virtual_transaction_class_init();
@@ -582,6 +630,7 @@ struct mailbox virtual_mailbox = {
                index_storage_get_seq_range,
                index_storage_get_uid_range,
                index_storage_get_expunged_uids,
+               virtual_get_virtual_uid,
                virtual_mail_alloc,
                index_header_lookup_init,
                index_header_lookup_ref,
index 0ae6ded4113fc958cfd58fb1180f564d13a9d2b6..707aae200a24c5b1bf1a453787e5163856741fc7 100644 (file)
@@ -103,6 +103,9 @@ struct virtual_mailbox {
        uint32_t highest_mailbox_id;
        uint32_t search_args_crc32;
 
+       char *vseq_lookup_prev_mailbox;
+       struct virtual_backend_box *vseq_lookup_prev_bbox;
+
        /* Mailboxes this virtual mailbox consists of, sorted by mailbox_id */
        ARRAY_TYPE(virtual_backend_box) backend_boxes;