index_storage_get_seq_range,
index_storage_get_uid_range,
index_storage_get_expunged_uids,
+ NULL,
index_mail_alloc,
index_header_lookup_init,
index_header_lookup_ref,
index_storage_get_seq_range,
index_storage_get_uid_range,
index_storage_get_expunged_uids,
+ NULL,
dbox_mail_alloc,
index_header_lookup_init,
index_header_lookup_ref,
index_storage_get_seq_range,
index_storage_get_uid_range,
index_storage_get_expunged_uids,
+ NULL,
index_mail_alloc,
index_header_lookup_init,
index_header_lookup_ref,
index_storage_get_seq_range,
index_storage_get_uid_range,
index_storage_get_expunged_uids,
+ NULL,
index_mail_alloc,
index_header_lookup_init,
index_header_lookup_ref,
index_storage_get_seq_range,
index_storage_get_uid_range,
index_storage_get_expunged_uids,
+ NULL,
index_mail_alloc,
index_header_lookup_init,
index_header_lookup_ref,
bool (*get_expunged_uids)(struct mailbox *box, uint64_t modseq,
const ARRAY_TYPE(seq_range) *uids,
ARRAY_TYPE(seq_range) *expunged_uids);
+ bool (*get_virtual_uid)(struct mailbox *box,
+ const char *backend_mailbox,
+ uint32_t backend_uidvalidity,
+ uint32_t backend_uid, uint32_t *uid_r);
struct mail *
(*mail_alloc)(struct mailbox_transaction_context *t,
return box->v.get_expunged_uids(box, modseq, uids, expunged_uids);
}
+bool mailbox_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
+ uint32_t backend_uidvalidity,
+ uint32_t backend_uid, uint32_t *uid_r)
+{
+ if (box->v.get_virtual_uid == NULL)
+ return FALSE;
+ return box->v.get_virtual_uid(box, backend_mailbox, backend_uidvalidity,
+ backend_uid, uid_r);
+}
+
struct mailbox_header_lookup_ctx *
mailbox_header_lookup_init(struct mailbox *box, const char *const headers[])
{
bool mailbox_get_expunged_uids(struct mailbox *box, uint64_t modseq,
const ARRAY_TYPE(seq_range) *uids,
ARRAY_TYPE(seq_range) *expunged_uids);
+/* If box is a virtual mailbox, look up UID for the given backend message.
+ Returns TRUE if found, FALSE if not. */
+bool mailbox_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
+ uint32_t backend_uidvalidity,
+ uint32_t backend_uid, uint32_t *uid_r);
/* Initialize header lookup for given headers. */
struct mailbox_header_lookup_ctx *
#include <curl/curl.h>
#define SOLR_CMDBUF_SIZE (1024*64)
+#define SOLR_MAX_ROWS 100000
struct solr_fts_backend_build_context {
struct fts_backend_build_context ctx;
solr_quote_str(str, backend->box->storage->ns->user->username);
t_array_init(&uids, 1);
- if (solr_connection_select(solr_conn, str_c(str), &uids, NULL) < 0)
+ if (solr_connection_select(solr_conn, str_c(str),
+ NULL, NULL, &uids, NULL) < 0)
return -1;
uidvals = array_get(&uids, &count);
{
}
+static bool solr_virtual_uid_map(const char *mailbox, uint32_t uidvalidity,
+ uint32_t *uid, void *context)
+{
+ struct mailbox *box = context;
+
+ return mailbox_get_virtual_uid(box, mailbox, uidvalidity,
+ *uid, uid);
+}
+
static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
ARRAY_TYPE(seq_range) *definite_uids,
ARRAY_TYPE(seq_range) *maybe_uids,
unsigned int i, count;
struct mailbox_status status;
string_t *str;
+ bool virtual;
+ virtual = strcmp(box->storage->name, "virtual") == 0;
mailbox_get_status(box, STATUS_UIDVALIDITY, &status);
str = t_str_new(256);
- str_printfa(str, "fl=uid,score&rows=%u&sort=uid%%20asc&q=",
- status.uidnext);
+ if (!virtual) {
+ str_printfa(str, "fl=uid,score&rows=%u&sort=uid%%20asc&q=",
+ status.uidnext);
+ } else {
+ str_printfa(str, "fl=uid,score,box,uidv&rows=%u"
+ "&sort=box%%20asc,uid%%20asc&q=",
+ SOLR_MAX_ROWS);
+ }
/* build a lucene search query from the fields */
fields = array_get(&ctx->fields, &count);
/* use a separate filter query for selecting the mailbox. it shouldn't
affect the score and there could be some caching benefits too. */
- str_printfa(str, "&fq=uidv:%u%%20box:", status.uidvalidity);
- solr_quote_str(str, box->name);
- str_append(str, "%20user:");
+ str_append(str, "&fq=user:");
solr_quote_str(str, box->storage->ns->user->username);
+ /* FIXME: limit what mailboxes to search with virtual storage */
+ if (!virtual) {
+ str_printfa(str, "%%20uidv:%u%%20box:", status.uidvalidity);
+ solr_quote_str(str, box->name);
+ }
+
array_clear(maybe_uids);
- return solr_connection_select(solr_conn, str_c(str),
- definite_uids, scores);
+ if (!virtual) {
+ return solr_connection_select(solr_conn, str_c(str), NULL, NULL,
+ definite_uids, scores);
+ } else {
+ return solr_connection_select(solr_conn, str_c(str),
+ solr_virtual_uid_map, box,
+ definite_uids, scores);
+ }
}
struct fts_backend fts_backend_solr = {
enum solr_xml_content_state {
SOLR_XML_CONTENT_STATE_NONE = 0,
SOLR_XML_CONTENT_STATE_UID,
- SOLR_XML_CONTENT_STATE_SCORE
+ SOLR_XML_CONTENT_STATE_SCORE,
+ SOLR_XML_CONTENT_STATE_MAILBOX,
+ SOLR_XML_CONTENT_STATE_UIDVALIDITY
};
struct solr_lookup_xml_context {
enum solr_xml_content_state content_state;
int depth;
- uint32_t uid;
+ uint32_t uid, uidvalidity;
float score;
+ char *mailbox;
+
+ solr_uid_map_callback_t *callback;
+ void *context;
ARRAY_TYPE(seq_range) *uids;
ARRAY_TYPE(fts_score_map) *scores;
ctx->state++;
ctx->uid = 0;
ctx->score = 0;
+ i_free_and_null(ctx->mailbox);
+ ctx->uidvalidity = 0;
}
break;
case SOLR_XML_RESPONSE_STATE_DOC:
ctx->content_state = SOLR_XML_CONTENT_STATE_UID;
else if (strcmp(name_attr, "score") == 0)
ctx->content_state = SOLR_XML_CONTENT_STATE_SCORE;
+ else if (strcmp(name_attr, "box") == 0)
+ ctx->content_state = SOLR_XML_CONTENT_STATE_MAILBOX;
+ else if (strcmp(name_attr, "uidv") == 0)
+ ctx->content_state = SOLR_XML_CONTENT_STATE_UIDVALIDITY;
else
break;
ctx->state++;
struct fts_score_map *score;
if (ctx->uid == 0) {
- i_error("fts_solr: missing uid");
+ i_error("fts_solr: Query didn't return uid");
return;
}
+ if (ctx->callback != NULL) {
+ if (ctx->mailbox == NULL) {
+ i_error("fts_solr: Query didn't return mailbox");
+ return;
+ }
+ if (!ctx->callback(ctx->mailbox, ctx->uidvalidity,
+ &ctx->uid, ctx->context))
+ return;
+ }
+
seq_range_array_add(ctx->uids, 0, ctx->uid);
if (ctx->scores != NULL && ctx->score != 0) {
score = array_append_space(ctx->scores);
ctx->depth--;
}
+static int uint32_parse(const char *str, int len, uint32_t *value_r)
+{
+ uint32_t value = 0;
+ int i;
+
+ for (i = 0; i < len; i++) {
+ if (str[i] < '0' || str[i] > '9')
+ break;
+ value = value*10 + str[i]-'0';
+ }
+ if (i != len)
+ return -1;
+
+ *value_r = value;
+ return 0;
+}
+
static void solr_lookup_xml_data(void *context, const char *str, int len)
{
struct solr_lookup_xml_context *ctx = context;
- uint32_t uid;
- int i;
switch (ctx->content_state) {
case SOLR_XML_CONTENT_STATE_NONE:
break;
case SOLR_XML_CONTENT_STATE_UID:
- for (i = 0, uid = 0; i < len; i++) {
- if (str[i] < '0' || str[i] > '9')
- break;
- uid = uid*10 + str[i]-'0';
- }
- if (i != len) {
+ if (uint32_parse(str, len, &ctx->uid) < 0)
i_error("fts_solr: received invalid uid");
- break;
- }
- ctx->uid = uid;
break;
case SOLR_XML_CONTENT_STATE_SCORE:
T_BEGIN {
ctx->score = strtod(t_strndup(str, len), NULL);
} T_END;
break;
+ case SOLR_XML_CONTENT_STATE_MAILBOX:
+ i_free(ctx->mailbox);
+ ctx->mailbox = i_strndup(str, len);
+ break;
+ case SOLR_XML_CONTENT_STATE_UIDVALIDITY:
+ if (uint32_parse(str, len, &ctx->uidvalidity) < 0)
+ i_error("fts_solr: received invalid uidvalidity");
+ break;
}
}
int solr_connection_select(struct solr_connection *conn, const char *query,
+ solr_uid_map_callback_t *callback, void *context,
ARRAY_TYPE(seq_range) *uids,
ARRAY_TYPE(fts_score_map) *scores)
{
memset(&solr_lookup_context, 0, sizeof(solr_lookup_context));
solr_lookup_context.uids = uids;
solr_lookup_context.scores = scores;
+ solr_lookup_context.callback = callback;
+ solr_lookup_context.context = context;
i_free_and_null(conn->http_failure);
conn->xml_failed = FALSE;
#include "seq-range-array.h"
#include "fts-api.h"
+/* Returns TRUE if UID conversion was done, FALSE if uid should be skipped. */
+typedef bool solr_uid_map_callback_t(const char *mailbox, uint32_t uidvalidity,
+ uint32_t *uid, void *context);
+
struct solr_connection *solr_connection_init(const char *url, bool debug);
void solr_connection_deinit(struct solr_connection *conn);
const char *str);
int solr_connection_select(struct solr_connection *conn, const char *query,
+ solr_uid_map_callback_t *callback, void *context,
ARRAY_TYPE(seq_range) *uids,
ARRAY_TYPE(fts_score_map) *scores);
int solr_connection_post(struct solr_connection *conn, const char *cmd);
struct fts_backend_build_context *build;
uint32_t last_uid, last_uid_locked, seq1, seq2;
+ if (fctx->fbox->virtual) {
+ /* FIXME: update all mailboxes */
+ return 0;
+ }
+
if (fts_backend_get_last_uid(backend, &last_uid) < 0)
return -1;
return ret;
}
+static bool
+fts_mailbox_search_next_update_seq_virtual(struct mail_search_context *ctx)
+{
+ struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
+ struct fts_search_context *fctx = FTS_CONTEXT(ctx);
+
+ while (fbox->module_ctx.super.search_next_update_seq(ctx)) {
+ if (!fctx->seqs_set)
+ return TRUE;
+
+ /* virtual mailbox searches don't return sequences sorted.
+ just check if the suggested sequence exists. */
+ if (seq_range_exists(&fctx->definite_seqs, ctx->seq)) {
+ fts_mailbox_search_args_definite_set(fctx);
+ return TRUE;
+ }
+ if (seq_range_exists(&fctx->maybe_seqs, ctx->seq))
+ return TRUE;
+ mail_search_args_reset(ctx->args->args, FALSE);
+ }
+ return FALSE;
+}
+
static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
{
struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction);
struct fts_mailbox *fbox;
fbox = i_new(struct fts_mailbox, 1);
+ fbox->virtual = strcmp(box->storage->name, "virtual") == 0;
fbox->env = env;
fbox->module_ctx.super = box->v;
box->v.close = fts_mailbox_close;
box->v.search_init = fts_mailbox_search_init;
box->v.search_next_nonblock = fts_mailbox_search_next_nonblock;
- box->v.search_next_update_seq = fts_mailbox_search_next_update_seq;
+ box->v.search_next_update_seq = fbox->virtual ?
+ fts_mailbox_search_next_update_seq_virtual :
+ fts_mailbox_search_next_update_seq;
box->v.search_deinit = fts_mailbox_search_deinit;
box->v.mail_alloc = fts_mail_alloc;
box->v.transaction_begin = fts_transaction_begin;
struct fts_backend *backend_fast;
const char *env;
+ unsigned int virtual:1;
unsigned int backend_set:1;
};
mbox->storage = storage;
mbox->path = p_strdup(pool, path);
+ mbox->vseq_lookup_prev_mailbox = i_strdup("");
mbox->virtual_ext_id =
mail_index_ext_register(index, "virtual", 0,
array_free(&bboxes[i]->uids);
}
array_free(&mbox->backend_boxes);
+ i_free(mbox->vseq_lookup_prev_mailbox);
return index_storage_mailbox_close(box) < 0 ? -1 : ret;
}
return ret;
}
+static int virtual_backend_uidmap_cmp(const void *key, const void *data)
+{
+ const uint32_t *uid = key;
+ const struct virtual_backend_uidmap *map = data;
+
+ return *uid < map->real_uid ? -1 :
+ *uid > map->real_uid ? 1 : 0;
+}
+
+static bool
+virtual_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
+ uint32_t backend_uidvalidity,
+ uint32_t backend_uid, uint32_t *uid_r)
+{
+ struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
+ struct virtual_backend_box *bbox;
+ struct mailbox_status status;
+ const struct virtual_backend_uidmap *uids;
+ unsigned int count;
+
+ if (strcmp(mbox->vseq_lookup_prev_mailbox, backend_mailbox) == 0)
+ bbox = mbox->vseq_lookup_prev_bbox;
+ else {
+ i_free(mbox->vseq_lookup_prev_mailbox);
+ mbox->vseq_lookup_prev_mailbox = i_strdup(backend_mailbox);
+
+ bbox = virtual_backend_box_lookup_name(mbox, backend_mailbox);
+ mbox->vseq_lookup_prev_bbox = bbox;
+ }
+ if (bbox == NULL)
+ return FALSE;
+
+ mailbox_get_status(bbox->box, STATUS_UIDVALIDITY, &status);
+ if (status.uidvalidity != backend_uidvalidity)
+ return FALSE;
+
+ uids = array_get(&bbox->uids, &count);
+ uids = bsearch(&backend_uid, uids, count, sizeof(*uids),
+ virtual_backend_uidmap_cmp);
+ if (uids == NULL)
+ return FALSE;
+
+ *uid_r = uids->virtual_uid;
+ return TRUE;
+}
+
static void virtual_class_init(void)
{
virtual_transaction_class_init();
index_storage_get_seq_range,
index_storage_get_uid_range,
index_storage_get_expunged_uids,
+ virtual_get_virtual_uid,
virtual_mail_alloc,
index_header_lookup_init,
index_header_lookup_ref,
uint32_t highest_mailbox_id;
uint32_t search_args_crc32;
+ char *vseq_lookup_prev_mailbox;
+ struct virtual_backend_box *vseq_lookup_prev_bbox;
+
/* Mailboxes this virtual mailbox consists of, sorted by mailbox_id */
ARRAY_TYPE(virtual_backend_box) backend_boxes;