AS_HELP_STRING([--with-lucene], [Build with CLucene full text search support]),
TEST_WITH(lucene, $withval),
want_lucene=no)
-if test "$want_lucene" = "yes"; then
- AC_ERROR([CLucene support isn't working very well. Use Solr instead.])
-fi
AM_CONDITIONAL(BUILD_LUCENE, test "$want_lucene" = "yes")
AC_ARG_WITH(solr,
This is the Solr schema file, place it into solr/conf/schema.xml. You may
want to modify the tokenizers and filters.
-->
-<schema name="dovecot" version="1.1">
+<schema name="dovecot" version="2.1">
<types>
<!-- IMAP has 32bit unsigned ints but java ints are signed, so use longs -->
<fieldType name="string" class="solr.StrField" omitNorms="true"/>
<fields>
- <field name="id" type="string" indexed="true" stored="true" required="true" />
- <field name="uid" type="slong" indexed="true" stored="true" required="true" />
- <field name="uidv" type="long" indexed="true" stored="true" required="true" />
- <field name="box" type="string" indexed="true" stored="true" required="true" />
- <field name="user" type="string" indexed="true" stored="true" required="true" />
- <field name="ns" type="string" indexed="true" stored="true" required="false" />
- <field name="last_uid" type="boolean" indexed="true" stored="false" />
- <field name="hdr" type="text" indexed="true" stored="false" />
- <field name="body" type="text" indexed="true" stored="false" />
+ <field name="id" type="string" indexed="true" stored="true" required="true" />
+ <field name="uid" type="slong" indexed="true" stored="true" required="true" />
+ <field name="box" type="string" indexed="true" stored="true" required="true" />
+ <field name="user" type="string" indexed="true" stored="true" required="true" />
+
+ <field name="hdr" type="text" indexed="true" stored="false" />
+ <field name="body" type="text" indexed="true" stored="false" />
+
+ <field name="from" type="text" indexed="true" stored="false" />
+ <field name="to" type="text" indexed="true" stored="false" />
+ <field name="cc" type="text" indexed="true" stored="false" />
+ <field name="bcc" type="text" indexed="true" stored="false" />
+ <field name="subject" type="text" indexed="true" stored="false" />
</fields>
<uniqueKey>id</uniqueKey>
#include "lib.h"
#include "array.h"
-#include "mkdir-parents.h"
+#include "hash.h"
+#include "hex-binary.h"
+#include "file-dotlock.h"
+#include "mail-namespace.h"
#include "mail-storage-private.h"
#include "lucene-wrapper.h"
#include "fts-lucene-plugin.h"
+#include <wchar.h>
+
#define LUCENE_INDEX_DIR_NAME "lucene-indexes"
-#define LUCENE_LOCK_SUBDIR_NAME "locks"
+#define LUCENE_EXPUNGE_FILENAME "pending-expunges"
+#define LUCENE_OPTIMIZE_BATCH_MSGS_COUNT 100
-#define LUCENE_CONTEXT(obj) \
- MODULE_CONTEXT(obj, fts_lucene_storage_module)
+struct lucene_fts_backend {
+ struct fts_backend backend;
+ char *dir_path;
-struct lucene_mail_storage {
- union mail_storage_module_context module_ctx;
struct lucene_index *index;
struct mailbox *selected_box;
- int refcount;
-};
-struct lucene_fts_backend {
- struct fts_backend backend;
- struct lucene_mail_storage *lstorage;
- struct mailbox *box;
+ unsigned int dir_created:1;
+ unsigned int updating:1;
};
-struct lucene_fts_backend_build_context {
- struct fts_backend_build_context ctx;
+struct lucene_fts_backend_update_context {
+ struct fts_backend_update_context ctx;
+
+ struct mailbox *box;
+ uint32_t last_uid;
uint32_t uid;
- bool hdr;
-};
+ char *hdr_name;
-static MODULE_CONTEXT_DEFINE_INIT(fts_lucene_storage_module,
- &mail_storage_module_register);
+ unsigned int added_msgs, expunges;
+};
-static void fts_backend_select(struct lucene_fts_backend *backend)
+static int fts_backend_lucene_mkdir(struct lucene_fts_backend *backend)
{
- if (backend->lstorage->selected_box != backend->box) {
- lucene_index_select_mailbox(backend->lstorage->index,
- mailbox_get_name(backend->box));
- backend->lstorage->selected_box = backend->box;
- }
+ if (backend->dir_created)
+ return 0;
+
+ backend->dir_created = TRUE;
+ return mailbox_list_mkdir_root(backend->backend.ns->list,
+ backend->dir_path,
+ MAILBOX_LIST_PATH_TYPE_INDEX);
}
-static struct fts_backend *fts_backend_lucene_init(struct mailbox *box)
+static int
+fts_backend_select(struct lucene_fts_backend *backend, struct mailbox *box)
{
- struct lucene_mail_storage *lstorage;
- struct lucene_fts_backend *backend;
- const char *path, *lock_path;
-
- lstorage = LUCENE_CONTEXT(box->storage);
- if (lstorage == NULL) {
- path = mailbox_list_get_path(box->list, "INBOX",
- MAILBOX_LIST_PATH_TYPE_INDEX);
- if (path == NULL) {
- /* in-memory indexes */
- if (box->storage->set->mail_debug)
- i_debug("fts squat: Disabled with in-memory indexes");
- return NULL;
- }
+ struct mailbox_metadata metadata;
+ buffer_t buf;
+ unsigned char guid_hex[MAILBOX_GUID_HEX_LENGTH];
+ wchar_t wguid_hex[MAILBOX_GUID_HEX_LENGTH];
+ unsigned int i;
- path = t_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL);
- lock_path = t_strdup_printf("%s/"LUCENE_LOCK_SUBDIR_NAME, path);
- if (mkdir_parents(lock_path, 0700) < 0 && errno != EEXIST) {
- i_error("mkdir_parents(%s) failed: %m", lock_path);
- return NULL;
+ if (backend->selected_box == box)
+ return 0;
+
+ if (fts_backend_lucene_mkdir(backend) < 0)
+ return -1;
+
+ if (box != NULL) {
+ if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
+ &metadata) < 0) {
+ i_error("fts-lucene: Couldn't get mailbox %s GUID: %s",
+ box->vname, mailbox_get_last_error(box, NULL));
+ return -1;
}
- lstorage = i_new(struct lucene_mail_storage, 1);
- lstorage->index = lucene_index_init(path, lock_path);
- MODULE_CONTEXT_SET(box->storage, fts_lucene_storage_module,
- lstorage);
+ buffer_create_data(&buf, guid_hex, sizeof(guid_hex));
+ binary_to_hex_append(&buf, metadata.guid, MAIL_GUID_128_SIZE);
+ for (i = 0; i < N_ELEMENTS(wguid_hex); i++)
+ wguid_hex[i] = guid_hex[i];
+
+ lucene_index_select_mailbox(backend->index, wguid_hex);
+ } else {
+ lucene_index_unselect_mailbox(backend->index);
}
- lstorage->refcount++;
+ backend->selected_box = box;
+ return 0;
+}
+
+static struct fts_backend *fts_backend_lucene_alloc(void)
+{
+ struct lucene_fts_backend *backend;
backend = i_new(struct lucene_fts_backend, 1);
backend->backend = fts_backend_lucene;
- backend->lstorage = lstorage;
- backend->box = box;
return &backend->backend;
}
+static int
+fts_backend_lucene_init(struct fts_backend *_backend,
+ const char **error_r ATTR_UNUSED)
+{
+ struct lucene_fts_backend *backend =
+ (struct lucene_fts_backend *)_backend;
+ struct mailbox_list *list = _backend->ns->list;
+ const char *path;
+
+ path = mailbox_list_get_path(list, NULL,
+ MAILBOX_LIST_PATH_TYPE_INDEX);
+ i_assert(path != NULL); /* fts already checked this */
+
+ backend->dir_path = i_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL);
+ backend->index = lucene_index_init(backend->dir_path);
+ return 0;
+}
+
static void fts_backend_lucene_deinit(struct fts_backend *_backend)
{
struct lucene_fts_backend *backend =
(struct lucene_fts_backend *)_backend;
- if (--backend->lstorage->refcount == 0) {
- MODULE_CONTEXT_UNSET(backend->box->storage,
- fts_lucene_storage_module);
- lucene_index_deinit(backend->lstorage->index);
- i_free(backend->lstorage);
- }
+ lucene_index_deinit(backend->index);
+ i_free(backend->dir_path);
i_free(backend);
}
static int
fts_backend_lucene_get_last_uid(struct fts_backend *_backend,
- uint32_t *last_uid_r)
+ struct mailbox *box, uint32_t *last_uid_r)
{
struct lucene_fts_backend *backend =
(struct lucene_fts_backend *)_backend;
- fts_backend_select(backend);
- return lucene_index_get_last_uid(backend->lstorage->index, last_uid_r);
+ if (fts_index_get_last_uid(box, last_uid_r))
+ return 0;
+
+ /* either nothing has been indexed, or the index was corrupted.
+ do it the slow way. */
+ if (fts_backend_select(backend, box) < 0)
+ return -1;
+ if (lucene_index_get_last_uid(backend->index, last_uid_r) < 0)
+ return -1;
+
+ (void)fts_index_set_last_uid(box, *last_uid_r);
+ return 0;
}
-static int
-fts_backend_lucene_build_init(struct fts_backend *_backend,
- uint32_t *last_uid_r,
- struct fts_backend_build_context **ctx_r)
+static struct fts_backend_update_context *
+fts_backend_lucene_update_init(struct fts_backend *_backend)
{
struct lucene_fts_backend *backend =
(struct lucene_fts_backend *)_backend;
- struct lucene_fts_backend_build_context *ctx;
- uint32_t last_uid;
+ struct lucene_fts_backend_update_context *ctx;
- fts_backend_select(backend);
- if (lucene_index_build_init(backend->lstorage->index,
- &last_uid) < 0)
- return -1;
+ i_assert(!backend->updating);
- ctx = i_new(struct lucene_fts_backend_build_context, 1);
+ ctx = i_new(struct lucene_fts_backend_update_context, 1);
ctx->ctx.backend = _backend;
- ctx->uid = last_uid + 1;
+ backend->updating = TRUE;
- *last_uid_r = last_uid;
- *ctx_r = &ctx->ctx;
- return 0;
+ if (fts_backend_lucene_mkdir(backend) < 0)
+ ctx->ctx.failed = TRUE;
+ if (lucene_index_build_init(backend->index) < 0)
+ ctx->ctx.failed = TRUE;
+ return &ctx->ctx;
+}
+
+static bool
+fts_backend_lucene_need_optimize(struct lucene_fts_backend_update_context *ctx)
+{
+ struct lucene_fts_backend *backend =
+ (struct lucene_fts_backend *)ctx->ctx.backend;
+ const struct dotlock_settings dotlock_set = {
+ .timeout = 1,
+ .stale_timeout = 30,
+ .use_excl_lock = TRUE
+ };
+ struct dotlock *dotlock;
+ const char *path;
+ char buf[MAX_INT_STRLEN];
+ unsigned int expunges = 0;
+ uint32_t numdocs;
+ int fdw, fdr;
+
+ if (ctx->added_msgs >= LUCENE_OPTIMIZE_BATCH_MSGS_COUNT)
+ return TRUE;
+
+ if (ctx->expunges == 0)
+ return FALSE;
+
+ if (lucene_index_get_doc_count(backend->index, &numdocs) < 0)
+ return FALSE;
+
+ /* update pending expunges count */
+ path = t_strconcat(backend->dir_path, "/"LUCENE_EXPUNGE_FILENAME, NULL);
+ fdw = file_dotlock_open(&dotlock_set, path, 0, &dotlock);
+ if (fdw == -1)
+ return FALSE;
+
+ fdr = open(path, O_RDONLY);
+ if (fdr == -1) {
+ if (errno != ENOENT)
+ i_error("open(%s) failed: %m", path);
+ } else {
+ if (read(fdr, buf, sizeof(buf)) < 0)
+ i_error("read(%s) failed: %m", path);
+ if (str_to_uint(buf, &expunges) < 0)
+ i_error("%s is corrupted", path);
+ if (close(fdr) < 0)
+ i_error("close(%s) failed: %m", path);
+ }
+ expunges += ctx->expunges;
+
+ i_snprintf(buf, sizeof(buf), "%u", expunges);
+ if (write(fdw, buf, strlen(buf)) < 0)
+ i_error("write(%s) failed: %m", path);
+ if (close(fdw) < 0)
+ i_error("close(%s) failed: %m", path);
+ return numdocs / expunges <= 50; /* >2% of index has been expunged */
+}
+
+static int
+fts_backend_lucene_update_deinit(struct fts_backend_update_context *_ctx)
+{
+ struct lucene_fts_backend_update_context *ctx =
+ (struct lucene_fts_backend_update_context *)_ctx;
+ struct lucene_fts_backend *backend =
+ (struct lucene_fts_backend *)_ctx->backend;
+ int ret = _ctx->failed ? -1 : 0;
+
+ i_assert(backend->updating);
+
+ backend->updating = FALSE;
+ lucene_index_build_deinit(backend->index);
+
+ if (fts_backend_lucene_need_optimize(ctx))
+ (void)fts_backend_optimize(_ctx->backend);
+
+ i_free(ctx);
+ return ret;
}
static void
-fts_backend_lucene_build_hdr(struct fts_backend_build_context *_ctx,
- uint32_t uid)
+fts_backend_lucene_update_set_mailbox(struct fts_backend_update_context *_ctx,
+ struct mailbox *box)
{
- struct lucene_fts_backend_build_context *ctx =
- (struct lucene_fts_backend_build_context *)_ctx;
+ struct lucene_fts_backend_update_context *ctx =
+ (struct lucene_fts_backend_update_context *)_ctx;
+ struct lucene_fts_backend *backend =
+ (struct lucene_fts_backend *)_ctx->backend;
- i_assert(uid >= ctx->uid);
+ if (ctx->last_uid != 0) {
+ (void)fts_index_set_last_uid(ctx->box, ctx->last_uid);
+ ctx->last_uid = 0;
+ }
+ ctx->box = box;
- ctx->uid = uid;
- ctx->hdr = TRUE;
+ if (fts_backend_select(backend, box) < 0)
+ _ctx->failed = TRUE;
}
-static bool
-fts_backend_lucene_build_body_begin(struct fts_backend_build_context *_ctx,
- uint32_t uid, const char *content_type,
- const char *content_disposition ATTR_UNUSED)
+static void
+fts_backend_lucene_update_expunge(struct fts_backend_update_context *_ctx,
+ uint32_t uid ATTR_UNUSED)
{
- struct lucene_fts_backend_build_context *ctx =
- (struct lucene_fts_backend_build_context *)_ctx;
+ struct lucene_fts_backend_update_context *ctx =
+ (struct lucene_fts_backend_update_context *)_ctx;
- i_assert(uid >= ctx->uid);
+ ctx->expunges++;
+}
- if (!fts_backend_default_can_index(content_type))
- return FALSE;
+static bool
+fts_backend_lucene_update_set_build_key(struct fts_backend_update_context *_ctx,
+ const struct fts_backend_build_key *key)
+{
+ struct lucene_fts_backend_update_context *ctx =
+ (struct lucene_fts_backend_update_context *)_ctx;
+
+ switch (key->type) {
+ case FTS_BACKEND_BUILD_KEY_HDR:
+ case FTS_BACKEND_BUILD_KEY_MIME_HDR:
+ i_assert(key->hdr_name != NULL);
+
+ i_free(ctx->hdr_name);
+ ctx->hdr_name = i_strdup(key->hdr_name);
+ break;
+ case FTS_BACKEND_BUILD_KEY_BODY_PART:
+ i_free_and_null(ctx->hdr_name);
+ break;
+ case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
+ i_unreached();
+ }
+
+ if (key->uid != ctx->last_uid) {
+ i_assert(key->uid >= ctx->last_uid);
+ ctx->last_uid = key->uid;
+ ctx->added_msgs++;
+ }
- ctx->uid = uid;
- ctx->hdr = FALSE;
+ ctx->uid = key->uid;
return TRUE;
}
+static void
+fts_backend_lucene_update_unset_build_key(struct fts_backend_update_context *_ctx)
+{
+ struct lucene_fts_backend_update_context *ctx =
+ (struct lucene_fts_backend_update_context *)_ctx;
+
+ ctx->uid = 0;
+ i_free_and_null(ctx->hdr_name);
+}
+
static int
-fts_backend_lucene_build_more(struct fts_backend_build_context *_ctx,
- const unsigned char *data, size_t size)
+fts_backend_lucene_update_build_more(struct fts_backend_update_context *_ctx,
+ const unsigned char *data, size_t size)
{
- struct lucene_fts_backend_build_context *ctx =
- (struct lucene_fts_backend_build_context *)_ctx;
+ struct lucene_fts_backend_update_context *ctx =
+ (struct lucene_fts_backend_update_context *)_ctx;
struct lucene_fts_backend *backend =
(struct lucene_fts_backend *)_ctx->backend;
+ int ret;
+
+ i_assert(ctx->uid != 0);
if (_ctx->failed)
return -1;
- i_assert(backend->lstorage->selected_box == backend->box);
- return lucene_index_build_more(backend->lstorage->index,
- ctx->uid, data, size, ctx->hdr);
+ T_BEGIN {
+ ret = lucene_index_build_more(backend->index, ctx->uid,
+ data, size, ctx->hdr_name);
+ } T_END;
+ return ret;
+}
+
+static int
+fts_backend_lucene_refresh(struct fts_backend *_backend ATTR_UNUSED)
+{
+ return 0;
+}
+
+static int
+optimize_mailbox_get_uids(struct mailbox *box, ARRAY_TYPE(seq_range) *uids)
+{
+ struct mailbox_status status;
+
+ if (mailbox_get_status(box, STATUS_MESSAGES, &status) < 0)
+ return -1;
+
+ if (status.messages > 0) T_BEGIN {
+ ARRAY_TYPE(seq_range) seqs;
+
+ t_array_init(&seqs, 2);
+ seq_range_array_add_range(&seqs, 1, status.messages);
+ mailbox_get_uid_range(box, &seqs, uids);
+ } T_END;
+ return 0;
}
static int
-fts_backend_lucene_build_deinit(struct fts_backend_build_context *ctx)
+optimize_mailbox_update_last_uid(struct mailbox *box,
+ const ARRAY_TYPE(seq_range) *missing_uids)
+{
+ struct mailbox_status status;
+ const struct seq_range *range;
+ unsigned int count;
+ uint32_t last_uid;
+
+ /* FIXME: missing_uids from the middle of mailbox could probably
+ be handled better. they now create duplicates on next indexing? */
+
+ range = array_get(missing_uids, &count);
+ if (count > 0)
+ last_uid = range[0].seq1 - 1;
+ else {
+ mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
+ last_uid = status.uidnext - 1;
+ }
+ return fts_index_set_last_uid(box, last_uid);
+}
+
+static int fts_backend_lucene_optimize(struct fts_backend *_backend)
{
struct lucene_fts_backend *backend =
- (struct lucene_fts_backend *)ctx->backend;
- int ret = ctx->failed ? -1 : 0;
+ (struct lucene_fts_backend *)_backend;
+ struct mailbox_list_iterate_context *iter;
+ const struct mailbox_info *info;
+ struct mailbox *box;
+ ARRAY_TYPE(seq_range) uids, missing_uids;
+ int ret = 0;
+
+ i_array_init(&uids, 128);
+ i_array_init(&missing_uids, 32);
+ iter = mailbox_list_iter_init(_backend->ns->list, "*",
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
+ while ((info = mailbox_list_iter_next(iter)) != NULL) {
+ if ((info->flags &
+ (MAILBOX_NOSELECT | MAILBOX_NONEXISTENT)) != 0)
+ continue;
+
+ box = mailbox_alloc(info->ns->list, info->name,
+ MAILBOX_FLAG_KEEP_RECENT);
+ if (fts_backend_select(backend, box) < 0 ||
+ optimize_mailbox_get_uids(box, &uids) < 0 ||
+ lucene_index_optimize_scan(backend->index, &uids,
+ &missing_uids) < 0 ||
+ optimize_mailbox_update_last_uid(box, &missing_uids))
+ ret = -1;
+
+ fts_backend_select(backend, NULL);
+ mailbox_free(&box);
+ array_clear(&uids);
+ array_clear(&missing_uids);
+ }
+ if (mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
- i_assert(backend->lstorage->selected_box == backend->box);
- lucene_index_build_deinit(backend->lstorage->index);
- i_free(ctx);
+ /* FIXME: optionally go through mailboxes in index and delete those
+ that don't exist anymre */
+ if (lucene_index_optimize_finish(backend->index) < 0)
+ ret = -1;
+
+ array_free(&uids);
+ array_free(&missing_uids);
return ret;
}
-static void
-fts_backend_lucene_expunge(struct fts_backend *_backend, struct mail *mail)
+static int
+fts_backend_lucene_lookup(struct fts_backend *_backend, struct mailbox *box,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_result *result)
{
struct lucene_fts_backend *backend =
(struct lucene_fts_backend *)_backend;
+ int ret;
- fts_backend_select(backend);
- (void)lucene_index_expunge(backend->lstorage->index, mail->uid);
+ if (fts_backend_select(backend, box) < 0)
+ return -1;
+ T_BEGIN {
+ ret = lucene_index_lookup(backend->index, args, and_args,
+ result);
+ } T_END;
+ return ret;
}
-static void
-fts_backend_lucene_expunge_finish(struct fts_backend *_backend ATTR_UNUSED,
- struct mailbox *box ATTR_UNUSED,
- bool committed ATTR_UNUSED)
+/* a char* hash function from ASU -- from glib */
+static unsigned int wstr_hash(const void *p)
{
+ const wchar_t *s = p;
+ unsigned int g, h = 0;
+
+ while (*s != '\0') {
+ h = (h << 4) + *s;
+ if ((g = h & 0xf0000000UL)) {
+ h = h ^ (g >> 24);
+ h = h ^ g;
+ }
+ s++;
+ }
+
+ return h;
}
static int
-fts_backend_lucene_refresh(struct fts_backend *_backend ATTR_UNUSED)
+mailboxes_get_guids(struct mailbox *const boxes[],
+ struct hash_table *guids, struct fts_multi_result *result)
{
+ ARRAY_DEFINE(box_results, struct fts_result);
+ struct fts_result *box_result;
+ const char *guid;
+ wchar_t *guid_dup;
+ unsigned int i, j;
+
+ p_array_init(&box_results, result->pool, 32);
+ for (i = 0; boxes[i] != NULL; i++) {
+ if (fts_mailbox_get_guid(boxes[i], &guid) < 0)
+ return -1;
+
+ i_assert(strlen(guid) == MAILBOX_GUID_HEX_LENGTH);
+ guid_dup = t_new(wchar_t, MAILBOX_GUID_HEX_LENGTH + 1);
+ for (j = 0; j < MAILBOX_GUID_HEX_LENGTH; j++)
+ guid_dup[j] = guid[j];
+
+ box_result = array_append_space(&box_results);
+ box_result->box = boxes[i];
+ hash_table_insert(guids, guid_dup, box_result);
+ }
+
+ (void)array_append_space(&box_results);
+ result->box_results = array_idx_modifiable(&box_results, 0);
return 0;
}
static int
-fts_backend_lucene_lookup(struct fts_backend *_backend,
- const char *key, enum fts_lookup_flags flags,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids)
+fts_backend_lucene_lookup_multi(struct fts_backend *_backend,
+ struct mailbox *const boxes[],
+ struct mail_search_arg *args, bool and_args,
+ struct fts_multi_result *result)
{
struct lucene_fts_backend *backend =
(struct lucene_fts_backend *)_backend;
+ int ret;
- i_assert((flags & FTS_LOOKUP_FLAG_INVERT) == 0);
+ if (fts_backend_lucene_mkdir(backend) < 0)
+ return -1;
- array_clear(maybe_uids);
- fts_backend_select(backend);
- return lucene_index_lookup(backend->lstorage->index,
- flags, key, definite_uids);
+ T_BEGIN {
+ struct hash_table *guids;
+
+ guids = hash_table_create(default_pool, default_pool, 0,
+ wstr_hash,
+ (hash_cmp_callback_t *)wcscmp);
+ ret = mailboxes_get_guids(boxes, guids, result);
+ if (ret == 0) {
+ ret = lucene_index_lookup_multi(backend->index,
+ guids, args, and_args,
+ result);
+ }
+ hash_table_destroy(&guids);
+ } T_END;
+ return ret;
}
struct fts_backend fts_backend_lucene = {
.flags = 0,
{
+ fts_backend_lucene_alloc,
fts_backend_lucene_init,
fts_backend_lucene_deinit,
fts_backend_lucene_get_last_uid,
- NULL,
- fts_backend_lucene_build_init,
- fts_backend_lucene_build_hdr,
- fts_backend_lucene_build_body_begin,
- NULL,
- fts_backend_lucene_build_more,
- fts_backend_lucene_build_deinit,
- fts_backend_lucene_expunge,
- fts_backend_lucene_expunge_finish,
+ fts_backend_lucene_update_init,
+ fts_backend_lucene_update_deinit,
+ fts_backend_lucene_update_set_mailbox,
+ fts_backend_lucene_update_expunge,
+ fts_backend_lucene_update_set_build_key,
+ fts_backend_lucene_update_unset_build_key,
+ fts_backend_lucene_update_build_more,
fts_backend_lucene_refresh,
+ fts_backend_lucene_optimize,
+ fts_backend_default_can_lookup,
fts_backend_lucene_lookup,
- NULL,
- NULL
+ fts_backend_lucene_lookup_multi
}
};
#include "array.h"
#include "env-util.h"
#include "unichar.h"
+#include "hash.h"
#include "str.h"
-#include "str-sanitize.h"
+#include "strescape.h"
+#include "mail-search.h"
#include "lucene-wrapper.h"
#include <dirent.h>
#include <sys/stat.h>
};
#include <CLucene.h>
+#include <CLucene/util/CLStreams.h>
+#include <CLucene/search/MultiPhraseQuery.h>
/* Lucene's default is 10000. Use it here also.. */
#define MAX_TERMS_PER_DOCUMENT 10000
-/* If all the files in the lucene index directory are older than this many
- seconds, assume we can delete stale locks */
-#define STALE_INDEX_SECS 60
-/* When index is determined to be stale, delete all locks older than this */
-#define STALE_LOCK_SECS 60
-/* Minimum interval between staleness checks */
-#define STALENESS_CHECK_INTERVAL 10
-
using namespace lucene::document;
using namespace lucene::index;
using namespace lucene::search;
using namespace lucene::queryParser;
using namespace lucene::analysis;
+using namespace lucene::analysis;
+using namespace lucene::util;
struct lucene_index {
- char *path, *lock_path;
- wchar_t *mailbox_name;
-
- time_t last_stale_check;
- bool lock_error;
+ char *path;
+ wchar_t mailbox_guid[MAILBOX_GUID_HEX_LENGTH + 1];
IndexReader *reader;
IndexWriter *writer;
Analyzer *analyzer;
Document *doc;
- uint32_t prev_uid, last_uid;
+ uint32_t prev_uid;
};
-static bool lucene_dir_scan(const char *dir, const char *skip_path,
- time_t stale_stamp, bool unlink_staled)
-{
- DIR *d;
- struct dirent *dp;
- struct stat st;
- string_t *path;
- unsigned int dir_len;
- bool found_nonstale = FALSE;
-
- d = opendir(dir);
- if (d == NULL) {
- i_error("opendir(%s) failed: %m", dir);
- return TRUE;
- }
-
- t_push();
- path = t_str_new(256);
- str_append(path, dir);
- str_append_c(path, '/');
- dir_len = str_len(path);
-
- while ((dp = readdir(d)) != NULL) {
- if (*dp->d_name == '.') {
- if (dp->d_name[1] == '\0')
- continue;
- if (dp->d_name[1] == '.' && dp->d_name[2] == '\0')
- continue;
- }
-
- str_truncate(path, dir_len);
- str_append(path, dp->d_name);
-
- if (skip_path != NULL &&
- strcmp(str_c(path), skip_path) == 0)
- continue;
-
- if (stat(str_c(path), &st) < 0) {
- if (errno != ENOENT)
- i_error("stat(%s) failed: %m", str_c(path));
- found_nonstale = TRUE;
- } else if (st.st_ctime <= stale_stamp &&
- st.st_mtime <= stale_stamp) {
- if (unlink_staled) {
- if (unlink(str_c(path)) < 0 &&
- errno != ENOENT) {
- i_error("unlink(%s) failed: %m",
- str_c(path));
- }
- }
- } else {
- found_nonstale = TRUE;
- }
- }
- if (closedir(d) < 0)
- i_error("closedir(%s) failed: %m", dir);
- t_pop();
- return found_nonstale;
-}
-
-static void lucene_delete_stale_locks(struct lucene_index *index)
-{
- time_t now;
-
- now = time(NULL);
- if (index->last_stale_check + STALENESS_CHECK_INTERVAL > now)
- return;
- index->last_stale_check = now;
-
- if (lucene_dir_scan(index->path, index->lock_path,
- now - STALE_INDEX_SECS, FALSE)) {
- /* the index is probably being updated */
- return;
- }
- (void)lucene_dir_scan(index->lock_path, NULL,
- now - STALE_LOCK_SECS, TRUE);
-}
-
-struct lucene_index *lucene_index_init(const char *path, const char *lock_path)
+struct lucene_index *lucene_index_init(const char *path)
{
struct lucene_index *index;
- env_put(t_strconcat(LUCENE_LOCK_DIR_ENV_1"=", lock_path, NULL));
-
index = i_new(struct lucene_index, 1);
index->path = i_strdup(path);
- index->lock_path = i_strdup(lock_path);
index->analyzer = _CLNEW standard::StandardAnalyzer();
-
- lucene_delete_stale_locks(index);
return index;
}
{
lucene_index_close(index);
_CLDELETE(index->analyzer);
- i_free(index->mailbox_name);
i_free(index->path);
- i_free(index->lock_path);
i_free(index);
}
static void
-lucene_utf8_to_tchar(const char *src, wchar_t *dest, size_t destsize)
+lucene_utf8_n_to_tchar(const unsigned char *src, size_t srcsize,
+ wchar_t *dest, size_t destsize)
{
ARRAY_TYPE(unichars) dest_arr;
- buffer_t buf = { 0, 0 };
+ buffer_t buf = { 0, 0, { 0, 0, 0, 0, 0 } };
i_assert(sizeof(wchar_t) == sizeof(unichar_t));
buffer_create_data(&buf, dest, sizeof(wchar_t) * destsize);
array_create_from_buffer(&dest_arr, &buf, sizeof(wchar_t));
- if (uni_utf8_to_ucs4(src, &dest_arr) < 0)
+ if (uni_utf8_to_ucs4_n(src, srcsize, &dest_arr) < 0)
i_unreached();
i_assert(array_count(&dest_arr)+1 == destsize);
dest[destsize-1] = 0;
}
-static void
-lucene_utf8_n_to_tchar(const unsigned char *src, size_t srcsize,
- wchar_t *dest, size_t destsize)
+static const wchar_t *t_lucene_utf8_to_tchar(const char *str)
{
ARRAY_TYPE(unichars) dest_arr;
- buffer_t buf = { 0, 0 };
+ const unichar_t *ret;
i_assert(sizeof(wchar_t) == sizeof(unichar_t));
- buffer_create_data(&buf, dest, sizeof(wchar_t) * destsize);
- array_create_from_buffer(&dest_arr, &buf, sizeof(wchar_t));
- if (uni_utf8_to_ucs4_n(src, srcsize, &dest_arr) < 0)
+ t_array_init(&dest_arr, strlen(str) + 1);
+ if (uni_utf8_to_ucs4(str, &dest_arr) < 0)
i_unreached();
- i_assert(array_count(&dest_arr)+1 == destsize);
- dest[destsize-1] = 0;
+ (void)array_append_space(&dest_arr);
+ ret = array_idx(&dest_arr, 0);
+ return (const wchar_t *)ret;
}
void lucene_index_select_mailbox(struct lucene_index *index,
- const char *mailbox_name)
+ const wchar_t guid[MAILBOX_GUID_HEX_LENGTH])
{
- size_t size;
-
- i_free(index->mailbox_name);
+ memcpy(index->mailbox_guid, guid,
+ MAILBOX_GUID_HEX_LENGTH * sizeof(wchar_t));
+ index->mailbox_guid[MAILBOX_GUID_HEX_LENGTH] = '\0';
+}
- size = uni_utf8_strlen_n(mailbox_name, (size_t)-1) + 1;
- index->mailbox_name = i_new(wchar_t, size);
- lucene_utf8_to_tchar(mailbox_name, index->mailbox_name, size);
+void lucene_index_unselect_mailbox(struct lucene_index *index)
+{
+ memset(index->mailbox_guid, 0, sizeof(index->mailbox_guid));
}
static void lucene_handle_error(struct lucene_index *index, CLuceneError &err,
{
const char *what = err.what();
- if (err.number() == CL_ERR_IO && strncasecmp(what, "Lock", 4) == 0) {
- /* "Lock obtain timed out". delete any stale locks. */
- lucene_delete_stale_locks(index);
- if (index->lock_error) {
- /* we've already complained about this */
- return;
- }
- index->lock_error = TRUE;
- }
i_error("lucene index %s: %s failed: %s", index->path, msg, what);
}
return 0;
}
-static int
-lucene_index_get_last_uid_int(struct lucene_index *index, bool delete_old)
+int lucene_index_get_last_uid(struct lucene_index *index, uint32_t *last_uid_r)
{
- ARRAY_TYPE(uint32_t) delete_doc_ids;
- uint32_t del_id;
int ret = 0;
- bool deleted = false;
- index->last_uid = 0;
+ *last_uid_r = 0;
if ((ret = lucene_index_open_search(index)) <= 0)
return ret;
- /* find all the existing last_uids for selected mailbox.
- if there are more than one, delete the smaller ones. this is normal
- behavior because we can't update/delete documents in writer, so
- we'll do it only in here.. */
- Term mailbox_term(_T("box"), index->mailbox_name);
- Term last_uid_term(_T("last_uid"), _T("*"));
- TermQuery mailbox_query(&mailbox_term);
- WildcardQuery last_uid_query(&last_uid_term);
+ Term mailbox_term(_T("box"), index->mailbox_guid);
+ TermQuery query(&mailbox_term);
- BooleanQuery query;
- query.add(&mailbox_query, true, false);
- query.add(&last_uid_query, true, false);
-
- t_push();
- t_array_init(&delete_doc_ids, 10);
- int32_t last_doc_id = -1;
+ uint32_t last_uid = 0;
try {
Hits *hits = index->searcher->search(&query);
- for (int32_t i = 0; i < hits->length(); i++) {
+ for (size_t i = 0; i < hits->length(); i++) {
uint32_t uid;
if (lucene_doc_get_uid(index, &hits->doc(i),
- _T("last_uid"), &uid) < 0) {
+ _T("uid"), &uid) < 0) {
ret = -1;
break;
}
- if (uid > index->last_uid) {
- if (last_doc_id >= 0) {
- del_id = last_doc_id;
- array_append_i(&delete_doc_ids.arr,
- (void *)&del_id, 1);
- }
- index->last_uid = uid;
- last_doc_id = hits->id(i);
- } else {
- del_id = hits->id(i);
- array_append_i(&delete_doc_ids.arr,
- (void *)&del_id, 1);
- }
- }
- if (delete_old && array_count(&delete_doc_ids) > 0) {
- const uint32_t *ids;
- unsigned int i, count;
-
- ids = array_get(&delete_doc_ids, &count);
- for (i = 0; i < count; i++)
- index->reader->deleteDocument(ids[i]);
- deleted = true;
+ if (uid > last_uid)
+ last_uid = uid;
}
- index->lock_error = FALSE;
_CLDELETE(hits);
} catch (CLuceneError &err) {
lucene_handle_error(index, err, "last_uid search");
ret = -1;
}
-
- if (deleted) {
- /* the index was modified. we'll need to release the locks
- before opening a writer */
- lucene_index_close(index);
- }
- t_pop();
+ *last_uid_r = last_uid;
return ret;
}
-int lucene_index_get_last_uid(struct lucene_index *index, uint32_t *last_uid_r)
+int lucene_index_get_doc_count(struct lucene_index *index, uint32_t *count_r)
{
- /* delete the old last_uids in here, since we've not write-locked
- the index yet */
- if (lucene_index_get_last_uid_int(index, true) < 0)
- return -1;
+ int ret;
- *last_uid_r = index->last_uid;
- return 0;
+ if (index->reader == NULL) {
+ lucene_index_close(index);
+ if ((ret = lucene_index_open(index)) < 0)
+ return -1;
+ if (ret == 0) {
+ *count_r = 0;
+ return 0;
+ }
+ }
+ return index->reader->numDocs();
}
-int lucene_index_build_init(struct lucene_index *index, uint32_t *last_uid_r)
+int lucene_index_build_init(struct lucene_index *index)
{
- i_assert(index->mailbox_name != NULL);
-
- /* set this even if we fail so fts-storage won't crash */
- *last_uid_r = index->last_uid;
-
lucene_index_close(index);
bool exists = IndexReader::indexExists(index->path);
try {
index->writer = _CLNEW IndexWriter(index->path,
index->analyzer, !exists);
- index->lock_error = FALSE;
} catch (CLuceneError &err) {
lucene_handle_error(index, err, "IndexWriter()");
return -1;
}
index->writer->setMaxFieldLength(MAX_TERMS_PER_DOCUMENT);
-
- if (lucene_index_get_last_uid_int(index, false) < 0)
- return -1;
- *last_uid_r = index->last_uid;
return 0;
}
int lucene_index_build_more(struct lucene_index *index, uint32_t uid,
const unsigned char *data, size_t size,
- bool headers)
+ const char *hdr_name)
{
- size_t destsize;
-
- i_assert(uid > index->last_uid);
- i_assert(size > 0);
-
- destsize = uni_utf8_strlen_n(data, size) + 1;
- wchar_t dest[destsize];
- lucene_utf8_n_to_tchar(data, size, dest, destsize);
+ wchar_t id[MAX_INT_STRLEN];
+ size_t namesize, datasize;
if (uid != index->prev_uid) {
- wchar_t id[MAX_INT_STRLEN];
-
if (lucene_index_build_flush(index) < 0)
return -1;
index->prev_uid = uid;
index->doc = _CLNEW Document();
swprintf(id, N_ELEMENTS(id), L"%u", uid);
- index->doc->add(*new Field(_T("uid"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
- index->doc->add(*new Field(_T("box"), index->mailbox_name, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
+ index->doc->add(*_CLNEW Field(_T("uid"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
+ index->doc->add(*_CLNEW Field(_T("box"), index->mailbox_guid, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
}
- if (headers)
- index->doc->add(*new Field(_T("headers"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED));
- else
- index->doc->add(*new Field(_T("body"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED));
- return 0;
-}
-
-static int lucene_index_update_last_uid(struct lucene_index *index)
-{
- Document doc;
- wchar_t id[MAX_INT_STRLEN];
-
- swprintf(id, N_ELEMENTS(id), L"%u", index->last_uid);
- doc.add(*new Field(_T("last_uid"), id, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
- doc.add(*new Field(_T("box"), index->mailbox_name, Field::STORE_YES | Field::INDEX_UNTOKENIZED));
-
- try {
- index->writer->addDocument(&doc);
- return 0;
- } catch (CLuceneError &err) {
- lucene_handle_error(index, err, "IndexWriter::addDocument()");
- return -1;
+ datasize = uni_utf8_strlen_n(data, size) + 1;
+ wchar_t dest[datasize];
+ lucene_utf8_n_to_tchar(data, size, dest, datasize);
+
+ if (hdr_name != NULL) {
+ /* hdr_name should be ASCII, but don't break in case it isn't */
+ namesize = uni_utf8_strlen(hdr_name) + 1;
+ wchar_t wname[namesize];
+ lucene_utf8_n_to_tchar((const unsigned char *)hdr_name,
+ strlen(hdr_name), wname, namesize);
+ index->doc->add(*_CLNEW Field(_T("hdr"), wname, Field::STORE_NO | Field::INDEX_UNTOKENIZED));
+ index->doc->add(*_CLNEW Field(_T("hdr"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED));
+
+ if (fts_header_want_indexed(hdr_name))
+ index->doc->add(*_CLNEW Field(wname, dest, Field::STORE_NO | Field::INDEX_TOKENIZED));
+ } else if (size > 0) {
+ index->doc->add(*_CLNEW Field(_T("body"), dest, Field::STORE_NO | Field::INDEX_TOKENIZED));
}
+ return 0;
}
int lucene_index_build_deinit(struct lucene_index *index)
/* no changes. */
return 0;
}
-
- if (index->prev_uid > index->last_uid)
- index->last_uid = index->prev_uid;
index->prev_uid = 0;
if (index->writer == NULL) {
if (lucene_index_build_flush(index) < 0)
ret = -1;
- if (lucene_index_update_last_uid(index) < 0)
- ret = -1;
try {
index->writer->optimize();
return ret;
}
-int lucene_index_expunge(struct lucene_index *index, uint32_t uid)
-{
- wchar_t id[MAX_INT_STRLEN];
- int ret;
-
- if ((ret = lucene_index_open_search(index)) <= 0)
- return ret;
+struct uid_id_map {
+ uint32_t imap_uid;
+ int32_t lucene_id;
+};
+ARRAY_DEFINE_TYPE(uid_id_map, struct uid_id_map);
- swprintf(id, N_ELEMENTS(id), L"%u", uid);
+static int uid_id_map_cmp(const struct uid_id_map *u1,
+ const struct uid_id_map *u2)
+{
+ if (u1->imap_uid < u2->imap_uid)
+ return -1;
+ if (u1->imap_uid > u2->imap_uid)
+ return 1;
+ return 0;
+}
- Term mailbox_term(_T("box"), index->mailbox_name);
- Term uid_term(_T("uid"), id);
- TermQuery mailbox_query(&mailbox_term);
- TermQuery uid_query(&uid_term);
+static int get_mailbox_uid_id_map(struct lucene_index *index,
+ ARRAY_TYPE(uid_id_map) *uid_id_map)
+{
+ int ret = 0;
- BooleanQuery query;
- query.add(&mailbox_query, true, false);
- query.add(&uid_query, true, false);
+ /* get a sorted map of imap uid -> lucene id */
+ Term mailbox_term(_T("box"), index->mailbox_guid);
+ TermQuery query(&mailbox_term);
try {
Hits *hits = index->searcher->search(&query);
- for (int32_t i = 0; i < hits->length(); i++)
- index->reader->deleteDocument(hits->id(i));
- index->lock_error = FALSE;
+ for (size_t i = 0; i < hits->length(); i++) {
+ uint32_t uid;
+
+ if (lucene_doc_get_uid(index, &hits->doc(i),
+ _T("uid"), &uid) < 0) {
+ ret = -1;
+ break;
+ }
+ struct uid_id_map *ui = array_append_space(uid_id_map);
+ ui->imap_uid = uid;
+ ui->lucene_id = hits->id(i);
+ }
_CLDELETE(hits);
} catch (CLuceneError &err) {
lucene_handle_error(index, err, "expunge search");
ret = -1;
}
+ array_sort(uid_id_map, uid_id_map_cmp);
+ return ret;
+}
+
+int lucene_index_optimize_scan(struct lucene_index *index,
+ const ARRAY_TYPE(seq_range) *existing_uids,
+ ARRAY_TYPE(seq_range) *missing_uids_r)
+{
+ ARRAY_TYPE(uid_id_map) uid_id_map_arr;
+ const struct uid_id_map *uid_id_map;
+ struct seq_range_iter iter;
+ unsigned int n, i, count;
+ uint32_t uid;
+ int ret;
+
+ if ((ret = lucene_index_open_search(index)) <= 0)
+ return ret;
+
+ i_array_init(&uid_id_map_arr, 128);
+ if (get_mailbox_uid_id_map(index, &uid_id_map_arr) < 0)
+ return -1;
+ uid_id_map = array_get(&uid_id_map_arr, &count);
+
+ seq_range_array_iter_init(&iter, existing_uids); n = i = 0;
+ while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+ while (i < count && uid_id_map[i].imap_uid < uid) {
+ /* expunged message */
+ index->reader->deleteDocument(uid_id_map[i].lucene_id);
+ i++;
+ }
+
+ if (i == count || uid_id_map[i].imap_uid > uid) {
+ /* uid is missing from index */
+ seq_range_array_add(missing_uids_r, 0, uid);
+ } else {
+ i++;
+ }
+ }
+ for (; i < count; i++)
+ index->reader->deleteDocument(uid_id_map[i].lucene_id);
+
+ array_free(&uid_id_map_arr);
+ return ret;
+}
+
+int lucene_index_optimize_finish(struct lucene_index *index)
+{
+ int ret = 0;
+
+ if (IndexReader::isLocked(index->path))
+ IndexReader::unlock(index->path);
+
+ IndexWriter *writer =
+ _CLNEW IndexWriter(index->path, index->analyzer, false);
try {
- index->reader->close();
+ writer->optimize();
} catch (CLuceneError &err) {
- lucene_handle_error(index, err, "IndexReader::close()");
+ lucene_handle_error(index, err, "IndexWriter::optimize()");
ret = -1;
}
-
- lucene_index_close(index);
+ _CLDELETE(writer);
return ret;
}
-int lucene_index_lookup(struct lucene_index *index, enum fts_lookup_flags flags,
- const char *key, ARRAY_TYPE(seq_range) *result)
+// Mostly copy&pasted from CLucene's QueryParser
+static Query* getFieldQuery(Analyzer *analyzer, const TCHAR* _field, const TCHAR* queryText) {
+ // Use the analyzer to get all the tokens, and then build a TermQuery,
+ // PhraseQuery, or nothing based on the term count
+
+ StringReader reader(queryText);
+ TokenStream* source = analyzer->tokenStream(_field, &reader);
+
+ CLVector<CL_NS(analysis)::Token*, Deletor::Object<CL_NS(analysis)::Token> > v;
+ CL_NS(analysis)::Token* t = NULL;
+ int32_t positionCount = 0;
+ bool severalTokensAtSamePosition = false;
+
+ while (true) {
+ t = _CLNEW Token();
+ try {
+ Token* _t = source->next(t);
+ if (_t == NULL) _CLDELETE(t);
+ }_CLCATCH_ERR(CL_ERR_IO, _CLLDELETE(source);_CLLDELETE(t);,{
+ t = NULL;
+ });
+ if (t == NULL)
+ break;
+ v.push_back(t);
+ if (t->getPositionIncrement() != 0)
+ positionCount += t->getPositionIncrement();
+ else
+ severalTokensAtSamePosition = true;
+ }
+ try {
+ source->close();
+ }
+ _CLCATCH_ERR_CLEANUP(CL_ERR_IO, {_CLLDELETE(source);_CLLDELETE(t);} ); /* cleanup */
+ _CLLDELETE(source);
+
+ if (v.size() == 0)
+ return NULL;
+ else if (v.size() == 1) {
+ Term* tm = _CLNEW Term(_field, v.at(0)->termBuffer());
+ Query* ret = _CLNEW PrefixQuery( tm );
+ _CLDECDELETE(tm);
+ return ret;
+ } else {
+ if (severalTokensAtSamePosition) {
+ if (positionCount == 1) {
+ // no phrase query:
+ BooleanQuery* q = _CLNEW BooleanQuery(true);
+ for(size_t i=0; i<v.size(); i++ ){
+ Term* tm = _CLNEW Term(_field, v.at(i)->termBuffer());
+ q->add(_CLNEW TermQuery(tm), true, BooleanClause::SHOULD);
+ _CLDECDELETE(tm);
+ }
+ return q;
+ }else {
+ MultiPhraseQuery* mpq = _CLNEW MultiPhraseQuery();
+ CLArrayList<Term*> multiTerms;
+ int32_t position = -1;
+ for (size_t i = 0; i < v.size(); i++) {
+ t = v.at(i);
+ if (t->getPositionIncrement() > 0 && multiTerms.size() > 0) {
+ ValueArray<Term*> termsArray(multiTerms.size());
+ multiTerms.toArray(termsArray.values);
+ mpq->add(&termsArray,position);
+ multiTerms.clear();
+ }
+ position += t->getPositionIncrement();
+ multiTerms.push_back(_CLNEW Term(_field, t->termBuffer()));
+ }
+ ValueArray<Term*> termsArray(multiTerms.size());
+ multiTerms.toArray(termsArray.values);
+ mpq->add(&termsArray,position);
+ return mpq;
+ }
+ }else {
+ PhraseQuery* pq = _CLNEW PhraseQuery();
+ int32_t position = -1;
+
+ for (size_t i = 0; i < v.size(); i++) {
+ t = v.at(i);
+ Term* tm = _CLNEW Term(_field, t->termBuffer());
+ position += t->getPositionIncrement();
+ pq->add(tm,position);
+ _CLDECDELETE(tm);
+ }
+ return pq;
+ }
+ }
+}
+
+static Query *
+lucene_get_query(struct lucene_index *index,
+ const TCHAR *key, const char *value)
+{
+ const TCHAR *wvalue = t_lucene_utf8_to_tchar(value);
+ return getFieldQuery(index->analyzer, key, wvalue);
+}
+
+static bool
+lucene_add_definite_query(struct lucene_index *index, BooleanQuery &query,
+ struct mail_search_arg *arg, bool and_args)
{
- const char *quoted_key;
+ Query *q;
+
+ if (arg->match_not && !and_args) {
+ /* FIXME: we could handle this by doing multiple queries.. */
+ return false;
+ }
+
+ switch (arg->type) {
+ case SEARCH_TEXT: {
+ BooleanQuery *bq = _CLNEW BooleanQuery();
+ Query *q1 = lucene_get_query(index, _T("hdr"), arg->value.str);
+ Query *q2 = lucene_get_query(index, _T("body"), arg->value.str);
+
+ bq->add(q1, true, BooleanClause::SHOULD);
+ bq->add(q2, true, BooleanClause::SHOULD);
+ q = bq;
+ break;
+ }
+ case SEARCH_BODY:
+ q = lucene_get_query(index, _T("body"), arg->value.str);
+ break;
+ case SEARCH_HEADER:
+ case SEARCH_HEADER_ADDRESS:
+ case SEARCH_HEADER_COMPRESS_LWSP:
+ if (!fts_header_want_indexed(arg->hdr_field_name))
+ return false;
+
+ q = lucene_get_query(index,
+ t_lucene_utf8_to_tchar(arg->hdr_field_name),
+ arg->value.str);
+ break;
+ default:
+ return false;
+ }
+
+ if (!and_args)
+ query.add(q, true, BooleanClause::SHOULD);
+ else if (!arg->match_not)
+ query.add(q, true, BooleanClause::MUST);
+ else
+ query.add(q, true, BooleanClause::MUST_NOT);
+ return true;
+}
+
+static bool
+lucene_add_maybe_query(struct lucene_index *index, BooleanQuery &query,
+ struct mail_search_arg *arg, bool and_args)
+{
+ Query *q;
+
+ if (arg->match_not && !and_args) {
+ /* FIXME: we could handle this by doing multiple queries.. */
+ return false;
+ }
+
+ switch (arg->type) {
+ case SEARCH_HEADER:
+ case SEARCH_HEADER_ADDRESS:
+ case SEARCH_HEADER_COMPRESS_LWSP:
+ if (fts_header_want_indexed(arg->hdr_field_name))
+ return false;
+
+ /* we can check if the search key exists in some header and
+ filter out the messages that have no chance of matching */
+ q = lucene_get_query(index, _T("hdr"), arg->value.str);
+ break;
+ default:
+ return false;
+ }
+
+ if (!and_args)
+ query.add(q, true, BooleanClause::SHOULD);
+ else if (!arg->match_not)
+ query.add(q, true, BooleanClause::MUST);
+ else
+ query.add(q, true, BooleanClause::MUST_NOT);
+ return true;
+}
+
+static int
+lucene_index_search(struct lucene_index *index,
+ Query &search_query, struct fts_result *result,
+ ARRAY_TYPE(seq_range) *uids_r)
+{
+ struct fts_score_map *score;
int ret = 0;
- i_assert((flags & (FTS_LOOKUP_FLAG_HEADER|FTS_LOOKUP_FLAG_BODY)) != 0);
+ BooleanQuery query;
+ query.add(&search_query, BooleanClause::MUST);
- if (lucene_index_open_search(index) <= 0)
- return -1;
+ Term mailbox_term(_T("box"), index->mailbox_guid);
+ TermQuery mailbox_query(&mailbox_term);
+ query.add(&mailbox_query, BooleanClause::MUST);
- t_push();
- quoted_key = strchr(key, ' ') == NULL ?
- t_strdup_printf("%s*", key) :
- t_strdup_printf("\"%s\"", key);
- unsigned int keysize = uni_utf8_strlen_n(quoted_key, (size_t)-1) + 1;
- wchar_t wkey[keysize];
- lucene_utf8_to_tchar(quoted_key, wkey, keysize);
- t_pop();
-
- BooleanQuery lookup_query;
- Query *content_query1 = NULL, *content_query2 = NULL;
try {
- if ((flags & FTS_LOOKUP_FLAG_HEADER) != 0) {
- content_query1 = QueryParser::parse(wkey, _T("headers"),
- index->analyzer);
- lookup_query.add(content_query1, false, false);
- }
- if ((flags & FTS_LOOKUP_FLAG_BODY) != 0) {
- content_query2 = QueryParser::parse(wkey, _T("body"),
- index->analyzer);
- lookup_query.add(content_query2, false, false);
+ Hits *hits = index->searcher->search(&query);
+
+ uint32_t last_uid = 0;
+ if (result != NULL)
+ result->scores_sorted = true;
+
+ for (size_t i = 0; i < hits->length(); i++) {
+ uint32_t uid;
+
+ if (lucene_doc_get_uid(index, &hits->doc(i),
+ _T("uid"), &uid) < 0) {
+ ret = -1;
+ break;
+ }
+
+ if (result != NULL) {
+ if (uid < last_uid)
+ result->scores_sorted = false;
+ last_uid = uid;
+
+ seq_range_array_add(uids_r, 0, uid);
+ score = array_append_space(&result->scores);
+ score->uid = uid;
+ score->score = hits->score(i);
+ }
}
+ _CLDELETE(hits);
+ return ret;
} catch (CLuceneError &err) {
- if (getenv("DEBUG") != NULL) {
- i_info("lucene: QueryParser::parse(%s) failed: %s",
- str_sanitize(key, 40), err.what());
- }
- if (content_query1 != NULL)
- _CLDELETE(content_query1);
- lucene_index_close(index);
+ lucene_handle_error(index, err, "search");
return -1;
}
+}
+
+int lucene_index_lookup(struct lucene_index *index,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_result *result)
+{
+ struct mail_search_arg *arg;
+
+ if (lucene_index_open_search(index) <= 0)
+ return -1;
+
+ BooleanQuery def_query;
+ bool have_definites = false;
+
+ for (arg = args; arg != NULL; arg = arg->next) {
+ if (lucene_add_definite_query(index, def_query, arg, and_args)) {
+ arg->match_always = true;
+ have_definites = true;
+ }
+ }
+
+ if (have_definites) {
+ if (lucene_index_search(index, def_query, result,
+ &result->definite_uids) < 0)
+ return -1;
+ }
+
+ BooleanQuery maybe_query;
+ bool have_maybies = false;
+
+ for (arg = args; arg != NULL; arg = arg->next) {
+ if (lucene_add_maybe_query(index, maybe_query, arg, and_args)) {
+ arg->match_always = true;
+ have_maybies = true;
+ }
+ }
+
+ if (have_maybies) {
+ if (lucene_index_search(index, maybe_query, NULL,
+ &result->maybe_uids) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static int
+lucene_index_search_multi(struct lucene_index *index, struct hash_table *guids,
+ Query &search_query, struct fts_multi_result *result)
+{
+ struct fts_score_map *score;
+ int ret = 0;
BooleanQuery query;
- Term mailbox_term(_T("box"), index->mailbox_name);
- TermQuery mailbox_query(&mailbox_term);
- query.add(&lookup_query, true, false);
- query.add(&mailbox_query, true, false);
+ query.add(&search_query, BooleanClause::MUST);
+
+ BooleanQuery mailbox_query;
+ struct hash_iterate_context *iter;
+ void *key, *value;
+ iter = hash_table_iterate_init(guids);
+ while (hash_table_iterate(iter, &key, &value)) {
+ Term *term = _CLNEW Term(_T("box"), (wchar_t *)key);
+ TermQuery *q = _CLNEW TermQuery(term);
+ mailbox_query.add(q, true, BooleanClause::SHOULD);
+ }
+ hash_table_iterate_deinit(&iter);
+ query.add(&mailbox_query, BooleanClause::MUST);
try {
Hits *hits = index->searcher->search(&query);
- for (int32_t i = 0; i < hits->length(); i++) {
+ for (size_t i = 0; i < hits->length(); i++) {
uint32_t uid;
+ Field *field = hits->doc(i).getField(_T("box"));
+ const TCHAR *box_guid = field == NULL ? NULL : field->stringValue();
+ if (box_guid == NULL) {
+ i_error("lucene: Corrupted FTS index %s: No mailbox for document",
+ index->path);
+ ret = -1;
+ break;
+ }
+ struct fts_result *br = (struct fts_result *)
+ hash_table_lookup(guids, (const void *)box_guid);
+ if (br == NULL) {
+ i_warning("lucene: Returned unexpected mailbox with GUID %ls", box_guid);
+ continue;
+ }
+
if (lucene_doc_get_uid(index, &hits->doc(i),
_T("uid"), &uid) < 0) {
ret = -1;
break;
}
- seq_range_array_add(result, 0, uid);
+ if (!array_is_created(&br->definite_uids)) {
+ p_array_init(&br->definite_uids, result->pool, 32);
+ p_array_init(&br->scores, result->pool, 32);
+ }
+ seq_range_array_add(&br->definite_uids, 0, uid);
+ score = array_append_space(&br->scores);
+ score->uid = uid;
+ score->score = hits->score(i);
}
- index->lock_error = FALSE;
_CLDELETE(hits);
+ return ret;
} catch (CLuceneError &err) {
- lucene_handle_error(index, err, "search");
- ret = -1;
+ lucene_handle_error(index, err, "multi search");
+ return -1;
}
+}
+int lucene_index_lookup_multi(struct lucene_index *index,
+ struct hash_table *guids,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_multi_result *result)
+{
+ struct mail_search_arg *arg;
- if (content_query1 != NULL)
- _CLDELETE(content_query1);
- if (content_query2 != NULL)
- _CLDELETE(content_query2);
- return ret;
+ if (lucene_index_open_search(index) <= 0)
+ return -1;
+
+ BooleanQuery def_query;
+ bool have_definites = false;
+
+ for (arg = args; arg != NULL; arg = arg->next) {
+ if (lucene_add_definite_query(index, def_query, arg, and_args)) {
+ arg->match_always = true;
+ have_definites = true;
+ }
+ }
+
+ if (have_definites) {
+ if (lucene_index_search_multi(index, guids,
+ def_query, result) < 0)
+ return -1;
+ }
+ return 0;
}
#include "fts-api-private.h"
-struct lucene_index *lucene_index_init(const char *path, const char *lock_path);
+#define MAILBOX_GUID_HEX_LENGTH (MAIL_GUID_128_SIZE*2)
+
+struct lucene_index *lucene_index_init(const char *path);
void lucene_index_deinit(struct lucene_index *index);
void lucene_index_select_mailbox(struct lucene_index *index,
- const char *mailbox_name);
+ const wchar_t guid[MAILBOX_GUID_HEX_LENGTH]);
+void lucene_index_unselect_mailbox(struct lucene_index *index);
int lucene_index_get_last_uid(struct lucene_index *index, uint32_t *last_uid_r);
+int lucene_index_get_doc_count(struct lucene_index *index, uint32_t *count_r);
-int lucene_index_build_init(struct lucene_index *index, uint32_t *last_uid_r);
+int lucene_index_build_init(struct lucene_index *index);
int lucene_index_build_more(struct lucene_index *index, uint32_t uid,
const unsigned char *data, size_t size,
- bool headers);
+ const char *hdr_name);
int lucene_index_build_deinit(struct lucene_index *index);
-int lucene_index_expunge(struct lucene_index *index, uint32_t uid);
+int lucene_index_optimize_scan(struct lucene_index *index,
+ const ARRAY_TYPE(seq_range) *existing_uids,
+ ARRAY_TYPE(seq_range) *missing_uids_r);
+int lucene_index_optimize_finish(struct lucene_index *index);
+
+int lucene_index_lookup(struct lucene_index *index,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_result *result);
-int lucene_index_lookup(struct lucene_index *index, enum fts_lookup_flags flags,
- const char *key, ARRAY_TYPE(seq_range) *result);
+int lucene_index_lookup_multi(struct lucene_index *index,
+ struct hash_table *guids,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_multi_result *result);
#endif
lib21_fts_solr_plugin_la_SOURCES = \
fts-backend-solr.c \
+ fts-backend-solr-old.c \
fts-solr-plugin.c \
solr-connection.c
--- /dev/null
+/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "hash.h"
+#include "strescape.h"
+#include "unichar.h"
+#include "mail-storage-private.h"
+#include "mailbox-list-private.h"
+#include "mail-search.h"
+#include "fts-api.h"
+#include "solr-connection.h"
+#include "fts-solr-plugin.h"
+
+#include <ctype.h>
+
+#define SOLR_CMDBUF_SIZE (1024*64)
+#define SOLR_MAX_MULTI_ROWS 100000
+
+struct solr_fts_backend {
+ struct fts_backend backend;
+ char *id_username, *id_namespace;
+ struct mail_namespace *default_ns;
+};
+
+struct solr_fts_backend_update_context {
+ struct fts_backend_update_context ctx;
+
+ struct mailbox *cur_box;
+ char *id_box_name;
+
+ struct solr_connection_post *post;
+ uint32_t prev_uid, uid_validity;
+ string_t *cmd, *hdr;
+
+ bool headers_open;
+ bool documents_added;
+};
+
+static struct solr_connection *solr_conn = NULL;
+
+static bool is_valid_xml_char(unichar_t chr)
+{
+ /* Valid characters in XML:
+
+ #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] |
+ [#x10000-#x10FFFF]
+
+ This function gets called only for #x80 and higher */
+ if (chr > 0xd7ff && chr < 0xe000)
+ return FALSE;
+ if (chr > 0xfffd && chr < 0x10000)
+ return FALSE;
+ return chr < 0x10ffff;
+}
+
+static void
+xml_encode_data(string_t *dest, const unsigned char *data, unsigned int len)
+{
+ unichar_t chr;
+ unsigned int i;
+
+ for (i = 0; i < len; i++) {
+ switch (data[i]) {
+ case '&':
+ str_append(dest, "&");
+ break;
+ case '<':
+ str_append(dest, "<");
+ break;
+ case '>':
+ str_append(dest, ">");
+ break;
+ case '\t':
+ case '\n':
+ case '\r':
+ /* exceptions to the following control char check */
+ str_append_c(dest, data[i]);
+ break;
+ default:
+ if (data[i] < 32) {
+ /* SOLR doesn't like control characters.
+ replace them with spaces. */
+ str_append_c(dest, ' ');
+ } else if (data[i] >= 0x80) {
+ /* make sure the character is valid for XML
+ so we don't get XML parser errors */
+ unsigned int char_len =
+ uni_utf8_char_bytes(data[i]);
+ if (i + char_len <= len &&
+ uni_utf8_get_char_n(data + i, char_len, &chr) == 1 &&
+ is_valid_xml_char(chr))
+ str_append_n(dest, data + i, char_len);
+ else {
+ str_append_n(dest, utf8_replacement_char,
+ UTF8_REPLACEMENT_CHAR_LEN);
+ }
+ i += char_len - 1;
+ } else {
+ str_append_c(dest, data[i]);
+ }
+ break;
+ }
+ }
+}
+
+static void xml_encode(string_t *dest, const char *str)
+{
+ xml_encode_data(dest, (const unsigned char *)str, strlen(str));
+}
+
+static const char *solr_escape_id_str(const char *str)
+{
+ string_t *tmp;
+ const char *p;
+
+ for (p = str; *p != '\0'; p++) {
+ if (*p == '/' || *p == '!')
+ break;
+ }
+ if (*p == '\0')
+ return str;
+
+ tmp = t_str_new(64);
+ for (p = str; *p != '\0'; p++) {
+ switch (*p) {
+ case '/':
+ str_append(tmp, "!\\");
+ break;
+ case '!':
+ str_append(tmp, "!!");
+ break;
+ default:
+ str_append_c(tmp, *p);
+ break;
+ }
+ }
+ return str_c(tmp);
+}
+
+static void solr_quote(string_t *dest, const char *str)
+{
+ str_append_c(dest, '"');
+ str_append(dest, str_escape(str));
+ str_append_c(dest, '"');
+}
+
+static void solr_quote_http(string_t *dest, const char *str)
+{
+ str_append(dest, "%22");
+ solr_connection_http_escape(solr_conn, dest, str);
+ str_append(dest, "%22");
+}
+
+static void fts_solr_set_default_ns(struct solr_fts_backend *backend)
+{
+ struct mail_namespace *ns = backend->backend.ns;
+ struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(ns->user);
+ const struct fts_solr_settings *set = &fuser->set;
+ const char *str;
+
+ if (backend->default_ns != NULL)
+ return;
+
+ if (set->default_ns_prefix != NULL) {
+ backend->default_ns =
+ mail_namespace_find_prefix(ns->user->namespaces,
+ set->default_ns_prefix);
+ if (backend->default_ns == NULL) {
+ i_error("fts_solr: default_ns setting points to "
+ "nonexistent namespace");
+ }
+ }
+ if (backend->default_ns == NULL) {
+ backend->default_ns =
+ mail_namespace_find_inbox(ns->user->namespaces);
+ }
+ while (backend->default_ns->alias_for != NULL)
+ backend->default_ns = backend->default_ns->alias_for;
+
+ if (ns != backend->default_ns) {
+ str = solr_escape_id_str(ns->prefix);
+ backend->id_namespace = i_strdup(str);
+ }
+}
+
+static void fts_box_name_get_root(struct mail_namespace **ns, const char **name)
+{
+ struct mail_namespace *orig_ns = *ns;
+
+ while ((*ns)->alias_for != NULL)
+ *ns = (*ns)->alias_for;
+
+ if (**name == '\0' && *ns != orig_ns &&
+ ((*ns)->flags & NAMESPACE_FLAG_INBOX_USER) != 0) {
+ /* ugly workaround to allow selecting INBOX from a Maildir/
+ when it's not in the inbox=yes namespace. */
+ *name = "INBOX";
+ }
+}
+
+static const char *
+fts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r)
+{
+ struct mail_namespace *ns = mailbox_get_namespace(box);
+ const char *name = box->name;
+
+ fts_box_name_get_root(&ns, &name);
+ *ns_r = ns;
+ return name;
+}
+
+static struct fts_backend *fts_backend_solr_alloc(void)
+{
+ struct solr_fts_backend *backend;
+
+ backend = i_new(struct solr_fts_backend, 1);
+ backend->backend = fts_backend_solr_old;
+ return &backend->backend;
+}
+
+static int
+fts_backend_solr_init(struct fts_backend *_backend,
+ const char **error_r ATTR_UNUSED)
+{
+ struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+ struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_backend->ns->user);
+ const struct fts_solr_settings *set = &fuser->set;
+ const char *str;
+
+ if (solr_conn == NULL)
+ solr_conn = solr_connection_init(set->url, set->debug);
+
+ str = solr_escape_id_str(_backend->ns->user->username);
+ backend->id_username = i_strdup(str);
+ return 0;
+}
+
+static void fts_backend_solr_deinit(struct fts_backend *_backend)
+{
+ struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+
+ i_free(backend->id_namespace);
+ i_free(backend->id_username);
+ i_free(backend);
+}
+
+static void
+solr_add_ns_query(string_t *str, struct solr_fts_backend *backend,
+ struct mail_namespace *ns, bool neg)
+{
+ while (ns->alias_for != NULL)
+ ns = ns->alias_for;
+
+ if (ns == backend->default_ns || *ns->prefix == '\0') {
+ if (!neg)
+ str_append(str, " -ns:[* TO *]");
+ else
+ str_append(str, " +ns:[* TO *]");
+ } else {
+ if (!neg)
+ str_append(str, " +ns:");
+ else
+ str_append(str, " -ns:");
+ solr_quote(str, ns->prefix);
+ }
+}
+
+static void
+solr_add_ns_query_http(string_t *str, struct solr_fts_backend *backend,
+ struct mail_namespace *ns)
+{
+ string_t *tmp;
+
+ tmp = t_str_new(64);
+ solr_add_ns_query(tmp, backend, ns, FALSE);
+ solr_connection_http_escape(solr_conn, str, str_c(tmp));
+}
+
+static int
+fts_backend_solr_get_last_uid_fallback(struct solr_fts_backend *backend,
+ struct mailbox *box,
+ uint32_t *last_uid_r)
+{
+ struct mail_namespace *ns;
+ struct mailbox_status status;
+ struct solr_result **results;
+ const struct seq_range *uidvals;
+ const char *box_name;
+ unsigned int count;
+ string_t *str;
+ pool_t pool;
+ int ret = 0;
+
+ str = t_str_new(256);
+ str_append(str, "fl=uid&rows=1&sort=uid+desc&q=");
+
+ box_name = fts_box_get_root(box, &ns);
+
+ mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
+ str_printfa(str, "uidv:%u+box:", status.uidvalidity);
+ solr_quote_http(str, box_name);
+ solr_add_ns_query_http(str, backend, ns);
+ str_append(str, "+user:");
+ solr_quote_http(str, ns->user->username);
+
+ pool = pool_alloconly_create("solr last uid lookup", 1024);
+ if (solr_connection_select(solr_conn, str_c(str),
+ pool, &results) < 0)
+ ret = -1;
+ else if (results[0] == NULL) {
+ /* no UIDs */
+ *last_uid_r = 0;
+ } else {
+ uidvals = array_get(&results[0]->uids, &count);
+ i_assert(count > 0);
+ if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
+ *last_uid_r = uidvals[0].seq1;
+ } else {
+ i_error("fts_solr: Last UID lookup returned multiple rows");
+ ret = -1;
+ }
+ }
+ pool_unref(&pool);
+ return ret;
+}
+
+static int
+fts_backend_solr_get_last_uid(struct fts_backend *_backend,
+ struct mailbox *box, uint32_t *last_uid_r)
+{
+ struct solr_fts_backend *backend =
+ (struct solr_fts_backend *)_backend;
+
+ if (fts_index_get_last_uid(box, last_uid_r))
+ return 0;
+
+ /* either nothing has been indexed, or the index was corrupted.
+ do it the slow way. */
+ if (fts_backend_solr_get_last_uid_fallback(backend, box, last_uid_r) < 0)
+ return -1;
+
+ (void)fts_index_set_last_uid(box, *last_uid_r);
+ return 0;
+}
+
+static struct fts_backend_update_context *
+fts_backend_solr_update_init(struct fts_backend *_backend)
+{
+ struct solr_fts_backend *backend =
+ (struct solr_fts_backend *)_backend;
+ struct solr_fts_backend_update_context *ctx;
+
+ ctx = i_new(struct solr_fts_backend_update_context, 1);
+ ctx->ctx.backend = _backend;
+ ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
+ ctx->hdr = str_new(default_pool, 4096);
+ fts_solr_set_default_ns(backend);
+ return &ctx->ctx;
+}
+
+static void xml_encode_id(struct solr_fts_backend_update_context *ctx,
+ string_t *str, uint32_t uid)
+{
+ struct solr_fts_backend *backend =
+ (struct solr_fts_backend *)ctx->ctx.backend;
+
+ if (uid != 0)
+ str_printfa(str, "%u/", uid);
+ else
+ str_append(str, "L/");
+
+ if (backend->id_namespace != NULL) {
+ xml_encode(str, backend->id_namespace);
+ str_append_c(str, '/');
+ }
+ str_printfa(str, "%u/", ctx->uid_validity);
+ xml_encode(str, backend->id_username);
+ str_append_c(str, '/');
+ xml_encode(str, ctx->id_box_name);
+}
+
+static void
+fts_backend_solr_add_doc_prefix(struct solr_fts_backend_update_context *ctx,
+ uint32_t uid)
+{
+ struct solr_fts_backend *backend =
+ (struct solr_fts_backend *)ctx->ctx.backend;
+ struct mailbox *box = ctx->cur_box;
+ struct mail_namespace *ns;
+ const char *box_name;
+
+ ctx->documents_added = TRUE;
+
+ str_printfa(ctx->cmd, "<doc>"
+ "<field name=\"uid\">%u</field>"
+ "<field name=\"uidv\">%u</field>",
+ uid, ctx->uid_validity);
+
+ box_name = fts_box_get_root(box, &ns);
+
+ if (ns != backend->default_ns) {
+ str_append(ctx->cmd, "<field name=\"ns\">");
+ xml_encode(ctx->cmd, ns->prefix);
+ str_append(ctx->cmd, "</field>");
+ }
+ str_append(ctx->cmd, "<field name=\"box\">");
+ xml_encode(ctx->cmd, box_name);
+ str_append(ctx->cmd, "</field><field name=\"user\">");
+ xml_encode(ctx->cmd, ns->user->username);
+ str_append(ctx->cmd, "</field>");
+}
+
+static int
+fts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx)
+{
+ if (ctx->post == NULL)
+ return 0;
+
+ str_append(ctx->cmd, "</doc></add>");
+
+ solr_connection_post_more(ctx->post, str_data(ctx->cmd),
+ str_len(ctx->cmd));
+ return solr_connection_post_end(ctx->post);
+}
+
+static int
+fts_backend_solr_update_deinit(struct fts_backend_update_context *_ctx)
+{
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
+ const char *str;
+ int ret;
+
+ ret = fts_backed_solr_build_commit(ctx);
+
+ /* commit and wait until the documents we just indexed are
+ visible to the following search */
+ str = t_strdup_printf("<commit waitFlush=\"false\" "
+ "waitSearcher=\"%s\"/>",
+ ctx->documents_added ? "true" : "false");
+ if (solr_connection_post(solr_conn, str) < 0)
+ ret = -1;
+
+ str_free(&ctx->cmd);
+ str_free(&ctx->hdr);
+ i_free(ctx->id_box_name);
+ i_free(ctx);
+ return ret;
+}
+
+static void
+fts_backend_solr_update_set_mailbox(struct fts_backend_update_context *_ctx,
+ struct mailbox *box)
+{
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
+ struct mailbox_status status;
+ struct mail_namespace *ns;
+
+ ctx->cur_box = box;
+ ctx->uid_validity = 0;
+ i_free_and_null(ctx->id_box_name);
+
+ if (box != NULL) {
+ ctx->id_box_name = i_strdup(fts_box_get_root(box, &ns));
+
+ mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
+ ctx->uid_validity = status.uidvalidity;
+ }
+}
+
+static void
+fts_backend_solr_update_expunge(struct fts_backend_update_context *_ctx,
+ uint32_t uid)
+{
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
+
+ T_BEGIN {
+ string_t *cmd;
+
+ cmd = t_str_new(256);
+ str_append(cmd, "<delete><id>");
+ xml_encode_id(ctx, cmd, uid);
+ str_append(cmd, "</id></delete>");
+
+ (void)solr_connection_post(solr_conn, str_c(cmd));
+ } T_END;
+}
+
+static void
+fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx,
+ uint32_t uid)
+{
+ if (ctx->post == NULL) {
+ i_assert(ctx->prev_uid == 0);
+
+ ctx->post = solr_connection_post_begin(solr_conn);
+ str_append(ctx->cmd, "<add>");
+ } else {
+ ctx->headers_open = FALSE;
+ str_append(ctx->cmd, "<field name=\"hdr\">");
+ str_append_str(ctx->cmd, ctx->hdr);
+ str_append(ctx->cmd, "</field>");
+ str_truncate(ctx->hdr, 0);
+
+ str_append(ctx->cmd, "</doc>");
+ }
+ ctx->prev_uid = uid;
+
+ fts_backend_solr_add_doc_prefix(ctx, uid);
+ str_printfa(ctx->cmd, "<field name=\"id\">");
+ xml_encode_id(ctx, ctx->cmd, uid);
+ str_append(ctx->cmd, "</field>");
+}
+
+static bool
+fts_backend_solr_update_set_build_key(struct fts_backend_update_context *_ctx,
+ const struct fts_backend_build_key *key)
+{
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
+
+ if (key->uid != ctx->prev_uid)
+ fts_backend_solr_uid_changed(ctx, key->uid);
+
+ switch (key->type) {
+ case FTS_BACKEND_BUILD_KEY_HDR:
+ case FTS_BACKEND_BUILD_KEY_MIME_HDR:
+ xml_encode(ctx->hdr, key->hdr_name);
+ str_append(ctx->hdr, ": ");
+ ctx->headers_open = TRUE;
+ break;
+ case FTS_BACKEND_BUILD_KEY_BODY_PART:
+ ctx->headers_open = FALSE;
+ str_append(ctx->cmd, "<field name=\"body\">");
+ break;
+ case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
+ i_unreached();
+ }
+ return TRUE;
+}
+
+static void
+fts_backend_solr_update_unset_build_key(struct fts_backend_update_context *_ctx)
+{
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
+
+ if (!ctx->headers_open)
+ str_append(ctx->cmd, "</field>");
+ else
+ str_append_c(ctx->hdr, '\n');
+}
+
+static int
+fts_backend_solr_update_build_more(struct fts_backend_update_context *_ctx,
+ const unsigned char *data, size_t size)
+{
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
+
+ xml_encode_data(ctx->cmd, data, size);
+ if (str_len(ctx->cmd) > SOLR_CMDBUF_SIZE-128) {
+ solr_connection_post_more(ctx->post, str_data(ctx->cmd),
+ str_len(ctx->cmd));
+ str_truncate(ctx->cmd, 0);
+ }
+ return 0;
+}
+
+static int fts_backend_solr_refresh(struct fts_backend *backend ATTR_UNUSED)
+{
+ return 0;
+}
+
+static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED)
+{
+ return 0;
+}
+
+static bool
+solr_add_definite_query(string_t *str, struct mail_search_arg *arg)
+{
+ switch (arg->type) {
+ case SEARCH_TEXT: {
+ if (arg->match_not)
+ str_append_c(str, '-');
+ str_append(str, "(hdr:");
+ solr_quote_http(str, arg->value.str);
+ str_append(str, "+OR+body:");
+ solr_quote_http(str, arg->value.str);
+ str_append(str, ")");
+ break;
+ }
+ case SEARCH_BODY:
+ if (arg->match_not)
+ str_append_c(str, '-');
+ str_append(str, "body:");
+ solr_quote_http(str, arg->value.str);
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static bool
+solr_add_definite_query_args(string_t *str, struct mail_search_arg *arg,
+ bool and_args)
+{
+ unsigned int last_len;
+
+ last_len = str_len(str);
+ for (; arg != NULL; arg = arg->next) {
+ if (solr_add_definite_query(str, arg)) {
+ arg->match_always = TRUE;
+ last_len = str_len(str);
+ if (and_args)
+ str_append(str, "+AND+");
+ else
+ str_append(str, "+OR+");
+ }
+ }
+ if (str_len(str) == last_len)
+ return FALSE;
+
+ str_truncate(str, last_len);
+ return TRUE;
+}
+
+static int
+fts_backend_solr_lookup(struct fts_backend *_backend, struct mailbox *box,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_result *result)
+{
+ struct solr_fts_backend *backend =
+ (struct solr_fts_backend *)_backend;
+ struct mail_namespace *ns;
+ struct mailbox_status status;
+ string_t *str;
+ const char *box_name;
+ pool_t pool;
+ struct solr_result **results;
+ int ret;
+
+ fts_solr_set_default_ns(backend);
+ mailbox_get_open_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT,
+ &status);
+
+ str = t_str_new(256);
+ str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=",
+ status.uidnext);
+
+ if (!solr_add_definite_query_args(str, args, and_args)) {
+ /* can't search this query */
+ return 0;
+ }
+
+ /* use a separate filter query for selecting the mailbox. it shouldn't
+ affect the score and there could be some caching benefits too. */
+ str_append(str, "&fq=%2Buser:");
+ solr_quote_http(str, box->storage->user->username);
+ box_name = fts_box_get_root(box, &ns);
+ str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity);
+ solr_quote_http(str, box_name);
+ solr_add_ns_query_http(str, backend, ns);
+
+ pool = pool_alloconly_create("fts solr search", 1024);
+ ret = solr_connection_select(solr_conn, str_c(str), pool, &results);
+ if (ret == 0 && results[0] != NULL) {
+ array_append_array(&result->definite_uids, &results[0]->uids);
+ array_append_array(&result->scores, &results[0]->scores);
+ }
+ result->scores_sorted = TRUE;
+ pool_unref(&pool);
+ return ret;
+}
+
+static char *
+mailbox_get_id(struct solr_fts_backend *backend, struct mail_namespace *ns,
+ const char *mailbox, uint32_t uidvalidity)
+{
+ string_t *str = t_str_new(64);
+
+ str_printfa(str, "%u\001", uidvalidity);
+ str_append(str, mailbox);
+ if (ns != backend->default_ns)
+ str_printfa(str, "\001%s", ns->prefix);
+ return str_c_modifiable(str);
+}
+
+static int
+solr_search_multi(struct solr_fts_backend *backend, string_t *str,
+ struct mailbox *const boxes[],
+ struct fts_multi_result *result)
+{
+ struct solr_result **solr_results;
+ struct fts_result *fts_result;
+ ARRAY_DEFINE(fts_results, struct fts_result);
+ struct mail_namespace *ns;
+ struct mailbox_status status;
+ struct hash_table *mailboxes;
+ struct mailbox *box;
+ const char *box_name;
+ char *box_id;
+ unsigned int i, len;
+
+ /* use a separate filter query for selecting the mailbox. it shouldn't
+ affect the score and there could be some caching benefits too. */
+ str_append(str, "&fq=%2Buser:");
+ if (backend->backend.ns->owner != NULL)
+ solr_quote_http(str, backend->backend.ns->owner->username);
+ else
+ str_append(str, "%22%22");
+
+ mailboxes = hash_table_create(default_pool, default_pool, 0,
+ str_hash, (hash_cmp_callback_t *)strcmp);
+ str_append(str, "%2B(");
+ len = str_len(str);
+ for (i = 0; boxes[i] != NULL; i++) {
+ if (str_len(str) != len)
+ str_append(str, "+OR+");
+
+ box_name = fts_box_get_root(boxes[i], &ns);
+ mailbox_get_open_status(boxes[i], STATUS_UIDVALIDITY, &status);
+ str_printfa(str, "%%2B(%%2Buidv:%u+%%2Bbox:", status.uidvalidity);
+ solr_quote_http(str, box_name);
+ solr_add_ns_query_http(str, backend, ns);
+ str_append_c(str, ')');
+
+ box_id = mailbox_get_id(backend, ns, box_name, status.uidvalidity);
+ hash_table_insert(mailboxes, box_id, boxes[i]);
+ }
+ str_append_c(str, ')');
+
+ if (solr_connection_select(solr_conn, str_c(str),
+ result->pool, &solr_results) < 0) {
+ hash_table_destroy(&mailboxes);
+ return -1;
+ }
+
+ p_array_init(&fts_results, result->pool, 32);
+ for (i = 0; solr_results[i] != NULL; i++) {
+ box = hash_table_lookup(mailboxes, solr_results[i]->box_id);
+ if (box == NULL) {
+ i_warning("fts_solr: Lookup returned unexpected mailbox "
+ "with id=%s", solr_results[i]->box_id);
+ continue;
+ }
+ fts_result = array_append_space(&fts_results);
+ fts_result->box = box;
+ fts_result->definite_uids = solr_results[i]->uids;
+ fts_result->scores = solr_results[i]->scores;
+ fts_result->scores_sorted = TRUE;
+ }
+ (void)array_append_space(&fts_results);
+ result->box_results = array_idx_modifiable(&fts_results, 0);
+ hash_table_destroy(&mailboxes);
+ return 0;
+}
+
+static int
+fts_backend_solr_lookup_multi(struct fts_backend *_backend,
+ struct mailbox *const boxes[],
+ struct mail_search_arg *args, bool and_args,
+ struct fts_multi_result *result)
+{
+ struct solr_fts_backend *backend =
+ (struct solr_fts_backend *)_backend;
+ string_t *str;
+
+ fts_solr_set_default_ns(backend);
+
+ str = t_str_new(256);
+ str_printfa(str, "fl=ns,box,uidv,uid,score&rows=%u&sort=box+asc,uid+asc&q=",
+ SOLR_MAX_MULTI_ROWS);
+
+ if (solr_add_definite_query_args(str, args, and_args)) {
+ if (solr_search_multi(backend, str, boxes, result) < 0)
+ return -1;
+ }
+ /* FIXME: maybe_uids could be handled also with some more work.. */
+ return 0;
+}
+
+struct fts_backend fts_backend_solr_old = {
+ .name = "solr_old",
+ .flags = 0,
+
+ {
+ fts_backend_solr_alloc,
+ fts_backend_solr_init,
+ fts_backend_solr_deinit,
+ fts_backend_solr_get_last_uid,
+ fts_backend_solr_update_init,
+ fts_backend_solr_update_deinit,
+ fts_backend_solr_update_set_mailbox,
+ fts_backend_solr_update_expunge,
+ fts_backend_solr_update_set_build_key,
+ fts_backend_solr_update_unset_build_key,
+ fts_backend_solr_update_build_more,
+ fts_backend_solr_refresh,
+ fts_backend_solr_optimize,
+ fts_backend_default_can_lookup,
+ fts_backend_solr_lookup,
+ fts_backend_solr_lookup_multi
+ }
+};
#include "lib.h"
#include "array.h"
#include "str.h"
+#include "hash.h"
#include "strescape.h"
#include "unichar.h"
#include "mail-storage-private.h"
#include "mailbox-list-private.h"
-#include "fts-mailbox.h"
+#include "mail-search.h"
+#include "fts-api.h"
#include "solr-connection.h"
#include "fts-solr-plugin.h"
#include <ctype.h>
#define SOLR_CMDBUF_SIZE (1024*64)
-#define SOLR_MAX_ROWS 100000
-#define FTS_SOLR_MAX_BOX_INC_PATTERNS 5
-#define FTS_SOLR_MAX_BOX_EXC_PATTERNS 5
+#define SOLR_MAX_MULTI_ROWS 100000
struct solr_fts_backend {
struct fts_backend backend;
- char *id_username, *id_namespace, *id_box_name;
- struct mail_namespace *default_ns;
};
-struct solr_fts_backend_build_context {
- struct fts_backend_build_context ctx;
+struct solr_fts_backend_update_context {
+ struct fts_backend_update_context ctx;
- struct solr_connection_post *post;
- uint32_t prev_uid, uid_validity;
- string_t *cmd;
- bool headers;
- bool field_open;
-};
-
-struct solr_virtual_uid_map_context {
- struct fts_backend *backend;
- struct mailbox *box;
-};
+ struct mailbox *cur_box;
+ char box_guid[MAILBOX_GUID_HEX_LENGTH+1];
-struct fts_backend_solr_get_last_uids_context {
- struct fts_backend *backend;
- pool_t pool;
- ARRAY_TYPE(fts_backend_uid_map) *last_uids;
+ struct solr_connection_post *post;
+ uint32_t prev_uid;
+ string_t *cmd, *hdr, *hdr_fields;
- struct mailbox *box;
+ bool headers_open;
+ bool cur_header_index;
+ bool documents_added;
};
static struct solr_connection *solr_conn = NULL;
-static void fts_box_name_get_root(struct mail_namespace **ns, const char **name)
-{
- struct mail_namespace *orig_ns = *ns;
-
- while ((*ns)->alias_for != NULL)
- *ns = (*ns)->alias_for;
-
- if (**name == '\0' && *ns != orig_ns &&
- ((*ns)->flags & NAMESPACE_FLAG_INBOX_USER) != 0) {
- /* ugly workaround to allow selecting INBOX from a Maildir/
- when it's not in the inbox=yes namespace. */
- *name = "INBOX";
- }
-}
-
-static const char *
-fts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r)
-{
- struct mail_namespace *ns = mailbox_get_namespace(box);
- const char *name = box->name;
-
- fts_box_name_get_root(&ns, &name);
- *ns_r = ns;
- return name;
-}
-
static bool is_valid_xml_char(unichar_t chr)
{
/* Valid characters in XML:
xml_encode_data(dest, (const unsigned char *)str, strlen(str));
}
-static const char *solr_escape_id_str(const char *str)
-{
- string_t *tmp;
- const char *p;
-
- for (p = str; *p != '\0'; p++) {
- if (*p == '/' || *p == '!')
- break;
- }
- if (*p == '\0')
- return str;
-
- tmp = t_str_new(64);
- for (p = str; *p != '\0'; p++) {
- switch (*p) {
- case '/':
- str_append(tmp, "!\\");
- break;
- case '!':
- str_append(tmp, "!!");
- break;
- default:
- str_append_c(tmp, *p);
- break;
- }
- }
- return str_c(tmp);
-}
-
-static void solr_quote(string_t *dest, const char *str)
-{
- str_append_c(dest, '"');
- str_append(dest, str_escape(str));
- str_append_c(dest, '"');
-}
-
static void solr_quote_http(string_t *dest, const char *str)
{
str_append(dest, "%22");
str_append(dest, "%22");
}
-static struct fts_backend *
-fts_backend_solr_init(struct mailbox *box)
+static struct fts_backend *fts_backend_solr_alloc(void)
{
- struct fts_solr_user *fuser =
- FTS_SOLR_USER_CONTEXT(box->storage->user);
- const struct fts_solr_settings *set = &fuser->set;
struct solr_fts_backend *backend;
- struct mail_namespace *ns;
- const char *str, *box_name;
-
-
- box_name = fts_box_get_root(box, &ns);
- i_assert(*box_name != '\0');
-
- if (solr_conn == NULL)
- solr_conn = solr_connection_init(set->url, set->debug);
backend = i_new(struct solr_fts_backend, 1);
- if (set->default_ns_prefix != NULL) {
- backend->default_ns =
- mail_namespace_find_prefix(ns->user->namespaces,
- set->default_ns_prefix);
- if (backend->default_ns == NULL) {
- i_fatal("fts_solr: default_ns setting points to "
- "nonexistent namespace");
- }
- } else {
- backend->default_ns =
- mail_namespace_find_inbox(ns->user->namespaces);
- }
- while (backend->default_ns->alias_for != NULL)
- backend->default_ns = backend->default_ns->alias_for;
-
- str = solr_escape_id_str(ns->user->username);
- backend->id_username = i_strdup(str);
- if (ns != backend->default_ns) {
- str = solr_escape_id_str(ns->prefix);
- backend->id_namespace = i_strdup(str);
- }
- backend->id_box_name = i_strdup(box_name);
backend->backend = fts_backend_solr;
-
return &backend->backend;
}
-static void fts_backend_solr_deinit(struct fts_backend *_backend)
+static int
+fts_backend_solr_init(struct fts_backend *_backend,
+ const char **error_r ATTR_UNUSED)
{
- struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+ struct fts_solr_user *fuser = FTS_SOLR_USER_CONTEXT(_backend->ns->user);
+ const struct fts_solr_settings *set = &fuser->set;
- i_free(backend->id_box_name);
- i_free(backend->id_namespace);
- i_free(backend->id_username);
- i_free(backend);
+ if (solr_conn == NULL)
+ solr_conn = solr_connection_init(set->url, set->debug);
+ return 0;
}
-static void
-solr_add_ns_query(string_t *str, struct fts_backend *_backend,
- struct mail_namespace *ns, bool neg)
+static void fts_backend_solr_deinit(struct fts_backend *_backend)
{
struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
- while (ns->alias_for != NULL)
- ns = ns->alias_for;
-
- if (ns == backend->default_ns || *ns->prefix == '\0') {
- if (!neg)
- str_append(str, " -ns:[* TO *]");
- else
- str_append(str, " +ns:[* TO *]");
- } else {
- if (!neg)
- str_append(str, " +ns:");
- else
- str_append(str, " -ns:");
- solr_quote(str, ns->prefix);
- }
-}
-
-static void
-solr_add_ns_query_http(string_t *str, struct fts_backend *backend,
- struct mail_namespace *ns)
-{
- string_t *tmp;
-
- tmp = t_str_new(64);
- solr_add_ns_query(tmp, backend, ns, FALSE);
- solr_connection_http_escape(solr_conn, str, str_c(tmp));
+ i_free(backend);
}
-static int fts_backend_solr_get_last_uid_fallback(struct fts_backend *backend,
- uint32_t *last_uid_r)
+static int
+get_last_uid_fallback(struct fts_backend *_backend, struct mailbox *box,
+ uint32_t *last_uid_r)
{
- struct mailbox *box = backend->box;
- struct mail_namespace *ns;
- struct mailbox_status status;
- ARRAY_TYPE(seq_range) uids;
const struct seq_range *uidvals;
- const char *box_name;
+ const char *box_guid;
unsigned int count;
+ struct solr_result **results;
string_t *str;
+ pool_t pool;
+ int ret = 0;
str = t_str_new(256);
str_append(str, "fl=uid&rows=1&sort=uid+desc&q=");
- box_name = fts_box_get_root(box, &ns);
+ if (fts_mailbox_get_guid(box, &box_guid) < 0)
+ return -1;
- mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
- str_printfa(str, "uidv:%u+box:", status.uidvalidity);
- solr_quote_http(str, box_name);
- solr_add_ns_query_http(str, backend, ns);
- str_append(str, "+user:");
- solr_quote_http(str, ns->user->username);
+ str_printfa(str, "box:%s+user:", box_guid);
+ if (_backend->ns->owner != NULL)
+ solr_quote_http(str, _backend->ns->owner->username);
+ else
+ str_append(str, "%22%22");
- t_array_init(&uids, 1);
+ pool = pool_alloconly_create("solr last uid lookup", 1024);
if (solr_connection_select(solr_conn, str_c(str),
- NULL, NULL, &uids, NULL) < 0)
- return -1;
-
- uidvals = array_get(&uids, &count);
- if (count == 0) {
- /* nothing indexed yet for this mailbox */
+ pool, &results) < 0)
+ ret = -1;
+ else if (results[0] == NULL) {
+ /* no UIDs */
*last_uid_r = 0;
- } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
- *last_uid_r = uidvals[0].seq1;
} else {
- i_error("fts_solr: Last UID lookup returned multiple rows");
- return -1;
+ uidvals = array_get(&results[0]->uids, &count);
+ i_assert(count > 0);
+ if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
+ *last_uid_r = uidvals[0].seq1;
+ } else {
+ i_error("fts_solr: Last UID lookup returned multiple rows");
+ ret = -1;
+ }
}
- return 0;
+ pool_unref(&pool);
+ return ret;
}
-static int fts_backend_solr_get_last_uid(struct fts_backend *backend,
- uint32_t *last_uid_r)
+static int
+fts_backend_solr_get_last_uid(struct fts_backend *_backend,
+ struct mailbox *box, uint32_t *last_uid_r)
{
- struct mailbox *box = backend->box;
- struct mail_namespace *ns;
- struct mailbox_status status;
- ARRAY_TYPE(seq_range) uids;
- const struct seq_range *uidvals;
- const char *box_name;
- unsigned int count;
- string_t *str;
-
- str = t_str_new(256);
- str_append(str, "fl=uid&rows=1&q=last_uid:TRUE+");
-
- box_name = fts_box_get_root(box, &ns);
-
- mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
- str_printfa(str, "uidv:%u+box:", status.uidvalidity);
- solr_quote_http(str, box_name);
- solr_add_ns_query_http(str, backend, ns);
- str_append(str, "+user:");
- solr_quote_http(str, ns->user->username);
+ if (fts_index_get_last_uid(box, last_uid_r))
+ return 0;
- t_array_init(&uids, 1);
- if (solr_connection_select(solr_conn, str_c(str),
- NULL, NULL, &uids, NULL) < 0)
+ /* either nothing has been indexed, or the index was corrupted.
+ do it the slow way. */
+ if (get_last_uid_fallback(_backend, box, last_uid_r) < 0)
return -1;
- uidvals = array_get(&uids, &count);
- if (count == 0) {
- /* either nothing is indexed or we're converting from an
- older database format without the last_uid fields */
- return fts_backend_solr_get_last_uid_fallback(backend,
- last_uid_r);
- } else if (count == 1 && uidvals[0].seq1 == uidvals[0].seq2) {
- *last_uid_r = uidvals[0].seq1;
- } else {
- i_error("fts_solr: Last UID lookup returned multiple rows");
- return -1;
- }
+ (void)fts_index_set_last_uid(box, *last_uid_r);
return 0;
}
-static struct mail_namespace *
-solr_get_namespaces(struct fts_backend *_backend,
- struct mailbox *box, const char *ns_prefix)
+static struct fts_backend_update_context *
+fts_backend_solr_update_init(struct fts_backend *_backend)
{
- struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
- struct mail_namespace *namespaces = box->storage->user->namespaces;
+ struct solr_fts_backend_update_context *ctx;
- if (ns_prefix == NULL)
- return backend->default_ns;
- else
- return mail_namespace_find_prefix(namespaces, ns_prefix);
+ ctx = i_new(struct solr_fts_backend_update_context, 1);
+ ctx->ctx.backend = _backend;
+ ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
+ ctx->hdr = str_new(default_pool, 4096);
+ ctx->hdr_fields = str_new(default_pool, 1024);
+ return &ctx->ctx;
}
-static bool
-solr_virtual_get_last_uids(const char *ns_prefix, const char *mailbox,
- uint32_t uidvalidity, uint32_t *uid, void *context)
+static void xml_encode_id(struct solr_fts_backend_update_context *ctx,
+ string_t *str, uint32_t uid)
{
- struct fts_backend_solr_get_last_uids_context *ctx = context;
- struct fts_backend_uid_map *map;
- struct mail_namespace *ns;
-
- ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix);
- for (; ns != NULL; ns = ns->alias_chain_next) T_BEGIN {
- const char *vname = mailbox_list_get_vname(ns->list, mailbox);
- map = array_append_space(ctx->last_uids);
- map->mailbox = p_strdup(ctx->pool, vname);
- map->uidvalidity = uidvalidity;
- map->uid = *uid;
- } T_END;
- return FALSE;
+ str_printfa(str, "%u/%s", uid, ctx->box_guid);
+ if (ctx->ctx.backend->ns->owner != NULL) {
+ str_append_c(str, '/');
+ xml_encode(str, ctx->ctx.backend->ns->owner->username);
+ }
}
static void
-solr_add_pattern(string_t *str, const struct mailbox_virtual_pattern *pattern)
+fts_backend_solr_doc_open(struct solr_fts_backend_update_context *ctx,
+ uint32_t uid)
{
- struct mail_namespace *ns = pattern->ns;
- const char *name, *p;
-
- name = pattern->pattern;
- fts_box_name_get_root(&ns, &name);
+ ctx->documents_added = TRUE;
- if (strcmp(name, "*") == 0) {
- str_append(str, "[* TO *]");
- return;
- }
-
- /* first check if there are any wildcards in the pattern */
- for (p = name; *p != '\0'; p++) {
- if (*p == '%' || *p == '*')
- break;
- }
- if (*p == '\0') {
- /* full mailbox name */
- solr_quote(str, name);
- return;
- }
+ str_printfa(ctx->cmd, "<doc>"
+ "<field name=\"uid\">%u</field>"
+ "<field name=\"box\">%s</field>",
+ uid, ctx->box_guid);
+ str_append(ctx->cmd, "<field name=\"user\">");
+ if (ctx->ctx.backend->ns->owner != NULL)
+ xml_encode(ctx->cmd, ctx->ctx.backend->ns->owner->username);
+ str_append(ctx->cmd, "</field>");
- /* there are at least some wildcards. */
- for (p = name; *p != '\0'; p++) {
- if (*p == '%' || *p == '*') {
- if (p == name || (p[-1] != '%' && p[-1] != '*'))
- str_append_c(str, '*');
- } else {
- if (!i_isalnum(*p))
- str_append_c(str, '\\');
- str_append_c(str, *p);
- }
- }
+ str_printfa(ctx->cmd, "<field name=\"id\">");
+ xml_encode_id(ctx, ctx->cmd, uid);
+ str_append(ctx->cmd, "</field>");
}
static void
-fts_backend_solr_filter_mailboxes(struct fts_backend *_backend,
- string_t *str, struct mailbox *box)
+fts_backend_solr_doc_close(struct solr_fts_backend_update_context *ctx)
{
- ARRAY_TYPE(mailbox_virtual_patterns) includes_arr, excludes_arr;
- struct mail_namespace *ns;
- const struct mailbox_virtual_pattern *includes, *excludes;
- unsigned int i, inc_count, exc_count;
- string_t *fq;
-
- t_array_init(&includes_arr, 16);
- t_array_init(&excludes_arr, 16);
- fts_mailbox_get_virtual_box_patterns(box, &includes_arr, &excludes_arr);
- includes = array_get(&includes_arr, &inc_count);
- excludes = array_get(&excludes_arr, &exc_count);
- i_assert(inc_count > 0);
-
- /* First see if there are any patterns that begin with a wildcard.
- Solr doesn't allow them, so in that case we'll need to return
- all mailboxes. */
- for (i = 0; i < inc_count; i++) {
- if (*includes[i].pattern == '*' ||
- *includes[i].pattern == '%')
- break;
- }
-
- fq = t_str_new(128);
- if (i == inc_count && inc_count <= FTS_SOLR_MAX_BOX_INC_PATTERNS) {
- /* we can filter what mailboxes we want returned */
- str_append_c(fq, '(');
- for (i = 0; i < inc_count; i++) {
- if (i != 0)
- str_append(fq, " OR +");
- str_append_c(fq, '(');
- str_append(fq, "+box:");
- solr_add_pattern(fq, &includes[i]);
- solr_add_ns_query(fq, _backend, includes[i].ns, FALSE);
- str_append_c(fq, ')');
- }
- str_append_c(fq, ')');
- }
- exc_count = I_MIN(FTS_SOLR_MAX_BOX_EXC_PATTERNS, exc_count);
- for (i = 0; i < exc_count; i++) {
- if (str_len(fq) > 0)
- str_append_c(fq, ' ');
- str_append(fq, "NOT (");
- str_append(fq, "box:");
- solr_add_pattern(fq, &excludes[i]);
-
- for (ns = excludes[i].ns; ns->alias_for != NULL; )
- ns = ns->alias_for;
- solr_add_ns_query(fq, _backend, ns, FALSE);
- str_append_c(fq, ')');
+ ctx->headers_open = FALSE;
+ if (str_len(ctx->hdr) > 0) {
+ str_append(ctx->cmd, "<field name=\"hdr\">");
+ str_append_str(ctx->cmd, ctx->hdr);
+ str_append(ctx->cmd, "</field>");
+ str_truncate(ctx->hdr, 0);
}
- if (str_len(fq) > 0) {
- str_append(str, "&fq=");
- solr_connection_http_escape(solr_conn, str, str_c(fq));
+ if (str_len(ctx->hdr_fields) > 0) {
+ str_append_str(ctx->cmd, ctx->hdr_fields);
+ str_truncate(ctx->hdr_fields, 0);
}
+ str_append(ctx->cmd, "</doc>");
}
static int
-fts_backend_solr_get_all_last_uids(struct fts_backend *backend, pool_t pool,
- ARRAY_TYPE(fts_backend_uid_map) *last_uids)
+fts_backed_solr_build_commit(struct solr_fts_backend_update_context *ctx)
{
- struct fts_backend_solr_get_last_uids_context ctx;
- string_t *str;
+ if (ctx->post == NULL)
+ return 0;
- memset(&ctx, 0, sizeof(ctx));
- ctx.backend = backend;
- ctx.pool = pool;
- ctx.last_uids = last_uids;
- ctx.box = backend->box;
+ fts_backend_solr_doc_close(ctx);
+ str_append(ctx->cmd, "</add>");
- str = t_str_new(256);
- str_printfa(str, "fl=uid,box,uidv,ns&rows=%u&q=last_uid:TRUE+user:",
- SOLR_MAX_ROWS);
- solr_quote_http(str, backend->box->storage->user->username);
- fts_backend_solr_filter_mailboxes(backend, str, backend->box);
-
- return solr_connection_select(solr_conn, str_c(str),
- solr_virtual_get_last_uids, &ctx,
- NULL, NULL);
+ solr_connection_post_more(ctx->post, str_data(ctx->cmd),
+ str_len(ctx->cmd));
+ return solr_connection_post_end(ctx->post);
}
static int
-fts_backend_solr_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
- struct fts_backend_build_context **ctx_r)
+fts_backend_solr_update_deinit(struct fts_backend_update_context *_ctx)
{
- struct solr_fts_backend_build_context *ctx;
- struct mailbox_status status;
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
+ const char *str;
+ int ret = _ctx->failed ? -1 : 0;
- *last_uid_r = (uint32_t)-1;
-
- ctx = i_new(struct solr_fts_backend_build_context, 1);
- ctx->ctx.backend = backend;
- ctx->cmd = str_new(default_pool, SOLR_CMDBUF_SIZE);
+ if (fts_backed_solr_build_commit(ctx) < 0)
+ ret = -1;
- mailbox_get_open_status(backend->box, STATUS_UIDVALIDITY, &status);
- ctx->uid_validity = status.uidvalidity;
+ /* commit and wait until the documents we just indexed are
+ visible to the following search */
+ str = t_strdup_printf("<commit waitFlush=\"false\" "
+ "waitSearcher=\"%s\"/>",
+ ctx->documents_added ? "true" : "false");
+ if (solr_connection_post(solr_conn, str) < 0)
+ ret = -1;
- *ctx_r = &ctx->ctx;
- return 0;
+ str_free(&ctx->cmd);
+ str_free(&ctx->hdr);
+ str_free(&ctx->hdr_fields);
+ i_free(ctx);
+ return ret;
}
static void
-fts_backend_solr_add_doc_prefix(struct solr_fts_backend_build_context *ctx,
- uint32_t uid)
+fts_backend_solr_update_set_mailbox(struct fts_backend_update_context *_ctx,
+ struct mailbox *box)
{
- struct solr_fts_backend *backend =
- (struct solr_fts_backend *)ctx->ctx.backend;
- struct mailbox *box = ctx->ctx.backend->box;
- struct mail_namespace *ns;
- const char *box_name;
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
+ const char *box_guid;
- str_printfa(ctx->cmd, "<doc>"
- "<field name=\"uid\">%u</field>"
- "<field name=\"uidv\">%u</field>",
- uid, ctx->uid_validity);
+ ctx->cur_box = box;
- box_name = fts_box_get_root(box, &ns);
+ if (box != NULL) {
+ if (fts_mailbox_get_guid(box, &box_guid) < 0)
+ _ctx->failed = TRUE;
- if (ns != backend->default_ns) {
- str_append(ctx->cmd, "<field name=\"ns\">");
- xml_encode(ctx->cmd, ns->prefix);
- str_append(ctx->cmd, "</field>");
+ i_assert(strlen(box_guid) == sizeof(ctx->box_guid)-1);
+ memcpy(ctx->box_guid, box_guid, sizeof(ctx->box_guid)-1);
+ } else {
+ memset(ctx->box_guid, 0, sizeof(ctx->box_guid));
}
- str_append(ctx->cmd, "<field name=\"box\">");
- xml_encode(ctx->cmd, box_name);
- str_append(ctx->cmd, "</field><field name=\"user\">");
- xml_encode(ctx->cmd, ns->user->username);
- str_append(ctx->cmd, "</field>");
}
-static void xml_encode_id(string_t *str, struct fts_backend *_backend,
- uint32_t uid, uint32_t uid_validity)
+static void
+fts_backend_solr_update_expunge(struct fts_backend_update_context *_ctx,
+ uint32_t uid)
{
- struct solr_fts_backend *backend = (struct solr_fts_backend *)_backend;
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
- if (uid != 0)
- str_printfa(str, "%u/", uid);
- else
- str_append(str, "L/");
- if (backend->id_namespace != NULL) {
- xml_encode(str, backend->id_namespace);
- str_append_c(str, '/');
- }
- str_printfa(str, "%u/", uid_validity);
- xml_encode(str, backend->id_username);
- str_append_c(str, '/');
- xml_encode(str, backend->id_box_name);
+ T_BEGIN {
+ string_t *cmd;
+
+ cmd = t_str_new(256);
+ str_append(cmd, "<delete><id>");
+ xml_encode_id(ctx, cmd, uid);
+ str_append(cmd, "</id></delete>");
+
+ (void)solr_connection_post(solr_conn, str_c(cmd));
+ } T_END;
}
static void
-fts_backend_solr_uid_changed(struct solr_fts_backend_build_context *ctx,
+fts_backend_solr_uid_changed(struct solr_fts_backend_update_context *ctx,
uint32_t uid)
{
if (ctx->post == NULL) {
+ i_assert(ctx->prev_uid == 0);
+
ctx->post = solr_connection_post_begin(solr_conn);
str_append(ctx->cmd, "<add>");
} else {
- if (ctx->field_open) {
- str_append(ctx->cmd, "</field>");
- ctx->field_open = FALSE;
- }
- str_append(ctx->cmd, "</doc>");
+ fts_backend_solr_doc_close(ctx);
}
ctx->prev_uid = uid;
- ctx->headers = FALSE;
-
- fts_backend_solr_add_doc_prefix(ctx, uid);
- str_printfa(ctx->cmd, "<field name=\"id\">");
- xml_encode_id(ctx->cmd, ctx->ctx.backend, uid, ctx->uid_validity);
- str_append(ctx->cmd, "</field>");
+ fts_backend_solr_doc_open(ctx, uid);
}
-static void
-fts_backend_solr_build_hdr(struct fts_backend_build_context *_ctx,
- uint32_t uid)
+static bool
+fts_backend_solr_update_set_build_key(struct fts_backend_update_context *_ctx,
+ const struct fts_backend_build_key *key)
{
- struct solr_fts_backend_build_context *ctx =
- (struct solr_fts_backend_build_context *)_ctx;
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
- if (uid != ctx->prev_uid)
- fts_backend_solr_uid_changed(ctx, uid);
- else {
- i_assert(!ctx->headers);
+ if (key->uid != ctx->prev_uid)
+ fts_backend_solr_uid_changed(ctx, key->uid);
- if (ctx->field_open) {
- str_append(ctx->cmd, "</field>");
- ctx->field_open = FALSE;
+ switch (key->type) {
+ case FTS_BACKEND_BUILD_KEY_HDR:
+ if (fts_header_want_indexed(key->hdr_name)) {
+ ctx->cur_header_index = TRUE;
+ str_printfa(ctx->hdr_fields, "<field name=\"%s\">",
+ t_str_lcase(key->hdr_name));
}
+ /* fall through */
+ case FTS_BACKEND_BUILD_KEY_MIME_HDR:
+ xml_encode(ctx->hdr, key->hdr_name);
+ str_append(ctx->hdr, ": ");
+ ctx->headers_open = TRUE;
+ break;
+ case FTS_BACKEND_BUILD_KEY_BODY_PART:
+ ctx->headers_open = FALSE;
+ str_append(ctx->cmd, "<field name=\"body\">");
+ break;
+ case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
+ i_unreached();
}
-
- i_assert(!ctx->field_open);
- ctx->field_open = TRUE;
- ctx->headers = TRUE;
- str_append(ctx->cmd, "<field name=\"hdr\">");
+ return TRUE;
}
-static bool
-fts_backend_solr_build_body_begin(struct fts_backend_build_context *_ctx,
- uint32_t uid, const char *content_type,
- const char *content_disposition ATTR_UNUSED)
+static void
+fts_backend_solr_update_unset_build_key(struct fts_backend_update_context *_ctx)
{
- struct solr_fts_backend_build_context *ctx =
- (struct solr_fts_backend_build_context *)_ctx;
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
- if (!fts_backend_default_can_index(content_type))
- return FALSE;
-
- if (uid != ctx->prev_uid)
- fts_backend_solr_uid_changed(ctx, uid);
+ if (!ctx->headers_open)
+ str_append(ctx->cmd, "</field>");
else {
- /* body comes first, then headers */
- i_assert(!ctx->headers);
+ /* this is called individually for each header line.
+ headers are finished only when key changes to body */
+ str_append_c(ctx->hdr, '\n');
}
- if (!ctx->field_open) {
- ctx->field_open = TRUE;
- ctx->headers = FALSE;
- str_append(ctx->cmd, "<field name=\"body\">");
+ if (ctx->cur_header_index) {
+ str_append(ctx->hdr_fields, "</field>");
+ ctx->cur_header_index = FALSE;
}
- return TRUE;
}
static int
-fts_backend_solr_build_more(struct fts_backend_build_context *_ctx,
- const unsigned char *data, size_t size)
+fts_backend_solr_update_build_more(struct fts_backend_update_context *_ctx,
+ const unsigned char *data, size_t size)
{
- struct solr_fts_backend_build_context *ctx =
- (struct solr_fts_backend_build_context *)_ctx;
+ struct solr_fts_backend_update_context *ctx =
+ (struct solr_fts_backend_update_context *)_ctx;
+
+ if (_ctx->failed)
+ return -1;
+
+ if (ctx->headers_open) {
+ if (ctx->cur_header_index)
+ xml_encode_data(ctx->hdr_fields, data, size);
+ xml_encode_data(ctx->hdr, data, size);
+ } else {
+ i_assert(!ctx->cur_header_index);
+ xml_encode_data(ctx->cmd, data, size);
+ }
- xml_encode_data(ctx->cmd, data, size);
if (str_len(ctx->cmd) > SOLR_CMDBUF_SIZE-128) {
solr_connection_post_more(ctx->post, str_data(ctx->cmd),
str_len(ctx->cmd));
return 0;
}
-static int
-fts_backed_solr_build_commit(struct solr_fts_backend_build_context *ctx)
+static int fts_backend_solr_refresh(struct fts_backend *backend ATTR_UNUSED)
{
- int ret;
-
- if (ctx->post == NULL)
- return 0;
-
- if (ctx->field_open) {
- str_append(ctx->cmd, "</field>");
- ctx->field_open = FALSE;
- }
- str_append(ctx->cmd, "</doc>");
-
- /* Update the mailbox's last_uid field, replacing the existing
- document. Note that since there is no locking, it's possible that
- if another session is indexing at the same time, the last_uid value
- may shrink. This doesn't really matter, we'll simply do more work
- in future by reindexing some messages. */
- fts_backend_solr_add_doc_prefix(ctx, ctx->prev_uid);
- str_printfa(ctx->cmd, "<field name=\"last_uid\">TRUE</field>"
- "<field name=\"id\">");
- xml_encode_id(ctx->cmd, ctx->ctx.backend, 0, ctx->uid_validity);
- str_append(ctx->cmd, "</field></doc></add>");
-
- solr_connection_post_more(ctx->post, str_data(ctx->cmd),
- str_len(ctx->cmd));
- ret = solr_connection_post_end(ctx->post);
- /* commit and wait until the documents we just indexed are
- visible to the following search */
- if (solr_connection_post(solr_conn, "<commit waitFlush=\"false\" "
- "waitSearcher=\"true\"/>") < 0)
- ret = -1;
- return ret;
+ return 0;
}
-static int
-fts_backend_solr_build_deinit(struct fts_backend_build_context *_ctx)
+static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED)
{
- struct solr_fts_backend_build_context *ctx =
- (struct solr_fts_backend_build_context *)_ctx;
- int ret;
-
- ret = fts_backed_solr_build_commit(ctx);
- str_free(&ctx->cmd);
- i_free(ctx);
- return ret;
+ return 0;
}
-static void
-fts_backend_solr_expunge(struct fts_backend *backend, struct mail *mail)
+static bool
+solr_add_definite_query(string_t *str, struct mail_search_arg *arg)
{
- struct mailbox_status status;
+ switch (arg->type) {
+ case SEARCH_TEXT: {
+ if (arg->match_not)
+ str_append_c(str, '-');
+ str_append(str, "(hdr:");
+ solr_quote_http(str, arg->value.str);
+ str_append(str, "+OR+body:");
+ solr_quote_http(str, arg->value.str);
+ str_append(str, ")");
+ break;
+ }
+ case SEARCH_BODY:
+ if (arg->match_not)
+ str_append_c(str, '-');
+ str_append(str, "body:");
+ solr_quote_http(str, arg->value.str);
+ break;
+ case SEARCH_HEADER:
+ case SEARCH_HEADER_ADDRESS:
+ case SEARCH_HEADER_COMPRESS_LWSP:
+ if (!fts_header_want_indexed(arg->hdr_field_name))
+ return FALSE;
+
+ if (arg->match_not)
+ str_append_c(str, '-');
+ str_append(str, arg->hdr_field_name);
+ str_append_c(str, ':');
+ solr_quote_http(str, arg->value.str);
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
- mailbox_get_open_status(mail->box, STATUS_UIDVALIDITY, &status);
+static bool
+solr_add_definite_query_args(string_t *str, struct mail_search_arg *arg,
+ bool and_args)
+{
+ unsigned int last_len;
+
+ last_len = str_len(str);
+ for (; arg != NULL; arg = arg->next) {
+ if (solr_add_definite_query(str, arg)) {
+ arg->match_always = TRUE;
+ last_len = str_len(str);
+ if (and_args)
+ str_append(str, "+AND+");
+ else
+ str_append(str, "+OR+");
+ }
+ }
+ if (str_len(str) == last_len)
+ return FALSE;
- T_BEGIN {
- string_t *cmd;
+ str_truncate(str, last_len);
+ return TRUE;
+}
- cmd = t_str_new(256);
- str_append(cmd, "<delete><id>");
- xml_encode_id(cmd, backend, mail->uid, status.uidvalidity);
- str_append(cmd, "</id></delete>");
+static bool
+solr_add_maybe_query(string_t *str, struct mail_search_arg *arg)
+{
+ switch (arg->type) {
+ case SEARCH_HEADER:
+ case SEARCH_HEADER_ADDRESS:
+ case SEARCH_HEADER_COMPRESS_LWSP:
+ if (fts_header_want_indexed(arg->hdr_field_name))
+ return FALSE;
+ if (arg->match_not) {
+ /* all matches would be definite, but all non-matches
+ would be maybies. too much trouble to optimize. */
+ return FALSE;
+ }
- (void)solr_connection_post(solr_conn, str_c(cmd));
- } T_END;
+ /* we can check if the search key exists in some header and
+ filter out the messages that have no chance of matching */
+ str_append(str, "hdr:");
+ solr_quote_http(str, arg->value.str);
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
}
-static void
-fts_backend_solr_expunge_finish(struct fts_backend *backend ATTR_UNUSED,
- struct mailbox *box ATTR_UNUSED,
- bool committed ATTR_UNUSED)
-{
- solr_connection_post(solr_conn,
- "<commit waitFlush=\"false\" waitSearcher=\"false\"/>");
-}
+static bool
+solr_add_maybe_query_args(string_t *str, struct mail_search_arg *arg,
+ bool and_args)
+{
+ unsigned int last_len;
+
+ last_len = str_len(str);
+ for (; arg != NULL; arg = arg->next) {
+ if (solr_add_maybe_query(str, arg)) {
+ arg->match_always = TRUE;
+ last_len = str_len(str);
+ if (and_args)
+ str_append(str, "+AND+");
+ else
+ str_append(str, "+OR+");
+ }
+ }
+ if (str_len(str) == last_len)
+ return FALSE;
-static int fts_backend_solr_refresh(struct fts_backend *backend ATTR_UNUSED)
-{
- return 0;
+ str_truncate(str, last_len);
+ return TRUE;
}
-static bool solr_virtual_uid_map(const char *ns_prefix, const char *mailbox,
- uint32_t uidvalidity, uint32_t *uid,
- void *context)
+static int solr_search(struct fts_backend *_backend, string_t *str,
+ const char *box_guid, ARRAY_TYPE(seq_range) *uids_r,
+ ARRAY_TYPE(fts_score_map) *scores_r)
{
- struct solr_virtual_uid_map_context *ctx = context;
- struct mail_namespace *ns;
- bool convert_inbox, ret;
-
- ns = solr_get_namespaces(ctx->backend, ctx->box, ns_prefix);
- convert_inbox = (ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 &&
- strcmp(mailbox, "INBOX") == 0;
- for (; ns != NULL; ns = ns->alias_chain_next) {
- T_BEGIN {
- const char *vname = convert_inbox ? ns->prefix :
- mailbox_list_get_vname(ns->list, mailbox);
- ret = fts_mailbox_get_virtual_uid(ctx->box, vname,
- uidvalidity,
- *uid, uid);
- } T_END;
- if (ret)
- return TRUE;
+ pool_t pool = pool_alloconly_create("fts solr search", 1024);
+ struct solr_result **results;
+ int ret;
+
+ /* use a separate filter query for selecting the mailbox. it shouldn't
+ affect the score and there could be some caching benefits too. */
+ str_printfa(str, "&fq=%%2Bbox:%s+%%2Buser:", box_guid);
+ if (_backend->ns->owner != NULL)
+ solr_quote_http(str, _backend->ns->owner->username);
+ else
+ str_append(str, "%22%22");
+
+ ret = solr_connection_select(solr_conn, str_c(str), pool, &results);
+ if (ret == 0 && results[0] != NULL) {
+ array_append_array(uids_r, &results[0]->uids);
+ array_append_array(scores_r, &results[0]->scores);
}
- return FALSE;
+ pool_unref(&pool);
+ return ret;
}
-static int fts_backend_solr_lookup(struct fts_backend_lookup_context *ctx,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids,
- ARRAY_TYPE(fts_score_map) *scores)
+static int
+fts_backend_solr_lookup(struct fts_backend *_backend, struct mailbox *box,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_result *result)
{
- struct mailbox *box = ctx->backend->box;
- struct mail_namespace *ns;
- struct solr_virtual_uid_map_context uid_map_ctx;
- const struct fts_backend_lookup_field *fields;
- const char *box_name;
- unsigned int i, count;
struct mailbox_status status;
string_t *str;
- bool virtual;
+ const char *box_guid;
+ unsigned int prefix_len;
- virtual = strcmp(box->storage->name, "virtual") == 0;
- mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
+ if (fts_mailbox_get_guid(box, &box_guid) < 0)
+ return -1;
+ mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
str = t_str_new(256);
- if (!virtual) {
- str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=",
- status.uidnext);
- } else {
- str_printfa(str, "fl=uid,score,box,uidv,ns&rows=%u"
- "&sort=box+asc,uid+asc&q=",
- SOLR_MAX_ROWS);
+ str_printfa(str, "fl=uid,score&rows=%u&sort=uid+asc&q=",
+ status.uidnext);
+ prefix_len = str_len(str);
+
+ if (solr_add_definite_query_args(str, args, and_args)) {
+ if (solr_search(_backend, str, box_guid,
+ &result->definite_uids, &result->scores) < 0)
+ return -1;
}
+ str_truncate(str, prefix_len);
+ if (solr_add_maybe_query_args(str, args, and_args)) {
+ if (solr_search(_backend, str, box_guid,
+ &result->maybe_uids, &result->scores) < 0)
+ return -1;
+ }
+ result->scores_sorted = TRUE;
+ return 0;
+}
- /* build a lucene search query from the fields */
- fields = array_get(&ctx->fields, &count);
- for (i = 0; i < count; i++) {
- if (i > 0)
- str_append_c(str, '+');
-
- if ((fields[i].flags & FTS_LOOKUP_FLAG_INVERT) != 0)
- str_append_c(str, '-');
-
- if ((fields[i].flags & FTS_LOOKUP_FLAG_HEADER) == 0) {
- /* body only */
- i_assert((fields[i].flags & FTS_LOOKUP_FLAG_BODY) != 0);
- str_append(str, "body:");
- solr_quote_http(str, fields[i].key);
- } else if ((fields[i].flags & FTS_LOOKUP_FLAG_BODY) == 0) {
- /* header only */
- str_append(str, "hdr:");
- solr_quote_http(str, fields[i].key);
- } else {
- /* both */
- str_append(str, "(body:");
- solr_quote_http(str, fields[i].key);
- str_append(str, "+OR+hdr:");
- solr_quote_http(str, fields[i].key);
- str_append_c(str, ')');
- }
- }
+static int
+solr_search_multi(struct fts_backend *_backend, string_t *str,
+ struct mailbox *const boxes[],
+ struct fts_multi_result *result)
+{
+ struct solr_result **solr_results;
+ struct fts_result *fts_result;
+ ARRAY_DEFINE(fts_results, struct fts_result);
+ struct hash_table *mailboxes;
+ struct mailbox *box;
+ const char *box_guid;
+ unsigned int i, len;
/* use a separate filter query for selecting the mailbox. it shouldn't
affect the score and there could be some caching benefits too. */
str_append(str, "&fq=%2Buser:");
- solr_quote_http(str, box->storage->user->username);
- if (virtual)
- fts_backend_solr_filter_mailboxes(ctx->backend, str, box);
- else {
- box_name = fts_box_get_root(box, &ns);
- str_printfa(str, "+%%2Buidv:%u+%%2Bbox:", status.uidvalidity);
- solr_quote_http(str, box_name);
- solr_add_ns_query_http(str, ctx->backend, ns);
+ if (_backend->ns->owner != NULL)
+ solr_quote_http(str, _backend->ns->owner->username);
+ else
+ str_append(str, "%22%22");
+
+ mailboxes = hash_table_create(default_pool, default_pool, 0,
+ str_hash, (hash_cmp_callback_t *)strcmp);
+ str_append(str, "%2B(");
+ len = str_len(str);
+ for (i = 0; boxes[i] != NULL; i++) {
+ if (fts_mailbox_get_guid(boxes[i], &box_guid) < 0)
+ continue;
+
+ if (str_len(str) != len)
+ str_append(str, "+OR+");
+ str_printfa(str, "box:%s", box_guid);
+ hash_table_insert(mailboxes, t_strdup_noconst(box_guid),
+ boxes[i]);
}
+ str_append_c(str, ')');
- array_clear(maybe_uids);
- if (!virtual) {
- return solr_connection_select(solr_conn, str_c(str), NULL, NULL,
- definite_uids, scores);
- } else {
- memset(&uid_map_ctx, 0, sizeof(uid_map_ctx));
- uid_map_ctx.backend = ctx->backend;
- uid_map_ctx.box = box;
- return solr_connection_select(solr_conn, str_c(str),
- solr_virtual_uid_map,
- &uid_map_ctx,
- definite_uids, scores);
+ if (solr_connection_select(solr_conn, str_c(str),
+ result->pool, &solr_results) < 0) {
+ hash_table_destroy(&mailboxes);
+ return -1;
}
+
+ p_array_init(&fts_results, result->pool, 32);
+ for (i = 0; solr_results[i] != NULL; i++) {
+ box = hash_table_lookup(mailboxes, solr_results[i]->box_id);
+ if (box == NULL) {
+ i_warning("fts_solr: Lookup returned unexpected mailbox "
+ "with guid=%s", solr_results[i]->box_id);
+ continue;
+ }
+ fts_result = array_append_space(&fts_results);
+ fts_result->box = box;
+ fts_result->definite_uids = solr_results[i]->uids;
+ fts_result->scores = solr_results[i]->scores;
+ fts_result->scores_sorted = TRUE;
+ }
+ (void)array_append_space(&fts_results);
+ result->box_results = array_idx_modifiable(&fts_results, 0);
+ hash_table_destroy(&mailboxes);
+ return 0;
+}
+
+static int
+fts_backend_solr_lookup_multi(struct fts_backend *backend,
+ struct mailbox *const boxes[],
+ struct mail_search_arg *args, bool and_args,
+ struct fts_multi_result *result)
+{
+ string_t *str;
+
+ str = t_str_new(256);
+ str_printfa(str, "fl=box,uid,score&rows=%u&sort=box+asc,uid+asc&q=",
+ SOLR_MAX_MULTI_ROWS);
+
+ if (solr_add_definite_query_args(str, args, and_args)) {
+ if (solr_search_multi(backend, str, boxes, result) < 0)
+ return -1;
+ }
+ /* FIXME: maybe_uids could be handled also with some more work.. */
+ return 0;
}
struct fts_backend fts_backend_solr = {
.name = "solr",
- .flags = FTS_BACKEND_FLAG_VIRTUAL_LOOKUPS,
+ .flags = 0,
{
+ fts_backend_solr_alloc,
fts_backend_solr_init,
fts_backend_solr_deinit,
fts_backend_solr_get_last_uid,
- fts_backend_solr_get_all_last_uids,
- fts_backend_solr_build_init,
- fts_backend_solr_build_hdr,
- fts_backend_solr_build_body_begin,
- NULL,
- fts_backend_solr_build_more,
- fts_backend_solr_build_deinit,
- fts_backend_solr_expunge,
- fts_backend_solr_expunge_finish,
+ fts_backend_solr_update_init,
+ fts_backend_solr_update_deinit,
+ fts_backend_solr_update_set_mailbox,
+ fts_backend_solr_update_expunge,
+ fts_backend_solr_update_set_build_key,
+ fts_backend_solr_update_unset_build_key,
+ fts_backend_solr_update_build_more,
fts_backend_solr_refresh,
- NULL,
- NULL,
- fts_backend_solr_lookup
+ fts_backend_solr_optimize,
+ fts_backend_default_can_lookup,
+ fts_backend_solr_lookup,
+ fts_backend_solr_lookup_multi
}
};
void fts_solr_plugin_init(struct module *module)
{
fts_backend_register(&fts_backend_solr);
+ fts_backend_register(&fts_backend_solr_old);
mail_storage_hooks_add(module, &fts_solr_mail_storage_hooks);
}
void fts_solr_plugin_deinit(void)
{
fts_backend_unregister(fts_backend_solr.name);
+ fts_backend_unregister(fts_backend_solr_old.name);
mail_storage_hooks_remove(&fts_solr_mail_storage_hooks);
}
extern const char *fts_solr_plugin_dependencies[];
extern struct fts_backend fts_backend_solr;
+extern struct fts_backend fts_backend_solr_old;
extern MODULE_CONTEXT_DEFINE(fts_solr_user_module, &mail_user_module_register);
void fts_solr_plugin_init(struct module *module);
#include "lib.h"
#include "array.h"
+#include "hash.h"
#include "str.h"
#include "strescape.h"
#include "solr-connection.h"
float score;
char *mailbox, *ns;
- solr_uid_map_callback_t *callback;
- void *context;
-
- ARRAY_TYPE(seq_range) *uids;
- ARRAY_TYPE(fts_score_map) *scores;
+ pool_t result_pool;
+ /* box_id -> solr_result */
+ struct hash_table *mailboxes;
+ ARRAY_DEFINE(results, struct solr_result *);
};
struct solr_connection_post {
}
}
+static struct solr_result *
+solr_result_get(struct solr_lookup_xml_context *ctx, const char *box_id)
+{
+ struct solr_result *result;
+ char *box_id_dup;
+
+ result = hash_table_lookup(ctx->mailboxes, box_id);
+ if (result != NULL)
+ return result;
+
+ box_id_dup = p_strdup(ctx->result_pool, box_id);
+ result = p_new(ctx->result_pool, struct solr_result, 1);
+ result->box_id = box_id_dup;
+ p_array_init(&result->uids, ctx->result_pool, 32);
+ p_array_init(&result->scores, ctx->result_pool, 32);
+ hash_table_insert(ctx->mailboxes, box_id_dup, result);
+ array_append(&ctx->results, &result, 1);
+ return result;
+}
+
static void solr_lookup_add_doc(struct solr_lookup_xml_context *ctx)
{
struct fts_score_map *score;
+ struct solr_result *result;
+ const char *box_id;
if (ctx->uid == 0) {
i_error("fts_solr: Query didn't return uid");
return;
}
- if (ctx->callback != NULL) {
- if (ctx->mailbox == NULL) {
- i_error("fts_solr: Query didn't return mailbox");
- return;
- }
- if (!ctx->callback(ctx->ns, ctx->mailbox, ctx->uidvalidity,
- &ctx->uid, ctx->context))
- return;
+ if (ctx->mailbox == NULL) {
+ /* looking up from a single mailbox only */
+ box_id = "";
+ } else if (ctx->uidvalidity != 0) {
+ /* old style lookup */
+ string_t *str = t_str_new(64);
+ str_printfa(str, "%u\001", ctx->uidvalidity);
+ str_append(str, ctx->mailbox);
+ if (ctx->ns != NULL)
+ str_printfa(str, "\001%s", ctx->ns);
+ box_id = str_c(str);
+ } else {
+ /* new style lookup */
+ box_id = ctx->mailbox;
}
+ result = solr_result_get(ctx, box_id);
- seq_range_array_add(ctx->uids, 0, ctx->uid);
- if (ctx->scores != NULL && ctx->score != 0) {
- score = array_append_space(ctx->scores);
+ seq_range_array_add(&result->uids, 0, ctx->uid);
+ if (ctx->score != 0) {
+ score = array_append_space(&result->scores);
score->uid = ctx->uid;
score->score = ctx->score;
}
i_assert(ctx->depth >= (int)ctx->state);
+ if (ctx->state == SOLR_XML_RESPONSE_STATE_CONTENT &&
+ ctx->content_state == SOLR_XML_CONTENT_STATE_MAILBOX &&
+ ctx->mailbox == NULL) {
+ /* mailbox is namespace prefix */
+ ctx->mailbox = i_strdup("");
+ }
+
if (ctx->depth == (int)ctx->state) {
- if (ctx->state == SOLR_XML_RESPONSE_STATE_DOC)
- solr_lookup_add_doc(ctx);
+ if (ctx->state == SOLR_XML_RESPONSE_STATE_DOC) {
+ T_BEGIN {
+ solr_lookup_add_doc(ctx);
+ } T_END;
+ }
ctx->state--;
ctx->content_state = SOLR_XML_CONTENT_STATE_NONE;
}
}
int solr_connection_select(struct solr_connection *conn, const char *query,
- solr_uid_map_callback_t *callback, void *context,
- ARRAY_TYPE(seq_range) *uids,
- ARRAY_TYPE(fts_score_map) *scores)
+ pool_t pool, struct solr_result ***box_results_r)
{
struct solr_lookup_xml_context solr_lookup_context;
CURLcode ret;
long httpret;
+ int parse_ret;
i_assert(!conn->posting);
memset(&solr_lookup_context, 0, sizeof(solr_lookup_context));
- solr_lookup_context.uids = uids;
- solr_lookup_context.scores = scores;
- solr_lookup_context.callback = callback;
- solr_lookup_context.context = context;
+ solr_lookup_context.result_pool = pool;
+ solr_lookup_context.mailboxes =
+ hash_table_create(default_pool, default_pool, 0,
+ str_hash, (hash_cmp_callback_t *)strcmp);
+ p_array_init(&solr_lookup_context.results, pool, 32);
i_free_and_null(conn->http_failure);
conn->xml_failed = FALSE;
i_error("fts_solr: Lookup failed: %s", conn->http_failure);
return -1;
}
- return solr_xml_parse(conn, NULL, 0, TRUE);
+ parse_ret = solr_xml_parse(conn, NULL, 0, TRUE);
+ hash_table_destroy(&solr_lookup_context.mailboxes);
+
+ (void)array_append_space(&solr_lookup_context.results);
+ *box_results_r = array_idx_modifiable(&solr_lookup_context.results, 0);
+ return parse_ret;
}
struct solr_connection_post *
#include "seq-range-array.h"
#include "fts-api.h"
-/* Returns TRUE if UID conversion was done, FALSE if uid should be skipped. */
-typedef bool solr_uid_map_callback_t(const char *ns_prefix, const char *mailbox,
- uint32_t uidvalidity, uint32_t *uid,
- void *context);
+struct solr_result {
+ const char *box_id;
+
+ ARRAY_TYPE(seq_range) uids;
+ ARRAY_TYPE(fts_score_map) scores;
+};
struct solr_connection *solr_connection_init(const char *url, bool debug);
void solr_connection_deinit(struct solr_connection *conn);
const char *str);
int solr_connection_select(struct solr_connection *conn, const char *query,
- solr_uid_map_callback_t *callback, void *context,
- ARRAY_TYPE(seq_range) *uids,
- ARRAY_TYPE(fts_score_map) *scores);
+ pool_t pool, struct solr_result ***box_results_r);
int solr_connection_post(struct solr_connection *conn, const char *cmd);
struct solr_connection_post *
#include "lib.h"
#include "array.h"
+#include "str.h"
#include "mail-user.h"
#include "mail-namespace.h"
#include "mail-storage-private.h"
struct squat_fts_backend {
struct fts_backend backend;
+
+ struct mailbox *box;
struct squat_trie *trie;
+
+ unsigned int partial_len, full_len;
+ bool refresh;
};
-struct squat_fts_backend_build_context {
- struct fts_backend_build_context ctx;
+struct squat_fts_backend_update_context {
+ struct fts_backend_update_context ctx;
struct squat_trie_build_context *build_ctx;
+
enum squat_index_type squat_type;
uint32_t uid;
+ string_t *hdr;
+
+ bool failed;
};
-static void
-fts_backend_squat_set(struct squat_fts_backend *backend, const char *str)
+static struct fts_backend *fts_backend_squat_alloc(void)
+{
+ struct squat_fts_backend *backend;
+
+ backend = i_new(struct squat_fts_backend, 1);
+ backend->backend = fts_backend_squat;
+ return &backend->backend;
+}
+
+static int
+fts_backend_squat_init(struct fts_backend *_backend, const char **error_r)
{
- const char *const *tmp;
+ struct squat_fts_backend *backend =
+ (struct squat_fts_backend *)_backend;
+ const char *const *tmp, *env;
unsigned int len;
- for (tmp = t_strsplit_spaces(str, " "); *tmp != NULL; tmp++) {
+ env = mail_user_plugin_getenv(_backend->ns->user, "fts_squat");
+ if (env == NULL)
+ return 0;
+
+ for (tmp = t_strsplit_spaces(env, " "); *tmp != NULL; tmp++) {
if (strncmp(*tmp, "partial=", 8) == 0) {
if (str_to_uint(*tmp + 8, &len) < 0 || len == 0) {
- i_fatal("fts_squat: Invalid partial len: %s",
- *tmp + 8);
+ *error_r = t_strdup_printf(
+ "Invalid partial length: %s", *tmp + 8);
+ return -1;
}
- squat_trie_set_partial_len(backend->trie, len);
+ backend->partial_len = len;
} else if (strncmp(*tmp, "full=", 5) == 0) {
if (str_to_uint(*tmp + 5, &len) < 0 || len == 0) {
- i_fatal("fts_squat: Invalid full len: %s",
- *tmp + 5);
+ *error_r = t_strdup_printf(
+ "Invalid full length: %s", *tmp + 5);
+ return -1;
}
- squat_trie_set_full_len(backend->trie, len);
+ backend->full_len = len;
} else {
- i_fatal("fts_squat: Invalid setting: %s", *tmp);
+ *error_r = t_strdup_printf("Invalid setting: %s", *tmp);
+ return -1;
}
}
+ return 0;
}
-static struct fts_backend *fts_backend_squat_init(struct mailbox *box)
+static void
+fts_backend_squat_unset_box(struct squat_fts_backend *backend)
+{
+ if (backend->trie != NULL)
+ squat_trie_deinit(&backend->trie);
+ backend->box = NULL;
+}
+
+static void fts_backend_squat_deinit(struct fts_backend *_backend)
+{
+ struct squat_fts_backend *backend =
+ (struct squat_fts_backend *)_backend;
+
+ fts_backend_squat_unset_box(backend);
+ i_free(backend);
+}
+
+static void
+fts_backend_squat_set_box(struct squat_fts_backend *backend,
+ struct mailbox *box)
{
const struct mailbox_permissions *perm = mailbox_get_permissions(box);
- struct squat_fts_backend *backend;
struct mail_storage *storage;
struct mailbox_status status;
- const char *path, *env;
+ const char *path;
enum squat_index_flags flags = 0;
+ if (backend->box == box)
+ return;
+ fts_backend_squat_unset_box(backend);
+
storage = mailbox_get_storage(box);
path = mailbox_list_get_path(box->list, box->name,
MAILBOX_LIST_PATH_TYPE_INDEX);
- if (*path == '\0') {
- /* in-memory indexes */
- if (storage->set->mail_debug)
- i_debug("fts squat: Disabled with in-memory indexes");
- return NULL;
- }
+ i_assert(*path != '\0'); /* fts already checked this */
mailbox_get_open_status(box, STATUS_UIDVALIDITY, &status);
if (storage->set->mmap_disable)
if (storage->set->dotlock_use_excl)
flags |= SQUAT_INDEX_FLAG_DOTLOCK_USE_EXCL;
- backend = i_new(struct squat_fts_backend, 1);
- backend->backend = fts_backend_squat;
backend->trie =
squat_trie_init(t_strconcat(path, "/"SQUAT_FILE_PREFIX, NULL),
status.uidvalidity,
flags, perm->file_create_mode,
perm->file_create_gid);
- env = mail_user_plugin_getenv(box->storage->user, "fts_squat");
- if (env != NULL)
- fts_backend_squat_set(backend, env);
- return &backend->backend;
+ if (backend->partial_len != 0)
+ squat_trie_set_partial_len(backend->trie, backend->partial_len);
+ if (backend->full_len != 0)
+ squat_trie_set_full_len(backend->trie, backend->full_len);
+ backend->box = box;
}
-static void fts_backend_squat_deinit(struct fts_backend *_backend)
-{
- struct squat_fts_backend *backend =
- (struct squat_fts_backend *)_backend;
-
- squat_trie_deinit(&backend->trie);
- i_free(backend);
-}
-
-static int fts_backend_squat_get_last_uid(struct fts_backend *_backend,
- uint32_t *last_uid_r)
+static int
+fts_backend_squat_get_last_uid(struct fts_backend *_backend,
+ struct mailbox *box, uint32_t *last_uid_r)
{
struct squat_fts_backend *backend =
(struct squat_fts_backend *)_backend;
+ fts_backend_squat_set_box(backend, box);
return squat_trie_get_last_uid(backend->trie, last_uid_r);
}
-static int
-fts_backend_squat_build_init(struct fts_backend *_backend, uint32_t *last_uid_r,
- struct fts_backend_build_context **ctx_r)
+static struct fts_backend_update_context *
+fts_backend_squat_update_init(struct fts_backend *_backend)
{
- struct squat_fts_backend *backend =
- (struct squat_fts_backend *)_backend;
- struct squat_fts_backend_build_context *ctx;
- struct squat_trie_build_context *build_ctx;
+ struct squat_fts_backend_update_context *ctx;
- if (squat_trie_build_init(backend->trie, last_uid_r, &build_ctx) < 0)
- return -1;
-
- ctx = i_new(struct squat_fts_backend_build_context, 1);
+ ctx = i_new(struct squat_fts_backend_update_context, 1);
ctx->ctx.backend = _backend;
- ctx->build_ctx = build_ctx;
-
- *ctx_r = &ctx->ctx;
- return 0;
-}
-
-static void
-fts_backend_squat_build_hdr(struct fts_backend_build_context *_ctx,
- uint32_t uid)
-{
- struct squat_fts_backend_build_context *ctx =
- (struct squat_fts_backend_build_context *)_ctx;
-
- ctx->squat_type = SQUAT_INDEX_TYPE_HEADER;
- ctx->uid = uid;
-}
-
-static bool
-fts_backend_squat_build_body_begin(struct fts_backend_build_context *_ctx,
- uint32_t uid, const char *content_type,
- const char *content_disposition ATTR_UNUSED)
-{
- struct squat_fts_backend_build_context *ctx =
- (struct squat_fts_backend_build_context *)_ctx;
-
- if (!fts_backend_default_can_index(content_type))
- return FALSE;
-
- ctx->squat_type = SQUAT_INDEX_TYPE_BODY;
- ctx->uid = uid;
- return TRUE;
-}
-
-static int
-fts_backend_squat_build_more(struct fts_backend_build_context *_ctx,
- const unsigned char *data, size_t size)
-{
- struct squat_fts_backend_build_context *ctx =
- (struct squat_fts_backend_build_context *)_ctx;
-
- return squat_trie_build_more(ctx->build_ctx, ctx->uid, ctx->squat_type,
- data, size);
+ ctx->hdr = str_new(default_pool, 1024*32);
+ return &ctx->ctx;
}
static int get_all_msg_uids(struct mailbox *box, ARRAY_TYPE(seq_range) *uids)
}
static int
-fts_backend_squat_build_deinit(struct fts_backend_build_context *_ctx)
+fts_backend_squat_update_uid_changed(struct squat_fts_backend_update_context *ctx)
+{
+ int ret = 0;
+
+ if (ctx->uid == 0)
+ return 0;
+
+ if (squat_trie_build_more(ctx->build_ctx, ctx->uid,
+ SQUAT_INDEX_TYPE_HEADER,
+ str_data(ctx->hdr), str_len(ctx->hdr)) < 0)
+ ret = -1;
+ str_truncate(ctx->hdr, 0);
+ return ret;
+}
+
+static int
+fts_backend_squat_build_deinit(struct squat_fts_backend_update_context *ctx)
{
- struct squat_fts_backend_build_context *ctx =
- (struct squat_fts_backend_build_context *)_ctx;
+ struct squat_fts_backend *backend =
+ (struct squat_fts_backend *)ctx->ctx.backend;
ARRAY_TYPE(seq_range) uids;
- int ret;
+ int ret = 0;
+
+ if (ctx->build_ctx == NULL)
+ return 0;
+
+ if (fts_backend_squat_update_uid_changed(ctx) < 0)
+ ret = -1;
i_array_init(&uids, 1024);
- if (get_all_msg_uids(ctx->ctx.backend->box, &uids) < 0)
- ret = squat_trie_build_deinit(&ctx->build_ctx, NULL);
- else {
+ if (get_all_msg_uids(backend->box, &uids) < 0) {
+ (void)squat_trie_build_deinit(&ctx->build_ctx, NULL);
+ ret = -1;
+ } else {
seq_range_array_invert(&uids, 2, (uint32_t)-2);
- ret = squat_trie_build_deinit(&ctx->build_ctx, &uids);
+ if (squat_trie_build_deinit(&ctx->build_ctx, &uids) < 0)
+ ret = -1;
}
array_free(&uids);
+ return ret;
+}
+
+static int
+fts_backend_squat_update_deinit(struct fts_backend_update_context *_ctx)
+{
+ struct squat_fts_backend_update_context *ctx =
+ (struct squat_fts_backend_update_context *)_ctx;
+ int ret = ctx->failed ? -1 : 0;
+
+ if (fts_backend_squat_build_deinit(ctx) < 0)
+ ret = -1;
+ str_free(&ctx->hdr);
i_free(ctx);
return ret;
}
static void
-fts_backend_squat_expunge(struct fts_backend *_backend ATTR_UNUSED,
- struct mail *mail ATTR_UNUSED)
+fts_backend_squat_update_set_mailbox(struct fts_backend_update_context *_ctx,
+ struct mailbox *box)
{
+ struct squat_fts_backend_update_context *ctx =
+ (struct squat_fts_backend_update_context *)_ctx;
+ struct squat_fts_backend *backend =
+ (struct squat_fts_backend *)ctx->ctx.backend;
+
+ if (fts_backend_squat_build_deinit(ctx) < 0)
+ ctx->failed = TRUE;
+ fts_backend_squat_set_box(backend, box);
+
+ if (squat_trie_build_init(backend->trie, &ctx->build_ctx) < 0)
+ ctx->failed = TRUE;
}
static void
-fts_backend_squat_expunge_finish(struct fts_backend *_backend ATTR_UNUSED,
- struct mailbox *box ATTR_UNUSED,
- bool committed ATTR_UNUSED)
+fts_backend_squat_update_expunge(struct fts_backend_update_context *_ctx ATTR_UNUSED,
+ uint32_t last_uid ATTR_UNUSED)
{
/* FIXME */
}
+static bool
+fts_backend_squat_update_set_build_key(struct fts_backend_update_context *_ctx,
+ const struct fts_backend_build_key *key)
+{
+ struct squat_fts_backend_update_context *ctx =
+ (struct squat_fts_backend_update_context *)_ctx;
+
+ if (ctx->failed)
+ return FALSE;
+
+ if (key->uid != ctx->uid) {
+ if (fts_backend_squat_update_uid_changed(ctx) < 0)
+ ctx->failed = TRUE;
+ }
+
+ switch (key->type) {
+ case FTS_BACKEND_BUILD_KEY_HDR:
+ case FTS_BACKEND_BUILD_KEY_MIME_HDR:
+ str_printfa(ctx->hdr, "%s: ", key->hdr_name);
+ ctx->squat_type = SQUAT_INDEX_TYPE_HEADER;
+ break;
+ case FTS_BACKEND_BUILD_KEY_BODY_PART:
+ ctx->squat_type = SQUAT_INDEX_TYPE_BODY;
+ break;
+ case FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY:
+ i_unreached();
+ }
+ ctx->uid = key->uid;
+ return TRUE;
+}
+
+static void
+fts_backend_squat_update_unset_build_key(struct fts_backend_update_context *_ctx)
+{
+ struct squat_fts_backend_update_context *ctx =
+ (struct squat_fts_backend_update_context *)_ctx;
+
+ if (ctx->squat_type == SQUAT_INDEX_TYPE_HEADER)
+ str_append_c(ctx->hdr, '\n');
+}
+
+static int
+fts_backend_squat_update_build_more(struct fts_backend_update_context *_ctx,
+ const unsigned char *data, size_t size)
+{
+ struct squat_fts_backend_update_context *ctx =
+ (struct squat_fts_backend_update_context *)_ctx;
+
+ if (ctx->squat_type == SQUAT_INDEX_TYPE_HEADER) {
+ str_append_n(ctx->hdr, data, size);
+ return 0;
+ }
+ return squat_trie_build_more(ctx->build_ctx, ctx->uid, ctx->squat_type,
+ data, size);
+}
+
static int fts_backend_squat_refresh(struct fts_backend *_backend)
{
struct squat_fts_backend *backend =
(struct squat_fts_backend *)_backend;
- return squat_trie_refresh(backend->trie);
+ backend->refresh = TRUE;
+ return 0;
+}
+
+static int fts_backend_squat_optimize(struct fts_backend *_backend ATTR_UNUSED)
+{
+ /* FIXME: drop expunged messages */
+ return 0;
+}
+
+static int squat_lookup_arg(struct squat_fts_backend *backend,
+ const struct mail_search_arg *arg, bool and_args,
+ ARRAY_TYPE(seq_range) *definite_uids,
+ ARRAY_TYPE(seq_range) *maybe_uids)
+{
+ enum squat_index_type squat_type;
+ ARRAY_TYPE(seq_range) tmp_definite_uids, tmp_maybe_uids;
+ uint32_t last_uid;
+ int ret;
+
+ switch (arg->type) {
+ case SEARCH_TEXT:
+ squat_type = SQUAT_INDEX_TYPE_HEADER |
+ SQUAT_INDEX_TYPE_BODY;
+ break;
+ case SEARCH_BODY:
+ squat_type = SQUAT_INDEX_TYPE_BODY;
+ break;
+ case SEARCH_HEADER:
+ case SEARCH_HEADER_ADDRESS:
+ case SEARCH_HEADER_COMPRESS_LWSP:
+ squat_type = SQUAT_INDEX_TYPE_HEADER;
+ break;
+ default:
+ return 0;
+ }
+
+ i_array_init(&tmp_definite_uids, 128);
+ i_array_init(&tmp_maybe_uids, 128);
+
+ ret = squat_trie_lookup(backend->trie, arg->value.str, squat_type,
+ &tmp_definite_uids, &tmp_maybe_uids);
+ if (arg->match_not) {
+ /* definite -> non-match
+ maybe -> maybe
+ non-match -> maybe */
+ array_clear(&tmp_maybe_uids);
+
+ if (squat_trie_get_last_uid(backend->trie, &last_uid) < 0)
+ i_unreached();
+ seq_range_array_add_range(&tmp_maybe_uids, 1, last_uid);
+ seq_range_array_remove_seq_range(&tmp_maybe_uids,
+ &tmp_definite_uids);
+ array_clear(&tmp_definite_uids);
+ }
+
+ if (and_args) {
+ /* AND:
+ definite && definite -> definite
+ definite && maybe -> maybe
+ maybe && maybe -> maybe */
+
+ /* put definites among maybies, so they can be intersected */
+ seq_range_array_merge(maybe_uids, definite_uids);
+ seq_range_array_merge(&tmp_maybe_uids, &tmp_definite_uids);
+
+ seq_range_array_intersect(maybe_uids, &tmp_maybe_uids);
+ seq_range_array_intersect(definite_uids, &tmp_definite_uids);
+ /* remove duplicate maybies that are also definites */
+ seq_range_array_remove_seq_range(maybe_uids, definite_uids);
+ } else {
+ /* OR:
+ definite || definite -> definite
+ definite || maybe -> definite
+ maybe || maybe -> maybe */
+
+ /* remove maybies that are now definites */
+ seq_range_array_remove_seq_range(&tmp_maybe_uids,
+ definite_uids);
+ seq_range_array_remove_seq_range(maybe_uids,
+ &tmp_definite_uids);
+
+ seq_range_array_merge(definite_uids, &tmp_definite_uids);
+ seq_range_array_merge(maybe_uids, &tmp_maybe_uids);
+ }
+
+ array_free(&tmp_definite_uids);
+ array_free(&tmp_maybe_uids);
+ return ret < 0 ? -1 : 1;
}
static int
-fts_backend_squat_lookup(struct fts_backend *_backend, const char *key,
- enum fts_lookup_flags flags,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids)
+fts_backend_squat_lookup(struct fts_backend *_backend, struct mailbox *box,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_result *result)
{
struct squat_fts_backend *backend =
(struct squat_fts_backend *)_backend;
- enum squat_index_type squat_type = 0;
+ int ret;
- i_assert((flags & FTS_LOOKUP_FLAG_INVERT) == 0);
+ fts_backend_squat_set_box(backend, box);
+ if (backend->refresh) {
+ if (squat_trie_refresh(backend->trie) < 0)
+ return -1;
+ backend->refresh = FALSE;
+ }
- if ((flags & FTS_LOOKUP_FLAG_HEADER) != 0)
- squat_type |= SQUAT_INDEX_TYPE_HEADER;
- if ((flags & FTS_LOOKUP_FLAG_BODY) != 0)
- squat_type |= SQUAT_INDEX_TYPE_BODY;
- i_assert(squat_type != 0);
+ for (; args != NULL; args = args->next) {
+ ret = squat_lookup_arg(backend, args, and_args,
+ &result->definite_uids,
+ &result->maybe_uids);
+ if (ret < 0)
+ return -1;
+ if (ret > 0)
+ args->match_always = TRUE;
- return squat_trie_lookup(backend->trie, key, squat_type,
- definite_uids, maybe_uids);
+ }
+ return 0;
}
struct fts_backend fts_backend_squat = {
.flags = 0,
{
+ fts_backend_squat_alloc,
fts_backend_squat_init,
fts_backend_squat_deinit,
fts_backend_squat_get_last_uid,
- NULL,
- fts_backend_squat_build_init,
- fts_backend_squat_build_hdr,
- fts_backend_squat_build_body_begin,
- NULL,
- fts_backend_squat_build_more,
- fts_backend_squat_build_deinit,
- fts_backend_squat_expunge,
- fts_backend_squat_expunge_finish,
+ fts_backend_squat_update_init,
+ fts_backend_squat_update_deinit,
+ fts_backend_squat_update_set_mailbox,
+ fts_backend_squat_update_expunge,
+ fts_backend_squat_update_set_build_key,
+ fts_backend_squat_update_unset_build_key,
+ fts_backend_squat_update_build_more,
fts_backend_squat_refresh,
+ fts_backend_squat_optimize,
+ fts_backend_default_can_lookup,
fts_backend_squat_lookup,
- NULL,
NULL
}
};
enum squat_index_type index_type;
bool data_header = TRUE, first = TRUE, skip_body = FALSE;
bool mime_header = TRUE;
- uint32_t last_uid;
size_t trie_mem, uidlist_mem;
clock_t clock_start, clock_end;
struct timeval tv_start, tv_end;
if (fd == -1)
return 1;
- if (squat_trie_build_init(trie, &last_uid, &build_ctx) < 0)
+ if (squat_trie_build_init(trie, &build_ctx) < 0)
return 1;
valid = buffer_create_dynamic(default_pool, 4096);
return fd;
}
-int squat_trie_build_init(struct squat_trie *trie, uint32_t *last_uid_r,
+int squat_trie_build_init(struct squat_trie *trie,
struct squat_trie_build_context **ctx_r)
{
struct squat_trie_build_context *ctx;
ctx->uidlist_build_ctx = uidlist_build_ctx;
ctx->first_uid = trie->root.next_uid;
- *last_uid_r = I_MAX((trie->root.next_uid+1)/2, 1) - 1;
*ctx_r = ctx;
return 0;
}
int squat_trie_refresh(struct squat_trie *trie);
-int squat_trie_build_init(struct squat_trie *trie, uint32_t *last_uid_r,
+int squat_trie_build_init(struct squat_trie *trie,
struct squat_trie_build_context **ctx_r);
/* bodies must be added before headers */
int squat_trie_build_more(struct squat_trie_build_context *ctx,
lib20_fts_plugin_la_SOURCES = \
fts-api.c \
- fts-mailbox.c \
+ fts-build.c \
+ fts-build-indexer.c \
+ fts-build-mailbox.c \
+ fts-build-virtual.c \
fts-plugin.c \
fts-search.c \
+ fts-search-serialize.c \
fts-storage.c
noinst_HEADERS = \
fts-api.h \
fts-api-private.h \
- fts-mailbox.h \
+ fts-build.h \
+ fts-build-private.h \
fts-plugin.h \
+ fts-search-serialize.h \
fts-storage.h
#include "fts-api.h"
+#define MAILBOX_GUID_HEX_LENGTH (MAIL_GUID_128_SIZE*2)
+
struct fts_backend_vfuncs {
- struct fts_backend *(*init)(struct mailbox *box);
+ struct fts_backend *(*alloc)(void);
+ int (*init)(struct fts_backend *backend, const char **error_r);
void (*deinit)(struct fts_backend *backend);
- int (*get_last_uid)(struct fts_backend *backend, uint32_t *last_uid_r);
- int (*get_all_last_uids)(struct fts_backend *backend, pool_t pool,
- ARRAY_TYPE(fts_backend_uid_map) *last_uids);
-
- int (*build_init)(struct fts_backend *backend, uint32_t *last_uid_r,
- struct fts_backend_build_context **ctx_r);
- void (*build_hdr)(struct fts_backend_build_context *ctx, uint32_t uid);
- bool (*build_body_begin)(struct fts_backend_build_context *ctx,
- uint32_t uid, const char *content_type,
- const char *content_disposition);
- void (*build_body_end)(struct fts_backend_build_context *ctx);
- int (*build_more)(struct fts_backend_build_context *ctx,
- const unsigned char *data, size_t size);
- int (*build_deinit)(struct fts_backend_build_context *ctx);
-
- void (*expunge)(struct fts_backend *backend, struct mail *mail);
- void (*expunge_finish)(struct fts_backend *backend,
- struct mailbox *box, bool committed);
+ int (*get_last_uid)(struct fts_backend *backend, struct mailbox *box,
+ uint32_t *last_uid_r);
- int (*refresh)(struct fts_backend *backend);
+ struct fts_backend_update_context *
+ (*update_init)(struct fts_backend *backend);
+ int (*update_deinit)(struct fts_backend_update_context *ctx);
+
+ void (*update_set_mailbox)(struct fts_backend_update_context *ctx,
+ struct mailbox *box);
+ void (*update_expunge)(struct fts_backend_update_context *ctx,
+ uint32_t uid);
- int (*lookup)(struct fts_backend *backend, const char *key,
- enum fts_lookup_flags flags,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids);
- int (*filter)(struct fts_backend *backend, const char *key,
- enum fts_lookup_flags flags,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids);
-
- int (*lookup2)(struct fts_backend_lookup_context *ctx,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids,
- ARRAY_TYPE(fts_score_map) *scores);
+ /* Start a build for specified key */
+ bool (*update_set_build_key)(struct fts_backend_update_context *ctx,
+ const struct fts_backend_build_key *key);
+ /* Finish a build for specified key - guaranteed to be called */
+ void (*update_unset_build_key)(struct fts_backend_update_context *ctx);
+ /* Add data for current build key */
+ int (*update_build_more)(struct fts_backend_update_context *ctx,
+ const unsigned char *data, size_t size);
+
+ int (*refresh)(struct fts_backend *backend);
+ int (*optimize)(struct fts_backend *backend);
+
+ bool (*can_lookup)(struct fts_backend *backend,
+ const struct mail_search_arg *args);
+ int (*lookup)(struct fts_backend *backend, struct mailbox *box,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_result *result);
+ int (*lookup_multi)(struct fts_backend *backend,
+ struct mailbox *const boxes[],
+ struct mail_search_arg *args, bool and_args,
+ struct fts_multi_result *result);
};
enum fts_backend_flags {
- /* Backend supports virtual mailbox lookups. */
- FTS_BACKEND_FLAG_VIRTUAL_LOOKUPS = 0x02,
/* Backend supports indexing binary MIME parts */
- FTS_BACKEND_FLAG_BINARY_MIME_PARTS = 0x04
+ FTS_BACKEND_FLAG_BINARY_MIME_PARTS = 0x01
};
struct fts_backend {
enum fts_backend_flags flags;
struct fts_backend_vfuncs v;
- struct mailbox *box;
+ struct mail_namespace *ns;
- unsigned int building:1;
+ unsigned int updating:1;
};
-struct fts_backend_build_context {
+struct fts_backend_update_context {
struct fts_backend *backend;
- unsigned int failed:1;
-};
+ struct mailbox *cur_box, *backend_box;
-struct fts_backend_lookup_field {
- const char *key;
- enum fts_lookup_flags flags;
+ unsigned int build_key_open:1;
+ unsigned int failed:1;
};
-struct fts_backend_lookup_context {
- struct fts_backend *backend;
- pool_t pool;
+struct fts_index_header {
+ uint32_t last_indexed_uid;
- ARRAY_DEFINE(fields, struct fts_backend_lookup_field);
+ /* highest UID and number of messages when optimization was last
+ time done */
+ uint32_t last_optimize_uid;
+ uint32_t last_optimize_msgcount;
};
void fts_backend_register(const struct fts_backend *backend);
void fts_backend_unregister(const char *name);
-bool fts_backend_default_can_index(const char *content_type);
+bool fts_backend_default_can_lookup(struct fts_backend *backend,
+ const struct mail_search_arg *args);
void fts_filter_uids(ARRAY_TYPE(seq_range) *definite_dest,
const ARRAY_TYPE(seq_range) *definite_filter,
ARRAY_TYPE(seq_range) *maybe_dest,
const ARRAY_TYPE(seq_range) *maybe_filter);
+/* Returns TRUE if ok, FALSE if no fts header */
+bool fts_index_get_last_uid(struct mailbox *box, uint32_t *last_uid_r);
+int fts_index_set_last_uid(struct mailbox *box, uint32_t last_uid);
+
+/* Returns TRUE if FTS backend should index the header for optimizing
+ separate lookups */
+bool fts_header_want_indexed(const char *hdr_name);
+
+int fts_mailbox_get_guid(struct mailbox *box, const char **guid_r);
+
#endif
#include "lib.h"
#include "array.h"
+#include "hex-binary.h"
+#include "mail-index.h"
+#include "mail-storage-private.h"
+#include "mail-search.h"
+#include "../virtual/virtual-storage.h"
#include "fts-api-private.h"
static ARRAY_DEFINE(backends, const struct fts_backend *);
return NULL;
}
-struct fts_backend *
-fts_backend_init(const char *backend_name, struct mailbox *box)
+int fts_backend_init(const char *backend_name, struct mail_namespace *ns,
+ const char **error_r, struct fts_backend **backend_r)
{
const struct fts_backend *be;
struct fts_backend *backend;
be = fts_backend_class_lookup(backend_name);
if (be == NULL) {
- i_error("Unknown FTS backend: %s", backend_name);
- return NULL;
+ *error_r = "Unknown backend";
+ return -1;
}
- backend = be->v.init(box);
- if (backend == NULL)
- return NULL;
- backend->box = box;
- return backend;
+ backend = be->v.alloc();
+ backend->ns = ns;
+ if (backend->v.init(backend, error_r) < 0) {
+ i_free(backend);
+ return -1;
+ }
+ *backend_r = backend;
+ return 0;
}
void fts_backend_deinit(struct fts_backend **_backend)
backend->v.deinit(backend);
}
-int fts_backend_get_last_uid(struct fts_backend *backend, uint32_t *last_uid_r)
+int fts_backend_get_last_uid(struct fts_backend *backend, struct mailbox *box,
+ uint32_t *last_uid_r)
{
- return backend->v.get_last_uid(backend, last_uid_r);
+ if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) == 0) {
+ /* virtual mailboxes themselves don't have any indexes,
+ so catch this call here */
+ if (!fts_index_get_last_uid(box, last_uid_r))
+ *last_uid_r = 0;
+ return 0;
+ }
+
+ return backend->v.get_last_uid(backend, box, last_uid_r);
}
-int fts_backend_get_all_last_uids(struct fts_backend *backend, pool_t pool,
- ARRAY_TYPE(fts_backend_uid_map) *last_uids)
+bool fts_backend_is_updating(struct fts_backend *backend)
{
- return backend->v.get_all_last_uids(backend, pool, last_uids);
+ return backend->updating;
}
-int fts_backend_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
- struct fts_backend_build_context **ctx_r)
+struct fts_backend_update_context *
+fts_backend_update_init(struct fts_backend *backend)
{
- int ret;
+ i_assert(!backend->updating);
- i_assert(!backend->building);
-
- ret = backend->v.build_init(backend, last_uid_r, ctx_r);
- if (ret == 0)
- backend->building = TRUE;
- return ret;
+ backend->updating = TRUE;
+ return backend->v.update_init(backend);
}
-void fts_backend_build_hdr(struct fts_backend_build_context *ctx, uint32_t uid)
+static void fts_backend_set_cur_mailbox(struct fts_backend_update_context *ctx)
{
- ctx->backend->v.build_hdr(ctx, uid);
+ fts_backend_update_unset_build_key(ctx);
+ if (ctx->backend_box != ctx->cur_box) {
+ ctx->backend->v.update_set_mailbox(ctx, ctx->cur_box);
+ ctx->backend_box = ctx->cur_box;
+ }
}
-bool fts_backend_build_body_begin(struct fts_backend_build_context *ctx,
- uint32_t uid, const char *content_type,
- const char *content_disposition)
+int fts_backend_update_deinit(struct fts_backend_update_context **_ctx)
{
- return ctx->backend->v.build_body_begin(ctx, uid, content_type,
- content_disposition);
+ struct fts_backend_update_context *ctx = *_ctx;
+
+ *_ctx = NULL;
+
+ ctx->cur_box = NULL;
+ fts_backend_set_cur_mailbox(ctx);
+
+ return ctx->backend->v.update_deinit(ctx);
}
-void fts_backend_build_body_end(struct fts_backend_build_context *ctx)
+void fts_backend_update_set_mailbox(struct fts_backend_update_context *ctx,
+ struct mailbox *box)
{
- if (ctx->backend->v.build_body_end != NULL)
- ctx->backend->v.build_body_end(ctx);
+ if (ctx->backend_box != NULL && box != ctx->backend_box) {
+ /* make sure we don't reference the backend box anymore */
+ ctx->backend->v.update_set_mailbox(ctx, NULL);
+ ctx->backend_box = NULL;
+ }
+ ctx->cur_box = box;
}
-int fts_backend_build_more(struct fts_backend_build_context *ctx,
- const unsigned char *data, size_t size)
+void fts_backend_update_expunge(struct fts_backend_update_context *ctx,
+ uint32_t uid)
{
- return ctx->backend->v.build_more(ctx, data, size);
+ fts_backend_set_cur_mailbox(ctx);
+ ctx->backend->v.update_expunge(ctx, uid);
}
-int fts_backend_build_deinit(struct fts_backend_build_context **_ctx)
+bool fts_backend_update_set_build_key(struct fts_backend_update_context *ctx,
+ const struct fts_backend_build_key *key)
{
- struct fts_backend_build_context *ctx = *_ctx;
+ fts_backend_set_cur_mailbox(ctx);
- *_ctx = NULL;
- ctx->backend->building = FALSE;
- return ctx->backend->v.build_deinit(ctx);
+ if (!ctx->backend->v.update_set_build_key(ctx, key))
+ return FALSE;
+ ctx->build_key_open = TRUE;
+ return TRUE;
}
-bool fts_backend_is_building(struct fts_backend *backend)
+void fts_backend_update_unset_build_key(struct fts_backend_update_context *ctx)
{
- return backend->building;
+ if (ctx->build_key_open) {
+ ctx->backend->v.update_unset_build_key(ctx);
+ ctx->build_key_open = FALSE;
+ }
}
-void fts_backend_expunge(struct fts_backend *backend, struct mail *mail)
+int fts_backend_update_build_more(struct fts_backend_update_context *ctx,
+ const unsigned char *data, size_t size)
{
- backend->v.expunge(backend, mail);
-}
+ i_assert(ctx->build_key_open);
-void fts_backend_expunge_finish(struct fts_backend *backend,
- struct mailbox *box, bool committed)
-{
- backend->v.expunge_finish(backend, box, committed);
+ return ctx->backend->v.update_build_more(ctx, data, size);
}
int fts_backend_refresh(struct fts_backend *backend)
return backend->v.refresh(backend);
}
+int fts_backend_optimize(struct fts_backend *backend)
+{
+ return backend->v.optimize == NULL ? 0 :
+ backend->v.optimize(backend);
+}
+
static void
fts_merge_maybies(ARRAY_TYPE(seq_range) *dest_maybe,
const ARRAY_TYPE(seq_range) *dest_definite,
seq_range_array_intersect(definite_dest, definite_filter);
}
-static void fts_lookup_invert(ARRAY_TYPE(seq_range) *definite_uids,
- const ARRAY_TYPE(seq_range) *maybe_uids)
+bool fts_backend_default_can_lookup(struct fts_backend *backend,
+ const struct mail_search_arg *args)
{
- /* we'll begin by inverting definite UIDs */
- seq_range_array_invert(definite_uids, 1, (uint32_t)-1);
-
- /* from that list remove UIDs in the maybe list.
- the maybe list itself isn't touched. */
- (void)seq_range_array_remove_seq_range(definite_uids, maybe_uids);
+ for (; args != NULL; args = args->next) {
+ switch (args->type) {
+ case SEARCH_OR:
+ case SEARCH_SUB:
+ case SEARCH_INTHREAD:
+ if (fts_backend_default_can_lookup(backend,
+ args->value.subargs))
+ return TRUE;
+ break;
+ case SEARCH_HEADER:
+ case SEARCH_HEADER_ADDRESS:
+ case SEARCH_HEADER_COMPRESS_LWSP:
+ case SEARCH_BODY:
+ case SEARCH_TEXT:
+ return TRUE;
+ default:
+ break;
+ }
+ }
+ return FALSE;
}
-static int fts_backend_lookup(struct fts_backend *backend, const char *key,
- enum fts_lookup_flags flags,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids)
+bool fts_backend_can_lookup(struct fts_backend *backend,
+ const struct mail_search_arg *args)
{
- int ret;
+ return backend->v.can_lookup(backend, args);
+}
- ret = backend->v.lookup(backend, key, flags & ~FTS_LOOKUP_FLAG_INVERT,
- definite_uids, maybe_uids);
- if (unlikely(ret < 0))
+static int fts_score_map_sort(const struct fts_score_map *m1,
+ const struct fts_score_map *m2)
+{
+ if (m1->uid < m2->uid)
return -1;
- if ((flags & FTS_LOOKUP_FLAG_INVERT) != 0)
- fts_lookup_invert(definite_uids, maybe_uids);
+ if (m1->uid > m2->uid)
+ return 1;
return 0;
}
-static int fts_backend_filter(struct fts_backend *backend, const char *key,
- enum fts_lookup_flags flags,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids)
+int fts_backend_lookup(struct fts_backend *backend, struct mailbox *box,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_result *result)
{
- ARRAY_TYPE(seq_range) tmp_definite, tmp_maybe;
- int ret;
+ array_clear(&result->definite_uids);
+ array_clear(&result->maybe_uids);
+ array_clear(&result->scores);
- if (backend->v.filter != NULL) {
- return backend->v.filter(backend, key, flags,
- definite_uids, maybe_uids);
- }
+ if (backend->v.lookup(backend, box, args, and_args, result) < 0)
+ return -1;
- /* do this ourself */
- i_array_init(&tmp_definite, 64);
- i_array_init(&tmp_maybe, 64);
- ret = fts_backend_lookup(backend, key, flags,
- &tmp_definite, &tmp_maybe);
- if (ret < 0) {
- array_clear(definite_uids);
- array_clear(maybe_uids);
- } else {
- fts_filter_uids(definite_uids, &tmp_definite,
- maybe_uids, &tmp_maybe);
+ if (!result->scores_sorted && array_is_created(&result->scores)) {
+ array_sort(&result->scores, fts_score_map_sort);
+ result->scores_sorted = TRUE;
}
- array_free(&tmp_maybe);
- array_free(&tmp_definite);
- return ret;
+ return 0;
}
-struct fts_backend_lookup_context *
-fts_backend_lookup_init(struct fts_backend *backend)
+int fts_backend_lookup_multi(struct fts_backend *backend,
+ struct mailbox *const boxes[],
+ struct mail_search_arg *args, bool and_args,
+ struct fts_multi_result *result)
{
- struct fts_backend_lookup_context *ctx;
- pool_t pool;
-
- pool = pool_alloconly_create("fts backend lookup", 256);
- ctx = p_new(pool, struct fts_backend_lookup_context, 1);
- ctx->pool = pool;
- ctx->backend = backend;
- p_array_init(&ctx->fields, pool, 8);
- return ctx;
+ i_assert(boxes[0] != NULL);
+
+ return backend->v.lookup_multi(backend, boxes, args, and_args, result);
}
-void fts_backend_lookup_add(struct fts_backend_lookup_context *ctx,
- const char *key, enum fts_lookup_flags flags)
+static bool
+fts_index_get_header(struct mailbox *box, struct fts_index_header *hdr_r,
+ uint32_t *ext_id_r)
{
- struct fts_backend_lookup_field *field;
+ const void *data;
+ size_t data_size;
+
+ *ext_id_r = mail_index_ext_register(box->index, "fts",
+ sizeof(struct fts_index_header),
+ 0, 0);
+ mail_index_get_header_ext(box->view, *ext_id_r, &data, &data_size);
+ if (data_size < sizeof(*hdr_r)) {
+ memset(hdr_r, 0, sizeof(*hdr_r));
+ return FALSE;
+ }
- field = array_append_space(&ctx->fields);
- field->key = p_strdup(ctx->pool, key);
- field->flags = flags;
+ memcpy(hdr_r, data, data_size);
+ return TRUE;
}
-static int fts_backend_lookup_old(struct fts_backend_lookup_context *ctx,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids)
+bool fts_index_get_last_uid(struct mailbox *box, uint32_t *last_uid_r)
{
- const struct fts_backend_lookup_field *fields;
- unsigned int i, count;
+ struct fts_index_header hdr;
+ uint32_t ext_id;
- fields = array_get(&ctx->fields, &count);
- i_assert(count > 0);
-
- if (fts_backend_lookup(ctx->backend, fields[0].key, fields[0].flags,
- definite_uids, maybe_uids) < 0)
- return -1;
- for (i = 1; i < count; i++) {
- if (fts_backend_filter(ctx->backend,
- fields[i].key, fields[i].flags,
- definite_uids, maybe_uids) < 0)
- return -1;
+ if (!fts_index_get_header(box, &hdr, &ext_id)) {
+ *last_uid_r = 0;
+ return FALSE;
}
- return 0;
+
+ *last_uid_r = hdr.last_indexed_uid;
+ return TRUE;
}
-int fts_backend_lookup_deinit(struct fts_backend_lookup_context **_ctx,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids,
- ARRAY_TYPE(fts_score_map) *scores)
+int fts_index_set_last_uid(struct mailbox *box, uint32_t last_uid)
{
- struct fts_backend_lookup_context *ctx = *_ctx;
- int ret;
+ struct mail_index_transaction *trans;
+ struct fts_index_header hdr;
+ uint32_t ext_id;
- *_ctx = NULL;
- if (ctx->backend->v.lookup2 != NULL) {
- ret = ctx->backend->v.lookup2(ctx, definite_uids, maybe_uids,
- scores);
- } else {
- array_clear(scores);
- ret = fts_backend_lookup_old(ctx, definite_uids, maybe_uids);
+ (void)fts_index_get_header(box, &hdr, &ext_id);
+
+ hdr.last_indexed_uid = last_uid;
+ trans = mail_index_transaction_begin(box->view, 0);
+ mail_index_update_header_ext(trans, ext_id, 0, &hdr, sizeof(hdr));
+ return mail_index_transaction_commit(&trans);
+}
+
+static const char *indexed_headers[] = {
+ "From", "To", "Cc", "Bcc", "Subject"
+};
+
+bool fts_header_want_indexed(const char *hdr_name)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(indexed_headers); i++) {
+ if (strcasecmp(hdr_name, indexed_headers[i]) == 0)
+ return TRUE;
}
- pool_unref(&ctx->pool);
- return ret;
+ return FALSE;
}
-bool fts_backend_default_can_index(const char *content_type)
+int fts_mailbox_get_guid(struct mailbox *box, const char **guid_r)
{
- return strncasecmp(content_type, "text/", 5) == 0 ||
- strcasecmp(content_type, "message/rfc822") == 0;
+ struct mailbox_metadata metadata;
+ buffer_t buf;
+ unsigned char guid_hex[MAILBOX_GUID_HEX_LENGTH];
+
+ if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0)
+ return -1;
+
+ buffer_create_data(&buf, guid_hex, sizeof(guid_hex));
+ binary_to_hex_append(&buf, metadata.guid, MAIL_GUID_128_SIZE);
+ *guid_r = t_strndup(guid_hex, sizeof(guid_hex));
+ return 0;
}
struct mail;
struct mailbox;
-struct fts_backend_build_context;
+struct mail_namespace;
+struct mail_search_arg;
-#include "seq-range-array.h"
+struct fts_backend;
-enum fts_lookup_flags {
- /* Search within header and/or body.
- At least one of these must be set. */
- FTS_LOOKUP_FLAG_HEADER = 0x01,
- FTS_LOOKUP_FLAG_BODY = 0x02,
+#include "seq-range-array.h"
- /* The key must NOT be found */
- FTS_LOOKUP_FLAG_INVERT = 0x04
+enum fts_backend_build_key_type {
+ /* Header */
+ FTS_BACKEND_BUILD_KEY_HDR,
+ /* MIME part header */
+ FTS_BACKEND_BUILD_KEY_MIME_HDR,
+ /* MIME body part */
+ FTS_BACKEND_BUILD_KEY_BODY_PART,
+ /* Binary MIME body part, if backend supports binary data */
+ FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY
};
-struct fts_backend_uid_map {
- const char *mailbox;
- uint32_t uidvalidity;
+struct fts_backend_build_key {
uint32_t uid;
+ enum fts_backend_build_key_type type;
+
+ /* for _KEY_HDR: */
+ const char *hdr_name;
+
+ /* for _KEY_BODY_PART and _KEY_BODY_PART_BINARY: */
+
+ /* Contains a valid parsed "type/subtype" string. For messages without
+ (valid) Content-Type: header, it's set to "text/plain". */
+ const char *body_content_type;
+ /* Content-Disposition: header without parsing/validation if it exists,
+ otherwise NULL. */
+ const char *body_content_disposition;
};
-ARRAY_DEFINE_TYPE(fts_backend_uid_map, struct fts_backend_uid_map);
struct fts_score_map {
uint32_t uid;
};
ARRAY_DEFINE_TYPE(fts_score_map, struct fts_score_map);
-struct fts_backend *
-fts_backend_init(const char *backend_name, struct mailbox *box);
+struct fts_result {
+ struct mailbox *box;
+
+ ARRAY_TYPE(seq_range) definite_uids;
+ /* The maybe_uids is useful with backends that can only filter out
+ messages, but can't definitively say if the search matched a
+ message. */
+ ARRAY_TYPE(seq_range) maybe_uids;
+ ARRAY_TYPE(fts_score_map) scores;
+ bool scores_sorted;
+};
+
+struct fts_multi_result {
+ pool_t pool;
+ /* box=NULL-terminated array of mailboxes and matching UIDs,
+ all allocated from the given pool. */
+ struct fts_result *box_results;
+};
+
+int fts_backend_init(const char *backend_name, struct mail_namespace *ns,
+ const char **error_r, struct fts_backend **backend_r);
void fts_backend_deinit(struct fts_backend **backend);
/* Get the last_uid for the mailbox. */
-int fts_backend_get_last_uid(struct fts_backend *backend, uint32_t *last_uid_r);
-/* Get last_uids for all mailboxes that might be backend mailboxes for a
- virtual mailbox. The backend can use mailbox_get_virtual_backend_boxes() or
- mailbox_get_virtual_box_patterns() functions to get the list of mailboxes.
-
- Depending on virtual mailbox configuration, this function may also return
- mailboxes that don't even match the virtual mailbox patterns. The caller
- needs to be able to ignore the unnecessary ones. */
-int fts_backend_get_all_last_uids(struct fts_backend *backend, pool_t pool,
- ARRAY_TYPE(fts_backend_uid_map) *last_uids);
-
-/* Initialize adding new data to the index. last_uid_r is set to the last
- indexed message's IMAP UID */
-int fts_backend_build_init(struct fts_backend *backend, uint32_t *last_uid_r,
- struct fts_backend_build_context **ctx_r);
-/* Switch to building index for mail's headers or MIME part headers. */
-void fts_backend_build_hdr(struct fts_backend_build_context *ctx, uint32_t uid);
-/* Switch to building index for the next body part. If backend doesn't want
- to index this body part (based on content type/disposition check), it can
- return FALSE and caller will skip to next part. The backend must return
- TRUE for all text/xxx and message/rfc822 content types.
-
- The content_type contains a valid parsed "type/subtype" string. For messages
- without (valid) Content-Type header, the content_type is set to "text/plain".
- The content_disposition is passed without parsing/validation if it exists,
- otherwise it's NULL. */
-bool fts_backend_build_body_begin(struct fts_backend_build_context *ctx,
- uint32_t uid, const char *content_type,
- const char *content_disposition);
-/* Called once when the whole body part has been sent. */
-void fts_backend_build_body_end(struct fts_backend_build_context *ctx);
-/* Add more content to the index for the currently selected header/body part.
- The data must contain only full valid UTF-8 characters, but it doesn't need
- to be NUL-terminated. size contains the data size in bytes, not characters.
- This function may be called many times and the data block sizes may be
- small. Backend returns 0 if ok, -1 if build should be aborted. */
-int fts_backend_build_more(struct fts_backend_build_context *ctx,
- const unsigned char *data, size_t size);
-/* Finish adding new data to the index. */
-int fts_backend_build_deinit(struct fts_backend_build_context **ctx);
-
-/* Returns TRUE if there exists a build context. */
-bool fts_backend_is_building(struct fts_backend *backend);
-
-/* Expunge given mail from the backend. Note that the transaction may still
- fail later, so backend shouldn't do anything irreversible. */
-void fts_backend_expunge(struct fts_backend *backend, struct mail *mail);
-/* Called after transaction has been committed or rollbacked. */
-void fts_backend_expunge_finish(struct fts_backend *backend,
- struct mailbox *box, bool committed);
-
-/* Refresh database to make sure we see latest changes from lookups.
+int fts_backend_get_last_uid(struct fts_backend *backend, struct mailbox *box,
+ uint32_t *last_uid_r);
+
+/* Returns TRUE if there exists an update context. */
+bool fts_backend_is_updating(struct fts_backend *backend);
+
+/* Start an index update. */
+struct fts_backend_update_context *
+fts_backend_update_init(struct fts_backend *backend);
+/* Finish an index update. Returns 0 if ok, -1 if some updates failed.
+ If updates failed, the index is in unspecified state. */
+int fts_backend_update_deinit(struct fts_backend_update_context **ctx);
+
+/* Switch to updating the specified mailbox. box may also be set to NULL to
+ make sure the previous mailbox won't tried to be accessed anymore. */
+void fts_backend_update_set_mailbox(struct fts_backend_update_context *ctx,
+ struct mailbox *box);
+/* Expunge the specified mail. */
+void fts_backend_update_expunge(struct fts_backend_update_context *ctx,
+ uint32_t uid);
+
+/* Switch to building index for specified key. If backend doesn't want to
+ index this key, it can return FALSE and caller will skip to next key. */
+bool fts_backend_update_set_build_key(struct fts_backend_update_context *ctx,
+ const struct fts_backend_build_key *key);
+/* Make sure that if _build_more() is called, we'll assert-crash. */
+void fts_backend_update_unset_build_key(struct fts_backend_update_context *ctx);
+/* Add more content to the index for the currently specified build key.
+ Non-BODY_PART_BINARY data must contain only full valid UTF-8 characters,
+ but it doesn't need to be NUL-terminated. size contains the data size in
+ bytes, not characters. This function may be called many times and the data
+ block sizes may be small. Backend returns 0 if ok, -1 if build should be
+ aborted. */
+int fts_backend_update_build_more(struct fts_backend_update_context *ctx,
+ const unsigned char *data, size_t size);
+
+/* Refresh index to make sure we see latest changes from lookups.
Returns 0 if ok, -1 if error. */
int fts_backend_refresh(struct fts_backend *backend);
-
-/* Start building a FTS lookup. */
-struct fts_backend_lookup_context *
-fts_backend_lookup_init(struct fts_backend *backend);
-/* Add a new search key to the lookup. The keys are ANDed together. */
-void fts_backend_lookup_add(struct fts_backend_lookup_context *ctx,
- const char *key, enum fts_lookup_flags flags);
-/* Finish the lookup and return found UIDs. The definite_uids are returned
- to client directly, while for maybe_uids Dovecot first verifies (by
- opening and reading the mail) that they really do contain the searched
- keys. The maybe_uids is useful with backends that can only filter out
- messages, but can't definitively say if the search matched a message. */
-int fts_backend_lookup_deinit(struct fts_backend_lookup_context **ctx,
- ARRAY_TYPE(seq_range) *definite_uids,
- ARRAY_TYPE(seq_range) *maybe_uids,
- ARRAY_TYPE(fts_score_map) *scores);
+/* Optimize the index. This can be a somewhat heavy operation. */
+int fts_backend_optimize(struct fts_backend *backend);
+
+/* Returns TRUE if fts_backend_lookup() should even be tried for the
+ given args. */
+bool fts_backend_can_lookup(struct fts_backend *backend,
+ const struct mail_search_arg *args);
+/* Do a FTS lookup for the given search args. Backends can support different
+ kinds of search arguments, so match_always=TRUE must be set to all search
+ args that were actually used to produce the search results. The other args
+ are handled by the regular search code. The backends MUST ignore all args
+ that have subargs (SEARCH_OR, SEARCH_SUB), since they are looked up
+ separately.
+
+ and_args specifies if the args should be ANDed or ORed together.
+
+ The arrays in result must be initialized by caller. */
+int fts_backend_lookup(struct fts_backend *backend, struct mailbox *box,
+ struct mail_search_arg *args, bool and_args,
+ struct fts_result *result);
+
+/* Search from multiple mailboxes. result->pool must be initialized. */
+int fts_backend_lookup_multi(struct fts_backend *backend,
+ struct mailbox *const boxes[],
+ struct mail_search_arg *args, bool and_args,
+ struct fts_multi_result *result);
#endif
--- /dev/null
+/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "network.h"
+#include "istream.h"
+#include "write-full.h"
+#include "strescape.h"
+#include "mail-user.h"
+#include "mail-storage-private.h"
+#include "fts-api-private.h"
+#include "fts-build-private.h"
+
+#define INDEXER_SOCKET_NAME "indexer"
+#define INDEXER_WAIT_MSECS 250
+#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
+
+struct indexer_fts_storage_build_context {
+ struct fts_storage_build_context ctx;
+
+ char *path;
+ int fd;
+ struct istream *input;
+};
+
+static int
+fts_build_indexer_init(struct fts_backend *backend, struct mailbox *box,
+ struct fts_storage_build_context **build_ctx_r)
+{
+ struct indexer_fts_storage_build_context *ctx;
+ struct mailbox_status status;
+ uint32_t last_uid, seq1, seq2;
+ const char *path, *cmd;
+ int fd;
+
+ if (fts_backend_get_last_uid(backend, box, &last_uid) < 0)
+ return -1;
+
+ mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
+ if (status.uidnext == last_uid+1) {
+ /* everything is already indexed */
+ return 0;
+ }
+
+ mailbox_get_seq_range(box, last_uid+1, (uint32_t)-1, &seq1, &seq2);
+ if (seq1 == 0) {
+ /* no new messages (last messages in mailbox were expunged) */
+ return 0;
+ }
+
+ path = t_strconcat(box->storage->user->set->base_dir,
+ "/"INDEXER_SOCKET_NAME, NULL);
+ fd = net_connect_unix_with_retries(path, 1000);
+ if (fd == -1) {
+ i_error("net_connect_unix(%s) failed: %m", path);
+ return -1;
+ }
+
+ cmd = t_strdup_printf(INDEXER_HANDSHAKE"PREPEND\t1\t%s\t%s\n",
+ str_tabescape(box->storage->user->username),
+ str_tabescape(box->vname));
+ if (write_full(fd, cmd, strlen(cmd)) < 0) {
+ i_error("write(%s) failed: %m", path);
+ (void)close(fd);
+ return -1;
+ }
+
+ /* connect to indexer and request immediate indexing of the mailbox */
+ ctx = i_new(struct indexer_fts_storage_build_context, 1);
+ ctx->ctx.mail_count = 100;
+ ctx->path = i_strdup(path);
+ ctx->fd = fd;
+ ctx->input = i_stream_create_fd(fd, 128, FALSE);
+
+ *build_ctx_r = &ctx->ctx;
+ return 1;
+}
+
+static int
+fts_build_indexer_deinit(struct fts_storage_build_context *_ctx)
+{
+ struct indexer_fts_storage_build_context *ctx =
+ (struct indexer_fts_storage_build_context *)_ctx;
+
+ i_stream_destroy(&ctx->input);
+ if (close(ctx->fd) < 0)
+ i_error("close(%s) failed: %m", ctx->path);
+ i_free(ctx->path);
+ return 0;
+}
+
+static int
+fts_build_indexer_input(struct indexer_fts_storage_build_context *ctx)
+{
+ const char *line;
+ int percentage;
+
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ /* initial reply: <tag> \t OK
+ following: <tag> \t <percentage> */
+ if (strncmp(line, "1\t", 2) != 0) {
+ i_error("indexer sent invalid reply: %s", line);
+ return -1;
+ }
+ line += 2;
+ if (strcmp(line, "OK") == 0)
+ continue;
+ if (str_to_int(line, &percentage) < 0 || percentage > 100) {
+ i_error("indexer sent invalid percentage: %s", line);
+ return -1;
+ }
+ if (percentage < 0) {
+ /* indexing failed */
+ i_error("indexer failed to index mailbox %s",
+ ctx->ctx.box->vname);
+ return -1;
+ }
+ ctx->ctx.mail_idx = percentage;
+ if (percentage == 100) {
+ /* finished */
+ return 1;
+ }
+ }
+ if (ctx->input->eof || ctx->input->stream_errno != 0) {
+ i_error("indexer disconnected unexpectedly");
+ return -1;
+ }
+ return 0;
+}
+
+static int fts_build_indexer_more(struct fts_storage_build_context *_ctx)
+{
+ struct indexer_fts_storage_build_context *ctx =
+ (struct indexer_fts_storage_build_context *)_ctx;
+ struct ioloop *ioloop;
+ struct io *io;
+ struct timeout *to;
+ int ret;
+
+ if ((ret = fts_build_indexer_input(ctx)) != 0)
+ return ret;
+
+ /* wait for a while for the reply. FIXME: once search API supports
+ asynchronous waits, get rid of this wait and use the mail IO loop */
+ ioloop = io_loop_create();
+ io = io_add(ctx->fd, IO_READ, io_loop_stop, ioloop);
+ to = timeout_add(INDEXER_WAIT_MSECS, io_loop_stop, ioloop);
+ io_loop_run(ioloop);
+ io_remove(&io);
+ timeout_remove(&to);
+ io_loop_destroy(&ioloop);
+
+ return fts_build_indexer_input(ctx);
+}
+
+const struct fts_storage_build_vfuncs fts_storage_build_indexer_vfuncs = {
+ fts_build_indexer_init,
+ fts_build_indexer_deinit,
+ fts_build_indexer_more
+};
--- /dev/null
+/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "mail-storage-private.h"
+#include "mail-search-build.h"
+#include "fts-api-private.h"
+#include "fts-build-private.h"
+
+#define FTS_SEARCH_NONBLOCK_COUNT 50
+
+static int
+fts_build_mailbox_init(struct fts_backend *backend, struct mailbox *box,
+ struct fts_storage_build_context **build_ctx_r)
+{
+ struct fts_storage_build_context *ctx;
+ struct mail_search_args *search_args;
+ struct fts_backend_update_context *update_ctx;
+ struct mailbox_status status;
+ uint32_t last_uid, seq1, seq2;
+
+ if (fts_backend_get_last_uid(backend, box, &last_uid) < 0)
+ return -1;
+
+ mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
+ if (status.uidnext == last_uid+1) {
+ /* everything is already indexed */
+ return 0;
+ }
+
+ mailbox_get_seq_range(box, last_uid+1, (uint32_t)-1, &seq1, &seq2);
+ if (seq1 == 0) {
+ /* no new messages (last messages in mailbox were expunged) */
+ return 0;
+ }
+
+ update_ctx = fts_backend_update_init(backend);
+ fts_backend_update_set_mailbox(update_ctx, box);
+
+ search_args = mail_search_build_init();
+ mail_search_build_add_seqset(search_args, seq1, seq2);
+
+ ctx = i_new(struct fts_storage_build_context, 1);
+ ctx->update_ctx = update_ctx;
+ ctx->mail_count = seq2 - seq1 + 1;
+
+ ctx->trans = mailbox_transaction_begin(box, 0);
+ ctx->search_ctx = mailbox_search_init(ctx->trans, search_args,
+ NULL, 0, NULL);
+ ctx->search_ctx->progress_hidden = TRUE;
+ mail_search_args_unref(&search_args);
+
+ *build_ctx_r = ctx;
+ return 1;
+}
+
+static int fts_build_mailbox_deinit(struct fts_storage_build_context *ctx)
+{
+ int ret;
+
+ ret = mailbox_search_deinit(&ctx->search_ctx);
+ (void)mailbox_transaction_commit(&ctx->trans);
+ return ret;
+}
+
+static int fts_build_mailbox_more(struct fts_storage_build_context *ctx)
+{
+ struct mail *mail = NULL;
+ unsigned int count = 0;
+ int ret;
+
+ while (mailbox_search_next(ctx->search_ctx, &mail)) {
+ T_BEGIN {
+ ret = fts_build_mail(ctx, mail);
+ } T_END;
+
+ if (ret < 0)
+ return -1;
+
+ ctx->mail_idx++;
+ if (++count == FTS_SEARCH_NONBLOCK_COUNT)
+ return 0;
+ }
+ return 1;
+}
+
+const struct fts_storage_build_vfuncs fts_storage_build_mailbox_vfuncs = {
+ fts_build_mailbox_init,
+ fts_build_mailbox_deinit,
+ fts_build_mailbox_more
+};
--- /dev/null
+#ifndef FTS_BUILD_PRIVATE_H
+#define FTS_BUILD_PRIVATE_H
+
+#include "fts-build.h"
+
+struct fts_storage_build_context;
+
+#define FTS_SEARCH_NONBLOCK_COUNT 50
+
+struct fts_storage_build_vfuncs {
+ int (*init)(struct fts_backend *backend, struct mailbox *box,
+ struct fts_storage_build_context **build_ctx_r);
+ int (*deinit)(struct fts_storage_build_context *ctx);
+ int (*more)(struct fts_storage_build_context *ctx);
+};
+
+struct fts_storage_build_context {
+ struct mailbox *box;
+ struct fts_backend_update_context *update_ctx;
+ struct fts_storage_build_vfuncs v;
+
+ struct timeval search_start_time, last_notify;
+ unsigned int mail_idx, mail_count;
+
+ struct mailbox_transaction_context *trans;
+ struct mail_search_context *search_ctx;
+
+ uint32_t uid;
+ char *content_type, *content_disposition;
+
+ unsigned int binary_mime_parts:1;
+ unsigned int notified:1;
+ unsigned int failed:1;
+};
+
+extern const struct fts_storage_build_vfuncs fts_storage_build_mailbox_vfuncs;
+extern const struct fts_storage_build_vfuncs fts_storage_build_virtual_vfuncs;
+extern const struct fts_storage_build_vfuncs fts_storage_build_indexer_vfuncs;
+
+int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail);
+
+#endif
--- /dev/null
+/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "mail-storage-private.h"
+#include "mail-search-build.h"
+#include "../virtual/virtual-storage.h"
+#include "fts-api-private.h"
+#include "fts-storage.h"
+#include "fts-build-private.h"
+
+#define FTS_SEARCH_NONBLOCK_COUNT 50
+
+struct virtual_fts_storage_build_context {
+ struct fts_storage_build_context ctx;
+
+ struct fts_backend *update_backend;
+ uint32_t virtual_last_uid;
+
+ ARRAY_TYPE(mailboxes) mailboxes;
+ unsigned int mailbox_idx;
+};
+
+static int
+fts_mailbox_get_seqs(struct mailbox *box, uint32_t *seq1_r, uint32_t *seq2_r)
+{
+ struct mailbox_status status;
+ uint32_t last_uid;
+
+ if (fts_backend_get_last_uid(fts_mailbox_backend(box),
+ box, &last_uid) < 0)
+ return -1;
+
+ mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
+ if (status.uidnext <= last_uid+1)
+ *seq1_r = *seq2_r = 0;
+ else {
+ mailbox_get_seq_range(box, last_uid+1, (uint32_t)-1,
+ seq1_r, seq2_r);
+ }
+ return 0;
+}
+
+static int
+fts_build_virtual_mailboxes_get(struct virtual_fts_storage_build_context *ctx)
+{
+ struct virtual_mailbox *vbox =
+ (struct virtual_mailbox *)ctx->ctx.box;
+ struct mailbox *const *boxp;
+ ARRAY_TYPE(mailboxes) all_mailboxes;
+ uint32_t seq1, seq2;
+
+ t_array_init(&all_mailboxes, 64);
+ i_array_init(&ctx->mailboxes, 64);
+ vbox->vfuncs.get_virtual_backend_boxes(ctx->ctx.box,
+ &all_mailboxes, TRUE);
+
+ array_foreach(&all_mailboxes, boxp) {
+ if (fts_mailbox_get_seqs(*boxp, &seq1, &seq2) < 0) {
+ array_free(&ctx->mailboxes);
+ return -1;
+ }
+ if (seq1 != 0) {
+ ctx->ctx.mail_count += seq2 - seq1 + 1;
+ array_append(&ctx->mailboxes, boxp, 1);
+ }
+ }
+ return 0;
+}
+
+static void
+fts_build_virtual_mailbox_close(struct virtual_fts_storage_build_context *ctx)
+{
+ if (mailbox_search_deinit(&ctx->ctx.search_ctx) < 0)
+ ctx->ctx.failed = TRUE;
+ (void)mailbox_transaction_commit(&ctx->ctx.trans);
+}
+
+static bool
+fts_build_virtual_mailbox_next(struct virtual_fts_storage_build_context *ctx)
+{
+ struct mail_search_args *search_args;
+ struct mailbox *const *boxes, *box;
+ struct fts_backend *backend;
+ unsigned int count;
+ uint32_t seq1, seq2;
+
+ boxes = array_get(&ctx->mailboxes, &count);
+ if (ctx->mailbox_idx == count)
+ return FALSE;
+ box = boxes[ctx->mailbox_idx++];
+
+ if (ctx->ctx.trans != NULL)
+ fts_build_virtual_mailbox_close(ctx);
+
+ if (fts_mailbox_get_seqs(box, &seq1, &seq2) < 0) {
+ ctx->ctx.failed = TRUE;
+ return fts_build_virtual_mailbox_next(ctx);
+ }
+
+ backend = fts_mailbox_backend(box);
+ if (ctx->update_backend != backend) {
+ if (ctx->ctx.update_ctx != NULL) {
+ if (fts_backend_update_deinit(&ctx->ctx.update_ctx) < 0)
+ ctx->ctx.failed = TRUE;
+ }
+ ctx->update_backend = backend;
+ ctx->ctx.update_ctx = fts_backend_update_init(backend);
+ }
+
+
+ fts_backend_update_set_mailbox(ctx->ctx.update_ctx, box);
+ search_args = mail_search_build_init();
+ mail_search_build_add_seqset(search_args, seq1, seq2);
+
+ ctx->ctx.trans = mailbox_transaction_begin(box, 0);
+ ctx->ctx.search_ctx = mailbox_search_init(ctx->ctx.trans, search_args,
+ NULL, 0, NULL);
+ ctx->ctx.search_ctx->progress_hidden = TRUE;
+ mail_search_args_unref(&search_args);
+ return TRUE;
+}
+
+static int
+fts_build_virtual_init(struct fts_backend *backend, struct mailbox *box,
+ struct fts_storage_build_context **build_ctx_r)
+{
+ struct virtual_fts_storage_build_context *ctx;
+ struct mailbox_status status;
+ uint32_t last_uid;
+
+ /* first do a quick check: is the virtual mailbox's last indexed
+ UID up to date? */
+ if (fts_backend_get_last_uid(backend, box, &last_uid) < 0)
+ return -1;
+
+ mailbox_get_open_status(box, STATUS_UIDNEXT, &status);
+ if (status.uidnext == last_uid+1)
+ return 0;
+
+ /* nope. we'll need to go through its mailboxes and check their
+ indexes. FIXME: we could optimize by going through only those
+ mailboxes that exist in >last_uid mails */
+ ctx = i_new(struct virtual_fts_storage_build_context, 1);
+ ctx->ctx.box = box;
+ ctx->virtual_last_uid = status.uidnext - 1;
+
+ if (fts_build_virtual_mailboxes_get(ctx) < 0) {
+ i_free(ctx);
+ return -1;
+ }
+ fts_build_virtual_mailbox_next(ctx);
+
+ *build_ctx_r = &ctx->ctx;
+ return 1;
+}
+
+static int fts_build_virtual_deinit(struct fts_storage_build_context *_ctx)
+{
+ struct virtual_fts_storage_build_context *ctx =
+ (struct virtual_fts_storage_build_context *)_ctx;
+
+ if (!_ctx->failed) {
+ (void)fts_index_set_last_uid(ctx->ctx.box,
+ ctx->virtual_last_uid);
+ }
+
+ fts_build_virtual_mailbox_close(ctx);
+ array_free(&ctx->mailboxes);
+ return 0;
+}
+
+static int fts_build_virtual_more(struct fts_storage_build_context *_ctx)
+{
+ struct virtual_fts_storage_build_context *ctx =
+ (struct virtual_fts_storage_build_context *)_ctx;
+ struct mail *mail;
+ unsigned int count = 0;
+ int ret;
+
+ while (mailbox_search_next(_ctx->search_ctx, &mail)) {
+ T_BEGIN {
+ ret = fts_build_mail(_ctx, mail);
+ } T_END;
+
+ if (ret < 0)
+ return -1;
+
+ _ctx->mail_idx++;
+ if (++count == FTS_SEARCH_NONBLOCK_COUNT)
+ return 0;
+ }
+
+ if (fts_build_virtual_mailbox_next(ctx))
+ return fts_build_virtual_more(_ctx);
+ return 1;
+}
+
+const struct fts_storage_build_vfuncs fts_storage_build_virtual_vfuncs = {
+ fts_build_virtual_init,
+ fts_build_virtual_deinit,
+ fts_build_virtual_more
+};
--- /dev/null
+/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "str.h"
+#include "time-util.h"
+#include "rfc822-parser.h"
+#include "message-address.h"
+#include "message-parser.h"
+#include "message-decoder.h"
+#include "../virtual/virtual-storage.h"
+#include "fts-api-private.h"
+#include "fts-build-private.h"
+
+#define FTS_BUILD_NOTIFY_INTERVAL_SECS 10
+
+static void fts_build_parse_content_type(struct fts_storage_build_context *ctx,
+ const struct message_header_line *hdr)
+{
+ struct rfc822_parser_context parser;
+ string_t *content_type;
+
+ rfc822_parser_init(&parser, hdr->full_value, hdr->full_value_len, NULL);
+ (void)rfc822_skip_lwsp(&parser);
+
+ T_BEGIN {
+ content_type = t_str_new(64);
+ if (rfc822_parse_content_type(&parser, content_type) >= 0) {
+ i_free(ctx->content_type);
+ ctx->content_type = i_strdup(str_c(content_type));
+ }
+ } T_END;
+}
+
+static void
+fts_build_parse_content_disposition(struct fts_storage_build_context *ctx,
+ const struct message_header_line *hdr)
+{
+ /* just pass it as-is to backend. */
+ i_free(ctx->content_disposition);
+ ctx->content_disposition =
+ i_strndup(hdr->full_value, hdr->full_value_len);
+}
+
+static void fts_parse_mail_header(struct fts_storage_build_context *ctx,
+ const struct message_block *raw_block)
+{
+ const struct message_header_line *hdr = raw_block->hdr;
+
+ if (strcasecmp(hdr->name, "Content-Type") == 0)
+ fts_build_parse_content_type(ctx, hdr);
+ else if (strcasecmp(hdr->name, "Content-Disposition") == 0)
+ fts_build_parse_content_disposition(ctx, hdr);
+}
+
+static void fts_build_mail_header(struct fts_storage_build_context *ctx,
+ const struct message_block *block)
+{
+ const struct message_header_line *hdr = block->hdr;
+ struct fts_backend_build_key key;
+
+ if (hdr->eoh)
+ return;
+
+ /* hdr->full_value is always set because we get the block from
+ message_decoder */
+ memset(&key, 0, sizeof(key));
+ key.uid = ctx->uid;
+ key.type = block->part->physical_pos == 0 ?
+ FTS_BACKEND_BUILD_KEY_HDR : FTS_BACKEND_BUILD_KEY_MIME_HDR;
+ key.hdr_name = hdr->name;
+
+ if (!fts_backend_update_set_build_key(ctx->update_ctx, &key))
+ return;
+
+ if (!message_header_is_address(hdr->name)) {
+ /* regular unstructured header */
+ (void)fts_backend_update_build_more(ctx->update_ctx,
+ hdr->full_value,
+ hdr->full_value_len);
+ } else T_BEGIN {
+ /* message address. normalize it to give better
+ search results. */
+ struct message_address *addr;
+ string_t *str;
+
+ addr = message_address_parse(pool_datastack_create(),
+ hdr->full_value,
+ hdr->full_value_len,
+ -1U, TRUE);
+ str = t_str_new(hdr->full_value_len);
+ message_address_write(str, addr);
+
+ (void)fts_backend_update_build_more(ctx->update_ctx,
+ str_data(str),
+ str_len(str));
+ } T_END;
+}
+
+static bool fts_build_body_begin(struct fts_storage_build_context *ctx)
+{
+ const char *content_type;
+ struct fts_backend_build_key key;
+
+ memset(&key, 0, sizeof(key));
+ key.uid = ctx->uid;
+
+ content_type = ctx->content_type != NULL ?
+ ctx->content_type : "text/plain";
+ if (strncmp(content_type, "text/", 5) == 0 ||
+ strncmp(content_type, "message/", 8) == 0) {
+ /* text body parts */
+ key.type = FTS_BACKEND_BUILD_KEY_BODY_PART;
+ } else {
+ /* possibly binary */
+ if (!ctx->binary_mime_parts)
+ return FALSE;
+ key.type = FTS_BACKEND_BUILD_KEY_BODY_PART_BINARY;
+ }
+ key.body_content_type = content_type;
+ key.body_content_disposition = ctx->content_disposition;
+ return fts_backend_update_set_build_key(ctx->update_ctx, &key);
+}
+
+int fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
+{
+ enum message_decoder_flags decoder_flags = MESSAGE_DECODER_FLAG_DTCASE;
+ struct istream *input;
+ struct message_parser_ctx *parser;
+ struct message_decoder_context *decoder;
+ struct message_block raw_block, block;
+ struct message_part *prev_part, *parts;
+ bool skip_body = FALSE, body_part = FALSE;
+ int ret;
+
+ ctx->uid = mail->uid;
+
+ if (mail_get_stream(mail, NULL, NULL, &input) < 0)
+ return -1;
+
+ prev_part = NULL;
+ parser = message_parser_init(pool_datastack_create(), input,
+ MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE,
+ 0);
+
+
+ if (ctx->binary_mime_parts)
+ decoder_flags |= MESSAGE_DECODER_FLAG_RETURN_BINARY;
+ decoder = message_decoder_init(decoder_flags);
+ for (;;) {
+ ret = message_parser_parse_next_block(parser, &raw_block);
+ i_assert(ret != 0);
+ if (ret < 0) {
+ if (input->stream_errno == 0)
+ ret = 0;
+ break;
+ }
+
+ if (raw_block.part != prev_part) {
+ /* body part changed. we're now parsing the end of
+ boundary, possibly followed by message epilogue */
+ fts_backend_update_unset_build_key(ctx->update_ctx);
+ prev_part = raw_block.part;
+ i_free_and_null(ctx->content_type);
+ i_free_and_null(ctx->content_disposition);
+
+ if (raw_block.size != 0) {
+ /* multipart. skip until beginning of next
+ part's headers */
+ skip_body = TRUE;
+ }
+ }
+
+ if (raw_block.hdr != NULL) {
+ /* always handle headers */
+ } else if (raw_block.size == 0) {
+ /* end of headers */
+ skip_body = !fts_build_body_begin(ctx);
+ body_part = TRUE;
+ } else {
+ if (skip_body)
+ continue;
+ }
+
+ if (!message_decoder_decode_next_block(decoder, &raw_block,
+ &block))
+ continue;
+
+ if (block.hdr != NULL) {
+ fts_parse_mail_header(ctx, &raw_block);
+ fts_build_mail_header(ctx, &block);
+ } else if (block.size == 0) {
+ /* end of headers */
+ } else {
+ i_assert(body_part);
+ if (fts_backend_update_build_more(ctx->update_ctx,
+ block.data,
+ block.size) < 0) {
+ ret = -1;
+ break;
+ }
+ }
+ }
+ if (message_parser_deinit(&parser, &parts) < 0)
+ mail_set_cache_corrupted(mail, MAIL_FETCH_MESSAGE_PARTS);
+ message_decoder_deinit(&decoder);
+ return ret;
+}
+
+static void fts_build_notify(struct fts_storage_build_context *ctx)
+{
+ double completed_frac;
+ unsigned int eta_secs;
+
+ if (ioloop_time - ctx->last_notify.tv_sec < FTS_BUILD_NOTIFY_INTERVAL_SECS)
+ return;
+ ctx->last_notify = ioloop_timeval;
+
+ if (ctx->box->storage->callbacks.notify_ok == NULL ||
+ ctx->mail_idx == 0)
+ return;
+
+ /* mail_count is counted before indexing actually begins.
+ by the time the mailbox is actually indexed it may already
+ have more (or less) mails. so mail_idx can be higher than
+ mail_count. */
+ completed_frac = ctx->mail_idx >= ctx->mail_count ? 1 :
+ (double)ctx->mail_idx / ctx->mail_count;
+
+ if (completed_frac >= 0.000001) {
+ unsigned int elapsed_msecs, est_total_msecs;
+
+ elapsed_msecs = timeval_diff_msecs(&ioloop_timeval,
+ &ctx->search_start_time);
+ est_total_msecs = elapsed_msecs / completed_frac;
+ eta_secs = (est_total_msecs - elapsed_msecs) / 1000;
+ } else {
+ eta_secs = 0;
+ }
+
+ T_BEGIN {
+ const char *text;
+
+ text = t_strdup_printf("Indexed %d%% of the mailbox, "
+ "ETA %d:%02d", (int)(completed_frac * 100.0),
+ eta_secs/60, eta_secs%60);
+ ctx->box->storage->callbacks.
+ notify_ok(ctx->box, text,
+ ctx->box->storage->callback_context);
+ ctx->notified = TRUE;
+ } T_END;
+}
+
+int fts_build_init(struct fts_backend *backend, struct mailbox *box,
+ bool precache,
+ struct fts_storage_build_context **build_ctx_r)
+{
+ const struct fts_storage_build_vfuncs *v;
+ int ret;
+
+ *build_ctx_r = NULL;
+
+ /* unless we're precaching (i.e. indexer service, doveadm index)
+ use the indexer service */
+ if (!precache)
+ v = &fts_storage_build_indexer_vfuncs;
+ else if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) == 0)
+ v = &fts_storage_build_virtual_vfuncs;
+ else
+ v = &fts_storage_build_mailbox_vfuncs;
+
+ if ((ret = v->init(backend, box, build_ctx_r)) <= 0)
+ return ret;
+
+ (*build_ctx_r)->box = box;
+ (*build_ctx_r)->v = *v;
+ (*build_ctx_r)->binary_mime_parts =
+ (backend->flags & FTS_BACKEND_FLAG_BINARY_MIME_PARTS) != 0;
+ return 1;
+}
+
+int fts_build_deinit(struct fts_storage_build_context **_ctx)
+{
+ struct fts_storage_build_context *ctx = *_ctx;
+ int ret = ctx->failed ? -1 : 0;
+
+ *_ctx = NULL;
+
+ if (ctx->v.deinit(ctx) < 0)
+ ret = -1;
+ if (ctx->update_ctx != NULL) {
+ if (fts_backend_update_deinit(&ctx->update_ctx) < 0)
+ ret = -1;
+ }
+ if (ctx->notified) {
+ /* we notified at least once */
+ ctx->box->storage->callbacks.
+ notify_ok(ctx->box, "Mailbox indexing finished",
+ ctx->box->storage->callback_context);
+ }
+ i_free(ctx->content_type);
+ i_free(ctx->content_disposition);
+ i_free(ctx);
+ return ret;
+}
+
+int fts_build_more(struct fts_storage_build_context *ctx)
+{
+ int ret;
+
+ if ((ret = ctx->v.more(ctx)) < 0) {
+ ctx->failed = TRUE;
+ return -1;
+ }
+
+ fts_build_notify(ctx);
+ return ret;
+}
--- /dev/null
+#ifndef FTS_BUILD_H
+#define FTS_BUILD_H
+
+struct fts_storage_build_context;
+
+/* Initialize building. Returns 1 if we need to build (build_ctx set),
+ 0 if not (build_ctx NULL) or -1 if error. */
+int fts_build_init(struct fts_backend *backend, struct mailbox *box,
+ bool precache,
+ struct fts_storage_build_context **build_ctx_r);
+/* Returns 0 if ok, -1 if error. */
+int fts_build_deinit(struct fts_storage_build_context **ctx);
+
+/* Build more. Returns 1 if finished, 0 if this function needs to be called
+ again, -1 if error. */
+int fts_build_more(struct fts_storage_build_context *ctx);
+
+#endif
+++ /dev/null
-/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
-
-#include "lib.h"
-#include "mail-storage.h"
-#include "fts-mailbox.h"
-#include "../virtual/virtual-storage.h"
-
-bool fts_mailbox_get_virtual_uid(struct mailbox *box,
- const char *backend_mailbox,
- uint32_t backend_uidvalidity,
- uint32_t backend_uid, uint32_t *uid_r)
-{
- struct virtual_mailbox *vbox;
-
- if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) != 0)
- return FALSE;
-
- vbox = (struct virtual_mailbox *)box;
- return vbox->vfuncs.get_virtual_uid(box, backend_mailbox,
- backend_uidvalidity,
- backend_uid, uid_r);
-}
-
-void fts_mailbox_get_virtual_backend_boxes(struct mailbox *box,
- ARRAY_TYPE(mailboxes) *mailboxes,
- bool only_with_msgs)
-{
- struct virtual_mailbox *vbox;
-
- if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) != 0)
- array_append(mailboxes, &box, 1);
- else {
- vbox = (struct virtual_mailbox *)box;
- vbox->vfuncs.get_virtual_backend_boxes(box, mailboxes,
- only_with_msgs);
- }
-}
-
-void fts_mailbox_get_virtual_box_patterns(struct mailbox *box,
- ARRAY_TYPE(mailbox_virtual_patterns) *includes,
- ARRAY_TYPE(mailbox_virtual_patterns) *excludes)
-{
- struct virtual_mailbox *vbox;
-
- if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) != 0) {
- struct mailbox_virtual_pattern pat;
-
- memset(&pat, 0, sizeof(pat));
- pat.ns = mailbox_list_get_namespace(box->list);
- pat.pattern = box->name;
- array_append(includes, &pat, 1);
- } else {
- vbox = (struct virtual_mailbox *)box;
- vbox->vfuncs.get_virtual_box_patterns(box, includes, excludes);
- }
-}
+++ /dev/null
-#ifndef FTS_MAILBOX_H
-#define FTS_MAILBOX_H
-
-/* If box is a virtual mailbox, look up UID for the given backend message.
- Returns TRUE if found, FALSE if not. */
-bool fts_mailbox_get_virtual_uid(struct mailbox *box,
- const char *backend_mailbox,
- uint32_t backend_uidvalidity,
- uint32_t backend_uid, uint32_t *uid_r);
-/* If box is a virtual mailbox, return all backend mailboxes. If
- only_with_msgs=TRUE, return only those mailboxes that have at least one
- message existing in the virtual mailbox. */
-void fts_mailbox_get_virtual_backend_boxes(struct mailbox *box,
- ARRAY_TYPE(mailboxes) *mailboxes,
- bool only_with_msgs);
-/* If mailbox is a virtual mailbox, return all mailbox list patterns that
- are used to figure out which mailboxes belong to the virtual mailbox. */
-void fts_mailbox_get_virtual_box_patterns(struct mailbox *box,
- ARRAY_TYPE(mailbox_virtual_patterns) *includes,
- ARRAY_TYPE(mailbox_virtual_patterns) *excludes);
-
-#endif
#include "lib.h"
#include "mail-storage-hooks.h"
+#include "fts-storage.h"
#include "fts-plugin.h"
#include <stdlib.h>
const char *fts_plugin_version = DOVECOT_VERSION;
static struct mail_storage_hooks fts_mail_storage_hooks = {
+ .mailbox_list_created = fts_mailbox_list_created,
.mailbox_allocated = fts_mailbox_allocated,
.mail_allocated = fts_mail_allocated
};
#ifndef FTS_PLUGIN_H
#define FTS_PLUGIN_H
-void fts_mailbox_allocated(struct mailbox *box);
-void fts_mail_allocated(struct mail *mail);
-
void fts_plugin_init(struct module *module);
void fts_plugin_deinit(void);
--- /dev/null
+/* Copyright (c) 2011 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "mail-search.h"
+#include "fts-search-serialize.h"
+
+#define HAVE_SUBARGS(arg) \
+ ((arg)->type == SEARCH_SUB || (arg)->type == SEARCH_OR)
+
+void fts_search_serialize(buffer_t *buf, const struct mail_search_arg *args)
+{
+ char chr;
+
+ for (; args != NULL; args = args->next) {
+ chr = (args->match_always ? 1 : 0) |
+ (args->nonmatch_always ? 2 : 0);
+ buffer_append_c(buf, chr);
+
+ if (HAVE_SUBARGS(args))
+ fts_search_serialize(buf, args->value.subargs);
+ }
+}
+
+static void fts_search_deserialize_idx(struct mail_search_arg *args,
+ const buffer_t *buf, unsigned int *idx)
+{
+ const char *data = buf->data;
+
+ for (; args != NULL; args = args->next) {
+ i_assert(*idx < buf->used);
+
+ args->match_always = (data[*idx] & 1) != 0;
+ args->nonmatch_always = (data[*idx] & 2) != 0;
+ args->result = args->match_always ? 1 :
+ (args->nonmatch_always ? 0 : -1);
+ *idx += 1;
+
+ if (HAVE_SUBARGS(args)) {
+ fts_search_deserialize_idx(args->value.subargs,
+ buf, idx);
+ }
+ }
+}
+
+void fts_search_deserialize(struct mail_search_arg *args,
+ const buffer_t *buf)
+{
+ unsigned int idx = 0;
+
+ fts_search_deserialize_idx(args, buf, &idx);
+ i_assert(idx == buf->used);
+}
+
+static void
+fts_search_deserialize_add_idx(struct mail_search_arg *args,
+ const buffer_t *buf, unsigned int *idx,
+ bool matches)
+{
+ const char *data = buf->data;
+
+ for (; args != NULL; args = args->next) {
+ i_assert(*idx < buf->used);
+
+ if (data[*idx] != 0) {
+ if (matches) {
+ args->match_always = TRUE;
+ args->result = 1;
+ } else {
+ args->nonmatch_always = TRUE;
+ args->result = 0;
+ }
+ }
+ *idx += 1;
+
+ if (HAVE_SUBARGS(args)) {
+ fts_search_deserialize_add_idx(args->value.subargs,
+ buf, idx, matches);
+ }
+ }
+}
+
+void fts_search_deserialize_add_matches(struct mail_search_arg *args,
+ const buffer_t *buf)
+{
+ unsigned int idx = 0;
+
+ fts_search_deserialize_add_idx(args, buf, &idx, TRUE);
+ i_assert(idx == buf->used);
+}
+
+void fts_search_deserialize_add_nonmatches(struct mail_search_arg *args,
+ const buffer_t *buf)
+{
+ unsigned int idx = 0;
+
+ fts_search_deserialize_add_idx(args, buf, &idx, FALSE);
+ i_assert(idx == buf->used);
+}
--- /dev/null
+#ifndef FTS_SEARCH_SERIALIZE_H
+#define FTS_SEARCH_SERIALIZE_H
+
+/* serialize [non]match_always fields (clearing buffer) */
+void fts_search_serialize(buffer_t *buf, const struct mail_search_arg *args);
+/* add/remove [non]match_always fields in search args */
+void fts_search_deserialize(struct mail_search_arg *args,
+ const buffer_t *buf);
+/* add match_always=TRUE fields to search args */
+void fts_search_deserialize_add_matches(struct mail_search_arg *args,
+ const buffer_t *buf);
+/* add nonmatch_always=TRUE fields to search args */
+void fts_search_deserialize_add_nonmatches(struct mail_search_arg *args,
+ const buffer_t *buf);
+
+#endif
#include "str.h"
#include "seq-range-array.h"
#include "mail-search.h"
-#include "mail-storage-private.h"
+#include "../virtual/virtual-storage.h"
#include "fts-api-private.h"
+#include "fts-search-serialize.h"
#include "fts-storage.h"
static void
-uid_range_to_seqs(struct mailbox *box, const ARRAY_TYPE(seq_range) *uid_range,
+uid_range_to_seqs(struct fts_search_context *fctx,
+ const ARRAY_TYPE(seq_range) *uid_range,
ARRAY_TYPE(seq_range) *seq_range)
{
const struct seq_range *range;
uint32_t seq1, seq2;
range = array_get(uid_range, &count);
- i_array_init(seq_range, count);
+ if (!array_is_created(seq_range))
+ p_array_init(seq_range, fctx->result_pool, count);
for (i = 0; i < count; i++) {
- mailbox_get_seq_range(box, range[i].seq1, range[i].seq2,
+ mailbox_get_seq_range(fctx->box, range[i].seq1, range[i].seq2,
&seq1, &seq2);
if (seq1 != 0)
seq_range_array_add_range(seq_range, seq1, seq2);
}
}
-static void fts_uid_results_to_seq(struct fts_search_context *fctx)
+static int fts_search_lookup_level_single(struct fts_search_context *fctx,
+ struct mail_search_arg *args,
+ bool and_args)
{
- ARRAY_TYPE(seq_range) uid_range;
+ struct fts_search_level *level;
+ struct fts_result result;
- uid_range = fctx->definite_seqs;
- uid_range_to_seqs(fctx->t->box, &uid_range, &fctx->definite_seqs);
- array_free(&uid_range);
+ memset(&result, 0, sizeof(result));
+ p_array_init(&result.definite_uids, fctx->result_pool, 32);
+ p_array_init(&result.maybe_uids, fctx->result_pool, 32);
+ p_array_init(&result.scores, fctx->result_pool, 32);
- uid_range = fctx->maybe_seqs;
- uid_range_to_seqs(fctx->t->box, &uid_range, &fctx->maybe_seqs);
- array_free(&uid_range);
+ mail_search_args_reset(args, TRUE);
+ if (fts_backend_lookup(fctx->backend, fctx->box, args, and_args,
+ &result) < 0)
+ return -1;
+
+ level = array_append_space(&fctx->levels);
+ level->args_matches = buffer_create_dynamic(fctx->result_pool, 16);
+ fts_search_serialize(level->args_matches, args);
+
+ uid_range_to_seqs(fctx, &result.definite_uids, &level->definite_seqs);
+ uid_range_to_seqs(fctx, &result.maybe_uids, &level->maybe_seqs);
+ level->score_map = result.scores;
+ return 0;
}
-static int fts_search_lookup_arg(struct fts_search_context *fctx,
- struct mail_search_arg *arg)
+static void
+level_scores_add_vuids(struct virtual_mailbox *vbox,
+ struct fts_search_level *level, struct fts_result *br)
{
- enum fts_lookup_flags flags = 0;
- const char *key;
-
- switch (arg->type) {
- case SEARCH_HEADER:
- case SEARCH_HEADER_COMPRESS_LWSP:
- /* we can filter out messages that don't have the header,
- but we can't trust definite results list. */
- flags = FTS_LOOKUP_FLAG_HEADER;
- key = arg->value.str;
- if (*key == '\0') {
- /* we're only checking the existence
- of the header. */
- key = t_str_ucase(arg->hdr_field_name);
- }
- break;
- case SEARCH_TEXT:
- flags = FTS_LOOKUP_FLAG_HEADER;
- case SEARCH_BODY:
- flags |= FTS_LOOKUP_FLAG_BODY;
- key = arg->value.str;
- break;
- default:
- /* can't filter this */
- return 0;
+ const struct fts_score_map *scores;
+ unsigned int i, count;
+ ARRAY_TYPE(seq_range) backend_uids;
+ ARRAY_TYPE(uint32_t) vuids_arr;
+ const uint32_t *vuids;
+ struct fts_score_map *score;
+
+ scores = array_get(&br->scores, &count);
+ t_array_init(&vuids_arr, count);
+ t_array_init(&backend_uids, 64);
+ for (i = 0; i < count; i++)
+ seq_range_array_add(&backend_uids, 0, scores[i].uid);
+ vbox->vfuncs.get_virtual_uid_map(&vbox->box, br->box,
+ &backend_uids, &vuids_arr);
+
+ i_assert(array_count(&vuids_arr) == array_count(&br->scores));
+ vuids = array_get(&vuids_arr, &count);
+ for (i = 0; i < count; i++) {
+ score = array_append_space(&level->score_map);
+ score->uid = vuids[i];
+ score->score = scores[i].score;
}
- if (arg->match_not)
- flags |= FTS_LOOKUP_FLAG_INVERT;
+}
- if (!fctx->refreshed) {
- if (fts_backend_refresh(fctx->fbox->backend) < 0)
- return -1;
- fctx->refreshed = TRUE;
+static int
+mailbox_cmp_fts_backend(struct mailbox *const *m1, struct mailbox *const *m2)
+{
+ struct fts_backend *b1, *b2;
+
+ b1 = fts_mailbox_backend(*m1);
+ b2 = fts_mailbox_backend(*m2);
+ if (b1 < b2)
+ return -1;
+ if (b1 > b2)
+ return 1;
+ return 0;
+}
+
+static int
+multi_add_lookup_result(struct fts_search_context *fctx,
+ struct fts_search_level *level,
+ struct mail_search_arg *args,
+ struct fts_multi_result *result)
+{
+ struct virtual_mailbox *vbox = (struct virtual_mailbox *)fctx->box;
+ ARRAY_TYPE(seq_range) vuids;
+ size_t orig_size;
+ unsigned int i;
+
+ orig_size = level->args_matches->used;
+ fts_search_serialize(level->args_matches, args);
+ if (orig_size > 0) {
+ if (level->args_matches->used != orig_size * 2 ||
+ memcmp(level->args_matches->data,
+ CONST_PTR_OFFSET(level->args_matches->data,
+ orig_size), orig_size) != 0)
+ i_panic("incompatible fts backends for namespaces");
+ buffer_set_used_size(level->args_matches, orig_size);
}
- /* note that the key is in UTF-8 decomposed titlecase */
- fctx->lookup_ctx = fts_backend_lookup_init(fctx->fbox->backend);
- fts_backend_lookup_add(fctx->lookup_ctx, key, flags);
+ t_array_init(&vuids, 64);
+ for (i = 0; result->box_results[i].box != NULL; i++) {
+ struct fts_result *br = &result->box_results[i];
+
+ array_clear(&vuids);
+ if (array_is_created(&br->definite_uids)) {
+ vbox->vfuncs.get_virtual_uids(fctx->box, br->box,
+ &br->definite_uids,
+ &vuids);
+ }
+ uid_range_to_seqs(fctx, &vuids, &level->definite_seqs);
+
+ array_clear(&vuids);
+ if (array_is_created(&br->maybe_uids)) {
+ vbox->vfuncs.get_virtual_uids(fctx->box, br->box,
+ &br->maybe_uids, &vuids);
+ }
+ uid_range_to_seqs(fctx, &vuids, &level->maybe_seqs);
+
+ if (array_is_created(&br->scores))
+ level_scores_add_vuids(vbox, level, br);
+ }
return 0;
}
-void fts_search_lookup(struct fts_search_context *fctx)
+static int fts_search_lookup_level_multi(struct fts_search_context *fctx,
+ struct mail_search_arg *args,
+ bool and_args)
{
- struct mail_search_arg *arg;
- int ret;
+ struct virtual_mailbox *vbox = (struct virtual_mailbox *)fctx->box;
+ ARRAY_TYPE(mailboxes) mailboxes_arr, tmp_mailboxes;
+ struct mailbox *const *mailboxes;
+ struct fts_backend *backend;
+ struct fts_search_level *level;
+ struct fts_multi_result result;
+ unsigned int i, j, mailbox_count;
- if (fctx->best_arg == NULL)
- return;
+ p_array_init(&mailboxes_arr, fctx->result_pool, 8);
+ vbox->vfuncs.get_virtual_backend_boxes(fctx->box, &mailboxes_arr, TRUE);
+ array_sort(&mailboxes_arr, mailbox_cmp_fts_backend);
- i_array_init(&fctx->definite_seqs, 64);
- i_array_init(&fctx->maybe_seqs, 64);
- i_array_init(&fctx->score_map, 64);
+ memset(&result, 0, sizeof(result));
+ result.pool = fctx->result_pool;
- /* start lookup with the best arg */
- T_BEGIN {
- ret = fts_search_lookup_arg(fctx, fctx->best_arg);
- } T_END;
- /* filter the rest */
- for (arg = fctx->args->args; arg != NULL && ret == 0; arg = arg->next) {
- if (arg != fctx->best_arg) {
- T_BEGIN {
- ret = fts_search_lookup_arg(fctx, arg);
- } T_END;
+ level = array_append_space(&fctx->levels);
+ level->args_matches = buffer_create_dynamic(fctx->result_pool, 16);
+ p_array_init(&level->score_map, fctx->result_pool, 1);
+
+ mailboxes = array_get(&mailboxes_arr, &mailbox_count);
+ t_array_init(&tmp_mailboxes, mailbox_count);
+ for (i = 0; i < mailbox_count; i = j) {
+ array_clear(&tmp_mailboxes);
+ array_append(&tmp_mailboxes, &mailboxes[i], 1);
+
+ backend = fts_mailbox_backend(mailboxes[i]);
+ for (j = i + 1; j < mailbox_count; j++) {
+ if (fts_mailbox_backend(mailboxes[j]) != backend)
+ break;
+ array_append(&tmp_mailboxes, &mailboxes[j], 1);
}
+ (void)array_append_space(&tmp_mailboxes);
+
+ mail_search_args_reset(args, TRUE);
+ if (fts_backend_lookup_multi(backend,
+ array_idx(&tmp_mailboxes, 0),
+ args, and_args, &result) < 0)
+ return -1;
+
+ if (multi_add_lookup_result(fctx, level, args, &result) < 0)
+ return -1;
}
+ return 0;
+}
+
+static int fts_search_lookup_level(struct fts_search_context *fctx,
+ struct mail_search_arg *args,
+ bool and_args)
+{
+ if (!fctx->virtual_mailbox) {
+ if (fts_search_lookup_level_single(fctx, args, and_args) < 0)
+ return -1;
+ } else T_BEGIN {
+ if (fts_search_lookup_level_multi(fctx, args, and_args) < 0)
+ return -1;
+ } T_END;
+
+ for (; args != NULL; args = args->next) {
+ if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
+ continue;
- if (fctx->lookup_ctx != NULL) {
- fts_backend_lookup_deinit(&fctx->lookup_ctx,
- &fctx->definite_seqs,
- &fctx->maybe_seqs,
- &fctx->score_map);
+ if (fts_search_lookup_level(fctx, args->value.subargs,
+ args->type == SEARCH_SUB) < 0)
+ return -1;
}
+ return 0;
+}
+
+static void
+fts_search_merge_scores_and(ARRAY_TYPE(fts_score_map) *dest,
+ const ARRAY_TYPE(fts_score_map) *src)
+{
+ struct fts_score_map *dest_map;
+ const struct fts_score_map *src_map;
+ unsigned int desti, srci, dest_count, src_count;
+
+ dest_map = array_get_modifiable(dest, &dest_count);
+ src_map = array_get(src, &src_count);
- if (ret == 0) {
- fctx->seqs_set = TRUE;
- fts_uid_results_to_seq(fctx);
+ /* arg_scores are summed to current scores. we could drop UIDs that
+ don't exist in both, but that's just extra work so don't bother */
+ for (desti = srci = 0; desti < dest_count && srci < src_count;) {
+ if (dest_map[desti].uid < src_map[srci].uid)
+ desti++;
+ else if (dest_map[desti].uid > src_map[srci].uid)
+ srci++;
+ else {
+ if (dest_map[desti].score < src_map[srci].score)
+ dest_map[desti].score = src_map[srci].score;
+ desti++; srci++;
+ }
}
}
-static bool arg_is_better(const struct mail_search_arg *new_arg,
- const struct mail_search_arg *old_arg)
+static void
+fts_search_merge_scores_or(ARRAY_TYPE(fts_score_map) *dest,
+ const ARRAY_TYPE(fts_score_map) *src)
{
- if (old_arg == NULL)
- return TRUE;
- if (new_arg == NULL)
- return FALSE;
-
- /* avoid NOTs */
- if (old_arg->match_not && !new_arg->match_not)
- return TRUE;
- if (!old_arg->match_not && new_arg->match_not)
- return FALSE;
-
- /* prefer not to use headers. they have a larger possibility of
- having lots of identical strings */
- if (old_arg->type == SEARCH_HEADER ||
- old_arg->type == SEARCH_HEADER_COMPRESS_LWSP)
- return TRUE;
- else if (new_arg->type == SEARCH_HEADER ||
- new_arg->type == SEARCH_HEADER_COMPRESS_LWSP)
- return FALSE;
-
- return strlen(new_arg->value.str) > strlen(old_arg->value.str);
+ ARRAY_TYPE(fts_score_map) src2;
+ const struct fts_score_map *src_map, *src2_map;
+ unsigned int srci, src2i, src_count, src2_count;
+
+ t_array_init(&src2, array_count(dest));
+ array_append_array(&src2, dest);
+ array_clear(dest);
+
+ src_map = array_get(src, &src_count);
+ src2_map = array_get(&src2, &src2_count);
+
+ /* add any missing UIDs to current scores. if any existing UIDs have
+ lower scores than in arg_scores, increase them. */
+ for (srci = src2i = 0; srci < src_count || src2i < src2_count;) {
+ if (src2i == src2_count ||
+ src_map[srci].uid < src2_map[src2i].uid) {
+ array_append(dest, &src_map[srci], 1);
+ srci++;
+ } else if (srci == src_count ||
+ src_map[srci].uid > src2_map[src2i].uid) {
+ array_append(dest, &src2_map[src2i], 1);
+ src2i++;
+ } else {
+ i_assert(src_map[srci].uid == src2_map[src2i].uid);
+ if (src_map[srci].score > src2_map[src2i].score)
+ array_append(dest, &src_map[srci], 1);
+ else
+ array_append(dest, &src2_map[src2i], 1);
+ srci++; src2i++;
+ }
+ }
}
static void
-fts_search_args_find_best(struct mail_search_arg *args,
- struct mail_search_arg **best_arg)
+fts_search_merge_scores_level(struct fts_search_context *fctx,
+ struct mail_search_arg *args, unsigned int *idx,
+ bool and_args, ARRAY_TYPE(fts_score_map) *scores)
{
+ const struct fts_search_level *level;
+ ARRAY_TYPE(fts_score_map) arg_scores;
+
+ i_assert(array_count(scores) == 0);
+
+ /*
+ The (simplified) args can look like:
+
+ A and B and (C or D) and (E or F) and ...
+ A or B or (C and D) or (E and F) or ...
+
+ The A op B part's scores are in level->scores. The child args'
+ scores are in the sub levels' scores.
+ */
+
+ level = array_idx(&fctx->levels, *idx);
+ array_append_array(scores, &level->score_map);
+
+ t_array_init(&arg_scores, 64);
for (; args != NULL; args = args->next) {
- switch (args->type) {
- case SEARCH_BODY:
- case SEARCH_TEXT:
- case SEARCH_HEADER:
- case SEARCH_HEADER_COMPRESS_LWSP:
- if (arg_is_better(args, *best_arg))
- *best_arg = args;
- break;
- default:
- break;
- }
+ if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
+ continue;
+
+ *idx += 1;
+ array_clear(&arg_scores);
+ fts_search_merge_scores_level(fctx, args->value.subargs, idx,
+ args->type == SEARCH_OR,
+ &arg_scores);
+
+ if (and_args)
+ fts_search_merge_scores_and(scores, &arg_scores);
+ else T_BEGIN {
+ fts_search_merge_scores_or(scores, &arg_scores);
+ } T_END;
}
}
-void fts_search_analyze(struct fts_search_context *fctx)
+static void fts_search_merge_scores(struct fts_search_context *fctx)
{
- fts_search_args_find_best(fctx->args->args, &fctx->best_arg);
+ unsigned int idx = 0;
+
+ fts_search_merge_scores_level(fctx, fctx->args->args, &idx,
+ TRUE, &fctx->scores->score_map);
+}
+
+void fts_search_lookup(struct fts_search_context *fctx)
+{
+ uint32_t last_uid, seq1, seq2;
+
+ i_assert(array_count(&fctx->levels) == 0);
+ i_assert(fctx->args->simplified);
+
+ if (fts_backend_refresh(fctx->backend) < 0)
+ return;
+ if (fts_backend_get_last_uid(fctx->backend, fctx->box, &last_uid) < 0)
+ return;
+ mailbox_get_seq_range(fctx->box, last_uid+1, (uint32_t)-1,
+ &seq1, &seq2);
+ fctx->first_unindexed_seq = seq1 != 0 ? seq1 : (uint32_t)-1;
+
+ fts_search_serialize(fctx->orig_matches, fctx->args->args);
+
+ if (fts_search_lookup_level(fctx, fctx->args->args, TRUE) == 0) {
+ fctx->fts_lookup_success = TRUE;
+ fts_search_merge_scores(fctx);
+ }
+
+ fts_search_deserialize(fctx->args->args, fctx->orig_matches);
}
/* Copyright (c) 2006-2011 Dovecot authors, see the included COPYING file */
#include "lib.h"
-#include "ioloop.h"
#include "array.h"
-#include "str.h"
-#include "unichar.h"
-#include "istream.h"
-#include "time-util.h"
-#include "rfc822-parser.h"
-#include "message-parser.h"
-#include "message-decoder.h"
-#include "mail-namespace.h"
-#include "mail-search-build.h"
+#include "mail-search.h"
#include "mail-storage-private.h"
#include "mailbox-list-private.h"
+#include "../virtual/virtual-storage.h"
#include "fts-api-private.h"
-#include "fts-mailbox.h"
-#include "fts-storage.h"
+#include "fts-build.h"
+#include "fts-search-serialize.h"
#include "fts-plugin.h"
+#include "fts-storage.h"
#include <stdlib.h>
MODULE_CONTEXT(obj, fts_storage_module)
#define FTS_MAIL_CONTEXT(obj) \
MODULE_CONTEXT(obj, fts_mail_module)
+#define FTS_LIST_CONTEXT(obj) \
+ MODULE_CONTEXT(obj, fts_mailbox_list_module)
-#define FTS_SEARCH_NONBLOCK_COUNT 50
-#define FTS_BUILD_NOTIFY_INTERVAL_SECS 10
-
-struct fts_mail {
- union mail_module_context module_ctx;
- char score[30];
+struct fts_mailbox_list {
+ union mailbox_list_module_context module_ctx;
+ struct fts_backend *backend;
};
-struct fts_storage_build_context {
- struct mail_search_context *search_ctx;
- struct mail_search_args *search_args;
- struct fts_backend_build_context *build;
-
- struct timeval search_start_time, last_notify;
-
- uint32_t uid;
- string_t *headers;
- char *content_type, *content_disposition;
+struct fts_mailbox {
+ union mailbox_module_context module_ctx;
+ struct fts_backend_update_context *sync_update_ctx;
};
struct fts_transaction_context {
union mailbox_transaction_module_context module_ctx;
- struct fts_storage_build_context *build_ctx;
- ARRAY_TYPE(fts_score_map) *score_map;
- struct mail *mail;
-
- uint32_t last_uid;
+ struct fts_scores *scores;
+};
- unsigned int free_mail:1;
- unsigned int expunges:1;
+struct fts_mail {
+ union mail_module_context module_ctx;
+ char score[30];
};
static MODULE_CONTEXT_DEFINE_INIT(fts_storage_module,
&mail_storage_module_register);
static MODULE_CONTEXT_DEFINE_INIT(fts_mail_module, &mail_module_register);
+static MODULE_CONTEXT_DEFINE_INIT(fts_mailbox_list_module,
+ &mailbox_list_module_register);
-static void fts_mailbox_free(struct mailbox *box)
-{
- struct fts_mailbox *fbox = FTS_CONTEXT(box);
-
- if (fbox->backend != NULL)
- fts_backend_deinit(&fbox->backend);
-
- fbox->module_ctx.super.free(box);
- i_free(fbox);
-}
-
-static int fts_build_mail_flush_headers(struct fts_storage_build_context *ctx)
-{
- if (str_len(ctx->headers) == 0)
- return 0;
-
- i_assert(uni_utf8_data_is_valid(ctx->headers->data, ctx->headers->used));
-
- fts_backend_build_hdr(ctx->build, ctx->uid);
- if (fts_backend_build_more(ctx->build, str_data(ctx->headers),
- str_len(ctx->headers)) < 0)
- return -1;
-
- str_truncate(ctx->headers, 0);
- return 0;
-}
-
-static void fts_build_parse_content_type(struct fts_storage_build_context *ctx,
- const struct message_header_line *hdr)
-{
- struct rfc822_parser_context parser;
- string_t *content_type;
-
- rfc822_parser_init(&parser, hdr->full_value, hdr->full_value_len, NULL);
- (void)rfc822_skip_lwsp(&parser);
-
- T_BEGIN {
- content_type = t_str_new(64);
- if (rfc822_parse_content_type(&parser, content_type) >= 0) {
- i_free(ctx->content_type);
- ctx->content_type = i_strdup(str_c(content_type));
- }
- } T_END;
-}
-
-static void
-fts_build_parse_content_disposition(struct fts_storage_build_context *ctx,
- const struct message_header_line *hdr)
-{
- /* just pass it as-is to backend. */
- i_free(ctx->content_disposition);
- ctx->content_disposition =
- i_strndup(hdr->full_value, hdr->full_value_len);
-}
-
-static void fts_parse_mail_header(struct fts_storage_build_context *ctx,
- const struct message_block *raw_block)
-{
- const struct message_header_line *hdr = raw_block->hdr;
-
- if (strcasecmp(hdr->name, "Content-Type") == 0)
- fts_build_parse_content_type(ctx, hdr);
- else if (strcasecmp(hdr->name, "Content-Disposition") == 0)
- fts_build_parse_content_disposition(ctx, hdr);
-}
-
-static void fts_build_mail_header(struct fts_storage_build_context *ctx,
- const struct message_block *block)
-{
- const struct message_header_line *hdr = block->hdr;
-
- /* hdr->full_value is always set because we get the block from
- message_decoder */
- str_append(ctx->headers, hdr->name);
- str_append_n(ctx->headers, hdr->middle, hdr->middle_len);
- str_append_n(ctx->headers, hdr->full_value, hdr->full_value_len);
- if (!hdr->no_newline)
- str_append_c(ctx->headers, '\n');
-}
-
-static int
-fts_build_mail(struct fts_storage_build_context *ctx, struct mail *mail)
-{
- enum message_decoder_flags decoder_flags = MESSAGE_DECODER_FLAG_DTCASE;
- struct istream *input;
- struct message_parser_ctx *parser;
- struct message_decoder_context *decoder;
- struct message_block raw_block, block;
- struct message_part *prev_part, *parts;
- bool skip_body = FALSE, body_part = FALSE;
- int ret;
-
- ctx->uid = mail->uid;
-
- if (mail_get_stream(mail, NULL, NULL, &input) < 0)
- return -1;
-
- prev_part = NULL;
- parser = message_parser_init(pool_datastack_create(), input,
- MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE,
- 0);
-
-
- if ((ctx->build->backend->flags & FTS_BACKEND_FLAG_BINARY_MIME_PARTS) != 0)
- decoder_flags |= MESSAGE_DECODER_FLAG_RETURN_BINARY;
- decoder = message_decoder_init(decoder_flags);
- for (;;) {
- ret = message_parser_parse_next_block(parser, &raw_block);
- i_assert(ret != 0);
- if (ret < 0) {
- if (input->stream_errno == 0)
- ret = 0;
- break;
- }
-
- if (raw_block.part != prev_part) {
- /* body part changed. we're now parsing the end of
- boundary, possibly followed by message epilogue */
- if (!skip_body && prev_part != NULL) {
- i_assert(body_part);
- fts_backend_build_body_end(ctx->build);
- }
- prev_part = raw_block.part;
- i_free_and_null(ctx->content_type);
- i_free_and_null(ctx->content_disposition);
-
- if (raw_block.size != 0) {
- /* multipart. skip until beginning of next
- part's headers */
- skip_body = TRUE;
- }
- }
-
- if (raw_block.hdr != NULL) {
- /* always handle headers */
- } else if (raw_block.size == 0) {
- /* end of headers */
- const char *content_type = ctx->content_type == NULL ?
- "text/plain" : ctx->content_type;
-
- skip_body = !fts_backend_build_body_begin(ctx->build,
- ctx->uid, content_type,
- ctx->content_disposition);
- body_part = TRUE;
- } else {
- if (skip_body)
- continue;
- }
-
- if (!message_decoder_decode_next_block(decoder, &raw_block,
- &block))
- continue;
-
- if (block.hdr != NULL) {
- fts_parse_mail_header(ctx, &raw_block);
- fts_build_mail_header(ctx, &block);
- } else if (block.size == 0) {
- /* end of headers */
- str_append_c(ctx->headers, '\n');
- } else {
- i_assert(body_part);
- if (fts_backend_build_more(ctx->build,
- block.data, block.size) < 0) {
- ret = -1;
- break;
- }
- }
- }
- if (!skip_body && body_part)
- fts_backend_build_body_end(ctx->build);
- if (message_parser_deinit(&parser, &parts) < 0)
- mail_set_cache_corrupted(mail, MAIL_FETCH_MESSAGE_PARTS);
- message_decoder_deinit(&decoder);
-
- if (ret == 0) {
- /* Index all headers at the end. This is required for Squat,
- because it can handle only incremental UIDs. */
- ret = fts_build_mail_flush_headers(ctx);
- }
- return ret;
-}
-
-static int fts_build_init_seq(struct fts_search_context *fctx,
- struct mailbox_transaction_context *t,
- uint32_t seq1, uint32_t seq2, uint32_t last_uid)
-{
- struct mail_search_args *search_args;
- struct fts_storage_build_context *ctx;
- struct fts_backend_build_context *build;
- uint32_t last_uid_locked;
-
- if (fctx->best_arg->type == SEARCH_HEADER ||
- fctx->best_arg->type == SEARCH_HEADER_COMPRESS_LWSP) {
- /* we're not updating the index just for header lookups */
- if (seq1 < fctx->first_nonindexed_seq)
- fctx->first_nonindexed_seq = seq1;
- return 0;
- }
-
- if (fts_backend_build_init(fctx->fbox->backend,
- &last_uid_locked, &build) < 0)
- return -1;
- if (last_uid != last_uid_locked && last_uid_locked != (uint32_t)-1) {
- /* changed, need to get again the sequences */
- last_uid = last_uid_locked;
- mailbox_get_seq_range(t->box, last_uid+1, (uint32_t)-1,
- &seq1, &seq2);
- if (seq1 == 0) {
- /* no new messages */
- (void)fts_backend_build_deinit(&build);
- return 0;
- }
- }
-
- search_args = mail_search_build_init();
- mail_search_build_add_seqset(search_args, seq1, seq2);
-
- ctx = i_new(struct fts_storage_build_context, 1);
- ctx->build = build;
- ctx->headers = str_new(default_pool, 512);
- ctx->search_ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
- ctx->search_ctx->progress_hidden = TRUE;
- ctx->search_args = search_args;
-
- fctx->build_ctx = ctx;
- return 1;
-}
-
-static int fts_build_init_trans(struct fts_search_context *fctx,
- struct mailbox_transaction_context *t)
-{
- uint32_t last_uid, seq1, seq2;
- int ret;
-
- if (fts_backend_get_last_uid(fctx->fbox->backend, &last_uid) < 0)
- return -1;
-
- mailbox_get_seq_range(t->box, last_uid+1, (uint32_t)-1, &seq1, &seq2);
- if (seq1 == 0) {
- /* no new messages */
- return 0;
- }
-
- ret = fts_build_init_seq(fctx, t, seq1, seq2, last_uid);
- return ret < 0 ? -1 : 0;
-}
-
-static int
-fts_build_init_box(struct fts_search_context *fctx, struct mailbox *box,
- uint32_t last_uid)
-{
- uint32_t seq1, seq2;
-
- mailbox_get_seq_range(box, last_uid + 1, (uint32_t)-1, &seq1, &seq2);
- if (seq1 == 0)
- return 0;
-
- fctx->virtual_ctx.trans = mailbox_transaction_begin(box, 0);
- return fts_build_init_seq(fctx, fctx->virtual_ctx.trans,
- seq1, seq2, last_uid);
-}
-
-static int mailbox_name_cmp(const struct fts_orig_mailboxes *box1,
- const struct fts_orig_mailboxes *box2)
-{
- int ret;
-
- T_BEGIN {
- const char *vname1, *vname2;
-
- vname1 = mailbox_list_get_vname(box1->ns->list, box1->name);
- vname2 = mailbox_list_get_vname(box2->ns->list, box2->name);
- ret = strcmp(vname1, vname2);
- } T_END;
- return ret;
-}
-
-static int
-fts_backend_uid_map_mailbox_cmp(const struct fts_backend_uid_map *map1,
- const struct fts_backend_uid_map *map2)
-{
- return strcmp(map1->mailbox, map2->mailbox);
-}
-
-static int fts_build_init_virtual_next(struct fts_search_context *fctx)
-{
- struct fts_search_virtual_context *vctx = &fctx->virtual_ctx;
- struct mailbox_status status;
- const struct fts_orig_mailboxes *boxes;
- const struct fts_backend_uid_map *last_uids;
- unsigned int boxi, uidi, box_count, last_uid_count;
- int ret, vret = 0;
-
- if (vctx->pool == NULL)
- return 0;
-
- if (fctx->virtual_ctx.trans != NULL)
- (void)mailbox_transaction_commit(&fctx->virtual_ctx.trans);
-
- boxes = array_get(&vctx->orig_mailboxes, &box_count);
- last_uids = array_get(&vctx->last_uids, &last_uid_count);
-
- boxi = vctx->boxi;
- uidi = vctx->uidi;
- while (vret == 0 && boxi < box_count && uidi < last_uid_count) {
- T_BEGIN {
- const char *vname;
-
- vname = mailbox_list_get_vname(boxes[boxi].ns->list,
- boxes[boxi].name);
- ret = strcmp(vname, last_uids[uidi].mailbox);
- } T_END;
- if (ret == 0) {
- /* match. check also that uidvalidity matches. */
- mailbox_get_open_status(boxes[boxi].box,
- STATUS_UIDVALIDITY, &status);
- if (status.uidvalidity != last_uids[uidi].uidvalidity) {
- uidi++;
- continue;
- }
- vret = fts_build_init_box(fctx, boxes[boxi].box,
- last_uids[uidi].uid);
- boxi++;
- uidi++;
- } else if (ret > 0) {
- /* not part of this virtual mailbox */
- uidi++;
- } else {
- /* no messages indexed in the mailbox */
- vret = fts_build_init_box(fctx, boxes[boxi].box, 0);
- boxi++;
- }
- }
- while (vret == 0 && boxi < box_count) {
- vret = fts_build_init_box(fctx, boxes[boxi].box, 0);
- boxi++;
- }
- vctx->boxi = boxi;
- vctx->uidi = uidi;
- return vret;
-}
-
-static const char *
-fts_box_get_root(struct mailbox *box, struct mail_namespace **ns_r)
-{
- struct mail_namespace *ns = mailbox_get_namespace(box);
- const char *name = box->name;
-
- while (ns->alias_for != NULL)
- ns = ns->alias_for;
- *ns_r = ns;
-
- if (*name == '\0' && ns != mailbox_get_namespace(box) &&
- (ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0) {
- /* ugly workaround to allow selecting INBOX from a Maildir/
- when it's not in the inbox=yes namespace. */
- return "INBOX";
- }
- return name;
-}
-
-static int fts_build_init_virtual(struct fts_search_context *fctx)
-{
- struct fts_search_virtual_context *vctx = &fctx->virtual_ctx;
- ARRAY_TYPE(mailboxes) mailboxes;
- struct mailbox *const *boxes;
- const struct fts_orig_mailboxes *orig_boxes;
- struct fts_orig_mailboxes orig_box;
- unsigned int i, box_count;
- int ret;
-
- t_array_init(&mailboxes, 64);
- fts_mailbox_get_virtual_backend_boxes(fctx->t->box, &mailboxes, TRUE);
- boxes = array_get_modifiable(&mailboxes, &box_count);
-
- vctx->pool = pool_alloconly_create("fts virtual build", 1024);
- p_array_init(&vctx->orig_mailboxes, vctx->pool, box_count);
- memset(&orig_box, 0, sizeof(orig_box));
- for (i = 0; i < box_count; i++) {
- orig_box.box = boxes[i];
- orig_box.name = fts_box_get_root(boxes[i], &orig_box.ns);
- array_append(&vctx->orig_mailboxes, &orig_box, 1);
- }
-
- orig_boxes = array_get(&vctx->orig_mailboxes, &box_count);
- if (box_count <= 0) {
- if (box_count == 0) {
- /* empty virtual mailbox */
- return 0;
- }
- /* virtual mailbox is built from only a single mailbox
- (currently). check that directly. */
- fctx->virtual_ctx.trans =
- mailbox_transaction_begin(orig_boxes[0].box, 0);
- ret = fts_build_init_trans(fctx, fctx->virtual_ctx.trans);
- return ret;
- }
-
- /* virtual mailbox is built from multiple mailboxes. figure out which
- ones need updating. */
- p_array_init(&vctx->last_uids, vctx->pool, 64);
- if (fts_backend_get_all_last_uids(fctx->fbox->backend, vctx->pool,
- &vctx->last_uids) < 0) {
- pool_unref(&vctx->pool);
- return -1;
- }
-
- array_sort(&vctx->orig_mailboxes, mailbox_name_cmp);
- array_sort(&vctx->last_uids, fts_backend_uid_map_mailbox_cmp);
-
- ret = fts_build_init_virtual_next(fctx);
- return ret < 0 ? -1 : 0;
-}
-
-static int fts_build_init(struct fts_search_context *fctx)
-{
- struct mailbox_status status;
- int ret;
-
- mailbox_get_open_status(fctx->t->box, STATUS_MESSAGES | STATUS_UIDNEXT,
- &status);
- if (status.messages == fctx->fbox->last_messages_count &&
- status.uidnext == fctx->fbox->last_uidnext) {
- /* no new messages since last check */
- return 0;
- }
-
- if (fctx->fbox->virtual &&
- (fctx->fbox->backend->flags & FTS_BACKEND_FLAG_VIRTUAL_LOOKUPS) != 0)
- ret = fts_build_init_virtual(fctx);
- else
- ret = fts_build_init_trans(fctx, fctx->t);
- if (ret == 0 && fctx->build_ctx == NULL) {
- /* index was up-to-date */
- fctx->fbox->last_messages_count = status.messages;
- fctx->fbox->last_uidnext = status.uidnext;
- }
- return ret;
-}
-
-static int fts_build_deinit(struct fts_storage_build_context **_ctx)
+static void fts_scores_unref(struct fts_scores **_scores)
{
- struct fts_storage_build_context *ctx = *_ctx;
- struct mailbox *box = ctx->search_ctx->transaction->box;
- struct fts_mailbox *fbox = FTS_CONTEXT(box);
- struct mailbox_status status;
- int ret = 0;
-
- *_ctx = NULL;
-
- if (mailbox_search_deinit(&ctx->search_ctx) < 0)
- ret = -1;
-
- if (fts_backend_build_deinit(&ctx->build) < 0)
- ret = -1;
+ struct fts_scores *scores = *_scores;
- if (ret == 0) {
- mailbox_get_open_status(box, STATUS_MESSAGES | STATUS_UIDNEXT,
- &status);
- fbox->last_messages_count = status.messages;
- fbox->last_uidnext = status.uidnext;
+ *_scores = NULL;
+ if (--scores->refcount == 0) {
+ array_free(&scores->score_map);
+ i_free(scores);
}
-
- if (ioloop_time - ctx->search_start_time.tv_sec >=
- FTS_BUILD_NOTIFY_INTERVAL_SECS &&
- box->storage->callbacks.notify_ok != NULL) {
- /* we notified at least once */
- box->storage->callbacks.
- notify_ok(box, "Mailbox indexing finished",
- box->storage->callback_context);
- }
-
- str_free(&ctx->headers);
- mail_search_args_unref(&ctx->search_args);
- i_free(ctx->content_type);
- i_free(ctx->content_disposition);
- i_free(ctx);
- return ret;
-}
-
-static void
-fts_build_notify(struct fts_storage_build_context *ctx, uint32_t seq)
-{
- struct mailbox *box = ctx->search_ctx->transaction->box;
-
- if (ctx->last_notify.tv_sec == 0) {
- /* set the search time in here, in case a plugin
- already spent some time indexing the mailbox */
- ctx->search_start_time = ioloop_timeval;
- } else if (box->storage->callbacks.notify_ok != NULL) {
- double completed_frac;
- unsigned int eta_secs;
- const struct seq_range *range;
- uint32_t seq_diff;
-
- range = array_idx(&ctx->search_args->args->value.seqset, 0);
-
- seq_diff = range->seq2 - range->seq1;
-
- if (seq_diff != 0) {
- completed_frac = (double)(seq - range->seq1) / seq_diff;
-
- if (completed_frac >= 0.000001) {
- unsigned int elapsed_msecs, est_total_msecs;
-
- elapsed_msecs = timeval_diff_msecs(&ioloop_timeval,
- &ctx->search_start_time);
- est_total_msecs = elapsed_msecs / completed_frac;
- eta_secs = (est_total_msecs - elapsed_msecs) / 1000;
- } else {
- eta_secs = 0;
- }
- } else {
- completed_frac = 0.0;
- eta_secs = 0;
- }
-
- T_BEGIN {
- const char *text;
-
- text = t_strdup_printf("Indexed %d%% of the mailbox, "
- "ETA %d:%02d", (int)(completed_frac * 100.0),
- eta_secs/60, eta_secs%60);
- box->storage->callbacks.
- notify_ok(box, text,
- box->storage->callback_context);
- } T_END;
- }
- ctx->last_notify = ioloop_timeval;
-}
-
-static int fts_build_more(struct fts_storage_build_context *ctx)
-{
- struct mail *mail = NULL;
- unsigned int count = 0;
- int ret;
-
- while (mailbox_search_next(ctx->search_ctx, &mail)) {
- T_BEGIN {
- ret = fts_build_mail(ctx, mail);
- } T_END;
-
- if (ret < 0)
- return -1;
-
- if (++count == FTS_SEARCH_NONBLOCK_COUNT)
- return 0;
- }
-
- if (ioloop_time - ctx->last_notify.tv_sec >=
- FTS_BUILD_NOTIFY_INTERVAL_SECS && mail != NULL)
- fts_build_notify(ctx, mail->seq);
- return 1;
-}
-
-static void fts_search_init_lookup(struct mail_search_context *ctx,
- struct fts_search_context *fctx)
-{
- fts_search_lookup(fctx);
-
- if (fctx->seqs_set &&
- strcmp(ctx->transaction->box->storage->name, "virtual") != 0) {
- ctx->progress_max = array_count(&fctx->definite_seqs) +
- array_count(&fctx->maybe_seqs);
- }
- ctx->progress_cur = 0;
}
static bool fts_try_build_init(struct mail_search_context *ctx,
struct fts_search_context *fctx)
{
- if (fctx->best_arg == NULL) {
- /* don't use FTS for this search */
- fctx->build_initialized = TRUE;
- return TRUE;
- }
-
- if (fts_backend_is_building(fctx->fbox->backend)) {
+ if (fts_backend_is_updating(fctx->backend)) {
/* this process is already building the indexes */
return FALSE;
}
fctx->build_initialized = TRUE;
- if (fts_build_init(fctx) < 0) {
- fctx->fbox->backend = NULL;
- return TRUE;
- }
-
- if (fctx->build_ctx == NULL) {
+ switch (fts_build_init(fctx->backend, ctx->transaction->box, FALSE,
+ &fctx->build_ctx)) {
+ case -1:
+ break;
+ case 0:
/* the index was up to date */
- fts_search_init_lookup(ctx, fctx);
- } else {
- /* hide "searching" notifications */
+ fts_search_lookup(fctx);
+ break;
+ case 1:
+ /* hide "searching" notifications while building index */
ctx->progress_hidden = TRUE;
+ break;
}
return TRUE;
}
+static bool fts_want_build_args(const struct mail_search_arg *args)
+{
+ /* we want to update index only when searching from message body.
+ it's not worth the wait for searching only from headers, which
+ could be in cache file already */
+ for (; args != NULL; args = args->next) {
+ switch (args->type) {
+ case SEARCH_OR:
+ case SEARCH_SUB:
+ case SEARCH_INTHREAD:
+ if (fts_want_build_args(args->value.subargs))
+ return TRUE;
+ break;
+ case SEARCH_BODY:
+ case SEARCH_TEXT:
+ return TRUE;
+ default:
+ break;
+ }
+ }
+ return FALSE;
+}
+
static struct mail_search_context *
fts_mailbox_search_init(struct mailbox_transaction_context *t,
struct mail_search_args *args,
{
struct fts_transaction_context *ft = FTS_CONTEXT(t);
struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
+ struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(t->box->list);
struct mail_search_context *ctx;
struct fts_search_context *fctx;
ctx = fbox->module_ctx.super.search_init(t, args, sort_program,
wanted_fields, wanted_headers);
+ if (!fts_backend_can_lookup(flist->backend, args->args))
+ return ctx;
+
fctx = i_new(struct fts_search_context, 1);
- fctx->fbox = fbox;
+ fctx->box = t->box;
+ fctx->backend = flist->backend;
fctx->t = t;
fctx->args = args;
- fctx->first_nonindexed_seq = (uint32_t)-1;
+ fctx->result_pool = pool_alloconly_create("fts results", 1024*32);
+ fctx->orig_matches = buffer_create_dynamic(default_pool, 64);
+ i_array_init(&fctx->levels, 8);
+ fctx->scores = i_new(struct fts_scores, 1);
+ fctx->scores->refcount = 1;
+ i_array_init(&fctx->scores->score_map, 64);
MODULE_CONTEXT_SET(ctx, fts_storage_module, fctx);
- if (fbox->backend == NULL)
- return ctx;
+ fctx->virtual_mailbox =
+ strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
- ft->score_map = &fctx->score_map;
+ /* transaction contains the last search's scores. they can be
+ queried later with mail_get_special() */
+ if (ft->scores != NULL)
+ fts_scores_unref(&ft->scores);
+ ft->scores = fctx->scores;
+ ft->scores->refcount++;
- fts_search_analyze(fctx);
- (void)fts_try_build_init(ctx, fctx);
+ if (fts_want_build_args(args->args))
+ (void)fts_try_build_init(ctx, fctx);
+ else {
+ fctx->build_initialized = TRUE;
+ fts_search_lookup(fctx);
+ }
return ctx;
}
-static int fts_mailbox_search_build_more(struct mail_search_context *ctx)
+static bool fts_mailbox_build_continue(struct mail_search_context *ctx)
{
struct fts_search_context *fctx = FTS_CONTEXT(ctx);
- int ret = 1;
+ int ret;
- while (fctx->build_ctx != NULL) {
+ if (fctx == NULL)
+ return TRUE;
+
+ if (!fctx->build_initialized) {
+ /* we're still waiting for this process (but another command)
+ to finish building the indexes */
+ if (!fts_try_build_init(ctx, fctx))
+ return FALSE;
+ }
+
+ if (fctx->build_ctx != NULL) {
/* this command is still building the indexes */
ret = fts_build_more(fctx->build_ctx);
if (ret == 0)
- return 0;
-
- /* finished / error */
+ return FALSE;
ctx->progress_hidden = FALSE;
if (fts_build_deinit(&fctx->build_ctx) < 0)
ret = -1;
- if (ret > 0) {
- if (fts_build_init_virtual_next(fctx) == 0) {
- /* all finished */
- fts_search_init_lookup(ctx, fctx);
- }
- }
+ if (ret > 0)
+ fts_search_lookup(fctx);
}
- return ret;
+ return TRUE;
}
static bool
struct mail **mail_r, bool *tryagain_r)
{
struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
- struct fts_search_context *fctx = FTS_CONTEXT(ctx);
- if (!fctx->build_initialized) {
- /* we're still waiting for this process (but another command)
- to finish building the indexes */
- if (!fts_try_build_init(ctx, fctx)) {
- *tryagain_r = TRUE;
- return FALSE;
- }
- }
-
- if (fts_mailbox_search_build_more(ctx) == 0) {
+ if (!fts_mailbox_build_continue(ctx)) {
*tryagain_r = TRUE;
return FALSE;
}
- /* if we're here, the indexes are either built or they're not used */
return fbox->module_ctx.super.
search_next_nonblock(ctx, mail_r, tryagain_r);
}
static void
-fts_mailbox_search_args_definite_set(struct fts_search_context *fctx)
+fts_search_apply_results_level(struct mail_search_context *ctx,
+ struct mail_search_arg *args, unsigned int *idx)
{
- struct mail_search_arg *arg;
+ struct fts_search_context *fctx = FTS_CONTEXT(ctx);
+ const struct fts_search_level *level;
- for (arg = fctx->args->args; arg != NULL; arg = arg->next) {
- switch (arg->type) {
- case SEARCH_TEXT:
- case SEARCH_BODY:
- arg->result = 1;
- break;
- default:
- break;
- }
- }
-}
+ level = array_idx(&fctx->levels, *idx);
-static bool search_nonindexed(struct mail_search_context *ctx)
-{
- struct fts_search_context *fctx = FTS_CONTEXT(ctx);
- struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
- struct mailbox_status status;
+ if (array_is_created(&level->definite_seqs) &&
+ seq_range_exists(&level->definite_seqs, ctx->seq))
+ fts_search_deserialize_add_matches(args, level->args_matches);
+ else if (!array_is_created(&level->maybe_seqs) ||
+ !seq_range_exists(&level->maybe_seqs, ctx->seq))
+ fts_search_deserialize_add_nonmatches(args, level->args_matches);
- mailbox_get_open_status(ctx->transaction->box,
- STATUS_MESSAGES, &status);
+ for (; args != NULL; args = args->next) {
+ if (args->type != SEARCH_OR && args->type != SEARCH_SUB)
+ continue;
- fctx->seqs_set = FALSE;
- ctx->seq = fctx->first_nonindexed_seq - 1;
- ctx->progress_cur = ctx->seq;
- ctx->progress_max = status.messages;
- return fbox->module_ctx.super.search_next_update_seq(ctx);
+ *idx += 1;
+ fts_search_apply_results_level(ctx, args->value.subargs, idx);
+ }
}
static bool fts_mailbox_search_next_update_seq(struct mail_search_context *ctx)
{
struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
struct fts_search_context *fctx = FTS_CONTEXT(ctx);
- struct seq_range *def_range, *maybe_range, *range;
- unsigned int def_count, maybe_count;
- uint32_t wanted_seq;
- bool use_maybe, ret;
+ unsigned int idx;
- if (!fctx->seqs_set)
+ if (fctx == NULL || !fctx->fts_lookup_success) {
+ /* fts lookup not done for this search */
return fbox->module_ctx.super.search_next_update_seq(ctx);
-
- wanted_seq = ctx->seq + 1;
- /* fts_search_lookup() was called successfully */
- for (;;) {
- def_range = array_get_modifiable(&fctx->definite_seqs,
- &def_count);
- maybe_range = array_get_modifiable(&fctx->maybe_seqs,
- &maybe_count);
- /* if we're ahead of current positions, skip them */
- while (fctx->definite_idx < def_count &&
- wanted_seq > def_range[fctx->definite_idx].seq2)
- fctx->definite_idx++;
- while (fctx->maybe_idx < maybe_count &&
- wanted_seq > maybe_range[fctx->maybe_idx].seq2)
- fctx->maybe_idx++;
-
- /* use whichever is lower of definite/maybe */
- if (fctx->definite_idx == def_count) {
- if (fctx->maybe_idx == maybe_count) {
- /* look for the non-indexed mails */
- if (fctx->first_nonindexed_seq == (uint32_t)-1)
- return FALSE;
- return search_nonindexed(ctx);
- }
- use_maybe = TRUE;
- } else if (fctx->maybe_idx == maybe_count) {
- use_maybe = FALSE;
- } else {
- use_maybe = maybe_range[fctx->maybe_idx].seq1 <
- def_range[fctx->definite_idx].seq2;
- }
-
- if (use_maybe)
- range = maybe_range + fctx->maybe_idx;
- else
- range = def_range + fctx->definite_idx;
-
- i_assert(range->seq1 <= range->seq2);
- if (wanted_seq > range->seq1) {
- /* current sequence is already larger than where
- range begins, so use the current sequence. */
- range->seq1 = wanted_seq+1;
- } else {
- wanted_seq = range->seq1;
- range->seq1++;
- }
- if (range->seq1 > range->seq2)
- range->seq2 = 0;
-
- /* ctx->seq points to previous sequence we want */
- ctx->seq = wanted_seq - 1;
- ret = fbox->module_ctx.super.search_next_update_seq(ctx);
- if (!ret || wanted_seq == ctx->seq)
- break;
- wanted_seq = ctx->seq;
- mail_search_args_reset(ctx->args->args, FALSE);
}
- if (!use_maybe) {
- /* we have definite results, update args */
- fts_mailbox_search_args_definite_set(fctx);
- }
-
- if (ctx->seq + 1 >= fctx->first_nonindexed_seq) {
- /* this is a virtual mailbox and we're searching headers.
- some mailboxes had more messages indexed than others.
- to avoid duplicates or jumping around, ignore the rest of
- the search results and just go through the messages in
- order. */
- return search_nonindexed(ctx);
- }
+ /* restore original [non]matches */
+ fts_search_deserialize(ctx->args->args, fctx->orig_matches);
- ctx->progress_cur = fctx->definite_idx + fctx->maybe_idx;
- return ret;
-}
-
-static bool
-fts_mailbox_search_next_update_seq_virtual(struct mail_search_context *ctx)
-{
- struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
- struct fts_search_context *fctx = FTS_CONTEXT(ctx);
-
- while (fbox->module_ctx.super.search_next_update_seq(ctx)) {
- if (!fctx->seqs_set)
- return TRUE;
+ if (!fbox->module_ctx.super.search_next_update_seq(ctx))
+ return FALSE;
- /* virtual mailbox searches don't return sequences sorted.
- just check if the suggested sequence exists. */
- if (seq_range_exists(&fctx->definite_seqs, ctx->seq)) {
- fts_mailbox_search_args_definite_set(fctx);
- return TRUE;
- }
- if (seq_range_exists(&fctx->maybe_seqs, ctx->seq))
- return TRUE;
- mail_search_args_reset(ctx->args->args, FALSE);
+ if (ctx->seq >= fctx->first_unindexed_seq) {
+ /* we've not indexed this far */
+ return TRUE;
}
- return FALSE;
+
+ /* apply [non]matches based on the FTS lookup results */
+ idx = 0;
+ fts_search_apply_results_level(ctx, ctx->args->args, &idx);
+ return TRUE;
}
static int fts_mailbox_search_deinit(struct mail_search_context *ctx)
{
- struct fts_transaction_context *ft = FTS_CONTEXT(ctx->transaction);
struct fts_mailbox *fbox = FTS_CONTEXT(ctx->transaction->box);
struct fts_search_context *fctx = FTS_CONTEXT(ctx);
- if (ft->score_map == &fctx->score_map)
- ft->score_map = NULL;
+ if (fctx != NULL) {
+ if (fctx->build_ctx != NULL) {
+ /* the search was cancelled */
+ (void)fts_build_deinit(&fctx->build_ctx);
+ }
- if (fctx->build_ctx != NULL) {
- /* the search was cancelled */
- (void)fts_build_deinit(&fctx->build_ctx);
+ buffer_free(&fctx->orig_matches);
+ array_free(&fctx->levels);
+ pool_unref(&fctx->result_pool);
+ fts_scores_unref(&fctx->scores);
+ i_free(fctx);
}
-
- if (array_is_created(&fctx->definite_seqs))
- array_free(&fctx->definite_seqs);
- if (array_is_created(&fctx->maybe_seqs))
- array_free(&fctx->maybe_seqs);
- if (array_is_created(&fctx->score_map))
- array_free(&fctx->score_map);
- if (fctx->virtual_ctx.trans != NULL)
- (void)mailbox_transaction_commit(&fctx->virtual_ctx.trans);
- if (fctx->virtual_ctx.pool != NULL)
- pool_unref(&fctx->virtual_ctx.pool);
- i_free(fctx);
return fbox->module_ctx.super.search_deinit(ctx);
}
-static void fts_mail_expunge(struct mail *_mail)
-{
- struct mail_private *mail = (struct mail_private *)_mail;
- struct fts_mail *fmail = FTS_MAIL_CONTEXT(mail);
- struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
- struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
-
- ft->expunges = TRUE;
- if (fbox->backend != NULL)
- fts_backend_expunge(fbox->backend, _mail);
-
- fmail->module_ctx.super.expunge(_mail);
-}
-
static int fts_score_cmp(const uint32_t *uid, const struct fts_score_map *score)
{
return *uid < score->uid ? -1 :
struct fts_transaction_context *ft = FTS_CONTEXT(_mail->transaction);
const struct fts_score_map *scores;
- if (field != MAIL_FETCH_SEARCH_SCORE || ft->score_map == NULL ||
- !array_is_created(ft->score_map))
+ if (field != MAIL_FETCH_SEARCH_SCORE || ft->scores == NULL)
scores = NULL;
else {
- scores = array_bsearch(ft->score_map, &_mail->uid,
+ scores = array_bsearch(&ft->scores->score_map, &_mail->uid,
fts_score_cmp);
}
if (scores != NULL) {
struct fts_mailbox *fbox = FTS_CONTEXT(_mail->box);
struct fts_mail *fmail;
- if (fbox == NULL || fbox->backend == NULL)
+ if (fbox == NULL)
return;
fmail = p_new(mail->pool, struct fts_mail, 1);
fmail->module_ctx.super = *v;
mail->vlast = &fmail->module_ctx.super;
- v->expunge = fts_mail_expunge;
v->get_special = fts_mail_get_special;
MODULE_CONTEXT_SET(mail, fts_mail_module, fmail);
}
-static void fts_box_backends_init(struct mailbox *box)
-{
- struct fts_mailbox *fbox = FTS_CONTEXT(box);
-
- fbox->backend = fts_backend_init(fbox->env, box);
- if (fbox->backend == NULL && box->storage->set->mail_debug)
- i_debug("fts: Backend not enabled by the fts setting");
-}
-
static struct mailbox_transaction_context *
fts_transaction_begin(struct mailbox *box,
enum mailbox_transaction_flags flags)
ft = i_new(struct fts_transaction_context, 1);
- /* the backend creation is delayed until the first transaction is
- started. at that point the mailbox has been synced at least once. */
- if (!fbox->backend_set) {
- fts_box_backends_init(box);
- fbox->backend_set = TRUE;
- }
-
t = fbox->module_ctx.super.transaction_begin(box, flags);
MODULE_CONTEXT_SET(t, fts_storage_module, ft);
return t;
}
-static void
-fts_storage_build_context_deinit(struct fts_storage_build_context *build_ctx)
-{
- (void)fts_backend_build_deinit(&build_ctx->build);
- str_free(&build_ctx->headers);
- i_free(build_ctx);
-}
-
-static void
-fts_transaction_finish(struct mailbox *box, struct fts_transaction_context *ft,
- bool committed)
-{
- struct fts_mailbox *fbox = FTS_CONTEXT(box);
-
- if (ft->expunges)
- fts_backend_expunge_finish(fbox->backend, box, committed);
- i_free(ft);
-}
-
static void fts_transaction_rollback(struct mailbox_transaction_context *t)
{
- struct mailbox *box = t->box;
- struct fts_mailbox *fbox = FTS_CONTEXT(box);
+ struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
- if (ft->build_ctx != NULL) {
- fts_storage_build_context_deinit(ft->build_ctx);
- ft->build_ctx = NULL;
- }
- if (ft->free_mail)
- mail_free(&ft->mail);
-
+ if (ft->scores != NULL)
+ fts_scores_unref(&ft->scores);
fbox->module_ctx.super.transaction_rollback(t);
- fts_transaction_finish(box, ft, FALSE);
}
static int
fts_transaction_commit(struct mailbox_transaction_context *t,
struct mail_transaction_commit_changes *changes_r)
{
- struct mailbox *box = t->box;
- struct fts_mailbox *fbox = FTS_CONTEXT(box);
+ struct fts_mailbox *fbox = FTS_CONTEXT(t->box);
struct fts_transaction_context *ft = FTS_CONTEXT(t);
- int ret;
- if (ft->build_ctx != NULL) {
- fts_storage_build_context_deinit(ft->build_ctx);
- ft->build_ctx = NULL;
+ if (ft->scores != NULL)
+ fts_scores_unref(&ft->scores);
+ return fbox->module_ctx.super.transaction_commit(t, changes_r);
+}
+
+static void fts_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
+ enum mailbox_sync_type sync_type)
+{
+ struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
+ struct fts_mailbox *fbox = FTS_CONTEXT(box);
+
+ if (fbox->module_ctx.super.sync_notify != NULL)
+ fbox->module_ctx.super.sync_notify(box, uid, sync_type);
+
+ if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE) {
+ if (uid == 0 && fbox->sync_update_ctx != NULL) {
+ /* this sync is finished */
+ (void)fts_backend_update_deinit(&fbox->sync_update_ctx);
+ }
+ return;
}
- if (ft->free_mail)
- mail_free(&ft->mail);
- ret = fbox->module_ctx.super.transaction_commit(t, changes_r);
- fts_transaction_finish(box, ft, ret == 0);
- return ret;
+ if (fbox->sync_update_ctx == NULL) {
+ fbox->sync_update_ctx = fts_backend_update_init(flist->backend);
+ fts_backend_update_set_mailbox(fbox->sync_update_ctx, box);
+ }
+ fts_backend_update_expunge(fbox->sync_update_ctx, uid);
}
static int fts_update(struct mailbox *box)
{
- struct mailbox_transaction_context *t;
- struct mail_search_args *search_args;
- struct mail_search_arg *arg;
- struct mail_search_context *ctx;
- struct fts_search_context *fctx;
+ struct fts_storage_build_context *build_ctx;
+ struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
int ret = 0;
- t = mailbox_transaction_begin(box, 0);
- search_args = mail_search_build_init();
- mail_search_build_add_all(search_args);
- arg = mail_search_build_add(search_args, SEARCH_BODY);
- arg->value.str = "xyzzy";
+ if ((ret = fts_build_init(flist->backend, box,
+ TRUE, &build_ctx)) <= 0) {
+ if (box->storage->set->mail_debug)
+ i_debug("%s: FTS index is up to date", box->vname);
+ return ret;
+ }
- ctx = mailbox_search_init(t, search_args, NULL, 0, NULL);
- mail_search_args_unref(&search_args);
+ if (box->storage->set->mail_debug)
+ i_debug("%s: Updating FTS index", box->vname);
- fctx = FTS_CONTEXT(ctx);
- if (fctx->build_initialized) {
- while ((ret = fts_mailbox_search_build_more(ctx)) == 0) ;
- }
- (void)mailbox_search_deinit(&ctx);
- (void)mailbox_transaction_commit(&t);
+ while ((ret = fts_build_more(build_ctx)) == 0) ;
+ if (fts_build_deinit(&build_ctx) < 0)
+ ret = -1;
return ret < 0 ? -1 : 0;
}
return -1;
ctx = NULL;
- return !precache ? 0 : fts_update(box);
+ if (precache) {
+ if (fts_update(box) < 0) {
+ mail_storage_set_critical(box->storage,
+ "FTS index update for mailbox %s failed",
+ box->vname);
+ return -1;
+ }
+ }
+ return 0;
}
-static void fts_mailbox_init(struct mailbox *box, const char *env)
+void fts_mailbox_allocated(struct mailbox *box)
{
+ struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
struct mailbox_vfuncs *v = box->vlast;
struct fts_mailbox *fbox;
- fbox = i_new(struct fts_mailbox, 1);
- fbox->virtual = strcmp(box->storage->name, "virtual") == 0;
- fbox->env = env;
+ if (flist == NULL)
+ return;
+
+ fbox = p_new(box->pool, struct fts_mailbox, 1);
fbox->module_ctx.super = *v;
box->vlast = &fbox->module_ctx.super;
- v->free = fts_mailbox_free;
v->search_init = fts_mailbox_search_init;
v->search_next_nonblock = fts_mailbox_search_next_nonblock;
- v->search_next_update_seq = fbox->virtual ?
- fts_mailbox_search_next_update_seq_virtual :
- fts_mailbox_search_next_update_seq;
+ v->search_next_update_seq = fts_mailbox_search_next_update_seq;
v->search_deinit = fts_mailbox_search_deinit;
v->transaction_begin = fts_transaction_begin;
v->transaction_rollback = fts_transaction_rollback;
v->transaction_commit = fts_transaction_commit;
+ v->sync_notify = fts_mailbox_sync_notify;
v->sync_deinit = fts_sync_deinit;
MODULE_CONTEXT_SET(box, fts_storage_module, fbox);
}
-void fts_mailbox_allocated(struct mailbox *box)
+static void fts_mailbox_list_deinit(struct mailbox_list *list)
+{
+ struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(list);
+
+ fts_backend_deinit(&flist->backend);
+ flist->module_ctx.super.deinit(list);
+}
+
+void fts_mailbox_list_created(struct mailbox_list *list)
+{
+ struct fts_backend *backend;
+ const char *name, *path, *error;
+
+ name = mail_user_plugin_getenv(list->ns->user, "fts");
+ if (name == NULL) {
+ if (list->mail_set->mail_debug)
+ i_debug("fts: No fts setting - plugin disabled");
+ return;
+ }
+
+ path = mailbox_list_get_path(list, NULL,
+ MAILBOX_LIST_PATH_TYPE_INDEX);
+ if (*path == '\0') {
+ if (list->mail_set->mail_debug) {
+ i_debug("fts: Indexes disabled for namespace '%s'",
+ list->ns->prefix);
+ }
+ return;
+ }
+
+ if (fts_backend_init(name, list->ns, &error, &backend) < 0) {
+ i_error("fts: Failed to initialize backend '%s': %s",
+ name, error);
+ } else {
+ struct fts_mailbox_list *flist;
+ struct mailbox_list_vfuncs *v = list->vlast;
+
+ flist = p_new(list->pool, struct fts_mailbox_list, 1);
+ flist->module_ctx.super = *v;
+ flist->backend = backend;
+ list->vlast = &flist->module_ctx.super;
+ v->deinit = fts_mailbox_list_deinit;
+ MODULE_CONTEXT_SET(list, fts_mailbox_list_module, flist);
+ }
+}
+
+struct fts_backend *fts_mailbox_backend(struct mailbox *box)
{
- const char *env;
+ struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
- env = mail_user_plugin_getenv(box->storage->user, "fts");
- if (env != NULL)
- fts_mailbox_init(box, env);
+ return flist->backend;
}
#ifndef FTS_STORAGE_H
#define FTS_STORAGE_H
-struct fts_mailbox {
- union mailbox_module_context module_ctx;
- struct fts_backend *backend;
-
- unsigned int last_messages_count, last_uidnext;
+#include "mail-storage-private.h"
+#include "fts-api.h"
- const char *env;
- unsigned int virtual:1;
- unsigned int backend_set:1;
-};
-
-struct fts_orig_mailboxes {
- const char *name;
- struct mail_namespace *ns;
- struct mailbox *box;
+struct fts_scores {
+ int refcount;
+ ARRAY_TYPE(fts_score_map) score_map;
};
-struct fts_search_virtual_context {
- pool_t pool;
-
- struct mailbox_transaction_context *trans;
- ARRAY_DEFINE(orig_mailboxes, struct fts_orig_mailboxes);
- ARRAY_TYPE(fts_backend_uid_map) last_uids;
-
- unsigned int boxi, uidi;
+struct fts_search_level {
+ ARRAY_TYPE(seq_range) definite_seqs, maybe_seqs;
+ buffer_t *args_matches;
+ ARRAY_TYPE(fts_score_map) score_map;
};
struct fts_search_context {
union mail_search_module_context module_ctx;
- struct fts_mailbox *fbox;
+ struct fts_backend *backend;
+ struct mailbox *box;
struct mailbox_transaction_context *t;
struct mail_search_args *args;
- struct mail_search_arg *best_arg;
- struct fts_backend_lookup_context *lookup_ctx;
- ARRAY_TYPE(seq_range) definite_seqs, maybe_seqs;
- ARRAY_TYPE(fts_score_map) score_map;
- unsigned int definite_idx, maybe_idx;
- uint32_t first_nonindexed_seq;
+ pool_t result_pool;
+ ARRAY_DEFINE(levels, struct fts_search_level);
+ buffer_t *orig_matches;
+
+ uint32_t first_unindexed_seq;
+
+ /* final scores, combined from all levels */
+ struct fts_scores *scores;
struct fts_storage_build_context *build_ctx;
- struct fts_search_virtual_context virtual_ctx;
+ unsigned int virtual_mailbox:1;
unsigned int build_initialized:1;
- unsigned int seqs_set:1;
- unsigned int refreshed:1;
+ unsigned int fts_lookup_success:1;
};
/* Figure out if we want to use full text search indexes and update
void fts_search_analyze(struct fts_search_context *fctx);
/* Perform the actual index lookup and update definite_uids and maybe_uids. */
void fts_search_lookup(struct fts_search_context *fctx);
+/* Returns FTS backend for the given mailbox. */
+struct fts_backend *fts_mailbox_backend(struct mailbox *box);
+
+void fts_mail_allocated(struct mail *mail);
+void fts_mailbox_allocated(struct mailbox *box);
+void fts_mailbox_list_created(struct mailbox_list *list);
#endif
VIRTUAL_INDEX_PREFIX);
mbox->storage = storage;
- mbox->vseq_lookup_prev_mailbox = i_strdup("");
-
- mbox->virtual_ext_id =
- mail_index_ext_register(mbox->box.index, "virtual", 0,
- sizeof(struct virtual_mail_index_record),
- sizeof(uint32_t));
+ mbox->virtual_ext_id = (uint32_t)-1;
return &mbox->box;
}
array_free(&bboxes[i]->sync_pending_removes);
array_free(&bboxes[i]->uids);
}
- i_free_and_null(mbox->vseq_lookup_prev_mailbox);
}
static int virtual_mailbox_exists(struct mailbox *box,
virtual_mailbox_close_internal(mbox);
return -1;
}
- return index_storage_mailbox_open(box, FALSE);
+ if (index_storage_mailbox_open(box, FALSE) < 0)
+ return -1;
+
+ mbox->virtual_ext_id =
+ mail_index_ext_register(mbox->box.index, "virtual", 0,
+ sizeof(struct virtual_mail_index_record),
+ sizeof(uint32_t));
+ return 0;
}
static void virtual_mailbox_close(struct mailbox *box)
}
}
-static int virtual_backend_uidmap_cmp(const uint32_t *uid,
- const struct virtual_backend_uidmap *map)
+static void
+virtual_get_virtual_uids(struct mailbox *box,
+ struct mailbox *backend_mailbox,
+ const ARRAY_TYPE(seq_range) *backend_uids,
+ ARRAY_TYPE(seq_range) *virtual_uids_r)
{
- return *uid < map->real_uid ? -1 :
- *uid > map->real_uid ? 1 : 0;
+ struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
+ struct virtual_backend_box *bbox;
+ const struct virtual_backend_uidmap *uids;
+ struct seq_range_iter iter;
+ unsigned int n, i, count;
+ uint32_t uid;
+
+ if (mbox->lookup_prev_bbox != NULL &&
+ strcmp(mbox->lookup_prev_bbox->box->vname, backend_mailbox->vname) == 0)
+ bbox = mbox->lookup_prev_bbox;
+ else {
+ bbox = virtual_backend_box_lookup_name(mbox, backend_mailbox->vname);
+ mbox->lookup_prev_bbox = bbox;
+ }
+ if (bbox == NULL)
+ return;
+
+ uids = array_get(&bbox->uids, &count); i = 0;
+ seq_range_array_iter_init(&iter, backend_uids); n = 0;
+ while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+ while (i < count && uids[i].real_uid < uid) i++;
+ if (i < count && uids[i].real_uid == uid) {
+ seq_range_array_add(virtual_uids_r, 0,
+ uids[i].virtual_uid);
+ i++;
+ }
+ }
}
-static bool
-virtual_get_virtual_uid(struct mailbox *box, const char *backend_mailbox,
- uint32_t backend_uidvalidity,
- uint32_t backend_uid, uint32_t *uid_r)
+static void
+virtual_get_virtual_uid_map(struct mailbox *box,
+ struct mailbox *backend_mailbox,
+ const ARRAY_TYPE(seq_range) *backend_uids,
+ ARRAY_TYPE(uint32_t) *virtual_uids_r)
{
struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
struct virtual_backend_box *bbox;
- struct mailbox_status status;
const struct virtual_backend_uidmap *uids;
+ struct seq_range_iter iter;
+ unsigned int n, i, count;
+ uint32_t uid;
- if (strcmp(mbox->vseq_lookup_prev_mailbox, backend_mailbox) == 0)
- bbox = mbox->vseq_lookup_prev_bbox;
+ if (mbox->lookup_prev_bbox != NULL &&
+ strcmp(mbox->lookup_prev_bbox->box->vname, backend_mailbox->vname) == 0)
+ bbox = mbox->lookup_prev_bbox;
else {
- i_free(mbox->vseq_lookup_prev_mailbox);
- mbox->vseq_lookup_prev_mailbox = i_strdup(backend_mailbox);
-
- bbox = virtual_backend_box_lookup_name(mbox, backend_mailbox);
- mbox->vseq_lookup_prev_bbox = bbox;
+ bbox = virtual_backend_box_lookup_name(mbox, backend_mailbox->vname);
+ mbox->lookup_prev_bbox = bbox;
}
if (bbox == NULL)
- return FALSE;
-
- mailbox_get_open_status(bbox->box, STATUS_UIDVALIDITY, &status);
- if (status.uidvalidity != backend_uidvalidity)
- return FALSE;
+ return;
- uids = array_bsearch(&bbox->uids, &backend_uid,
- virtual_backend_uidmap_cmp);
- if (uids == NULL)
- return FALSE;
+ uids = array_get(&bbox->uids, &count); i = 0;
+ seq_range_array_iter_init(&iter, backend_uids); n = 0;
+ while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+ while (i < count && uids[i].real_uid < uid) i++;
+ if (i == count || uids[i].real_uid > uid) {
+ uint32_t zero = 0;
- *uid_r = uids->virtual_uid;
- return TRUE;
+ array_append(virtual_uids_r, &zero, 1);
+ } else {
+ array_append(virtual_uids_r, &uids[i].virtual_uid, 1);
+ i++;
+ }
+ }
}
static void
}
}
-static void
-virtual_get_virtual_box_patterns(struct mailbox *box,
- ARRAY_TYPE(mailbox_virtual_patterns) *includes,
- ARRAY_TYPE(mailbox_virtual_patterns) *excludes)
-{
- struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
-
- array_append_array(includes, &mbox->list_include_patterns);
- array_append_array(excludes, &mbox->list_exclude_patterns);
-}
-
static bool virtual_is_inconsistent(struct mailbox *box)
{
struct virtual_mailbox *mbox = (struct virtual_mailbox *)box;
};
struct virtual_mailbox_vfuncs virtual_mailbox_vfuncs = {
- virtual_get_virtual_uid,
- virtual_get_virtual_backend_boxes,
- virtual_get_virtual_box_patterns
+ virtual_get_virtual_uids,
+ virtual_get_virtual_uid_map,
+ virtual_get_virtual_backend_boxes
};
ARRAY_DEFINE_TYPE(virtual_backend_box, struct virtual_backend_box *);
struct virtual_mailbox_vfuncs {
- bool (*get_virtual_uid)(struct mailbox *box,
- const char *backend_mailbox,
- uint32_t backend_uidvalidity,
- uint32_t backend_uid, uint32_t *uid_r);
+ /* convert backend UIDs to virtual UIDs. if some backend UID doesn't
+ exist in mailbox, it's simply ignored */
+ void (*get_virtual_uids)(struct mailbox *box,
+ struct mailbox *backend_mailbox,
+ const ARRAY_TYPE(seq_range) *backend_uids,
+ ARRAY_TYPE(seq_range) *virtual_uids_r);
+ /* like get_virtual_uids(), but if a backend UID doesn't exist,
+ convert it to 0. */
+ void (*get_virtual_uid_map)(struct mailbox *box,
+ struct mailbox *backend_mailbox,
+ const ARRAY_TYPE(seq_range) *backend_uids,
+ ARRAY_TYPE(uint32_t) *virtual_uids_r);
void (*get_virtual_backend_boxes)(struct mailbox *box,
ARRAY_TYPE(mailboxes) *mailboxes,
bool only_with_msgs);
- void (*get_virtual_box_patterns)(struct mailbox *box,
- ARRAY_TYPE(mailbox_virtual_patterns) *includes,
- ARRAY_TYPE(mailbox_virtual_patterns) *excludes);
};
struct virtual_mailbox {
uint32_t highest_mailbox_id;
uint32_t search_args_crc32;
- char *vseq_lookup_prev_mailbox;
- struct virtual_backend_box *vseq_lookup_prev_bbox;
+ struct virtual_backend_box *lookup_prev_bbox;
uint32_t sync_virtual_next_uid;
/* Mailboxes this virtual mailbox consists of, sorted by mailbox_id */
projects / thirdparty / dovecot / core.git / commitdiff
