fts-lucene: Automatically delete corrupted indexes and mark all mailboxes unindexed.

diff --git a/src/plugins/fts-lucene/doveadm-fts-lucene.c b/src/plugins/fts-lucene/doveadm-fts-lucene.c
index d5ed91addc532f00661a9b5e5105ad6df7733a89..c59c3c6399c7595ae9d291662964610aaa66cd5e 100644 (file)
--- a/src/plugins/fts-lucene/doveadm-fts-lucene.c
+++ b/src/plugins/fts-lucene/doveadm-fts-lucene.c
@@ -23,7 +23,7 @@ static void cmd_dump_fts_lucene(int argc ATTR_UNUSED, char *argv[])
        bool first = TRUE;
 
        memset(&prev_guid, 0, sizeof(prev_guid));
-       index = lucene_index_init(argv[1], NULL, NULL);
+       index = lucene_index_init(argv[1], NULL, NULL, NULL);
        iter = lucene_index_iter_init(index);
        while ((rec = lucene_index_iter_next(iter)) != NULL) {
                if (memcmp(prev_guid, rec->mailbox_guid,

diff --git a/src/plugins/fts-lucene/fts-backend-lucene.c b/src/plugins/fts-lucene/fts-backend-lucene.c
index 757a8d6fa08ef700ecc0a817ebfc750d2db8b6c4..463f23d31a754b71eeeea8b10d883b33b329aa73 100644 (file)
--- a/src/plugins/fts-lucene/fts-backend-lucene.c
+++ b/src/plugins/fts-lucene/fts-backend-lucene.c
@@ -136,10 +136,12 @@ fts_backend_lucene_init(struct fts_backend *_backend,
        backend->dir_path = i_strconcat(path, "/"LUCENE_INDEX_DIR_NAME, NULL);
        if (fuser != NULL) {
                backend->index = lucene_index_init(backend->dir_path,
+                                                  _backend->ns->list,
                                                   fuser->set.textcat_dir,
                                                   fuser->set.textcat_conf);
        } else {
                backend->index = lucene_index_init(backend->dir_path,
+                                                  _backend->ns->list,
                                                   NULL, NULL);
        }
 
@@ -403,7 +405,7 @@ static int fts_backend_lucene_rescan(struct fts_backend *_backend)
        struct lucene_fts_backend *backend =
                (struct lucene_fts_backend *)_backend;
 
-       if (lucene_index_rescan(backend->index, _backend->ns->list) < 0)
+       if (lucene_index_rescan(backend->index) < 0)
                return -1;
        return lucene_index_optimize(backend->index);
 }
@@ -418,7 +420,7 @@ static int fts_backend_lucene_optimize(struct fts_backend *_backend)
                                            backend->expunge_log);
        if (ret == 0) {
                /* log was corrupted, need to rescan */
-               ret = lucene_index_rescan(backend->index, _backend->ns->list);
+               ret = lucene_index_rescan(backend->index);
        }
        if (ret >= 0)
                ret = lucene_index_optimize(backend->index);

diff --git a/src/plugins/fts-lucene/lucene-wrapper.cc b/src/plugins/fts-lucene/lucene-wrapper.cc
index 3b62c3ceebf45cb4d36bad4c83fea357b125a38b..8ee46da7b2cca5d4b888de3c04f730484bf19a20 100644 (file)
--- a/src/plugins/fts-lucene/lucene-wrapper.cc
+++ b/src/plugins/fts-lucene/lucene-wrapper.cc
@@ -6,6 +6,7 @@ extern "C" {
 #include "unichar.h"
 #include "hash.h"
 #include "hex-binary.h"
+#include "unlink-directory.h"
 #include "mail-index.h"
 #include "mail-search.h"
 #include "mail-namespace.h"
@@ -46,6 +47,8 @@ struct lucene_analyzer {
 
 struct lucene_index {
        char *path;
+       struct mailbox_list *list;
+
        char *textcat_dir, *textcat_conf;
        wchar_t mailbox_guid[MAILBOX_GUID_HEX_LENGTH + 1];
 
@@ -62,7 +65,6 @@ struct lucene_index {
 
 struct rescan_context {
        struct lucene_index *index;
-       struct mailbox_list *list;
 
        struct mailbox *box;
        mail_guid_128_t box_guid;
@@ -83,7 +85,11 @@ static void *textcat = NULL;
 static bool textcat_broken = FALSE;
 static int textcat_refcount = 0;
 
+static void rescan_clear_unseen_mailboxes(struct mailbox_list *list,
+                                         struct hash_table *guids);
+
 struct lucene_index *lucene_index_init(const char *path,
+                                      struct mailbox_list *list,
                                       const char *textcat_dir,
                                       const char *textcat_conf)
 {
@@ -91,6 +97,7 @@ struct lucene_index *lucene_index_init(const char *path,
 
        index = i_new(struct lucene_index, 1);
        index->path = i_strdup(path);
+       index->list = list;
        index->textcat_dir = i_strdup(textcat_dir);
        index->textcat_conf = i_strdup(textcat_conf);
 #ifdef HAVE_LUCENE_TEXTCAT
@@ -183,7 +190,18 @@ static void lucene_handle_error(struct lucene_index *index, CLuceneError &err,
 {
        const char *what = err.what();
 
-       i_error("lucene index %s: %s failed: %s", index->path, msg, what);
+       i_error("lucene index %s: %s failed (#%d): %s",
+               index->path, msg, err.number(), what);
+
+       if (index->list != NULL &&
+           (err.number() == CL_ERR_CorruptIndex ||
+            err.number() == CL_ERR_IO)) {
+               /* delete corrupted index. most IO errors are also about
+                  missing files and other such corruption.. */
+               if (unlink_directory(index->path, TRUE) < 0 && errno != ENOENT)
+                       i_error("unlink_directory(%s) failed: %m", index->path);
+               rescan_clear_unseen_mailboxes(index->list, NULL);
+       }
 }
 
 static int lucene_index_open(struct lucene_index *index)
@@ -560,7 +578,8 @@ rescan_open_mailbox(struct rescan_context *ctx, Document *doc)
 
        if (ctx->box != NULL)
                rescan_finish(ctx);
-       ctx->box = mailbox_alloc_guid(ctx->list, guid, MAILBOX_FLAG_KEEP_RECENT);
+       ctx->box = mailbox_alloc_guid(ctx->index->list,
+                                     guid, MAILBOX_FLAG_KEEP_RECENT);
        if (mailbox_open(ctx->box) < 0) {
                enum mail_error error;
                const char *errstr;
@@ -634,7 +653,8 @@ rescan_next(struct rescan_context *ctx, Document *doc)
        }
 }
 
-static void rescan_clear_unseen_mailboxes(struct rescan_context *ctx)
+static void rescan_clear_unseen_mailboxes(struct mailbox_list *list,
+                                         struct hash_table *guids)
 {
        const enum mailbox_list_iter_flags iter_flags =
                (enum mailbox_list_iter_flags)
@@ -645,13 +665,14 @@ static void rescan_clear_unseen_mailboxes(struct rescan_context *ctx)
        struct mailbox *box;
        struct mailbox_metadata metadata;
 
-       iter = mailbox_list_iter_init(ctx->list, "*", iter_flags);
+       iter = mailbox_list_iter_init(list, "*", iter_flags);
        while ((info = mailbox_list_iter_next(iter)) != NULL) {
-               box = mailbox_alloc(ctx->list, info->name,
+               box = mailbox_alloc(list, info->name,
                                    MAILBOX_FLAG_KEEP_RECENT);
                if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
                                         &metadata) == 0 &&
-                   hash_table_lookup(ctx->guids, metadata.guid) == NULL) {
+                   (guids == NULL ||
+                    hash_table_lookup(guids, metadata.guid) == NULL)) {
                        /* this mailbox had no records in lucene index.
                           make sure its last indexed uid is 0 */
                        (void)fts_index_set_last_uid(box, 0);
@@ -661,8 +682,7 @@ static void rescan_clear_unseen_mailboxes(struct rescan_context *ctx)
        (void)mailbox_list_iter_deinit(&iter);
 }
 
-int lucene_index_rescan(struct lucene_index *index,
-                       struct mailbox_list *list)
+int lucene_index_rescan(struct lucene_index *index)
 {
        static const TCHAR *sort_fields[] = { _T("box"), _T("uid"), NULL };
        struct rescan_context ctx;
@@ -670,6 +690,8 @@ int lucene_index_rescan(struct lucene_index *index,
        bool failed = false;
        int ret;
 
+       i_assert(index->list != NULL);
+
        if ((ret = lucene_index_open_search(index)) < 0)
                return ret;
 
@@ -679,7 +701,6 @@ int lucene_index_rescan(struct lucene_index *index,
 
        memset(&ctx, 0, sizeof(ctx));
        ctx.index = index;
-       ctx.list = list;
        ctx.pool = pool_alloconly_create("guids", 1024);
        ctx.guids = hash_table_create(default_pool, ctx.pool, 0,
                                      mail_guid_128_hash,
@@ -709,7 +730,7 @@ int lucene_index_rescan(struct lucene_index *index,
                rescan_finish(&ctx);
        array_free(&ctx.uids);
 
-       rescan_clear_unseen_mailboxes(&ctx);
+       rescan_clear_unseen_mailboxes(index->list, ctx.guids);
        hash_table_destroy(&ctx.guids);
        pool_unref(&ctx.pool);
        return failed ? -1 : 0;

diff --git a/src/plugins/fts-lucene/lucene-wrapper.h b/src/plugins/fts-lucene/lucene-wrapper.h
index a4108e87b80a738ab962bd90bf92ba5248bce280..749cfbd872455a53e8c4d30a7acf70ca24f98016 100644 (file)
--- a/src/plugins/fts-lucene/lucene-wrapper.h
+++ b/src/plugins/fts-lucene/lucene-wrapper.h
@@ -16,6 +16,7 @@ struct lucene_index_record {
 };
 
 struct lucene_index *lucene_index_init(const char *path,
+                                      struct mailbox_list *list,
                                       const char *textcat_dir,
                                       const char *textcat_conf);
 void lucene_index_deinit(struct lucene_index *index);
@@ -33,8 +34,7 @@ int lucene_index_build_more(struct lucene_index *index, uint32_t uid,
 int lucene_index_build_deinit(struct lucene_index *index);
 
 void lucene_index_close(struct lucene_index *index);
-int lucene_index_rescan(struct lucene_index *index,
-                       struct mailbox_list *list);
+int lucene_index_rescan(struct lucene_index *index);
 int lucene_index_expunge_from_log(struct lucene_index *index,
                                  struct fts_expunge_log *log);
 int lucene_index_optimize(struct lucene_index *index);