fts-lucene: Added whitespace_chars subsetting to fts_lucene.

A value of "@." could be useful so that user@domain.tld allows searching
user, domain and tld separately instead of requiring the whole string to
match.

diff --git a/src/plugins/fts-lucene/fts-lucene-plugin.c b/src/plugins/fts-lucene/fts-lucene-plugin.c
index 23fa85af3bd76aba103907e71128361a9f3408c5..f5e0ebcf1a1726cd1fadc577a2de95d472ab9d2e 100644 (file)
--- a/src/plugins/fts-lucene/fts-lucene-plugin.c
+++ b/src/plugins/fts-lucene/fts-lucene-plugin.c
@@ -26,6 +26,8 @@ fts_lucene_plugin_init_settings(struct mail_user *user,
                        set->textcat_conf = p_strdup(user->pool, *tmp + 13);
                } else if (strncmp(*tmp, "textcat_dir=", 12) == 0) {
                        set->textcat_dir = p_strdup(user->pool, *tmp + 12);
+               } else if (strncmp(*tmp, "whitespace_chars=", 17) == 0) {
+                       set->whitespace_chars = p_strdup(user->pool, *tmp + 17);
                } else {
                        i_error("fts_lucene: Invalid setting: %s", *tmp);
                        return -1;
@@ -39,6 +41,8 @@ fts_lucene_plugin_init_settings(struct mail_user *user,
                i_error("fts_lucene: textcat_dir set, but textcat_conf unset");
                return -1;
        }
+       if (set->whitespace_chars == NULL)
+               set->whitespace_chars = "";
 #ifndef HAVE_LUCENE_STEMMER
        if (set->default_language != NULL) {
                i_error("fts_lucene: default_language set, "
@@ -61,9 +65,11 @@ fts_lucene_plugin_init_settings(struct mail_user *user,
 
 uint32_t fts_lucene_settings_checksum(const struct fts_lucene_settings *set)
 {
-       /* only the default language change matters */
-       return set->default_language == NULL ? 0 :
-               crc32_str(set->default_language);
+       uint32_t crc;
+
+       crc = crc32_str(set->default_language);
+       crc = crc32_str_more(crc, set->whitespace_chars);
+       return crc;
 }
 
 static void fts_lucene_mail_user_created(struct mail_user *user)

diff --git a/src/plugins/fts-lucene/fts-lucene-plugin.h b/src/plugins/fts-lucene/fts-lucene-plugin.h
index 9d70eb96110f6e18a25b0131b0d6fc567ce81be8..7dfc40156fd34a8954542fcef65f135914948527 100644 (file)
--- a/src/plugins/fts-lucene/fts-lucene-plugin.h
+++ b/src/plugins/fts-lucene/fts-lucene-plugin.h
@@ -11,6 +11,7 @@
 struct fts_lucene_settings {
        const char *default_language;
        const char *textcat_conf, *textcat_dir;
+       const char *whitespace_chars;
 };
 
 struct fts_lucene_user {

diff --git a/src/plugins/fts-lucene/lucene-wrapper.cc b/src/plugins/fts-lucene/lucene-wrapper.cc
index 8f5bd4ba425d8fe7894295c0f0c068845f6b87d1..da02595245faa123739d7c5ff1f80616fd6d1bd1 100644 (file)
--- a/src/plugins/fts-lucene/lucene-wrapper.cc
+++ b/src/plugins/fts-lucene/lucene-wrapper.cc
@@ -143,6 +143,21 @@ void lucene_index_deinit(struct lucene_index *index)
        i_free(index);
 }
 
+static void lucene_data_translate(struct lucene_index *index,
+                                 wchar_t *data, unsigned int len)
+{
+       const char *whitespace_chars = index->set.whitespace_chars;
+       unsigned int i;
+
+       if (*whitespace_chars == '\0')
+               return;
+
+       for (i = 0; i < len; i++) {
+               if (strchr(whitespace_chars, data[i]) != NULL)
+                       data[i] = ' ';
+       }
+}
+
 void lucene_utf8_n_to_tchar(const unsigned char *src, size_t srcsize,
                            wchar_t *dest, size_t destsize)
 {
@@ -159,10 +174,14 @@ void lucene_utf8_n_to_tchar(const unsigned char *src, size_t srcsize,
        dest[destsize-1] = 0;
 }
 
-static const wchar_t *t_lucene_utf8_to_tchar(const char *str)
+static const wchar_t *
+t_lucene_utf8_to_tchar(struct lucene_index *index,
+                      const char *str, bool translate)
 {
        ARRAY_TYPE(unichars) dest_arr;
-       const unichar_t *ret;
+       const unichar_t *chars;
+       wchar_t *ret;
+       unsigned int len;
 
        i_assert(sizeof(wchar_t) == sizeof(unichar_t));
 
@@ -170,8 +189,11 @@ static const wchar_t *t_lucene_utf8_to_tchar(const char *str)
        if (uni_utf8_to_ucs4(str, &dest_arr) < 0)
                i_unreached();
        (void)array_append_space(&dest_arr);
-       ret = array_idx(&dest_arr, 0);
-       return (const wchar_t *)ret;
+
+       chars = array_get_modifiable(&dest_arr, &len);
+       ret = (wchar_t *)chars;
+       lucene_data_translate(index, ret, len - 1);
+       return ret;
 }
 
 void lucene_index_select_mailbox(struct lucene_index *index,
@@ -478,6 +500,7 @@ int lucene_index_build_more(struct lucene_index *index, uint32_t uid,
        datasize = uni_utf8_strlen_n(data, size) + 1;
        wchar_t dest[datasize];
        lucene_utf8_n_to_tchar(data, size, dest, datasize);
+       lucene_data_translate(index, dest, datasize);
 
        if (hdr_name != NULL) {
                /* hdr_name should be ASCII, but don't break in case it isn't */
@@ -1010,7 +1033,7 @@ static Query *
 lucene_get_query_str(struct lucene_index *index,
                     const TCHAR *key, const char *str, bool fuzzy)
 {
-       const TCHAR *wvalue = t_lucene_utf8_to_tchar(str);
+       const TCHAR *wvalue = t_lucene_utf8_to_tchar(index, str, TRUE);
        Analyzer *analyzer = guess_analyzer(index, str, strlen(str));
        if (analyzer == NULL)
                analyzer = index->default_analyzer;
@@ -1067,7 +1090,7 @@ lucene_add_definite_query(struct lucene_index *index, BooleanQuery &query,
                }
 
                q = lucene_get_query(index,
-                                    t_lucene_utf8_to_tchar(arg->hdr_field_name),
+                                    t_lucene_utf8_to_tchar(index, arg->hdr_field_name, FALSE),
                                     arg);
                break;
        default: