lib-fts: Added normalizer-simple for doing normalization without libicu.

diff --git a/src/lib-fts/Makefile.am b/src/lib-fts/Makefile.am
index 80960afe6783ba11d7988adf83e1fc2f0666abb7..c3b485555682ed1360fa1f961ddbd50e7e1b1a32 100644 (file)
--- a/src/lib-fts/Makefile.am
+++ b/src/lib-fts/Makefile.am
@@ -61,6 +61,7 @@ libfts_la_LIBADD = \
 libfts_la_SOURCES = \
        fts-filter.c \
        fts-filter-normalizer-icu.c \
+       fts-filter-normalizer-simple.c \
        fts-filter-stopwords.c \
        fts-filter-stemmer-snowball.c \
        fts-language.c \
@@ -88,13 +89,9 @@ test_libs = \
        ../lib/liblib.la
 test_deps = $(noinst_LTLIBRARIES) $(test_libs)
 
-filter_deps = \
-       fts-filter.lo fts-filter-stopwords.lo \
-       fts-filter-stemmer-snowball.lo fts-filter-normalizer-icu.lo
-
 test_fts_filter_SOURCES = test-fts-filter.c
-test_fts_filter_LDADD = $(filter_deps) $(test_libs) $(STEMMER_LIBS) $(TEXTCAT_LIBS) $(NORMALIZER_LIBS)
-test_fts_filter_DEPENDENCIES = $(test_deps) $(filter_deps)
+test_fts_filter_LDADD = libfts.la $(test_libs)
+test_fts_filter_DEPENDENCIES = libfts.la $(test_deps)
 
 if BUILD_FTS_EXTTEXTCAT
 TEST_FTS_LANGUAGE = test-fts-language

diff --git a/src/lib-fts/fts-filter-normalizer-simple.c b/src/lib-fts/fts-filter-normalizer-simple.c
new file mode 100644 (file)
index 0000000..7cff5ea
--- /dev/null
+++ b/src/lib-fts/fts-filter-normalizer-simple.c
@@ -0,0 +1,77 @@
+/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "unichar.h"
+#include "str.h"
+#include "fts-filter.h"
+#include "fts-filter-private.h"
+#include "fts-language.h"
+
+struct fts_filter_normalizer_simple {
+       struct fts_filter filter;
+       string_t *str;
+};
+
+static bool
+fts_filter_normalizer_simple_supports(const struct fts_language *lang ATTR_UNUSED)
+{
+       return TRUE;
+}
+
+static void
+fts_filter_normalizer_simple_destroy(struct fts_filter *_filter)
+{
+       struct fts_filter_normalizer_simple *filter =
+               (struct fts_filter_normalizer_simple *)_filter;
+
+       str_free(&filter->str);
+       i_free(filter);
+}
+
+static int
+fts_filter_normalizer_simple_create(const struct fts_language *lang ATTR_UNUSED,
+                                   const char *const *settings,
+                                   struct fts_filter **filter_r,
+                                   const char **error_r)
+{
+       struct fts_filter_normalizer_simple *filter;
+
+       if (settings[0] != NULL) {
+               *error_r = t_strdup_printf("Unknown setting: %s", settings[0]);
+               return -1;
+       }
+       filter = i_new(struct fts_filter_normalizer_simple, 1);
+       filter->filter = *fts_filter_normalizer_simple;
+       filter->str = str_new(default_pool, 128);
+
+       *filter_r = &filter->filter;
+       return 0;
+}
+
+static const char *
+fts_filter_normalizer_simple_filter(struct fts_filter *_filter,
+                                   const char *token)
+{
+       struct fts_filter_normalizer_simple *filter =
+               (struct fts_filter_normalizer_simple *)_filter;
+
+       str_truncate(filter->str, 0);
+       if (uni_utf8_to_decomposed_titlecase(token, strlen(token),
+                                            filter->str) < 0)
+               return NULL;
+       return str_c(filter->str);
+}
+
+static const struct fts_filter_vfuncs normalizer_filter_vfuncs = {
+       fts_filter_normalizer_simple_supports,
+       fts_filter_normalizer_simple_create,
+       fts_filter_normalizer_simple_filter,
+       fts_filter_normalizer_simple_destroy
+};
+
+static const struct fts_filter fts_filter_normalizer_simple_real = {
+       .class_name = SIMPLE_NORMALIZER_FILTER_NAME,
+       .v = &normalizer_filter_vfuncs
+};
+
+const struct fts_filter *fts_filter_normalizer_simple = &fts_filter_normalizer_simple_real;

diff --git a/src/lib-fts/fts-filter.c b/src/lib-fts/fts-filter.c
index 08b9b55d8e544eadd28dd1d89308c8d4afb1a1da..6cb6e0205954cd6c172f161590ade51654acbaee 100644 (file)
--- a/src/lib-fts/fts-filter.c
+++ b/src/lib-fts/fts-filter.c
@@ -15,6 +15,7 @@ void fts_filters_init(void)
        fts_filter_register(fts_filter_stopwords);
        fts_filter_register(fts_filter_stemmer_snowball);
        fts_filter_register(fts_filter_normalizer_icu);
+       fts_filter_register(fts_filter_normalizer_simple);
 }
 
 void fts_filters_deinit(void)

diff --git a/src/lib-fts/fts-filter.h b/src/lib-fts/fts-filter.h
index 1628d349ecf41f92de6f037cf9a54162197b76ba..4ccd50dc317a3246eef646a8c5d31864527ce8cf 100644 (file)
--- a/src/lib-fts/fts-filter.h
+++ b/src/lib-fts/fts-filter.h
@@ -33,6 +33,10 @@ extern const struct fts_filter *fts_filter_stemmer_snowball;
 extern const struct fts_filter *fts_filter_normalizer_icu;
 #define ICU_NORMALIZER_FILTER_NAME "normalizer-icu"
 
+/* Normalization using i;unicode-casemap (RFC 5051) */
+extern const struct fts_filter *fts_filter_normalizer_simple;
+#define SIMPLE_NORMALIZER_FILTER_NAME "normalizer-simple"
+
 /* Register all built-in filters. */
 void fts_filters_init(void);
 void fts_filters_deinit(void);