From: Timo Sirainen <tss@iki.fi>
Date: Tue, 21 Apr 2015 17:16:36 +0000 (+0300)
Subject: lib-fts: Added normalizer-simple for doing normalization without libicu.
X-Git-Tag: 2.2.17.rc1~179
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ec930ce90b17fb63ff035c1c87d994800de092f1;p=thirdparty%2Fdovecot%2Fcore.git

lib-fts: Added normalizer-simple for doing normalization without libicu.
---

diff --git a/src/lib-fts/Makefile.am b/src/lib-fts/Makefile.am
index 80960afe67..c3b4855556 100644
--- a/src/lib-fts/Makefile.am
+++ b/src/lib-fts/Makefile.am
@@ -61,6 +61,7 @@ libfts_la_LIBADD = \
 libfts_la_SOURCES = \
 	fts-filter.c \
 	fts-filter-normalizer-icu.c \
+	fts-filter-normalizer-simple.c \
 	fts-filter-stopwords.c \
 	fts-filter-stemmer-snowball.c \
 	fts-language.c \
@@ -88,13 +89,9 @@ test_libs = \
 	../lib/liblib.la
 test_deps = $(noinst_LTLIBRARIES) $(test_libs)
 
-filter_deps = \
-	fts-filter.lo fts-filter-stopwords.lo \
-	fts-filter-stemmer-snowball.lo fts-filter-normalizer-icu.lo
-
 test_fts_filter_SOURCES = test-fts-filter.c
-test_fts_filter_LDADD = $(filter_deps) $(test_libs) $(STEMMER_LIBS) $(TEXTCAT_LIBS) $(NORMALIZER_LIBS)
-test_fts_filter_DEPENDENCIES = $(test_deps) $(filter_deps)
+test_fts_filter_LDADD = libfts.la $(test_libs)
+test_fts_filter_DEPENDENCIES = libfts.la $(test_deps)
 
 if BUILD_FTS_EXTTEXTCAT
 TEST_FTS_LANGUAGE = test-fts-language
diff --git a/src/lib-fts/fts-filter-normalizer-simple.c b/src/lib-fts/fts-filter-normalizer-simple.c
new file mode 100644
index 0000000000..7cff5eac83
--- /dev/null
+++ b/src/lib-fts/fts-filter-normalizer-simple.c
@@ -0,0 +1,77 @@
+/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "unichar.h"
+#include "str.h"
+#include "fts-filter.h"
+#include "fts-filter-private.h"
+#include "fts-language.h"
+
+struct fts_filter_normalizer_simple {
+	struct fts_filter filter;
+	string_t *str;
+};
+
+static bool
+fts_filter_normalizer_simple_supports(const struct fts_language *lang ATTR_UNUSED)
+{
+	return TRUE;
+}
+
+static void
+fts_filter_normalizer_simple_destroy(struct fts_filter *_filter)
+{
+	struct fts_filter_normalizer_simple *filter =
+		(struct fts_filter_normalizer_simple *)_filter;
+
+	str_free(&filter->str);
+	i_free(filter);
+}
+
+static int
+fts_filter_normalizer_simple_create(const struct fts_language *lang ATTR_UNUSED,
+				    const char *const *settings,
+				    struct fts_filter **filter_r,
+				    const char **error_r)
+{
+	struct fts_filter_normalizer_simple *filter;
+
+	if (settings[0] != NULL) {
+		*error_r = t_strdup_printf("Unknown setting: %s", settings[0]);
+		return -1;
+	}
+	filter = i_new(struct fts_filter_normalizer_simple, 1);
+	filter->filter = *fts_filter_normalizer_simple;
+	filter->str = str_new(default_pool, 128);
+
+	*filter_r = &filter->filter;
+	return 0;
+}
+
+static const char *
+fts_filter_normalizer_simple_filter(struct fts_filter *_filter,
+				    const char *token)
+{
+	struct fts_filter_normalizer_simple *filter =
+		(struct fts_filter_normalizer_simple *)_filter;
+
+	str_truncate(filter->str, 0);
+	if (uni_utf8_to_decomposed_titlecase(token, strlen(token),
+					     filter->str) < 0)
+		return NULL;
+	return str_c(filter->str);
+}
+
+static const struct fts_filter_vfuncs normalizer_filter_vfuncs = {
+	fts_filter_normalizer_simple_supports,
+	fts_filter_normalizer_simple_create,
+	fts_filter_normalizer_simple_filter,
+	fts_filter_normalizer_simple_destroy
+};
+
+static const struct fts_filter fts_filter_normalizer_simple_real = {
+	.class_name = SIMPLE_NORMALIZER_FILTER_NAME,
+	.v = &normalizer_filter_vfuncs
+};
+
+const struct fts_filter *fts_filter_normalizer_simple = &fts_filter_normalizer_simple_real;
diff --git a/src/lib-fts/fts-filter.c b/src/lib-fts/fts-filter.c
index 08b9b55d8e..6cb6e02059 100644
--- a/src/lib-fts/fts-filter.c
+++ b/src/lib-fts/fts-filter.c
@@ -15,6 +15,7 @@ void fts_filters_init(void)
 	fts_filter_register(fts_filter_stopwords);
 	fts_filter_register(fts_filter_stemmer_snowball);
 	fts_filter_register(fts_filter_normalizer_icu);
+	fts_filter_register(fts_filter_normalizer_simple);
 }
 
 void fts_filters_deinit(void)
diff --git a/src/lib-fts/fts-filter.h b/src/lib-fts/fts-filter.h
index 1628d349ec..4ccd50dc31 100644
--- a/src/lib-fts/fts-filter.h
+++ b/src/lib-fts/fts-filter.h
@@ -33,6 +33,10 @@ extern const struct fts_filter *fts_filter_stemmer_snowball;
 extern const struct fts_filter *fts_filter_normalizer_icu;
 #define ICU_NORMALIZER_FILTER_NAME "normalizer-icu"
 
+/* Normalization using i;unicode-casemap (RFC 5051) */
+extern const struct fts_filter *fts_filter_normalizer_simple;
+#define SIMPLE_NORMALIZER_FILTER_NAME "normalizer-simple"
+
 /* Register all built-in filters. */
 void fts_filters_init(void);
 void fts_filters_deinit(void);