From: Timo Sirainen <tss@iki.fi>
Date: Sat, 9 May 2015 11:26:42 +0000 (+0300)
Subject: lib-fts: Added "lowercase" filter.
X-Git-Tag: 2.2.17.rc1~37
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=acfcf88e4dd529e4b2409f43bc9713cbc0169347;p=thirdparty%2Fdovecot%2Fcore.git

lib-fts: Added "lowercase" filter.
For now it handles only ASCII characters, but that's enough for our use.
---

diff --git a/src/lib-fts/Makefile.am b/src/lib-fts/Makefile.am
index e4a04f17c7..a096de699e 100644
--- a/src/lib-fts/Makefile.am
+++ b/src/lib-fts/Makefile.am
@@ -61,6 +61,7 @@ libfts_la_LIBADD = \
 
 libfts_la_SOURCES = \
 	fts-filter.c \
+	fts-filter-lowercase.c \
 	fts-filter-normalizer-icu.c \
 	fts-filter-stopwords.c \
 	fts-filter-stemmer-snowball.c \
diff --git a/src/lib-fts/fts-filter-lowercase.c b/src/lib-fts/fts-filter-lowercase.c
new file mode 100644
index 0000000000..4074c8411d
--- /dev/null
+++ b/src/lib-fts/fts-filter-lowercase.c
@@ -0,0 +1,61 @@
+/* Copyright (c) 2015 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "fts-filter.h"
+#include "fts-filter-private.h"
+#include "fts-language.h"
+
+static bool
+fts_filter_lowercase_supports(const struct fts_language *lang ATTR_UNUSED)
+{
+	return TRUE;
+}
+
+static void
+fts_filter_lowercase_destroy(struct fts_filter *filter)
+{
+	i_free(filter);
+}
+
+static int
+fts_filter_lowercase_create(const struct fts_language *lang ATTR_UNUSED,
+			    const char *const *settings,
+			    struct fts_filter **filter_r,
+			    const char **error_r)
+{
+	struct fts_filter *filter;
+
+	if (settings[0] != NULL) {
+		*error_r = t_strdup_printf("Unknown setting: %s", settings[0]);
+		return -1;
+	}
+	filter = i_new(struct fts_filter, 1);
+	*filter = *fts_filter_lowercase;
+
+	*filter_r = filter;
+	return 0;
+}
+
+static int
+fts_filter_lowercase_filter(struct fts_filter *_filter ATTR_UNUSED,
+			    const char **token,
+			    const char **error_r ATTR_UNUSED)
+{
+	*token = t_str_lcase(*token);
+	return 1;
+}
+
+static const struct fts_filter_vfuncs normalizer_filter_vfuncs = {
+	fts_filter_lowercase_supports,
+	fts_filter_lowercase_create,
+	fts_filter_lowercase_filter,
+	fts_filter_lowercase_destroy
+};
+
+static const struct fts_filter fts_filter_lowercase_real = {
+	.class_name = LOWERCASE_FILTER_NAME,
+	.v = &normalizer_filter_vfuncs
+};
+
+const struct fts_filter *fts_filter_lowercase = &fts_filter_lowercase_real;
diff --git a/src/lib-fts/fts-filter.c b/src/lib-fts/fts-filter.c
index 2ffbfdb2c4..74350dc7a6 100644
--- a/src/lib-fts/fts-filter.c
+++ b/src/lib-fts/fts-filter.c
@@ -15,6 +15,7 @@ void fts_filters_init(void)
 	fts_filter_register(fts_filter_stopwords);
 	fts_filter_register(fts_filter_stemmer_snowball);
 	fts_filter_register(fts_filter_normalizer_icu);
+	fts_filter_register(fts_filter_lowercase);
 }
 
 void fts_filters_deinit(void)
diff --git a/src/lib-fts/fts-filter.h b/src/lib-fts/fts-filter.h
index 07aa4bae2b..128a5b4f76 100644
--- a/src/lib-fts/fts-filter.h
+++ b/src/lib-fts/fts-filter.h
@@ -33,6 +33,10 @@ extern const struct fts_filter *fts_filter_stemmer_snowball;
 extern const struct fts_filter *fts_filter_normalizer_icu;
 #define ICU_NORMALIZER_FILTER_NAME "normalizer-icu"
 
+/* Lowecases the input. Currently only ASCII data is lowercased. */
+extern const struct fts_filter *fts_filter_lowercase;
+#define LOWERCASE_FILTER_NAME "lowercase"
+
 /* Register all built-in filters. */
 void fts_filters_init(void);
 void fts_filters_deinit(void);
diff --git a/src/lib-fts/test-fts-filter.c b/src/lib-fts/test-fts-filter.c
index 867574344a..e2e666ca9e 100644
--- a/src/lib-fts/test-fts-filter.c
+++ b/src/lib-fts/test-fts-filter.c
@@ -11,6 +11,35 @@
 static const char *const stopword_settings[] = {"stopwords_dir", TEST_STOPWORDS_DIR, NULL};
 static struct fts_language english_language = { .name = "en" };
 
+static void test_fts_filter_lowercase(void)
+{
+	struct {
+		const char *input;
+		const char *output;
+	} tests[] = {
+		{ "foo", "foo" },
+		{ "FOO", "foo" },
+		{ "fOo", "foo" }
+	};
+	const struct fts_filter *filter_class;
+	struct fts_filter *filter;
+	const char *error;
+	const char *token;
+	unsigned int i;
+
+	test_begin("fts filter lowercase");
+	filter_class = fts_filter_find(LOWERCASE_FILTER_NAME);
+	test_assert(fts_filter_create(filter_class, NULL, &english_language, NULL, &filter, &error) == 0);
+
+	for (i = 0; i < N_ELEMENTS(tests); i++) {
+		token = tests[i].input;
+		test_assert_idx(fts_filter_filter(filter, &token, &error) > 0 &&
+				strcmp(token, tests[i].output) == 0, 0);
+	}
+	fts_filter_unref(&filter);
+	test_end();
+}
+
 static void test_fts_filter_stopwords_eng(void)
 {
 	const struct fts_filter *filter_class;
@@ -521,6 +550,7 @@ static void test_fts_filter_normalizer_stopwords_stemmer_eng(void)
 int main(void)
 {
 	static void (*test_functions[])(void) = {
+		test_fts_filter_lowercase,
 		test_fts_filter_stopwords_eng,
 		test_fts_filter_stopwords_fin,
 		test_fts_filter_stopwords_fra,