From: Teemu Huovila Date: Sat, 9 May 2015 08:06:45 +0000 (+0300) Subject: fts: Change filter API to be able to return errors X-Git-Tag: 2.2.17.rc1~69 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=db090e2a48bcc5ce162af0c36eef04407421379d;p=thirdparty%2Fdovecot%2Fcore.git fts: Change filter API to be able to return errors Modify fts_filter_filter() to return integer status codes. It returns 1 if a token was returned, 0 if it was filtered away and -1 on error. --- diff --git a/src/lib-fts/fts-filter-normalizer-icu.c b/src/lib-fts/fts-filter-normalizer-icu.c index f7e40f2d1c..e2783738f8 100644 --- a/src/lib-fts/fts-filter-normalizer-icu.c +++ b/src/lib-fts/fts-filter-normalizer-icu.c @@ -98,8 +98,9 @@ static int make_uchar(const char *src, UChar **dst, int32_t *dst_uchars_r) return 0; } -static int make_utf8(const UChar *src, char **dst, const char **error_r) +static int make_utf8(const UChar *src, const char **_dst, const char **error_r) { + char *dst; char *retp = NULL; int32_t dsize = 0; int32_t dsize_actual = 0; @@ -120,9 +121,9 @@ static int make_utf8(const UChar *src, char **dst, const char **error_r) i_assert(NULL == retp); dsize++; /* room for '\0' byte */ - *dst = t_malloc(dsize); + dst = t_malloc(dsize); err = U_ZERO_ERROR; - retp = u_strToUTF8WithSub(*dst, dsize, &dsize_actual, src, usrc_len, + retp = u_strToUTF8WithSub(dst, dsize, &dsize_actual, src, usrc_len, UNICODE_REPLACEMENT_CHAR, &sub_num, &err); if (U_FAILURE(err)) i_panic("Lib ICU u_strToUTF8WithSub() failed: %s", @@ -137,8 +138,9 @@ static int make_utf8(const UChar *src, char **dst, const char **error_r) " Substitutions (%d) were made.", sub_num); return -1; } - i_assert(retp == *dst); + i_assert(retp == dst); + *_dst = dst; return 0; } @@ -212,27 +214,24 @@ fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED, return 0; } -/* Returns 0 on success and -1 on error. */ -/* TODO: delay errors until _deinit() and return some other values? */ -static const char * -fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char *token) +static int +fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char **token) { UErrorCode err = U_ZERO_ERROR; UChar *utext = NULL; int32_t utext_cap = 0; int32_t utext_len = -1; int32_t utext_limit; - char *normalized = NULL; struct fts_filter_normalizer *np = (struct fts_filter_normalizer *)filter; /* TODO: fix error handling */ if (np->error != NULL) - return NULL; + goto err_exit; - if (make_uchar(token, &utext, &utext_cap) < 0) { + if (make_uchar(*token, &utext, &utext_cap) < 0) { fts_filter_normalizer_icu_error(&np->error, "Conversion to UChar failed"); - return NULL; + goto err_exit; } /* TODO: Some problems here. How much longer can the result @@ -249,8 +248,9 @@ fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char *token) size utrans_transUChars indicated */ utext_len++; /* room for '\0' bytes(2) */ utext_cap = utext_len; - if (make_uchar(token, &utext, &utext_cap) < 0) - return NULL; + if (make_uchar(*token, &utext, &utext_cap) < 0) { + goto err_exit; + } i_assert(utext_cap == utext_len); utext_limit = u_strlen(utext); utext_len = -1; @@ -262,13 +262,17 @@ fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char *token) if (U_FAILURE(err)) { icu_error(&np->error, err, "utrans_transUChars()"); - return NULL; + goto err_exit; } - if (make_utf8(utext, &normalized, &np->error) < 0) - return NULL; + if (make_utf8(utext, token, &np->error) < 0) { + goto err_exit; + } - return normalized; + return 1; + err_exit: + *token = NULL; + return -1; } #else @@ -289,7 +293,7 @@ fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED, return -1; } -static const char * +static int fts_filter_normalizer_icu_filter(struct fts_filter *filter ATTR_UNUSED, const char *token ATTR_UNUSED) { diff --git a/src/lib-fts/fts-filter-normalizer-simple.c b/src/lib-fts/fts-filter-normalizer-simple.c index 7cff5eac83..e06c0f948f 100644 --- a/src/lib-fts/fts-filter-normalizer-simple.c +++ b/src/lib-fts/fts-filter-normalizer-simple.c @@ -48,18 +48,21 @@ fts_filter_normalizer_simple_create(const struct fts_language *lang ATTR_UNUSED, return 0; } -static const char * +static int fts_filter_normalizer_simple_filter(struct fts_filter *_filter, - const char *token) + const char **token) { struct fts_filter_normalizer_simple *filter = (struct fts_filter_normalizer_simple *)_filter; str_truncate(filter->str, 0); - if (uni_utf8_to_decomposed_titlecase(token, strlen(token), - filter->str) < 0) - return NULL; - return str_c(filter->str); + if (uni_utf8_to_decomposed_titlecase(*token, strlen(*token), + filter->str) < 0) { + *token = NULL; + return -1; + } + *token = str_c(filter->str); + return 1; } static const struct fts_filter_vfuncs normalizer_filter_vfuncs = { diff --git a/src/lib-fts/fts-filter-private.h b/src/lib-fts/fts-filter-private.h index 86f78dc3b8..77de275bcb 100644 --- a/src/lib-fts/fts-filter-private.h +++ b/src/lib-fts/fts-filter-private.h @@ -17,7 +17,7 @@ struct fts_filter_vfuncs { const char *const *settings, struct fts_filter **filter_r, const char **error_r); - const char * (*filter)(struct fts_filter *filter, const char *token); + int (*filter)(struct fts_filter *filter, const char **token); void (*destroy)(struct fts_filter *filter); }; diff --git a/src/lib-fts/fts-filter-stemmer-snowball.c b/src/lib-fts/fts-filter-stemmer-snowball.c index fc4ade39f3..b9de6510fe 100644 --- a/src/lib-fts/fts-filter-stemmer-snowball.c +++ b/src/lib-fts/fts-filter-stemmer-snowball.c @@ -66,18 +66,20 @@ fts_filter_stemmer_snowball_create(const struct fts_language *lang, return 0; } -static const char * +static int fts_filter_stemmer_snowball_filter(struct fts_filter *filter, - const char *token) + const char **token) { const sb_symbol *base; int len; struct fts_filter_stemmer_snowball *sp = (struct fts_filter_stemmer_snowball *) filter; - base = sb_stemmer_stem(sp->stemmer, (const unsigned char *)token, strlen(token)); + base = sb_stemmer_stem(sp->stemmer, (const unsigned char *)*token, strlen(*token)); len = sb_stemmer_length(sp->stemmer); - return t_strdup_until(base, base + len); + *token = t_strdup_until(base, base + len); + + return *token != NULL? 1: -1; } #else @@ -101,11 +103,11 @@ fts_filter_stemmer_snowball_destroy(struct fts_filter *stemmer ATTR_UNUSED) { } -static const char * +static int fts_filter_stemmer_snowball_filter(struct fts_filter *filter ATTR_UNUSED, - const char *token ATTR_UNUSED) + const char **token ATTR_UNUSED) { - return NULL; + return -1; } #endif diff --git a/src/lib-fts/fts-filter-stopwords.c b/src/lib-fts/fts-filter-stopwords.c index af830b5332..a18487a1be 100644 --- a/src/lib-fts/fts-filter-stopwords.c +++ b/src/lib-fts/fts-filter-stopwords.c @@ -125,18 +125,20 @@ fts_filter_stopwords_create(const struct fts_language *lang, return ret; } -static const char * -fts_filter_stopwords_filter(struct fts_filter *filter, const char *token) +static int +fts_filter_stopwords_filter(struct fts_filter *filter, const char **token) { const char *stopword; struct fts_filter_stopwords *sp = (struct fts_filter_stopwords *) filter; - stopword = hash_table_lookup(sp->stopwords, token); - if (stopword != NULL) - return NULL; + stopword = hash_table_lookup(sp->stopwords, *token); + if (stopword != NULL) { + *token = NULL; + return 0; + } else - return token; + return 1; } const struct fts_filter_vfuncs stopwords_filter_vfuncs = { diff --git a/src/lib-fts/fts-filter.c b/src/lib-fts/fts-filter.c index 6cb6e02059..0ec2229ed4 100644 --- a/src/lib-fts/fts-filter.c +++ b/src/lib-fts/fts-filter.c @@ -92,19 +92,19 @@ void fts_filter_unref(struct fts_filter **_fpp) /* TODO: Avoid multiple allocations by using a buffer in v->filter? Do this non-recursively? */ -const char * -fts_filter_filter(struct fts_filter *filter, const char *token) +int +fts_filter_filter(struct fts_filter *filter, const char **token) { - const char *filtered = NULL; + int ret; if (filter->parent == NULL) return filter->v->filter(filter, token); - filtered = fts_filter_filter(filter->parent, token); + ret = fts_filter_filter(filter->parent, token); - if(filtered != NULL) - return filter->v->filter(filter, filtered); + if(ret > 0) + return filter->v->filter(filter, token); - return NULL; + return ret; } diff --git a/src/lib-fts/fts-filter.h b/src/lib-fts/fts-filter.h index 4ccd50dc31..de5232218c 100644 --- a/src/lib-fts/fts-filter.h +++ b/src/lib-fts/fts-filter.h @@ -58,8 +58,11 @@ int fts_filter_create(const struct fts_filter *filter_class, void fts_filter_ref(struct fts_filter *filter); void fts_filter_unref(struct fts_filter **filter); -/* Returns the filtered token or NULL, if it was completely removed */ -const char * -fts_filter_filter(struct fts_filter *filter, const char *token); +/* Returns 1 if token is returned in *token, 0 if token was filtered + out and -1 on error. + Input is also given via *token. +*/ +int +fts_filter_filter(struct fts_filter *filter, const char **token); #endif diff --git a/src/lib-fts/test-fts-filter.c b/src/lib-fts/test-fts-filter.c index 6ca6b47ab9..914e463104 100644 --- a/src/lib-fts/test-fts-filter.c +++ b/src/lib-fts/test-fts-filter.c @@ -24,7 +24,7 @@ static void test_fts_filter_stopwords_eng(void) "drive", NULL, NULL, NULL, "reason", NULL, NULL, NULL, "sing"}; const char **ip, **op; - const char *filtered; + const char *token; test_begin("fts filter stopwords, English"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); @@ -34,12 +34,14 @@ static void test_fts_filter_stopwords_eng(void) ip = input; op = output; while (*ip != NULL) { - filtered = fts_filter_filter(filter, *ip); - if (filtered == NULL) + token = *ip; + ret = fts_filter_filter(filter, &token); + test_assert(ret >= 0); + if (ret == 0) test_assert(*op == NULL); else { test_assert(*op != NULL); - test_assert(strcmp(*ip, filtered) == 0); + test_assert(strcmp(*ip, token) == 0); } op++; ip++; @@ -66,7 +68,7 @@ static void test_fts_filter_stopwords_fin(void) {"kuka", "kenet", "keneen", "testi", "eiv\xC3\xA4t", NULL}; const char *output2[] = {NULL, NULL, NULL, "testi", NULL}; const char **ip, **op; - const char *filtered; + const char *token; test_begin("fts filter stopwords, Finnish"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); @@ -76,12 +78,14 @@ static void test_fts_filter_stopwords_fin(void) ip = input; op = output; while (*ip != NULL) { - filtered = fts_filter_filter(filter, *ip); - if (filtered == NULL) + token = *ip; + ret = fts_filter_filter(filter, &token); + test_assert(ret >= 0); + if (ret == 0) test_assert(*op == NULL); else { test_assert(*op != NULL); - test_assert(strcmp(*ip, filtered) == 0); + test_assert(strcmp(*ip, token) == 0); } op++; ip++; @@ -95,12 +99,13 @@ static void test_fts_filter_stopwords_fin(void) ip = input2; op = output2; while (*ip != NULL) { - filtered = fts_filter_filter(filter, *ip); - if (filtered == NULL) + token = *ip; + ret = fts_filter_filter(filter, &token); + if (ret == 0) test_assert(*op == NULL); else { test_assert(*op != NULL); - test_assert(strcmp(*ip, filtered) == 0); + test_assert(strcmp(*ip, token) == 0); } op++; ip++; @@ -127,7 +132,7 @@ static void test_fts_filter_stopwords_fra(void) "quelconque", NULL, "l\xE2\x80\x99""av\xC3\xA8nement",}; const char **ip, **op; - const char *filtered; + const char *token; test_begin("fts filter stopwords, French"); filter_class = fts_filter_find(STOPWORDS_FILTER_NAME); @@ -137,12 +142,14 @@ static void test_fts_filter_stopwords_fra(void) ip = input; op = output; while (*ip != NULL) { - filtered = fts_filter_filter(filter, *ip); - if (filtered == NULL) + token = *ip; + ret = fts_filter_filter(filter, &token); + test_assert(ret >= 0); + if (ret == 0) test_assert(*op == NULL); else { test_assert(*op != NULL); - test_assert(strcmp(*ip, filtered) == 0); + test_assert(strcmp(*ip, token) == 0); } op++; ip++; @@ -177,7 +184,7 @@ static void test_fts_filter_stemmer_snowball_stem_english(void) struct fts_filter *stemmer; const char *error; struct fts_language language = { .name = "EN" }; - const char *base = NULL; + const char *token = NULL; const char * const tokens[] = { "dries" ,"friendlies", "All", "human", "beings", "are", "born", "free", "and", "equal", "in", "dignity", "and", @@ -199,9 +206,10 @@ static void test_fts_filter_stemmer_snowball_stem_english(void) test_assert(ret == 0); bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { - base = fts_filter_filter(stemmer, *tpp); - test_assert(base != NULL); - test_assert(null_strcmp(base, *bpp) == 0); + token = *tpp; + ret = fts_filter_filter(stemmer, &token); + test_assert(token != NULL); + test_assert(null_strcmp(token, *bpp) == 0); bpp++; } fts_filter_unref(&stemmer); @@ -216,7 +224,7 @@ static void test_fts_filter_stemmer_snowball_stem_french(void) struct fts_filter *stemmer; const char *error; struct fts_language language = { .name = "fRench" }; - const char *base = NULL; + const char *token = NULL; const char * const tokens[] = { "Tous", "les", "\xC3\xAAtres", "humains", "naissent", "libres", "et", "\xC3\xA9gaux", "en", "dignit\xC3\xA9", @@ -233,9 +241,10 @@ static void test_fts_filter_stemmer_snowball_stem_french(void) test_assert(ret == 0); bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { - base = fts_filter_filter(stemmer, *tpp); - test_assert(base != NULL); - test_assert(null_strcmp(base, *bpp) == 0); + token = *tpp; + ret = fts_filter_filter(stemmer, &token); + test_assert(token != NULL); + test_assert(null_strcmp(token, *bpp) == 0); bpp++; } fts_filter_unref(&stemmer); @@ -251,7 +260,7 @@ static void test_fts_filter_stopwords_stemmer_eng(void) struct fts_filter *filter; const char *error; struct fts_language language = { .name = "eN" }; - const char *base = NULL; + const char *token = NULL; const char * const tokens[] = { "dries" ,"friendlies", "All", "human", "beings", "are", "born", "free", "and", "equal", "in", "dignity", "and", @@ -279,12 +288,13 @@ static void test_fts_filter_stopwords_stemmer_eng(void) bpp = bases; for (tpp=tokens; *tpp != NULL; tpp++) { - base = fts_filter_filter(stemmer, *tpp); - if (base == NULL) + token = *tpp; + ret = fts_filter_filter(stemmer, &token); + if (ret == 0) test_assert(*bpp == NULL); else { test_assert(*bpp != NULL); - test_assert(null_strcmp(*bpp, base) == 0); + test_assert(null_strcmp(*bpp, token) == 0); } bpp++; } @@ -322,7 +332,7 @@ static void test_fts_filter_normalizer_swedish_short(void) const char * const settings[] = {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC", NULL}; const char *error = NULL; - const char *normalized = NULL; + const char *token = NULL; unsigned int i; test_begin("fts filter normalizer Swedish short text"); @@ -333,8 +343,9 @@ static void test_fts_filter_normalizer_swedish_short(void) test_assert(ret == 0); for (i = 0; i < N_ELEMENTS(input); i++) { if (input[i] != NULL) { - test_assert_idx((normalized = fts_filter_filter(norm, input[i])) != NULL, i); - test_assert_idx(null_strcmp(normalized, expected_output[i]) == 0, i); + token = input[i]; + test_assert_idx(fts_filter_filter(norm, &token) == 1, i); + test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); } } fts_filter_unref(&norm); @@ -366,7 +377,7 @@ static void test_fts_filter_normalizer_swedish_short_default_id(void) "vem kan segla forutan vind?\naaooaa" }; const char *error = NULL; - const char *normalized = NULL; + const char *token = NULL; unsigned int i; test_begin("fts filter normalizer Swedish short text using default ID"); @@ -377,8 +388,9 @@ static void test_fts_filter_normalizer_swedish_short_default_id(void) test_assert(ret == 0); for (i = 0; i < N_ELEMENTS(input); i++) { if (input[i] != NULL) { - test_assert_idx((normalized = fts_filter_filter(norm, input[i])) != NULL, i); - test_assert_idx(null_strcmp(normalized, expected_output[i]) == 0, i); + token = input[i]; + test_assert_idx(fts_filter_filter(norm, &token) == 1, i); + test_assert_idx(null_strcmp(token, expected_output[i]) == 0, i); } } fts_filter_unref(&norm); @@ -398,7 +410,7 @@ static void test_fts_filter_normalizer_french(void) {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL}; char buf[4096] = {0}; const char *error = NULL; - const char *normalized = NULL; + const char *tokens; int ret; unsigned char sha512_digest[SHA512_RESULTLEN]; struct sha512_ctx ctx; @@ -424,11 +436,11 @@ static void test_fts_filter_normalizer_french(void) test_assert(input != NULL); sha512_init(&ctx); while (NULL != fgets(buf, sizeof(buf), input)) { - - if ((normalized = fts_filter_filter(norm, buf)) == NULL){ + tokens = buf; + if (fts_filter_filter(norm, &tokens) != 1){ break; } - sha512_loop(&ctx, normalized, strlen(normalized)); + sha512_loop(&ctx, tokens, strlen(tokens)); } fclose(input); sha512_result(&ctx, sha512_digest); @@ -470,7 +482,7 @@ static void test_fts_filter_normalizer_stopwords_stemmer_eng(void) //{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC", NULL}; {"id", "Lower", NULL}; struct fts_language language = { .name = "En" }; - const char *base = NULL; + const char *token = NULL; const char * const tokens[] = { "dries" ,"friendlies", "All", "human", "beings", "are", "born", "free", "and", "equal", "in", "dignity", "and", @@ -503,12 +515,13 @@ static void test_fts_filter_normalizer_stopwords_stemmer_eng(void) bpp = bases; for (tpp = tokens; *tpp != NULL; tpp++) { - base = fts_filter_filter(stemmer, *tpp); - if (base == NULL) + token = *tpp; + ret = fts_filter_filter(stemmer, &token); + if (ret == 0) test_assert(*bpp == NULL); else { test_assert(*bpp != NULL); - test_assert(strcasecmp(*bpp, base) == 0); + test_assert(strcasecmp(*bpp, token) == 0); } bpp++; } diff --git a/src/plugins/fts/fts-build-mail.c b/src/plugins/fts/fts-build-mail.c index 4bf9ac132d..212e969644 100644 --- a/src/plugins/fts/fts-build-mail.c +++ b/src/plugins/fts/fts-build-mail.c @@ -249,9 +249,11 @@ fts_build_add_tokens_with_filter(struct fts_mail_build_context *ctx, tokenizer = fts_user_get_index_tokenizer(ctx->update_ctx->backend->ns->user); while ((ret = fts_tokenizer_next(tokenizer, data, size, &token)) > 0) { if (filter != NULL) { - token = fts_filter_filter(filter, token); - if (token == NULL) + ret = fts_filter_filter(filter, &token); + if (ret == 0) continue; + if (ret < 0) + break; } if (fts_backend_update_build_more(ctx->update_ctx, (const void *)token, diff --git a/src/plugins/fts/fts-search-args.c b/src/plugins/fts/fts-search-args.c index ce8795888e..b7f2bcd106 100644 --- a/src/plugins/fts/fts-search-args.c +++ b/src/plugins/fts/fts-search-args.c @@ -64,6 +64,7 @@ fts_backend_dovecot_expand_lang_tokens(const ARRAY_TYPE(fts_user_language) *lang struct fts_user_language *const *langp; ARRAY_TYPE(const_string) tokens; const char *token2; + int ret; t_array_init(&tokens, 4); /* first add the word exactly as it without any tokenization */ @@ -73,9 +74,10 @@ fts_backend_dovecot_expand_lang_tokens(const ARRAY_TYPE(fts_user_language) *lang /* add the word filtered */ array_foreach(languages, langp) { - token2 = (*langp)->filter == NULL ? token : - fts_filter_filter((*langp)->filter, token); - if (token2 != NULL) { + token2 = t_strdup(token); + if ((*langp)->filter != NULL) + ret = fts_filter_filter((*langp)->filter, &token2); + if (ret > 0) { token2 = t_strdup(token2); array_append(&tokens, &token2, 1); }