lib-fts: Make sure address tokenizer can't return empty tokens.

This happened when address was a token that first looked like it could be
a valid address, but then got truncated due to reaching maxlen, followed
by truncating the UTF8-sequence and finally all the rest of the '-' or
'.' chars that were valid at the beginning of the address are stripped
away by fts_tokenizer_delete_trailing_invalid_char(), leaving nothing left.

Fixes:
Panic: file fts-tokenizer.c: line 206 (fts_tokenizer_next): assertion failed: (ret <= 0 || (*token_r)[0] != '\0')

diff --git a/src/lib-fts/fts-tokenizer-address.c b/src/lib-fts/fts-tokenizer-address.c
index 2b9380c499ae35c43d7ee9049d75e7196bafead5..125a612ae62ce2f1904ebde03d9e38cb9aee6975 100644 (file)
--- a/src/lib-fts/fts-tokenizer-address.c
+++ b/src/lib-fts/fts-tokenizer-address.c
@@ -75,7 +75,7 @@ static void fts_tokenizer_email_address_destroy(struct fts_tokenizer *_tok)
        i_free(tok);
 }
 
-static void
+static bool
 fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok,
                                     const char **token_r)
 {
@@ -98,6 +98,7 @@ fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok,
                fts_tokenizer_delete_trailing_invalid_char(data, &len);
        *token_r = len == 0 ? "" :
                t_strndup(data, len);
+       return len > 0;
 }
 
 static bool
@@ -243,8 +244,8 @@ fts_tokenizer_email_address_next(struct fts_tokenizer *_tok,
 
        if (tok->state == EMAIL_ADDRESS_PARSER_STATE_COMPLETE) {
                *skip_r = pos;
-               fts_tokenizer_address_current_token(tok, token_r);
-               return 1;
+               if (fts_tokenizer_address_current_token(tok, token_r))
+                       return 1;
        }
 
        /* end of data, output lingering tokens. first the parents data, then
@@ -261,8 +262,8 @@ fts_tokenizer_email_address_next(struct fts_tokenizer *_tok,
                        return 1;
 
                if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN) {
-                       fts_tokenizer_address_current_token(tok, token_r);
-                       return 1;
+                       if (fts_tokenizer_address_current_token(tok, token_r))
+                               return 1;
                }
                tok->state = EMAIL_ADDRESS_PARSER_STATE_NONE;
        }
@@ -310,9 +311,11 @@ fts_tokenizer_email_address_next(struct fts_tokenizer *_tok,
                        break;
                case EMAIL_ADDRESS_PARSER_STATE_COMPLETE:
                        *skip_r = pos;
-                       if (!fts_tokenizer_address_parent_data(tok, token_r))
-                               fts_tokenizer_address_current_token(tok, token_r);
-                       return 1;
+                       if (fts_tokenizer_address_parent_data(tok, token_r))
+                               return 1;
+                       if (fts_tokenizer_address_current_token(tok, token_r))
+                               return 1;
+                       break;
                default:
                        i_unreached();
                }

diff --git a/src/lib-fts/test-fts-tokenizer.c b/src/lib-fts/test-fts-tokenizer.c
index 55aef40e26b8708067a30c2d5f58259b73372171..eb606410d4f1c435572b96489d1fdb4edd86f4b6 100644 (file)
--- a/src/lib-fts/test-fts-tokenizer.c
+++ b/src/lib-fts/test-fts-tokenizer.c
@@ -452,6 +452,23 @@ static void test_fts_tokenizer_delete_trailing_partial_char(void)
        test_end();
 }
 
+static void test_fts_tokenizer_address_maxlen(void)
+{
+       const char *const settings[] = {"maxlen", "5", NULL};
+       const char *input = "...\357\277\275@a";
+       struct fts_tokenizer *tok;
+       const char *token, *error;
+
+       test_begin("fts tokenizer address maxlen");
+       test_assert(fts_tokenizer_create(fts_tokenizer_email_address, NULL, settings, &tok, &error) == 0);
+
+       while (fts_tokenizer_next(tok, (const unsigned char *)input,
+                                 strlen(input), &token, &error) > 0) ;
+       while (fts_tokenizer_final(tok, &token, &error) > 0) ;
+       fts_tokenizer_unref(&tok);
+       test_end();
+}
+
 int main(void)
 {
        static void (*test_functions[])(void) = {
@@ -462,6 +479,7 @@ int main(void)
                test_fts_tokenizer_address_only,
                test_fts_tokenizer_address_parent_simple,
                test_fts_tokenizer_address_parent_tr29,
+               test_fts_tokenizer_address_maxlen,
                test_fts_tokenizer_address_search,
                test_fts_tokenizer_delete_trailing_partial_char,
                NULL