lib-fts: Implemented "search" parameter to fts-tokenizer-address.

diff --git a/src/lib-fts/fts-tokenizer-address.c b/src/lib-fts/fts-tokenizer-address.c
index 8cda5d161bafa4dc2db44569983c23d53580321a..cfada46952de7cbbcfa99708e4a946c0fcf955a2 100644 (file)
--- a/src/lib-fts/fts-tokenizer-address.c
+++ b/src/lib-fts/fts-tokenizer-address.c
@@ -76,17 +76,30 @@ fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok,
        return 1;
 }
 
-static int
+static bool
 fts_tokenizer_address_parent_data(struct email_address_fts_tokenizer *tok,
                                   const char **token_r)
 {
-       /* TODO: search option removes address from data here. */
-       if (tok->search && tok->state >= EMAIL_ADDRESS_PARSER_STATE_DOMAIN)
-               i_debug("Would remove current token");
+       if (tok->tokenizer.parent == NULL || str_len(tok->parent_data) == 0)
+               return FALSE;
+
+       if (tok->search && tok->state >= EMAIL_ADDRESS_PARSER_STATE_DOMAIN) {
+               /* we're searching and we want to find only the full
+                  user@domain (not "user" and "domain"). we'll do this by
+                  not feeding the last user@domain to parent tokenizer. */
+               unsigned int parent_prefix_len =
+                       str_len(tok->parent_data) - str_len(tok->last_word);
+               i_assert(str_len(tok->parent_data) >= str_len(tok->last_word) &&
+                        strcmp(str_c(tok->parent_data) + parent_prefix_len,
+                               str_c(tok->last_word)) == 0);
+               str_truncate(tok->parent_data, parent_prefix_len);
+               if (str_len(tok->parent_data) == 0)
+                       return FALSE;
+       }
 
        *token_r = t_strdup(str_c(tok->parent_data));
        str_truncate(tok->parent_data, 0);
-       return 1;
+       return TRUE;
 }
 
 /* Used to rewind past characters that can not be the start of a new localpart.
@@ -204,8 +217,8 @@ fts_tokenizer_email_address_next(struct fts_tokenizer *_tok,
        /* end of data, output lingering tokens. first the parents data, then
           possibly our token, if complete enough */
        if (size == 0) {
-               if (tok->tokenizer.parent != NULL && str_len(tok->parent_data) > 0)
-                       return fts_tokenizer_address_parent_data(tok, token_r);
+               if (fts_tokenizer_address_parent_data(tok, token_r))
+                       return 1;
 
                if (tok->state == EMAIL_ADDRESS_PARSER_STATE_DOMAIN &&
                    !domain_is_empty(tok))
@@ -254,16 +267,10 @@ fts_tokenizer_email_address_next(struct fts_tokenizer *_tok,
 
                        break;
                case EMAIL_ADDRESS_PARSER_STATE_COMPLETE:
-                       /* skip tailing non-atext */
-                       local_skip = skip_nonlocal_part(data+pos, size - pos);
-                       *skip_r = pos + local_skip;
-                       fts_tokenizer_address_update_parent(tok, data+pos,
-                                                           local_skip);
-                       if (tok->tokenizer.parent != NULL)
-                               return fts_tokenizer_address_parent_data(tok, token_r);
-                       else {
-                               return fts_tokenizer_address_current_token(tok, token_r);
-                       }
+                       *skip_r = pos;
+                       if (fts_tokenizer_address_parent_data(tok, token_r))
+                               return 1;
+                       return fts_tokenizer_address_current_token(tok, token_r);
                default:
                        i_unreached();
                }

diff --git a/src/lib-fts/test-fts-tokenizer.c b/src/lib-fts/test-fts-tokenizer.c
index 0fa41bf193caba4ad29ee3ac57212343e657a66c..1912335836df0656bf45dc2849f1cf73ab6d0bd1 100644 (file)
--- a/src/lib-fts/test-fts-tokenizer.c
+++ b/src/lib-fts/test-fts-tokenizer.c
@@ -518,6 +518,48 @@ static void test_fts_tokenizer_address_rand(void)
        test_end();
 }
 
+static void test_fts_tokenizer_address_search(void)
+{
+       static const unsigned char input[] =
+               "@invalid invalid@ abc@example.com, "
+               "Bar Baz <bar@example.org>, "
+               "foo@domain";
+       static const char *const expected_output[] = {
+               "invalid", "invalid", "abc@example.com", "Bar", "Baz",
+               "bar@example.org", "foo@domain", NULL
+       };
+       static const char *const settings[] = { "search", "" };
+       struct fts_tokenizer *tok, *gen_tok;
+       const char * const *eopp = expected_output;
+       const char *token, *error;
+       unsigned int i;
+       int ret;
+
+       test_begin("fts tokenizer search email address + parent, input one character at a time");
+       fts_tokenizers_init();
+
+       test_assert(fts_tokenizer_create(fts_tokenizer_generic, NULL, NULL, &gen_tok, &error) == 0);
+       test_assert(fts_tokenizer_create(fts_tokenizer_email_address, gen_tok, settings, &tok, &error) == 0);
+
+       for (i = 0; i <= sizeof(input)-1; ) {
+               ret = i < sizeof(input)-1 ?
+                       fts_tokenizer_next(tok, &input[i], 1, &token) :
+                       fts_tokenizer_next(tok, NULL, 0, &token);
+               if (ret == 0) {
+                       i++;
+                       continue;
+               }
+               test_assert(*eopp != NULL);
+               test_assert(null_strcmp(token, *eopp) == 0);
+               eopp++;
+       }
+       test_assert(*eopp == NULL);
+       fts_tokenizer_unref(&tok);
+       fts_tokenizer_unref(&gen_tok);
+       fts_tokenizers_deinit();
+       test_end();
+}
+
 int main(void)
 {
        static void (*test_functions[])(void) = {
@@ -534,6 +576,7 @@ int main(void)
                test_fts_tokenizer_address_char,
                test_fts_tokenizer_address_line,
                test_fts_tokenizer_address_rand,
+               test_fts_tokenizer_address_search,
                NULL
        };