From: Phil Carmody <phil@dovecot.fi>
Date: Wed, 16 May 2018 14:04:35 +0000 (+0300)
Subject: lib-fts: tokenizer-generic - move state history setting into helper
X-Git-Tag: 2.3.9~1284
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a664a335e314f4a6aa9f4e179e862059c1f8a878;p=thirdparty%2Fdovecot%2Fcore.git

lib-fts: tokenizer-generic - move state history setting into helper

We can read the value directly, but for encapsulation it's best to do
the shifting of the token type history into a helper in a similar way
as how it is done for tr29 tokenising.

Signed-off-by: Phil Carmody <phil@dovecot.fi>
---

diff --git a/src/lib-fts/fts-tokenizer-generic.c b/src/lib-fts/fts-tokenizer-generic.c
index a49b1655fd..2cec8f1e00 100644
--- a/src/lib-fts/fts-tokenizer-generic.c
+++ b/src/lib-fts/fts-tokenizer-generic.c
@@ -101,6 +101,13 @@ fts_tokenizer_generic_destroy(struct fts_tokenizer *_tok)
 	i_free(tok);
 }
 
+static inline void
+shift_prev_type(struct generic_fts_tokenizer *tok, enum letter_type lt)
+{
+	tok->prev_prev_type = tok->prev_type;
+	tok->prev_type = lt;
+}
+
 static bool
 fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok,
                                            const char **token_r)
@@ -128,7 +135,7 @@ fts_tokenizer_generic_simple_current_token(struct generic_fts_tokenizer *tok,
 		t_strndup(tok->token->data, len);
 	buffer_set_used_size(tok->token, 0);
 	tok->untruncated_length = 0;
-	tok->prev_type = LETTER_TYPE_NONE;
+	shift_prev_type(tok, LETTER_TYPE_NONE);
 	return len > 0;
 }
 
@@ -221,7 +228,7 @@ fts_tokenizer_generic_simple_next(struct fts_tokenizer *_tok,
 			   subsequent apostrophes are handled by prefix
 			   skipping or by ignoring empty tokens - they will be
 			   dropped in any case. */
-			tok->prev_type = LETTER_TYPE_NONE;
+			shift_prev_type(tok, LETTER_TYPE_NONE);
 		} else if (apostrophe) {
 			/* all apostrophes require special handling */
 			const unsigned char apostrophe_char = '\'';
@@ -230,9 +237,9 @@ fts_tokenizer_generic_simple_next(struct fts_tokenizer *_tok,
 			if (tok->token->used > 0)
 				tok_append_truncated(tok, &apostrophe_char, 1);
 			start = i + char_size;
-			tok->prev_type = LETTER_TYPE_SINGLE_QUOTE;
+			shift_prev_type(tok, LETTER_TYPE_SINGLE_QUOTE);
 		} else {
-			tok->prev_type = LETTER_TYPE_NONE;
+			shift_prev_type(tok, LETTER_TYPE_NONE);
 		}
 	}
 	/* word boundary not found yet */