[Minor] Properly set flag on text tokens

diff --git a/src/libstat/tokenizers/tokenizers.c b/src/libstat/tokenizers/tokenizers.c
index 0902ceb05578bbb5f2e095aef1d8b608f3890ba7..ac7f8be8525f7128cc088de8861150b464b0752f 100644 (file)
--- a/src/libstat/tokenizers/tokenizers.c
+++ b/src/libstat/tokenizers/tokenizers.c
@@ -393,7 +393,7 @@ start_over:
                                                        if (rspamd_utf_word_valid (text, text + len, last, p)) {
                                                                token.begin = text + last;
                                                                token.len = p - last;
-                                                               token.flags = 0;
+                                                               token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT;
                                                        }
                                                }
                                        }
@@ -407,7 +407,7 @@ start_over:
                                                if (rspamd_utf_word_valid (text, text + len, last, p)) {
                                                        token.begin = text + last;
                                                        token.len = p - last;
-                                                       token.flags = 0;
+                                                       token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT;
                                                }
                                        }
                                        else {
@@ -415,7 +415,7 @@ start_over:
                                                if (rspamd_utf_word_valid (text, text + len, last, p)) {
                                                        token.begin = text + last;
                                                        token.len = p - last;
-                                                       token.flags = 0;
+                                                       token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT;
                                                }
                                        }
                                }
@@ -423,6 +423,7 @@ start_over:
                                        if (rspamd_utf_word_valid (text, text + len, last, p)) {
                                                token.begin = text + last;
                                                token.len = p - last;
+                                               token.flags = RSPAMD_STAT_TOKEN_FLAG_TEXT;
                                        }
                                }