From: Vsevolod Stakhov <vsevolod@highsecure.ru>
Date: Tue, 22 Nov 2016 13:00:50 +0000 (+0000)
Subject: [Fix] Filter incorrect training data
X-Git-Tag: 1.4.1~105
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ade230a994a585a451e1410bd6a606c5e583068e;p=thirdparty%2Frspamd.git

[Fix] Filter incorrect training data
---

diff --git a/src/plugins/lua/fann_redis.lua b/src/plugins/lua/fann_redis.lua
index 64d1bf142b..aa4efd4c63 100644
--- a/src/plugins/lua/fann_redis.lua
+++ b/src/plugins/lua/fann_redis.lua
@@ -492,15 +492,19 @@ local function train_fann(_, ev_base, elt)
       local inputs = {}
       local outputs = {}
 
+      local n = rspamd_config:get_symbols_count() + rspamd_count_metatokens()
+      local filt = function(elts)
+        return #elts == n
+      end
+
       fun.each(function(spam_sample, ham_sample)
         table.insert(inputs, fun.totable(spam_sample))
         table.insert(outputs, {1.0})
         table.insert(inputs, fun.totable(ham_sample))
         table.insert(outputs, {-1.0})
-      end, fun.zip(spam_elts, ham_elts))
+      end, fun.zip(fun.filter(filt, spam_elts), fun.filter(filt, ham_elts)))
 
       -- Now we can train fann
-      local n = rspamd_config:get_symbols_count() + rspamd_count_metatokens()
       if not fanns[elt] or not fanns[elt].fann_train
         or n ~= fanns[elt].fann_train:get_inputs() then
         -- Create fann if it does not exist