max_string_line_length = 500
max_comment_line_length = 500
+max_line_length = 140
else
common.symbols[sym] = s
if not auth_results[auth_type] then
- auth_results[auth_type] = {key}
+ auth_results[auth_type] = { key }
else
table.insert(auth_results[auth_type], key)
end
-- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=fA8VVvJ8;
-- dkim=neutral (body hash did not verify) header.d=example.com header.s=sel header.b=f8pM8o90;
- for _,dres in ipairs(dkim_results) do
+ for _, dres in ipairs(dkim_results) do
local ar_string = 'none'
if dres.result == 'reject' then
hdr = hdr .. ' (policy=' .. lua_util.maybe_smtp_quote_value(opts[2]) .. ')'
hdr = hdr .. ' header.from=' .. lua_util.maybe_smtp_quote_value(opts[1])
elseif key ~= 'none' then
- local t = {opts[1]:match('^([^%s]+) : (.*)$')}
+ local t = { opts[1]:match('^([^%s]+) : (.*)$') }
if #t > 0 then
local dom = t[1]
local rsn = t[2]
-- Main type
local sender
local sender_type
- local smtp_from = task:get_from({'smtp','orig'})
+ local smtp_from = task:get_from({ 'smtp', 'orig' })
if smtp_from and
smtp_from[1] and
hdr = string.format('%s=%s', auth_type, key)
end
-
table.insert(hdr_parts, hdr)
end
end
end
local u = task:get_user()
- local smtp_from = task:get_from({'smtp','orig'})
+ local smtp_from = task:get_from({ 'smtp', 'orig' })
if u and smtp_from then
- local hdr = {[1] = 'auth=pass'}
+ local hdr = { [1] = 'auth=pass' }
if settings['add_smtp_user'] then
- table.insert(hdr,'smtp.auth=' .. lua_util.maybe_smtp_quote_value(u))
+ table.insert(hdr, 'smtp.auth=' .. lua_util.maybe_smtp_quote_value(u))
end
if smtp_from[1]['addr'] then
- table.insert(hdr,'smtp.mailfrom=' .. lua_util.maybe_smtp_quote_value(smtp_from[1]['addr']))
+ table.insert(hdr, 'smtp.mailfrom=' .. lua_util.maybe_smtp_quote_value(smtp_from[1]['addr']))
end
- table.insert(hdr_parts, table.concat(hdr,' '))
+ table.insert(hdr_parts, table.concat(hdr, ' '))
end
if #hdr_parts > 0 then
local S = lpeg.S
local V = lpeg.V
local C = lpeg.C
- local space = S(" ")^0
- local doublequoted = space * P'"' * ((1 - S'"\r\n\f\\') + (P'\\' * 1))^0 * '"' * space
- local comment = space * P{ "(" * ((1 - S"()") + V(1))^0 * ")" } * space
- local name = C((1 - S('=(" '))^1) * space
+ local space = S(" ") ^ 0
+ local doublequoted = space * P '"' * ((1 - S '"\r\n\f\\') + (P '\\' * 1)) ^ 0 * '"' * space
+ local comment = space * P { "(" * ((1 - S "()") + V(1)) ^ 0 * ")" } * space
+ local name = C((1 - S('=(" ')) ^ 1) * space
local pair = lpeg.Cg(name * "=" * space * name) * space
- aar_elt_grammar = lpeg.Cf(lpeg.Ct("") * (pair + comment + doublequoted)^1, rawset)
+ aar_elt_grammar = lpeg.Cf(lpeg.Ct("") * (pair + comment + doublequoted) ^ 1, rawset)
end
return aar_elt_grammar:match(elt)
local function save_cached_key(date_str, secret_key, region, service, req_type, key)
local numdate = tonumber(date_str)
-- expire old buckets
- for k,_ in pairs(cached_keys) do
+ for k, _ in pairs(cached_keys) do
if k < numdate then
cached_keys[k] = nil
end
end
-
local bucket = cached_keys[tonumber(date_str)]
local idx = string.format('%s.%s.%s.%s', secret_key, region, service, req_type)
end
local hmac1 = rspamd_crypto_hash.create_specific_keyed("AWS4" .. secret_key, "sha256", date_str):bin()
- local hmac2 = rspamd_crypto_hash.create_specific_keyed(hmac1, "sha256",region):bin()
+ local hmac2 = rspamd_crypto_hash.create_specific_keyed(hmac1, "sha256", region):bin()
local hmac3 = rspamd_crypto_hash.create_specific_keyed(hmac2, "sha256", service):bin()
local final_key = rspamd_crypto_hash.create_specific_keyed(hmac3, "sha256", req_type):bin()
end, headers_to_sign))
local header_names = lua_util.keys(hdr_canon)
table.sort(header_names)
- for _,hn in ipairs(header_names) do
+ for _, hn in ipairs(header_names) do
local v = hdr_canon[hn]
lua_util.debugm(N, 'update signature with the header %s, %s',
hn, v)
lua_util.debugm(N, 'headers list to sign: %s', hdrs_list)
sha_ctx:update(string.format('\n%s\n%s', hdrs_list, hex_hash))
- return sha_ctx:hex(),hdrs_list
+ return sha_ctx:hex(), hdrs_list
end
exports.aws_canon_request_hash = aws_canon_request_hash
-local aws_authorization_hdr_args_schema = ts.shape{
+local aws_authorization_hdr_args_schema = ts.shape {
date = ts.string + ts['nil'] / today_canonical,
secret_key = ts.string,
- method = ts.string + ts['nil'] / function() return 'GET' end,
+ method = ts.string + ts['nil'] / function()
+ return 'GET'
+ end,
uri = ts.string,
region = ts.string,
- service = ts.string + ts['nil'] / function() return 's3' end,
- req_type = ts.string + ts['nil'] / function() return 'aws4_request' end,
+ service = ts.string + ts['nil'] / function()
+ return 's3'
+ end,
+ req_type = ts.string + ts['nil'] / function()
+ return 'aws4_request'
+ end,
headers = ts.map_of(ts.string, ts.string),
key_id = ts.string,
}
--
--]]
local function aws_authorization_hdr(tbl, transformed)
- local res,err
+ local res, err
if not transformed then
- res,err = aws_authorization_hdr_args_schema:transform(tbl)
+ res, err = aws_authorization_hdr_args_schema:transform(tbl)
assert(res, err)
else
res = tbl
local signing_key = aws_signing_key(res.date, res.secret_key, res.region, res.service,
res.req_type)
assert(signing_key ~= nil)
- local signed_sha,signed_hdrs = aws_canon_request_hash(res.method, res.uri,
+ local signed_sha, signed_hdrs = aws_canon_request_hash(res.method, res.uri,
res.headers)
if not signed_sha then
lua_util.debugm(N, "string to sign: %s", string_to_sign)
local hmac = rspamd_crypto_hash.create_specific_keyed(signing_key, 'sha256', string_to_sign):hex()
lua_util.debugm(N, "hmac: %s", hmac)
- local auth_hdr = string.format('AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/%s,'..
+ local auth_hdr = string.format('AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/%s,' ..
'SignedHeaders=%s,Signature=%s',
res.key_id, res.date, res.region, res.service, res.req_type,
signed_hdrs, hmac)
--
--]]
local function aws_request_enrich(tbl, content)
- local res,err = aws_authorization_hdr_args_schema:transform(tbl)
+ local res, err = aws_authorization_hdr_args_schema:transform(tbl)
assert(res, err)
local content_sha256 = rspamd_crypto_hash.create_specific('sha256', content):hex()
local hdrs = res.headers
assert(aws_canon_request_hash('GET', '/test.txt', test_request_hdrs) ==
'7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972')
-assert(aws_authorization_hdr{
+assert(aws_authorization_hdr {
date = '20130524',
region = 'us-east-1',
headers = test_request_hdrs,
end
if in_class then
- return false,string.format(
+ return false, string.format(
'already in class %s; probability %.2f%%',
cl, math.abs((prob - 0.5) * 200.0))
end
local mime_rcpts = 'undef'
local mr = task:get_recipients('mime')
if mr then
- for _,r in ipairs(mr) do
+ for _, r in ipairs(mr) do
if mime_rcpts == 'undef' then
mime_rcpts = r.addr
else
end
-- We have autolearn config so let's figure out what is requested
- local verdict,score = lua_verdict.get_specific_verdict("bayes", task)
- local learn_spam,learn_ham = false, false
+ local verdict, score = lua_verdict.get_specific_verdict("bayes", task)
+ local learn_spam, learn_ham = false, false
if verdict == 'passthrough' then
-- No need to autolearn
local ham_learns = task:get_mempool():get_variable('ham_learns', 'int64') or 0
local min_balance = 0.9
- if conf.min_balance then min_balance = conf.min_balance end
+ if conf.min_balance then
+ min_balance = conf.min_balance
+ end
if spam_learns > 0 or ham_learns > 0 then
local max_ratio = 1.0 / min_balance
local spam_learns_ratio = spam_learns / (ham_learns + 1)
- if spam_learns_ratio > max_ratio and learn_spam then
+ if spam_learns_ratio > max_ratio and learn_spam then
lua_util.debugm(N, task,
'skip learning spam, balance is not satisfied: %s < %s; %s spam learns; %s ham learns',
spam_learns_ratio, min_balance, spam_learns, ham_learns)
end
local ham_learns_ratio = ham_learns / (spam_learns + 1)
- if ham_learns_ratio > max_ratio and learn_ham then
+ if ham_learns_ratio > max_ratio and learn_ham then
lua_util.debugm(N, task,
'skip learning ham, balance is not satisfied: %s < %s; %s spam learns; %s ham learns',
ham_learns_ratio, min_balance, spam_learns, ham_learns)
local function gen_keys(tbl)
if implicit_array then
- for _,v in ipairs(tbl) do
+ for _, v in ipairs(tbl) do
if v.name then
- table.insert(keys, {v.name, v})
+ table.insert(keys, { v.name, v })
v.name = nil
else
-- Very tricky to distinguish:
-- group {name = "foo" ... } + group "blah" { ... }
- for gr_name,gr in pairs(v) do
+ for gr_name, gr in pairs(v) do
if type(gr_name) ~= 'number' then
-- We can also have implicit arrays here
local gr_implicit = is_implicit(gr)
if gr_implicit then
- for _,gr_elt in ipairs(gr) do
- table.insert(keys, {gr_name, gr_elt})
+ for _, gr_elt in ipairs(gr) do
+ table.insert(keys, { gr_name, gr_elt })
end
else
- table.insert(keys, {gr_name, gr})
+ table.insert(keys, { gr_name, gr })
end
end
end
end
else
if tbl.name then
- table.insert(keys, {tbl.name, tbl})
+ table.insert(keys, { tbl.name, tbl })
tbl.name = nil
else
- for k,v in pairs(tbl) do
+ for k, v in pairs(tbl) do
if type(k) ~= 'number' then
-- We can also have implicit arrays here
local sym_implicit = is_implicit(v)
if sym_implicit then
- for _,elt in ipairs(v) do
- table.insert(keys, {k, elt})
+ for _, elt in ipairs(v) do
+ table.insert(keys, { k, elt })
end
else
- table.insert(keys, {k, v})
+ table.insert(keys, { k, v })
end
end
end
end
local function group_transform(cfg, k, v)
- if v.name then k = v.name end
+ if v.name then
+ k = v.name
+ end
local new_group = {
symbols = {}
}
- if v.enabled then new_group.enabled = v.enabled end
- if v.disabled then new_group.disabled = v.disabled end
- if v.max_score then new_group.max_score = v.max_score end
+ if v.enabled then
+ new_group.enabled = v.enabled
+ end
+ if v.disabled then
+ new_group.disabled = v.disabled
+ end
+ if v.max_score then
+ new_group.max_score = v.max_score
+ end
if v.symbol then
- for sk,sv in metric_pairs(v.symbol) do
+ for sk, sv in metric_pairs(v.symbol) do
if sv.name then
sk = sv.name
sv.name = nil -- Remove field
end
end
- if not cfg.group then cfg.group = {} end
+ if not cfg.group then
+ cfg.group = {}
+ end
if cfg.group[k] then
cfg.group[k] = lua_util.override_defaults(cfg.group[k], new_group)
for gr_name, gr in pairs(groups) do
if not gr.symbols then
local cnt = 0
- for _,_ in pairs(gr) do cnt = cnt + 1 end
+ for _, _ in pairs(gr) do
+ cnt = cnt + 1
+ end
if cnt == 0 then
logger.debugx('group %s is empty', gr_name)
-- merged group definition
local function merge_groups(groups)
local ret = {}
- for k,gr in pairs(groups) do
+ for k, gr in pairs(groups) do
if type(k) == 'number' then
- for key,sec in pairs(gr) do
+ for key, sec in pairs(gr) do
ret[key] = sec
end
else
if rspamd_util.file_exists(local_stat) and
rspamd_util.file_exists(local_bayes) then
- logger.warnx(rspamd_config, 'conflicting files %s and %s are found: '..
+ logger.warnx(rspamd_config, 'conflicting files %s and %s are found: ' ..
'Rspamd classifier configuration might be broken!', local_stat, local_bayes)
end
end
local function surbl_section_convert(cfg, section)
local rbl_section = cfg.rbl.rbls
local wl = section.whitelist
- for name,value in pairs(section.rules or {}) do
+ for name, value in pairs(section.rules or {}) do
if rbl_section[name] then
logger.warnx(rspamd_config, 'conflicting names in surbl and rbl rules: %s, prefer surbl rule!',
name)
converted.whitelist = wl
end
- for k,v in pairs(value) do
+ for k, v in pairs(value) do
local skip = false
-- Rename
- if k == 'suffix' then k = 'rbl' end
- if k == 'ips' then k = 'returncodes' end
- if k == 'bits' then k = 'returnbits' end
- if k == 'noip' then k = 'no_ip' end
+ if k == 'suffix' then
+ k = 'rbl'
+ end
+ if k == 'ips' then
+ k = 'returncodes'
+ end
+ if k == 'bits' then
+ k = 'returnbits'
+ end
+ if k == 'noip' then
+ k = 'no_ip'
+ end
-- Crappy legacy
if k == 'options' then
if v == 'noip' or v == 'no_ip' then
local function emails_section_convert(cfg, section)
local rbl_section = cfg.rbl.rbls
local wl = section.whitelist
- for name,value in pairs(section.rules or {}) do
+ for name, value in pairs(section.rules or {}) do
if rbl_section[name] then
logger.warnx(rspamd_config, 'conflicting names in emails and rbl rules: %s, prefer emails rule!',
name)
converted.whitelist = wl
end
- for k,v in pairs(value) do
+ for k, v in pairs(value) do
local skip = false
-- Rename
- if k == 'dnsbl' then k = 'rbl' end
- if k == 'check_replyto' then k = 'replyto' end
- if k == 'hashlen' then k = 'hash_len' end
- if k == 'encoding' then k = 'hash_format' end
- if k == 'domain_only' then k = 'emails_domainonly' end
- if k == 'delimiter' then k = 'emails_delimiter' end
+ if k == 'dnsbl' then
+ k = 'rbl'
+ end
+ if k == 'check_replyto' then
+ k = 'replyto'
+ end
+ if k == 'hashlen' then
+ k = 'hash_len'
+ end
+ if k == 'encoding' then
+ k = 'hash_format'
+ end
+ if k == 'domain_only' then
+ k = 'emails_domainonly'
+ end
+ if k == 'delimiter' then
+ k = 'emails_delimiter'
+ end
if k == 'skip_body' then
skip = true
if v then
logger.errx('no actions defined')
else
-- Perform sanity check for actions
- local actions_defs = {'no action', 'no_action', -- In case if that's added
- 'greylist', 'add header', 'add_header',
- 'rewrite subject', 'rewrite_subject', 'quarantine',
- 'reject', 'discard'}
+ local actions_defs = { 'no action', 'no_action', -- In case if that's added
+ 'greylist', 'add header', 'add_header',
+ 'rewrite subject', 'rewrite_subject', 'quarantine',
+ 'reject', 'discard' }
if not cfg.actions['no action'] and not cfg.actions['no_action'] and
- not cfg.actions['accept'] then
- for _,d in ipairs(actions_defs) do
+ not cfg.actions['accept'] then
+ for _, d in ipairs(actions_defs) do
if cfg.actions[d] then
local action_score = nil
elseif type(action_score) == 'number' and action_score < 0 then
cfg.actions['no_action'] = cfg.actions[d] - 0.001
logger.infox(rspamd_config, 'set no_action score to: %s, as action %s has negative score',
- cfg.actions['no_action'], d)
+ cfg.actions['no_action'], d)
break
end
end
actions_set['grow_factor'] = true
actions_set['subject'] = true
- for k,_ in pairs(cfg.actions) do
+ for k, _ in pairs(cfg.actions) do
if not actions_set[k] then
logger.warnx(rspamd_config, 'unknown element in actions section: %s', k)
end
'reject',
'discard'
}
- for i=1,(#actions_order - 1) do
+ for i = 1, (#actions_order - 1) do
local act = actions_order[i]
if cfg.actions[act] and type(cfg.actions[act]) == 'number' then
local score = cfg.actions[act]
- for j=i+1,#actions_order do
+ for j = i + 1, #actions_order do
local next_act = actions_order[j]
if cfg.actions[next_act] and type(cfg.actions[next_act]) == 'number' then
local next_score = cfg.actions[next_act]
if next_score <= score then
- logger.errx(rspamd_config, 'invalid actions thresholds order: action %s (%s) must have lower '..
+ logger.errx(rspamd_config, 'invalid actions thresholds order: action %s (%s) must have lower ' ..
'score than action %s (%s)', act, score, next_act, next_score)
ret = false
end
-- Again: legacy stuff :(
if not cfg.dkim.sign_headers then
local sec = cfg.dkim_signing
- if sec and sec[1] then sec = cfg.dkim_signing[1] end
+ if sec and sec[1] then
+ sec = cfg.dkim_signing[1]
+ end
if sec and sec.sign_headers then
cfg.dkim.sign_headers = sec.sign_headers
end
-- DKIM signing/ARC legacy
- for _, mod in ipairs({'dkim_signing', 'arc'}) do
+ for _, mod in ipairs({ 'dkim_signing', 'arc' }) do
if cfg[mod] then
if cfg[mod].auth_only ~= nil then
if cfg[mod].sign_authenticated ~= nil then
end
-- Try to find some obvious issues with configuration
- for k,v in pairs(cfg) do
- if type(v) == 'table' and v[k] and type (v[k]) == 'table' then
+ for k, v in pairs(cfg) do
+ if type(v) == 'table' and v[k] and type(v[k]) == 'table' then
logger.errx('nested section: %s { %s { ... } }, it is likely a configuration error',
- k, k)
+ k, k)
end
end
}
end
- if not cfg.reputation.rules then cfg.reputation.rules = {} end
+ if not cfg.reputation.rules then
+ cfg.reputation.rules = {}
+ end
- if not fun.any(function(_, v) return v.selector and v.selector.ip end,
+ if not fun.any(function(_, v)
+ return v.selector and v.selector.ip
+ end,
cfg.reputation.rules) then
logger.infox(rspamd_config, 'attach ip reputation element to use it')
end
local function ch_number(a)
- if (a+2^52)-2^52 == a then
+ if (a + 2 ^ 52) - 2 ^ 52 == a then
-- Integer
return tostring(math.floor(a))
end
-- Converts an array to a string suitable for clickhouse
local function array_to_string(ar)
- for i,elt in ipairs(ar) do
+ for i, elt in ipairs(ar) do
local t = type(elt)
if t == 'string' then
ar[i] = string.format('\'%s\'', clickhouse_quote(elt))
-- Converts a row into TSV, taking extra care about arrays
local function row_to_tsv(row)
- for i,elt in ipairs(row) do
+ for i, elt in ipairs(row) do
local t = type(elt)
if t == 'table' then
row[i] = '[' .. array_to_string(elt) .. ']'
local parser = ucl.parser()
local res, err
if type(s) == 'string' then
- res,err = parser:parse_string(s)
+ res, err = parser:parse_string(s)
else
- res,err = parser:parse_text(s)
+ res, err = parser:parse_text(s)
end
if not res then
local res, err
if type(s) == 'string' then
- res,err = parser:parse_string(s)
+ res, err = parser:parse_string(s)
else
- res,err = parser:parse_text(s)
+ res, err = parser:parse_text(s)
end
if not res then
return 'bad json', {}
end
- return nil,json
+ return nil, json
end
-- Helper to generate HTTP closure
local function mk_http_select_cb(upstream, params, ok_cb, fail_cb, row_cb)
local function http_cb(err_message, code, data, _)
if code ~= 200 or err_message then
- if not err_message then err_message = data end
+ if not err_message then
+ err_message = data
+ end
local ip_addr = upstream:get_addr():to_string(true)
if fail_cb then
else
local ip_addr = upstream:get_addr():to_string(true)
rspamd_logger.errx(params.log_obj,
- "request failed on clickhouse server %s: %s",
- ip_addr, 'failed to parse reply')
+ "request failed on clickhouse server %s: %s",
+ ip_addr, 'failed to parse reply')
end
end
end
local function mk_http_insert_cb(upstream, params, ok_cb, fail_cb)
local function http_cb(err_message, code, data, _)
if code ~= 200 or err_message then
- if not err_message then err_message = data end
+ if not err_message then
+ err_message = data
+ end
local ip_addr = upstream:get_addr():to_string(true)
if fail_cb then
upstream:ok()
if ok_cb then
- local err,parsed = parse_clickhouse_response_json(data)
+ local err, parsed = parse_clickhouse_response_json(data)
if err then
fail_cb(params, err, data)
-- @example
--
--]]
-exports.select = function (upstream, settings, params, query, ok_cb, fail_cb, row_cb)
+exports.select = function(upstream, settings, params, query, ok_cb, fail_cb, row_cb)
local http_params = {}
- for k,v in pairs(params) do http_params[k] = v end
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
http_params.callback = mk_http_select_cb(upstream, http_params, ok_cb, fail_cb, row_cb)
http_params.gzip = settings.use_gzip
-- @example
--
--]]
-exports.select_sync = function (upstream, settings, params, query, row_cb)
+exports.select_sync = function(upstream, settings, params, query, row_cb)
local http_params = {}
- for k,v in pairs(params) do http_params[k] = v end
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
http_params.gzip = settings.use_gzip
http_params.mime_type = 'text/plain'
-- @example
--
--]]
-exports.insert = function (upstream, settings, params, query, rows,
- ok_cb, fail_cb)
+exports.insert = function(upstream, settings, params, query, rows,
+ ok_cb, fail_cb)
local http_params = {}
- for k,v in pairs(params) do http_params[k] = v end
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb)
http_params.gzip = settings.use_gzip
http_params.user = settings.user
http_params.password = settings.password
http_params.method = 'POST'
- http_params.body = {rspamd_text.fromtable(rows, '\n'), '\n'}
+ http_params.body = { rspamd_text.fromtable(rows, '\n'), '\n' }
http_params.log_obj = params.task or params.config
if not http_params.url then
-- @example
--
--]]
-exports.generic = function (upstream, settings, params, query,
+exports.generic = function(upstream, settings, params, query,
ok_cb, fail_cb)
local http_params = {}
- for k,v in pairs(params) do http_params[k] = v end
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
http_params.callback = mk_http_insert_cb(upstream, http_params, ok_cb, fail_cb)
http_params.gzip = settings.use_gzip
-- @example
--
--]]
-exports.generic_sync = function (upstream, settings, params, query)
+exports.generic_sync = function(upstream, settings, params, query)
local http_params = {}
- for k,v in pairs(params) do http_params[k] = v end
+ for k, v in pairs(params) do
+ http_params[k] = v
+ end
http_params.gzip = settings.use_gzip
http_params.mime_type = 'text/plain'
return response.content, response
else
lua_util.debugm(N, http_params.log_obj, "clickhouse generic response: %1", response)
- local e,obj = parse_clickhouse_response_json(params, response.content)
+ local e, obj = parse_clickhouse_response_json(params, response.content)
if e then
- return e,nil
+ return e, nil
end
return nil, obj
end
local function gen_grammar()
if not ical_grammar then
local wsp = l.S(" \t\v\f")
- local crlf = (l.P"\r"^-1 * l.P"\n") + l.P"\r"
- local eol = (crlf * #crlf) + (crlf - (crlf^-1 * wsp))
- local name = l.C((l.P(1) - (l.P":"))^1) / function(v) return (v:gsub("[\n\r]+%s","")) end
- local value = l.C((l.P(1) - eol)^0) / function(v) return (v:gsub("[\n\r]+%s","")) end
- ical_grammar = name * ":" * wsp^0 * value * eol^-1
+ local crlf = (l.P "\r" ^ -1 * l.P "\n") + l.P "\r"
+ local eol = (crlf * #crlf) + (crlf - (crlf ^ -1 * wsp))
+ local name = l.C((l.P(1) - (l.P ":")) ^ 1) / function(v)
+ return (v:gsub("[\n\r]+%s", ""))
+ end
+ local value = l.C((l.P(1) - eol) ^ 0) / function(v)
+ return (v:gsub("[\n\r]+%s", ""))
+ end
+ ical_grammar = name * ":" * wsp ^ 0 * value * eol ^ -1
end
return ical_grammar
local function extract_text_data(specific)
local fun = require "fun"
- local tbl = fun.totable(fun.map(function(e) return e[2]:lower() end, specific.elts))
+ local tbl = fun.totable(fun.map(function(e)
+ return e[2]:lower()
+ end, specific.elts))
return table.concat(tbl, '\n')
end
-- Keys that can have visible urls
-local url_keys = lua_util.list_to_hash{
+local url_keys = lua_util.list_to_hash {
'description',
'location',
'summary',
}
local function process_ical(input, mpart, task)
- local control={n='\n', r=''}
+ local control = { n = '\n', r = '' }
local rspamd_url = require "rspamd_url"
local escaper = l.Ct((gen_grammar() / function(key, value)
value = value:gsub("\\(.)", control)
local local_urls = rspamd_url.all(task:get_mempool(), value)
if local_urls and #local_urls > 0 then
- for _,u in ipairs(local_urls) do
+ for _, u in ipairs(local_urls) do
lua_util.debugm(N, task, 'ical: found URL in ical key "%s": %s',
- key, tostring(u))
+ key, tostring(u))
task:inject_url(u, mpart)
end
end
end
lua_util.debugm(N, task, 'ical: ical key %s = "%s"',
key, value)
- return {key, value}
- end)^1)
+ return { key, value }
+ end) ^ 1)
local elts = escaper:match(input)
local content_modules = {
ical = {
- mime_type = {"text/calendar", "application/calendar"},
+ mime_type = { "text/calendar", "application/calendar" },
module = require "lua_content/ical",
- extensions = {'ics'},
+ extensions = { 'ics' },
output = "text"
},
vcf = {
- mime_type = {"text/vcard", "application/vcard"},
+ mime_type = { "text/vcard", "application/vcard" },
module = require "lua_content/vcard",
- extensions = {'vcf'},
+ extensions = { 'vcf' },
output = "text"
},
pdf = {
mime_type = "application/pdf",
module = require "lua_content/pdf",
- extensions = {'pdf'},
+ extensions = { 'pdf' },
output = "table"
},
}
local function init()
modules_by_mime_type = {}
modules_by_extension = {}
- for k,v in pairs(content_modules) do
+ for k, v in pairs(content_modules) do
if v.mime_type then
if type(v.mime_type) == 'table' then
- for _,mt in ipairs(v.mime_type) do
- modules_by_mime_type[mt] = {k, v}
+ for _, mt in ipairs(v.mime_type) do
+ modules_by_mime_type[mt] = { k, v }
end
else
- modules_by_mime_type[v.mime_type] = {k, v}
+ modules_by_mime_type[v.mime_type] = { k, v }
end
end
if v.extensions then
- for _,ext in ipairs(v.extensions) do
- modules_by_extension[ext] = {k, v}
+ for _, ext in ipairs(v.extensions) do
+ modules_by_extension[ext] = { k, v }
end
end
end
end
-
return exports
\ No newline at end of file
rspamd_trie.flags.no_start)
local function compile_pats(patterns, indexes, compile_flags)
local strs = {}
- for what,data in pairs(patterns) do
- for i,pat in ipairs(data.patterns) do
+ for what, data in pairs(patterns) do
+ for i, pat in ipairs(data.patterns) do
strs[#strs + 1] = pat
- indexes[#indexes + 1] = {what, data, pat, i}
+ indexes[#indexes + 1] = { what, data, pat, i }
end
end
local S = lpeg.S
local V = lpeg.V
local C = lpeg.C
- local D = R'09' -- Digits
+ local D = R '09' -- Digits
local grammar_elts = {}
end
local function pdf_id_unescape(s)
- return (s:gsub('#%d%d', function (cc)
+ return (s:gsub('#%d%d', function(cc)
return string.char(tonumber(cc:sub(2), 16))
end))
end
- local delim = S'()<>[]{}/%'
- grammar_elts.ws = S'\0 \r\n\t\f'
- local hex = R'af' + R'AF' + D
+ local delim = S '()<>[]{}/%'
+ grammar_elts.ws = S '\0 \r\n\t\f'
+ local hex = R 'af' + R 'AF' + D
-- Comments.
- local eol = P'\r\n' + '\n'
- local line = (1 - S'\r\n\f')^0 * eol^-1
- grammar_elts.comment = P'%' * line
+ local eol = P '\r\n' + '\n'
+ local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1
+ grammar_elts.comment = P '%' * line
-- Numbers.
- local sign = S'+-'^-1
- local decimal = D^1
- local float = D^1 * P'.' * D^0 + P'.' * D^1
+ local sign = S '+-' ^ -1
+ local decimal = D ^ 1
+ local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1
grammar_elts.number = C(sign * (float + decimal)) / tonumber
-- String
- grammar_elts.str = P{ "(" * C(((1 - S"()\\") + (P '\\' * 1) + V(1))^0) / pdf_string_unescape * ")" }
- grammar_elts.hexstr = P{"<" * C(hex^0) / pdf_hexstring_unescape * ">"}
+ grammar_elts.str = P { "(" * C(((1 - S "()\\") + (P '\\' * 1) + V(1)) ^ 0) / pdf_string_unescape * ")" }
+ grammar_elts.hexstr = P { "<" * C(hex ^ 0) / pdf_hexstring_unescape * ">" }
-- Identifier
- grammar_elts.id = P{'/' * C((1-(delim + grammar_elts.ws))^1) / pdf_id_unescape}
+ grammar_elts.id = P { '/' * C((1 - (delim + grammar_elts.ws)) ^ 1) / pdf_id_unescape }
-- Booleans (who care about them?)
grammar_elts.boolean = C(P("true") + P("false"))
-- Stupid references
- grammar_elts.ref = lpeg.Ct{lpeg.Cc("%REF%") * C(D^1) * " " * C(D^1) * " " * "R"}
+ grammar_elts.ref = lpeg.Ct { lpeg.Cc("%REF%") * C(D ^ 1) * " " * C(D ^ 1) * " " * "R" }
return grammar_elts
end
local V = lpeg.V
local gen = generic_grammar_elts()
- return lpeg.P{
+ return lpeg.P {
"EXPR";
- EXPR = gen.ws^0 * V("ELT")^0 * gen.ws^0,
+ EXPR = gen.ws ^ 0 * V("ELT") ^ 0 * gen.ws ^ 0,
ELT = V("ARRAY") + V("DICT") + V("ATOM"),
- ATOM = gen.ws^0 * (gen.comment + gen.boolean + gen.ref +
- gen.number + V("STRING") + gen.id) * gen.ws^0,
- DICT = "<<" * gen.ws^0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR")^0, rawset) * gen.ws^0 * ">>",
- KV_PAIR = lpeg.Cg(gen.id * gen.ws^0 * V("ELT") * gen.ws^0),
- ARRAY = "[" * gen.ws^0 * lpeg.Ct(V("ELT")^0) * gen.ws^0 * "]",
- STRING = lpeg.P{gen.str + gen.hexstr},
+ ATOM = gen.ws ^ 0 * (gen.comment + gen.boolean + gen.ref +
+ gen.number + V("STRING") + gen.id) * gen.ws ^ 0,
+ DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>",
+ KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ELT") * gen.ws ^ 0),
+ ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ELT") ^ 0) * gen.ws ^ 0 * "]",
+ STRING = lpeg.P { gen.str + gen.hexstr },
}
end
local S = lpeg.S
return P("q") + P("Q") + P("h")
- + S("WSsFfBb") * P("*")^0 + P("n")
+ + S("WSsFfBb") * P("*") ^ 0 + P("n")
end
local function gen_graphics_binary()
local text_quote_op = P('"')
local font_op = P("Tf")
- return lpeg.P{
+ return lpeg.P {
"EXPR";
- EXPR = gen.ws^0 * lpeg.Ct(V("COMMAND")^0),
+ EXPR = gen.ws ^ 0 * lpeg.Ct(V("COMMAND") ^ 0),
COMMAND = (V("UNARY") + V("BINARY") + V("TERNARY") + V("NARY") + V("TEXT") +
- V("FONT") + gen.comment) * gen.ws^0,
+ V("FONT") + gen.comment) * gen.ws ^ 0,
UNARY = unary_ops,
- BINARY = V("ARG") / empty * gen.ws^1 * binary_ops,
- TERNARY = V("ARG") / empty * gen.ws^1 * V("ARG") / empty * gen.ws^1 * ternary_ops,
- NARY = (gen.number / 0 * gen.ws^1)^1 * (gen.id / empty * gen.ws^0)^-1 * nary_op,
+ BINARY = V("ARG") / empty * gen.ws ^ 1 * binary_ops,
+ TERNARY = V("ARG") / empty * gen.ws ^ 1 * V("ARG") / empty * gen.ws ^ 1 * ternary_ops,
+ NARY = (gen.number / 0 * gen.ws ^ 1) ^ 1 * (gen.id / empty * gen.ws ^ 0) ^ -1 * nary_op,
ARG = V("ARRAY") + V("DICT") + V("ATOM"),
ATOM = (gen.comment + gen.boolean + gen.ref +
gen.number + V("STRING") + gen.id),
- DICT = "<<" * gen.ws^0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR")^0, rawset) * gen.ws^0 * ">>",
- KV_PAIR = lpeg.Cg(gen.id * gen.ws^0 * V("ARG") * gen.ws^0),
- ARRAY = "[" * gen.ws^0 * lpeg.Ct(V("ARG")^0) * gen.ws^0 * "]",
- STRING = lpeg.P{gen.str + gen.hexstr},
- TEXT = (V("TEXT_ARG") * gen.ws^1 * text_binary_op) +
- (V("ARG") / 0 * gen.ws^1 * V("ARG") / 0 * gen.ws^1 * V("TEXT_ARG") * gen.ws^1 * text_quote_op),
- FONT = (V("FONT_ARG") * gen.ws^1 * (gen.number / 0) * gen.ws^1 * font_op),
+ DICT = "<<" * gen.ws ^ 0 * lpeg.Cf(lpeg.Ct("") * V("KV_PAIR") ^ 0, rawset) * gen.ws ^ 0 * ">>",
+ KV_PAIR = lpeg.Cg(gen.id * gen.ws ^ 0 * V("ARG") * gen.ws ^ 0),
+ ARRAY = "[" * gen.ws ^ 0 * lpeg.Ct(V("ARG") ^ 0) * gen.ws ^ 0 * "]",
+ STRING = lpeg.P { gen.str + gen.hexstr },
+ TEXT = (V("TEXT_ARG") * gen.ws ^ 1 * text_binary_op) +
+ (V("ARG") / 0 * gen.ws ^ 1 * V("ARG") / 0 * gen.ws ^ 1 * V("TEXT_ARG") * gen.ws ^ 1 * text_quote_op),
+ FONT = (V("FONT_ARG") * gen.ws ^ 1 * (gen.number / 0) * gen.ws ^ 1 * font_op),
FONT_ARG = lpeg.Ct(lpeg.Cc("%font%") * gen.id),
TEXT_ARG = lpeg.Ct(V("STRING")) + V("TEXT_ARRAY"),
TEXT_ARRAY = "[" *
- lpeg.Ct(((gen.ws^0 * (gen.ws^0 * (gen.number / 0)^0 * gen.ws^0 * (gen.str + gen.hexstr)))^1)) * gen.ws^0 * "]",
+ lpeg.Ct(((gen.ws ^ 0 * (gen.ws ^ 0 * (gen.number / 0) ^ 0 * gen.ws ^ 0 * (gen.str + gen.hexstr))) ^ 1)) * gen.ws ^ 0 * "]",
}
end
if dict.Filter then
local filt = dict.Filter
if type(filt) == 'string' then
- filt = {filt}
+ filt = { filt }
end
if dict.DecodeParms then
if type(decode_params) == 'table' then
if decode_params.Predictor then
- return nil,'predictor exists'
+ return nil, 'predictor exists'
end
end
end
- for _,f in ipairs(filt) do
+ for _, f in ipairs(filt) do
uncompressed = apply_pdf_filter(uncompressed, f)
- if not uncompressed then break end
+ if not uncompressed then
+ break
+ end
end
end
- return uncompressed,nil
+ return uncompressed, nil
end
-- Conditionally extract stream data from object and attach it as obj.uncompressed
tonumber(maybe_dereference_object(dict.Length, pdf, task)) or 0)
local real_stream = obj.stream.data:span(1, len)
- local uncompressed,filter_err = maybe_apply_filter(dict, real_stream, pdf, task)
+ local uncompressed, filter_err = maybe_apply_filter(dict, real_stream, pdf, task)
if uncompressed then
obj.uncompressed = uncompressed
end
end
-
local function parse_object_grammar(obj, task, pdf)
-- Parse grammar
local obj_dict_span
end
if obj_dict_span:len() < config.max_processing_size then
- local ret,obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span)
+ local ret, obj_or_err = pcall(pdf_outer_grammar.match, pdf_outer_grammar, obj_dict_span)
if ret then
if obj.stream then
if contents and type(contents) == 'table' then
if contents[1] == '%REF%' then
-- Single reference
- contents = {contents}
+ contents = { contents }
end
obj.contents = {}
- for _,c in ipairs(contents) do
+ for _, c in ipairs(contents) do
local cobj = maybe_dereference_object(c, pdf, task)
if cobj and type(cobj) == 'table' then
obj.contents[#obj.contents + 1] = cobj
---[[Disabled fonts extraction
- local fonts = obj.resources.Font
- if fonts and type(fonts) == 'table' then
- obj.fonts = {}
- for k,v in pairs(fonts) do
- obj.fonts[k] = maybe_dereference_object(v, pdf, task)
+ --[[Disabled fonts extraction
+ local fonts = obj.resources.Font
+ if fonts and type(fonts) == 'table' then
+ obj.fonts = {}
+ for k,v in pairs(fonts) do
+ obj.fonts[k] = maybe_dereference_object(v, pdf, task)
- if obj.fonts[k] then
- local font = obj.fonts[k]
+ if obj.fonts[k] then
+ local font = obj.fonts[k]
- if config.text_extraction then
- process_font(task, pdf, font, k)
- lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s',
- k, obj.major, obj.minor, font)
+ if config.text_extraction then
+ process_font(task, pdf, font, k)
+ lua_util.debugm(N, task, 'found font "%s" for object %s:%s -> %s',
+ k, obj.major, obj.minor, font)
+ end
+ end
end
end
- end
- end
-]]
+ ]]
lua_util.debugm(N, task, 'found resources for object %s:%s (%s): %s',
obj.major, obj.minor, obj.type, obj.resources)
local function compound_obj_grammar_gen()
if not compound_obj_grammar then
local gen = generic_grammar_elts()
- compound_obj_grammar = gen.ws^0 * (gen.comment * gen.ws^1)^0 *
- lpeg.Ct(lpeg.Ct(gen.number * gen.ws^1 * gen.number * gen.ws^0)^1)
+ compound_obj_grammar = gen.ws ^ 0 * (gen.comment * gen.ws ^ 1) ^ 0 *
+ lpeg.Ct(lpeg.Ct(gen.number * gen.ws ^ 1 * gen.number * gen.ws ^ 0) ^ 1)
end
return compound_obj_grammar
lua_util.debugm(N, task, 'compound elts (chunk length %s): %s',
#uncompressed, elts)
- for i,pair in ipairs(elts) do
- local obj_number,offset = pair[1], pair[2]
+ for i, pair in ipairs(elts) do
+ local obj_number, offset = pair[1], pair[2]
offset = offset + first
if offset < #uncompressed then
-- PDF 1.5 ObjStmt
local function extract_pdf_compound_objects(task, pdf)
- for i,obj in ipairs(pdf.objects or {}) do
+ for i, obj in ipairs(pdf.objects or {}) do
if i > 0 and i % 100 == 0 then
local now = rspamd_util.get_ticks()
-- Also get the starting span and try to match it versus obj re to get numbers
local obj_line_potential = first - 32
- if obj_line_potential < 1 then obj_line_potential = 1 end
+ if obj_line_potential < 1 then
+ obj_line_potential = 1
+ end
local prev_obj_end = pdf.end_objects[end_pos - 1]
if end_pos > 1 and prev_obj_end >= obj_line_potential and prev_obj_end < first then
obj_line_potential = prev_obj_end + 1
max_start_pos = math.min(config.max_pdf_objects, #pdf.start_streams)
max_end_pos = math.min(config.max_pdf_objects, #pdf.end_streams)
- for _,obj in ipairs(pdf.objects) do
+ for _, obj in ipairs(pdf.objects) do
while start_pos <= max_start_pos and end_pos <= max_end_pos do
local first = pdf.start_streams[start_pos]
local last = pdf.end_streams[end_pos]
-- Strip the first \n
while first < last do
local chr = input:byte(first)
- if chr ~= 13 and chr ~= 10 then break end
+ if chr ~= 13 and chr ~= 10 then
+ break
+ end
first = first + 1
end
local len = last - first
-- Now we have objects and we need to attach streams that are in bounds
attach_pdf_streams(task, input, pdf)
-- Parse grammar for outer objects
- for i,obj in ipairs(pdf.objects) do
+ for i, obj in ipairs(pdf.objects) do
if i > 0 and i % 100 == 0 then
local now = rspamd_util.get_ticks()
end
-- Now we might probably have all objects being processed
- for i,obj in ipairs(pdf.objects) do
+ for i, obj in ipairs(pdf.objects) do
if obj.dict then
-- Types processing
if i > 0 and i % 100 == 0 then
end
local function search_text(task, pdf)
- for _,obj in ipairs(pdf.objects) do
+ for _, obj in ipairs(pdf.objects) do
if obj.type == 'Page' and obj.contents then
local text = {}
- for _,tobj in ipairs(obj.contents) do
+ for _, tobj in ipairs(obj.contents) do
maybe_extract_object_stream(tobj, pdf, task)
local matches = pdf_text_trie:match(tobj.uncompressed or '')
if matches then
local starts = {}
local ends = {}
- for npat,matched_positions in pairs(matches) do
+ for npat, matched_positions in pairs(matches) do
if npat == 1 then
- for _,pos in ipairs(matched_positions) do
+ for _, pos in ipairs(matched_positions) do
starts[#starts + 1] = pos
end
else
- for _,pos in ipairs(matched_positions) do
+ for _, pos in ipairs(matched_positions) do
ends[#ends + 1] = pos
end
end
end
offsets_to_blocks(starts, ends, text_blocks)
- for _,bl in ipairs(text_blocks) do
+ for _, bl in ipairs(text_blocks) do
if bl.len > 2 then
-- To remove \s+ET\b pattern (it can leave trailing space or not but it doesn't matter)
bl.len = bl.len - 2
-- tobj.major, tobj.minor, bl.data)
if bl.len < config.max_processing_size then
- local ret,obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar,
+ local ret, obj_or_err = pcall(pdf_text_grammar.match, pdf_text_grammar,
bl.data)
if ret then
return
end
- for k,v in pairs(dict) do
+ for k, v in pairs(dict) do
if type(v) == 'table' then
recursive_object_traverse(obj, v, rec + 1)
elseif k == 'URI' then
v = maybe_dereference_object(v, pdf, task)
if type(v) == 'string' then
- local url = rspamd_url.create(task:get_mempool(), v, {'content'})
+ local url = rspamd_url.create(task:get_mempool(), v, { 'content' })
if url then
lua_util.debugm(N, task, 'found url %s in object %s:%s',
end
end
- for _,obj in ipairs(pdf.objects) do
+ for _, obj in ipairs(pdf.objects) do
if obj.dict and type(obj.dict) == 'table' then
recursive_object_traverse(obj, obj.dict, 0)
end
-- Output object that excludes all internal stuff
local pdf_output = lua_util.shallowcopy(pdf_object)
local grouped_processors = {}
- for npat,matched_positions in pairs(matches) do
+ for npat, matched_positions in pairs(matches) do
local index = pdf_indexes[npat]
- local proc_key,loc_npat = index[1], index[4]
+ local proc_key, loc_npat = index[1], index[4]
if not grouped_processors[proc_key] then
grouped_processors[proc_key] = {
end
local proc = grouped_processors[proc_key]
-- Fill offsets
- for _,pos in ipairs(matched_positions) do
- proc.offsets[#proc.offsets + 1] = {pos, loc_npat}
+ for _, pos in ipairs(matched_positions) do
+ proc.offsets[#proc.offsets + 1] = { pos, loc_npat }
end
end
- for name,processor in pairs(grouped_processors) do
+ for name, processor in pairs(grouped_processors) do
-- Sort by offset
lua_util.debugm(N, task, "pdf: process group %s with %s matches",
name, #processor.offsets)
- table.sort(processor.offsets, function(e1, e2) return e1[1] < e2[1] end)
+ table.sort(processor.offsets, function(e1, e2)
+ return e1[1] < e2[1]
+ end)
processor.processor_func(input, task, processor.offsets, pdf_object, pdf_output)
end
end
else
-- All hashes
- for h,sc in pairs(pdf_object.scripts) do
+ for h, sc in pairs(pdf_object.scripts) do
if config.min_js_fuzzy and #sc.data >= config.min_js_fuzzy then
lua_util.debugm(N, task, "pdf: add fuzzy hash from JavaScript: %s; size = %s; object: %s:%s",
sc.hash,
local nencoded = 0
local close_encoded = 0
local last_encoded
- for _,match in ipairs(positions) do
+ for _, match in ipairs(positions) do
if match[2] == 1 then
-- netsh
suspicious_factor = suspicious_factor + 0.5
pdf_object[output_key] = {}
end
local shift = #pdf_object[output_key]
- for i,pos in ipairs(positions) do
+ for i, pos in ipairs(positions) do
pdf_object[output_key][i + shift] = pos[1]
end
end
local function gen_grammar()
if not vcard_grammar then
local wsp = l.S(" \t\v\f")
- local crlf = (l.P"\r"^-1 * l.P"\n") + l.P"\r"
- local eol = (crlf * #crlf) + (crlf - (crlf^-1 * wsp))
- local name = l.C((l.P(1) - (l.P":"))^1) / function(v) return (v:gsub("[\n\r]+%s","")) end
- local value = l.C((l.P(1) - eol)^0) / function(v) return (v:gsub("[\n\r]+%s","")) end
- vcard_grammar = name * ":" * wsp^0 * value * eol^-1
+ local crlf = (l.P "\r" ^ -1 * l.P "\n") + l.P "\r"
+ local eol = (crlf * #crlf) + (crlf - (crlf ^ -1 * wsp))
+ local name = l.C((l.P(1) - (l.P ":")) ^ 1) / function(v)
+ return (v:gsub("[\n\r]+%s", ""))
+ end
+ local value = l.C((l.P(1) - eol) ^ 0) / function(v)
+ return (v:gsub("[\n\r]+%s", ""))
+ end
+ vcard_grammar = name * ":" * wsp ^ 0 * value * eol ^ -1
end
return vcard_grammar
local exports = {}
local function process_vcard(input, mpart, task)
- local control={n='\n', r=''}
+ local control = { n = '\n', r = '' }
local rspamd_url = require "rspamd_url"
local escaper = l.Ct((gen_grammar() / function(key, value)
value = value:gsub("\\(.)", control)
local local_urls = rspamd_url.all(task:get_mempool(), value)
if local_urls and #local_urls > 0 then
- for _,u in ipairs(local_urls) do
+ for _, u in ipairs(local_urls) do
lua_util.debugm(N, task, 'vcard: found URL in vcard %s',
tostring(u))
task:inject_url(u, mpart)
end
lua_util.debugm(N, task, 'vcard: vcard key %s = "%s"',
key, value)
- return {key, value}
- end)^1)
+ return { key, value }
+ end) ^ 1)
local elts = escaper:match(input)
return {
tag = 'vcard',
- extract_text = function() return nil end, -- NYI
+ extract_text = function()
+ return nil
+ end, -- NYI
elts = elts
}
end
-- Check for DKIM_REJECT
local sym_check = 'R_DKIM_REJECT'
- if N == 'arc' then sym_check = 'ARC_REJECT' end
+ if N == 'arc' then
+ sym_check = 'ARC_REJECT'
+ end
if task:has_symbol(sym_check) then
local sym = task:get_symbol(sym_check)[1]
logger.infox(task, 'skip signing for %s: violation %s found: %s',
if not (domain and selector and key) then
logger.errx(task, 'missing required headers to sign email')
- return false,{}
+ return false, {}
end
-- Now check if we need to check the existing auth
end
lua_util.debugm(N, task, 'no sign header %s', headers.sign_header)
- return false,{}
+ return false, {}
end
local function prepare_dkim_signing(N, task, settings)
local is_local, is_sign_networks, is_authed
if settings.use_http_headers then
- local res,tbl = parse_dkim_http_headers(N, task, settings)
+ local res, tbl = parse_dkim_http_headers(N, task, settings)
if not res then
if not settings.allow_headers_fallback then
- return res,{}
+ return res, {}
else
lua_util.debugm(N, task, 'failed to read http headers, fallback to normal schema')
end
else
- return res,tbl
+ return res, tbl
end
end
local ret = settings.sign_condition(task)
if not ret then
- return false,{}
+ return false, {}
end
if ret[1] then
- return true,ret
+ return true, ret
else
- return true,{ret}
+ return true, { ret }
end
end
if metric_action == 'reject' or metric_action == 'drop' then
-- No need to sign what we are already rejecting/dropping
lua_util.debugm(N, task, 'task result is already %s, no need to sign', metric_action)
- return false,{}
+ return false, {}
end
if metric_action == 'soft reject' then
-- Same here, we are going to delay an email, signing is just a waste of time
lua_util.debugm(N, task, 'task result is %s, skip signing', metric_action)
- return false,{}
+ return false, {}
end
-- For spam actions, there is no clear distinction
if metric_action ~= 'no action' and type(settings.skip_spam_sign) == 'boolean' and settings.skip_spam_sign then
lua_util.debugm(N, task, 'task result is %s, no need to sign', metric_action)
- return false,{}
+ return false, {}
end
end
lua_util.debugm(N, task, 'mail was sent to us')
else
lua_util.debugm(N, task, 'mail is ineligible for signing')
- return false,{}
+ return false, {}
end
local efrom = task:get_from('smtp')
if #(((efrom or E)[1] or E).addr or '') == 0 then
if not settings.allow_envfrom_empty then
lua_util.debugm(N, task, 'empty envelope from not allowed')
- return false,{}
+ return false, {}
else
empty_envelope = true
end
local hfrom = task:get_from('mime')
if not settings.allow_hdrfrom_multiple and (hfrom or E)[2] then
lua_util.debugm(N, task, 'multiple header from not allowed')
- return false,{}
+ return false, {}
end
local eto = task:get_recipients(0)
lua_util.debugm(N, task,
'skip signing: is_sign_network: %s, is_authed: %s, is_local: %s',
is_sign_networks, is_authed, is_local)
- return false,{}
+ return false, {}
end
if not hfrom or not hfrom[1] or not hfrom[1].addr then
lua_util.debugm(N, task,
'signing_table: cannot get data when no header from is presented')
- return false,{}
+ return false, {}
end
local sign_entry = settings.signing_table:get_key(hfrom[1].addr)
end
if settings.key_table then
- -- Now search in key table
- local key_entry = settings.key_table:get_key(sign_entry)
+ -- Now search in key table
+ local key_entry = settings.key_table:get_key(sign_entry)
if key_entry then
local parts = lua_util.str_split(key_entry, ':')
if not selector then
logger.errx(task, 'no selector defined for sign_entry %s, key_entry %s',
sign_entry, key_entry)
- return false,{}
+ return false, {}
end
local res = {
hdom, selector, res.domain)
end
- return true,{res}
+ return true, { res }
elseif #parts == 3 then
-- domain, selector, key
local selector = parts[2]
hdom, selector, res.domain)
end
- return true,{res}
+ return true, { res }
else
logger.errx(task, 'invalid key entry for sign entry %s: %s; when signing %s domain',
sign_entry, key_entry, hdom)
- return false,{}
+ return false, {}
end
elseif settings.use_vault then
-- Sign table is presented, the rest is covered by vault
else
logger.errx(task, 'missing key entry for sign entry %s; when signing %s domain',
sign_entry, hdom)
- return false,{}
+ return false, {}
end
else
logger.errx(task, 'cannot get key entry for signing entry %s, when signing %s domain',
sign_entry, hdom)
- return false,{}
+ return false, {}
end
else
lua_util.debugm(N, task,
'signing_table: no entry for %s', hfrom[1].addr)
- return false,{}
+ return false, {}
end
else
if settings.use_domain_sign_networks and is_sign_networks then
if not dkim_domain then
lua_util.debugm(N, task, 'could not extract dkim domain')
- return false,{}
+ return false, {}
end
if settings.use_esld then
lua_util.debugm(N, task, 'domain mismatch allowed for empty envelope: %1 != %2', hdom, edom)
else
lua_util.debugm(N, task, 'domain mismatch not allowed: %1 != %2', hdom, edom)
- return false,{}
+ return false, {}
end
end
end
if auser and not settings.allow_username_mismatch then
if not udom then
lua_util.debugm(N, task, 'couldnt find domain in username')
- return false,{}
+ return false, {}
end
if settings.use_esld then
udom = rspamd_util.get_tld(udom)
end
if udom ~= dkim_domain then
lua_util.debugm(N, task, 'user domain mismatch')
- return false,{}
+ return false, {}
end
end
}
else
lua_util.debugm(N, task, 'domain %s is not designated for vault',
- dkim_domain)
- return false,{}
+ dkim_domain)
+ return false, {}
end
else
-- TODO: try every domain in the vault
if settings.domain[dkim_domain] then
-- support old style selector/paths
if settings.domain[dkim_domain].selector or
- settings.domain[dkim_domain].path then
+ settings.domain[dkim_domain].path then
local k = {}
k.selector = settings.domain[dkim_domain].selector
k.key = settings.domain[dkim_domain].path
if ret then
table.insert(p, k)
lua_util.debugm(N, task, 'using mempool selector %s with key %s',
- k.selector, k.key)
+ k.selector, k.key)
end
end
if #p == 0 and not settings.try_fallback then
lua_util.debugm(N, task, 'dkim unconfigured and fallback disabled')
- return false,{}
+ return false, {}
end
if not settings.use_redis then
end
insert_or_update_prop(N, task, p, 'selector',
- 'default selector', settings.selector)
+ 'default selector', settings.selector)
if settings.check_violation then
if not check_violation(N, task, p.domain) then
- return false,{}
+ return false, {}
end
end
insert_or_update_prop(N, task, p, 'domain', 'dkim_domain',
- dkim_domain)
+ dkim_domain)
- return true,p
+ return true, p
end
exports.prepare_dkim_signing = prepare_dkim_signing
false, -- is write
redis_key_cb, --callback
'HGET', -- command
- {settings.key_prefix, rk} -- arguments
+ { settings.key_prefix, rk } -- arguments
)
if not rret then
err_func(task,
- string.format( "cannot make request to load DKIM key for %s", rk))
+ string.format("cannot make request to load DKIM key for %s", rk))
end
end
false, -- is write
redis_selector_cb, --callback
'HGET', -- command
- {settings.selector_prefix, p.domain} -- arguments
+ { settings.selector_prefix, p.domain } -- arguments
)
if not rret then
err_func(task, string.format("cannot make Redis request to load DKIM selector for domain %s",
full_url, err, body))
else
local parser = ucl.parser()
- local res,parser_err = parser:parse_string(body)
+ local res, parser_err = parser:parse_string(body)
if not res then
err_func(task, string.format('vault reply for %s (data=%s) cannot be parsed: %s',
full_url, body, parser_err))
end
end
- local ret = http.request{
+ local ret = http.request {
task = task,
url = full_url,
callback = vault_callback,
local lua_maps = require "lua_maps"
-- Used to convert plain options to the maps
local maps_opts = {
- sign_networks = {'radix', 'DKIM signing networks'},
- path_map = {'map', 'Paths to DKIM signing keys'},
- selector_map = {'map', 'DKIM selectors'},
- signing_table = {'glob', 'DKIM signing table'},
- key_table = {'glob', 'DKIM keys table'},
- vault_domains = {'glob', 'DKIM signing domains in vault'},
- whitelisted_signers_map = {'set', 'ARC trusted signers domains'}
+ sign_networks = { 'radix', 'DKIM signing networks' },
+ path_map = { 'map', 'Paths to DKIM signing keys' },
+ selector_map = { 'map', 'DKIM selectors' },
+ signing_table = { 'glob', 'DKIM signing table' },
+ key_table = { 'glob', 'DKIM keys table' },
+ vault_domains = { 'glob', 'DKIM signing domains in vault' },
+ whitelisted_signers_map = { 'set', 'ARC trusted signers domains' }
}
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
local maybe_map = maps_opts[k]
if maybe_map then
settings[k] = lua_maps.map_add_from_ucl(v, maybe_map[1], maybe_map[2])
elseif k == 'sign_condition' then
- local ret,f = lua_util.callback_from_string(v)
+ local ret, f = lua_util.callback_from_string(v)
if ret then
settings[k] = f
else
local ffi = require 'ffi'
-ffi.cdef[[
+ffi.cdef [[
struct GString {
char *str;
size_t len;
local ffi = require 'ffi'
-ffi.cdef[[
+ffi.cdef [[
struct rspamd_dkim_sign_context_s;
struct rspamd_dkim_key_s;
struct rspamd_task;
elseif format == 'raw' then
format = ffi.C.RSPAMD_DKIM_KEY_RAW
else
- return nil,'unknown key format'
+ return nil, 'unknown key format'
end
end
return ffi.C.rspamd_dkim_sign_key_load(what, #what, format, nil)
end
-local default_dkim_headers =
-"(o)from:(o)sender:(o)reply-to:(o)subject:(o)date:(o)message-id:" ..
-"(o)to:(o)cc:(o)mime-version:(o)content-type:(o)content-transfer-encoding:" ..
-"resent-to:resent-cc:resent-from:resent-sender:resent-message-id:" ..
-"(o)in-reply-to:(o)references:list-id:list-owner:list-unsubscribe:" ..
-"list-subscribe:list-post:(o)openpgp:(o)autocrypt"
+local default_dkim_headers = "(o)from:(o)sender:(o)reply-to:(o)subject:(o)date:(o)message-id:" ..
+ "(o)to:(o)cc:(o)mime-version:(o)content-type:(o)content-transfer-encoding:" ..
+ "resent-to:resent-cc:resent-from:resent-sender:resent-message-id:" ..
+ "(o)in-reply-to:(o)references:list-id:list-owner:list-unsubscribe:" ..
+ "list-subscribe:list-post:(o)openpgp:(o)autocrypt"
local function create_sign_context(task, privkey, dkim_headers, sign_type)
if not task or not privkey then
- return nil,'invalid arguments'
+ return nil, 'invalid arguments'
end
if not dkim_headers then
elseif sign_type == 'arc-seal' then
sign_type = ffi.C.RSPAMD_DKIM_ARC_SEAL
else
- return nil,'invalid sign type'
+ return nil, 'invalid sign type'
end
-
return ffi.C.rspamd_create_dkim_sign_context(task:topointer(), privkey,
1, 1, dkim_headers, sign_type, nil)
end
local function do_sign(task, sign_context, selector, domain,
expire, len, arc_idx)
if not task or not sign_context or not selector or not domain then
- return nil,'invalid arguments'
+ return nil, 'invalid arguments'
end
- if not expire then expire = 0 end
- if not len then len = 0 end
- if not arc_idx then arc_idx = 0 end
+ if not expire then
+ expire = 0
+ end
+ if not len then
+ len = 0
+ end
+ if not arc_idx then
+ arc_idx = 0
+ end
local gstring = ffi.C.rspamd_dkim_sign(task:topointer(), selector, domain, expire, len, arc_idx, nil, sign_context)
if not gstring then
- return nil,'cannot sign'
+ return nil, 'cannot sign'
end
local ret = ffi.string(gstring.str, gstring.len)
return o ~= NULL
end
else
- local ret,result_or_err = pcall(require, 'ffi')
+ local ret, result_or_err = pcall(require, 'ffi')
if not ret then
return {}
exports.spf = require "lua_ffi/spf"
exports.linalg = require "lua_ffi/linalg"
-for k,v in pairs(ffi) do
+for k, v in pairs(ffi) do
-- Preserve all stuff to use lua_ffi as ffi itself
exports[k] = v
end
local exports = {}
-ffi.cdef[[
+ffi.cdef [[
void kad_sgemm_simple(int trans_A, int trans_B, int M, int N, int K, const float *A, const float *B, float *C);
bool kad_ssyev_simple (int N, float *A, float *output);
]]
local function table_to_ffi(a, m, n)
local a_conv = ffi.new("float[?]", m * n)
- for i=1,m or #a do
- for j=1,n or #a[1] do
+ for i = 1, m or #a do
+ for j = 1, n or #a[1] do
a_conv[(i - 1) * n + (j - 1)] = a[i][j]
end
end
local function ffi_to_table(a, m, n)
local res = {}
- for i=0,m-1 do
+ for i = 0, m - 1 do
res[i + 1] = {}
- for j=0,n-1 do
+ for j = 0, n - 1 do
res[i + 1][j + 1] = a[i * n + j]
end
end
local res = ffi.new("float[?]", n)
if ffi.C.kad_ssyev_simple(n, ffi.cast('float*', a), res) then
- return res,a
+ return res, a
end
return nil
local ffi = require 'ffi'
-ffi.cdef[[
+ffi.cdef [[
enum spf_mech_e {
SPF_FAIL,
SPF_SOFT_FAIL,
local digstr = ffi.new("char[64]")
ffi.C.rspamd_snprintf(digstr, 64, "0x%xuL", rec.digest)
res.digest = ffi.string(digstr)
- for i = 1,nelts do
+ for i = 1, nelts do
res.addrs[i] = spf_addr_tolua(elts[i - 1])
end
min_width = 500,
min_length = 64,
text_multiplier = 4.0, -- divide min_bytes by 4 for texts
- mime_types = {"application/*"},
+ mime_types = { "application/*" },
scan_archives = true,
short_text_direct_hash = true,
text_shingles = true,
local default_policy = policies.recommended
-local policy_schema = ts.shape{
+local policy_schema = ts.shape {
min_bytes = ts.number + ts.string / tonumber,
min_height = ts.number + ts.string / tonumber,
min_width = ts.number + ts.string / tonumber,
skip_images = ts.boolean,
}
-
local exports = {}
rspamd_logger.warnx(rspamd_config, "overriding policy %s", name)
end
- local parsed_policy,err = policy_schema:transform(policy)
+ local parsed_policy, err = policy_schema:transform(policy)
if not parsed_policy then
rspamd_logger.errx(rspamd_config, 'invalid fuzzy rule policy %s: %s',
end
local function check_text_part(task, part, rule, text)
- local allow_direct,allow_shingles = false,false
+ local allow_direct, allow_shingles = false, false
local id = part:get_id()
lua_util.debugm(N, task, 'check text part %s', id)
allow_direct = check_length(task, part, rule)
end
- return allow_direct,allow_shingles
+ return allow_direct, allow_shingles
end
--local function has_sane_text_parts(task)
local function check_image_part(task, part, rule, image)
if rule.skip_images then
lua_util.debugm(N, task, 'skip image part as images are disabled')
- return false,false
+ return false, false
end
local id = part:get_id()
if height and width then
if height < min_height or width < min_width then
lua_util.debugm(N, task, 'skip image part %s as it does not meet minimum sizes: %sx%s < %sx%s',
- id, width, height, min_width, min_height)
+ id, width, height, min_width, min_height)
return false, false
else
lua_util.debugm(N, task, 'allow image part %s: %sx%s',
end
end
- return check_length(task, part, rule),false
+ return check_length(task, part, rule), false
end
local function mime_types_check(task, part, rule)
- local t,st = part:get_type()
+ local t, st = part:get_type()
- if not t then return false, false end
+ if not t then
+ return false, false
+ end
local ct = string.format('%s/%s', t, st)
local detected_ct
- t,st = part:get_detected_type()
+ t, st = part:get_detected_type()
if t then
detected_ct = string.format('%s/%s', t, st)
else
end
opts = fun.tomap(fun.map(function(opt)
local elts = lua_util.str_split(opt, ':')
- return elts[1],elts[2]
+ return elts[1], elts[2]
end, opts))
if opts[id] and opts[id] == '-' then
lua_util.debugm(N, task, 'explicitly check binary part %s: bad mime type %s', id, ct)
- return check_length(task, part, rule),false
+ return check_length(task, part, rule), false
end
if rule.mime_types then
end, rule.mime_types) then
lua_util.debugm(N, task, 'found mime type match for part %s: %s (%s detected)',
id, ct, detected_ct)
- return check_length(task, part, rule),false
+ return check_length(task, part, rule), false
end
return false, false
end
- return false,false
+ return false, false
end
exports.check_mime_part = function(task, part, rule_id)
if not rule then
rspamd_logger.errx(task, 'cannot find rule with id %s', rule_id)
- return false,false
+ return false, false
end
if part:is_text() then
-- Always send archives
lua_util.debugm(N, task, 'check archive part %s', part:get_id())
- return true,false
+ return true, false
end
if part:is_specific() then
if type(sp) == 'table' and sp.fuzzy_hashes then
lua_util.debugm(N, task, 'check specific part %s', part:get_id())
- return true,false
+ return true, false
end
end
return mime_types_check(task, part, rule)
end
- return false,false
+ return false, false
end
exports.cleanup_rules = function()
local P = lpeg.P
local R = lpeg.R
local S = lpeg.S
-local D = R'09' -- Digits
+local D = R '09' -- Digits
local I = R('AZ', 'az', '\127\255') + '_' -- Identifiers
local B = -(I + D) -- Word boundary
local EOS = -lpeg.P(1) -- end of string
-- Pattern for long strings and long comments.
-local longstring = #(P'[[' + (P'[' * P'='^0 * '[')) * P(function(input, index)
+local longstring = #(P '[[' + (P '[' * P '=' ^ 0 * '[')) * P(function(input, index)
local level = input:match('^%[(=*)%[', index)
if level then
local _, last = input:find(']' .. level .. ']', index, true)
- if last then return last + 1 end
+ if last then
+ return last + 1
+ end
end
end)
-- String literals.
-local singlequoted = P"'" * ((1 - S"'\r\n\f\\") + (P'\\' * 1))^0 * "'"
-local doublequoted = P'"' * ((1 - S'"\r\n\f\\') + (P'\\' * 1))^0 * '"'
+local singlequoted = P "'" * ((1 - S "'\r\n\f\\") + (P '\\' * 1)) ^ 0 * "'"
+local doublequoted = P '"' * ((1 - S '"\r\n\f\\') + (P '\\' * 1)) ^ 0 * '"'
-- Comments.
-local eol = P'\r\n' + '\n'
-local line = (1 - S'\r\n\f')^0 * eol^-1
-local singleline = P'--' * line
-local multiline = P'--' * longstring
+local eol = P '\r\n' + '\n'
+local line = (1 - S '\r\n\f') ^ 0 * eol ^ -1
+local singleline = P '--' * line
+local multiline = P '--' * longstring
-- Numbers.
-local sign = S'+-'^-1
-local decimal = D^1
-local hexadecimal = P'0' * S'xX' * R('09', 'AF', 'af') ^ 1
-local float = D^1 * P'.' * D^0 + P'.' * D^1
-local maybeexp = (float + decimal) * (S'eE' * sign * D^1)^-1
+local sign = S '+-' ^ -1
+local decimal = D ^ 1
+local hexadecimal = P '0' * S 'xX' * R('09', 'AF', 'af') ^ 1
+local float = D ^ 1 * P '.' * D ^ 0 + P '.' * D ^ 1
+local maybeexp = (float + decimal) * (S 'eE' * sign * D ^ 1) ^ -1
local function compile_keywords(keywords)
local list = {}
end
-- Identifiers
-local ident = I * (I + D)^0
-local expr = ('.' * ident)^0
+local ident = I * (I + D) ^ 0
+local expr = ('.' * ident) ^ 0
local patterns = {
- {'whitespace', S'\r\n\f\t\v '^1},
- {'constant', (P'true' + 'false' + 'nil') * B},
- {'string', singlequoted + doublequoted + longstring},
- {'comment', multiline + singleline},
- {'number', hexadecimal + maybeexp},
- {'operator', P'not' + '...' + 'and' + '..' + '~=' + '==' + '>=' + '<='
- + 'or' + S']{=>^[<;)*(%}+-:,/.#'},
- {'keyword', compile_keywords([[
+ { 'whitespace', S '\r\n\f\t\v ' ^ 1 },
+ { 'constant', (P 'true' + 'false' + 'nil') * B },
+ { 'string', singlequoted + doublequoted + longstring },
+ { 'comment', multiline + singleline },
+ { 'number', hexadecimal + maybeexp },
+ { 'operator', P 'not' + '...' + 'and' + '..' + '~=' + '==' + '>=' + '<='
+ + 'or' + S ']{=>^[<;)*(%}+-:,/.#' },
+ { 'keyword', compile_keywords([[
break do else elseif end for function if in local repeat return then until while
- ]])},
- {'identifier', lpeg.Cmt(ident,
+ ]]) },
+ { 'identifier', lpeg.Cmt(ident,
function(input, index)
return expr:match(input, index)
end)
},
- {'error', 1},
+ { 'error', 1 },
}
local compiled
local function compile_patterns()
if not compiled then
local function process(elt)
- local n,grammar = elt[1],elt[2]
+ local n, grammar = elt[1], elt[2]
return lpeg.Cc(n) * lpeg.P(grammar) * lpeg.Cp()
end
local any = process(patterns[1])
local out = {}
for kind, text, lnum, cnum in exports.gmatch(input) do
- out[#out + 1] = {kind, text, lnum, cnum}
+ out[#out + 1] = { kind, text, lnum, cnum }
end
return out
local N = "lua_magic"
local msoffice_trie
local msoffice_patterns = {
- doc = {[[WordDocument]]},
- xls = {[[Workbook]], [[Book]]},
- ppt = {[[PowerPoint Document]], [[Current User]]},
- vsd = {[[VisioDocument]]},
+ doc = { [[WordDocument]] },
+ xls = { [[Workbook]], [[Book]] },
+ ppt = { [[PowerPoint Document]], [[Current User]] },
+ vsd = { [[VisioDocument]] },
}
local msoffice_trie_clsid
local msoffice_clsids = {
- doc = {[[0609020000000000c000000000000046]]},
- xls = {[[1008020000000000c000000000000046]], [[2008020000000000c000000000000046]]},
- ppt = {[[108d81649b4fcf1186ea00aa00b929e8]]},
- msg = {[[46f0060000000000c000000000000046]], [[0b0d020000000000c000000000000046]]},
- msi = {[[84100c0000000000c000000000000046]]},
+ doc = { [[0609020000000000c000000000000046]] },
+ xls = { [[1008020000000000c000000000000046]], [[2008020000000000c000000000000046]] },
+ ppt = { [[108d81649b4fcf1186ea00aa00b929e8]] },
+ msg = { [[46f0060000000000c000000000000046]], [[0b0d020000000000c000000000000046]] },
+ msi = { [[84100c0000000000c000000000000046]] },
}
local zip_trie
local zip_patterns = {
[[mimetypeapplication/vnd\.oasis\.opendocument\.formula]],
[[mimetypeapplication/vnd\.oasis\.opendocument\.chart]]
},
- odp = {[[mimetypeapplication/vnd\.oasis\.opendocument\.presentation]]},
- epub = {[[epub\+zip]]},
- asice = {[[mimetypeapplication/vnd\.etsi\.asic-e\+zipPK]]},
- asics = {[[mimetypeapplication/vnd\.etsi\.asic-s\+zipPK]]},
+ odp = { [[mimetypeapplication/vnd\.oasis\.opendocument\.presentation]] },
+ epub = { [[epub\+zip]] },
+ asice = { [[mimetypeapplication/vnd\.etsi\.asic-e\+zipPK]] },
+ asics = { [[mimetypeapplication/vnd\.etsi\.asic-s\+zipPK]] },
}
local txt_trie
local txt_patterns = {
html = {
- {[=[(?i)<html[\s>]]=], 32},
- {[[(?i)<script\b]], 20}, -- Commonly used by spammers
- {[[<script\s+type="text\/javascript">]], 31}, -- Another spammy pattern
- {[[(?i)<\!DOCTYPE HTML\b]], 33},
- {[[(?i)<body\b]], 20},
- {[[(?i)<table\b]], 20},
- {[[(?i)<a\s]], 10},
- {[[(?i)<p\b]], 10},
- {[[(?i)<div\b]], 10},
- {[[(?i)<span\b]], 10},
+ { [=[(?i)<html[\s>]]=], 32 },
+ { [[(?i)<script\b]], 20 }, -- Commonly used by spammers
+ { [[<script\s+type="text\/javascript">]], 31 }, -- Another spammy pattern
+ { [[(?i)<\!DOCTYPE HTML\b]], 33 },
+ { [[(?i)<body\b]], 20 },
+ { [[(?i)<table\b]], 20 },
+ { [[(?i)<a\s]], 10 },
+ { [[(?i)<p\b]], 10 },
+ { [[(?i)<div\b]], 10 },
+ { [[(?i)<span\b]], 10 },
},
csv = {
- {[[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20}
+ { [[(?:[-a-zA-Z0-9_]+\s*,){2,}(?:[-a-zA-Z0-9_]+,?[ ]*[\r\n])]], 20 }
},
ics = {
- {[[^BEGIN:VCALENDAR\r?\n]], 40},
+ { [[^BEGIN:VCALENDAR\r?\n]], 40 },
},
vcf = {
- {[[^BEGIN:VCARD\r?\n]], 40},
+ { [[^BEGIN:VCARD\r?\n]], 40 },
},
xml = {
- {[[<\?xml\b.+\?>]], 31},
+ { [[<\?xml\b.+\?>]], 31 },
}
}
rspamd_trie.flags.no_start)
local function compile_pats(patterns, indexes, transform_func, compile_flags)
local strs = {}
- for ext,pats in pairs(patterns) do
- for _,pat in ipairs(pats) do
+ for ext, pats in pairs(patterns) do
+ for _, pat in ipairs(pats) do
-- These are utf16 strings in fact...
strs[#strs + 1] = transform_func(pat)
- indexes[#indexes + 1] = {ext, pat}
+ indexes[#indexes + 1] = { ext, pat }
end
end
return '^' ..
table.concat(
fun.totable(
- fun.map(function(c) return c .. [[\x{00}]] end,
+ fun.map(function(c)
+ return c .. [[\x{00}]]
+ end,
fun.iter(pat))))
end
local function msoffice_clsid_transform(pat)
local hex_table = {}
- for i=1,#pat,2 do
+ for i = 1, #pat, 2 do
local subc = pat:sub(i, i + 1)
hex_table[#hex_table + 1] = string.format('\\x{%s}', subc)
end
msoffice_clsid_transform)
-- Misc zip patterns at the initial fragment
zip_trie = compile_pats(zip_patterns, zip_patterns_indexes,
- function(pat) return pat end)
+ function(pat)
+ return pat
+ end)
-- Text patterns at the initial fragment
txt_trie = compile_pats(txt_patterns, txt_patterns_indexes,
- function(pat_tbl) return pat_tbl[1] end,
+ function(pat_tbl)
+ return pat_tbl[1]
+ end,
bit.bor(rspamd_trie.flags.re,
rspamd_trie.flags.dot_all,
rspamd_trie.flags.no_start))
return nil
end
- local bom,sec_size = rspamd_util.unpack('<I2<I2', input:span(29, 4))
+ local bom, sec_size = rspamd_util.unpack('<I2<I2', input:span(29, 4))
if bom == 0xFFFE then
bom = '<'
else
lua_util.debugm(N, log_obj, "bom file!: %s", bom)
- bom = '>'; sec_size = bit.bswap(sec_size)
+ bom = '>';
+ sec_size = bit.bswap(sec_size)
end
if sec_size < 7 or sec_size > 31 then
-- Extract clsid
local matches = msoffice_trie_clsid:match(input:span(offset + 80, 16))
if matches then
- for n,_ in pairs(matches) do
+ for n, _ in pairs(matches) do
if msoffice_clsid_indexes[n] then
lua_util.debugm(N, log_obj, "found valid clsid for %s",
msoffice_clsid_indexes[n][1])
- return true,msoffice_clsid_indexes[n][1]
+ return true, msoffice_clsid_indexes[n][1]
end
end
end
- return true,nil
+ return true, nil
elseif dtype == 2 then
local matches = msoffice_trie:match(input:span(offset, 64))
if matches then
- for n,_ in pairs(matches) do
+ for n, _ in pairs(matches) do
if msoffice_patterns_indexes[n] then
- return true,msoffice_patterns_indexes[n][1]
+ return true, msoffice_patterns_indexes[n][1]
end
end
end
- return true,nil
+ return true, nil
elseif dtype >= 0 and dtype < 5 then
-- Bad type
- return true,nil
+ return true, nil
end
end
- return false,nil
+ return false, nil
end
repeat
- local res,ext = process_dir_entry(directory_offset)
+ local res, ext = process_dir_entry(directory_offset)
if res and ext then
- return ext,60
+ return ext, 60
end
if not res then
return res[ex1] > res[ex2]
end)
- return extensions[1],res[extensions[1]]
+ return extensions[1], res[extensions[1]]
end
return nil
if arch_type == 'zip' then
-- Find specific files/folders in zip file
local files = arch:get_files(100) or {}
- for _,file in ipairs(files) do
+ for _, file in ipairs(files) do
if file == '[Content_Types].xml' then
add_msoffice_confidence(10)
elseif file:sub(1, 3) == 'xl/' then
end
end
- local ext,weight = process_top_detected(res)
+ local ext, weight = process_top_detected(res)
if weight >= 40 then
- return ext,weight
+ return ext, weight
end
-- Apply misc Zip detection logic
local matches = zip_trie:match(start_span)
if matches then
- for n,_ in pairs(matches) do
+ for n, _ in pairs(matches) do
if zip_patterns_indexes[n] then
lua_util.debugm(N, log_obj, "found zip pattern for %s",
zip_patterns_indexes[n][1])
- return zip_patterns_indexes[n][1],40
+ return zip_patterns_indexes[n][1], 40
end
end
end
end
end
- return arch_type:lower(),40
+ return arch_type:lower(), 40
end
local csv_grammar
-- Returns a grammar that will count commas
local function get_csv_grammar()
if not csv_grammar then
- local lpeg = require'lpeg'
+ local lpeg = require 'lpeg'
- local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P'""' / '"')^0) * '"' +
- lpeg.C((1 - lpeg.S',\n"')^0)
+ local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P '""' / '"') ^ 0) * '"' +
+ lpeg.C((1 - lpeg.S ',\n"') ^ 0)
- csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P( (lpeg.P(',') +
- lpeg.P('\t')) * field)^1 * (lpeg.S'\r\n' + -1),
- function(acc) return acc + 1 end)
+ csv_grammar = lpeg.Cf(lpeg.Cc(0) * field * lpeg.P((lpeg.P(',') +
+ lpeg.P('\t')) * field) ^ 1 * (lpeg.S '\r\n' + -1),
+ function(acc)
+ return acc + 1
+ end)
end
return csv_grammar
while b >= 127 and idx < len do
-- utf8 part
if bit.band(b, 0xe0) == 0xc0 and remain > 1 and
- bit.band(bytes[idx + 1], 0xc0) == 0x80 then
- return true,1
+ bit.band(bytes[idx + 1], 0xc0) == 0x80 then
+ return true, 1
elseif bit.band(b, 0xf0) == 0xe0 and remain > 2 and
- bit.band(bytes[idx + 1], 0xc0) == 0x80 and
- bit.band(bytes[idx + 2], 0xc0) == 0x80 then
- return true,2
+ bit.band(bytes[idx + 1], 0xc0) == 0x80 and
+ bit.band(bytes[idx + 2], 0xc0) == 0x80 then
+ return true, 2
elseif bit.band(b, 0xf8) == 0xf0 and remain > 3 and
- bit.band(bytes[idx + 1], 0xc0) == 0x80 and
- bit.band(bytes[idx + 2], 0xc0) == 0x80 and
- bit.band(bytes[idx + 3], 0xc0) == 0x80 then
- return true,3
+ bit.band(bytes[idx + 1], 0xc0) == 0x80 and
+ bit.band(bytes[idx + 2], 0xc0) == 0x80 and
+ bit.band(bytes[idx + 3], 0xc0) == 0x80 then
+ return true, 3
end
n8bit = n8bit + 1
end
if n8bit >= 3 then
- return true,n8bit
+ return true, n8bit
end
- return false,0
+ return false, 0
end
-- Convert to string as LuaJIT can optimise string.sub (and fun.iter) but not C calls
if (b < 0x20) and not (b == 0x0d or b == 0x0a or b == 0x09) then
non_printable = non_printable + 1
elseif b >= 127 then
- local c,nskip = rough_8bit_check(bytes, i, tlen - i, tlen)
+ local c, nskip = rough_8bit_check(bytes, i, tlen - i, tlen)
if not c then
non_printable = non_printable + 1
local parent = part:get_parent()
if parent then
- local parent_type,parent_subtype = parent:get_type()
+ local parent_type, parent_subtype = parent:get_type()
if parent_type == 'multipart' and parent_subtype == 'encrypted' then
-- Skip text heuristics for encrypted parts
end
local content = part:get_content()
- local mtype,msubtype = part:get_type()
+ local mtype, msubtype = part:get_type()
local clen = #content
local is_text
if matches then
-- Require at least 2 occurrences of those patterns
- for n,positions in pairs(matches) do
- local ext,weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2]
+ for n, positions in pairs(matches) do
+ local ext, weight = txt_patterns_indexes[n][1], txt_patterns_indexes[n][2][2]
if ext then
res[ext] = (res[ext] or 0) + weight * #positions
lua_util.debugm(N, log_obj, "found txt pattern for %s: %s, total: %s; %s/%s announced",
end
end
- if res.html and res.html >= 40 then
+ if res.html and res.html >= 40 then
-- HTML has priority over something like js...
return 'html', res.html
end
-- Content type stuff
if (mtype == 'text' or mtype == 'application') and
- (msubtype == 'html' or msubtype == 'xhtml+xml') then
+ (msubtype == 'html' or msubtype == 'xhtml+xml') then
return 'html', 21
end
local function has_extension(file, ext)
local ext_len = ext:len()
return file:len() > ext_len + 1
- and file:sub(-ext_len):lower() == ext
- and file:sub(-ext_len - 1, -ext_len - 1) == '.'
+ and file:sub(-ext_len):lower() == ext
+ and file:sub(-ext_len - 1, -ext_len - 1) == '.'
end
if fname and (has_extension(fname, 'htm') or has_extension(fname, 'html')) then
- return 'html',21
+ return 'html', 21
end
if mtype ~= 'text' then
return nil
end
- return 'txt',40
+ return 'txt', 40
end
end
end
weight = weight + 30
end
- return 'pdf',weight
+ return 'pdf', weight
end
exports.pe_part_heuristic = function(input, log_obj, pos, part)
return
end
- return 'exe',30
+ return 'exe', 30
end
return exports
str, pattern.ext)
end
-
if max_short_offset < match.position then
max_short_offset = match.position
end
end
if not compiled_patterns then
- for ext,pattern in pairs(patterns) do
+ for ext, pattern in pairs(patterns) do
assert(types[ext], 'not found type: ' .. ext)
pattern.ext = ext
- for _,match in ipairs(pattern.matches) do
+ for _, match in ipairs(pattern.matches) do
if match.string then
if match.relative_position and not match.position then
match.position = match.relative_position + #match.string
elseif match.hex then
local hex_table = {}
- for i=1,#match.hex,2 do
+ for i = 1, #match.hex, 2 do
local subc = match.hex:sub(i, i + 1)
hex_table[#hex_table + 1] = string.format('\\x{%s}', subc)
end
compile_flags = bit.bor(compile_flags, rspamd_trie.flags.single_match)
compile_flags = bit.bor(compile_flags, rspamd_trie.flags.no_start)
compiled_patterns = rspamd_trie.create(fun.totable(
- fun.map(function(t) return t[1] end, processed_patterns)),
+ fun.map(function(t)
+ return t[1]
+ end, processed_patterns)),
compile_flags
)
compiled_short_patterns = rspamd_trie.create(fun.totable(
- fun.map(function(t) return t[1] end, short_patterns)),
+ fun.map(function(t)
+ return t[1]
+ end, short_patterns)),
compile_flags
)
compiled_tail_patterns = rspamd_trie.create(fun.totable(
- fun.map(function(t) return t[1] end, tail_patterns)),
+ fun.map(function(t)
+ return t[1]
+ end, tail_patterns)),
compile_flags
)
res[ext] = res[ext] + 1
end
- lua_util.debugm(N, log_obj,'add pattern for %s, weight %s, total weight %s',
+ lua_util.debugm(N, log_obj, 'add pattern for %s, weight %s, total weight %s',
ext, weight, res[ext])
end
local function match_position(pos, expected)
- local cmp = function(a, b) return a == b end
+ local cmp = function(a, b)
+ return a == b
+ end
if type(expected) == 'table' then
-- Something like {'>', 0}
if expected[1] == '>' then
- cmp = function(a, b) return a > b end
+ cmp = function(a, b)
+ return a > b
+ end
elseif expected[1] == '>=' then
- cmp = function(a, b) return a >= b end
+ cmp = function(a, b)
+ return a >= b
+ end
elseif expected[1] == '<' then
- cmp = function(a, b) return a < b end
+ cmp = function(a, b)
+ return a < b
+ end
elseif expected[1] == '<=' then
- cmp = function(a, b) return a <= b end
+ cmp = function(a, b)
+ return a <= b
+ end
elseif expected[1] == '!=' then
- cmp = function(a, b) return a ~= b end
+ cmp = function(a, b)
+ return a ~= b
+ end
end
expected = expected[2]
end
return cmp(pos, expected)
end
- for npat,matched_positions in pairs(matches) do
+ for npat, matched_positions in pairs(matches) do
local pat_data = processed_tbl[npat]
local pattern = pat_data[3]
local match = pat_data[2]
if match.position then
local position = match.position
- for _,pos in ipairs(matched_positions) do
+ for _, pos in ipairs(matched_positions) do
lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)',
pattern.ext, pos, offset)
if match_position(pos + offset, position) then
if match.heuristic then
- local ext,weight = match.heuristic(input, log_obj, pos + offset, part)
+ local ext, weight = match.heuristic(input, log_obj, pos + offset, part)
if ext then
add_result(weight, ext)
-- Match all positions
local all_right = true
local matched_pos = 0
- for _,position in ipairs(match.positions) do
+ for _, position in ipairs(match.positions) do
local matched = false
- for _,pos in ipairs(matched_positions) do
+ for _, pos in ipairs(matched_positions) do
lua_util.debugm(N, log_obj, 'found match %s at offset %s(from %s)',
pattern.ext, pos, offset)
if not match_position(pos + offset, position) then
if all_right then
if match.heuristic then
- local ext,weight = match.heuristic(input, log_obj, matched_pos + offset, part)
+ local ext, weight = match.heuristic(input, log_obj, matched_pos + offset, part)
if ext then
add_result(weight, ext)
return res[ex1] > res[ex2]
end)
- return extensions,res[extensions[1]]
+ return extensions, res[extensions[1]]
end
return nil
end
exports.detect = function(part, log_obj)
- if not log_obj then log_obj = rspamd_config end
+ if not log_obj then
+ log_obj = rspamd_config
+ end
local input = part:get_content()
local res = {}
input = rspamd_text.fromstring(input)
end
-
if type(input) == 'userdata' then
local inplen = #input
compiled_short_patterns, short_patterns, log_obj, res, part)
-- Check if we have enough data or go to long patterns
- local extensions,confidence = process_detected(res)
+ local extensions, confidence = process_detected(res)
if extensions and #extensions > 0 and confidence > 30 then
-- We are done on short patterns
- return extensions[1],types[extensions[1]]
+ return extensions[1], types[extensions[1]]
end
-- No way, let's check data in chunks or just the whole input if it is small enough
if #input > exports.chunk_size * 3 then
-- Chunked version as input is too long
- local chunk1, chunk2 =
- input:span(1, exports.chunk_size * 2),
+ local chunk1, chunk2 = input:span(1, exports.chunk_size * 2),
input:span(inplen - exports.chunk_size, exports.chunk_size)
local offset1, offset2 = 0, inplen - exports.chunk_size
local extensions = process_detected(res)
if extensions and #extensions > 0 then
- return extensions[1],types[extensions[1]]
+ return extensions[1], types[extensions[1]]
end
-- Nothing found
end
exports.detect_mime_part = function(part, log_obj)
- local ext,weight = heuristics.mime_part_heuristic(part, log_obj)
+ local ext, weight = heuristics.mime_part_heuristic(part, log_obj)
if ext and weight and weight > 20 then
- return ext,types[ext]
+ return ext, types[ext]
end
ext = exports.detect(part, log_obj)
if ext then
- return ext,types[ext]
+ return ext, types[ext]
end
-- Text/html and other parts
- ext,weight = heuristics.text_part_heuristic(part, log_obj)
+ ext, weight = heuristics.text_part_heuristic(part, log_obj)
if ext and weight and weight > 20 then
- return ext,types[ext]
+ return ext, types[ext]
end
end
matches = {
{
string = [[%PDF-[12]\.\d]],
- position = {'<=', 1024},
+ position = { '<=', 1024 },
weight = 60,
heuristic = heuristics.pdf_format_heuristic
},
{
string = [[%FDF-[12]\.\d]],
- position = {'<=', 1024},
+ position = { '<=', 1024 },
weight = 60,
heuristic = heuristics.pdf_format_heuristic
},
-- PE part
{
string = [[PE\x{00}\x{00}]],
- position = {'>=', 0x3c + 4},
+ position = { '>=', 0x3c + 4 },
weight = 15,
heuristic = heuristics.pe_part_heuristic,
}
matches = {
{
string = [[(?i)@\s*ECHO\s+OFF]],
- position = {'>=', 0},
+ position = { '>=', 0 },
weight = 60,
},
}
matches = {
{
hex = [[4d53434600000000]], -- Can be anywhere for SFX :(
- position = {'>=', 8},
+ position = { '>=', 8 },
weight = 60,
},
}
matches = {
{
string = [[\x{01}CD001\x{01}]],
- position = {'>=', 0x8000 + 7}, -- first 32k is unused
+ position = { '>=', 0x8000 + 7 }, -- first 32k is unused
weight = 60,
},
}
return st:hex()
end
-local function starts(where,st)
- return string.sub(where,1,string.len(st))==st
+local function starts(where, st)
+ return string.sub(where, 1, string.len(st)) == st
end
-local function cut_prefix(where,st)
- return string.sub(where,#st + 1)
+local function cut_prefix(where, st)
+ return string.sub(where, #st + 1)
end
-local function maybe_adjust_type(data,mtype)
+local function maybe_adjust_type(data, mtype)
local function check_prefix(prefix, t)
if starts(data, prefix) then
data = cut_prefix(data, prefix)
end
local known_types = {
- {'regexp;', 'regexp'},
- {'re;', 'regexp'},
- {'regexp_multi;', 'regexp_multi'},
- {'re_multi;', 'regexp_multi'},
- {'glob;', 'glob'},
- {'glob_multi;', 'glob_multi'},
- {'radix;', 'radix'},
- {'ipnet;', 'radix'},
- {'set;', 'set'},
- {'hash;', 'hash'},
- {'plain;', 'hash'},
- {'cdb;', 'cdb'},
- {'cdb:/', 'cdb'},
+ { 'regexp;', 'regexp' },
+ { 're;', 'regexp' },
+ { 'regexp_multi;', 'regexp_multi' },
+ { 're_multi;', 'regexp_multi' },
+ { 'glob;', 'glob' },
+ { 'glob_multi;', 'glob_multi' },
+ { 'radix;', 'radix' },
+ { 'ipnet;', 'radix' },
+ { 'set;', 'set' },
+ { 'hash;', 'hash' },
+ { 'plain;', 'hash' },
+ { 'cdb;', 'cdb' },
+ { 'cdb:/', 'cdb' },
}
if mtype == 'callback' then
return mtype
end
- for _,t in ipairs(known_types) do
+ for _, t in ipairs(known_types) do
if check_prefix(t[1], t[2]) then
- return data,mtype
+ return data, mtype
end
end
-- No change
- return data,mtype
+ return data, mtype
end
-
-local external_map_schema = ts.shape{
+local external_map_schema = ts.shape {
external = ts.equivalent(true), -- must be true
backend = ts.string, -- where to get data, required
- method = ts.one_of{"body", "header", "query"}, -- how to pass input
- encode = ts.one_of{"json", "messagepack"}:is_optional(), -- how to encode input (if relevant)
+ method = ts.one_of { "body", "header", "query" }, -- how to pass input
+ encode = ts.one_of { "json", "messagepack" }:is_optional(), -- how to encode input (if relevant)
timeout = (ts.number + ts.string / lua_util.parse_time_interval):is_optional(),
}
local function url_encode_string(str)
str = string.gsub(str, "([^%w _%%%-%.~])",
- function(c) return string.format("%%%02X", string.byte(c)) end)
+ function(c)
+ return string.format("%%%02X", string.byte(c))
+ end)
str = string.gsub(str, " ", "+")
return str
end
-- query/header and no encode
if map_config.method == 'query' then
local params_table = {}
- for k,v in pairs(key) do
+ for k, v in pairs(key) do
if type(v) == 'string' then
table.insert(params_table, string.format('%s=%s', url_encode_string(k), url_encode_string(v)))
end
end
end
- local ret = rspamd_http.request{
+ local ret = rspamd_http.request {
task = task,
url = url,
callback = map_callback,
end
if type(opt) == 'string' then
- opt,mtype = maybe_adjust_type(opt, mtype)
+ opt, mtype = maybe_adjust_type(opt, mtype)
local cache_key = map_hash_key(opt, mtype)
if not callback and maps_cache[cache_key] then
rspamd_logger.infox(rspamd_config, 'reuse url for %s(%s)',
return maps_cache[cache_key]
end
-- We have a single string, so we treat it as a map
- local map = rspamd_config:add_map{
+ local map = rspamd_config:add_map {
type = mtype,
description = description,
url = opt,
local cache_key = lua_util.table_digest(opt)
if not callback and maps_cache[cache_key] then
rspamd_logger.infox(rspamd_config, 'reuse url for complex map definition %s: %s',
- cache_key:sub(1,8), description)
+ cache_key:sub(1, 8), description)
return maps_cache[cache_key]
end
if opt[1] then
-- Adjust each element if needed
local adjusted
- for i,source in ipairs(opt) do
- local nsrc,ntype = maybe_adjust_type(source, mtype)
+ for i, source in ipairs(opt) do
+ local nsrc, ntype = maybe_adjust_type(source, mtype)
if mtype ~= ntype then
if not adjusted then
end
else
-- Plain table
- local map = rspamd_config:add_map{
+ local map = rspamd_config:add_map {
type = mtype,
description = description,
url = opt,
elseif mtype == 'regexp' or mtype == 'glob' then
if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then
-- Plain table
- local map = rspamd_config:add_map{
+ local map = rspamd_config:add_map {
type = mtype,
description = description,
url = opt,
return ret
end
else
- local map = rspamd_config:add_map{
+ local map = rspamd_config:add_map {
type = mtype,
description = description,
url = {
else
if string.find(opt[1], '^/%a') or string.find(opt[1], '^http') then
-- Plain table
- local map = rspamd_config:add_map{
+ local map = rspamd_config:add_map {
type = mtype,
description = description,
url = opt,
local data = {}
local nelts = 0
-- Plain array of keys, count merely numeric elts
- for _,elt in ipairs(opt) do
+ for _, elt in ipairs(opt) do
if type(elt) == 'string' then
-- Numeric table
if mtype == 'hash' then
return nil
end
ret.foreach = function(_, func)
- for k,v in pairs(ret.__data) do
+ for k, v in pairs(ret.__data) do
if not func(k, v) then
return false
end
else
if opt.external then
-- External map definition, missing fields are handled by schema
- local parse_res,parse_err = external_map_schema(opt)
+ local parse_res, parse_err = external_map_schema(opt)
if parse_res then
ret.__upstreams = lua_util.http_upstreams_by_url(rspamd_config:get_mempool(), opt.backend)
else
-- Adjust lua specific augmentations in a trivial case
if type(opt.url) == 'string' then
- local nsrc,ntype = maybe_adjust_type(opt.url, mtype)
+ local nsrc, ntype = maybe_adjust_type(opt.url, mtype)
if nsrc and ntype then
opt.url = nsrc
mtype = ntype
end
end
-- We have some non-trivial object so let C code to deal with it somehow...
- local map = rspamd_config:add_map{
+ local map = rspamd_config:add_map {
type = mtype,
description = description,
url = opt,
local fun = require "fun"
if type(what) == "table" then
- return fun.any(function(elt) return rspamd_maybe_check_map(key, elt) end, what)
+ return fun.any(function(elt)
+ return rspamd_maybe_check_map(key, elt)
+ end, what)
end
if type(rspamd_maps) == "table" then
local mn
rspamd_logger.errx(rspamd_config, 'map add error %s for module %s', k, mname)
return false
end
- opts[k..'_orig'] = opts[k]
+ opts[k .. '_orig'] = opts[k]
opts[k] = map
elseif not v.optional then
rspamd_logger.errx(rspamd_config, 'cannot find non optional map %s for module %s', k, mname)
return true
end
-local direct_map_schema = ts.shape{ -- complex object
+local direct_map_schema = ts.shape { -- complex object
name = ts.string:is_optional(),
description = ts.string:is_optional(),
selector_alias = ts.string:is_optional(), -- an optional alias for the selectos framework
timeout = ts.number,
data = ts.array_of(ts.string):is_optional(),
-- Tableshape has no options support for something like key1 or key2?
- upstreams = ts.one_of{
+ upstreams = ts.one_of {
ts.string,
ts.array_of(ts.string),
- }:is_optional(),
- url = ts.one_of{
+ } :is_optional(),
+ url = ts.one_of {
ts.string,
ts.array_of(ts.string),
- }:is_optional(),
+ } :is_optional(),
}
-exports.map_schema = ts.one_of{
+exports.map_schema = ts.one_of {
ts.string, -- 'http://some_map'
ts.array_of(ts.string), -- ['foo', 'bar']
- ts.one_of{direct_map_schema, external_map_schema}
+ ts.one_of { direct_map_schema, external_map_schema }
}
return exports
if values then
if type(values) == 'table' then
- for _,val in ipairs(values) do
+ for _, val in ipairs(values) do
if res == 0 then
match_rule(val)
end
local res = elt.expr:process(process_atom)
if res > 0 then
- return res,matched
+ return res, matched
end
return nil
end
-
-exports.schema = ts.shape{
+exports.schema = ts.shape {
expression = ts.string,
rules = ts.array_of(
- ts.shape{
+ ts.shape {
selector = ts.string,
map = lua_maps.map_schema,
}
--
--]]
local function create(cfg, obj, module_name)
- if not module_name then module_name = 'lua_maps_expressions' end
+ if not module_name then
+ module_name = 'lua_maps_expressions'
+ end
if not obj or not obj.rules or not obj.expression then
rspamd_logger.errx(cfg, 'cannot add maps combination for module %s: required elements are missing',
module_name = module_name
}
- for name,rule in pairs(obj.rules) do
+ for name, rule in pairs(obj.rules) do
local sel = lua_selectors.create_selector_closure(cfg, rule.selector)
if not sel then
ret.expr = expr
if obj.symbol then
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual,ghost',
name = obj.symbol,
score = 0.0,
}
local size = task:get_size()
- for i = 1,#sizes do
+ for i = 1, #sizes do
if sizes[i] >= size then
- return {(1.0 * i) / #sizes}
+ return { (1.0 * i) / #sizes }
end
end
- return {0}
+ return { 0 }
end
local function meta_images_function(task)
local nsmall = 0
if images then
- for _,img in ipairs(images) do
+ for _, img in ipairs(images) do
if img:get_type() == 'png' then
npng = npng + 1
elseif img:get_type() == 'jpeg' then
nlarge = 1.0 * nlarge / ntotal
nsmall = 1.0 * nsmall / ntotal
end
- return {ntotal,njpg,npng,nlarge,nsmall}
+ return { ntotal, njpg, npng, nlarge, nsmall }
end
local function meta_nparts_function(task)
local parts = task:get_parts()
if parts then
- for _,p in ipairs(parts) do
+ for _, p in ipairs(parts) do
if p:is_attachment() then
nattachments = nattachments + 1
end
end
end
- return {(1.0 * ntextparts)/totalparts, (1.0 * nattachments)/totalparts}
+ return { (1.0 * ntextparts) / totalparts, (1.0 * nattachments) / totalparts }
end
local function meta_encoding_function(task)
local tp = task:get_text_parts()
if tp and #tp > 0 then
- for _,p in ipairs(tp) do
+ for _, p in ipairs(tp) do
if p:is_utf() then
nutf = nutf + 1
else
end
end
- return {nutf / #tp, nother / #tp}
+ return { nutf / #tp, nother / #tp }
end
- return {0, 0}
+ return { 0, 0 }
end
local function meta_recipients_function(task)
nsmtp = #(task:get_recipients('smtp'))
end
- if nmime > 0 then nmime = 1.0 / nmime end
- if nsmtp > 0 then nsmtp = 1.0 / nsmtp end
+ if nmime > 0 then
+ nmime = 1.0 / nmime
+ end
+ if nsmtp > 0 then
+ nsmtp = 1.0 / nsmtp
+ end
- return {nmime,nsmtp}
+ return { nmime, nsmtp }
end
local function meta_received_function(task)
secure_factor = secure_factor + 1.0
end
end,
- fun.filter(function(rc) return not rc.flags or not rc.flags['artificial'] end, rh))
+ fun.filter(function(rc)
+ return not rc.flags or not rc.flags['artificial']
+ end, rh))
if ntotal > 0 then
invalid_factor = invalid_factor / ntotal
end
end
- return {count_factor, invalid_factor, time_factor, secure_factor}
+ return { count_factor, invalid_factor, time_factor, secure_factor }
end
local function meta_urls_function(task)
- local has_urls,nurls = task:has_urls()
+ local has_urls, nurls = task:has_urls()
if has_urls and nurls > 0 then
- return {1.0 / nurls}
+ return { 1.0 / nurls }
end
- return {0}
+ return { 0 }
end
local function meta_words_function(task)
20,
}
- for i = 1,#lens do
+ for i = 1, #lens do
if lens[i] >= avg_len then
ret_len = (1.0 * i) / #lens
break
0, -- capital characters rate
0, -- numeric characters
}
- for _,p in ipairs(tp) do
+ for _, p in ipairs(tp) do
local stats = p:get_stats()
local len = p:get_length()
divisor = #tp
end
- for _,wr in ipairs(wres) do
+ for _, wr in ipairs(wres) do
table.insert(ret, wr / divisor)
end
},
}
-local meta_schema = ts.shape{
+local meta_schema = ts.shape {
cb = ts.func,
ninputs = ts.number,
names = ts.array_of(ts.string),
local function fill_metatokens_by_name()
metatokens_by_name = {}
- for _,mt in ipairs(metafunctions) do
- for i=1,mt.ninputs do
+ for _, mt in ipairs(metafunctions) do
+ for i = 1, mt.ninputs do
local name = mt.names[i]
metatokens_by_name[name] = function(task)
local cr = require "rspamd_cryptobox_hash"
local h = cr.create()
- for _,mt in ipairs(metafunctions) do
- for i=1,mt.ninputs do
+ for _, mt in ipairs(metafunctions) do
+ for i = 1, mt.ninputs do
local name = mt.names[i]
h:update(name)
end
if cached then
return cached
else
- for _,mt in ipairs(metafunctions) do
+ for _, mt in ipairs(metafunctions) do
local ct = mt.cb(task)
- for i,tok in ipairs(ct) do
+ for i, tok in ipairs(ct) do
lua_util.debugm(N, task, "metatoken: %s = %s",
mt.names[i], tok)
if tok ~= tok or tok == math.huge then
end
else
- for _,n in ipairs(names) do
+ for _, n in ipairs(names) do
if metatokens_by_name[n] then
local tok = metatokens_by_name[n](task)
if tok ~= tok or tok == math.huge then
end
return metatokens
- end
+end
exports.rspamd_gen_metatokens = rspamd_gen_metatokens
exports.gen_metatokens = rspamd_gen_metatokens
local function rspamd_gen_metatokens_table(task)
local metatokens = {}
- for _,mt in ipairs(metafunctions) do
+ for _, mt in ipairs(metafunctions) do
local ct = mt.cb(task)
- for i,tok in ipairs(ct) do
+ for i, tok in ipairs(ct) do
if tok ~= tok or tok == math.huge then
logger.errx(task, 'metatoken %s returned %s; replace it with 0 for sanity',
mt.names[i], tok)
local function rspamd_count_metatokens()
local ipairs = ipairs
local total = 0
- for _,mt in ipairs(metafunctions) do
+ for _, mt in ipairs(metafunctions) do
total = total + mt.ninputs
end
end
if is_multipart then
- out[#out + 1] = string.format('Content-Type: %s; charset=utf-8%s'..
+ out[#out + 1] = string.format('Content-Type: %s; charset=utf-8%s' ..
'Content-Transfer-Encoding: %s',
ct, newline_s, cte)
out[#out + 1] = ''
if content:sub(-(nlen), nlen + 1) == double_nline then
-- content without last newline
content = content:sub(-(#newline_s), #newline_s + 1) .. footer
- out[#out + 1] = {encode_func(content), true}
+ out[#out + 1] = { encode_func(content), true }
out[#out + 1] = ''
else
content = content .. footer
- out[#out + 1] = {encode_func(content), true}
+ out[#out + 1] = { encode_func(content), true }
out[#out + 1] = ''
end
local boundaries = {}
local cur_boundary
- for _,part in ipairs(task:get_parts()) do
+ for _, part in ipairs(task:get_parts()) do
local boundary = part:get_boundary()
if part:is_multipart() then
if cur_boundary then
local rh = part:get_raw_headers()
if #rh > 0 then
- out[#out + 1] = {rh, true}
+ out[#out + 1] = { rh, true }
end
elseif part:is_message() then
if boundary then
boundary)
end
- out[#out + 1] = {part:get_raw_headers(), true}
+ out[#out + 1] = { part:get_raw_headers(), true }
else
local append_footer = false
local skip_footer = part:is_attachment()
local parent = part:get_parent()
if parent then
- local t,st = parent:get_type()
+ local t, st = parent:get_type()
if t == 'multipart' and st == 'signed' then
-- Do not modify signed parts
do_append_footer(task, part, append_footer,
parent and parent:is_multipart(), out, state)
else
- out[#out + 1] = {part:get_raw_headers(), true}
- out[#out + 1] = {part:get_raw_content(), false}
+ out[#out + 1] = { part:get_raw_headers(), true }
+ out[#out + 1] = { part:get_raw_content(), false }
end
end
end
-- sort matches and form the table:
-- start .. end for inclusion position
local matches_flattened = {}
- for npat,matches in pairs(match_pos) do
- for _,m in ipairs(matches) do
- table.insert(matches_flattened, {m, npat})
+ for npat, matches in pairs(match_pos) do
+ for _, m in ipairs(matches) do
+ table.insert(matches_flattened, { m, npat })
end
end
-- Handle the case of empty match
if #matches_flattened == 0 then
- out[#out + 1] = {part:get_raw_headers(), true}
- out[#out + 1] = {part:get_raw_content(), false}
+ out[#out + 1] = { part:get_raw_headers(), true }
+ out[#out + 1] = { part:get_raw_content(), false }
return
end
if is_multipart then
- out[#out + 1] = {string.format('Content-Type: %s; charset="utf-8"%s'..
+ out[#out + 1] = { string.format('Content-Type: %s; charset="utf-8"%s' ..
'Content-Transfer-Encoding: %s',
- ct, newline_s, cte), true}
- out[#out + 1] = {'', true}
+ ct, newline_s, cte), true }
+ out[#out + 1] = { '', true }
else
state.new_cte = cte
end
state.has_matches = true
-- now sort flattened by start of match and eliminate all overlaps
- table.sort(matches_flattened, function(m1, m2) return m1[1][1] < m2[1][1] end)
+ table.sort(matches_flattened, function(m1, m2)
+ return m1[1][1] < m2[1][1]
+ end)
- for i=1,#matches_flattened - 1 do
+ for i = 1, #matches_flattened - 1 do
local st = matches_flattened[i][1][1] -- current start of match
local e = matches_flattened[i][1][2] -- current end of match
local max_npat = matches_flattened[i][2]
- for j=i+1,#matches_flattened do
+ for j = i + 1, #matches_flattened do
if matches_flattened[j][1][1] == st then
-- overlap
if matches_flattened[j][1][2] > e then
end
end
-- Maximum overlap for all matches
- for j=i,#matches_flattened do
+ for j = i, #matches_flattened do
if matches_flattened[j][1][1] == st then
if e > matches_flattened[j][1][2] then
matches_flattened[j][1][2] = e
end
end
-- Off-by one: match returns 0 based positions while we use 1 based in Lua
- for _,m in ipairs(matches_flattened) do
+ for _, m in ipairs(matches_flattened) do
m[1][1] = m[1][1] + 1
m[1][2] = m[1][2] + 1
end
local cur_start = 1
local fragments = {}
- for _,m in ipairs(matches_flattened) do
+ for _, m in ipairs(matches_flattened) do
if m[1][1] >= cur_start then
fragments[#fragments + 1] = content:sub(cur_start, m[1][1] - 1)
fragments[#fragments + 1] = replacements[m[2]]
end
-- Final stuff
- out[#out + 1] = {encode_func(rspamd_text.fromtable(fragments)), false}
+ out[#out + 1] = { encode_func(rspamd_text.fromtable(fragments)), false }
else
-- No matches
- out[#out + 1] = {part:get_raw_headers(), true}
- out[#out + 1] = {part:get_raw_content(), false}
+ out[#out + 1] = { part:get_raw_headers(), true }
+ out[#out + 1] = { part:get_raw_content(), false }
end
end
local boundaries = {}
local cur_boundary
- for _,part in ipairs(task:get_parts()) do
+ for _, part in ipairs(task:get_parts()) do
local boundary = part:get_boundary()
if part:is_multipart() then
if cur_boundary then
- out[#out + 1] = {string.format('--%s',
- boundaries[#boundaries]), true}
+ out[#out + 1] = { string.format('--%s',
+ boundaries[#boundaries]), true }
end
boundaries[#boundaries + 1] = boundary or '--XXX'
local rh = part:get_raw_headers()
if #rh > 0 then
- out[#out + 1] = {rh, true}
+ out[#out + 1] = { rh, true }
end
elseif part:is_message() then
if boundary then
if cur_boundary and boundary ~= cur_boundary then
-- Need to close boundary
- out[#out + 1] = {string.format('--%s--',
- boundaries[#boundaries]), true}
+ out[#out + 1] = { string.format('--%s--',
+ boundaries[#boundaries]), true }
table.remove(boundaries)
cur_boundary = nil
end
- out[#out + 1] = {string.format('--%s',
- boundary), true}
+ out[#out + 1] = { string.format('--%s',
+ boundary), true }
end
- out[#out + 1] = {part:get_raw_headers(), true}
+ out[#out + 1] = { part:get_raw_headers(), true }
else
local skip_replacement = part:is_attachment()
local parent = part:get_parent()
if parent then
- local t,st = parent:get_type()
+ local t, st = parent:get_type()
if t == 'multipart' and st == 'signed' then
-- Do not modify signed parts
if boundary then
if cur_boundary and boundary ~= cur_boundary then
-- Need to close boundary
- out[#out + 1] = {string.format('--%s--',
- boundaries[#boundaries]), true}
+ out[#out + 1] = { string.format('--%s--',
+ boundaries[#boundaries]), true }
table.remove(boundaries)
cur_boundary = boundary
end
- out[#out + 1] = {string.format('--%s',
- boundary), true}
+ out[#out + 1] = { string.format('--%s',
+ boundary), true }
end
if not skip_replacement then
parent and parent:is_multipart(), out, state)
else
-- Append as is
- out[#out + 1] = {part:get_raw_headers(), true}
- out[#out + 1] = {part:get_raw_content(), false}
+ out[#out + 1] = { part:get_raw_headers(), true }
+ out[#out + 1] = { part:get_raw_content(), false }
end
end
end
-- Close remaining
local b = table.remove(boundaries)
while b do
- out[#out + 1] = {string.format('--%s--', b), true}
+ out[#out + 1] = { string.format('--%s--', b), true }
if #boundaries > 0 then
- out[#out + 1] = {'', true}
+ out[#out + 1] = { '', true }
end
b = table.remove(boundaries)
end
--]]
exports.modify_headers = function(task, hdr_alterations, mode)
-- Assume default mode compatibility
- if not mode then mode = 'compat' end
+ if not mode then
+ mode = 'compat'
+ end
local add = hdr_alterations.add or {}
local remove = hdr_alterations.remove or {}
add_headers[hname] = {}
end
if not hdr_flattened[hname] then
- hdr_flattened[hname] = {add = {}}
+ hdr_flattened[hname] = { add = {} }
end
local add_tbl = hdr_flattened[hname].add
if hdr.value then
order = (tonumber(hdr.order) or -1),
value = hdr.value,
})
- table.insert(add_tbl, {tonumber(hdr.order) or -1, hdr.value})
+ table.insert(add_tbl, { tonumber(hdr.order) or -1, hdr.value })
elseif type(hdr) == 'table' then
- for _,v in ipairs(hdr) do
+ for _, v in ipairs(hdr) do
flatten_add_header(hname, v)
end
elseif type(hdr) == 'string' then
order = -1,
value = hdr,
})
- table.insert(add_tbl, {-1, hdr})
+ table.insert(add_tbl, { -1, hdr })
else
logger.errx(task, 'invalid modification of header: %s', hdr)
end
end
if hdr_alterations.order then
-- Get headers alterations ordered
- for _,hname in ipairs(hdr_alterations.order) do
+ for _, hname in ipairs(hdr_alterations.order) do
flatten_add_header(hname, add[hname])
end
else
- for hname,hdr in pairs(add) do
+ for hname, hdr in pairs(add) do
flatten_add_header(hname, hdr)
end
end
-
- for hname,hdr in pairs(remove) do
+ for hname, hdr in pairs(remove) do
if not hdr_flattened[hname] then
- hdr_flattened[hname] = {remove = {}}
+ hdr_flattened[hname] = { remove = {} }
end
if not hdr_flattened[hname].remove then
hdr_flattened[hname].remove = {}
if type(hdr) == 'number' then
table.insert(remove_tbl, hdr)
else
- for _,num in ipairs(hdr) do
+ for _, num in ipairs(hdr) do
table.insert(remove_tbl, num)
end
end
if mode == 'compat' then
-- Clear empty alterations in the compat mode
- if add_headers and not next(add_headers) then add_headers = nil end
- if hdr_alterations.remove and not next(hdr_alterations.remove) then hdr_alterations.remove = nil end
+ if add_headers and not next(add_headers) then
+ add_headers = nil
+ end
+ if hdr_alterations.remove and not next(hdr_alterations.remove) then
+ hdr_alterations.remove = nil
+ end
end
task:set_milter_reply({
add_headers = add_headers,
remove_headers = hdr_alterations.remove
})
- for hname,flat_rules in pairs(hdr_flattened) do
+ for hname, flat_rules in pairs(hdr_flattened) do
task:modify_header(hname, flat_rules)
end
end
local E = {}
local maybe_stringify_f = stringify_content and
- tostring or function(t) return t end
+ tostring or function(t)
+ return t
+ end
local result = {
size = task:get_size(),
digest = task:get_digest(),
local parts = task:get_parts() or E
result.parts = {}
- for _,part in ipairs(parts) do
+ for _, part in ipairs(parts) do
if not part:is_multipart() and not part:is_message() then
local p = {
size = part:get_length(),
detected_type = string.format('%s/%s', part:get_detected_type()),
filename = part:get_filename(),
content = maybe_stringify_f(part:get_content()),
- headers = part:get_headers(true) or E,
+ headers = part:get_headers(true) or E,
boundary = part:get_enclosing_boundary(),
}
table.insert(result.parts, p)
-- Service part: multipart container or message/rfc822
local p = {
type = string.format('%s/%s', part:get_type()),
- headers = part:get_headers(true) or E,
+ headers = part:get_headers(true) or E,
boundary = part:get_enclosing_boundary(),
size = 0,
}
local ts = require("tableshape").types
local function headers_schema()
- return ts.shape{
- order = ts.integer:describe('Header order in a message'),
+ return ts.shape {
+ order = ts.integer:describe('Header order in a message'),
raw = ts.string:describe('Raw header value'):is_optional(),
empty_separator = ts.boolean:describe('Whether header has an empty separator'),
separator = ts.string:describe('Separator between a header and a value'),
end
local function part_schema()
- return ts.shape{
- content = ts.string:describe('Decoded content'):is_optional(),
+ return ts.shape {
+ content = ts.string:describe('Decoded content'):is_optional(),
multipart_boundary = ts.string:describe('Multipart service boundary'):is_optional(),
size = ts.integer:describe('Size of the part'),
type = ts.string:describe('Announced type'):is_optional(),
end
local function email_addr_schema()
- return ts.shape{
- addr = ts.string:describe('Parsed address'):is_optional(),
+ return ts.shape {
+ addr = ts.string:describe('Parsed address'):is_optional(),
raw = ts.string:describe('Raw address'),
- flags = ts.shape{
+ flags = ts.shape {
valid = ts.boolean:describe('Valid address'):is_optional(),
ip = ts.boolean:describe('IP like address'):is_optional(),
braced = ts.boolean:describe('Have braces around address'):is_optional(),
backslash = ts.boolean:describe('Backslash in address'):is_optional(),
['8bit'] = ts.boolean:describe('8 bit characters in address'):is_optional(),
},
- user = ts.string:describe('Parsed user part'):is_optional(),
- name = ts.string:describe('Displayed name'):is_optional(),
- domain = ts.string:describe('Parsed domain part'):is_optional(),
+ user = ts.string:describe('Parsed user part'):is_optional(),
+ name = ts.string:describe('Displayed name'):is_optional(),
+ domain = ts.string:describe('Parsed domain part'):is_optional(),
}
end
local function envelope_schema()
- return ts.shape{
+ return ts.shape {
from_smtp = email_addr_schema():describe('SMTP from'):is_optional(),
recipients_smtp = ts.array_of(email_addr_schema()):describe('SMTP recipients'):is_optional(),
helo = ts.string:describe('SMTP Helo'):is_optional(),
}
end
- return ts.shape{
+ return ts.shape {
headers = ts.array_of(headers_schema()),
parts = ts.array_of(part_schema()),
digest = ts.pattern(string.format('^%s$', string.rep('%x', 32)))
- :describe('Message digest'),
- newlines = ts.one_of({"cr", "lf", "crlf"}):describe('Newlines type'),
+ :describe('Message digest'),
+ newlines = ts.one_of({ "cr", "lf", "crlf" }):describe('Newlines type'),
size = ts.integer:describe('Size of the message in bytes'),
envelope = envelope_schema()
}
-- All mime extensions with corresponding content types
exports.full_extensions_map = {
- {"323", "text/h323"},
- {"3g2", "video/3gpp2"},
- {"3gp", "video/3gpp"},
- {"3gp2", "video/3gpp2"},
- {"3gpp", "video/3gpp"},
- {"7z", {"application/x-7z-compressed", "application/7z"}},
- {"aa", "audio/audible"},
- {"AAC", "audio/aac"},
- {"aaf", "application/octet-stream"},
- {"aax", "audio/vnd.audible.aax"},
- {"ac3", "audio/ac3"},
- {"aca", "application/octet-stream"},
- {"accda", "application/msaccess.addin"},
- {"accdb", "application/msaccess"},
- {"accdc", "application/msaccess.cab"},
- {"accde", "application/msaccess"},
- {"accdr", "application/msaccess.runtime"},
- {"accdt", "application/msaccess"},
- {"accdw", "application/msaccess.webapplication"},
- {"accft", "application/msaccess.ftemplate"},
- {"acx", "application/internet-property-stream"},
- {"AddIn", "text/xml"},
- {"ade", "application/msaccess"},
- {"adobebridge", "application/x-bridge-url"},
- {"adp", "application/msaccess"},
- {"ADT", "audio/vnd.dlna.adts"},
- {"ADTS", "audio/aac"},
- {"afm", "application/octet-stream"},
- {"ai", "application/postscript"},
- {"aif", "audio/aiff"},
- {"aifc", "audio/aiff"},
- {"aiff", "audio/aiff"},
- {"air", "application/vnd.adobe.air-application-installer-package+zip"},
- {"amc", "application/mpeg"},
- {"anx", "application/annodex"},
- {"apk", "application/vnd.android.package-archive" },
- {"application", "application/x-ms-application"},
- {"art", "image/x-jg"},
- {"asa", "application/xml"},
- {"asax", "application/xml"},
- {"ascx", "application/xml"},
- {"asd", "application/octet-stream"},
- {"asf", "video/x-ms-asf"},
- {"ashx", "application/xml"},
- {"asi", "application/octet-stream"},
- {"asm", "text/plain"},
- {"asmx", "application/xml"},
- {"aspx", "application/xml"},
- {"asr", "video/x-ms-asf"},
- {"asx", "video/x-ms-asf"},
- {"atom", "application/atom+xml"},
- {"au", "audio/basic"},
- {"avi", "video/x-msvideo"},
- {"axa", "audio/annodex"},
- {"axs", "application/olescript"},
- {"axv", "video/annodex"},
- {"bas", "text/plain"},
- {"bcpio", "application/x-bcpio"},
- {"bin", "application/octet-stream"},
- {"bmp", {"image/bmp", "image/x-ms-bmp"}},
- {"c", "text/plain"},
- {"cab", "application/octet-stream"},
- {"caf", "audio/x-caf"},
- {"calx", "application/vnd.ms-office.calx"},
- {"cat", "application/vnd.ms-pki.seccat"},
- {"cc", "text/plain"},
- {"cd", "text/plain"},
- {"cdda", "audio/aiff"},
- {"cdf", "application/x-cdf"},
- {"cer", "application/x-x509-ca-cert"},
- {"cfg", "text/plain"},
- {"chm", "application/octet-stream"},
- {"class", "application/x-java-applet"},
- {"clp", "application/x-msclip"},
- {"cmd", "text/plain"},
- {"cmx", "image/x-cmx"},
- {"cnf", "text/plain"},
- {"cod", "image/cis-cod"},
- {"config", "application/xml"},
- {"contact", "text/x-ms-contact"},
- {"coverage", "application/xml"},
- {"cpio", "application/x-cpio"},
- {"cpp", "text/plain"},
- {"crd", "application/x-mscardfile"},
- {"crl", "application/pkix-crl"},
- {"crt", "application/x-x509-ca-cert"},
- {"cs", "text/plain"},
- {"csdproj", "text/plain"},
- {"csh", "application/x-csh"},
- {"csproj", "text/plain"},
- {"css", "text/css"},
- {"csv", {"application/vnd.ms-excel", "text/csv", "text/plain"}},
- {"cur", "application/octet-stream"},
- {"cxx", "text/plain"},
- {"dat", {"application/octet-stream", "application/ms-tnef"}},
- {"datasource", "application/xml"},
- {"dbproj", "text/plain"},
- {"dcr", "application/x-director"},
- {"def", "text/plain"},
- {"deploy", "application/octet-stream"},
- {"der", "application/x-x509-ca-cert"},
- {"dgml", "application/xml"},
- {"dib", "image/bmp"},
- {"dif", "video/x-dv"},
- {"dir", "application/x-director"},
- {"disco", "text/xml"},
- {"divx", "video/divx"},
- {"dll", "application/x-msdownload"},
- {"dll.config", "text/xml"},
- {"dlm", "text/dlm"},
- {"doc", "application/msword"},
- {"docm", "application/vnd.ms-word.document.macroEnabled.12"},
- {"docx", {
- "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
- "application/msword",
- "application/vnd.ms-word.document.12",
- "application/octet-stream",
- }},
- {"dot", "application/msword"},
- {"dotm", "application/vnd.ms-word.template.macroEnabled.12"},
- {"dotx", "application/vnd.openxmlformats-officedocument.wordprocessingml.template"},
- {"dsp", "application/octet-stream"},
- {"dsw", "text/plain"},
- {"dtd", "text/xml"},
- {"dtsConfig", "text/xml"},
- {"dv", "video/x-dv"},
- {"dvi", "application/x-dvi"},
- {"dwf", "drawing/x-dwf"},
- {"dwg", {"application/acad", "image/vnd.dwg"}},
- {"dwp", "application/octet-stream"},
- {"dxf", "application/x-dxf" },
- {"dxr", "application/x-director"},
- {"eml", "message/rfc822"},
- {"emz", "application/octet-stream"},
- {"eot", "application/vnd.ms-fontobject"},
- {"eps", "application/postscript"},
- {"etl", "application/etl"},
- {"etx", "text/x-setext"},
- {"evy", "application/envoy"},
- {"exe", {
- "application/x-dosexec",
- "application/x-msdownload",
- "application/x-executable",
- }},
- {"exe.config", "text/xml"},
- {"fdf", "application/vnd.fdf"},
- {"fif", "application/fractals"},
- {"filters", "application/xml"},
- {"fla", "application/octet-stream"},
- {"flac", "audio/flac"},
- {"flr", "x-world/x-vrml"},
- {"flv", "video/x-flv"},
- {"fsscript", "application/fsharp-script"},
- {"fsx", "application/fsharp-script"},
- {"generictest", "application/xml"},
- {"gif", "image/gif"},
- {"gpx", "application/gpx+xml"},
- {"group", "text/x-ms-group"},
- {"gsm", "audio/x-gsm"},
- {"gtar", "application/x-gtar"},
- {"gz", {"application/gzip", "application/x-gzip", "application/tlsrpt+gzip"}},
- {"h", "text/plain"},
- {"hdf", "application/x-hdf"},
- {"hdml", "text/x-hdml"},
- {"hhc", "application/x-oleobject"},
- {"hhk", "application/octet-stream"},
- {"hhp", "application/octet-stream"},
- {"hlp", "application/winhlp"},
- {"hpp", "text/plain"},
- {"hqx", "application/mac-binhex40"},
- {"hta", "application/hta"},
- {"htc", "text/x-component"},
- {"htm", "text/html"},
- {"html", "text/html"},
- {"htt", "text/webviewhtml"},
- {"hxa", "application/xml"},
- {"hxc", "application/xml"},
- {"hxd", "application/octet-stream"},
- {"hxe", "application/xml"},
- {"hxf", "application/xml"},
- {"hxh", "application/octet-stream"},
- {"hxi", "application/octet-stream"},
- {"hxk", "application/xml"},
- {"hxq", "application/octet-stream"},
- {"hxr", "application/octet-stream"},
- {"hxs", "application/octet-stream"},
- {"hxt", "text/html"},
- {"hxv", "application/xml"},
- {"hxw", "application/octet-stream"},
- {"hxx", "text/plain"},
- {"i", "text/plain"},
- {"ico", "image/x-icon"},
- {"ics", {"text/calendar", "application/ics", "application/octet-stream"}},
- {"idl", "text/plain"},
- {"ief", "image/ief"},
- {"iii", "application/x-iphone"},
- {"inc", "text/plain"},
- {"inf", "application/octet-stream"},
- {"ini", "text/plain"},
- {"inl", "text/plain"},
- {"ins", "application/x-internet-signup"},
- {"ipa", "application/x-itunes-ipa"},
- {"ipg", "application/x-itunes-ipg"},
- {"ipproj", "text/plain"},
- {"ipsw", "application/x-itunes-ipsw"},
- {"iqy", "text/x-ms-iqy"},
- {"isp", "application/x-internet-signup"},
- {"ite", "application/x-itunes-ite"},
- {"itlp", "application/x-itunes-itlp"},
- {"itms", "application/x-itunes-itms"},
- {"itpc", "application/x-itunes-itpc"},
- {"IVF", "video/x-ivf"},
- {"jar", "application/java-archive"},
- {"java", "application/octet-stream"},
- {"jck", "application/liquidmotion"},
- {"jcz", "application/liquidmotion"},
- {"jfif", {"image/jpeg", "image/pjpeg"}},
- {"jnlp", "application/x-java-jnlp-file"},
- {"jpb", "application/octet-stream"},
- {"jpe", {"image/jpeg", "image/pjpeg"}},
- {"jpeg", {"image/jpeg", "image/pjpeg"}},
- {"jpg", {"image/jpeg", "image/pjpeg"}},
- {"js", "application/javascript"},
- {"json", "application/json"},
- {"jsx", "text/jscript"},
- {"jsxbin", "text/plain"},
- {"latex", "application/x-latex"},
- {"library-ms", "application/windows-library+xml"},
- {"lit", "application/x-ms-reader"},
- {"loadtest", "application/xml"},
- {"lpk", "application/octet-stream"},
- {"lsf", "video/x-la-asf"},
- {"lst", "text/plain"},
- {"lsx", "video/x-la-asf"},
- {"lzh", "application/octet-stream"},
- {"m13", "application/x-msmediaview"},
- {"m14", "application/x-msmediaview"},
- {"m1v", "video/mpeg"},
- {"m2t", "video/vnd.dlna.mpeg-tts"},
- {"m2ts", "video/vnd.dlna.mpeg-tts"},
- {"m2v", "video/mpeg"},
- {"m3u", "audio/x-mpegurl"},
- {"m3u8", "audio/x-mpegurl"},
- {"m4a", {"audio/m4a", "audio/x-m4a"}},
- {"m4b", "audio/m4b"},
- {"m4p", "audio/m4p"},
- {"m4r", "audio/x-m4r"},
- {"m4v", "video/x-m4v"},
- {"mac", "image/x-macpaint"},
- {"mak", "text/plain"},
- {"man", "application/x-troff-man"},
- {"manifest", "application/x-ms-manifest"},
- {"map", "text/plain"},
- {"master", "application/xml"},
- {"mbox", "application/mbox"},
- {"mda", "application/msaccess"},
- {"mdb", "application/x-msaccess"},
- {"mde", "application/msaccess"},
- {"mdp", "application/octet-stream"},
- {"me", "application/x-troff-me"},
- {"mfp", "application/x-shockwave-flash"},
- {"mht", "message/rfc822"},
- {"mhtml", "message/rfc822"},
- {"mid", "audio/mid"},
- {"midi", "audio/mid"},
- {"mix", "application/octet-stream"},
- {"mk", "text/plain"},
- {"mmf", "application/x-smaf"},
- {"mno", "text/xml"},
- {"mny", "application/x-msmoney"},
- {"mod", "video/mpeg"},
- {"mov", "video/quicktime"},
- {"movie", "video/x-sgi-movie"},
- {"mp2", "video/mpeg"},
- {"mp2v", "video/mpeg"},
- {"mp3", {"audio/mpeg", "audio/mpeg3", "audio/mp3", "audio/x-mpeg-3"}},
- {"mp4", "video/mp4"},
- {"mp4v", "video/mp4"},
- {"mpa", "video/mpeg"},
- {"mpe", "video/mpeg"},
- {"mpeg", "video/mpeg"},
- {"mpf", "application/vnd.ms-mediapackage"},
- {"mpg", "video/mpeg"},
- {"mpp", "application/vnd.ms-project"},
- {"mpv2", "video/mpeg"},
- {"mqv", "video/quicktime"},
- {"ms", "application/x-troff-ms"},
- {"msg", "application/vnd.ms-outlook"},
- {"msi", {"application/x-msi", "application/octet-stream"}},
- {"mso", "application/octet-stream"},
- {"mts", "video/vnd.dlna.mpeg-tts"},
- {"mtx", "application/xml"},
- {"mvb", "application/x-msmediaview"},
- {"mvc", "application/x-miva-compiled"},
- {"mxp", "application/x-mmxp"},
- {"nc", "application/x-netcdf"},
- {"nsc", "video/x-ms-asf"},
- {"nws", "message/rfc822"},
- {"ocx", "application/octet-stream"},
- {"oda", "application/oda"},
- {"odb", "application/vnd.oasis.opendocument.database"},
- {"odc", "application/vnd.oasis.opendocument.chart"},
- {"odf", "application/vnd.oasis.opendocument.formula"},
- {"odg", "application/vnd.oasis.opendocument.graphics"},
- {"odh", "text/plain"},
- {"odi", "application/vnd.oasis.opendocument.image"},
- {"odl", "text/plain"},
- {"odm", "application/vnd.oasis.opendocument.text-master"},
- {"odp", "application/vnd.oasis.opendocument.presentation"},
- {"ods", "application/vnd.oasis.opendocument.spreadsheet"},
- {"odt", "application/vnd.oasis.opendocument.text"},
- {"oga", "audio/ogg"},
- {"ogg", "audio/ogg"},
- {"ogv", "video/ogg"},
- {"ogx", "application/ogg"},
- {"one", "application/onenote"},
- {"onea", "application/onenote"},
- {"onepkg", "application/onenote"},
- {"onetmp", "application/onenote"},
- {"onetoc", "application/onenote"},
- {"onetoc2", "application/onenote"},
- {"opus", "audio/ogg"},
- {"orderedtest", "application/xml"},
- {"osdx", "application/opensearchdescription+xml"},
- {"otf", "application/font-sfnt"},
- {"otg", "application/vnd.oasis.opendocument.graphics-template"},
- {"oth", "application/vnd.oasis.opendocument.text-web"},
- {"otp", "application/vnd.oasis.opendocument.presentation-template"},
- {"ots", "application/vnd.oasis.opendocument.spreadsheet-template"},
- {"ott", "application/vnd.oasis.opendocument.text-template"},
- {"oxt", "application/vnd.openofficeorg.extension"},
- {"p10", "application/pkcs10"},
- {"p12", "application/x-pkcs12"},
- {"p7b", "application/x-pkcs7-certificates"},
- {"p7c", "application/pkcs7-mime"},
- {"p7m", "application/pkcs7-mime", "application/x-pkcs7-mime"},
- {"p7r", "application/x-pkcs7-certreqresp"},
- {"p7s", {"application/pkcs7-signature", "application/x-pkcs7-signature", "text/plain"}},
- {"pbm", "image/x-portable-bitmap"},
- {"pcast", "application/x-podcast"},
- {"pct", "image/pict"},
- {"pcx", "application/octet-stream"},
- {"pcz", "application/octet-stream"},
- {"pdf", "application/pdf"},
- {"pfb", "application/octet-stream"},
- {"pfm", "application/octet-stream"},
- {"pfx", "application/x-pkcs12"},
- {"pgm", "image/x-portable-graymap"},
- {"pic", "image/pict"},
- {"pict", "image/pict"},
- {"pkgdef", "text/plain"},
- {"pkgundef", "text/plain"},
- {"pko", "application/vnd.ms-pki.pko"},
- {"pls", "audio/scpls"},
- {"pma", "application/x-perfmon"},
- {"pmc", "application/x-perfmon"},
- {"pml", "application/x-perfmon"},
- {"pmr", "application/x-perfmon"},
- {"pmw", "application/x-perfmon"},
- {"png", "image/png"},
- {"pnm", "image/x-portable-anymap"},
- {"pnt", "image/x-macpaint"},
- {"pntg", "image/x-macpaint"},
- {"pnz", "image/png"},
- {"pot", "application/vnd.ms-powerpoint"},
- {"potm", "application/vnd.ms-powerpoint.template.macroEnabled.12"},
- {"potx", "application/vnd.openxmlformats-officedocument.presentationml.template"},
- {"ppa", "application/vnd.ms-powerpoint"},
- {"ppam", "application/vnd.ms-powerpoint.addin.macroEnabled.12"},
- {"ppm", "image/x-portable-pixmap"},
- {"pps", "application/vnd.ms-powerpoint"},
- {"ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12"},
- {"ppsx", "application/vnd.openxmlformats-officedocument.presentationml.slideshow"},
- {"ppt", "application/vnd.ms-powerpoint"},
- {"pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12"},
- {"pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"},
- {"prf", "application/pics-rules"},
- {"prm", "application/octet-stream"},
- {"prx", "application/octet-stream"},
- {"ps", "application/postscript"},
- {"psc1", "application/PowerShell"},
- {"psd", "application/octet-stream"},
- {"psess", "application/xml"},
- {"psm", "application/octet-stream"},
- {"psp", "application/octet-stream"},
- {"pst", "application/vnd.ms-outlook"},
- {"pub", "application/x-mspublisher"},
- {"pwz", "application/vnd.ms-powerpoint"},
- {"qht", "text/x-html-insertion"},
- {"qhtm", "text/x-html-insertion"},
- {"qt", "video/quicktime"},
- {"qti", "image/x-quicktime"},
- {"qtif", "image/x-quicktime"},
- {"qtl", "application/x-quicktimeplayer"},
- {"qxd", "application/octet-stream"},
- {"ra", "audio/x-pn-realaudio"},
- {"ram", "audio/x-pn-realaudio"},
- {"rar", {"application/x-rar-compressed", "application/x-rar", "application/rar", "application/octet-stream"}},
- {"ras", "image/x-cmu-raster"},
- {"rat", "application/rat-file"},
- {"rc", "text/plain"},
- {"rc2", "text/plain"},
- {"rct", "text/plain"},
- {"rdlc", "application/xml"},
- {"reg", "text/plain"},
- {"resx", "application/xml"},
- {"rf", "image/vnd.rn-realflash"},
- {"rgb", "image/x-rgb"},
- {"rgs", "text/plain"},
- {"rm", "application/vnd.rn-realmedia"},
- {"rmi", "audio/mid"},
- {"rmp", "application/vnd.rn-rn_music_package"},
- {"roff", "application/x-troff"},
- {"rpm", "audio/x-pn-realaudio-plugin"},
- {"rqy", "text/x-ms-rqy"},
- {"rtf", {"application/rtf","application/msword", "text/richtext", "text/rtf"}},
- {"rtx", "text/richtext"},
- {"rvt", "application/octet-stream" },
- {"ruleset", "application/xml"},
- {"s", "text/plain"},
- {"safariextz", "application/x-safari-safariextz"},
- {"scd", "application/x-msschedule"},
- {"scr", "text/plain"},
- {"sct", "text/scriptlet"},
- {"sd2", "audio/x-sd2"},
- {"sdp", "application/sdp"},
- {"sea", "application/octet-stream"},
- {"searchConnector-ms", "application/windows-search-connector+xml"},
- {"setpay", "application/set-payment-initiation"},
- {"setreg", "application/set-registration-initiation"},
- {"settings", "application/xml"},
- {"sgimb", "application/x-sgimb"},
- {"sgml", "text/sgml"},
- {"sh", "application/x-sh"},
- {"shar", "application/x-shar"},
- {"shtml", "text/html"},
- {"sit", "application/x-stuffit"},
- {"sitemap", "application/xml"},
- {"skin", "application/xml"},
- {"skp", "application/x-koan" },
- {"sldm", "application/vnd.ms-powerpoint.slide.macroEnabled.12"},
- {"sldx", "application/vnd.openxmlformats-officedocument.presentationml.slide"},
- {"slk", "application/vnd.ms-excel"},
- {"sln", "text/plain"},
- {"slupkg-ms", "application/x-ms-license"},
- {"smd", "audio/x-smd"},
- {"smi", "application/octet-stream"},
- {"smx", "audio/x-smd"},
- {"smz", "audio/x-smd"},
- {"snd", "audio/basic"},
- {"snippet", "application/xml"},
- {"snp", "application/octet-stream"},
- {"sol", "text/plain"},
- {"sor", "text/plain"},
- {"spc", "application/x-pkcs7-certificates"},
- {"spl", "application/futuresplash"},
- {"spx", "audio/ogg"},
- {"src", "application/x-wais-source"},
- {"srf", "text/plain"},
- {"SSISDeploymentManifest", "text/xml"},
- {"ssm", "application/streamingmedia"},
- {"sst", "application/vnd.ms-pki.certstore"},
- {"stl", "application/vnd.ms-pki.stl"},
- {"sv4cpio", "application/x-sv4cpio"},
- {"sv4crc", "application/x-sv4crc"},
- {"svc", "application/xml"},
- {"svg", "image/svg+xml"},
- {"swf", "application/x-shockwave-flash"},
- {"step", "application/step"},
- {"stp", "application/step"},
- {"t", "application/x-troff"},
- {"tar", "application/x-tar"},
- {"tcl", "application/x-tcl"},
- {"testrunconfig", "application/xml"},
- {"testsettings", "application/xml"},
- {"tex", "application/x-tex"},
- {"texi", "application/x-texinfo"},
- {"texinfo", "application/x-texinfo"},
- {"tgz", "application/x-compressed"},
- {"thmx", "application/vnd.ms-officetheme"},
- {"thn", "application/octet-stream"},
- {"tif", {"image/tiff", "application/octet-stream"}},
- {"tiff", "image/tiff"},
- {"tlh", "text/plain"},
- {"tli", "text/plain"},
- {"toc", "application/octet-stream"},
- {"tr", "application/x-troff"},
- {"trm", "application/x-msterminal"},
- {"trx", "application/xml"},
- {"ts", "video/vnd.dlna.mpeg-tts"},
- {"tsv", "text/tab-separated-values"},
- {"ttf", "application/font-sfnt"},
- {"tts", "video/vnd.dlna.mpeg-tts"},
- {"txt", "text/plain"},
- {"u32", "application/octet-stream"},
- {"uls", "text/iuls"},
- {"user", "text/plain"},
- {"ustar", "application/x-ustar"},
- {"vb", "text/plain"},
- {"vbdproj", "text/plain"},
- {"vbk", "video/mpeg"},
- {"vbproj", "text/plain"},
- {"vbs", "text/vbscript"},
- {"vcf", {"text/x-vcard", "text/vcard"}},
- {"vcproj", "application/xml"},
- {"vcs", "text/plain"},
- {"vcxproj", "application/xml"},
- {"vddproj", "text/plain"},
- {"vdp", "text/plain"},
- {"vdproj", "text/plain"},
- {"vdx", "application/vnd.ms-visio.viewer"},
- {"vml", "text/xml"},
- {"vscontent", "application/xml"},
- {"vsct", "text/xml"},
- {"vsd", "application/vnd.visio"},
- {"vsi", "application/ms-vsi"},
- {"vsix", "application/vsix"},
- {"vsixlangpack", "text/xml"},
- {"vsixmanifest", "text/xml"},
- {"vsmdi", "application/xml"},
- {"vspscc", "text/plain"},
- {"vss", "application/vnd.visio"},
- {"vsscc", "text/plain"},
- {"vssettings", "text/xml"},
- {"vssscc", "text/plain"},
- {"vst", "application/vnd.visio"},
- {"vstemplate", "text/xml"},
- {"vsto", "application/x-ms-vsto"},
- {"vsw", "application/vnd.visio"},
- {"vsx", "application/vnd.visio"},
- {"vtx", "application/vnd.visio"},
- {"wav", {"audio/wav", "audio/vnd.wave", "audio/x-wav"}},
- {"wave", "audio/wav"},
- {"wax", "audio/x-ms-wax"},
- {"wbk", "application/msword"},
- {"wbmp", "image/vnd.wap.wbmp"},
- {"wcm", "application/vnd.ms-works"},
- {"wdb", "application/vnd.ms-works"},
- {"wdp", "image/vnd.ms-photo"},
- {"webarchive", "application/x-safari-webarchive"},
- {"webm", "video/webm"},
- {"webp", "image/webp"},
- {"webtest", "application/xml"},
- {"wiq", "application/xml"},
- {"wiz", "application/msword"},
- {"wks", "application/vnd.ms-works"},
- {"WLMP", "application/wlmoviemaker"},
- {"wlpginstall", "application/x-wlpg-detect"},
- {"wlpginstall3", "application/x-wlpg3-detect"},
- {"wm", "video/x-ms-wm"},
- {"wma", "audio/x-ms-wma"},
- {"wmd", "application/x-ms-wmd"},
- {"wmf", {"application/x-msmetafile", "image/wmf", "image/x-wmf"}},
- {"wml", "text/vnd.wap.wml"},
- {"wmlc", "application/vnd.wap.wmlc"},
- {"wmls", "text/vnd.wap.wmlscript"},
- {"wmlsc", "application/vnd.wap.wmlscriptc"},
- {"wmp", "video/x-ms-wmp"},
- {"wmv", "video/x-ms-wmv"},
- {"wmx", "video/x-ms-wmx"},
- {"wmz", "application/x-ms-wmz"},
- {"woff", "application/font-woff"},
- {"wpl", "application/vnd.ms-wpl"},
- {"wps", "application/vnd.ms-works"},
- {"wri", "application/x-mswrite"},
- {"wrl", "x-world/x-vrml"},
- {"wrz", "x-world/x-vrml"},
- {"wsc", "text/scriptlet"},
- {"wsdl", "text/xml"},
- {"wvx", "video/x-ms-wvx"},
- {"x", "application/directx"},
- {"xaf", "x-world/x-vrml"},
- {"xaml", "application/xaml+xml"},
- {"xap", "application/x-silverlight-app"},
- {"xbap", "application/x-ms-xbap"},
- {"xbm", "image/x-xbitmap"},
- {"xdr", "text/plain"},
- {"xht", "application/xhtml+xml"},
- {"xhtml", "application/xhtml+xml"},
- {"xla", "application/vnd.ms-excel"},
- {"xlam", "application/vnd.ms-excel.addin.macroEnabled.12"},
- {"xlc", "application/vnd.ms-excel"},
- {"xld", "application/vnd.ms-excel"},
- {"xlk", "application/vnd.ms-excel"},
- {"xll", "application/vnd.ms-excel"},
- {"xlm", "application/vnd.ms-excel"},
- {"xls", {
- "application/excel",
- "application/vnd.ms-excel",
- "application/vnd.ms-office",
- "application/x-excel",
- "application/octet-stream"
- }},
- {"xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12"},
- {"xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12"},
- {"xlsx", {
- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
- "application/vnd.ms-excel.12",
- "application/octet-stream"
- }},
- {"xlt", "application/vnd.ms-excel"},
- {"xltm", "application/vnd.ms-excel.template.macroEnabled.12"},
- {"xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template"},
- {"xlw", "application/vnd.ms-excel"},
- {"xml", {"application/xml", "text/xml", "application/octet-stream"}},
- {"xmp", "application/octet-stream" },
- {"xmta", "application/xml"},
- {"xof", "x-world/x-vrml"},
- {"XOML", "text/plain"},
- {"xpm", "image/x-xpixmap"},
- {"xps", "application/vnd.ms-xpsdocument"},
- {"xrm-ms", "text/xml"},
- {"xsc", "application/xml"},
- {"xsd", "text/xml"},
- {"xsf", "text/xml"},
- {"xsl", "text/xml"},
- {"xslt", "text/xml"},
- {"xsn", "application/octet-stream"},
- {"xss", "application/xml"},
- {"xspf", "application/xspf+xml"},
- {"xtp", "application/octet-stream"},
- {"xwd", "image/x-xwindowdump"},
- {"z", "application/x-compress"},
- {"zip", {
- "application/zip",
- "application/x-zip-compressed",
- "application/octet-stream"
- }},
- {"zlib", "application/zlib"},
+ { "323", "text/h323" },
+ { "3g2", "video/3gpp2" },
+ { "3gp", "video/3gpp" },
+ { "3gp2", "video/3gpp2" },
+ { "3gpp", "video/3gpp" },
+ { "7z", { "application/x-7z-compressed", "application/7z" } },
+ { "aa", "audio/audible" },
+ { "AAC", "audio/aac" },
+ { "aaf", "application/octet-stream" },
+ { "aax", "audio/vnd.audible.aax" },
+ { "ac3", "audio/ac3" },
+ { "aca", "application/octet-stream" },
+ { "accda", "application/msaccess.addin" },
+ { "accdb", "application/msaccess" },
+ { "accdc", "application/msaccess.cab" },
+ { "accde", "application/msaccess" },
+ { "accdr", "application/msaccess.runtime" },
+ { "accdt", "application/msaccess" },
+ { "accdw", "application/msaccess.webapplication" },
+ { "accft", "application/msaccess.ftemplate" },
+ { "acx", "application/internet-property-stream" },
+ { "AddIn", "text/xml" },
+ { "ade", "application/msaccess" },
+ { "adobebridge", "application/x-bridge-url" },
+ { "adp", "application/msaccess" },
+ { "ADT", "audio/vnd.dlna.adts" },
+ { "ADTS", "audio/aac" },
+ { "afm", "application/octet-stream" },
+ { "ai", "application/postscript" },
+ { "aif", "audio/aiff" },
+ { "aifc", "audio/aiff" },
+ { "aiff", "audio/aiff" },
+ { "air", "application/vnd.adobe.air-application-installer-package+zip" },
+ { "amc", "application/mpeg" },
+ { "anx", "application/annodex" },
+ { "apk", "application/vnd.android.package-archive" },
+ { "application", "application/x-ms-application" },
+ { "art", "image/x-jg" },
+ { "asa", "application/xml" },
+ { "asax", "application/xml" },
+ { "ascx", "application/xml" },
+ { "asd", "application/octet-stream" },
+ { "asf", "video/x-ms-asf" },
+ { "ashx", "application/xml" },
+ { "asi", "application/octet-stream" },
+ { "asm", "text/plain" },
+ { "asmx", "application/xml" },
+ { "aspx", "application/xml" },
+ { "asr", "video/x-ms-asf" },
+ { "asx", "video/x-ms-asf" },
+ { "atom", "application/atom+xml" },
+ { "au", "audio/basic" },
+ { "avi", "video/x-msvideo" },
+ { "axa", "audio/annodex" },
+ { "axs", "application/olescript" },
+ { "axv", "video/annodex" },
+ { "bas", "text/plain" },
+ { "bcpio", "application/x-bcpio" },
+ { "bin", "application/octet-stream" },
+ { "bmp", { "image/bmp", "image/x-ms-bmp" } },
+ { "c", "text/plain" },
+ { "cab", "application/octet-stream" },
+ { "caf", "audio/x-caf" },
+ { "calx", "application/vnd.ms-office.calx" },
+ { "cat", "application/vnd.ms-pki.seccat" },
+ { "cc", "text/plain" },
+ { "cd", "text/plain" },
+ { "cdda", "audio/aiff" },
+ { "cdf", "application/x-cdf" },
+ { "cer", "application/x-x509-ca-cert" },
+ { "cfg", "text/plain" },
+ { "chm", "application/octet-stream" },
+ { "class", "application/x-java-applet" },
+ { "clp", "application/x-msclip" },
+ { "cmd", "text/plain" },
+ { "cmx", "image/x-cmx" },
+ { "cnf", "text/plain" },
+ { "cod", "image/cis-cod" },
+ { "config", "application/xml" },
+ { "contact", "text/x-ms-contact" },
+ { "coverage", "application/xml" },
+ { "cpio", "application/x-cpio" },
+ { "cpp", "text/plain" },
+ { "crd", "application/x-mscardfile" },
+ { "crl", "application/pkix-crl" },
+ { "crt", "application/x-x509-ca-cert" },
+ { "cs", "text/plain" },
+ { "csdproj", "text/plain" },
+ { "csh", "application/x-csh" },
+ { "csproj", "text/plain" },
+ { "css", "text/css" },
+ { "csv", { "application/vnd.ms-excel", "text/csv", "text/plain" } },
+ { "cur", "application/octet-stream" },
+ { "cxx", "text/plain" },
+ { "dat", { "application/octet-stream", "application/ms-tnef" } },
+ { "datasource", "application/xml" },
+ { "dbproj", "text/plain" },
+ { "dcr", "application/x-director" },
+ { "def", "text/plain" },
+ { "deploy", "application/octet-stream" },
+ { "der", "application/x-x509-ca-cert" },
+ { "dgml", "application/xml" },
+ { "dib", "image/bmp" },
+ { "dif", "video/x-dv" },
+ { "dir", "application/x-director" },
+ { "disco", "text/xml" },
+ { "divx", "video/divx" },
+ { "dll", "application/x-msdownload" },
+ { "dll.config", "text/xml" },
+ { "dlm", "text/dlm" },
+ { "doc", "application/msword" },
+ { "docm", "application/vnd.ms-word.document.macroEnabled.12" },
+ { "docx", {
+ "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+ "application/msword",
+ "application/vnd.ms-word.document.12",
+ "application/octet-stream",
+ } },
+ { "dot", "application/msword" },
+ { "dotm", "application/vnd.ms-word.template.macroEnabled.12" },
+ { "dotx", "application/vnd.openxmlformats-officedocument.wordprocessingml.template" },
+ { "dsp", "application/octet-stream" },
+ { "dsw", "text/plain" },
+ { "dtd", "text/xml" },
+ { "dtsConfig", "text/xml" },
+ { "dv", "video/x-dv" },
+ { "dvi", "application/x-dvi" },
+ { "dwf", "drawing/x-dwf" },
+ { "dwg", { "application/acad", "image/vnd.dwg" } },
+ { "dwp", "application/octet-stream" },
+ { "dxf", "application/x-dxf" },
+ { "dxr", "application/x-director" },
+ { "eml", "message/rfc822" },
+ { "emz", "application/octet-stream" },
+ { "eot", "application/vnd.ms-fontobject" },
+ { "eps", "application/postscript" },
+ { "etl", "application/etl" },
+ { "etx", "text/x-setext" },
+ { "evy", "application/envoy" },
+ { "exe", {
+ "application/x-dosexec",
+ "application/x-msdownload",
+ "application/x-executable",
+ } },
+ { "exe.config", "text/xml" },
+ { "fdf", "application/vnd.fdf" },
+ { "fif", "application/fractals" },
+ { "filters", "application/xml" },
+ { "fla", "application/octet-stream" },
+ { "flac", "audio/flac" },
+ { "flr", "x-world/x-vrml" },
+ { "flv", "video/x-flv" },
+ { "fsscript", "application/fsharp-script" },
+ { "fsx", "application/fsharp-script" },
+ { "generictest", "application/xml" },
+ { "gif", "image/gif" },
+ { "gpx", "application/gpx+xml" },
+ { "group", "text/x-ms-group" },
+ { "gsm", "audio/x-gsm" },
+ { "gtar", "application/x-gtar" },
+ { "gz", { "application/gzip", "application/x-gzip", "application/tlsrpt+gzip" } },
+ { "h", "text/plain" },
+ { "hdf", "application/x-hdf" },
+ { "hdml", "text/x-hdml" },
+ { "hhc", "application/x-oleobject" },
+ { "hhk", "application/octet-stream" },
+ { "hhp", "application/octet-stream" },
+ { "hlp", "application/winhlp" },
+ { "hpp", "text/plain" },
+ { "hqx", "application/mac-binhex40" },
+ { "hta", "application/hta" },
+ { "htc", "text/x-component" },
+ { "htm", "text/html" },
+ { "html", "text/html" },
+ { "htt", "text/webviewhtml" },
+ { "hxa", "application/xml" },
+ { "hxc", "application/xml" },
+ { "hxd", "application/octet-stream" },
+ { "hxe", "application/xml" },
+ { "hxf", "application/xml" },
+ { "hxh", "application/octet-stream" },
+ { "hxi", "application/octet-stream" },
+ { "hxk", "application/xml" },
+ { "hxq", "application/octet-stream" },
+ { "hxr", "application/octet-stream" },
+ { "hxs", "application/octet-stream" },
+ { "hxt", "text/html" },
+ { "hxv", "application/xml" },
+ { "hxw", "application/octet-stream" },
+ { "hxx", "text/plain" },
+ { "i", "text/plain" },
+ { "ico", "image/x-icon" },
+ { "ics", { "text/calendar", "application/ics", "application/octet-stream" } },
+ { "idl", "text/plain" },
+ { "ief", "image/ief" },
+ { "iii", "application/x-iphone" },
+ { "inc", "text/plain" },
+ { "inf", "application/octet-stream" },
+ { "ini", "text/plain" },
+ { "inl", "text/plain" },
+ { "ins", "application/x-internet-signup" },
+ { "ipa", "application/x-itunes-ipa" },
+ { "ipg", "application/x-itunes-ipg" },
+ { "ipproj", "text/plain" },
+ { "ipsw", "application/x-itunes-ipsw" },
+ { "iqy", "text/x-ms-iqy" },
+ { "isp", "application/x-internet-signup" },
+ { "ite", "application/x-itunes-ite" },
+ { "itlp", "application/x-itunes-itlp" },
+ { "itms", "application/x-itunes-itms" },
+ { "itpc", "application/x-itunes-itpc" },
+ { "IVF", "video/x-ivf" },
+ { "jar", "application/java-archive" },
+ { "java", "application/octet-stream" },
+ { "jck", "application/liquidmotion" },
+ { "jcz", "application/liquidmotion" },
+ { "jfif", { "image/jpeg", "image/pjpeg" } },
+ { "jnlp", "application/x-java-jnlp-file" },
+ { "jpb", "application/octet-stream" },
+ { "jpe", { "image/jpeg", "image/pjpeg" } },
+ { "jpeg", { "image/jpeg", "image/pjpeg" } },
+ { "jpg", { "image/jpeg", "image/pjpeg" } },
+ { "js", "application/javascript" },
+ { "json", "application/json" },
+ { "jsx", "text/jscript" },
+ { "jsxbin", "text/plain" },
+ { "latex", "application/x-latex" },
+ { "library-ms", "application/windows-library+xml" },
+ { "lit", "application/x-ms-reader" },
+ { "loadtest", "application/xml" },
+ { "lpk", "application/octet-stream" },
+ { "lsf", "video/x-la-asf" },
+ { "lst", "text/plain" },
+ { "lsx", "video/x-la-asf" },
+ { "lzh", "application/octet-stream" },
+ { "m13", "application/x-msmediaview" },
+ { "m14", "application/x-msmediaview" },
+ { "m1v", "video/mpeg" },
+ { "m2t", "video/vnd.dlna.mpeg-tts" },
+ { "m2ts", "video/vnd.dlna.mpeg-tts" },
+ { "m2v", "video/mpeg" },
+ { "m3u", "audio/x-mpegurl" },
+ { "m3u8", "audio/x-mpegurl" },
+ { "m4a", { "audio/m4a", "audio/x-m4a" } },
+ { "m4b", "audio/m4b" },
+ { "m4p", "audio/m4p" },
+ { "m4r", "audio/x-m4r" },
+ { "m4v", "video/x-m4v" },
+ { "mac", "image/x-macpaint" },
+ { "mak", "text/plain" },
+ { "man", "application/x-troff-man" },
+ { "manifest", "application/x-ms-manifest" },
+ { "map", "text/plain" },
+ { "master", "application/xml" },
+ { "mbox", "application/mbox" },
+ { "mda", "application/msaccess" },
+ { "mdb", "application/x-msaccess" },
+ { "mde", "application/msaccess" },
+ { "mdp", "application/octet-stream" },
+ { "me", "application/x-troff-me" },
+ { "mfp", "application/x-shockwave-flash" },
+ { "mht", "message/rfc822" },
+ { "mhtml", "message/rfc822" },
+ { "mid", "audio/mid" },
+ { "midi", "audio/mid" },
+ { "mix", "application/octet-stream" },
+ { "mk", "text/plain" },
+ { "mmf", "application/x-smaf" },
+ { "mno", "text/xml" },
+ { "mny", "application/x-msmoney" },
+ { "mod", "video/mpeg" },
+ { "mov", "video/quicktime" },
+ { "movie", "video/x-sgi-movie" },
+ { "mp2", "video/mpeg" },
+ { "mp2v", "video/mpeg" },
+ { "mp3", { "audio/mpeg", "audio/mpeg3", "audio/mp3", "audio/x-mpeg-3" } },
+ { "mp4", "video/mp4" },
+ { "mp4v", "video/mp4" },
+ { "mpa", "video/mpeg" },
+ { "mpe", "video/mpeg" },
+ { "mpeg", "video/mpeg" },
+ { "mpf", "application/vnd.ms-mediapackage" },
+ { "mpg", "video/mpeg" },
+ { "mpp", "application/vnd.ms-project" },
+ { "mpv2", "video/mpeg" },
+ { "mqv", "video/quicktime" },
+ { "ms", "application/x-troff-ms" },
+ { "msg", "application/vnd.ms-outlook" },
+ { "msi", { "application/x-msi", "application/octet-stream" } },
+ { "mso", "application/octet-stream" },
+ { "mts", "video/vnd.dlna.mpeg-tts" },
+ { "mtx", "application/xml" },
+ { "mvb", "application/x-msmediaview" },
+ { "mvc", "application/x-miva-compiled" },
+ { "mxp", "application/x-mmxp" },
+ { "nc", "application/x-netcdf" },
+ { "nsc", "video/x-ms-asf" },
+ { "nws", "message/rfc822" },
+ { "ocx", "application/octet-stream" },
+ { "oda", "application/oda" },
+ { "odb", "application/vnd.oasis.opendocument.database" },
+ { "odc", "application/vnd.oasis.opendocument.chart" },
+ { "odf", "application/vnd.oasis.opendocument.formula" },
+ { "odg", "application/vnd.oasis.opendocument.graphics" },
+ { "odh", "text/plain" },
+ { "odi", "application/vnd.oasis.opendocument.image" },
+ { "odl", "text/plain" },
+ { "odm", "application/vnd.oasis.opendocument.text-master" },
+ { "odp", "application/vnd.oasis.opendocument.presentation" },
+ { "ods", "application/vnd.oasis.opendocument.spreadsheet" },
+ { "odt", "application/vnd.oasis.opendocument.text" },
+ { "oga", "audio/ogg" },
+ { "ogg", "audio/ogg" },
+ { "ogv", "video/ogg" },
+ { "ogx", "application/ogg" },
+ { "one", "application/onenote" },
+ { "onea", "application/onenote" },
+ { "onepkg", "application/onenote" },
+ { "onetmp", "application/onenote" },
+ { "onetoc", "application/onenote" },
+ { "onetoc2", "application/onenote" },
+ { "opus", "audio/ogg" },
+ { "orderedtest", "application/xml" },
+ { "osdx", "application/opensearchdescription+xml" },
+ { "otf", "application/font-sfnt" },
+ { "otg", "application/vnd.oasis.opendocument.graphics-template" },
+ { "oth", "application/vnd.oasis.opendocument.text-web" },
+ { "otp", "application/vnd.oasis.opendocument.presentation-template" },
+ { "ots", "application/vnd.oasis.opendocument.spreadsheet-template" },
+ { "ott", "application/vnd.oasis.opendocument.text-template" },
+ { "oxt", "application/vnd.openofficeorg.extension" },
+ { "p10", "application/pkcs10" },
+ { "p12", "application/x-pkcs12" },
+ { "p7b", "application/x-pkcs7-certificates" },
+ { "p7c", "application/pkcs7-mime" },
+ { "p7m", "application/pkcs7-mime", "application/x-pkcs7-mime" },
+ { "p7r", "application/x-pkcs7-certreqresp" },
+ { "p7s", { "application/pkcs7-signature", "application/x-pkcs7-signature", "text/plain" } },
+ { "pbm", "image/x-portable-bitmap" },
+ { "pcast", "application/x-podcast" },
+ { "pct", "image/pict" },
+ { "pcx", "application/octet-stream" },
+ { "pcz", "application/octet-stream" },
+ { "pdf", "application/pdf" },
+ { "pfb", "application/octet-stream" },
+ { "pfm", "application/octet-stream" },
+ { "pfx", "application/x-pkcs12" },
+ { "pgm", "image/x-portable-graymap" },
+ { "pic", "image/pict" },
+ { "pict", "image/pict" },
+ { "pkgdef", "text/plain" },
+ { "pkgundef", "text/plain" },
+ { "pko", "application/vnd.ms-pki.pko" },
+ { "pls", "audio/scpls" },
+ { "pma", "application/x-perfmon" },
+ { "pmc", "application/x-perfmon" },
+ { "pml", "application/x-perfmon" },
+ { "pmr", "application/x-perfmon" },
+ { "pmw", "application/x-perfmon" },
+ { "png", "image/png" },
+ { "pnm", "image/x-portable-anymap" },
+ { "pnt", "image/x-macpaint" },
+ { "pntg", "image/x-macpaint" },
+ { "pnz", "image/png" },
+ { "pot", "application/vnd.ms-powerpoint" },
+ { "potm", "application/vnd.ms-powerpoint.template.macroEnabled.12" },
+ { "potx", "application/vnd.openxmlformats-officedocument.presentationml.template" },
+ { "ppa", "application/vnd.ms-powerpoint" },
+ { "ppam", "application/vnd.ms-powerpoint.addin.macroEnabled.12" },
+ { "ppm", "image/x-portable-pixmap" },
+ { "pps", "application/vnd.ms-powerpoint" },
+ { "ppsm", "application/vnd.ms-powerpoint.slideshow.macroEnabled.12" },
+ { "ppsx", "application/vnd.openxmlformats-officedocument.presentationml.slideshow" },
+ { "ppt", "application/vnd.ms-powerpoint" },
+ { "pptm", "application/vnd.ms-powerpoint.presentation.macroEnabled.12" },
+ { "pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation" },
+ { "prf", "application/pics-rules" },
+ { "prm", "application/octet-stream" },
+ { "prx", "application/octet-stream" },
+ { "ps", "application/postscript" },
+ { "psc1", "application/PowerShell" },
+ { "psd", "application/octet-stream" },
+ { "psess", "application/xml" },
+ { "psm", "application/octet-stream" },
+ { "psp", "application/octet-stream" },
+ { "pst", "application/vnd.ms-outlook" },
+ { "pub", "application/x-mspublisher" },
+ { "pwz", "application/vnd.ms-powerpoint" },
+ { "qht", "text/x-html-insertion" },
+ { "qhtm", "text/x-html-insertion" },
+ { "qt", "video/quicktime" },
+ { "qti", "image/x-quicktime" },
+ { "qtif", "image/x-quicktime" },
+ { "qtl", "application/x-quicktimeplayer" },
+ { "qxd", "application/octet-stream" },
+ { "ra", "audio/x-pn-realaudio" },
+ { "ram", "audio/x-pn-realaudio" },
+ { "rar", { "application/x-rar-compressed", "application/x-rar", "application/rar", "application/octet-stream" } },
+ { "ras", "image/x-cmu-raster" },
+ { "rat", "application/rat-file" },
+ { "rc", "text/plain" },
+ { "rc2", "text/plain" },
+ { "rct", "text/plain" },
+ { "rdlc", "application/xml" },
+ { "reg", "text/plain" },
+ { "resx", "application/xml" },
+ { "rf", "image/vnd.rn-realflash" },
+ { "rgb", "image/x-rgb" },
+ { "rgs", "text/plain" },
+ { "rm", "application/vnd.rn-realmedia" },
+ { "rmi", "audio/mid" },
+ { "rmp", "application/vnd.rn-rn_music_package" },
+ { "roff", "application/x-troff" },
+ { "rpm", "audio/x-pn-realaudio-plugin" },
+ { "rqy", "text/x-ms-rqy" },
+ { "rtf", { "application/rtf", "application/msword", "text/richtext", "text/rtf" } },
+ { "rtx", "text/richtext" },
+ { "rvt", "application/octet-stream" },
+ { "ruleset", "application/xml" },
+ { "s", "text/plain" },
+ { "safariextz", "application/x-safari-safariextz" },
+ { "scd", "application/x-msschedule" },
+ { "scr", "text/plain" },
+ { "sct", "text/scriptlet" },
+ { "sd2", "audio/x-sd2" },
+ { "sdp", "application/sdp" },
+ { "sea", "application/octet-stream" },
+ { "searchConnector-ms", "application/windows-search-connector+xml" },
+ { "setpay", "application/set-payment-initiation" },
+ { "setreg", "application/set-registration-initiation" },
+ { "settings", "application/xml" },
+ { "sgimb", "application/x-sgimb" },
+ { "sgml", "text/sgml" },
+ { "sh", "application/x-sh" },
+ { "shar", "application/x-shar" },
+ { "shtml", "text/html" },
+ { "sit", "application/x-stuffit" },
+ { "sitemap", "application/xml" },
+ { "skin", "application/xml" },
+ { "skp", "application/x-koan" },
+ { "sldm", "application/vnd.ms-powerpoint.slide.macroEnabled.12" },
+ { "sldx", "application/vnd.openxmlformats-officedocument.presentationml.slide" },
+ { "slk", "application/vnd.ms-excel" },
+ { "sln", "text/plain" },
+ { "slupkg-ms", "application/x-ms-license" },
+ { "smd", "audio/x-smd" },
+ { "smi", "application/octet-stream" },
+ { "smx", "audio/x-smd" },
+ { "smz", "audio/x-smd" },
+ { "snd", "audio/basic" },
+ { "snippet", "application/xml" },
+ { "snp", "application/octet-stream" },
+ { "sol", "text/plain" },
+ { "sor", "text/plain" },
+ { "spc", "application/x-pkcs7-certificates" },
+ { "spl", "application/futuresplash" },
+ { "spx", "audio/ogg" },
+ { "src", "application/x-wais-source" },
+ { "srf", "text/plain" },
+ { "SSISDeploymentManifest", "text/xml" },
+ { "ssm", "application/streamingmedia" },
+ { "sst", "application/vnd.ms-pki.certstore" },
+ { "stl", "application/vnd.ms-pki.stl" },
+ { "sv4cpio", "application/x-sv4cpio" },
+ { "sv4crc", "application/x-sv4crc" },
+ { "svc", "application/xml" },
+ { "svg", "image/svg+xml" },
+ { "swf", "application/x-shockwave-flash" },
+ { "step", "application/step" },
+ { "stp", "application/step" },
+ { "t", "application/x-troff" },
+ { "tar", "application/x-tar" },
+ { "tcl", "application/x-tcl" },
+ { "testrunconfig", "application/xml" },
+ { "testsettings", "application/xml" },
+ { "tex", "application/x-tex" },
+ { "texi", "application/x-texinfo" },
+ { "texinfo", "application/x-texinfo" },
+ { "tgz", "application/x-compressed" },
+ { "thmx", "application/vnd.ms-officetheme" },
+ { "thn", "application/octet-stream" },
+ { "tif", { "image/tiff", "application/octet-stream" } },
+ { "tiff", "image/tiff" },
+ { "tlh", "text/plain" },
+ { "tli", "text/plain" },
+ { "toc", "application/octet-stream" },
+ { "tr", "application/x-troff" },
+ { "trm", "application/x-msterminal" },
+ { "trx", "application/xml" },
+ { "ts", "video/vnd.dlna.mpeg-tts" },
+ { "tsv", "text/tab-separated-values" },
+ { "ttf", "application/font-sfnt" },
+ { "tts", "video/vnd.dlna.mpeg-tts" },
+ { "txt", "text/plain" },
+ { "u32", "application/octet-stream" },
+ { "uls", "text/iuls" },
+ { "user", "text/plain" },
+ { "ustar", "application/x-ustar" },
+ { "vb", "text/plain" },
+ { "vbdproj", "text/plain" },
+ { "vbk", "video/mpeg" },
+ { "vbproj", "text/plain" },
+ { "vbs", "text/vbscript" },
+ { "vcf", { "text/x-vcard", "text/vcard" } },
+ { "vcproj", "application/xml" },
+ { "vcs", "text/plain" },
+ { "vcxproj", "application/xml" },
+ { "vddproj", "text/plain" },
+ { "vdp", "text/plain" },
+ { "vdproj", "text/plain" },
+ { "vdx", "application/vnd.ms-visio.viewer" },
+ { "vml", "text/xml" },
+ { "vscontent", "application/xml" },
+ { "vsct", "text/xml" },
+ { "vsd", "application/vnd.visio" },
+ { "vsi", "application/ms-vsi" },
+ { "vsix", "application/vsix" },
+ { "vsixlangpack", "text/xml" },
+ { "vsixmanifest", "text/xml" },
+ { "vsmdi", "application/xml" },
+ { "vspscc", "text/plain" },
+ { "vss", "application/vnd.visio" },
+ { "vsscc", "text/plain" },
+ { "vssettings", "text/xml" },
+ { "vssscc", "text/plain" },
+ { "vst", "application/vnd.visio" },
+ { "vstemplate", "text/xml" },
+ { "vsto", "application/x-ms-vsto" },
+ { "vsw", "application/vnd.visio" },
+ { "vsx", "application/vnd.visio" },
+ { "vtx", "application/vnd.visio" },
+ { "wav", { "audio/wav", "audio/vnd.wave", "audio/x-wav" } },
+ { "wave", "audio/wav" },
+ { "wax", "audio/x-ms-wax" },
+ { "wbk", "application/msword" },
+ { "wbmp", "image/vnd.wap.wbmp" },
+ { "wcm", "application/vnd.ms-works" },
+ { "wdb", "application/vnd.ms-works" },
+ { "wdp", "image/vnd.ms-photo" },
+ { "webarchive", "application/x-safari-webarchive" },
+ { "webm", "video/webm" },
+ { "webp", "image/webp" },
+ { "webtest", "application/xml" },
+ { "wiq", "application/xml" },
+ { "wiz", "application/msword" },
+ { "wks", "application/vnd.ms-works" },
+ { "WLMP", "application/wlmoviemaker" },
+ { "wlpginstall", "application/x-wlpg-detect" },
+ { "wlpginstall3", "application/x-wlpg3-detect" },
+ { "wm", "video/x-ms-wm" },
+ { "wma", "audio/x-ms-wma" },
+ { "wmd", "application/x-ms-wmd" },
+ { "wmf", { "application/x-msmetafile", "image/wmf", "image/x-wmf" } },
+ { "wml", "text/vnd.wap.wml" },
+ { "wmlc", "application/vnd.wap.wmlc" },
+ { "wmls", "text/vnd.wap.wmlscript" },
+ { "wmlsc", "application/vnd.wap.wmlscriptc" },
+ { "wmp", "video/x-ms-wmp" },
+ { "wmv", "video/x-ms-wmv" },
+ { "wmx", "video/x-ms-wmx" },
+ { "wmz", "application/x-ms-wmz" },
+ { "woff", "application/font-woff" },
+ { "wpl", "application/vnd.ms-wpl" },
+ { "wps", "application/vnd.ms-works" },
+ { "wri", "application/x-mswrite" },
+ { "wrl", "x-world/x-vrml" },
+ { "wrz", "x-world/x-vrml" },
+ { "wsc", "text/scriptlet" },
+ { "wsdl", "text/xml" },
+ { "wvx", "video/x-ms-wvx" },
+ { "x", "application/directx" },
+ { "xaf", "x-world/x-vrml" },
+ { "xaml", "application/xaml+xml" },
+ { "xap", "application/x-silverlight-app" },
+ { "xbap", "application/x-ms-xbap" },
+ { "xbm", "image/x-xbitmap" },
+ { "xdr", "text/plain" },
+ { "xht", "application/xhtml+xml" },
+ { "xhtml", "application/xhtml+xml" },
+ { "xla", "application/vnd.ms-excel" },
+ { "xlam", "application/vnd.ms-excel.addin.macroEnabled.12" },
+ { "xlc", "application/vnd.ms-excel" },
+ { "xld", "application/vnd.ms-excel" },
+ { "xlk", "application/vnd.ms-excel" },
+ { "xll", "application/vnd.ms-excel" },
+ { "xlm", "application/vnd.ms-excel" },
+ { "xls", {
+ "application/excel",
+ "application/vnd.ms-excel",
+ "application/vnd.ms-office",
+ "application/x-excel",
+ "application/octet-stream"
+ } },
+ { "xlsb", "application/vnd.ms-excel.sheet.binary.macroEnabled.12" },
+ { "xlsm", "application/vnd.ms-excel.sheet.macroEnabled.12" },
+ { "xlsx", {
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ "application/vnd.ms-excel.12",
+ "application/octet-stream"
+ } },
+ { "xlt", "application/vnd.ms-excel" },
+ { "xltm", "application/vnd.ms-excel.template.macroEnabled.12" },
+ { "xltx", "application/vnd.openxmlformats-officedocument.spreadsheetml.template" },
+ { "xlw", "application/vnd.ms-excel" },
+ { "xml", { "application/xml", "text/xml", "application/octet-stream" } },
+ { "xmp", "application/octet-stream" },
+ { "xmta", "application/xml" },
+ { "xof", "x-world/x-vrml" },
+ { "XOML", "text/plain" },
+ { "xpm", "image/x-xpixmap" },
+ { "xps", "application/vnd.ms-xpsdocument" },
+ { "xrm-ms", "text/xml" },
+ { "xsc", "application/xml" },
+ { "xsd", "text/xml" },
+ { "xsf", "text/xml" },
+ { "xsl", "text/xml" },
+ { "xslt", "text/xml" },
+ { "xsn", "application/octet-stream" },
+ { "xss", "application/xml" },
+ { "xspf", "application/xspf+xml" },
+ { "xtp", "application/octet-stream" },
+ { "xwd", "image/x-xwindowdump" },
+ { "z", "application/x-compress" },
+ { "zip", {
+ "application/zip",
+ "application/x-zip-compressed",
+ "application/octet-stream"
+ } },
+ { "zlib", "application/zlib" },
}
-- Used to match extension by content type
exports.reversed_extensions_map = {
- ["text/html"] = "html",
- ["text/css"] = "css",
- ["text/xml"] = "xml",
- ["image/gif"] = "gif",
- ["image/jpeg"] = "jpeg",
- ["application/javascript"] = "js",
- ["application/atom+xml"] = "atom",
- ["application/rss+xml"] = "rss",
- ["application/csv"] = "csv",
- ["text/mathml"] = "mml",
- ["text/plain"] = "txt",
- ["text/vnd.sun.j2me.app-descriptor"] = "jad",
- ["text/vnd.wap.wml"] = "wml",
- ["text/x-component"] = "htc",
- ["image/png"] = "png",
- ["image/svg+xml"] = "svg",
- ["image/tiff"] = "tiff",
- ["image/vnd.wap.wbmp"] = "wbmp",
- ["image/webp"] = "webp",
- ["image/x-icon"] = "ico",
- ["image/x-jng"] = "jng",
- ["image/x-ms-bmp"] = "bmp",
- ["font/woff"] = "woff",
- ["font/woff2"] = "woff2",
- ["application/java-archive"] = "jar",
- ["application/json"] = "json",
- ["application/mac-binhex40"] = "hqx",
- ["application/msword"] = "doc",
- ["application/pdf"] = "pdf",
- ["application/postscript"] = "ps",
- ["application/rtf"] = "rtf",
- ["application/vnd.apple.mpegurl"] = "m3u8",
- ["application/vnd.google-earth.kml+xml"] = "kml",
- ["application/vnd.google-earth.kmz"] = "kmz",
- ["application/vnd.ms-excel"] = "xls",
- ["application/vnd.ms-fontobject"] = "eot",
- ["application/vnd.ms-powerpoint"] = "ppt",
- ["application/vnd.oasis.opendocument.graphics"] = "odg",
- ["application/vnd.oasis.opendocument.presentation"] = "odp",
- ["application/vnd.oasis.opendocument.spreadsheet"] = "ods",
- ["application/vnd.oasis.opendocument.text"] = "odt",
- ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "pptx",
- ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "xlsx",
- ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "docx",
- ["application/x-7z-compressed"] = "7z",
- ["application/x-cocoa"] = "cco",
- ["application/x-java-archive-diff"] = "jardiff",
- ["application/x-java-jnlp-file"] = "jnlp",
- ["application/x-makeself"] = "run",
- ["application/x-perl"] = "pl",
- ["application/x-pilot"] = "pdb",
- ["application/x-rar-compressed"] = "rar",
- ["application/x-redhat-package-manager"] = "rpm",
- ["application/x-sea"] = "sea",
- ["application/x-shockwave-flash"] = "swf",
- ["application/x-stuffit"] = "sit",
- ["application/x-tcl"] = "tcl",
- ["application/x-x509-ca-cert"] = "crt",
- ["application/x-xpinstall"] = "xpi",
- ["application/xhtml+xml"] = "xhtml",
- ["application/xspf+xml"] = "xspf",
- ["application/zip"] = "zip",
- ["application/x-dosexec"] = "exe",
- ["application/x-msdownload"] = "exe",
- ["application/x-executable"] = "exe",
- ["text/x-msdos-batch"] = "bat",
+ ["text/html"] = "html",
+ ["text/css"] = "css",
+ ["text/xml"] = "xml",
+ ["image/gif"] = "gif",
+ ["image/jpeg"] = "jpeg",
+ ["application/javascript"] = "js",
+ ["application/atom+xml"] = "atom",
+ ["application/rss+xml"] = "rss",
+ ["application/csv"] = "csv",
+ ["text/mathml"] = "mml",
+ ["text/plain"] = "txt",
+ ["text/vnd.sun.j2me.app-descriptor"] = "jad",
+ ["text/vnd.wap.wml"] = "wml",
+ ["text/x-component"] = "htc",
+ ["image/png"] = "png",
+ ["image/svg+xml"] = "svg",
+ ["image/tiff"] = "tiff",
+ ["image/vnd.wap.wbmp"] = "wbmp",
+ ["image/webp"] = "webp",
+ ["image/x-icon"] = "ico",
+ ["image/x-jng"] = "jng",
+ ["image/x-ms-bmp"] = "bmp",
+ ["font/woff"] = "woff",
+ ["font/woff2"] = "woff2",
+ ["application/java-archive"] = "jar",
+ ["application/json"] = "json",
+ ["application/mac-binhex40"] = "hqx",
+ ["application/msword"] = "doc",
+ ["application/pdf"] = "pdf",
+ ["application/postscript"] = "ps",
+ ["application/rtf"] = "rtf",
+ ["application/vnd.apple.mpegurl"] = "m3u8",
+ ["application/vnd.google-earth.kml+xml"] = "kml",
+ ["application/vnd.google-earth.kmz"] = "kmz",
+ ["application/vnd.ms-excel"] = "xls",
+ ["application/vnd.ms-fontobject"] = "eot",
+ ["application/vnd.ms-powerpoint"] = "ppt",
+ ["application/vnd.oasis.opendocument.graphics"] = "odg",
+ ["application/vnd.oasis.opendocument.presentation"] = "odp",
+ ["application/vnd.oasis.opendocument.spreadsheet"] = "ods",
+ ["application/vnd.oasis.opendocument.text"] = "odt",
+ ["application/vnd.openxmlformats-officedocument.presentationml.presentation"] = "pptx",
+ ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"] = "xlsx",
+ ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"] = "docx",
+ ["application/x-7z-compressed"] = "7z",
+ ["application/x-cocoa"] = "cco",
+ ["application/x-java-archive-diff"] = "jardiff",
+ ["application/x-java-jnlp-file"] = "jnlp",
+ ["application/x-makeself"] = "run",
+ ["application/x-perl"] = "pl",
+ ["application/x-pilot"] = "pdb",
+ ["application/x-rar-compressed"] = "rar",
+ ["application/x-redhat-package-manager"] = "rpm",
+ ["application/x-sea"] = "sea",
+ ["application/x-shockwave-flash"] = "swf",
+ ["application/x-stuffit"] = "sit",
+ ["application/x-tcl"] = "tcl",
+ ["application/x-x509-ca-cert"] = "crt",
+ ["application/x-xpinstall"] = "xpi",
+ ["application/xhtml+xml"] = "xhtml",
+ ["application/xspf+xml"] = "xspf",
+ ["application/zip"] = "zip",
+ ["application/x-dosexec"] = "exe",
+ ["application/x-msdownload"] = "exe",
+ ["application/x-executable"] = "exe",
+ ["text/x-msdos-batch"] = "bat",
- ["audio/midi"] = "mid",
- ["audio/mpeg"] = "mp3",
- ["audio/ogg"] = "ogg",
- ["audio/x-m4a"] = "m4a",
- ["audio/x-realaudio"] = "ra",
- ["video/3gpp"] = "3gpp",
- ["video/mp2t"] = "ts",
- ["video/mp4"] = "mp4",
- ["video/mpeg"] = "mpeg",
- ["video/quicktime"] = "mov",
- ["video/webm"] = "webm",
- ["video/x-flv"] = "flv",
- ["video/x-m4v"] = "m4v",
- ["video/x-mng"] = "mng",
- ["video/x-ms-asf"] = "asx",
- ["video/x-ms-wmv"] = "wmv",
- ["video/x-msvideo"] = "avi",
+ ["audio/midi"] = "mid",
+ ["audio/mpeg"] = "mp3",
+ ["audio/ogg"] = "ogg",
+ ["audio/x-m4a"] = "m4a",
+ ["audio/x-realaudio"] = "ra",
+ ["video/3gpp"] = "3gpp",
+ ["video/mp2t"] = "ts",
+ ["video/mp4"] = "mp4",
+ ["video/mpeg"] = "mpeg",
+ ["video/quicktime"] = "mov",
+ ["video/webm"] = "webm",
+ ["video/x-flv"] = "flv",
+ ["video/x-m4v"] = "m4v",
+ ["video/x-mng"] = "mng",
+ ["video/x-ms-asf"] = "asx",
+ ["video/x-ms-wmv"] = "wmv",
+ ["video/x-msvideo"] = "avi",
}
return exports
-- Define callbacks
- avast_helo_cb = function (merr, mdata, conn)
+ avast_helo_cb = function(merr, mdata, conn)
-- Called when we have established a connection but not read anything
tcp_conn = conn
end
end
-
avast_scan_cb = function(merr)
-- Called when we have send request to avast and are waiting for reply
if no_connection_error(merr) then
local vname = string.match(data, 'stream: (.+) FOUND')
if string.find(vname, '^Heuristics%.Encrypted') then
rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix)
- common.yield_result(task, rule, 'File is encrypted: '.. vname,
+ common.yield_result(task, rule, 'File is encrypted: ' .. vname,
0.0, 'encrypted', maybe_part)
cached = 'ENCRYPTED'
elseif string.find(vname, '^Heuristics%.OLE2%.ContainsMacros') then
cached = 'MACRO'
elseif string.find(vname, '^Heuristics%.Limits%.Exceeded') then
rspamd_logger.errx(task, '%s: ClamAV Limits Exceeded', rule.log_prefix)
- common.yield_result(task, rule, 'Limits Exceeded: '.. vname, 0.0,
+ common.yield_result(task, rule, 'Limits Exceeded: ' .. vname, 0.0,
'fail', maybe_part)
elseif vname then
common.yield_result(task, rule, vname, 1.0, nil, maybe_part)
http.request({
ev_base = ev_base,
config = cfg,
- url = cloudmark_url(rule, addr, '/score/v2/max-message-size'),
+ url = cloudmark_url(rule, addr, '/score/v2/max-message-size'),
callback = max_message_size_cb,
})
end
if cloudmark_conf.upstreams then
- cloudmark_conf.symbols = {{ symbol = cloudmark_conf.symbol_spam, score = 5.0 }}
- cloudmark_conf.preloads = {cloudmark_preload}
+ cloudmark_conf.symbols = { { symbol = cloudmark_conf.symbol_spam, score = 5.0 } }
+ cloudmark_conf.preloads = { cloudmark_preload }
lua_util.add_debug_alias('external_services', cloudmark_conf.name)
return cloudmark_conf
end
local seen_data = false
local out = {}
- for k,v in pairs(tbl) do
+ for k, v in pairs(tbl) do
if v.data then
seen_data = true
table.insert(out, string.format('--%s\r\n', boundary))
if rule.add_headers and type(obj.appendHeaders) == 'table' then
local headers_add = fun.tomap(fun.map(function(h)
- return h.headerField,{
+ return h.headerField, {
order = 1, value = h.body
}
end, obj.appendHeaders))
local rcpt_to = task:get_recipients('smtp')
if rcpt_to then
request['rcptTo'] = {
- data = table.concat(fun.totable(fun.map(function(r) return r.addr end, rcpt_to)), ',')
+ data = table.concat(fun.totable(fun.map(function(r)
+ return r.addr
+ end, rcpt_to)), ',')
}
end
http.request(request_data)
else
- rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
'exceed', rule.log_prefix)
- task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '..
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
'retransmits exceed')
upstream:fail()
end
cloudmark_requery()
else
-- Parse the response
- if upstream then upstream:ok() end
+ if upstream then
+ upstream:ok()
+ end
if code ~= 200 then
rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
end
return {
- type = {'cloudmark', 'scanner'},
+ type = { 'cloudmark', 'scanner' },
description = 'Cloudmark cartridge interface',
configure = cloudmark_config,
check = cloudmark_check,
end
local function match_patterns(default_sym, found, patterns, dyn_weight)
- if type(patterns) ~= 'table' then return default_sym, dyn_weight end
+ if type(patterns) ~= 'table' then
+ return default_sym, dyn_weight
+ end
if not patterns[1] then
for sym, pat in pairs(patterns) do
if pat:match(found) then
local flags
if type(vname) == 'string' then
- threat_table = {vname}
+ threat_table = { vname }
elseif type(vname) == 'table' then
threat_table = vname
end
patterns = rule.patterns
symbol = rule.symbol
threat_info = rule.detection_category .. 'found'
- if not dyn_weight then dyn_weight = 1.0 end
+ if not dyn_weight then
+ dyn_weight = 1.0
+ end
elseif is_fail == 'fail' then
patterns = rule.patterns_fail
symbol = rule.symbol_fail
dyn_weight = 1.0
end
-
for _, tm in ipairs(threat_table) do
local symname, symscore = match_patterns(symbol, tm, patterns, dyn_weight)
if rule.whitelist and rule.whitelist:get_key(tm) then
local function message_not_too_large(task, content, rule)
local max_size = tonumber(rule.max_size)
- if not max_size then return true end
+ if not max_size then
+ return true
+ end
if #content > max_size then
rspamd_logger.infox(task, "skip %s check as it is too large: %s (%s is allowed)",
rule.log_prefix, #content, max_size)
local function message_not_too_small(task, content, rule)
local min_size = tonumber(rule.min_size)
- if not min_size then return true end
+ if not min_size then
+ return true
+ end
if #content < min_size then
rspamd_logger.infox(task, "skip %s check as it is too small: %s (%s is allowed)",
rule.log_prefix, #content, min_size)
if not text_part_above_limit then
rspamd_logger.infox(task, '%s: #words in all text parts is below text_part_min_words limit: %s',
- rule.log_prefix, rule.text_part_min_words)
+ rule.log_prefix, rule.text_part_min_words)
end
return text_part_above_limit
-- rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, "result is already reject")
-- return false
--elseif metric_result[1] > metric_result[2]*2 then
- if metric_result[1] > metric_result[2]*2 then
+ if metric_result[1] > metric_result[2] * 2 then
rspamd_logger.infox(task, '%s: aborting: %s', rule.log_prefix, 'score > 2 * reject_level: ' .. metric_result[1])
return false
elseif has_pre_result and metric_action == 'reject' then
else
lua_util.debugm(rule.name, task, '%s: got cached negative result for %s: %s',
- rule.log_prefix, key, threat_string[1])
+ rule.log_prefix, key, threat_string[1])
end
uncached = false
else
local f_dynamic_scan = dynamic_scan(task, rule)
if uncached and
- f_message_not_too_large and
- f_message_not_too_small and
- f_message_min_words and
- f_dynamic_scan then
+ f_message_not_too_large and
+ f_message_not_too_small and
+ f_message_min_words and
+ f_dynamic_scan then
fn()
false, -- is write
redis_av_cb, --callback
'GET', -- command
- {key} -- arguments)
+ { key } -- arguments)
) then
return true
end
local function save_cache(task, digest, rule, to_save, dyn_weight, maybe_part)
local key = digest
- if not dyn_weight then dyn_weight = 1.0 end
+ if not dyn_weight then
+ dyn_weight = 1.0
+ end
local function redis_set_cb(err)
-- Do nothing
rule.detection_category, to_save, key, err)
else
lua_util.debugm(rule.name, task, '%s: saved cached result for %s: %s - score %s - ttl %s',
- rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire)
+ rule.log_prefix, key, to_save, dyn_weight, rule.cache_expire)
end
end
to_save = table.concat(to_save, '\v')
end
- local value_tbl = {to_save, dyn_weight}
+ local value_tbl = { to_save, dyn_weight }
if maybe_part and rule.show_attachments and maybe_part:get_filename() then
local fname = maybe_part:get_filename()
table.insert(value_tbl, fname)
local ext = {}
for n = 1, 2 do
- ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil
+ ext[n] = #filename_parts > n and string.lower(filename_parts[#filename_parts + 1 - n]) or nil
end
- return ext[1],ext[2],filename_parts
+ return ext[1], ext[2], filename_parts
end
local function check_parts_match(task, rule)
local filter_func = function(p)
- local mtype,msubtype = p:get_type()
+ local mtype, msubtype = p:get_type()
local detected_ext = p:get_detected_ext()
local fname = p:get_filename()
local ext, ext2
if rule.scan_all_mime_parts == false then
- -- check file extension and filename regex matching
+ -- check file extension and filename regex matching
--lua_util.debugm(rule.name, task, '%s: filename: |%s|%s|', rule.log_prefix, fname)
if fname ~= nil then
- ext,ext2 = gen_extension(fname)
+ ext, ext2 = gen_extension(fname)
--lua_util.debugm(rule.name, task, '%s: extension, fname: |%s|%s|%s|', rule.log_prefix, ext, ext2, fname)
if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext')
or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then
if p:is_archive() then
local arch = p:get_archive()
local filelist = arch:get_files_full(1000)
- for _,f in ipairs(filelist) do
- ext,ext2 = gen_extension(f.name)
+ for _, f in ipairs(filelist) do
+ ext, ext2 = gen_extension(f.name)
if match_filter(task, rule, ext, rule.mime_parts_filter_ext, 'ext')
or match_filter(task, rule, ext2, rule.mime_parts_filter_ext, 'ext') then
lua_util.debugm(rule.name, task, '%s: extension matched in archive: |%s|%s|', rule.log_prefix, ext, ext2)
if rule.symbol_type == 'postfilter' and metric_action == 'reject' then
return true, 'result is already reject'
- elseif metric_result[1] > metric_result[2]*2 then
+ elseif metric_result[1] > metric_result[2] * 2 then
return true, 'score > 2 * reject_level: ' .. metric_result[1]
elseif has_pre_result and metric_action == 'reject' then
return true, 'pre_result reject is set'
local upstream = rule.upstreams:get_upstream_round_robin()
local addr = upstream:get_addr()
local retransmits = rule.retransmits
- local client = rule.client
+ local client = rule.client
local client_ip = task:get_from_ip()
if client_ip and client_ip:is_valid() then
local rcpts = task:get_recipients();
if rcpts then
local dcc_recipients = table.concat(fun.totable(fun.map(function(rcpt)
- return rcpt['addr'] end,
+ return rcpt['addr']
+ end,
rcpts)), '\n')
if dcc_recipients then
envrcpt = dcc_recipients
fuz2_max = 999999,
})
else
- rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
- 'exceed', rule.log_prefix)
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
end
end
else
-- Parse the response
- local _,_,result,disposition,header = tostring(data):find("(.-)\n(.-)\n(.-)$")
+ local _, _, result, disposition, header = tostring(data):find("(.-)\n(.-)\n(.-)$")
lua_util.debugm(rule.name, task, 'DCC result=%1 disposition=%2 header="%3"',
result, disposition, header)
if header then
-- Unfold header
header = header:gsub('\r?\n%s*', ' ')
- local _,_,info = header:find("; (.-)$")
+ local _, _, info = header:find("; (.-)$")
if (result == 'R') then
-- Reject
common.yield_result(task, rule, info, rule.default_score)
dcc_requery()
elseif result == 'A' then
- local opts = {}
- local score = 0.0
- info = info:lower()
- local rep = info:match('rep=([^=%s]+)')
+ local opts = {}
+ local score = 0.0
+ info = info:lower()
+ local rep = info:match('rep=([^=%s]+)')
- -- Adjust reputation if available
- if rep then rep = tonumber(rep) end
- if not rep then
- rep = 1.0
+ -- Adjust reputation if available
+ if rep then
+ rep = tonumber(rep)
+ end
+ if not rep then
+ rep = 1.0
+ end
+
+ local function check_threshold(what, num, lim)
+ local rnum
+ if num == 'many' then
+ rnum = lim
+ else
+ rnum = tonumber(num)
end
- local function check_threshold(what, num, lim)
- local rnum
- if num == 'many' then
- rnum = lim
- else
- rnum = tonumber(num)
- end
-
- if rnum and rnum >= lim then
- opts[#opts + 1] = string.format('%s=%s', what, num)
- score = score + rep / 3.0
- end
+ if rnum and rnum >= lim then
+ opts[#opts + 1] = string.format('%s=%s', what, num)
+ score = score + rep / 3.0
end
+ end
- info = info:lower()
- local body = info:match('body=([^=%s]+)')
+ info = info:lower()
+ local body = info:match('body=([^=%s]+)')
- if body then
- check_threshold('body', body, rule.body_max)
- end
+ if body then
+ check_threshold('body', body, rule.body_max)
+ end
- local fuz1 = info:match('fuz1=([^=%s]+)')
+ local fuz1 = info:match('fuz1=([^=%s]+)')
- if fuz1 then
- check_threshold('fuz1', fuz1, rule.fuz1_max)
- end
+ if fuz1 then
+ check_threshold('fuz1', fuz1, rule.fuz1_max)
+ end
- local fuz2 = info:match('fuz2=([^=%s]+)')
+ local fuz2 = info:match('fuz2=([^=%s]+)')
- if fuz2 then
- check_threshold('fuz2', fuz2, rule.fuz2_max)
- end
+ if fuz2 then
+ check_threshold('fuz2', fuz2, rule.fuz2_max)
+ end
- if #opts > 0 and score > 0 then
- task:insert_result(rule.symbol_bulk,
- score,
- opts)
- common.save_cache(task, digest, rule, opts, score)
+ if #opts > 0 and score > 0 then
+ task:insert_result(rule.symbol_bulk,
+ score,
+ opts)
+ common.save_cache(task, digest, rule, opts, score)
+ else
+ common.save_cache(task, digest, rule, 'OK')
+ if rule.log_clean then
+ rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
+ rule.log_prefix, info)
else
- common.save_cache(task, digest, rule, 'OK')
- if rule.log_clean then
- rspamd_logger.infox(task, '%s: clean, returned result A - info: %s',
- rule.log_prefix, info)
- else
- lua_util.debugm(rule.name, task, '%s: returned result A - info: %s',
- rule.log_prefix, info)
+ lua_util.debugm(rule.name, task, '%s: returned result A - info: %s',
+ rule.log_prefix, info)
end
end
elseif result == 'G' then
end
return {
- type = {'dcc','bulk', 'hash', 'scanner'},
+ type = { 'dcc', 'bulk', 'hash', 'scanner' },
description = 'dcc bulk scanner',
configure = dcc_config,
check = dcc_check,
local addr = upstream:get_addr()
local retransmits = rule.retransmits
local scan_id = task:get_queue_id()
- if not scan_id then scan_id = task:get_uid() end
+ if not scan_id then
+ scan_id = task:get_uid()
+ end
local header = string.format('SCAN STREAM %s SIZE %d\n', scan_id,
#content)
local footer = '\n'
local rspamd_util = require "rspamd_util"
local rspamd_version = rspamd_version
-
local N = 'icap'
local function icap_config(opts)
end
icap_conf.upstreams = upstream_list.create(rspamd_config,
- icap_conf.servers,
- icap_conf.default_port)
+ icap_conf.servers,
+ icap_conf.default_port)
if icap_conf.upstreams then
lua_util.add_debug_alias('external_services', icap_conf.name)
end
rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
- icap_conf.servers)
+ icap_conf.servers)
return nil
end
rspamd_version('main'),
rspamd_version('id'),
rspamd_util.get_hostname(),
- string.sub(task:get_uid(), 1,6))
+ string.sub(task:get_uid(), 1, 6))
end
-- Build the icap queries
end
local respond_headers = {
- -- Add main RESPMOD header before any other
- string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme),
- string.format('Host: %s\r\n', addr:to_string()),
+ -- Add main RESPMOD header before any other
+ string.format('RESPMOD icap://%s/%s ICAP/1.0\r\n', addr:to_string(), rule.scheme),
+ string.format('Host: %s\r\n', addr:to_string()),
}
local size = tonumber(#content)
addr = upstream:get_addr()
lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
- rule.log_prefix, addr, addr:get_port())
+ rule.log_prefix, addr, addr:get_port())
tcp_options.host = addr:to_string()
tcp_options.port = addr:get_port()
tcp.request(tcp_options)
else
- rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
- 'exceed - error: %s', rule.log_prefix, err_m or '')
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed - error: %s', rule.log_prefix, err_m or '')
common.yield_result(task, rule, string.format('failed - error: %s', err_m),
0.0, 'fail', maybe_part)
end
end
table.insert(respond_headers, '\r\n')
- for _,h in ipairs(resp_req_headers) do table.insert(respond_headers, h) end
+ for _, h in ipairs(resp_req_headers) do
+ table.insert(respond_headers, h)
+ end
table.insert(respond_headers, '\r\n')
- for _,h in ipairs(resp_http_headers) do table.insert(respond_headers, h) end
+ for _, h in ipairs(resp_http_headers) do
+ table.insert(respond_headers, h)
+ end
table.insert(respond_headers, '\r\n')
table.insert(respond_headers, chunked_size .. '\r\n')
table.insert(respond_headers, content)
elseif string.find(s, '^HTTP') then
icap_headers['http'] = tostring(s)
elseif string.find(s, '[%a%d-+]-:') then
- local _,_,key,value = tostring(s):find("([%a%d-+]-):%s?(.+)")
+ local _, _, key, value = tostring(s):find("([%a%d-+]-):%s?(.+)")
if key ~= nil then
icap_headers[key:lower()] = tostring(value)
end
if maybe_split and string.find(icap_threat, ',') then
local threats = lua_util.str_split(string.gsub(icap_threat, "%s", ""), ',') or {}
- for _,v in ipairs(threats) do
+ for _, v in ipairs(threats) do
table.insert(threat_table, v)
end
else
-- Generic ICAP Headers
if headers['x-infection-found'] then
- local _,_,icap_type,_,icap_threat =
- headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$")
+ local _, _, icap_type, _, icap_threat = headers['x-infection-found']:find("Type=(.-); Resolution=(.-); Threat=(.-);$")
-- Type=2 is typical for scan error returns
if icap_type and icap_type == '2' then
lua_util.debugm(rule.name, task,
'%s: icap X-Infection-Found: %s', rule.log_prefix, icap_threat)
threat_table_add(icap_threat, false)
- -- stupid workaround for unuseable x-infection-found header
- -- but also x-virus-name set (McAfee Web Gateway 9)
+ -- stupid workaround for unuseable x-infection-found header
+ -- but also x-virus-name set (McAfee Web Gateway 9)
elseif not icap_threat and headers['x-virus-name'] then
threat_table_add(headers['x-virus-name'], true)
else
lua_util.debugm(rule.name, task,
'%s: icap X-Virus-ID: %s', rule.log_prefix, headers['x-virus-id'])
threat_table_add(headers['x-virus-id'], true)
- -- FSecure X-Headers
+ -- FSecure X-Headers
elseif headers['x-fsecure-scan-result'] and headers['x-fsecure-scan-result'] ~= "clean" then
local infected_filename = ""
rule.log_prefix, infection_name, infected_filename)
threat_table_add(infection_name, true)
- -- McAfee Web Gateway manual extra headers
+ -- McAfee Web Gateway manual extra headers
elseif headers['x-mwg-block-reason'] and headers['x-mwg-block-reason'] ~= "" then
threat_table_add(headers['x-mwg-block-reason'], false)
- -- Sophos SAVDI special http headers
+ -- Sophos SAVDI special http headers
elseif headers['x-blocked'] and headers['x-blocked'] ~= "" then
threat_table_add(headers['x-blocked'], false)
elseif headers['x-block-reason'] and headers['x-block-reason'] ~= "" then
threat_table_add(headers['x-block-reason'], false)
- -- last try HTTP [4]xx return
+ -- last try HTTP [4]xx return
elseif headers.http and string.find(headers.http, '^HTTP%/[12]%.. [4]%d%d') then
threat_table_add(
- string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false)
+ string.format("pseudo-virus (blocked): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false)
elseif rule.use_http_3xx_as_threat and headers.http and string.find(headers.http, '^HTTP%/[12]%.. [3]%d%d')
- then
+ then
threat_table_add(
- string.format("pseudo-virus (redirect): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false)
+ string.format("pseudo-virus (redirect): %s", string.gsub(headers.http, 'HTTP%/[12]%.. ', '')), false)
end
if #threat_table > 0 then
end
else
rspamd_logger.errx(task, '%s: unhandled response |%s|',
- rule.log_prefix, string.gsub(result, "\r\n", ", "))
+ rule.log_prefix, string.gsub(result, "\r\n", ", "))
common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_http_headers.icap),
0.0, 'fail', maybe_part)
end
-- Threat found - close connection
connection:close()
elseif not icap_header_result
- and rule.use_http_result_header
- and icap_headers.encapsulated
- and not string.find(icap_headers.encapsulated, 'null%-body=0')
- then
+ and rule.use_http_result_header
+ and icap_headers.encapsulated
+ and not string.find(icap_headers.encapsulated, 'null%-body=0')
+ then
-- Try to read encapsulated HTTP Headers
lua_util.debugm(rule.name, task, '%s: no ICAP virus header found - try HTTP headers',
- rule.log_prefix)
+ rule.log_prefix)
connection:add_read(icap_r_respond_http_cb, '\r\n\r\n')
else
connection:close()
return false
else
rspamd_logger.errx(task, '%s: unhandled response |%s|',
- rule.log_prefix, string.gsub(result, "\r\n", ", "))
+ rule.log_prefix, string.gsub(result, "\r\n", ", "))
common.yield_result(task, rule, string.format('unhandled icap response: %s', icap_headers.icap),
0.0, 'fail', maybe_part)
end
if rule.x_client_header then
local client = task:get_from_ip()
- if client then add_respond_header('X-Client-IP', client:to_string()) end
+ if client then
+ add_respond_header('X-Client-IP', client:to_string())
+ end
end
-- F-Secure extra headers
if rule.x_rcpt_header then
local rcpt_to = task:get_principal_recipient()
- if rcpt_to then add_respond_header('X-Rcpt-To', rcpt_to) end
+ if rcpt_to then
+ add_respond_header('X-Rcpt-To', rcpt_to)
+ end
end
if rule.x_from_header then
local mail_from = task:get_principal_recipient()
- if mail_from and mail_from[1] then add_respond_header('X-Rcpt-To', mail_from[1].addr) end
+ if mail_from and mail_from[1] then
+ add_respond_header('X-Rcpt-To', mail_from[1].addr)
+ end
end
end
if icap_headers.connection and icap_headers.connection:lower() == 'close' then
lua_util.debugm(rule.name, task, '%s: OPTIONS request Connection: %s - using new connection',
- rule.log_prefix, icap_headers.connection)
+ rule.log_prefix, icap_headers.connection)
connection:close()
tcp_options.callback = icap_w_respond_cb
tcp_options.data = get_respond_query()
else
rspamd_logger.errx(task, '%s: RESPMOD method not advertised: Methods: %s',
- rule.log_prefix, icap_headers['methods'])
+ rule.log_prefix, icap_headers['methods'])
common.yield_result(task, rule, 'NO RESPMOD', 0.0,
'fail', maybe_part)
end
else
rspamd_logger.errx(task, '%s: OPTIONS query failed: %s',
- rule.log_prefix, icap_headers.icap or "-")
+ rule.log_prefix, icap_headers.icap or "-")
common.yield_result(task, rule, 'OPTIONS query failed', 0.0,
'fail', maybe_part)
end
end
return {
- type = {N, 'virus', 'virus', 'scanner'},
+ type = { N, 'virus', 'virus', 'scanner' },
description = 'generic icap antivirus',
configure = icap_config,
check = icap_check,
}
local function require_scanner(name)
- local sc = require ("lua_scanners/" .. name)
+ local sc = require("lua_scanners/" .. name)
exports[sc.name or name] = sc
end
return fun.tomap(fun.filter(function(_, elt)
return type(elt) == 'table' and elt.type and (
(type(elt.type) == 'string' and elt.type == t) or
- (type(elt.type) == 'table' and fun.any(function(tt) return tt == t end, elt.type))
+ (type(elt.type) == 'table' and fun.any(function(tt)
+ return tt == t
+ end, elt.type))
)
end, exports))
end
rspamd_util.close_file(message_fd)
end)
-
local function kaspersky_callback(err, data)
if err then
local req_body
if rule.use_files then
- local fname = string.format('%s/%s.tmp',
+ local fname = string.format('%s/%s.tmp',
rule.tmpdir, rspamd_util.random_hex(32))
local message_fd = rspamd_util.create_file(fname)
http.request(request_data)
else
- rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
'exceed', rule.log_prefix)
- task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '..
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
'retransmits exceed')
end
end
requery()
else
-- Parse the response
- if upstream then upstream:ok() end
+ if upstream then
+ upstream:ok()
+ end
if code ~= 200 then
rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
if why == 'PASSWORD PROTECTED' then
rspamd_logger.errx(task, '%s: File is encrypted', rule.log_prefix)
- common.yield_result(task, rule, 'File is encrypted: '.. why,
+ common.yield_result(task, rule, 'File is encrypted: ' .. why,
0.0, 'encrypted', maybe_part)
cached = 'ENCRYPTED'
else
callback = oletools_callback,
})
else
- rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
'exceed - err: %s', rule.log_prefix, error)
common.yield_result(task, rule,
'failed to scan, maximum retransmits exceed - err: ' .. error,
else
json_response = json_response .. tostring(data)
- if not string.find(json_response, '\t\n\n\t') and #data == 8192 then
+ if not string.find(json_response, '\t\n\n\t') and #data == 8192 then
lua_util.debugm(rule.name, task, '%s: no stop word: add_read - #json: %s / current packet: %s',
- rule.log_prefix, #json_response, #data)
+ rule.log_prefix, #json_response, #data)
conn:add_read(oletools_callback)
else
rule.log_prefix, v.message)
--common.yield_result(task, rule, 'failed - err: ' .. v.error, 0.0, 'fail')
- elseif type(v.analysis) == 'table' and type(v.macros) == 'table' then
+ elseif type(v.analysis) == 'table' and type(v.macros) == 'table' then
-- analysis + macro found - evaluate response
if type(v.analysis) == 'table' and #v.analysis == 0 and #v.macros == 0 then
lua_util.debugm(rule.name, task,
'%s: type: %s', rule.log_prefix, result[2]['type'])
- for _,m in ipairs(v.macros) do
- lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, '..
+ for _, m in ipairs(v.macros) do
+ lua_util.debugm(rule.name, task, '%s: macros found - code: %s, ole_stream: %s, ' ..
'vba_filename: %s', rule.log_prefix, m.code, m.ole_stream, m.vba_filename)
end
- for _,a in ipairs(v.analysis) do
- lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, '..
+ for _, a in ipairs(v.analysis) do
+ lua_util.debugm(rule.name, task, '%s: threat found - type: %s, keyword: %s, ' ..
'description: %s', rule.log_prefix, a.type, a.keyword, a.description)
if a.type == 'AutoExec' then
analysis_cat_table.autoexec = 'A'
table.insert(analysis_keyword_table, a.keyword)
elseif a.type == 'Suspicious' then
if rule.extended == true or
- (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings')
+ (a.keyword ~= 'Base64 Strings' and a.keyword ~= 'Hex Strings')
then
analysis_cat_table.suspicious = 'S'
table.insert(analysis_keyword_table, a.keyword)
local analysis_cat_table_keys_sorted = lua_util.keys(analysis_cat_table)
table.sort(analysis_cat_table_keys_sorted)
- for _,v in ipairs(analysis_cat_table_keys_sorted) do
+ for _, v in ipairs(analysis_cat_table_keys_sorted) do
table.insert(analysis_cat_table_values_sorted, analysis_cat_table[v])
end
end
return {
- type = {N, 'attachment scanner', 'hash', 'scanner'},
+ type = { N, 'attachment scanner', 'hash', 'scanner' },
description = 'oletools office macro scanner',
configure = oletools_config,
check = oletools_check,
-- SEE: https://github.com/p0f/p0f/blob/v3.06b/docs/README#L317
local S = {
BAD_QUERY = 0x0,
- OK = 0x10,
- NO_MATCH = 0x20
+ OK = 0x10,
+ NO_MATCH = 0x20
}
local N = 'p0f'
end
local function trim(...)
- local vars = {...}
+ local vars = { ... }
for k, v in ipairs(vars) do
-- skip numbers, trim only strings
-- API response must be 232 bytes long
if #data ~= 232 then
rspamd_logger.errx(task, 'malformed response from p0f on %s, %s bytes',
- rule.socket, #data)
+ rule.socket, #data)
common.yield_result(task, rule, 'Malformed Response: ' .. rule.socket,
- 0.0, 'fail')
+ 0.0, 'fail')
return
end
local _, status, _, _, _, uptime_min, _, _, _, distance, _, _, os_name,
- os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack(
+ os_flavor, _, _, link_type, _ = trim(rspamd_util.unpack(
'I4I4I4I4I4I4I4I4I4hbbc32c32c32c32c32c32', data))
if status ~= S.OK then
if status == S.BAD_QUERY then
rspamd_logger.errx(task, 'malformed p0f query on %s', rule.socket)
common.yield_result(task, rule, 'Malformed Query: ' .. rule.socket,
- 0.0, 'fail')
+ 0.0, 'fail')
end
return
local os_string = #os_name == 0 and 'unknown' or os_name .. ' ' .. os_flavor
task:get_mempool():set_variable('os_fingerprint', os_string, link_type,
- uptime_min, distance)
+ uptime_min, distance)
if link_type and #link_type > 0 then
common.yield_result(task, rule, {
os_string,
'link=' .. link_type,
- 'distance=' .. distance},
+ 'distance=' .. distance },
0.0)
else
common.yield_result(task, rule, {
os_string,
'link=unknown',
- 'distance=' .. distance},
+ 'distance=' .. distance },
0.0)
end
end
local query = rspamd_util.pack('I4 I1 c16', 0x50304601,
- ip:get_version(), ip2bin(ip))
+ ip:get_version(), ip2bin(ip))
tcp.request({
host = rule.socket,
if rule.redis_params then
local key = rule.prefix .. ip:to_string()
ret = lua_redis.redis_make_request(task,
- rule.redis_params,
- key,
- false,
- redis_get_cb,
- 'GET',
- { key }
+ rule.redis_params,
+ key,
+ false,
+ redis_get_cb,
+ 'GET',
+ { key }
)
end
end
return {
- type = {N, 'fingerprint', 'scanner'},
+ type = { N, 'fingerprint', 'scanner' },
description = 'passive OS fingerprinter',
configure = p0f_config,
check = p0f_check,
local common = require "lua_scanners/common"
local N = 'pyzor'
-local categories = {'pyzor','bulk', 'hash', 'scanner'}
+local categories = { 'pyzor', 'bulk', 'hash', 'scanner' }
local function pyzor_config(opts)
end
end
- if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then
- rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix)
- return
- end
+ if digest == 'da39a3ee5e6b4b0d3255bfef95601890afd80709' then
+ rspamd_logger.infox(task, '%s: not checking default digest', rule.log_prefix)
+ return
+ end
tcp.request({
task = task,
end
end
-
return {
type = categories,
description = 'pyzor bulk scanner',
return nil
end
-
local function razor_check(task, content, digest, rule)
local function razor_check_uncached ()
local upstream = rule.upstreams:get_upstream_round_robin()
retransmits = retransmits - 1
lua_util.debugm(rule.name, task, '%s: Request Error: %s - retries left: %s',
- rule.log_prefix, err, retransmits)
+ rule.log_prefix, err, retransmits)
-- Select a different upstream!
upstream = rule.upstreams:get_upstream_round_robin()
addr = upstream:get_addr()
lua_util.debugm(rule.name, task, '%s: retry IP: %s:%s',
- rule.log_prefix, addr, addr:get_port())
+ rule.log_prefix, addr, addr:get_port())
tcp.request({
task = task,
callback = razor_callback,
})
else
- rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
- 'exceed', rule.log_prefix)
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed', rule.log_prefix)
common.yield_result(task, rule, 'failed to scan and retransmits exceed', 0.0, 'fail')
end
end
end
common.save_cache(task, digest, rule, 'OK', rule.default_score)
else
- rspamd_logger.errx(task,"%s - unknown response from razorfy: %s", addr:to_string(), threat_string)
+ rspamd_logger.errx(task, "%s - unknown response from razorfy: %s", addr:to_string(), threat_string)
end
end
end
return {
- type = {'razor','spam', 'hash', 'scanner'},
+ type = { 'razor', 'spam', 'hash', 'scanner' },
description = 'razor bulk scanner',
configure = razor_config,
check = razor_check,
local function savapi_fin_cb(err, conn)
local vnames_reordered = {}
-- Swap table
- for virus,_ in pairs(vnames) do
+ for virus, _ in pairs(vnames) do
table.insert(vnames_reordered, virus)
end
lua_util.debugm(rule.name, task, "%s: number of virus names found %s", rule['type'], #vnames_reordered)
if #vnames_reordered > 0 then
local vname = {}
- for _,virus in ipairs(vnames_reordered) do
+ for _, virus in ipairs(vnames_reordered) do
table.insert(vname, virus)
end
if string.find(result, '100 PRODUCT') then
lua_util.debugm(rule.name, task, "%s: scanning file: %s",
rule['type'], fname)
- conn:add_write(savapi_scan1_cb, {string.format('SCAN %s\n',
- fname)})
+ conn:add_write(savapi_scan1_cb, { string.format('SCAN %s\n',
+ fname) })
else
rspamd_logger.errx(task, '%s: invalid product id %s', rule['type'],
rule['product_id'])
upstream = upstream,
timeout = rule['timeout'],
callback = savapi_callback_init,
- stop_pattern = {'\n'},
+ stop_pattern = { '\n' },
})
else
rspamd_logger.errx(task, '%s [%s]: failed to scan, maximum retransmits exceed', rule['symbol'], rule['type'])
-- 100 SAVAPI:4.0 greeting
if string.find(result, '100') then
- conn:add_write(savapi_greet1_cb, {string.format('SET PRODUCT %s\n', rule['product_id'])})
+ conn:add_write(savapi_greet1_cb, { string.format('SET PRODUCT %s\n', rule['product_id']) })
end
end
end
upstream = upstream,
timeout = rule['timeout'],
callback = savapi_callback_init,
- stop_pattern = {'\n'},
+ stop_pattern = { '\n' },
})
end
end
spamassassin_conf.upstreams = upstream_list.create(rspamd_config,
- spamassassin_conf.servers,
- spamassassin_conf.default_port)
+ spamassassin_conf.servers,
+ spamassassin_conf.default_port)
if spamassassin_conf.upstreams then
lua_util.add_debug_alias('external_services', spamassassin_conf.N)
end
rspamd_logger.errx(rspamd_config, 'cannot parse servers %s',
- spamassassin_conf.servers)
+ spamassassin_conf.servers)
return nil
end
local request_data = {
"HEADERS SPAMC/1.5\r\n",
"User: root\r\n",
- "Content-length: ".. #content .. "\r\n",
+ "Content-length: " .. #content .. "\r\n",
"\r\n",
content,
}
retransmits = retransmits - 1
lua_util.debugm(rule.N, task, '%s: Request Error: %s - retries left: %s',
- rule.log_prefix, error, retransmits)
+ rule.log_prefix, error, retransmits)
-- Select a different upstream!
upstream = rule.upstreams:get_upstream_round_robin()
addr = upstream:get_addr()
lua_util.debugm(rule.N, task, '%s: retry IP: %s:%s',
- rule.log_prefix, addr, addr:get_port())
+ rule.log_prefix, addr, addr:get_port())
tcp.request({
task = task,
callback = spamassassin_callback,
})
else
- rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
- 'exceed - err: %s', rule.log_prefix, error)
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
+ 'exceed - err: %s', rule.log_prefix, error)
common.yield_result(task, rule, 'failed to scan and retransmits exceed: ' .. error, 0.0, 'fail')
end
end
end
lua_util.debugm(rule.N, task, '%s: spam_score: %s, symbols: %s, int spam_score: |%s|, type spam_score: |%s|',
- rule.log_prefix, spam_score, symbols, tonumber(spam_score), type(spam_score))
+ rule.log_prefix, spam_score, symbols, tonumber(spam_score), type(spam_score))
if tonumber(spam_score) > 0 and #symbols > 0 and symbols ~= "none" then
end
return {
- type = {N,'spam', 'scanner'},
+ type = { N, 'spam', 'scanner' },
description = 'spamassassin spam scanner',
configure = spamassassin_config,
check = spamassassin_check,
score = 8.0,
description = 'VadeSecure decided message to be phishing'
},
- commercial = {
+ commercial = {
symbol = 'VADE_COMMERCIAL',
score = 0.0,
description = 'VadeSecure decided message to be commercial message'
},
- community = {
+ community = {
symbol = 'VADE_COMMUNITY',
score = 0.0,
description = 'VadeSecure decided message to be community message'
},
- transactional = {
+ transactional = {
symbol = 'VADE_TRANSACTIONAL',
score = 0.0,
description = 'VadeSecure decided message to be transactional message'
http.request(request_data)
else
- rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits '..
+ rspamd_logger.errx(task, '%s: failed to scan, maximum retransmits ' ..
'exceed', rule.log_prefix)
- task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and '..
+ task:insert_result(rule['symbol_fail'], 0.0, 'failed to scan and ' ..
'retransmits exceed')
end
end
vade_requery()
else
-- Parse the response
- if upstream then upstream:ok() end
+ if upstream then
+ upstream:ok()
+ end
if code ~= 200 then
rspamd_logger.errx(task, 'invalid HTTP code: %s, body: %s, headers: %s', code, body, headers)
task:insert_result(rule.symbol_fail, 1.0, 'Bad HTTP code: ' .. code)
end
return {
- type = {'vadesecure', 'scanner'},
+ type = { 'vadesecure', 'scanner' },
description = 'VadeSecure Filterd interface',
configure = vade_config,
check = vade_check,
else
local ucl = require "ucl"
local parser = ucl.parser()
- local res,json_err = parser:parse_string(body)
+ local res, json_err = parser:parse_string(body)
lua_util.debugm(rule.name, task, '%s: got reply data: "%s"',
rule.log_prefix, body)
local blake2b_key = cr_hash.create_specific('blake2'):update('rspamd'):bin()
local function digest_schema()
- return {ts.one_of{'hex', 'base32', 'bleach32', 'rbase32', 'base64'}:is_optional(),
- ts.one_of{'blake2', 'sha256', 'sha1', 'sha512', 'md5'}:is_optional()}
+ return { ts.one_of { 'hex', 'base32', 'bleach32', 'rbase32', 'base64' }:is_optional(),
+ ts.one_of { 'blake2', 'sha256', 'sha1', 'sha512', 'md5' }:is_optional() }
end
exports.digest_schema = digest_schema
return encode_digest(h, args)
end
-
local function get_cached_or_raw_digest(task, idx, mime_part, args)
if #args == 0 then
-- Optimise as we already have this hash in the API
return function(u)
local got_flags = u:get_flags()
for _, flag in ipairs(exclude_flags) do
- if got_flags[flag] then return false end
+ if got_flags[flag] then
+ return false
+ end
end
return true
end
return args[1], 'string'
end
- return '','string'
+ return '', 'string'
end,
['description'] = [[Return value from function's argument or an empty string,
For example, `id('Something')` returns a string 'Something']],
- ['args_schema'] = {ts.string:is_optional()}
+ ['args_schema'] = { ts.string:is_optional() }
},
-- Similar but for making lists
['list'] = {
return fun.map(tostring, args), 'string_list'
end
- return {},'string_list'
+ return {}, 'string_list'
end,
['description'] = [[Return a list from function's arguments or an empty list,
For example, `list('foo', 'bar')` returns a list {'foo', 'bar'}]],
['ip'] = {
['get_value'] = function(task)
local ip = task:get_ip()
- if ip and ip:is_valid() then return ip,'userdata' end
+ if ip and ip:is_valid() then
+ return ip, 'userdata'
+ end
return nil
end,
['description'] = [[Get source IP address]],
from = task:get_from(0)
end
if ((from or E)[1] or E).addr then
- return from[1],'table'
+ return from[1], 'table'
end
return nil
end,
rcpts = task:get_recipients(0)
end
if ((rcpts or E)[1] or E).addr then
- return rcpts,'table_list'
+ return rcpts, 'table_list'
end
return nil
end,
if not country then
return nil
else
- return country,'string'
+ return country, 'string'
end
end,
['description'] = [[Get country (ASN module must be executed first)]],
if not asn then
return nil
else
- return asn,'string'
+ return asn, 'string'
end
end,
['description'] = [[Get AS number (ASN module must be executed first)]],
if not auser then
return nil
else
- return auser,'string'
+ return auser, 'string'
end
end,
['description'] = 'Get authenticated user name',
-- Get principal recipient
['to'] = {
['get_value'] = function(task)
- return task:get_principal_recipient(),'string'
+ return task:get_principal_recipient(), 'string'
end,
['description'] = 'Get principal recipient',
},
-- Get content digest
['digest'] = {
['get_value'] = function(task)
- return task:get_digest(),'string'
+ return task:get_digest(), 'string'
end,
['description'] = 'Get content digest',
},
['get_value'] = function(task, args)
local parts = task:get_parts() or E
local digests = {}
- for i,p in ipairs(parts) do
+ for i, p in ipairs(parts) do
if p:is_attachment() then
table.insert(digests, common.get_cached_or_raw_digest(task, i, p, args))
end
end
if #digests > 0 then
- return digests,'string_list'
+ return digests, 'string_list'
end
return nil
local parts = task:get_parts() or E
local files = {}
- for _,p in ipairs(parts) do
+ for _, p in ipairs(parts) do
local fname = p:get_filename()
if fname then
table.insert(files, fname)
end
if #files > 0 then
- return files,'string_list'
+ return files, 'string_list'
end
return nil
local text_parts = task:get_text_parts() or E
local languages = {}
- for _,p in ipairs(text_parts) do
+ for _, p in ipairs(text_parts) do
local lang = p:get_language()
if lang then
table.insert(languages, lang)
end
if #languages > 0 then
- return languages,'string_list'
+ return languages, 'string_list'
end
return nil
-- Get helo value
['helo'] = {
['get_value'] = function(task)
- return task:get_helo(),'string'
+ return task:get_helo(), 'string'
end,
['description'] = 'Get helo value',
},
end
if args[2]:match('full') then
- return task:get_header_full(args[1], strong),'table_list'
+ return task:get_header_full(args[1], strong), 'table_list'
end
- return task:get_header(args[1], strong),'string'
+ return task:get_header(args[1], strong), 'string'
else
- return task:get_header(args[1]),'string'
+ return task:get_header(args[1]), 'string'
end
end,
['description'] = [[Get header with the name that is expected as an argument.
The optional second argument accepts list of flags:
- `full`: returns all headers with this name with all data (like task:get_header_full())
- `strong`: use case sensitive match when matching header's name]],
- ['args_schema'] = {ts.string,
- (ts.pattern("strong") + ts.pattern("full")):is_optional()}
+ ['args_schema'] = { ts.string,
+ (ts.pattern("strong") + ts.pattern("full")):is_optional() }
},
-- Get list of received headers (returns list of tables)
['received'] = {
return nil
end
if args[1] then
- return fun.map(function(r) return r[args[1]] end, rh), 'string_list'
+ return fun.map(function(r)
+ return r[args[1]]
+ end, rh), 'string_list'
end
- return rh,'table_list'
+ return rh, 'table_list'
end,
['description'] = [[Get list of received headers.
If no arguments specified, returns list of tables. Otherwise, selects a specific element,
return nil
end
if args[1] then
- return fun.map(function(r) return r[args[1]](r) end, urls), 'string_list'
+ return fun.map(function(r)
+ return r[args[1]](r)
+ end, urls), 'string_list'
end
- return urls,'userdata_list'
+ return urls, 'userdata_list'
end,
['description'] = [[Get list of all urls.
If no arguments specified, returns list of url objects. Otherwise, calls a specific method,
if not urls[1] then
return nil
end
- return urls,'userdata_list'
+ return urls, 'userdata_list'
end,
['description'] = [[Get most specific urls. Arguments are equal to the Lua API function]],
- ['args_schema'] = {ts.shape{
+ ['args_schema'] = { ts.shape {
limit = ts.number + ts.string / tonumber,
esld_limit = (ts.number + ts.string / tonumber):is_optional(),
exclude_flags = url_flags_ts,
flags = url_flags_ts,
- flags_mode = ts.one_of{'explicit'}:is_optional(),
+ flags_mode = ts.one_of { 'explicit' }:is_optional(),
prefix = ts.string:is_optional(),
need_content = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
need_emails = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
need_images = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
ignore_redirected = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
- }}
+ } }
},
['specific_urls_filter_map'] = {
['get_value'] = function(task, args)
if not urls[1] then
return nil
end
- return fun.filter(function(u) return map:get_key(tostring(u)) end, urls),'userdata_list'
+ return fun.filter(function(u)
+ return map:get_key(tostring(u))
+ end, urls), 'userdata_list'
end,
['description'] = [[Get most specific urls, filtered by some map. Arguments are equal to the Lua API function]],
- ['args_schema'] = {ts.string, ts.shape{
+ ['args_schema'] = { ts.string, ts.shape {
limit = ts.number + ts.string / tonumber,
esld_limit = (ts.number + ts.string / tonumber):is_optional(),
exclude_flags = url_flags_ts,
flags = url_flags_ts,
- flags_mode = ts.one_of{'explicit'}:is_optional(),
+ flags_mode = ts.one_of { 'explicit' }:is_optional(),
prefix = ts.string:is_optional(),
need_content = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
need_emails = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
need_images = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
ignore_redirected = (ts.boolean + ts.string / lua_util.toboolean):is_optional(),
- }}
+ } }
},
-- URLs filtered by flags
['urls_filtered'] = {
if not urls[1] then
return nil
end
- return urls,'userdata_list'
+ return urls, 'userdata_list'
end,
['description'] = [[Get list of all urls filtered by flags_include/exclude
(see rspamd_task:get_urls_filtered for description)]],
- ['args_schema'] = {ts.array_of{
+ ['args_schema'] = { ts.array_of {
url_flags_ts:is_optional(), url_flags_ts:is_optional()
- }}
+ } }
},
-- Get all emails
['emails'] = {
return nil
end
if args[1] then
- return fun.map(function(r) return r[args[1]](r) end, urls), 'string_list'
+ return fun.map(function(r)
+ return r[args[1]](r)
+ end, urls), 'string_list'
end
- return urls,'userdata_list'
+ return urls, 'userdata_list'
end,
['description'] = [[Get list of all emails.
If no arguments specified, returns list of url objects. Otherwise, calls a specific method,
['pool_var'] = {
['get_value'] = function(task, args)
local type = args[2] or 'string'
- return task:get_mempool():get_variable(args[1], type),(type)
+ return task:get_mempool():get_variable(args[1], type), (type)
end,
['description'] = [[Get specific pool var. The first argument must be variable name,
the second argument is optional and defines the type (string by default)]],
- ['args_schema'] = {ts.string, ts.string:is_optional()}
+ ['args_schema'] = { ts.string, ts.string:is_optional() }
},
-- Get value of specific key from task cache
['task_cache'] = {
end,
['description'] = [[Get value of specific key from task cache. The first argument must be
the key name]],
- ['args_schema'] = {ts.string}
+ ['args_schema'] = { ts.string }
},
-- Get specific HTTP request header. The first argument must be header name.
['request_header'] = {
['get_value'] = function(task, args)
local hdr = task:get_request_header(args[1])
if hdr then
- return hdr,'string'
+ return hdr, 'string'
end
return nil
end,
['description'] = [[Get specific HTTP request header.
The first argument must be header name.]],
- ['args_schema'] = {ts.string}
+ ['args_schema'] = { ts.string }
},
-- Get task date, optionally formatted
['time'] = {
['get_value'] = function(task, args)
local what = args[1] or 'message'
- local dt = task:get_date{format = what, gmt = true}
+ local dt = task:get_date { format = what, gmt = true }
if dt then
if args[2] then
-- Should be in format !xxx, as dt is in GMT
- return os.date(args[2], dt),'string'
+ return os.date(args[2], dt), 'string'
end
- return tostring(dt),'string'
+ return tostring(dt), 'string'
end
return nil
- `message`: timestamp as defined by `Date` header
The second argument is optional time format, see [os.date](http://pgl.yoyo.org/luai/i/os.date) description]],
- ['args_schema'] = {ts.one_of{'connect', 'message'}:is_optional(),
- ts.string:is_optional()}
+ ['args_schema'] = { ts.one_of { 'connect', 'message' }:is_optional(),
+ ts.string:is_optional() }
},
-- Get text words from a message
['words'] = {
- `norm`: normalised words (lowercased)
- `full`: list of tables
]],
- ['args_schema'] = { ts.one_of { 'stem', 'raw', 'norm', 'full' }:is_optional()},
+ ['args_schema'] = { ts.one_of { 'stem', 'raw', 'norm', 'full' }:is_optional() },
},
-- Get queue ID
['queueid'] = {
['get_value'] = function(task)
local queueid = task:get_queue_id()
- if queueid then return queueid,'string' end
+ if queueid then
+ return queueid, 'string'
+ end
return nil
end,
['description'] = [[Get queue ID]],
['uid'] = {
['get_value'] = function(task)
local uid = task:get_uid()
- if uid then return uid,'string' end
+ if uid then
+ return uid, 'string'
+ end
return nil
end,
['description'] = [[Get ID of the task being processed]],
['messageid'] = {
['get_value'] = function(task)
local mid = task:get_message_id()
- if mid then return mid,'string' end
+ if mid then
+ return mid, 'string'
+ end
return nil
end,
['description'] = [[Get message ID]],
['get_value'] = function(task, args)
local symbol = task:get_symbol(args[1], args[2])
if symbol then
- return symbol[1],'table'
+ return symbol[1], 'table'
end
end,
['description'] = 'Get specific symbol. The first argument must be the symbol name. ' ..
- 'The second argument is an optional shadow result name. ' ..
- 'Returns the symbol table. See task:get_symbol()',
- ['args_schema'] = {ts.string, ts.string:is_optional()}
+ 'The second argument is an optional shadow result name. ' ..
+ 'Returns the symbol table. See task:get_symbol()',
+ ['args_schema'] = { ts.string, ts.string:is_optional() }
},
-- Get full scan result
['scan_result'] = {
['get_value'] = function(task, args)
local res = task:get_metric_result(args[1])
if res then
- return res,'table'
+ return res, 'table'
end
end,
['description'] = 'Get full scan result (either default or shadow if shadow result name is specified)' ..
'Returns the result table. See task:get_metric_result()',
- ['args_schema'] = {ts.string:is_optional()}
+ ['args_schema'] = { ts.string:is_optional() }
},
-- Get list of metatokens as strings
['metatokens'] = {
if t == 'table' then
-- Table (very special)
if ud_or_table.value then
- return ud_or_table.value,'string'
+ return ud_or_table.value, 'string'
elseif ud_or_table.addr then
- return ud_or_table.addr,'string'
+ return ud_or_table.addr, 'string'
end
- return logger.slog("%s", ud_or_table),'string'
+ return logger.slog("%s", ud_or_table), 'string'
elseif (t == 'string' or t == 'text') and type(ud_or_table) == 'userdata' then
if ud_or_table.cookie and ud_or_table.cookie == text_cookie then
-- Preserve opaque
- return ud_or_table,'string'
+ return ud_or_table, 'string'
else
- return tostring(ud_or_table),'string'
+ return tostring(ud_or_table), 'string'
end
elseif t ~= 'nil' then
- return tostring(ud_or_table),'string'
+ return tostring(ud_or_table), 'string'
end
return nil
return pure_type(t)
end
- local input,etype = sel.selector.get_value(task, sel.selector.args)
+ local input, etype = sel.selector.get_value(task, sel.selector.args)
if not input then
lua_util.debugm(M, task, 'no value extracted for %s', sel.selector.name)
local first_elt = pipe[1]
if first_elt and (first_elt.method or
- fun.any(function(t) return t == 'userdata' or t == 'table' end, first_elt.types)) then
+ fun.any(function(t)
+ return t == 'userdata' or t == 'table'
+ end, first_elt.types)) then
-- Explicit conversion
local meth = first_elt
if meth.types[etype] then
lua_util.debugm(M, task, 'apply method `%s` to %s',
meth.name, etype)
- input,etype = meth.process(input, etype, meth.args)
+ input, etype = meth.process(input, etype, meth.args)
else
local pt = pure_type(etype)
meth.name, pt)
-- Map method to a list of inputs, excluding empty elements
-- We need to fold it down here to get a proper type resolution
- input = fun.totable(fun.filter(function(map_elt, _) return map_elt end,
+ input = fun.totable(fun.filter(function(map_elt, _)
+ return map_elt
+ end,
fun.map(function(list_elt)
local ret, ty = meth.process(list_elt, pt, meth.args)
etype = ty
etype = 'string'
else
lua_util.debugm(M, task, 'apply implicit map %s->string', pt)
- input = fun.filter(function(map_elt) return map_elt end,
+ input = fun.filter(function(map_elt)
+ return map_elt
+ end,
fun.map(function(list_elt)
local ret = implicit_tostring(pt, list_elt)
return ret
if pt and x.types['list'] then
-- Generic list processor
lua_util.debugm(M, task, 'apply list function `%s` to %s', x.name, t)
- return {x.process(value, t, x.args)}
+ return { x.process(value, t, x.args) }
elseif pt and x.map_type and x.types[pt] then
local map_type = x.map_type .. '_list'
lua_util.debugm(M, task, 'map `%s` to list of %s resulting %s',
x.name, pt, map_type)
-- Apply map, filtering empty values
return {
- fun.filter(function(map_elt) return map_elt end,
+ fun.filter(function(map_elt)
+ return map_elt
+ end,
fun.map(function(list_elt)
- if not list_elt then return nil end
+ if not list_elt then
+ return nil
+ end
local ret, _ = x.process(list_elt, pt, x.args)
return ret
end, value)),
end
lua_util.debugm(M, task, 'apply %s to %s', x.name, t)
- return {x.process(value, t, x.args)}
+ return { x.process(value, t, x.args) }
end
local res = fun.foldl(fold_function,
- {input, etype},
+ { input, etype },
pipe)
- if not res or not res[1] then return nil end -- Pipeline failed
+ if not res or not res[1] then
+ return nil
+ end -- Pipeline failed
if not allowed_type(res[2]) then
-- Search for implicit conversion
if pt then
lua_util.debugm(M, task, 'apply implicit map %s->string_list', pt)
- res[1] = fun.map(function(e) return implicit_tostring(pt, e) end, res[1])
+ res[1] = fun.map(function(e)
+ return implicit_tostring(pt, e)
+ end, res[1])
res[2] = 'string_list'
else
res[1] = implicit_tostring(res[2], res[1])
local function make_grammar()
local l = require "lpeg"
- local spc = l.S(" \t\n")^0
+ local spc = l.S(" \t\n") ^ 0
local cont = l.R("\128\191") -- continuation byte
local utf8_high = l.R("\194\223") * cont
+ l.R("\224\239") * cont * cont
+ l.R("\240\244") * cont * cont * cont
- local atom = l.C((l.R("az") + l.R("AZ") + l.R("09") + l.S("_-") + utf8_high)^1)
- local singlequoted_string = l.P "'" * l.C(((1 - l.S "'\r\n\f\\") + (l.P'\\' * 1))^0) * "'"
- local doublequoted_string = l.P '"' * l.C(((1 - l.S'"\r\n\f\\') + (l.P'\\' * 1))^0) * '"'
+ local atom = l.C((l.R("az") + l.R("AZ") + l.R("09") + l.S("_-") + utf8_high) ^ 1)
+ local singlequoted_string = l.P "'" * l.C(((1 - l.S "'\r\n\f\\") + (l.P '\\' * 1)) ^ 0) * "'"
+ local doublequoted_string = l.P '"' * l.C(((1 - l.S '"\r\n\f\\') + (l.P '\\' * 1)) ^ 0) * '"'
local argument = atom + singlequoted_string + doublequoted_string
local dot = l.P(".")
local semicolon = l.P(":")
local tbl_ebrace = spc * "}"
local ebrace = spc * ")"
local comma = spc * "," * spc
- local sel_separator = spc * l.S";*" * spc
+ local sel_separator = spc * l.S ";*" * spc
- return l.P{
+ return l.P {
"LIST";
- LIST = l.Ct(l.V("EXPR")) * (sel_separator * l.Ct(l.V("EXPR")))^0,
- EXPR = l.V("FUNCTION") * (semicolon * l.V("METHOD"))^-1 * (dot * l.V("PROCESSOR"))^0,
- PROCESSOR = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace)^0),
- FUNCTION = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace)^0),
- METHOD = l.Ct(atom / function(e) return '__' .. e end * spc * (obrace * l.V("ARG_LIST") * ebrace)^0),
- ARG_LIST = l.Ct((l.V("ARG") * comma^0)^0),
+ LIST = l.Ct(l.V("EXPR")) * (sel_separator * l.Ct(l.V("EXPR"))) ^ 0,
+ EXPR = l.V("FUNCTION") * (semicolon * l.V("METHOD")) ^ -1 * (dot * l.V("PROCESSOR")) ^ 0,
+ PROCESSOR = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0),
+ FUNCTION = l.Ct(atom * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0),
+ METHOD = l.Ct(atom / function(e)
+ return '__' .. e
+ end * spc * (obrace * l.V("ARG_LIST") * ebrace) ^ 0),
+ ARG_LIST = l.Ct((l.V("ARG") * comma ^ 0) ^ 0),
ARG = l.Cf(tbl_obrace * l.V("NAMED_ARG") * tbl_ebrace, rawset) + argument + l.V("LIST_ARGS"),
- NAMED_ARG = (l.Ct("") * l.Cg(argument * eqsign * (argument + l.V("LIST_ARGS")) * comma^0)^0),
+ NAMED_ARG = (l.Ct("") * l.Cg(argument * eqsign * (argument + l.V("LIST_ARGS")) * comma ^ 0) ^ 0),
LIST_ARGS = l.Ct(tbl_obrace * l.V("LIST_ARG") * tbl_ebrace),
- LIST_ARG = l.Cg(argument * comma^0)^0,
+ LIST_ARG = l.Cg(argument * comma ^ 0) ^ 0,
}
end
-- @function lua_selectors.parse_selector(cfg, str)
--]]
exports.parse_selector = function(cfg, str)
- local parsed = {parser:match(str)}
+ local parsed = { parser:match(str) }
local output = {}
- if not parsed or not parsed[1] then return nil end
+ if not parsed or not parsed[1] then
+ return nil
+ end
local function check_args(name, schema, args)
if schema then
if getmetatable(schema) then
-- Schema covers all arguments
- local res,err = schema:transform(args)
+ local res, err = schema:transform(args)
if not res then
logger.errx(rspamd_config, 'invalid arguments for %s: %s', name, err)
return false
else
- for i,elt in ipairs(res) do
+ for i, elt in ipairs(res) do
args[i] = elt
end
end
else
- for i,selt in ipairs(schema) do
- local res,err = selt:transform(args[i])
+ for i, selt in ipairs(schema) do
+ local res, err = selt:transform(args[i])
if err then
logger.errx(rspamd_config, 'invalid arguments for %s: argument number: %s, error: %s', name, i, err)
-- table of individual selectors
-- each selector: list of functions
-- each function: function name + optional list of arguments
- for _,sel in ipairs(parsed) do
+ for _, sel in ipairs(parsed) do
local res = {
selector = {},
processor_pipe = {},
local ret_type = type(ret)
- if ret_type == 'nil' then return nil end
+ if ret_type == 'nil' then
+ return nil
+ end
-- Now apply types heuristic
if ret_type == 'string' then
- return ret,'string'
+ return ret, 'string'
elseif ret_type == 'table' then
-- TODO: we need to ensure that 1) table is numeric 2) table has merely strings
- return ret,'string_list'
+ return ret, 'string_list'
else
return implicit_tostring(ret_type, ret)
end
exports.process_selectors = function(task, selectors_pipe)
local ret = {}
- for _,sel in ipairs(selectors_pipe) do
+ for _, sel in ipairs(selectors_pipe) do
local r = process_selector(task, sel)
-- If any element is nil, then the whole selector is nil
- if not r then return nil end
+ if not r then
+ return nil
+ end
table.insert(ret, r)
end
-- @function lua_selectors.combine_selectors(task, selectors, delimiter)
--]]
exports.combine_selectors = function(_, selectors, delimiter)
- if not delimiter then delimiter = '' end
+ if not delimiter then
+ delimiter = ''
+ end
- if not selectors then return nil end
+ if not selectors then
+ return nil
+ end
local have_tables, have_userdata
- for _,s in ipairs(selectors) do
+ for _, s in ipairs(selectors) do
if type(s) == 'table' then
have_tables = true
elseif type(s) == 'userdata' then
local tbl = {}
local res = {}
- for i,s in ipairs(selectors) do
+ for i, s in ipairs(selectors) do
if type(s) == 'string' then
rawset(tbl, i, fun.duplicate(s))
elseif type(s) == 'userdata' then
end
fun.each(function(...)
- table.insert(res, table.concat({...}, delimiter))
+ table.insert(res, table.concat({ ... }, delimiter))
end, fun.zip(lua_util.unpack(tbl)))
return res
local res = {}
local function fill(tbl)
- for _,s in ipairs(tbl) do
+ for _, s in ipairs(tbl) do
if type(s) == 'string' then
rawset(res, #res + 1, s)
elseif type(s) == 'userdata' then
tbl_len)
return
end
- for i=1,tbl_len,2 do
+ for i = 1, tbl_len, 2 do
local k = tostring(tbl[i])
local v = tbl[i + 1]
if type(v) == 'string' then
elseif type(v) == 'userdata' then
res[k] = tostring(v)
else
- res[k] = fun.totable(fun.map(function(elt) return tostring(elt) end, v))
+ res[k] = fun.totable(fun.map(function(elt)
+ return tostring(elt)
+ end, v))
end
end
end
end
local function display_selectors(tbl)
- return fun.tomap(fun.map(function(k,v)
+ return fun.tomap(fun.map(function(k, v)
return k, fun.tomap(fun.filter(function(kk, vv)
return type(vv) ~= 'function'
end, v))
},
['map_type'] = 'string',
['process'] = function(inp, _)
- return inp:lower(),'string'
+ return inp:lower(), 'string'
end,
['description'] = 'Returns the lowercased string',
},
['list'] = true,
},
['process'] = function(inp, t)
- return fun.head(inp),pure_type(t)
+ return fun.head(inp), pure_type(t)
end,
['description'] = 'Returns the first element',
},
['list'] = true,
},
['process'] = function(inp, t)
- return fun.nth(fun.length(inp), inp),pure_type(t)
+ return fun.nth(fun.length(inp), inp), pure_type(t)
end,
['description'] = 'Returns the last element',
},
['list'] = true,
},
['process'] = function(inp, t, args)
- return fun.nth(args[1] or 1, inp),pure_type(t)
+ return fun.nth(args[1] or 1, inp), pure_type(t)
end,
['description'] = 'Returns the nth element',
- ['args_schema'] = {ts.number + ts.string / tonumber}
+ ['args_schema'] = { ts.number + ts.string / tonumber }
},
['take_n'] = {
['types'] = {
['list'] = true,
},
['process'] = function(inp, t, args)
- return fun.take_n(args[1] or 1, inp),t
+ return fun.take_n(args[1] or 1, inp), t
end,
['description'] = 'Returns the n first elements',
- ['args_schema'] = {ts.number + ts.string / tonumber}
+ ['args_schema'] = { ts.number + ts.string / tonumber }
},
['drop_n'] = {
['types'] = {
['list'] = true,
},
['process'] = function(inp, t, args)
- return fun.drop_n(args[1] or 1, inp),t
+ return fun.drop_n(args[1] or 1, inp), t
end,
['description'] = 'Returns list without the first n elements',
- ['args_schema'] = {ts.number + ts.string / tonumber}
+ ['args_schema'] = { ts.number + ts.string / tonumber }
},
-- Joins strings into a single string using separator in the argument
['join'] = {
return table.concat(fun.totable(inp), args[1] or ''), 'string'
end,
['description'] = 'Joins strings into a single string using separator in the argument',
- ['args_schema'] = {ts.string:is_optional()}
+ ['args_schema'] = { ts.string:is_optional() }
},
-- Joins strings into a set of strings using N elements and a separator in the argument
['join_nth'] = {
local inp_t = fun.totable(inp)
local res = {}
- for i=1, #inp_t, step do
+ for i = 1, #inp_t, step do
table.insert(res, table.concat(inp_t, sep, i, i + step))
end
- return res,'string_list'
+ return res, 'string_list'
end,
['description'] = 'Joins strings into a set of strings using N elements and a separator in the argument',
- ['args_schema'] = {ts.number + ts.string / tonumber, ts.string:is_optional()}
+ ['args_schema'] = { ts.number + ts.string / tonumber, ts.string:is_optional() }
},
-- Joins tables into a table of strings
['join_tables'] = {
},
['process'] = function(inp, _, args)
local sep = args[1] or ''
- return fun.map(function(t) return table.concat(t, sep) end, inp), 'string_list'
+ return fun.map(function(t)
+ return table.concat(t, sep)
+ end, inp), 'string_list'
end,
['description'] = 'Joins tables into a table of strings',
- ['args_schema'] = {ts.string:is_optional()}
+ ['args_schema'] = { ts.string:is_optional() }
},
-- Sort strings
['sort'] = {
tmp[val] = true
end, inp)
- return fun.map(function(k, _) return k end, tmp), t
+ return fun.map(function(k, _)
+ return k
+ end, tmp), t
end,
['description'] = 'Returns a list of unique elements (using a hash table)',
},
},
['map_type'] = 'string',
['process'] = function(inp, _, args)
- return common.create_digest(inp, args),'string'
+ return common.create_digest(inp, args), 'string'
end,
['description'] = [[Create a digest from a string.
The first argument is encoding (`hex`, `base32` (and forms `bleach32`, `rbase32`), `base64`),
return inp:sub(start_pos, end_pos), 'string'
end,
['description'] = 'Extracts substring; the first argument is start, the second is the last (like in Lua)',
- ['args_schema'] = {(ts.number + ts.string / tonumber):is_optional(),
- (ts.number + ts.string / tonumber):is_optional()}
+ ['args_schema'] = { (ts.number + ts.string / tonumber):is_optional(),
+ (ts.number + ts.string / tonumber):is_optional() }
},
-- Prepends a string or a strings list
['prepend'] = {
end
end
flatten_table(res)
- return flattened_table,'string_list'
+ return flattened_table, 'string_list'
end
return nil
end,
['description'] = 'Regexp matching, returns all matches flattened in a single list',
- ['args_schema'] = {ts.string}
+ ['args_schema'] = { ts.string }
},
-- Returns a value if it exists in some map (or acts like a `filter` function)
['filter_map'] = {
local res = map:get_key(inp)
if res then
- return inp,t
+ return inp, t
end
return nil
end,
['description'] = 'Returns a value if it exists in some map (or acts like a `filter` function)',
- ['args_schema'] = {ts.string}
+ ['args_schema'] = { ts.string }
},
-- Returns a value if it exists in some map (or acts like a `filter` function)
['except_map'] = {
local res = map:get_key(inp)
if not res then
- return inp,t
+ return inp, t
end
return nil
end,
['description'] = 'Returns a value if it does not exists in some map (or acts like a `except` function)',
- ['args_schema'] = {ts.string}
+ ['args_schema'] = { ts.string }
},
-- Returns a value from some map corresponding to some key (or acts like a `map` function)
['apply_map'] = {
local res = map:get_key(inp)
if res then
- return res,t
+ return res, t
end
return nil
end,
['description'] = 'Returns a value from some map corresponding to some key (or acts like a `map` function)',
- ['args_schema'] = {ts.string}
+ ['args_schema'] = { ts.string }
},
-- Drops input value and return values from function's arguments or an empty string
['id'] = {
['map_type'] = 'string',
['process'] = function(_, _, args)
if args[1] and args[2] then
- return fun.map(tostring, args),'string_list'
+ return fun.map(tostring, args), 'string_list'
elseif args[1] then
- return args[1],'string'
+ return args[1], 'string'
end
- return '','string'
+ return '', 'string'
end,
['description'] = 'Drops input value and return values from function\'s arguments or an empty string',
['args_schema'] = (ts.string + ts.array_of(ts.string)):is_optional()
['map_type'] = 'string',
['process'] = function(inp, _, args)
if inp == args[1] then
- return inp,'string'
+ return inp, 'string'
end
return nil
end,
['description'] = [[Boolean function equal.
Returns either nil or its argument if input is equal to argument]],
- ['args_schema'] = {ts.string}
+ ['args_schema'] = { ts.string }
},
-- Boolean function in, returns either nil or its input if input is in args list
['in'] = {
},
['map_type'] = 'string',
['process'] = function(inp, t, args)
- for _,a in ipairs(args) do if a == inp then return inp,t end end
+ for _, a in ipairs(args) do
+ if a == inp then
+ return inp, t
+ end
+ end
return nil
end,
['description'] = [[Boolean function in.
},
['map_type'] = 'string',
['process'] = function(inp, t, args)
- for _,a in ipairs(args) do if a == inp then return nil end end
- return inp,t
+ for _, a in ipairs(args) do
+ if a == inp then
+ return nil
+ end
+ end
+ return inp, t
end,
['description'] = [[Boolean function not in.
Returns either nil or its input if input is not in args list]],
if inp then
return nil
else
- return (args[1] or 'true'),'string'
+ return (args[1] or 'true'), 'string'
end
end,
['description'] = [[Inverses input.
Empty string comes the first argument or 'true', non-empty string comes nil]],
- ['args_schema'] = {ts.string:is_optional()}
+ ['args_schema'] = { ts.string:is_optional() }
},
['ipmask'] = {
['types'] = {
if ip:get_version() == 4 then
local mask = tonumber(args[1])
- return ip:apply_mask(mask):to_string(),'string'
+ return ip:apply_mask(mask):to_string(), 'string'
else
-- IPv6 takes the second argument or the first one...
local mask_str = args[2] or args[1]
local mask = tonumber(mask_str)
- return ip:apply_mask(mask):to_string(),'string'
+ return ip:apply_mask(mask):to_string(), 'string'
end
end,
['description'] = 'Applies mask to IP address.' ..
- ' The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.',
- ['args_schema'] = {(ts.number + ts.string / tonumber),
- (ts.number + ts.string / tonumber):is_optional()}
+ ' The first argument is the mask for IPv4 addresses, the second is the mask for IPv6 addresses.',
+ ['args_schema'] = { (ts.number + ts.string / tonumber),
+ (ts.number + ts.string / tonumber):is_optional() }
},
-- Returns the string(s) with all non ascii chars replaced
['to_ascii'] = {
['process'] = function(inp, _, args)
if type(inp) == 'table' then
return fun.map(
- function(s)
- return string.gsub(tostring(s), '[\128-\255]', args[1] or '?')
- end, inp), 'string_list'
+ function(s)
+ return string.gsub(tostring(s), '[\128-\255]', args[1] or '?')
+ end, inp), 'string_list'
else
return string.gsub(tostring(inp), '[\128-\255]', '?'), 'string'
end
end,
['description'] = 'Returns the string with all non-ascii bytes replaced with the character ' ..
- 'given as second argument or `?`',
- ['args_schema'] = {ts.string:is_optional()}
+ 'given as second argument or `?`',
+ ['args_schema'] = { ts.string:is_optional() }
},
-- Extracts tld from a hostname
['get_tld'] = {
},
['map_type'] = 'string',
['process'] = function(inp, _, _)
- return rspamd_util.get_tld(inp),'string'
+ return rspamd_util.get_tld(inp), 'string'
end,
['description'] = 'Extracts tld from a hostname represented as a string',
['args_schema'] = {}
return rspamd_util.pack(string.rep(fmt, #res), lua_util.unpack(res)), 'string'
end,
['description'] = 'Converts a list of strings to numbers & returns a packed string',
- ['args_schema'] = {ts.string:is_optional()}
+ ['args_schema'] = { ts.string:is_optional() }
},
-- Filter nils from a list
['filter_string_nils'] = {
['string_list'] = true
},
['process'] = function(inp, _, _)
- return fun.filter(function(val) return type(val) == 'string' and val ~= 'nil' end, inp), 'string_list'
+ return fun.filter(function(val)
+ return type(val) == 'string' and val ~= 'nil'
+ end, inp), 'string_list'
end,
['description'] = 'Removes all nils from a list of strings (when converted implicitly)',
['args_schema'] = {}
},
['process'] = function(inp, _, args)
local res = {}
- for _,arg in ipairs(args) do
+ for _, arg in ipairs(args) do
local meth = inp[arg]
local ret = meth(inp)
- if ret then table.insert(res, tostring(ret)) end
+ if ret then
+ table.insert(res, tostring(ret))
+ end
end
- return res,'string_list'
+ return res, 'string_list'
end,
['description'] = 'Apply a list of method calls to the userdata object',
}
default_symbols = fun.totable(fun.filter(function(_, v)
return not v.allowed_ids or #v.allowed_ids == 0 or v.flags.explicit_disable
- end,all_symbols))
+ end, all_symbols))
local explicit_symbols = lua_util.keys(fun.filter(function(k, v)
return v.flags.explicit_disable
local symnames = lua_util.list_to_hash(lua_util.keys(all_symbols))
- for _,set in pairs(known_ids) do
+ for _, set in pairs(known_ids) do
local s = set.settings.apply or {}
set.symbols = lua_util.shallowcopy(symnames)
local enabled_symbols = {}
-- Remove all symbols from set.symbols aside of explicit_disable symbols
set.symbols = lua_util.list_to_hash(explicit_symbols)
seen_enabled = true
- for _,sym in ipairs(s.symbols_enabled) do
+ for _, sym in ipairs(s.symbols_enabled) do
enabled_symbols[sym] = true
set.symbols[sym] = true
end
end
if s.groups_enabled then
seen_enabled = true
- for _,gr in ipairs(s.groups_enabled) do
+ for _, gr in ipairs(s.groups_enabled) do
local syms = rspamd_config:get_group_symbols(gr)
if syms then
- for _,sym in ipairs(syms) do
+ for _, sym in ipairs(syms) do
enabled_symbols[sym] = true
set.symbols[sym] = true
end
-- Disabled map
if s.symbols_disabled then
seen_disabled = true
- for _,sym in ipairs(s.symbols_disabled) do
+ for _, sym in ipairs(s.symbols_disabled) do
disabled_symbols[sym] = true
set.symbols[sym] = false
end
end
if s.groups_disabled then
seen_disabled = true
- for _,gr in ipairs(s.groups_disabled) do
+ for _, gr in ipairs(s.groups_disabled) do
local syms = rspamd_config:get_group_symbols(gr)
if syms then
- for _,sym in ipairs(syms) do
+ for _, sym in ipairs(syms) do
disabled_symbols[sym] = true
set.symbols[sym] = false
end
end
-- Deal with complexity to avoid mess in C
- if not seen_enabled then enabled_symbols = nil end
- if not seen_disabled then disabled_symbols = nil end
+ if not seen_enabled then
+ enabled_symbols = nil
+ end
+ if not seen_disabled then
+ disabled_symbols = nil
+ end
if enabled_symbols or disabled_symbols then
-- Specify what symbols are really enabled for this settings id
end
-- We now iterate over all symbols and check for allowed_ids/forbidden_ids
- for k,v in pairs(all_symbols) do
+ for k, v in pairs(all_symbols) do
if v.allowed_ids and not v.flags.explicit_disable then
- for _,id in ipairs(v.allowed_ids) do
+ for _, id in ipairs(v.allowed_ids) do
if known_ids[id] then
local set = known_ids[id]
if not set.has_specific_symbols then
end
end
if v.forbidden_ids then
- for _,id in ipairs(v.forbidden_ids) do
+ for _, id in ipairs(v.forbidden_ids) do
if known_ids[id] then
local set = known_ids[id]
if not set.has_specific_symbols then
end
-- Now we create lists of symbols for each settings and digest
- for _,set in pairs(known_ids) do
- set.symbols = lua_util.keys(fun.filter(function(_, v) return v end, set.symbols))
+ for _, set in pairs(known_ids) do
+ set.symbols = lua_util.keys(fun.filter(function(_, v)
+ return v
+ end, set.symbols))
table.sort(set.symbols)
set.digest = lua_util.table_digest(set.symbols)
end
if not apply.scores then
apply.scores = {}
end
- for k,v in pairs(senabled) do
+ for k, v in pairs(senabled) do
if tonumber(v) then
-- Move to symbols as well
apply.scores[k] = tonumber(v)
if apply.symbols then
-- Check if added symbols are enabled
- for k,v in pairs(apply.symbols) do
+ for k, v in pairs(apply.symbols) do
local s
-- Check if we have ["sym1", "sym2" ...] or {"sym1": xx, "sym2": yy}
if type(k) == 'string' then
if not from_postload and not post_init_added then
-- Use high priority to ensure that settings are initialised early but not before all
-- plugins are loaded
- rspamd_config:add_post_init(function () register_settings_cb(true) end, 150)
+ rspamd_config:add_post_init(function()
+ register_settings_cb(true)
+ end, 150)
rspamd_config:add_config_unload(function()
if post_init_added then
known_ids = {}
exports.register_settings_id = register_settings_id
-
local function settings_by_id(id)
if not post_init_performed then
register_settings_cb(false)
return known_ids[id]
end
-
exports.settings_by_id = settings_by_id
exports.all_settings = function()
if not post_init_performed then
wantcode = wantcode or '2'
if merr then
callback(false, string.format('error on stage %s: %s',
- stage, merr))
+ stage, merr))
if conn then
conn:close()
end
local function data_done_cb(merr, mdata)
if no_error_read(merr, mdata, '3') then
if type(message) == 'string' or type(message) == 'userdata' then
- conn:add_write(pre_quit_cb, {message, CRLF.. '.' .. CRLF})
+ conn:add_write(pre_quit_cb, { message, CRLF .. '.' .. CRLF })
else
- table.insert(message, CRLF.. '.' .. CRLF)
+ table.insert(message, CRLF .. '.' .. CRLF)
conn:add_write(pre_quit_cb, message)
end
end
-- RCPT phase
local next_recipient
local function rcpt_done_cb_gen(i)
- return function (merr, mdata)
+ return function(merr, mdata)
if no_error_read(merr, mdata) then
if i == #opts.recipients then
conn:add_write(data_cb, 'DATA' .. CRLF)
end
local function rcpt_cb_gen(i)
- return function (merr, _)
+ return function(merr, _)
if no_error_write(merr, '2') then
conn:add_read(rcpt_done_cb_gen(i), CRLF)
end
if no_error_read(err, data) then
stage = 'helo'
conn:add_write(hello_cb, string.format('HELO %s%s',
- opts.helo, CRLF))
+ opts.helo, CRLF))
end
end
if type(opts.recipients) == 'string' then
- opts.recipients = {opts.recipients}
+ opts.recipients = { opts.recipients }
end
local tcp_opts = lua_util.shallowcopy(opts)
local N = "stat_tools" -- luacheck: ignore (maybe unused)
-- Performs synchronous conversion of redis schema
-local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire)
+local function convert_bayes_schema(redis_params, symbol_spam, symbol_ham, expire)
-- Old schema is the following one:
-- Keys are named <symbol>[<user>]
-- So we can expire individual records, measure most popular elements by zranges,
-- add new fields, such as tokens etc
- local res,conn = lua_redis.redis_connect_sync(redis_params, true)
+ local res, conn = lua_redis.redis_connect_sync(redis_params, true)
if not res then
logger.errx("cannot connect to redis server")
return nconverted
]]
- conn:add_cmd('EVAL', {lua_script, '3', symbol_spam, 'S', tostring(expire)})
+ conn:add_cmd('EVAL', { lua_script, '3', symbol_spam, 'S', tostring(expire) })
local ret
ret, res = conn:exec()
logger.messagex('converted %s elements from symbol %s', res, symbol_spam)
end
- conn:add_cmd('EVAL', {lua_script, '3', symbol_ham, 'H', tostring(expire)})
+ conn:add_cmd('EVAL', { lua_script, '3', symbol_ham, 'H', tostring(expire) })
ret, res = conn:exec()
if not ret then
end
]]
- conn:add_cmd('EVAL', {lua_script, '2', symbol_spam, 'learns_spam'})
- ret,res = conn:exec()
+ conn:add_cmd('EVAL', { lua_script, '2', symbol_spam, 'learns_spam' })
+ ret, res = conn:exec()
if not ret then
logger.errx('error converting metadata for symbol %s: %s', symbol_spam, res)
return false
end
- conn:add_cmd('EVAL', {lua_script, '2', symbol_ham, 'learns_ham'})
+ conn:add_cmd('EVAL', { lua_script, '2', symbol_ham, 'learns_ham' })
ret, res = conn:exec()
if not ret then
-- learn_cache_ham - name for sqlite database with ham learn cache
-- reset_previous - if true, then the old database is flushed (slow)
local function convert_sqlite_to_redis(redis_params,
- sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham,
- learn_cache_db, expire, reset_previous)
+ sqlite_db_spam, sqlite_db_ham, symbol_spam, symbol_ham,
+ learn_cache_db, expire, reset_previous)
local nusers = 0
local lim = 1000 -- Update each 1000 tokens
local users_map = {}
return false
end
- local res,conn = lua_redis.redis_connect_sync(redis_params, true)
+ local res, conn = lua_redis.redis_connect_sync(redis_params, true)
if not res then
logger.errx("cannot connect to redis server")
end
]]
-- Common keys
- for _,sym in ipairs({symbol_spam, symbol_ham}) do
+ for _, sym in ipairs({ symbol_spam, symbol_ham }) do
logger.messagex('Cleaning up old data for %s', sym)
- conn:add_cmd('EVAL', {script, '1', sym})
+ conn:add_cmd('EVAL', { script, '1', sym })
conn:exec()
- conn:add_cmd('DEL', {sym .. "_version"})
- conn:add_cmd('DEL', {sym .. "_keys"})
+ conn:add_cmd('DEL', { sym .. "_version" })
+ conn:add_cmd('DEL', { sym .. "_keys" })
conn:exec()
end
if learn_cache_db then
-- Cleanup learned_cache
logger.messagex('Cleaning up old data learned cache')
- conn:add_cmd('DEL', {"learned_ids"})
+ conn:add_cmd('DEL', { "learned_ids" })
conn:exec()
end
end
if is_spam then
hash_key = 'S'
end
- for _,tok in ipairs(tokens) do
+ for _, tok in ipairs(tokens) do
-- tok schema:
-- tok[1] = token_id (uint64 represented as a string)
-- tok[2] = token value (number)
-- tok[3] = user_map[user_id] or ''
local rkey = string.format('%s%s_%s', prefix, tok[3], tok[1])
- conn:add_cmd('HINCRBYFLOAT', {rkey, hash_key, tostring(tok[2])})
+ conn:add_cmd('HINCRBYFLOAT', { rkey, hash_key, tostring(tok[2]) })
if expire and expire ~= 0 then
- conn:add_cmd('EXPIRE', {rkey, tostring(expire)})
+ conn:add_cmd('EXPIRE', { rkey, tostring(expire) })
end
end
user = users_map[row.user]
end
- table.insert(tokens, {row.token, row.value, user})
+ table.insert(tokens, { row.token, row.value, user })
num = num + 1
total = total + 1
if num > lim then
-- TODO: we use the default 'RS' prefix, it can be false in case of
-- classifiers with labels
- local ret,err_str = send_batch(tokens, 'RS')
+ local ret, err_str = send_batch(tokens, 'RS')
if not ret then
logger.errx('Cannot send tokens to the redis server: ' .. err_str)
db:sql('COMMIT;')
end
-- Last batch
if #tokens > 0 then
- local ret,err_str = send_batch(tokens, 'RS')
+ local ret, err_str = send_batch(tokens, 'RS')
if not ret then
logger.errx('Cannot send tokens to the redis server: ' .. err_str)
db:sql('COMMIT;')
learns_elt = "learns_spam"
end
- for id,learned in pairs(learns) do
+ for id, learned in pairs(learns) do
local user = users_map[id]
- if not conn:add_cmd('HSET', {'RS' .. user, learns_elt, learned}) then
+ if not conn:add_cmd('HSET', { 'RS' .. user, learns_elt, learned }) then
logger.errx('Cannot update learns for user: ' .. user)
return false
end
- if not conn:add_cmd('SADD', {symbol .. '_keys', 'RS' .. user}) then
+ if not conn:add_cmd('SADD', { symbol .. '_keys', 'RS' .. user }) then
logger.errx('Cannot update learns for user: ' .. user)
return false
end
end
-- Set version
- conn:add_cmd('SET', {symbol..'_version', '2'})
+ conn:add_cmd('SET', { symbol .. '_version', '2' })
return conn:exec()
end
is_spam = '1'
end
- if not conn:add_cmd('HSET', {'learned_ids', digest, is_spam}) then
+ if not conn:add_cmd('HSET', { 'learned_ids', digest, is_spam }) then
logger.errx('Cannot add hash: ' .. digest)
ret = false
else
if ret then
logger.messagex('Converted %s cached items from sqlite3 learned cache to redis',
- total)
+ total)
else
logger.errx('Error occurred during sending data to redis')
end
end
local statfiles = cls.statfile
- for _,stf in ipairs(statfiles) do
+ for _, stf in ipairs(statfiles) do
local path = (stf.file or stf.path or stf.db or stf.dbname)
local symbol = stf.symbol or 'undefined'
if classifier then
if classifier[1] then
- for _,cls in ipairs(classifier) do
- if cls.bayes then cls = cls.bayes end
+ for _, cls in ipairs(classifier) do
+ if cls.bayes then
+ cls = cls.bayes
+ end
if cls.backend and cls.backend == 'sqlite3' then
parse_classifier(cls)
end
if classifier.bayes then
classifier = classifier.bayes
if classifier[1] then
- for _,cls in ipairs(classifier) do
+ for _, cls in ipairs(classifier) do
if cls.backend and cls.backend == 'sqlite3' then
parse_classifier(cls)
end
result.expire = expire
end
- return {classifier = {bayes = result}}
+ return { classifier = { bayes = result } }
end
exports.redis_classifier_from_sqlite = redis_classifier_from_sqlite
-- Postprocess classify_headers
local classify_headers_parsed = {}
- for _,v in ipairs(res_config.classify_headers) do
+ for _, v in ipairs(res_config.classify_headers) do
local s1, s2 = v:match("^([A-Z])[^%-]+%-([A-Z]).*$")
local hname
if classify_headers_parsed[hname] then
table.insert(classify_headers_parsed[hname], v)
else
- classify_headers_parsed[hname] = {v}
+ classify_headers_parsed[hname] = { v }
end
end
local empty_html = false
local online_text = false
- for _,part in ipairs(parts) do
+ for _, part in ipairs(parts) do
local fname = part:get_filename()
local sz = part:get_length()
end
]]--
- for k,hdrs in pairs(cf.classify_headers_parsed) do
- for _,hname in ipairs(hdrs) do
+ for k, hdrs in pairs(cf.classify_headers_parsed) do
+ for _, hname in ipairs(hdrs) do
local value = task:get_header(hname)
if value then
local function get_meta_stat_tokens(task, res, i)
local day_and_hour = os.date('%u:%H',
- task:get_date{format = 'message', gmt = true})
+ task:get_date { format = 'message', gmt = true })
rawset(res, i, string.format("#dt:%s", day_and_hour))
lua_util.debugm("bayes", task, "added day_of_week token: %s",
res[i])
local trace = task:get_symbol('DKIM_TRACE')
local dkim_opts = trace[1]['options']
if dkim_opts then
- for _,o in ipairs(dkim_opts) do
+ for _, o in ipairs(dkim_opts) do
local check_res = string.sub(o, -1)
local domain = string.sub(o, 1, -3)
if aur then
local spf = aur:match('spf=([a-z]+)')
- local dkim,dkim_domain = aur:match('dkim=([a-z]+) header.d=([a-z.%-]+)')
-
+ local dkim, dkim_domain = aur:match('dkim=([a-z]+) header.d=([a-z.%-]+)')
if spf then
table.insert(pol, 's=' .. spf)
if cf.classify_images then
local images = task:get_images() or E
- for _,img in ipairs(images) do
+ for _, img in ipairs(images) do
rawset(res, i, "image")
i = i + 1
rawset(res, i, tostring(img:get_height()))
end
if cf.classify_urls then
- local urls = lua_util.extract_specific_urls{task = task, limit = 5, esld_limit = 1}
+ local urls = lua_util.extract_specific_urls { task = task, limit = 5, esld_limit = 1 }
if urls then
- for _,u in ipairs(urls) do
+ for _, u in ipairs(urls) do
rawset(res, i, string.format("#u:%s", u:get_tld()))
lua_util.debugm("bayes", task, "added url token: %s",
res[i])
local exports = {}
local N = 'tcp_sync'
-local tcp_sync = {_conn = nil, _data = '', _eof = false, _addr = ''}
+local tcp_sync = { _conn = nil, _data = '', _eof = false, _addr = '' }
local metatable = {
- __tostring = function (self)
+ __tostring = function(self)
return "class {tcp_sync connect to: " .. self._addr .. "}"
end
}
--
--]]
function tcp_sync:read_until(pattern)
- repeat
+ repeat
local pos_start, pos_end = self._data:find(pattern, 1, true)
if pos_start then
local data = self._data:sub(1, pos_start - 1)
return self._conn:shutdown()
end
-exports.connect = function (args)
+exports.connect = function(args)
local is_ok, connection = rspamd_tcp.connect_sync(args)
if not is_ok then
return is_ok, connection
lua_util.debugm(N, log_obj, 'found compose tld for %s (host = %s)',
url_tld, url_host)
- for _,excl in ipairs(tld_elt.except_rules) do
- local matched,ret = excl[2](url_tld, url_host)
+ for _, excl in ipairs(tld_elt.except_rules) do
+ local matched, ret = excl[2](url_tld, url_host)
if matched then
lua_util.debugm(N, log_obj, 'found compose exclusion for %s (%s) -> %s',
url_host, excl[1], ret)
if matches then
local lua_pat_idx = math.huge
- for m,_ in pairs(matches) do
+ for m, _ in pairs(matches) do
if m < lua_pat_idx then
lua_pat_idx = m
end
if #tld_elt.compose_rules >= lua_pat_idx then
local lua_pat = tld_elt.compose_rules[lua_pat_idx]
- local matched,ret = lua_pat[2](url_tld, url_host)
+ local matched, ret = lua_pat[2](url_tld, url_host)
if not matched then
lua_util.debugm(N, log_obj, 'NOT found compose inclusion for %s (%s) -> %s',
end
else
-- Match one by one
- for _,lua_pat in ipairs(tld_elt.compose_rules) do
- local matched,ret = lua_pat[2](url_tld, url_host)
+ for _, lua_pat in ipairs(tld_elt.compose_rules) do
+ local matched, ret = lua_pat[2](url_tld, url_host)
if matched then
lua_util.debugm(N, log_obj, 'found compose inclusion for %s (%s) -> %s',
url_host, lua_pat[1], ret)
return function(_, host)
local matches = pat:search(host, false, true)
if matches then
- return true,matches[1][2]
+ return true, matches[1][2]
end
return false
pat = rspamd_regexp.create(tld_pattern_transform(pat))
return function(tld, host)
if pat:search(host) then
- return true,tld
+ return true, tld
end
return false
local lpeg = require "lpeg"
local singleline_comment = lpeg.P '#' * (1 - lpeg.S '\r\n\f') ^ 0
- local comments_strip_grammar = lpeg.C((1 - lpeg.P '#') ^ 1) * lpeg.S(' \t')^0 * singleline_comment^0
+ local comments_strip_grammar = lpeg.C((1 - lpeg.P '#') ^ 1) * lpeg.S(' \t') ^ 0 * singleline_comment ^ 0
local function process_tld_rule(tld_elt, l)
if l:sub(1, 1) == '!' then
-- Exclusion elt
- table.insert(tld_elt.except_rules, {l, exclude_elt_gen(l:sub(2))})
+ table.insert(tld_elt.except_rules, { l, exclude_elt_gen(l:sub(2)) })
else
- table.insert(tld_elt.compose_rules, {l, include_elt_gen(l)})
+ table.insert(tld_elt.compose_rules, { l, include_elt_gen(l) })
end
end
local function process_map_line(l)
-- Skip empty lines and comments
- if #l == 0 then return end
+ if #l == 0 then
+ return
+ end
l = comments_strip_grammar:match(l)
- if not l or #l == 0 then return end
+ if not l or #l == 0 then
+ return
+ end
-- Get TLD
local tld = rspamd_util.get_tld(l)
end
local multipattern_threshold = 1
- for tld,tld_elt in pairs(self.tlds) do
+ for tld, tld_elt in pairs(self.tlds) do
-- Sort patterns to have longest labels before shortest ones,
-- so we can ensure that they match before
table.sort(tld_elt.compose_rules, function(e1, e2)
- local _,ndots1 = string.gsub(e1[1], '(%.)', '')
- local _,ndots2 = string.gsub(e2[1], '(%.)', '')
+ local _, ndots1 = string.gsub(e1[1], '(%.)', '')
+ local _, ndots2 = string.gsub(e2[1], '(%.)', '')
return ndots1 > ndots2
end)
tlds = {},
}
- map = cfg:add_map{
+ map = cfg:add_map {
type = 'callback',
description = 'URL compose map',
url = map_obj,
- callback = function(input) compose_map_cb(ret, input) end,
+ callback = function(input)
+ compose_map_cb(ret, input)
+ end,
opaque_data = true,
}
if result then
if result.passthrough then
- return 'passthrough',nil
+ return 'passthrough', nil
end
local score = result.score
local action = result.action
if action == 'reject' and result.npositive > 1 then
- return 'spam',score
+ return 'spam', score
elseif action == 'no action' then
if score < 0 or result.nnegative > 3 then
- return 'ham',score
+ return 'ham', score
end
else
-- All colors of junk
if action == 'add header' or action == 'rewrite subject' then
if result.npositive > 2 then
- return 'junk',score
+ return 'junk', score
end
end
end
- return 'uncertain',score
+ return 'uncertain', score
end
end
exports.set_verdict_table = function(verdict_tbl, what)
assert(type(verdict_tbl) == 'table' and
- type(verdict_tbl.callback) == 'function' and
- type(verdict_tbl.possible_verdicts) == 'table')
+ type(verdict_tbl.callback) == 'function' and
+ type(verdict_tbl.possible_verdicts) == 'table')
if not what then
-- Default verdict
exports.adjust_passthrough_action = function(task)
local action = task:get_metric_action()
if action == 'soft reject' then
- local has_pr,_,_,module = task:has_pre_result()
+ local has_pr, _, _, module = task:has_pre_result()
if has_pr and module then
action = module
-- Returns a key used to be inserted into dmarc report sample
-exports.dmarc_report = function (task, settings, data)
+exports.dmarc_report = function(task, settings, data)
local rspamd_lua_utils = require "lua_util"
local E = {}
local res = table.concat({
ip, data.spf_ok, data.dkim_ok,
disposition_to_return, (data.sampled_out and 'sampled_out' or ''), data.domain,
- dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, data.spf_domain, data.spf_result}, ',')
+ dkim_pass, dkim_fail, dkim_temperror, dkim_permerror, data.spf_domain, data.spf_result }, ',')
return res
end
-
exports.gen_munging_callback = function(munging_opts, settings)
local rspamd_util = require "rspamd_util"
local lua_mime = require "lua_mime"
- return function (task)
+ return function(task)
if munging_opts.mitigate_allow_only then
if not task:has_symbol(settings.symbols.allow) then
lua_util.debugm(N, task, 'skip munging, no %s symbol',
end
end
if munging_opts.mitigate_strict_only then
- local s = task:get_symbol(settings.symbols.allow) or {[1] = {}}
+ local s = task:get_symbol(settings.symbols.allow) or { [1] = {} }
local sopts = s[1].options or {}
local seen_strict
- for _,o in ipairs(sopts) do
+ for _, o in ipairs(sopts) do
if o == 'reject' or o == 'quarantine' then
seen_strict = true
break
end
end
if munging_opts.munge_map_condition then
- local accepted,trace = munging_opts.munge_map_condition:process(task)
+ local accepted, trace = munging_opts.munge_map_condition:process(task)
if not accepted then
lua_util.debugm(task, 'skip munging, maps condition not satisfied: (%s)',
trace)
end
end
-- Now, look for domain for munging
- local mr = task:get_recipients({ 'mime', 'orig'})
+ local mr = task:get_recipients({ 'mime', 'orig' })
local rcpt_found
if mr then
- for _,r in ipairs(mr) do
+ for _, r in ipairs(mr) do
if r.domain and munging_opts.list_map:get_key(r.addr) then
rcpt_found = r
break
return
end
- local from = task:get_from({ 'mime', 'orig'})
+ local from = task:get_from({ 'mime', 'orig' })
if not from or not from[1] then
lua_util.debugm(task, 'skip munging, from is bad')
local add_hdrs = {
['From'] = { order = 1, value = hdr_encoded },
}
- local remove_hdrs = {['From'] = 0}
+ local remove_hdrs = { ['From'] = 0 }
local nreply = from.addr
if munging_opts.reply_goes_to_list then
remove_hdrs['Reply-To'] = 1
end
- add_hdrs['Reply-To'] = {order = 0, value = nreply}
+ add_hdrs['Reply-To'] = { order = 0, value = nreply }
- add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded}
+ add_hdrs['X-Original-From'] = { order = 0, value = orig_from_encoded }
lua_mime.modify_headers(task, {
remove = remove_hdrs,
add = add_hdrs
local function gen_dmarc_grammar()
local lpeg = require "lpeg"
lpeg.locale(lpeg)
- local space = lpeg.space^0
- local name = lpeg.C(lpeg.alpha^1) * space
- local sep = space * (lpeg.S("\\;") * space) + (lpeg.space^1)
- local value = lpeg.C(lpeg.P(lpeg.graph - sep)^1)
- local pair = lpeg.Cg(name * "=" * space * value) * sep^-1
- local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset)
+ local space = lpeg.space ^ 0
+ local name = lpeg.C(lpeg.alpha ^ 1) * space
+ local sep = space * (lpeg.S("\\;") * space) + (lpeg.space ^ 1)
+ local value = lpeg.C(lpeg.P(lpeg.graph - sep) ^ 1)
+ local pair = lpeg.Cg(name * "=" * space * value) * sep ^ -1
+ local list = lpeg.Cf(lpeg.Ct("") * pair ^ 0, rawset)
local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("DMARC1")
local record = version * sep * list
result.strict_dkim = true
elseif dkim_pol ~= 'r' then
failed_policy = 'adkim tag has invalid value: ' .. dkim_pol
- return false,failed_policy
+ return false, failed_policy
end
end
result.strict_spf = true
elseif spf_pol ~= 'r' then
failed_policy = 'aspf tag has invalid value: ' .. spf_pol
- return false,failed_policy
+ return false, failed_policy
end
end
result.dmarc_policy = 'quarantine'
elseif (policy ~= 'none') then
failed_policy = 'p tag has invalid value: ' .. policy
- return false,failed_policy
+ return false, failed_policy
end
end
result.dmarc_policy = 'none'
elseif (subdomain_policy ~= 'none') then
failed_policy = 'sp tag has invalid value: ' .. subdomain_policy
- return false,failed_policy
+ return false, failed_policy
end
end
result.pct = elts['pct']
end
result.raw_elts = elts
else
- return false,false -- Ignore garbage
+ return false, false -- Ignore garbage
end
return true, result
settings = lua_util.override_defaults(settings, module_config)
local redis_params = lua_redis.parse_redis_server('neural')
-
local redis_lua_script_vectors_len = "neural_train_size.lua"
local redis_lua_script_maybe_invalidate = "neural_maybe_invalidate.lua"
local redis_lua_script_maybe_lock = "neural_maybe_lock.lua"
local function load_scripts()
redis_script_id.vectors_len = lua_redis.load_redis_script_from_file(redis_lua_script_vectors_len,
- redis_params)
+ redis_params)
redis_script_id.maybe_invalidate = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_invalidate,
- redis_params)
+ redis_params)
redis_script_id.maybe_lock = lua_redis.load_redis_script_from_file(redis_lua_script_maybe_lock,
- redis_params)
+ redis_params)
redis_script_id.save_unlock = lua_redis.load_redis_script_from_file(redis_lua_script_save_unlock,
- redis_params)
+ redis_params)
end
local function create_ann(n, nlayers, rule)
- -- We ignore number of layers so far when using kann
+ -- We ignore number of layers so far when using kann
local nhidden = math.floor(n * (rule.hidden_layer_mult or 1.0) + 1.0)
local t = rspamd_kann.layer.input(n)
t = rspamd_kann.transform.relu(t)
-- scatter matrix is not filled with eigenvectors
lua_util.debugm(N, 'eigenvalues: %s', eigenvals)
local w = rspamd_tensor.new(2, max_inputs, #scatter_matrix[1])
- for i=1,max_inputs do
+ for i = 1, max_inputs do
w[i] = scatter_matrix[#scatter_matrix - i + 1]
end
local a = {}
local b = {}
- for i=1,n do
+ for i = 1, n do
r[i] = i
end
- local cmp = function(p, q) return p < q end
+ local cmp = function(p, q)
+ return p < q
+ end
- table.sort(r, function(p, q) return cmp(x[p], x[q]) end)
+ table.sort(r, function(p, q)
+ return cmp(x[p], x[q])
+ end)
- for i=1,n do
+ for i = 1, n do
a[i] = x[r[i]]
b[i] = y[r[i]]
end
local function get_scores(nn, input_vectors)
local scores = {}
- for i=1,#inputs do
+ for i = 1, #inputs do
local score = nn:apply1(input_vectors[i], nn.pca)[1]
- scores[#scores+1] = score
+ scores[#scores + 1] = score
end
return scores
end
local fpr = {}
- local fnr = {}
- local scores = get_scores(ann, inputs)
-
- scores, outputs = sort_relative(scores, outputs)
-
- local n_samples = #outputs
- local n_spam = 0
- local n_ham = 0
- local ham_count_ahead = {}
- local spam_count_ahead = {}
- local ham_count_behind = {}
- local spam_count_behind = {}
-
- ham_count_ahead[n_samples + 1] = 0
- spam_count_ahead[n_samples + 1] = 0
-
- for i=n_samples,1,-1 do
-
- if outputs[i][1] == 0 then
- n_ham = n_ham + 1
- ham_count_ahead[i] = 1
- spam_count_ahead[i] = 0
- else
- n_spam = n_spam + 1
- ham_count_ahead[i] = 0
- spam_count_ahead[i] = 1
- end
-
- ham_count_ahead[i] = ham_count_ahead[i] + ham_count_ahead[i + 1]
- spam_count_ahead[i] = spam_count_ahead[i] + spam_count_ahead[i + 1]
- end
-
- for i=1,n_samples do
+ local fnr = {}
+ local scores = get_scores(ann, inputs)
+
+ scores, outputs = sort_relative(scores, outputs)
+
+ local n_samples = #outputs
+ local n_spam = 0
+ local n_ham = 0
+ local ham_count_ahead = {}
+ local spam_count_ahead = {}
+ local ham_count_behind = {}
+ local spam_count_behind = {}
+
+ ham_count_ahead[n_samples + 1] = 0
+ spam_count_ahead[n_samples + 1] = 0
+
+ for i = n_samples, 1, -1 do
+
+ if outputs[i][1] == 0 then
+ n_ham = n_ham + 1
+ ham_count_ahead[i] = 1
+ spam_count_ahead[i] = 0
+ else
+ n_spam = n_spam + 1
+ ham_count_ahead[i] = 0
+ spam_count_ahead[i] = 1
+ end
+
+ ham_count_ahead[i] = ham_count_ahead[i] + ham_count_ahead[i + 1]
+ spam_count_ahead[i] = spam_count_ahead[i] + spam_count_ahead[i + 1]
+ end
+
+ for i = 1, n_samples do
if outputs[i][1] == 0 then
- ham_count_behind[i] = 1
- spam_count_behind[i] = 0
- else
- ham_count_behind[i] = 0
- spam_count_behind[i] = 1
- end
-
- if i ~= 1 then
- ham_count_behind[i] = ham_count_behind[i] + ham_count_behind[i - 1]
- spam_count_behind[i] = spam_count_behind[i] + spam_count_behind[i - 1]
- end
- end
-
- for i=1,n_samples do
- fpr[i] = 0
- fnr[i] = 0
-
- if (ham_count_ahead[i + 1] + ham_count_behind[i]) ~= 0 then
- fpr[i] = ham_count_ahead[i + 1] / (ham_count_ahead[i + 1] + ham_count_behind[i])
- end
-
- if (spam_count_behind[i] + spam_count_ahead[i + 1]) ~= 0 then
- fnr[i] = spam_count_behind[i] / (spam_count_behind[i] + spam_count_ahead[i + 1])
- end
- end
-
- local p = n_spam / (n_spam + n_ham)
-
- local cost = {}
- local min_cost_idx = 0
- local min_cost = math.huge
- for i=1,n_samples do
- cost[i] = ((1 - p) * alpha * fpr[i]) + (p * beta * fnr[i])
- if min_cost >= cost[i] then
- min_cost = cost[i]
- min_cost_idx = i
- end
- end
-
- return scores[min_cost_idx]
+ ham_count_behind[i] = 1
+ spam_count_behind[i] = 0
+ else
+ ham_count_behind[i] = 0
+ spam_count_behind[i] = 1
+ end
+
+ if i ~= 1 then
+ ham_count_behind[i] = ham_count_behind[i] + ham_count_behind[i - 1]
+ spam_count_behind[i] = spam_count_behind[i] + spam_count_behind[i - 1]
+ end
+ end
+
+ for i = 1, n_samples do
+ fpr[i] = 0
+ fnr[i] = 0
+
+ if (ham_count_ahead[i + 1] + ham_count_behind[i]) ~= 0 then
+ fpr[i] = ham_count_ahead[i + 1] / (ham_count_ahead[i + 1] + ham_count_behind[i])
+ end
+
+ if (spam_count_behind[i] + spam_count_ahead[i + 1]) ~= 0 then
+ fnr[i] = spam_count_behind[i] / (spam_count_behind[i] + spam_count_ahead[i + 1])
+ end
+ end
+
+ local p = n_spam / (n_spam + n_ham)
+
+ local cost = {}
+ local min_cost_idx = 0
+ local min_cost = math.huge
+ for i = 1, n_samples do
+ cost[i] = ((1 - p) * alpha * fpr[i]) + (p * beta * fnr[i])
+ if min_cost >= cost[i] then
+ min_cost = cost[i]
+ min_cost_idx = i
+ end
+ end
+
+ return scores[min_cost_idx]
end
-- This function is intended to extend lock for ANN during training
true, -- is write
redis_lock_extend_cb, --callback
'HINCRBY', -- command
- {ann_key, 'lock', '30'}
+ { ann_key, 'lock', '30' }
)
else
lua_util.debugm(N, rspamd_config, "stop lock extension as learning_spawned is false")
end
end
return true
- else -- Enough learns
+ else
+ -- Enough learns
rspamd_logger.infox(task, 'skip %s sample to keep spam/ham balance; too many spam samples: %s',
learn_type,
nspam)
-- Closure generator for unlock function
local function gen_unlock_cb(rule, set, ann_key)
- return function (err)
+ return function(err)
if err then
rspamd_logger.errx(rspamd_config, 'cannot unlock ANN %s:%s at %s from redis: %s',
rule.prefix, set.name, ann_key, err)
-- We also need to count metatokens:
local n = meta_functions.version
return string.format('%s%d_%s_%d_%s',
- settings.prefix, plugin_ver, rule.prefix, n, settings_name)
+ settings.prefix, plugin_ver, rule.prefix, n, settings_name)
end
-- This function receives training vectors, checks them, spawn learning and saves ANN in Redis
-- Used to show parsed vectors in a convenient format (for debugging only)
local function debug_vec(t)
local ret = {}
- for i,v in ipairs(t) do
+ for i, v in ipairs(t) do
if v ~= 0 then
ret[#ret + 1] = string.format('%d=%.2f', i, v)
end
-- KANN automatically shuffles those samples
-- 1.0 is used for spam and -1.0 is used for ham
-- It implies that output layer can express that (e.g. tanh output)
- for _,e in ipairs(params.spam_vec) do
+ for _, e in ipairs(params.spam_vec) do
inputs[#inputs + 1] = e
- outputs[#outputs + 1] = {1.0}
+ outputs[#outputs + 1] = { 1.0 }
--rspamd_logger.debugm(N, rspamd_config, 'spam vector: %s', debug_vec(e))
end
- for _,e in ipairs(params.ham_vec) do
+ for _, e in ipairs(params.ham_vec) do
inputs[#inputs + 1] = e
- outputs[#outputs + 1] = {-1.0}
+ outputs[#outputs + 1] = { -1.0 }
--rspamd_logger.debugm(N, rspamd_config, 'ham vector: %s', debug_vec(e))
end
rspamd_logger.errx(rspamd_config, 'ANN %s:%s: train error: observed nan in error cost!; value cost = %s',
params.rule.prefix, params.set.name,
value_cost)
- for i,e in ipairs(inputs) do
+ for i, e in ipairs(inputs) do
lua_util.debugm(N, rspamd_config, 'train vector %s -> %s',
debug_vec(e), outputs[i][1])
end
lua_util.debugm(N, rspamd_config, "start neural train for ANN %s:%s",
params.rule.prefix, params.set.name)
- local ret,err = pcall(train_ann.train1, train_ann,
+ local ret, err = pcall(train_ann.train1, train_ann,
inputs, outputs, {
lr = params.rule.train.learning_rate,
max_epoch = params.rule.train.max_iterations,
local roc_thresholds = {}
if params.rule.roc_enabled then
local spam_threshold = get_roc_thresholds(train_ann,
- inputs,
- outputs,
- 1 - params.rule.roc_misclassification_cost,
- params.rule.roc_misclassification_cost)
+ inputs,
+ outputs,
+ 1 - params.rule.roc_misclassification_cost,
+ params.rule.roc_misclassification_cost)
local ham_threshold = get_roc_thresholds(train_ann,
- inputs,
- outputs,
- params.rule.roc_misclassification_cost,
- 1 - params.rule.roc_misclassification_cost)
- roc_thresholds = {spam_threshold, ham_threshold}
+ inputs,
+ outputs,
+ params.rule.roc_misclassification_cost,
+ 1 - params.rule.roc_misclassification_cost)
+ roc_thresholds = { spam_threshold, ham_threshold }
rspamd_logger.messagex("ROC thresholds: (spam_threshold: %s, ham_threshold: %s)",
- roc_thresholds[1], roc_thresholds[2])
+ roc_thresholds[1], roc_thresholds[2])
end
if not seen_nan then
false, -- is write
gen_unlock_cb(params.rule, params.set, params.ann_key), --callback
'HDEL', -- command
- {params.ann_key, 'lock'}
+ { params.ann_key, 'lock' }
)
else
rspamd_logger.infox(rspamd_config, 'saved ANN %s:%s to redis: %s',
true, -- is write
gen_unlock_cb(params.rule, params.set, params.ann_key), --callback
'HDEL', -- command
- {params.ann_key, 'lock'}
+ { params.ann_key, 'lock' }
)
else
local parser = ucl.parser()
params.set.ann.redis_key, params.ann_key)
lua_redis.exec_redis_script(redis_script_id.save_unlock,
- {ev_base = params.ev_base, is_write = true},
+ { ev_base = params.ev_base, is_write = true },
redis_save_cb,
- {profile.redis_key,
- redis_ann_prefix(params.rule, params.set.name),
- ann_data,
- profile_serialized,
- tostring(params.rule.ann_expire),
- tostring(os.time()),
- params.ann_key, -- old key to unlock...
- roc_thresholds_serialized,
- pca_data,
+ { profile.redis_key,
+ redis_ann_prefix(params.rule, params.set.name),
+ ann_data,
+ profile_serialized,
+ tostring(params.rule.ann_expire),
+ tostring(os.time()),
+ params.ann_key, -- old key to unlock...
+ roc_thresholds_serialized,
+ pca_data,
})
end
end
fill_set_ann(params.set, params.ann_key)
end
- params.worker:spawn_process{
+ params.worker:spawn_process {
func = train,
on_complete = ann_trained,
proctitle = string.format("ANN train for %s/%s", params.rule.prefix, params.set.name),
-- Ensure that we have an array...
lua_util.debugm(N, rspamd_config, "use static profile for %s (%s): %s",
rule.prefix, selt.name, profile)
- if not profile[1] then profile = lua_util.keys(profile) end
+ if not profile[1] then
+ profile = lua_util.keys(profile)
+ end
selt.symbols = profile
else
lua_util.debugm(N, rspamd_config, "use dynamic cfg based profile for %s (%s)",
})
end
- for k,rule in pairs(settings.rules) do
+ for k, rule in pairs(settings.rules) do
if not rule.allowed_settings then
rule.allowed_settings = {}
elseif rule.allowed_settings == 'all' then
-- Now, for each allowed settings, we store sorted symbols + digest
-- We set table rule.settings[id] -> { name = name, symbols = symbols, digest = digest }
- for s,_ in pairs(rule.allowed_settings) do
+ for s, _ in pairs(rule.allowed_settings) do
-- Here, we have a name, set of symbols and
local settings_id = s
if type(settings_id) ~= 'number' then
}
process_settings_elt(rule, nelt)
- for id,ex in pairs(rule.settings) do
+ for id, ex in pairs(rule.settings) do
if type(ex) == 'table' then
if nelt and lua_util.distance_sorted(ex.symbols, nelt.symbols) == 0 then
-- Equal symbols, add reference
local sid = task:get_settings_id() or -1
local set = rule.settings[sid]
- if not set then return nil end
+ if not set then
+ return nil
+ end
while type(set) == 'number' do
-- Reference to another settings!
if not profile.zeros then
-- Fill zeros vector
local zeros = {}
- for i=1,meta_functions.count_metatokens() do
+ for i = 1, meta_functions.count_metatokens() do
zeros[i] = 0.0
end
- for _,_ in ipairs(profile.symbols) do
+ for _, _ in ipairs(profile.symbols) do
zeros[#zeros + 1] = 0.0
end
profile.zeros = zeros
local vec = lua_util.shallowcopy(profile.zeros)
local mt = meta_functions.rspamd_gen_metatokens(task)
- for i,v in ipairs(mt) do
+ for i, v in ipairs(mt) do
vec[i] = v
end
-- Common RBL plugin definitions
local check_types = {
- from = {
- connfilter = true,
- },
- received = {},
- helo = {
- connfilter = true,
- },
- urls = {},
- content_urls = {},
- emails = {},
- replyto = {},
- dkim = {},
- rdns = {
- connfilter = true,
- },
- selector = {
- require_argument = true,
- },
+ from = {
+ connfilter = true,
+ },
+ received = {},
+ helo = {
+ connfilter = true,
+ },
+ urls = {},
+ content_urls = {},
+ emails = {},
+ replyto = {},
+ dkim = {},
+ rdns = {
+ connfilter = true,
+ },
+ selector = {
+ require_argument = true,
+ },
}
local default_options = {
exclude_private_ips = ts.boolean:is_optional(),
exclude_users = ts.boolean:is_optional(),
from = ts.boolean:is_optional(),
- hash = ts.one_of{"sha1", "sha256", "sha384", "sha512", "md5", "blake2"}:is_optional(),
- hash_format = ts.one_of{"hex", "base32", "base64"}:is_optional(),
+ hash = ts.one_of { "sha1", "sha256", "sha384", "sha512", "md5", "blake2" }:is_optional(),
+ hash_format = ts.one_of { "hex", "base32", "base64" }:is_optional(),
hash_len = (ts.integer + ts.string / tonumber):is_optional(),
helo = ts.boolean:is_optional(),
ignore_default = ts.boolean:is_optional(), -- alias
replyto = ts.boolean:is_optional(),
requests_limit = (ts.integer + ts.string / tonumber):is_optional(),
require_symbols = (
- ts.array_of(ts.string) + (ts.string / function(s) return {s} end)
+ ts.array_of(ts.string) + (ts.string / function(s)
+ return { s }
+ end)
):is_optional(),
resolve_ip = ts.boolean:is_optional(),
return_bits = return_bits_schema:is_optional(),
return_codes = return_codes_schema:is_optional(),
returnbits = return_bits_schema:is_optional(),
returncodes = return_codes_schema:is_optional(),
- selector = ts.one_of{ts.string, ts.table}:is_optional(),
+ selector = ts.one_of { ts.string, ts.table }:is_optional(),
selector_flatten = ts.boolean:is_optional(),
symbol = ts.string:is_optional(),
symbols_prefixes = ts.map_of(ts.string, ts.string):is_optional(),
urls = ts.boolean:is_optional(),
whitelist = lua_maps.map_schema:is_optional(),
whitelist_exception = (
- ts.array_of(ts.string) + (ts.string / function(s) return {s} end)
+ ts.array_of(ts.string) + (ts.string / function(s)
+ return { s }
+ end)
):is_optional(),
checks = ts.array_of(ts.one_of(lua_util.keys(check_types))):is_optional(),
exclude_checks = ts.array_of(ts.one_of(lua_util.keys(check_types))):is_optional(),
if rule.checks then
local all_connfilter = true
local exclude_checks = lua_util.list_to_hash(rule.exclude_checks or {})
- for _,check in ipairs(rule.checks) do
+ for _, check in ipairs(rule.checks) do
if not exclude_checks[check] then
local check_type = check_types[check]
if check_type.require_argument then
if not rule[check] then
rspamd_logger.errx(rspamd_config, 'rbl rule %s has check %s which requires an argument',
- rule.symbol, check)
+ rule.symbol, check)
return nil
end
end
if not check_type then
rspamd_logger.errx(rspamd_config, 'rbl rule %s has invalid check type: %s',
- rule.symbol, check)
+ rule.symbol, check)
return nil
end
else
rspamd_logger.infox(rspamd_config, 'disable check %s in %s: excluded explicitly',
- check, rule.symbol)
+ check, rule.symbol)
end
end
rule.connfilter = all_connfilter
-- Now check if we have any check enabled at all
local check_found = false
- for k,_ in pairs(check_types) do
+ for k, _ in pairs(check_types) do
if type(rule[k]) ~= 'nil' then
check_found = true
break
-- Add default boolean flags to the schema
-for def_k,_ in pairs(default_options) do
+for def_k, _ in pairs(default_options) do
rule_schema_tbl[def_k:sub(#('default_') + 1)] = ts.boolean:is_optional()
end
if card > lim then
local to_delete = redis.call('ZRANGE', KEYS[1], 0, card - lim - 1)
if to_delete then
- for _,k in ipairs(to_delete) do
+ for _, k in ipairs(to_delete) do
local tb = cjson.decode(k)
if type(tb) == 'table' and type(tb.redis_key) == 'string' then
redis.call('DEL', tb.redis_key)
locked = tonumber(locked)
local expire = tonumber(KEYS[3])
if now > locked and (now - locked) < expire then
- return {tostring(locked), redis.call('HGET', KEYS[1], 'hostname') or 'unknown'}
+ return { tostring(locked), redis.call('HGET', KEYS[1], 'hostname') or 'unknown' }
end
end
redis.call('HSET', KEYS[1], 'lock', tostring(now))
local nham = 0
local ret = redis.call('SCARD', prefix .. '_spam_set')
-if ret then nspam = tonumber(ret) end
+if ret then
+ nspam = tonumber(ret)
+end
ret = redis.call('SCARD', prefix .. '_ham_set')
-if ret then nham = tonumber(ret) end
+if ret then
+ nham = tonumber(ret)
+end
-return {nspam,nham}
\ No newline at end of file
+return { nspam, nham }
\ No newline at end of file
-- New bucket
redis.call('HMSET', prefix, 'l', tostring(now), 'b', '0', 'dr', '10000', 'db', '10000', 'p', tostring(nrcpt))
redis.call('EXPIRE', prefix, KEYS[5])
- return {0, '0', '1', '1', '0'}
+ return { 0, '0', '1', '1', '0' }
end
last = tonumber(last)
-local burst,pending = unpack(redis.call('HMGET', prefix, 'b', 'p'))
-burst,pending = tonumber(burst or '0'),tonumber(pending or '0')
+local burst, pending = unpack(redis.call('HMGET', prefix, 'b', 'p'))
+burst, pending = tonumber(burst or '0'), tonumber(pending or '0')
-- Sanity to avoid races
-if burst < 0 then burst = 0 end
-if pending < 0 then pending = 0 end
+if burst < 0 then
+ burst = 0
+end
+if pending < 0 then
+ pending = 0
+end
pending = pending + nrcpt -- this message
-- Perform leak
if burst + pending > 0 then
-- If we have any time passed
if burst > 0 and last < now then
dynr = tonumber(redis.call('HGET', prefix, 'dr')) / 10000.0
- if dynr == 0 then dynr = 0.0001 end
+ if dynr == 0 then
+ dynr = 0.0001
+ end
leak_rate = leak_rate * dynr
leaked = ((now - last) * leak_rate)
- if leaked > burst then leaked = burst end
+ if leaked > burst then
+ leaked = burst
+ end
burst = burst - leaked
redis.call('HINCRBYFLOAT', prefix, 'b', -(leaked))
redis.call('HSET', prefix, 'l', tostring(now))
end
dynb = tonumber(redis.call('HGET', prefix, 'db')) / 10000.0
- if dynb == 0 then dynb = 0.0001 end
+ if dynb == 0 then
+ dynb = 0.0001
+ end
burst = burst + pending
if burst > 0 and burst > max_burst * dynb then
- return {1, tostring(burst - pending), tostring(dynr), tostring(dynb), tostring(leaked)}
+ return { 1, tostring(burst - pending), tostring(dynr), tostring(dynb), tostring(leaked) }
end
-- Increase pending if we allow ratelimit
redis.call('HINCRBY', prefix, 'p', nrcpt)
redis.call('HMSET', prefix, 'b', '0', 'p', tostring(nrcpt))
end
-return {0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked)}
\ No newline at end of file
+return { 0, tostring(burst), tostring(dynr), tostring(dynb), tostring(leaked) }
\ No newline at end of file
-- 2. Update the pending values based on the number of recipients (requests)
local pending = redis.call('HGET', prefix, 'p')
pending = tonumber(pending or '0')
-if pending < nrcpt then pending = 0 else pending = pending - nrcpt end
+if pending < nrcpt then
+ pending = 0
+else
+ pending = pending - nrcpt
+end
-- 3. Set the updated values back to Redis and update the expiration time for the bucket
redis.call('HMSET', prefix, 'p', tostring(pending), 'l', KEYS[2])
-- 2. Initialize a new bucket if the last hit time is not found (must not happen)
redis.call('HMSET', prefix, 'l', tostring(now), 'b', tostring(nrcpt), 'dr', '10000', 'db', '10000', 'p', '0')
redis.call('EXPIRE', prefix, KEYS[7])
- return {1, 1, 1}
+ return { 1, 1, 1 }
end
-- 3. Update the dynamic rate multiplier based on input parameters
end
-- 5. Update the burst and pending values based on the number of recipients (requests)
-local burst,pending = unpack(redis.call('HMGET', prefix, 'b', 'p'))
-burst,pending = tonumber(burst or '0'),tonumber(pending or '0')
-if burst < 0 then burst = nrcpt else burst = burst + nrcpt end
-if pending < nrcpt then pending = 0 else pending = pending - nrcpt end
+local burst, pending = unpack(redis.call('HMGET', prefix, 'b', 'p'))
+burst, pending = tonumber(burst or '0'), tonumber(pending or '0')
+if burst < 0 then
+ burst = nrcpt
+else
+ burst = burst + nrcpt
+end
+if pending < nrcpt then
+ pending = 0
+else
+ pending = pending - nrcpt
+end
-- 6. Set the updated values back to Redis and update the expiration time for the bucket
redis.call('HMSET', prefix, 'b', tostring(burst), 'p', tostring(pending), 'l', KEYS[2])
redis.call('EXPIRE', prefix, KEYS[7])
-- 7. Return the updated burst value, dynamic rate multiplier, and dynamic burst multiplier
-return {tostring(burst), tostring(dr), tostring(db)}
\ No newline at end of file
+return { tostring(burst), tostring(dr), tostring(db) }
\ No newline at end of file
for _, line in pairs(logs) do
line = lua_util.rspamd_str_split(line, " ")
- for i=4,(#line-1) do
+ for i = 4, (#line - 1) do
line[i] = line[i]:gsub("%s+", "")
if not symbols_set[line[i]] then
symbols_set[line[i]] = true
local fname = string.gsub(file, "(.*/)(.*)", "%2")
for line in fd:lines() do
- local start,stop = string.find(line, fname .. ':')
+ local start, stop = string.find(line, fname .. ':')
if start and stop then
table.insert(lines, string.sub(line, 1, start))
table.insert(messages, string.sub(line, stop + 1, -1))
end
-end
+ end
io.close(fd)
- return lines,messages
+ return lines, messages
end
function utility.get_all_logs(dirs)
-- Reads all log files in the directory and returns a list of logs.
if type(dirs) == 'string' then
- dirs = {dirs}
+ dirs = { dirs }
end
local all_logs = {}
local all_messages = {}
- for _,dir in ipairs(dirs) do
+ for _, dir in ipairs(dirs) do
if dir:sub(-1, -1) == "/" then
dir = dir:sub(1, -2)
local files = rspamd_util.glob(dir .. "/*.log")
for _, file in pairs(files) do
- local logs,messages = utility.read_log_file(file)
- for i=1,#logs do
+ local logs, messages = utility.read_log_file(file)
+ for i = 1, #logs do
table.insert(all_logs, logs[i])
table.insert(all_messages, messages[i])
end
end
else
- local logs,messages = utility.read_log_file(dir)
- for i=1,#logs do
+ local logs, messages = utility.read_log_file(dir)
+ for i = 1, #logs do
table.insert(all_logs, logs[i])
table.insert(all_messages, messages[i])
end
end
end
- return all_logs,all_messages
+ return all_logs, all_messages
end
function utility.get_all_symbol_scores(conf, ignore_symbols)
local symbols = conf:get_symbols_scores()
return fun.tomap(fun.map(function(name, elt)
- return name,elt['score']
+ return name, elt['score']
end, fun.filter(function(name, elt)
return not ignore_symbols[name]
end, symbols)))
true_negatives = true_negatives + 1
end
- for j=4, (#log-1) do
+ for j = 4, (#log - 1) do
if all_symbols_stats[log[j]] == nil then
all_symbols_stats[log[j]] = {
name = message,
-- Find slowest message
if ((tonumber(log[#log]) or 0) > file_stats.slowest) then
- file_stats.slowest = tonumber(log[#log])
- file_stats.slowest_file = message
+ file_stats.slowest = tonumber(log[#log])
+ file_stats.slowest_file = message
end
end
end
:argname('use_https')
local neural_profile = parser:command 'neural_profile'
- :description 'Generate symbols profile using data from Clickhouse'
+ :description 'Generate symbols profile using data from Clickhouse'
neural_profile:option '-w --where'
- :description 'WHERE clause for Clickhouse query'
- :argname('where')
+ :description 'WHERE clause for Clickhouse query'
+ :argname('where')
neural_profile:flag '-j --json'
- :description 'Write output as JSON'
- :argname('json')
+ :description 'Write output as JSON'
+ :argname('json')
neural_profile:option '--days'
- :description 'Number of days to collect stats for'
- :argname('days')
- :default('7')
+ :description 'Number of days to collect stats for'
+ :argname('days')
+ :default('7')
neural_profile:option '--limit -l'
- :description 'Maximum rows to fetch per day'
- :argname('limit')
+ :description 'Maximum rows to fetch per day'
+ :argname('limit')
neural_profile:option '--settings-id'
- :description 'Settings ID to query'
- :argname('settings_id')
- :default('')
+ :description 'Settings ID to query'
+ :argname('settings_id')
+ :default('')
local neural_train = parser:command 'neural_train'
- :description 'Train neural using data from Clickhouse'
+ :description 'Train neural using data from Clickhouse'
neural_train:option '--days'
- :description 'Number of days to query data for'
- :argname('days')
- :default('7')
+ :description 'Number of days to query data for'
+ :argname('days')
+ :default('7')
neural_train:option '--column-name-digest'
- :description 'Name of neural profile digest column in Clickhouse'
- :argname('column_name_digest')
- :default('NeuralDigest')
+ :description 'Name of neural profile digest column in Clickhouse'
+ :argname('column_name_digest')
+ :default('NeuralDigest')
neural_train:option '--column-name-vector'
- :description 'Name of neural training vector column in Clickhouse'
- :argname('column_name_vector')
- :default('NeuralMpack')
+ :description 'Name of neural training vector column in Clickhouse'
+ :argname('column_name_vector')
+ :default('NeuralMpack')
neural_train:option '--limit -l'
- :description 'Maximum rows to fetch per day'
- :argname('limit')
+ :description 'Maximum rows to fetch per day'
+ :argname('limit')
neural_train:option '--profile -p'
- :description 'Profile to use for training'
- :argname('profile')
- :default('default')
+ :description 'Profile to use for training'
+ :argname('profile')
+ :default('default')
neural_train:option '--rule -r'
- :description 'Rule to train'
- :argname('rule')
- :default('default')
+ :description 'Rule to train'
+ :argname('rule')
+ :default('default')
neural_train:option '--spam -s'
- :description 'WHERE clause to use for spam'
- :argname('spam')
- :default("Action == 'reject'")
+ :description 'WHERE clause to use for spam'
+ :argname('spam')
+ :default("Action == 'reject'")
neural_train:option '--ham -h'
- :description 'WHERE clause to use for ham'
- :argname('ham')
- :default('Score < 0')
+ :description 'WHERE clause to use for ham'
+ :argname('ham')
+ :default('Score < 0')
neural_train:option '--url -u'
- :description 'URL to use for training'
- :argname('url')
- :default('http://127.0.0.1:11334/plugins/neural/learn')
+ :description 'URL to use for training'
+ :argname('url')
+ :default('http://127.0.0.1:11334/plugins/neural/learn')
local http_params = {
config = rspamd_config,
}
local function load_config(config_file)
- local _r,err = rspamd_config:load_ucl(config_file)
+ local _r, err = rspamd_config:load_ucl(config_file)
if not _r then
rspamd_logger.errx('cannot load %s: %s', config_file, err)
os.exit(1)
end
- _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
if not _r then
rspamd_logger.errx('cannot process %s: %s', config_file, err)
os.exit(1)
elseif not all_symbols[k] then
remove[k] = 'nonexistent symbol'
else
- for fl,_ in pairs(all_symbols[k].flags or {}) do
+ for fl, _ in pairs(all_symbols[k].flags or {}) do
if skip_flags[fl] then
remove[k] = fl .. ' symbol'
break
local nsym = #r['Symbols.Names']
- for i = 1,nsym do
+ for i = 1, nsym do
local sym = r['Symbols.Names'][i]
local t = known_symbols[sym]
if not t then
end
-- Fill correlations
- for i = 1,nsym do
- for j = 1,nsym do
+ for i = 1, nsym do
+ for j = 1, nsym do
if i ~= j then
local sym = r['Symbols.Names'][i]
local inner_sym_name = r['Symbols.Names'][j]
local function post_neural_training(url, rule, spam_rows, ham_rows)
-- Prepare JSON payload
local payload = ucl.to_format(
- {
- ham_vec = ham_rows,
- rule = rule,
- spam_vec = spam_rows,
- }, 'json')
+ {
+ ham_vec = ham_rows,
+ rule = rule,
+ spam_vec = spam_rows,
+ }, 'json')
-- POST the payload
local err, response = rspamd_http.request({
limit = string.format(' LIMIT %d', num_limit) -- Contains leading space
end
-- Prepare query elements
- local conditions = {string.format("%s = '%s'", args.column_name_digest, symbols_digest)}
+ local conditions = { string.format("%s = '%s'", args.column_name_digest, symbols_digest) }
local query_fmt = 'SELECT %s FROM rspamd WHERE %s%s'
-- Run queries
- for _, the_where in ipairs({args.ham, args.spam}) do
+ for _, the_where in ipairs({ args.ham, args.spam }) do
-- Inform callback which group of vectors we're collecting
this_where = the_where
table.insert(conditions, the_where) -- should be 2nd from last condition
if this_where == args.ham then
if not want_ham then
break
- end
+ end
else
if not want_spam then
break
parser:flag "-a --all"
:description('Show all nodes, not just existing ones')
-
local function process_filename(fname)
local cdir = rspamd_paths['CONFDIR'] .. '/'
fname = fname:gsub(cdir, '')
local function output_dot(opts, nodes, adjacency)
rspamd_logger.messagex("digraph rspamd {")
- for k,node in pairs(nodes) do
- local attrs = {"shape=box"}
+ for k, node in pairs(nodes) do
+ local attrs = { "shape=box" }
local skip = false
if node.exists then
if node.priority >= 10 then
table.concat(attrs, ','))
end
end
- for _,adj in ipairs(adjacency) do
+ for _, adj in ipairs(adjacency) do
local attrs = {}
local skip = false
local nodes = {}
local function maybe_match_glob(file)
- for _,gl in ipairs(glob_traces) do
+ for _, gl in ipairs(glob_traces) do
if gl.re:match(file) then
return gl
end
end
end
- local _r,err = rspamd_config:load_ucl(opts['config'], trace_func)
+ local _r, err = rspamd_config:load_ucl(opts['config'], trace_func)
if not _r then
rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
os.exit(1)
output_dot(opts, nodes, adjacency)
end
-
local function handler(args)
local res = parser:parse(args)
parser:argument "path":args "*"
:description('Optional config paths')
parser:flag "--no-color"
- :description "Disable coloured output"
+ :description "Disable coloured output"
parser:flag "--short"
- :description "Show only option names"
+ :description "Show only option names"
parser:flag "--no-examples"
- :description "Do not show examples (implied by --short)"
+ :description "Do not show examples (implied by --short)"
local function maybe_print_color(key)
if not opts['no-color'] then
if type(value['required']) == 'boolean' then
if value['required'] then
print(string.format('%s\tRequired: %s', tabs,
- maybe_print_color(tostring(value['required']))))
+ maybe_print_color(tostring(value['required']))))
else
print(string.format('%s\tRequired: %s', tabs,
- tostring(value['required'])))
+ tostring(value['required'])))
end
end
if value['default'] then
local sorted = sort_values(res)
- for _,v in ipairs(sorted) do
+ for _, v in ipairs(sorted) do
print_help(v['key'], v['value'], '')
print('')
end
local reply = rspamd_util.readline(greet)
- if not reply then os.exit(0) end
- if #reply == 0 then reply = def_str end
+ if not reply then
+ os.exit(0)
+ end
+ if #reply == 0 then
+ reply = def_str
+ end
reply = reply:lower()
- if reply == 'y' or reply == 'yes' then return true end
+ if reply == 'y' or reply == 'yes' then
+ return true
+ end
return false
end
local function readline_default(greet, def_value)
local reply = rspamd_util.readline(greet)
- if not reply then os.exit(0) end
+ if not reply then
+ os.exit(0)
+ end
- if #reply == 0 then return def_value end
+ if #reply == 0 then
+ return def_value
+ end
return reply
end
local expire = '100d'
repeat
expire = readline_default("Expire time for new tokens [" .. expire .. "]: ",
- expire)
+ expire)
expire = lua_util.parse_time_interval(expire)
if not expire then
local function print_changes(changes)
local function print_change(k, c, where)
printf('File: %s, changes list:', highlight(local_conf .. '/'
- .. where .. '/'.. k))
+ .. where .. '/' .. k))
- for ek,ev in pairs(c) do
+ for ek, ev in pairs(c) do
printf("%s => %s", highlight(ek), rspamd_logger.slog("%s", ev))
end
end
end
local function apply_change(k, c, where)
- local fname = local_conf .. '/' .. where .. '/'.. k
+ local fname = local_conf .. '/' .. where .. '/' .. k
if not rspamd_util.file_exists(fname) then
printf("Create file %s", highlight(fname))
end
end
-
local function setup_controller(controller, changes)
printf("Setup %s and controller worker:", highlight("WebUI"))
printf("%s servers are not set:", highlight("Redis"))
printf("The following modules will be enabled if you add Redis servers:")
- for k,_ in pairs(rspamd_plugins_state.disabled_redis) do
+ for k, _ in pairs(rspamd_plugins_state.disabled_redis) do
printf("\t* %s", highlight(k))
end
if ask_yes_no("Do you wish to set Redis servers?", true) then
local read_servers = readline_default("Input read only servers separated by `,` [default: localhost]: ",
- "localhost")
+ "localhost")
local rs = parse_servers(read_servers)
if rs and #rs > 0 then
local function setup_dkim_signing(cfg, changes)
-- Remove the trailing slash of a pathname, if present.
local function remove_trailing_slash(path)
- if string.sub(path, -1) ~= "/" then return path end
+ if string.sub(path, -1) ~= "/" then
+ return path
+ end
return string.sub(path, 1, string.len(path) - 1)
end
local use_esld
local sign_domain = 'pet luacheck'
- local defined_auth_types = {'header', 'envelope', 'auth', 'recipient'}
+ local defined_auth_types = { 'header', 'envelope', 'auth', 'recipient' }
if sign_type == '4' then
repeat
sign_authenticated = true
end
- if fun.any(function(s) return s == sign_domain end, defined_auth_types) then
+ if fun.any(function(s)
+ return s == sign_domain
+ end, defined_auth_types) then
-- Allow mismatch
allow_mismatch = ask_yes_no(
string.format('Allow data %s, e.g. if mime from domain is not equal to authenticated user domain? ',
local dkim_keys_dir = rspamd_paths["DBDIR"] .. "/dkim/"
local prompt = string.format("Enter output directory for the keys [default: %s]: ",
- highlight(dkim_keys_dir))
+ highlight(dkim_keys_dir))
dkim_keys_dir = remove_trailing_slash(readline_default(prompt, dkim_keys_dir))
local ret, err = rspamd_util.mkdir(dkim_keys_dir, true)
local function print_domains()
printf("Domains configured:")
- for k,v in pairs(domains) do
+ for k, v in pairs(domains) do
printf("Domain: %s, selector: %s, privkey: %s", highlight(k),
v.selector, v.privkey)
end
until #domain ~= 0
local selector = readline_default("Enter selector [default: dkim]: ", 'dkim')
- if not selector then selector = 'dkim' end
+ if not selector then
+ selector = 'dkim'
+ end
local privkey_file = string.format("%s/%s.%s.key", dkim_keys_dir, domain,
selector)
if not rspamd_util.file_exists(privkey_file) then
if ask_yes_no("Do you want to create privkey " .. highlight(privkey_file),
- true) then
+ true) then
local pubkey_file = privkey_file .. ".pub"
local rspamd_cryptobox = require "rspamd_cryptobox"
local sk, pk = rspamd_cryptobox.generate_keypair("rsa", 2048)
}
until not ask_yes_no("Do you wish to add another DKIM domain?")
- changes.l['dkim_signing.conf'] = {domain = domains}
+ changes.l['dkim_signing.conf'] = { domain = domains }
local res_tbl = changes.l['dkim_signing.conf']
if sign_networks then
local symbol_spam, symbol_ham
-- Load symbols from statfiles
local statfiles = cls.statfile
- for _,stf in ipairs(statfiles) do
+ for _, stf in ipairs(statfiles) do
local symbol = stf.symbol or 'undefined'
local spam
end
local function get_version(conn)
- conn:add_cmd("SMEMBERS", {"RS_keys"})
+ conn:add_cmd("SMEMBERS", { "RS_keys" })
- local ret,members = conn:exec()
+ local ret, members = conn:exec()
-- Empty db
- if not ret or #members == 0 then return false,0 end
+ if not ret or #members == 0 then
+ return false, 0
+ end
-- We still need to check versions
local lua_script = [[
return ver
]]
- conn:add_cmd('EVAL', {lua_script, '1', 'RS'})
- local _,ver = conn:exec()
+ conn:add_cmd('EVAL', { lua_script, '1', 'RS' })
+ local _, ver = conn:exec()
- return true,tonumber(ver)
+ return true, tonumber(ver)
end
local function check_expire(conn)
return ttl
]]
- conn:add_cmd('EVAL', {lua_script, '0'})
- local _,ttl = conn:exec()
+ conn:add_cmd('EVAL', { lua_script, '0' })
+ local _, ttl = conn:exec()
return tonumber(ttl)
end
- local res,conn = lua_redis.redis_connect_sync(parsed_redis, true)
+ local res, conn = lua_redis.redis_connect_sync(parsed_redis, true)
if not res then
printf("Cannot connect to Redis server")
return false
end
if not cls.new_schema then
- local r,ver = get_version(conn)
- if not r then return false end
+ local r, ver = get_version(conn)
+ if not r then
+ return false
+ end
if ver ~= 2 then
if not ver then
printf('Key "RS_version" has not been found in Redis for %s/%s',
end
end
else
- local r,ver = get_version(conn)
- if not r then return false end
+ local r, ver = get_version(conn)
+ if not r then
+ return false
+ end
if ver ~= 2 then
printf("You have configured new schema for %s/%s but your DB has old version: %s",
- symbol_spam, symbol_ham, ver)
+ symbol_spam, symbol_ham, ver)
try_convert(false)
else
printf(
'You have configured new schema for %s/%s and your DB already has new layout (v. %s).' ..
' DB conversion is not needed.',
- symbol_spam, symbol_ham, ver)
+ symbol_spam, symbol_ham, ver)
end
end
end
if not redis_params then
printf('You have %d sqlite classifiers, but you have no Redis servers being set',
- #sqlite_configs)
+ #sqlite_configs)
return false
end
local reset_previous = ask_yes_no("Reset previous data?")
if ask_yes_no('Do you wish to convert them to Redis?', true) then
- for _,cls in ipairs(sqlite_configs) do
+ for _, cls in ipairs(sqlite_configs) do
if rspamd_util.file_exists(cls.db_spam) and rspamd_util.file_exists(cls.db_ham) then
if not lua_stat_tools.convert_sqlite_to_redis(parsed_redis, cls.db_spam,
cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, expire,
if classifier then
if classifier[1] then
- for _,cls in ipairs(classifier) do
- if cls.bayes then cls = cls.bayes end
+ for _, cls in ipairs(classifier) do
+ if cls.bayes then
+ cls = cls.bayes
+ end
if cls.backend and cls.backend == 'redis' then
check_redis_classifier(cls, changes)
end
classifier = classifier.bayes
if classifier[1] then
- for _,cls in ipairs(classifier) do
+ for _, cls in ipairs(classifier) do
if cls.backend and cls.backend == 'redis' then
check_redis_classifier(cls, changes)
end
local function find_worker(cfg, wtype)
if cfg.worker then
- for k,s in pairs(cfg.worker) do
+ for k, s in pairs(cfg.worker) do
if type(k) == 'number' and type(s) == 'table' then
- if s[wtype] then return s[wtype] end
+ if s[wtype] then
+ return s[wtype]
+ end
end
if type(s) == 'table' and s.type and s.type == wtype then
return s
end
- if type(k) == 'string' and k == wtype then return s end
+ if type(k) == 'string' and k == wtype then
+ return s
+ end
end
end
return nil
end
-
-
return {
handler = function(cmd_args)
local changes = {
local opts = parser:parse(cmd_args)
local args = opts['checks'] or {}
- local _r,err = rspamd_config:load_ucl(opts['config'])
+ local _r, err = rspamd_config:load_ucl(opts['config'])
if not _r then
rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
os.exit(1)
end
- _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
if not _r then
rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
os.exit(1)
if #args > 0 then
interactive_start = false
- for _,arg in ipairs(args) do
+ for _, arg in ipairs(args) do
if arg == 'all' then
checks = all_checks
elseif arg == 'list' then
printf(highlight(rspamd_logo))
printf('Available modules')
- for _,c in ipairs(all_checks) do
+ for _, c in ipairs(all_checks) do
printf('- %s', c)
end
return
end
local function has_check(check)
- for _,c in ipairs(checks) do
+ for _, c in ipairs(checks) do
if c == check then
return true
end
end
local nchanges = 0
- for _,_ in pairs(changes.l) do nchanges = nchanges + 1 end
- for _,_ in pairs(changes.o) do nchanges = nchanges + 1 end
+ for _, _ in pairs(changes.l) do
+ nchanges = nchanges + 1
+ end
+ for _, _ in pairs(changes.o) do
+ nchanges = nchanges + 1
+ end
if nchanges > 0 then
print_changes(changes)
local function gen_cookie(args, key)
local cr = require "rspamd_cryptobox"
- if not args.cookie then return end
+ if not args.cookie then
+ return
+ end
local function encrypt()
if #args.cookie > 31 then
extracted_cookie = args.cookie
end
- local dec_cookie,ts = cr.decrypt_cookie(key, extracted_cookie)
+ local dec_cookie, ts = cr.decrypt_cookie(key, extracted_cookie)
if dec_cookie then
if args.timestamp then
print('cannot decrypt cookie')
os.exit(1)
end
- end
+ end
if args.decrypt then
decrypt()
end
if res.key then
- local pattern = {'^'}
- for i=1,32 do pattern[i + 1] = '[a-zA-Z0-9]' end
+ local pattern = { '^' }
+ for i = 1, 32 do
+ pattern[i + 1] = '[a-zA-Z0-9]'
+ end
pattern[34] = '$'
if not res.key:match(table.concat(pattern, '')) then
rspamd_logger.messagex("Messages/sec: %s", (total_msgs / elapsed_time))
end
-
return {
name = 'corpustest',
- aliases = {'corpus_test', 'corpus'},
+ aliases = { 'corpus_test', 'corpus' },
handler = handler,
description = parser._description
}
:description 'Save private key to file instead of printing it to stdout'
parser:option '-b --bits'
:convert(function(input)
- local n = tonumber(input)
- if not n or n < 512 or n > 4096 then
- return nil
- end
- return n
- end)
+ local n = tonumber(input)
+ if not n or n < 512 or n > 4096 then
+ return nil
+ end
+ return n
+end)
:description 'Generate an RSA key with the specified number of bits (512 to 4096)'
:default(1024)
parser:option '-t --type'
:description 'Key type: RSA, ED25519 or ED25119-seed'
:convert {
- ['rsa'] = 'rsa',
- ['RSA'] = 'rsa',
- ['ed25519'] = 'ed25519',
- ['ED25519'] = 'ed25519',
- ['ed25519-seed'] = 'ed25519-seed',
- ['ED25519-seed'] = 'ed25519-seed',
- }
+ ['rsa'] = 'rsa',
+ ['RSA'] = 'rsa',
+ ['ed25519'] = 'ed25519',
+ ['ED25519'] = 'ed25519',
+ ['ed25519-seed'] = 'ed25519-seed',
+ ['ED25519-seed'] = 'ed25519-seed',
+}
:default 'rsa'
parser:option '-o --output'
:description 'Output public key in the following format: dns, dnskey or plain'
:convert {
- ['dns'] = 'dns',
- ['plain'] = 'plain',
- ['dnskey'] = 'dnskey',
- }
+ ['dns'] = 'dns',
+ ['plain'] = 'plain',
+ ['dnskey'] = 'dnskey',
+}
:default 'dns'
parser:option '--priv-output'
:description 'Output private key in the following format: PEM or DER (for RSA)'
:convert {
- ['pem'] = 'pem',
- ['der'] = 'der',
- }
+ ['pem'] = 'pem',
+ ['der'] = 'der',
+}
:default 'pem'
parser:flag '-f --force'
:description 'Force overwrite of existing files'
return pieces
end
-
local function print_public_key_dns(opts, base64_pk)
local key_type = opts.type == 'rsa' and 'rsa' or 'ed25519'
if #base64_pk < 255 - 2 then
-- Split it by parts
local parts = split_string(base64_pk)
io.write(string.format('%s._domainkey IN TXT ( "v=DKIM1; k=%s; "\n', opts.selector, key_type))
- for i,part in ipairs(parts) do
+ for i, part in ipairs(parts) do
if i == 1 then
io.write(string.format('\t"p=%s"\n', part))
else
local function gen_rsa_key(opts)
local rsa = require "rspamd_rsa"
- local sk,pk = rsa.keypair(opts.bits or 1024)
+ local sk, pk = rsa.keypair(opts.bits or 1024)
if opts.privkey then
if opts.force then
os.remove(opts.privkey)
end
local function gen_eddsa_key(opts)
- local sk,pk = rspamd_cryptobox.gen_dkim_keypair(opts.type)
+ local sk, pk = rspamd_cryptobox.gen_dkim_keypair(opts.type)
if opts.privkey and opts.force then
os.remove(opts.privkey)
local function handler(args)
local opts = parser:parse(args)
- if not opts then os.exit(1) end
+ if not opts then
+ os.exit(1)
+ end
if opts.type == 'rsa' then
gen_rsa_key(opts)
return {
name = 'dkim_keygen',
- aliases = {'dkimkeygen'},
+ aliases = { 'dkimkeygen' },
handler = handler,
description = parser._description
}
:description "Do not reset reporting data/send reports"
parser:argument "date"
- :description "Date to process (today by default)"
- :argname "<YYYYMMDD>"
- :args "*"
+ :description "Date to process (today by default)"
+ :argname "<YYYYMMDD>"
+ :args "*"
parser:option "-b --batch-size"
:description "Send reports in batches up to <batch-size> messages"
:argname "<number>"
local pool
local function load_config(opts)
- local _r,err = rspamd_config:load_ucl(opts['config'])
+ local _r, err = rspamd_config:load_ucl(opts['config'])
if not _r then
logger.errx('cannot parse %s: %s', opts['config'], err)
os.exit(1)
end
- _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
if not _r then
logger.errx('cannot process %s: %s', opts['config'], err)
os.exit(1)
-- Concat elements using redis_keys.join_char
local function redis_prefix(...)
- return table.concat({...}, dmarc_settings.reporting.redis_keys.join_char)
+ return table.concat({ ... }, dmarc_settings.reporting.redis_keys.join_char)
end
-
-
local function get_rua(rep_key)
local parts = lua_util.str_split(rep_key, dmarc_settings.reporting.redis_keys.join_char)
end
local function gen_uuid()
- local template ='xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'
- return string.gsub(template, '[xy]', function (c)
+ local template = 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'
+ return string.gsub(template, '[xy]', function(c)
local v = (c == 'x') and math.random(0, 0xf) or math.random(8, 0xb)
return string.format('%x', v)
end)
local quot = lpeg.P('"') / '"'
local apos = lpeg.P("'") / '''
local special = lt + gt + amp + quot + apos
- local grammar = lpeg.Cs((special + 1)^0)
+ local grammar = lpeg.Cs((special + 1) ^ 0)
return grammar
end
</auth_results>
</record>
]]
- return lua_util.jinja_template(xml_template, {data = data}, true)
+ return lua_util.jinja_template(xml_template, { data = data }, true)
end
-- Process a report entry stored in Redis splitting it to a lua table
if dkim_data and dkim_data ~= '' then
local dkim_elts = lua_util.str_split(dkim_data, '|')
for _, d in ipairs(dkim_elts) do
- table.insert(row.dkim_results, {domain = d, result = result})
+ table.insert(row.dkim_results, { domain = d, result = result })
end
end
end
logger.errx('cannot resolve %s: %s; exclude %s', resolve_str, results, rua_part)
else
local found = false
- for _,t in ipairs(results) do
+ for _, t in ipairs(results) do
if string.match(t, 'v=DMARC1') then
found = true
break
config = rspamd_config,
session = rspamadm_session,
type = 'txt',
- name = '_dmarc.' .. dmarc_domain ,
+ name = '_dmarc.' .. dmarc_domain,
})
if not is_ok or not results then
return nil
end
- for _,r in ipairs(results) do
- local processed,rec = dmarc_common.dmarc_check_record(rspamd_config, r, false)
+ for _, r in ipairs(results) do
+ local processed, rec = dmarc_common.dmarc_check_record(rspamd_config, r, false)
if processed and rec.rua then
-- We need to check or alter rua if needed
local processed_rua = process_rua(dmarc_domain, rec.rua)
-- Returns a list of recipients from a table as a string processing elements if needed
local function rcpt_list(tbl, func)
local res = {}
- for _,r in ipairs(tbl) do
+ for _, r in ipairs(tbl) do
if func then
table.insert(res, func(r))
else
local nreports = math.min(#reports - cur_batch + 1, opts.batch_size)
local next_start = cur_batch + nreports
lua_util.debugm(N, 'send data for %s domains (from %s to %s)',
- nreports, cur_batch, next_start-1)
+ nreports, cur_batch, next_start - 1)
-- Shared across all closures
local gen_args = {
cont_func = send_data_in_batches,
nreports = nreports,
next_start = next_start
}
- for i=cur_batch,next_start-1 do
+ for i = cur_batch, next_start - 1 do
local report = reports[i]
local send_opts = {
ev_base = rspamadm_ev_base,
end
local ret, results = lua_redis.request(redis_params, redis_attrs,
- {'EXISTS', rep_key})
+ { 'EXISTS', rep_key })
if not ret or not results or results == 0 then
return nil
-- Rename report key to avoid races
if not opts.no_opt then
lua_redis.request(redis_params, redis_attrs,
- {'RENAME', rep_key, rep_key .. '_processing'})
+ { 'RENAME', rep_key, rep_key .. '_processing' })
rep_key = rep_key .. '_processing'
end
if not dmarc_record then
if not opts.no_opt then
lua_redis.request(redis_params, redis_attrs,
- {'DEL', rep_key})
+ { 'DEL', rep_key })
end
logger.messagex('Cannot process reports for domain %s; invalid dmarc record', reporting_domain)
return nil
-- Get all reports for a domain
ret, results = lua_redis.request(redis_params, redis_attrs,
- {'ZRANGE', rep_key, '0', '-1', 'WITHSCORES'})
+ { 'ZRANGE', rep_key, '0', '-1', 'WITHSCORES' })
local report_entries = {}
table.insert(report_entries,
report_header(reporting_domain, start_time, end_time, dmarc_record))
- for i=1,#results,2 do
+ for i = 1, #results, 2 do
local xml_record = entry_to_xml(process_report_entry(results[i], results[i + 1]))
table.insert(report_entries, xml_record)
end
local rfooter = lua_util.jinja_template(report_footer, {
uuid = uuid,
}, true)
- local message = rspamd_text.fromtable{
+ local message = rspamd_text.fromtable {
(rhead:gsub("\n", "\r\n")),
rspamd_util.encode_base64(rspamd_util.gzip_compress(xml_to_compress), 73),
rfooter:gsub("\n", "\r\n"),
}
-
lua_util.debugm(N, 'got final message: %s', message)
if not opts.no_opt then
lua_redis.request(redis_params, redis_attrs,
- {'DEL', rep_key})
+ { 'DEL', rep_key })
end
local report_rcpts = lua_util.str_split(rcpt_string, ',')
if report_settings.bcc_addrs then
- for _,b in ipairs(report_settings.bcc_addrs) do
+ for _, b in ipairs(report_settings.bcc_addrs) do
table.insert(report_rcpts, b)
end
end
local function process_report_date(opts, start_time, end_time, date)
local idx_key = redis_prefix(dmarc_settings.reporting.redis_keys.index_prefix, date)
local ret, results = lua_redis.request(redis_params, redis_attrs,
- {'EXISTS', idx_key})
+ { 'EXISTS', idx_key })
if not ret or not results or results == 0 then
logger.messagex('No reports for %s', date)
-- Rename index key to avoid races
if not opts.no_opt then
lua_redis.request(redis_params, redis_attrs,
- {'RENAME', idx_key, idx_key .. '_processing'})
+ { 'RENAME', idx_key, idx_key .. '_processing' })
idx_key = idx_key .. '_processing'
end
ret, results = lua_redis.request(redis_params, redis_attrs,
- {'SMEMBERS', idx_key})
+ { 'SMEMBERS', idx_key })
if not ret or not results then
-- Remove bad key
if not opts.no_opt then
lua_redis.request(redis_params, redis_attrs,
- {'DEL', idx_key})
+ { 'DEL', idx_key })
end
logger.messagex('Cannot get reports for %s', date)
return {}
end
local reports = {}
- for _,rep in ipairs(results) do
+ for _, rep in ipairs(results) do
local report = prepare_report(opts, start_time, end_time, rep)
if report then
-- Remove processed key
if not opts.no_opt then
lua_redis.request(redis_params, redis_attrs,
- {'DEL', idx_key})
+ { 'DEL', idx_key })
end
return reports
os.exit(1)
end
- for _, e in ipairs({'email', 'domain', 'org_name'}) do
+ for _, e in ipairs({ 'email', 'domain', 'org_name' }) do
if not dmarc_settings.reporting[e] then
logger.errx('Missing required setting: dmarc.reporting.%s', e)
return
end
end
- local ret,results = lua_redis.request(redis_params, redis_attrs, {
+ local ret, results = lua_redis.request(redis_params, redis_attrs, {
'GET', 'rspamd_dmarc_last_collection'
})
local ndates = 0
local nreports = 0
local all_reports = {}
- for _,date in ipairs(opts.date) do
+ for _, date in ipairs(opts.date) do
lua_util.debugm(N, 'Process date %s', date)
local reports_for_date = process_report_date(opts, start_time, start_collection, date)
if #reports_for_date > 0 then
ndates = ndates + 1
nreports = nreports + #reports_for_date
- for _,r in ipairs(reports_for_date) do
+ for _, r in ipairs(reports_for_date) do
table.insert(all_reports, r)
end
end
ndates, nreports, nsuccess, nfail)
end
lua_redis.request(redis_params, redis_attrs,
- {'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2,
- tostring(start_collection)})
+ { 'SETEX', 'rspamd_dmarc_last_collection', dmarc_settings.reporting.keys_expire * 2,
+ tostring(start_collection) })
else
logger.messagex('Reporting collection has finished %s dates processed, %s reports: %s completed, %s failed',
ndates, nreports, nsuccess, nfail)
return {
name = 'dmarc_report',
- aliases = {'dmarc_reporting'},
+ aliases = { 'dmarc_reporting' },
handler = handler,
description = parser._description
}
\ No newline at end of file
:default(rspamd_paths["CONFDIR"] .. "/" .. "rspamd.conf")
local spf = parser:command "spf"
- :description "Extracts spf records"
+ :description "Extracts spf records"
spf:mutex(
spf:option "-d --domain"
:description "Domain to use"
end
local function load_config(opts)
- local _r,err = rspamd_config:load_ucl(opts['config'])
+ local _r, err = rspamd_config:load_ucl(opts['config'])
if not _r then
rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
os.exit(1)
end
- _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
if not _r then
rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
os.exit(1)
os.exit(1)
end
elseif opts.domain then
- task:set_from('smtp', {user = 'user', domain = opts.domain})
+ task:set_from('smtp', { user = 'user', domain = opts.domain })
else
io.stderr:write('Neither domain nor from specified\n')
os.exit(1)
end
local function display_spf_results(elt, colored)
- local dec = function(e) return e end
+ local dec = function(e)
+ return e
+ end
local policy_decode = function(e)
if e == rspamd_spf.policy.fail then
return 'reject'
end
if colored then
- dec = function(e) return highlight(e) end
+ dec = function(e)
+ return highlight(e)
+ end
- if elt.result == rspamd_spf.policy.pass then
- dec = function(e) return green(e) end
- elseif elt.result == rspamd_spf.policy.fail then
- dec = function(e) return red(e) end
+ if elt.result == rspamd_spf.policy.pass then
+ dec = function(e)
+ return green(e)
+ end
+ elseif elt.result == rspamd_spf.policy.fail then
+ dec = function(e)
+ return red(e)
+ end
end
end
if result then
printf('SPF record for %s; digest: %s',
highlight(opts.domain or opts.from), highlight(record:get_digest()))
- for _,elt in ipairs(record:get_elts()) do
+ for _, elt in ipairs(record:get_elts()) do
if result and error_or_addr and elt.str and elt.str == error_or_addr.str then
printf("%s", highlight('*** Matched ***'))
display_spf_results(elt, true)
return {
name = 'dnstool',
- aliases = {'dns', 'dns_tool'},
+ aliases = { 'dns', 'dns_tool' },
handler = handler,
description = parser._description
}
\ No newline at end of file
end
if password then
- ret = conn:add_cmd('AUTH', {password})
+ ret = conn:add_cmd('AUTH', { password })
if not ret then
return nil, 'Cannot queue command'
end
end
if db then
- ret = conn:add_cmd('SELECT', {db})
+ ret = conn:add_cmd('SELECT', { db })
if not ret then
return nil, 'Cannot queue command'
end
return true
end
-return function (_, res)
+return function(_, res)
local db = sqlite3.open(res['source_db'])
local shingles = {}
local digests = {}
local expire_in = math.floor(now - row.time + res['expiry'])
if expire_in >= 1 then
- table.insert(digests, {row.digest, row.flag, row.value, expire_in})
+ table.insert(digests, { row.digest, row.flag, row.value, expire_in })
num_batch_digests = num_batch_digests + 1
total_digests = total_digests + 1
for srow in db:rows('SELECT value, number FROM shingles WHERE digest_id = ' .. row.id) do
- table.insert(shingles, {srow.value, srow.number, expire_in, row.digest})
+ table.insert(shingles, { srow.value, srow.number, expire_in, row.digest })
total_shingles = total_shingles + 1
num_batch_shingles = num_batch_shingles + 1
end
end
local message = string.format(
- 'Migrated %d digests and %d shingles',
- total_digests, total_shingles
+ 'Migrated %d digests and %d shingles',
+ total_digests, total_shingles
)
if not update_counters(total_digests, res['redis_host'], redis_password, redis_db) then
message = message .. ' but failed to update counters'
parser:option "--sort"
:description "Sort order"
:convert {
- checked = "checked",
- matched = "matched",
- errors = "errors",
- name = "name"
- }
+ checked = "checked",
+ matched = "matched",
+ errors = "errors",
+ name = "name"
+}
local function add_data(target, src)
- for k,v in pairs(src) do
+ for k, v in pairs(src) do
if type(v) == 'number' then
if target[k] then
target[k] = target[k] + v
target[k] = v
end
elseif k == 'ips' then
- if not target['ips'] then target['ips'] = {} end
+ if not target['ips'] then
+ target['ips'] = {}
+ end
-- Iterate over IPs
- for ip,st in pairs(v) do
- if not target['ips'][ip] then target['ips'][ip] = {} end
+ for ip, st in pairs(v) do
+ if not target['ips'][ip] then
+ target['ips'][ip] = {}
+ end
add_data(target['ips'][ip], st)
end
elseif k == 'flags' then
- if not target['flags'] then target['flags'] = {} end
+ if not target['flags'] then
+ target['flags'] = {}
+ end
-- Iterate over Flags
- for flag,st in pairs(v) do
- if not target['flags'][flag] then target['flags'][flag] = {} end
+ for flag, st in pairs(v) do
+ if not target['flags'][flag] then
+ target['flags'][flag] = {}
+ end
add_data(target['flags'][flag], st)
end
elseif k == 'keypair' then
-- Sort by checked
local function sort_hash_table(tbl, sort_opts, key_key)
local res = {}
- for k,v in pairs(tbl) do
- table.insert(res, {[key_key] = k, data = v})
+ for k, v in pairs(tbl) do
+ table.insert(res, { [key_key] = k, data = v })
end
local function sort_order(elt)
if type(src) == 'table' then
if type(dst) == 'number' then
-- Convert dst to table
- dst = {dst}
+ dst = { dst }
elseif type(dst) == 'nil' then
dst = {}
end
- for i,v in ipairs(src) do
+ for i, v in ipairs(src) do
if dst[i] and k ~= 'fuzzy_stored' then
dst[i] = dst[i] + v
else
end
if type(r) == 'table' then
local res = {}
- for i,num in ipairs(r) do
+ for i, num in ipairs(r) do
res[i] = string.format('(%s: %s)', num_to_epoch(i), print_num(num))
end
opts = parser:parse(args)
if wrk then
- for _,pr in pairs(wrk) do
+ for _, pr in pairs(wrk) do
-- processes cycle
if pr['data'] then
local id = pr['id']
end
-- General stats
- for k,v in pairs(pr['data']) do
+ for k, v in pairs(pr['data']) do
if k ~= 'keys' and k ~= 'errors_ips' then
res_db[k] = add_result(res_db[k], v, k)
elseif k == 'errors_ips' then
if not res_db['errors_ips'] then
res_db['errors_ips'] = {}
end
- for ip,nerrors in pairs(v) do
+ for ip, nerrors in pairs(v) do
if not res_db['errors_ips'][ip] then
res_db['errors_ips'][ip] = nerrors
else
res_db['keys'] = res_keys
end
-- Go through keys in input
- for k,elts in pairs(pr['data']['keys']) do
+ for k, elts in pairs(pr['data']['keys']) do
-- keys cycle
if not res_keys[k] then
res_keys[k] = {}
add_data(res_keys[k], elts)
if elts['ips'] then
- for ip,v in pairs(elts['ips']) do
+ for ip, v in pairs(elts['ips']) do
if not res_ips[ip] then
res_ips[ip] = {}
end
end
-- General stats
- for db,st in pairs(res_databases) do
+ for db, st in pairs(res_databases) do
print(string.format('Statistics for storage %s', db))
- for k,v in pairs(st) do
+ for k, v in pairs(st) do
if k ~= 'keys' and k ~= 'errors_ips' then
print(string.format('%s: %s', k, print_result(v)))
end
print('\tIPs stat:')
local sorted_ips = sort_hash_table(key_stat['ips'], opts, 'ip')
- for _,v in ipairs(sorted_ips) do
+ for _, v in ipairs(sorted_ips) do
print(string.format('\t%s', v['ip']))
print_stat(v['data'], '\t\t')
print('')
if key_stat.flags then
print('')
print('\tFlags stat:')
- for flag,v in pairs(key_stat.flags) do
+ for flag, v in pairs(key_stat.flags) do
print(string.format('\t[%s]:', flag))
-- Remove irrelevant fields
v.checked = nil
if search_str and not sensitive then
search_str = string.lower(search_str)
end
- local inputs = res['input'] or {'stdin'}
+ local inputs = res['input'] or { 'stdin' }
for _, n in ipairs(inputs) do
local h, err
if buffer[hash] then
table.insert(buffer[hash], line)
else
- buffer[hash] = {line}
+ buffer[hash] = { line }
end
end
end
:argname "<file>"
)
verify:argument "file"
- :description "File to verify"
- :argname "<file>"
- :args "*"
+ :description "File to verify"
+ :argname "<file>"
+ :args "*"
verify:flag "-n --nist"
:description "Uses nistp curves (P256)"
verify:option "-s --suffix"
-- Default command is generate, so duplicate options to be compatible
parser:flag "-s --sign"
- :description "Generates a sign keypair instead of the encryption one"
+ :description "Generates a sign keypair instead of the encryption one"
parser:flag "-n --nist"
- :description "Uses nistp curves (P256)"
+ :description "Uses nistp curves (P256)"
parser:mutex(
parser:flag "-j --json"
- :description "Output JSON instead of UCL",
+ :description "Output JSON instead of UCL",
parser:flag "-u --ucl"
- :description "Output UCL"
- :default(true)
+ :description "Output UCL"
+ :default(true)
)
parser:option "-o --output"
:description "Write keypair to file"
local reply = rspamd_util.readline(greet)
- if not reply then os.exit(0) end
- if #reply == 0 then reply = def_str end
+ if not reply then
+ os.exit(0)
+ end
+ if #reply == 0 then
+ reply = def_str
+ end
reply = reply:lower()
- if reply == 'y' or reply == 'yes' then return true end
+ if reply == 'y' or reply == 'yes' then
+ return true
+ end
return false
end
local function sign_handler(opts)
if opts.file then
if type(opts.file) == 'string' then
- opts.file = {opts.file}
+ opts.file = { opts.file }
end
else
parser:error('no files to sign')
end
local ucl_parser = ucl.parser()
- local res,err = ucl_parser:parse_file(opts.keypair)
+ local res, err = ucl_parser:parse_file(opts.keypair)
if not res then
fatal(string.format('cannot load %s: %s', opts.keypair, err))
fatal("cannot load keypair: " .. opts.keypair)
end
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
local sig = rspamd_crypto.sign_file(kp, fname)
if not sig then
local function verify_handler(opts)
if opts.file then
if type(opts.file) == 'string' then
- opts.file = {opts.file}
+ opts.file = { opts.file }
end
else
parser:error('no files to verify')
if opts.keypair then
local ucl_parser = ucl.parser()
- local res,err = ucl_parser:parse_file(opts.keypair)
+ local res, err = ucl_parser:parse_file(opts.keypair)
if not res then
fatal(string.format('cannot load %s: %s', opts.keypair, err))
pk = kp:pk()
alg = kp:alg()
elseif opts.pubkey then
- if opts.nist then alg = 'nist' end
+ if opts.nist then
+ alg = 'nist'
+ end
pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg)
elseif opts.pubstr then
- if opts.nist then alg = 'nist' end
+ if opts.nist then
+ alg = 'nist'
+ end
pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg)
end
local valid = true
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
local sig_fname = string.format('%s.%s', fname, opts.suffix or 'sig')
local sig = rspamd_signature.load(sig_fname, alg)
local function encrypt_handler(opts)
if opts.file then
if type(opts.file) == 'string' then
- opts.file = {opts.file}
+ opts.file = { opts.file }
end
else
parser:error('no files to sign')
if opts.keypair then
local ucl_parser = ucl.parser()
- local res,err = ucl_parser:parse_file(opts.keypair)
+ local res, err = ucl_parser:parse_file(opts.keypair)
if not res then
fatal(string.format('cannot load %s: %s', opts.keypair, err))
pk = kp:pk()
alg = kp:alg()
elseif opts.pubkey then
- if opts.nist then alg = 'nist' end
+ if opts.nist then
+ alg = 'nist'
+ end
pk = rspamd_pubkey.load(opts.pubkey, 'sign', alg)
elseif opts.pubstr then
- if opts.nist then alg = 'nist' end
+ if opts.nist then
+ alg = 'nist'
+ end
pk = rspamd_pubkey.create(opts.pubstr, 'sign', alg)
end
fatal("cannot load keypair: " .. opts.keypair)
end
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
local enc = rspamd_crypto.encrypt_file(pk, fname, alg)
if not enc then
local function decrypt_handler(opts)
if opts.file then
if type(opts.file) == 'string' then
- opts.file = {opts.file}
+ opts.file = { opts.file }
end
else
parser:error('no files to decrypt')
end
local ucl_parser = ucl.parser()
- local res,err = ucl_parser:parse_file(opts.keypair)
+ local res, err = ucl_parser:parse_file(opts.keypair)
if not res then
fatal(string.format('cannot load %s: %s', opts.keypair, err))
fatal("cannot load keypair: " .. opts.keypair)
end
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
local decrypted = rspamd_crypto.decrypt_file(kp, fname)
if not decrypted then
return {
name = 'keypair',
- aliases = {'kp', 'key'},
+ aliases = { 'kp', 'key' },
handler = handler,
description = parser._description
}
\ No newline at end of file
parser:flag "-U --ucl"
:description "UCL output",
parser:flag "-M --messagepack"
- :description "MessagePack output"
+ :description "MessagePack output"
)
parser:flag "-C --compact"
:description "Use compact format"
:description "Output format ('raw', 'content', 'oneline', 'decoded', 'decoded_utf')"
:argname("<type>")
:convert {
- raw = "raw",
- content = "content",
- oneline = "content_oneline",
- decoded = "raw_parsed",
- decoded_utf = "raw_utf"
- }
+ raw = "raw",
+ content = "content",
+ oneline = "content_oneline",
+ decoded = "raw_parsed",
+ decoded_utf = "raw_utf"
+}
:default "content"
extract:flag "-w --words"
:description "Extracts words"
:description "Words format ('stem', 'norm', 'raw', 'full')"
:argname("<type>")
:convert {
- stem = "stem",
- norm = "norm",
- raw = "raw",
- full = "full",
- }
+ stem = "stem",
+ norm = "norm",
+ raw = "raw",
+ full = "full",
+}
:default "stem"
-
local stat = parser:command "stat st s"
- :description "Extracts statistical data from MIME messages"
+ :description "Extracts statistical data from MIME messages"
stat:argument "file"
:description "File to process"
:argname "<file>"
urls:flag "-H --host"
:description "Get hosts only",
urls:flag "-f --full"
- :description "Show piecewise urls as processed by Rspamd"
+ :description "Show piecewise urls as processed by Rspamd"
)
urls:flag "-u --unique"
:description "Reverse sort order"
local modify = parser:command "modify mod m"
- :description "Modifies MIME message"
+ :description "Modifies MIME message"
modify:argument "file"
:description "File to process"
:argname "<file>"
:argname "<file>"
local sign = parser:command "sign"
- :description "Performs DKIM signing"
+ :description "Performs DKIM signing"
sign:argument "file"
- :description "File to process"
- :argname "<file>"
- :args "+"
+ :description "File to process"
+ :argname "<file>"
+ :args "+"
sign:option "-d --domain"
:description "Use specific domain"
:description "ARC or DKIM signing"
:argname("<arc|dkim>")
:convert {
- ['arc'] = 'arc',
- ['dkim'] = 'dkim',
- }
+ ['arc'] = 'arc',
+ ['dkim'] = 'dkim',
+}
:default 'dkim'
sign:option "-o --output"
:description "Output format"
:argname("<message|signature>")
:convert {
- ['message'] = 'message',
- ['signature'] = 'signature',
- }
+ ['message'] = 'message',
+ ['signature'] = 'signature',
+}
:default 'message'
local dump = parser:command "dump"
:description "MessagePack output"
)
dump:flag "-s --split"
- :description "Split the output file contents such that no content is embedded"
+ :description "Split the output file contents such that no content is embedded"
dump:option "-o --outdir"
- :description "Output directory"
- :argname("<directory>")
+ :description "Output directory"
+ :argname("<directory>")
local function load_config(opts)
- local _r,err = rspamd_config:load_ucl(opts['config'])
+ local _r, err = rspamd_config:load_ucl(opts['config'])
if not _r then
rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
os.exit(1)
end
- _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
if not _r then
rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
os.exit(1)
fname = '-'
end
- local res,task = rspamd_task.load_from_file(fname, rspamd_config)
+ local res, task = rspamd_task.load_from_file(fname, rspamd_config)
if not res then
parser:error(string.format('cannot read message from %s: %s', fname,
local function output_fmt(opts)
local fmt = 'json'
- if opts.compact then fmt = 'json-compact' end
- if opts.ucl then fmt = 'ucl' end
- if opts.messagepack then fmt = 'msgpack' end
+ if opts.compact then
+ fmt = 'json-compact'
+ end
+ if opts.ucl then
+ fmt = 'ucl'
+ end
+ if opts.messagepack then
+ fmt = 'msgpack'
+ end
return fmt
end
if not opts.json and not opts.ucl then
table.insert(out,
rspamd_logger.slog('Part: %s: %s, language: %s, size: %s (%s raw), words: %s',
- part:get_mimepart():get_digest():sub(1,8),
- t,
- part:get_language(),
- part:get_length(), part:get_raw_length(),
- part:get_words_count()))
+ part:get_mimepart():get_digest():sub(1, 8),
+ t,
+ part:get_language(),
+ part:get_length(), part:get_raw_length(),
+ part:get_words_count()))
table.insert(out,
rspamd_logger.slog('Stats: %s',
- fun.foldl(function(acc, k, v)
- if acc ~= '' then
- return string.format('%s, %s:%s', acc, k, v)
- else
- return string.format('%s:%s', k,v)
- end
- end, '', part:get_stats())))
+ fun.foldl(function(acc, k, v)
+ if acc ~= '' then
+ return string.format('%s, %s:%s', acc, k, v)
+ else
+ return string.format('%s:%s', k, v)
+ end
+ end, '', part:get_stats())))
end
end
end
if opts.part then
if not opts.json and not opts.ucl then
- local mtype,msubtype = part:get_type()
- local det_mtype,det_msubtype = part:get_detected_type()
+ local mtype, msubtype = part:get_type()
+ local det_mtype, det_msubtype = part:get_detected_type()
table.insert(out,
rspamd_logger.slog('Mime Part: %s: %s/%s (%s/%s detected), filename: %s (%s detected ext), size: %s',
- part:get_digest():sub(1,8),
+ part:get_digest():sub(1, 8),
mtype, msubtype,
det_mtype, det_msubtype,
part:get_filename(),
end
end
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
local task = load_task(opts, fname)
out_elts[fname] = {}
if opts.text or opts.html then
local mp = task:get_parts() or {}
- for _,mime_part in ipairs(mp) do
+ for _, mime_part in ipairs(mp) do
local how = opts.output
local part
- if mime_part:is_text() then part = mime_part:get_text() end
+ if mime_part:is_text() then
+ part = mime_part:get_text()
+ end
if part and opts.text and not part:is_html() then
maybe_print_text_part_info(part, out_elts[fname])
end
hc:foreach_tag('any', function(tag)
- local elt = {}
- local ex = tag:get_extra()
- elt.tag = tag:get_type()
- if ex then
- elt.extra = ex
- end
- local content = tag:get_content()
- if content then
- elt.content = tostring(content)
- end
- local style = tag:get_style()
- if style then
- elt.style = style
- end
- table.insert(res, elt)
+ local elt = {}
+ local ex = tag:get_extra()
+ elt.tag = tag:get_type()
+ if ex then
+ elt.extra = ex
+ end
+ local content = tag:get_content()
+ if content then
+ elt.content = tostring(content)
+ end
+ local style = tag:get_style()
+ if style then
+ elt.style = style
+ end
+ table.insert(res, elt)
end)
table.insert(out_elts[fname], res)
- else -- opts.structure
+ else
+ -- opts.structure
table.insert(out_elts[fname], tostring(part:get_content(how)))
end
if opts.invisible then
print_elts(out_elts, opts, process_func)
-- To avoid use after free we postpone tasks destruction
- for _,task in ipairs(tasks) do task:destroy() end
+ for _, task in ipairs(tasks) do
+ task:destroy()
+ end
end
local function stat_handler(opts)
local process_func
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
local task = load_task(opts, fname)
out_elts[fname] = {}
process_func = function(e)
return string.format('%s (%d): "%s"+"%s", [%s]', e.data, e.win, e.t1 or "",
e.t2 or "", table.concat(fun.totable(
- fun.map(function(k) return k end, e.flags)), ","))
+ fun.map(function(k)
+ return k
+ end, e.flags)), ","))
end
elseif opts.fuzzy then
local parts = task:get_parts() or {}
local ret = string.format('part: %s(%s): %s', e.type, e.file or "", e.digest)
if opts.shingles and e.shingles then
local sgl = {}
- for _,s in ipairs(e.shingles) do
+ for _, s in ipairs(e.shingles) do
table.insert(sgl, string.format('%s: %s+%s+%s', s[1], s[2], s[3], s[4]))
end
end
return ret
end
- for _,part in ipairs(parts) do
+ for _, part in ipairs(parts) do
if not part:is_multipart() then
local text = part:get_text()
if text then
- local digest,shingles = text:get_fuzzy_hashes(task:get_mempool())
+ local digest, shingles = text:get_fuzzy_hashes(task:get_mempool())
table.insert(out_elts[fname], {
digest = digest,
shingles = shingles,
type = string.format('%s/%s',
- ({part:get_type()})[1],
- ({part:get_type()})[2])
+ ({ part:get_type() })[1],
+ ({ part:get_type() })[2])
})
else
table.insert(out_elts[fname], {
digest = part:get_digest(),
file = part:get_filename(),
type = string.format('%s/%s',
- ({part:get_type()})[1],
- ({part:get_type()})[2])
+ ({ part:get_type() })[1],
+ ({ part:get_type() })[2])
})
end
end
rspamd_url.init(rspamd_config:get_tld_path())
local out_elts = {}
- if opts.json then rspamd_logger.messagex('[') end
+ if opts.json then
+ rspamd_logger.messagex('[')
+ end
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
out_elts[fname] = {}
local task = load_task(opts, fname)
local elts = {}
end
end
- for _,u in ipairs(task:get_urls(true)) do
+ for _, u in ipairs(task:get_urls(true)) do
process_url(u)
end
end
end
-
- for s,u in lua_util.spairs(elts, sfunc) do
+ for s, u in lua_util.spairs(elts, sfunc) do
process_elt(s, u)
end
else
- for s,u in pairs(elts) do
+ for s, u in pairs(elts) do
process_elt(s, u)
end
end
html_footer = read_file(opts['html_footer'])
end
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
local task = load_task(opts, fname)
local newline_s = newline(task)
local seen_cte
local out = {} -- Start with headers
local function process_headers_cb(name, hdr)
- for _,h in ipairs(opts['remove_header']) do
+ for _, h in ipairs(opts['remove_header']) do
if name:match(h) then
return
end
end
- for _,h in ipairs(opts['rewrite_header']) do
- local hname,hpattern = h:match('^([^=]+)=(.+)$')
+ for _, h in ipairs(opts['rewrite_header']) do
+ local hname, hpattern = h:match('^([^=]+)=(.+)$')
if hname == name then
local new_value = string.format(hpattern, hdr.decoded)
new_value = string.format('%s:%s%s',
out[#out + 1] = hdr.raw:gsub('\r?\n?$', '')
end
- task:headers_foreach(process_headers_cb, {full = true})
+ task:headers_foreach(process_headers_cb, { full = true })
- for _,h in ipairs(opts['add_header']) do
- local hname,hvalue = h:match('^([^=]+)=(.+)$')
+ for _, h in ipairs(opts['add_header']) do
+ local hname, hvalue = h:match('^([^=]+)=(.+)$')
if hname and hvalue then
out[#out + 1] = string.format('%s: %s', hname,
out[#out + 1] = ''
if rewrite.out then
- for _,o in ipairs(rewrite.out) do
+ for _, o in ipairs(rewrite.out) do
out[#out + 1] = o
end
else
- out[#out + 1] = {task:get_rawbody(), false}
+ out[#out + 1] = { task:get_rawbody(), false }
end
- for _,o in ipairs(out) do
+ for _, o in ipairs(out) do
if type(o) == 'string' then
io.write(o)
io.write(newline_s)
os.exit(1)
end
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
local task = load_task(opts, fname)
local ctx = lua_dkim.create_sign_context(task, sign_key, nil, opts.algorithm)
end
if dump_content:save_in_file(outpath) then
wrote_filepath = outpath
- io.write(wrote_filepath.."\n")
+ io.write(wrote_filepath .. "\n")
else
io.stderr:write(string.format("Unable to save dump content to file: %s\n", outpath))
end
load_config(opts)
rspamd_url.init(rspamd_config:get_tld_path())
- for _,fname in ipairs(opts.file) do
+ for _, fname in ipairs(opts.file) do
local task = load_task(opts, fname)
local data, extension = get_dump_content(task, opts, fname)
write_dump_content(data, fname, extension, opts.outdir)
local command = opts.command
if type(opts.file) == 'string' then
- opts.file = {opts.file}
+ opts.file = { opts.file }
elseif type(opts.file) == 'none' then
opts.file = {}
end
return {
name = 'mime',
- aliases = {'mime_tool'},
+ aliases = { 'mime_tool' },
handler = handler,
description = parser._description
}
\ No newline at end of file
local ucl = require "ucl"
local parser = argparse()
- :name "rspamadm neural_test"
- :description "Test the neural network with labelled dataset"
- :help_description_margin(32)
+ :name "rspamadm neural_test"
+ :description "Test the neural network with labelled dataset"
+ :help_description_margin(32)
parser:option "-c --config"
:description "Path to config file"
:description 'Rule to test'
:argname('<rule>')
-
local HAM = "HAM"
local SPAM = "SPAM"
local function load_config(opts)
- local _r,err = rspamd_config:load_ucl(opts['config'])
+ local _r, err = rspamd_config:load_ucl(opts['config'])
if not _r then
rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
os.exit(1)
end
- _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
if not _r then
rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
os.exit(1)
end
end
-
local function scan_email(rspamc_path, host, n_parallel, path, timeout)
local rspamc_command = string.format("%s --connect %s -j --compact -n %s -t %.3f %s",
local function get_stats_from_scan_results(results, rules)
local rule_stats = {}
- for rule,_ in pairs(rules) do
- rule_stats[rule] = {tp = 0, tn = 0, fp = 0, fn = 0}
+ for rule, _ in pairs(rules) do
+ rule_stats[rule] = { tp = 0, tn = 0, fp = 0, fn = 0 }
end
- for _,result in ipairs(results) do
- for _,symbol in ipairs(result["symbols"]) do
- for name,rule in pairs(rules) do
+ for _, result in ipairs(results) do
+ for _, symbol in ipairs(result["symbols"]) do
+ for name, rule in pairs(rules) do
if rule.symbol_spam and rule.symbol_spam == symbol then
if result.type == HAM then
rule_stats[name].fp = rule_stats[name].fp + 1
end
end
- for rule,_ in pairs(rules) do
+ for rule, _ in pairs(rules) do
rule_stats[rule].fpr = rule_stats[rule].fp / (rule_stats[rule].fp + rule_stats[rule].tn)
rule_stats[rule].fnr = rule_stats[rule].fn / (rule_stats[rule].fn + rule_stats[rule].tp)
end
end
-
return {
name = "neuraltest",
- aliases = {"neural_test"},
+ aliases = { "neural_test" },
handler = handler,
description = parser._description
}
\ No newline at end of file
:default(rspamd_paths['CONFDIR'] .. '/rspamd.conf')
parser:command 'compile'
- :description 'Compile publicsuffix list if needed'
+ :description 'Compile publicsuffix list if needed'
local function load_config(config_file)
- local _r,err = rspamd_config:load_ucl(config_file)
+ local _r, err = rspamd_config:load_ucl(config_file)
if not _r then
rspamd_logger.errx('cannot load %s: %s', config_file, err)
os.exit(1)
end
- _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
if not _r then
rspamd_logger.errx('cannot process %s: %s', config_file, err)
os.exit(1)
load_config(cmd_opts.config_file)
-
if cmd_opts.command == 'compile' then
compile_handler(cmd_opts)
else
local logger = require "rspamd_logger"
local lua_util = require "lua_util"
-return function (_, res)
+return function(_, res)
local redis_params = lua_redis.try_load_redis_servers(res.redis, nil)
if res.expire then
res.expire = lua_util.parse_time_interval(res.expire)
return false
end
- for _,cls in ipairs(sqlite_params) do
+ for _, cls in ipairs(sqlite_params) do
if not stat_tools.convert_sqlite_to_redis(redis_params, cls.db_spam,
cls.db_ham, cls.symbol_spam, cls.symbol_ham, cls.learn_cache, res.expire,
res.reset_previous) then
logger.messagex('Converted classifier to the from sqlite to redis')
logger.messagex('Suggested configuration:')
logger.messagex(ucl.to_format(stat_tools.redis_classifier_from_sqlite(cls, res.expire),
- 'config'))
+ 'config'))
end
end
:argname "<file>"
:args "*"
restore:option "-b --batch-size"
- :description "Number of entires to process at once"
- :argname("<elts>")
- :convert(tonumber)
- :default(1000)
+ :description "Number of entires to process at once"
+ :argname("<elts>")
+ :convert(tonumber)
+ :default(1000)
restore:option "-m --mode"
:description "Number of entires to process at once"
:argname("<append|subtract|replace>")
:convert {
- ['append'] = 'append',
- ['subtract'] = 'subtract',
- ['replace'] = 'replace',
- }
+ ['append'] = 'append',
+ ['subtract'] = 'subtract',
+ ['replace'] = 'replace',
+}
:default 'append'
restore:flag "-n --no-operation"
- :description "Only show redis commands to be issued"
+ :description "Only show redis commands to be issued"
local function load_config(opts)
- local _r,err = rspamd_config:load_ucl(opts['config'])
+ local _r, err = rspamd_config:load_ucl(opts['config'])
if not _r then
rspamd_logger.errx('cannot parse %s: %s', opts['config'], err)
os.exit(1)
end
- _r,err = rspamd_config:parse_rcl({'logging', 'worker'})
+ _r, err = rspamd_config:parse_rcl({ 'logging', 'worker' })
if not _r then
rspamd_logger.errx('cannot process %s: %s', opts['config'], err)
os.exit(1)
local statfiles = cls.statfile
if statfiles[1] then
- for _,stf in ipairs(statfiles) do
+ for _, stf in ipairs(statfiles) do
if not stf.symbol then
- for k,v in pairs(stf) do
+ for k, v in pairs(stf) do
check_statfile_table(v, k)
end
else
end
end
else
- for stn,stf in pairs(statfiles) do
+ for stn, stf in pairs(statfiles) do
check_statfile_table(stf, stn)
end
end
local function redis_map_zip(ar)
local data = {}
- for j=1,#ar,2 do
+ for j = 1, #ar, 2 do
data[ar[j]] = ar[j + 1]
end
-- Used to clear tables
local clear_fcn = table.clear or function(tbl)
local keys = lua_util.keys(tbl)
- for _,k in ipairs(keys) do
+ for _, k in ipairs(keys) do
tbl[k] = nil
end
end
compress_ctx:stream(rspamd_text.fromtable(out), 'flush'):write()
end
else
- for _,o in ipairs(out) do
+ for _, o in ipairs(out) do
io.write(o)
end
end
out.cdb_builder:add('_lrnham_', rspamd_i64.fromstring(results.learns_ham or '0'))
end
- for _,o in ipairs(results.elts) do
+ for _, o in ipairs(results.elts) do
out.cdb_builder:add(o.key, o.value)
end
local cursor = 0
repeat
- conn:add_cmd('SCAN', {tostring(cursor),
- 'MATCH', pattern,
- 'COUNT', tostring(opts.batch_size)})
+ conn:add_cmd('SCAN', { tostring(cursor),
+ 'MATCH', pattern,
+ 'COUNT', tostring(opts.batch_size) })
local ret, results = conn:exec()
if not ret then
local elts = results[2]
local tokens = {}
- for _,e in ipairs(elts) do
- conn:add_cmd('HGETALL', {e})
+ for _, e in ipairs(elts) do
+ conn:add_cmd('HGETALL', { e })
end
-- This function returns many results, each for each command
-- So if we have batch 1000, then we would have 1000 tables in form
-- [result, {hash_content}]
- local all_results = {conn:exec()}
+ local all_results = { conn:exec() }
- for i=1,#all_results,2 do
+ for i = 1, #all_results, 2 do
local r, hash_content = all_results[i], all_results[i + 1]
if r then
-- List to a hash map
local data = redis_map_zip(hash_content)
- tokens[#tokens + 1] = {key = elts[(i + 1)/2], data = data}
+ tokens[#tokens + 1] = { key = elts[(i + 1) / 2], data = data }
end
end
-- Output keeping track of the commas
- for i,d in ipairs(tokens) do
+ for i, d in ipairs(tokens) do
if cursor == 0 and i == #tokens or not opts.json then
if opts.cdb then
table.insert(out[key].elts, {
local function dump_handler(opts)
local patterns_seen = {}
- for _,cls in ipairs(classifiers) do
- local res,conn = lua_redis.redis_connect_sync(cls.redis_params, false)
+ for _, cls in ipairs(classifiers) do
+ local res, conn = lua_redis.redis_connect_sync(cls.redis_params, false)
if not res then
rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params)
local function check_keys(sym)
local sym_keys_pattern = string.format("%s_keys", sym)
conn:add_cmd('SMEMBERS', { sym_keys_pattern })
- local ret,keys = conn:exec()
+ local ret, keys = conn:exec()
if not ret then
rspamd_logger.errx("cannot execute command to get keys: %s", keys)
out[#out + 1] = string.format('"%s": %s\n', sym_keys_pattern,
ucl.to_format(keys, 'json-compact'))
end
- for _,k in ipairs(keys) do
+ for _, k in ipairs(keys) do
local pat = string.format('%s*_*', k)
if not patterns_seen[pat] then
- conn:add_cmd('HGETALL', {k})
- local _ret,additional_keys = conn:exec()
+ conn:add_cmd('HGETALL', { k })
+ local _ret, additional_keys = conn:exec()
if _ret then
if opts.json then
end
local function obj_to_redis_arguments(obj, opts, cmd_pipe)
- local key,value = next(obj)
+ local key, value = next(obj)
if type(key) == 'string' then
if type(value) == 'table' then
if not value[1] then
if opts.mode == 'replace' then
local cmd = 'HMSET'
- local params = {key}
- for k,v in pairs(value) do
+ local params = { key }
+ for k, v in pairs(value) do
table.insert(params, k)
table.insert(params, v)
end
- table.insert(cmd_pipe, {cmd, params})
+ table.insert(cmd_pipe, { cmd, params })
else
local cmd = 'HINCRBYFLOAT'
local mult = 1.0
mult = (-mult)
end
- for k,v in pairs(value) do
+ for k, v in pairs(value) do
if tonumber(v) then
v = tonumber(v)
- table.insert(cmd_pipe, {cmd, {key, k, tostring(v * mult)}})
+ table.insert(cmd_pipe, { cmd, { key, k, tostring(v * mult) } })
else
- table.insert(cmd_pipe, {'HSET', {key, k, v}})
+ table.insert(cmd_pipe, { 'HSET', { key, k, v } })
end
end
end
else
-- Numeric table of elements (e.g. _keys) - it is actually a set in Redis
- for _,elt in ipairs(value) do
- table.insert(cmd_pipe, {'SADD', {key, elt}})
+ for _, elt in ipairs(value) do
+ table.insert(cmd_pipe, { 'SADD', { key, elt } })
end
end
end
local function execute_batch(batch, conns, opts)
local cmd_pipe = {}
- for _,cmd in ipairs(batch) do
+ for _, cmd in ipairs(batch) do
obj_to_redis_arguments(cmd, opts, cmd_pipe)
end
if opts.no_operation then
- for _,cmd in ipairs(cmd_pipe) do
+ for _, cmd in ipairs(cmd_pipe) do
rspamd_logger.messagex('%s %s', cmd[1], table.concat(cmd[2], ' '))
end
else
for _, conn in ipairs(conns) do
- for _,cmd in ipairs(cmd_pipe) do
+ for _, cmd in ipairs(cmd_pipe) do
local is_ok, err = conn:add_cmd(cmd[1], cmd[2])
if not is_ok then
end
local function restore_handler(opts)
- local files = opts.file or {'-'}
+ local files = opts.file or { '-' }
local conns = {}
- for _,cls in ipairs(classifiers) do
- local res,conn = lua_redis.redis_connect_sync(cls.redis_params, true)
+ for _, cls in ipairs(classifiers) do
+ local res, conn = lua_redis.redis_connect_sync(cls.redis_params, true)
if not res then
rspamd_logger.errx("cannot connect to redis server: %s", cls.redis_params)
local batch = {}
- for _,f in ipairs(files) do
+ for _, f in ipairs(files) do
local fd
if f ~= '-' then
fd = io.open(f, 'r')
for line in io.lines() do
local ucl_parser = ucl.parser()
local res, err
- res,err = ucl_parser:parse_string(line)
+ res, err = ucl_parser:parse_string(line)
if not res then
rspamd_logger.errx("%s: cannot read line %s: %s", f, cur_line, err)
end
end
- if fd then fd:close() end
+ if fd then
+ fd:close()
+ end
end
if #batch > 0 then
if classifier then
if classifier[1] then
- for _,cls in ipairs(classifier) do
- if cls.bayes then cls = cls.bayes end
+ for _, cls in ipairs(classifier) do
+ if cls.bayes then
+ cls = cls.bayes
+ end
if cls.backend and cls.backend == 'redis' then
check_redis_classifier(cls, obj)
end
classifier = classifier.bayes
if classifier[1] then
- for _,cls in ipairs(classifier) do
+ for _, cls in ipairs(classifier) do
if cls.backend and cls.backend == 'redis' then
check_redis_classifier(cls, obj)
end
end
if type(opts.file) == 'string' then
- opts.file = {opts.file}
+ opts.file = { opts.file }
elseif type(opts.file) == 'none' then
opts.file = {}
end
return {
name = 'statistics_dump',
- aliases = {'stat_dump', 'bayes_dump'},
+ aliases = { 'stat_dump', 'bayes_dump' },
handler = handler,
description = parser._description
}
\ No newline at end of file
:description "Store files with the new suffix"
:argname "<suffix>",
parser:flag "-i --inplace"
- :description "Replace input file(s)"
+ :description "Replace input file(s)"
)
local lua_util = require "lua_util"
local function set_env(opts, env)
if opts.env then
- for _,fname in ipairs(opts.env) do
+ for _, fname in ipairs(opts.env) do
for kv in assert(io.open(fname)):lines() do
if not kv:match('%s*#.*') then
- local k,v = kv:match('([^=%s]+)%s*=%s*(.+)')
+ local k, v = kv:match('([^=%s]+)%s*=%s*(.+)')
if k and v then
env[k] = v
end
if opts.lua_env then
- for _,fname in ipairs(opts.env) do
- local ret,res_or_err = pcall(loadfile(fname))
+ for _, fname in ipairs(opts.env) do
+ local ret, res_or_err = pcall(loadfile(fname))
if not ret then
io.write(string.format('cannot load %s: %s\n', fname, res_or_err))
else
if type(res_or_err) == 'table' then
- for k,v in pairs(res_or_err) do
+ for k, v in pairs(res_or_err) do
env[k] = lua_util.deepcopy(v)
end
else
local env = {}
set_env(opts, env)
- if not opts.file or #opts.file == 0 then opts.file = {'-'} end
- for _,fname in ipairs(opts.file) do
+ if not opts.file or #opts.file == 0 then
+ opts.file = { '-' }
+ end
+ for _, fname in ipairs(opts.file) do
local content = read_file(fname)
local res = lua_util.jinja_template(content, env, opts.no_vars)
:description "Output format ('ucl', 'json', 'json-compact', 'yaml')"
:argname("<type>")
:convert {
- ucl = "ucl",
- json = "json",
- ['json-compact'] = "json-compact",
- yaml = "yaml",
- }
- :default "ucl"
+ ucl = "ucl",
+ json = "json",
+ ['json-compact'] = "json-compact",
+ yaml = "yaml",
+}
+ :default "ucl"
parser:command "list ls l"
- :description "List elements in the vault"
+ :description "List elements in the vault"
local show = parser:command "show get"
- :description "Extract element from the vault"
+ :description "Extract element from the vault"
show:argument "domain"
- :description "Domain to create key for"
- :args "+"
+ :description "Domain to create key for"
+ :args "+"
local delete = parser:command "delete del rm remove"
- :description "Delete element from the vault"
+ :description "Delete element from the vault"
delete:argument "domain"
- :description "Domain to create delete key(s) for"
- :args "+"
-
+ :description "Domain to create delete key(s) for"
+ :args "+"
local newkey = parser:command "newkey new create"
:description "Add new key to the vault"
newkey:option "-A --algorithm"
:argname("<type>")
:convert {
- rsa = "rsa",
- ed25519 = "ed25519",
- eddsa = "ed25519",
- }
+ rsa = "rsa",
+ ed25519 = "ed25519",
+ eddsa = "ed25519",
+}
:default "rsa"
newkey:option "-b --bits"
:argname("<nbits>")
local function parse_vault_reply(data)
local p = ucl.parser()
- local res,parser_err = p:parse_string(data)
+ local res, parser_err = p:parse_string(data)
if not res then
- return nil,parser_err
+ return nil, parser_err
else
- return p:get_object(),nil
+ return p:get_object(), nil
end
end
local function maybe_print_vault_data(opts, data, func)
if data then
- local res,parser_err = parse_vault_reply(data)
+ local res, parser_err = parse_vault_reply(data)
if not res then
printf('vault reply for cannot be parsed: %s', parser_err)
local prefix = string.format("v=DKIM1; k=%s; p=", alg)
b64 = prefix .. b64
if #b64 < 255 then
- labels = {'"' .. b64 .. '"'}
+ labels = { '"' .. b64 .. '"' }
else
- for sl=1,#b64,256 do
+ for sl = 1, #b64, 256 do
table.insert(labels, '"' .. b64:sub(sl, sl + 255) .. '"')
end
end
local function show_handler(opts, domain)
local uri = vault_url(opts, domain)
- local err,data = rspamd_http.request{
+ local err, data = rspamd_http.request {
config = rspamd_config,
ev_base = rspamadm_ev_base,
session = rspamadm_session,
local function delete_handler(opts, domain)
local uri = vault_url(opts, domain)
- local err,data = rspamd_http.request{
+ local err, data = rspamd_http.request {
config = rspamd_config,
ev_base = rspamadm_ev_base,
session = rspamadm_session,
local function list_handler(opts)
local uri = vault_url(opts)
- local err,data = rspamd_http.request{
+ local err, data = rspamd_http.request {
config = rspamd_config,
ev_base = rspamadm_ev_base,
session = rspamadm_session,
local function create_and_push_key(opts, domain, existing)
local uri = vault_url(opts, domain)
- local sk,pk = genkey(opts)
+ local sk, pk = genkey(opts)
local res = {
selectors = {
}
}
- for _,sel in ipairs(existing) do
+ for _, sel in ipairs(existing) do
res.selectors[#res.selectors + 1] = sel
end
res.selectors[1].valid_end = os.time() + opts.expire * 3600 * 24
end
- local err,data = rspamd_http.request{
+ local err, data = rspamd_http.request {
config = rspamd_config,
ev_base = rspamadm_ev_base,
session = rspamadm_session,
maybe_print_vault_data(opts, data.content)
os.exit(1)
else
- maybe_printf(opts,'stored key for: %s, selector: %s', domain, opts.selector)
+ maybe_printf(opts, 'stored key for: %s, selector: %s', domain, opts.selector)
maybe_printf(opts, 'please place the corresponding public key as following:')
if opts.silent then
os.date("!%Y%m%d"))
end
- local err,data = rspamd_http.request{
+ local err, data = rspamd_http.request {
config = rspamd_config,
ev_base = rspamadm_ev_base,
session = rspamadm_session,
}
if is_http_error(err, data) or not data.content then
- create_and_push_key(opts, domain,{})
+ create_and_push_key(opts, domain, {})
else
-- Key exists
local rep = parse_vault_reply(data.content)
local elts = rep.data.selectors
if not elts then
- create_and_push_key(opts, domain,{})
+ create_and_push_key(opts, domain, {})
os.exit(0)
end
- for _,sel in ipairs(elts) do
+ for _, sel in ipairs(elts) do
if sel.alg == opts.algorithm then
printf('key with the specific algorithm %s is already presented at %s selector for %s domain',
opts.algorithm, sel.selector, domain)
selectors = {}
}
- local err,data = rspamd_http.request{
+ local err, data = rspamd_http.request {
config = rspamd_config,
ev_base = rspamadm_ev_base,
session = rspamadm_session,
table.insert(nkeys[sel.alg], sel)
end
- for _,sel in ipairs(elts) do
+ for _, sel in ipairs(elts) do
if sel.valid_end and sel.valid_end < os.time() then
if not opts.remove_expired then
insert_key(sel, false)
end
-- Now we need to ensure that all but one selectors have either expired or just a single key
- for alg,keys in pairs(nkeys) do
+ for alg, keys in pairs(nkeys) do
table.sort(keys, function(k1, k2)
if k1.valid_end and k2.valid_end then
return k1.valid_end > k2.valid_end
end)
-- Exclude the key with the highest expiration date and examine the rest
if not (#keys == 1 or fun.all(function(k)
- return k.valid_end and k.valid_end < os.time()
- end, fun.tail(keys))) then
+ return k.valid_end and k.valid_end < os.time()
+ end, fun.tail(keys))) then
printf('bad keys list for %s and %s algorithm', domain, alg)
fun.each(function(k)
if not k.valid_end then
if not opts.remove_expired then
-- OK to process
-- Insert keys for each algorithm in pairs <old_key(s)>, <new_key>
- local sk,pk = genkey({algorithm = alg, bits = keys[1].bits})
+ local sk, pk = genkey({ algorithm = alg, bits = keys[1].bits })
local selector = string.format('%s-%s', alg,
os.date("!%Y%m%d"))
table.insert(res.selectors, nelt)
end
- for _,k in ipairs(keys) do
+ for _, k in ipairs(keys) do
table.insert(res.selectors, k)
end
end
end
-- We can now store res in the vault
- err,data = rspamd_http.request{
+ err, data = rspamd_http.request {
config = rspamd_config,
ev_base = rspamadm_ev_base,
session = rspamadm_session,
maybe_print_vault_data(opts, data.content)
os.exit(1)
else
- for _,key in ipairs(res.selectors) do
- if not key.valid_end or key.valid_end > os.time() + opts.ttl * 3600 * 24 then
- maybe_printf(opts,'rolled key for: %s, new selector: %s', domain, key.selector)
+ for _, key in ipairs(res.selectors) do
+ if not key.valid_end or key.valid_end > os.time() + opts.ttl * 3600 * 24 then
+ maybe_printf(opts, 'rolled key for: %s, new selector: %s', domain, key.selector)
maybe_printf(opts, 'please place the corresponding public key as following:')
if opts.silent then
if command == 'list' then
list_handler(opts)
elseif command == 'show' then
- fun.each(function(d) show_handler(opts, d) end, opts.domain)
+ fun.each(function(d)
+ show_handler(opts, d)
+ end, opts.domain)
elseif command == 'newkey' then
- fun.each(function(d) newkey_handler(opts, d) end, opts.domain)
+ fun.each(function(d)
+ newkey_handler(opts, d)
+ end, opts.domain)
elseif command == 'roll' then
- fun.each(function(d) roll_handler(opts, d) end, opts.domain)
+ fun.each(function(d)
+ roll_handler(opts, d)
+ end, opts.domain)
elseif command == 'delete' then
- fun.each(function(d) delete_handler(opts, d) end, opts.domain)
+ fun.each(function(d)
+ delete_handler(opts, d)
+ end, opts.domain)
else
parser:error(string.format('command %s is not implemented', command))
end
local base58_dec = fun.tomap(fun.map(
function(c)
off = off + 1
- return c,(off - 1)
+ return c, (off - 1)
end,
"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"))
local hash = require "rspamd_cryptobox_hash"
local bytes = {}
- for i=1,25 do bytes[i] = 0 end
+ for i = 1, 25 do
+ bytes[i] = 0
+ end
-- Base58 decode loop
fun.each(function(ch)
local acc = base58_dec[ch] or 0
- for i=25,1,-1 do
+ for i = 25, 1, -1 do
acc = acc + (58 * bytes[i]);
bytes[i] = acc % 256
acc = math.floor(acc / 256);
end, word)
-- Now create a validation tag
local sha256 = hash.create_specific('sha256')
- for i=1,21 do
+ for i = 1, 21 do
sha256:update(string.char(bytes[i]))
end
sha256 = hash.create_specific('sha256', sha256:bin()):bin()
-- Compare tags
local valid = true
- for i=1,4 do
+ for i = 1, 4 do
if string.sub(sha256, i, i) ~= string.char(bytes[21 + i]) then
valid = false
end
-- Beach32 checksum combiner
local function polymod(...)
local chk = 1;
- local gen = {0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3};
- for _,t in ipairs({...}) do
- for _,v in ipairs(t) do
+ local gen = { 0x3b6a57b2, 0x26508e6d, 0x1ea119fa, 0x3d4233dd, 0x2a1462b3 };
+ for _, t in ipairs({ ... }) do
+ for _, v in ipairs(t) do
local top = bit.rshift(chk, 25)
chk = bit.bxor(bit.lshift(bit.band(chk, 0x1ffffff), 5), v)
- for i=1,5 do
+ for i = 1, 5 do
if bit.band(bit.rshift(top, i - 1), 0x1) ~= 0 then
chk = bit.bxor(chk, gen[i])
end
return polymod(hrpExpand(hrp), elts) == 1
end
-
local function gen_bleach32_table(input)
local d = {}
local i = 1
-- For semicolon
table.insert(polymod_tbl, 0)
- fun.each(function(byte) table.insert(polymod_tbl, byte) end, decoded)
+ fun.each(function(byte)
+ table.insert(polymod_tbl, byte)
+ end, decoded)
lua_util.debugm(N, task, 'final polymod table: %s', polymod_tbl)
return rspamd_util.btc_polymod(polymod_tbl)
return false
end
-
local parts = task:get_parts()
local top_type, top_subtype, params = parts[1]:get_type_full()
-- RFC 3464, RFC 8098
if top_type == 'multipart' and top_subtype == 'report' and params and
- (params['report-type'] == 'delivery-status' or params['report-type'] == 'disposition-notification') then
+ (params['report-type'] == 'delivery-status' or params['report-type'] == 'disposition-notification') then
-- Assume that inner parts are OK, don't check them to save time
return true, 1.0, 'DSN'
end
-- Check common bounce senders
if (from_user == 'postmaster' or from_user == 'mailer-daemon') then
bounce_sender = from_user
- -- MDaemon >= 14.5 sends multipart/report (RFC 3464) DSN covered above,
- -- but older versions send non-standard bounces with localized subjects and they
- -- are still around
+ -- MDaemon >= 14.5 sends multipart/report (RFC 3464) DSN covered above,
+ -- but older versions send non-standard bounces with localized subjects and they
+ -- are still around
elseif from_user == 'mdaemon' and task:has_header('X-MDDSN-Message') then
return true, 1.0, 'MDaemon'
end
end
if suspicious_factor > 0.5 then
- if suspicious_factor > 1.0 then suspicious_factor = 1.0 end
+ if suspicious_factor > 1.0 then
+ suspicious_factor = 1.0
+ end
task:insert_result('PDF_SUSPICIOUS', suspicious_factor, part:get_filename() or 'unknown')
end
local function process_specific_cb(task)
local parts = task:get_parts() or {}
- for _,p in ipairs(parts) do
+ for _, p in ipairs(parts) do
if p:is_specific() then
local data = p:get_specific()
end
end
-local id = rspamd_config:register_symbol{
+local id = rspamd_config:register_symbol {
type = 'callback',
name = 'SPECIFIC_CONTENT_CHECK',
callback = process_specific_cb
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
name = 'PDF_ENCRYPTED',
parent = id,
- groups = {"content", "pdf"},
+ groups = { "content", "pdf" },
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
name = 'PDF_JAVASCRIPT',
parent = id,
- groups = {"content", "pdf"},
+ groups = { "content", "pdf" },
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
name = 'PDF_SUSPICIOUS',
parent = id,
- groups = {"content", "pdf"},
+ groups = { "content", "pdf" },
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
name = 'PDF_LONG_TRAILER',
parent = id,
- groups = {"content", "pdf"},
+ groups = { "content", "pdf" },
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
name = 'PDF_MANY_OBJECTS',
parent = id,
- groups = {"content", "pdf"},
+ groups = { "content", "pdf" },
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
name = 'PDF_TIMEOUT',
parent = id,
- groups = {"content", "pdf"},
+ groups = { "content", "pdf" },
}
local ret, hashes
task:process_message()
if req_params.rule then
- ret,hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, req_params.rule)
+ ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, req_params.rule)
elseif req_params.flag then
- ret,hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, tonumber(req_params.flag))
+ ret, hashes = pcall(rspamd_plugins.fuzzy_check.hex_hashes, task, tonumber(req_params.flag))
else
conn:send_error(404, 'missing rule or flag')
return
end
if ret then
- conn:send_ucl({success = true, hashes = hashes})
+ conn:send_ucl({ success = true, hashes = hashes })
else
conn:send_error(500, 'cannot generate hashes')
end
end
end
-for plug,paths in pairs(controller_plugin_paths) do
+for plug, paths in pairs(controller_plugin_paths) do
if not rspamd_plugins[plug] then
rspamd_plugins[plug] = {}
end
local webui = rspamd_plugins[plug].webui
- for path,attrs in pairs(paths) do
+ for path, attrs in pairs(paths) do
if type(attrs) == 'table' then
if type(attrs.handler) ~= 'function' then
rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid handler: %s; ignore it',
end
else
rspamd_logger.infox(rspamd_config, 'controller plugin %s; webui path %s has invalid type: %s; ignore it',
- plug, path, type(attrs))
+ plug, path, type(attrs))
end
end
end
maps_cache = {}
maps_aliases = {}
local maps = rspamd_config:get_maps()
- for _,m in ipairs(maps) do
+ for _, m in ipairs(maps) do
-- We get the first url here and that's it
local url = m:get_uri()
if url ~= 'static' then
end
local results = {}
- for _,key in ipairs(keys_to_check) do
- for uri,m in pairs(maps_cache) do
+ for _, key in ipairs(keys_to_check) do
+ for uri, m in pairs(maps_cache) do
check_specific_map(key, uri, m, results, req_params.report_misses)
end
end
- conn:send_ucl{
+ conn:send_ucl {
success = (#results > 0),
results = results
}
if req_params.maps then
local map_names = lua_util.str_split(req_params.maps, ',')
maps_to_check = {}
- for _,mn in ipairs(map_names) do
+ for _, mn in ipairs(map_names) do
if maps_cache[mn] then
maps_to_check[mn] = maps_cache[mn]
else
end
local results = {}
- for _,key in ipairs(keys_to_check) do
- for uri,m in pairs(maps_to_check) do
+ for _, key in ipairs(keys_to_check) do
+ for uri, m in pairs(maps_to_check) do
check_specific_map(key, uri, m, results, req_params.report_misses)
end
end
- conn:send_ucl{
+ conn:send_ucl {
success = (#results > 0),
results = results
}
local function handle_list_maps(_, conn, _)
maybe_fill_maps_cache()
- conn:send_ucl{
+ conn:send_ucl {
maps = lua_util.keys(maps_cache),
aliases = maps_aliases
}
end
-local query_json_schema = ts.shape{
+local query_json_schema = ts.shape {
maps = ts.array_of(ts.string):is_optional(),
report_misses = ts.boolean:is_optional(),
values = ts.array_of(ts.string),
local results = {}
if obj.maps then
- for _,mn in ipairs(obj.maps) do
+ for _, mn in ipairs(obj.maps) do
if maps_cache[mn] then
maps_to_check[mn] = maps_cache[mn]
else
maps_to_check = maps_cache
end
- for _,key in ipairs(obj.values) do
- for uri,m in pairs(maps_to_check) do
+ for _, key in ipairs(obj.values) do
+ for uri, m in pairs(maps_to_check) do
check_specific_map(key, uri, m, results, report_misses)
end
end
- conn:send_ucl{
+ conn:send_ucl {
success = (#results > 0),
results = results
}
-- Controller neural plugin
-local learn_request_schema = ts.shape{
+local learn_request_schema = ts.shape {
ham_vec = ts.array_of(ts.array_of(ts.number)),
rule = ts.string:is_optional(),
spam_vec = ts.array_of(ts.array_of(ts.number)),
local set = neural_common.get_rule_settings(task, rule)
local version = ((set.ann or E).version or 0) + 1
- neural_common.spawn_train{
+ neural_common.spawn_train {
ev_base = task:get_ev_base(),
ann_key = neural_common.new_ann_key(rule, set, version),
set = set,
if req_params.selector and req_params.selector ~= '' then
local selector = lua_selectors.create_selector_closure(rspamd_config,
req_params.selector, '', true)
- conn:send_ucl({success = selector and true})
+ conn:send_ucl({ success = selector and true })
else
conn:send_error(404, 'missing selector')
end
else
task:process_message()
local elts = selector(task)
- conn:send_ucl({success = true, data = elts})
+ conn:send_ucl({ success = true, data = elts })
end
else
conn:send_error(404, 'missing selector')
local rspamd_util = require "rspamd_util"
rspamd_config.FWD_GOOGLE = {
- callback = function (task)
+ callback = function(task)
if not (task:has_from(1) and task:has_recipients(1)) then
return false
end
- local envfrom = task:get_from{'smtp', 'orig'}
+ local envfrom = task:get_from { 'smtp', 'orig' }
local envrcpts = task:get_recipients(1)
-- Forwarding will only be to a single recipient
- if #envrcpts > 1 then return false end
+ if #envrcpts > 1 then
+ return false
+ end
-- Get recipient and compute VERP address
local rcpt = envrcpts[1].addr:lower()
- local verp = rcpt:gsub('@','=')
+ local verp = rcpt:gsub('@', '=')
-- Get the user portion of the envfrom
local ef_user = envfrom[1].user:lower()
-- Check for a match
if ef_user:find('+caf_=' .. verp, 1, true) then
- local _,_,user = ef_user:find('^(.+)+caf_=')
+ local _, _, user = ef_user:find('^(.+)+caf_=')
if user then
user = user .. '@' .. envfrom[1].domain
return true, user
}
rspamd_config.FWD_YANDEX = {
- callback = function (task)
+ callback = function(task)
if not (task:has_from(1) and task:has_recipients(1)) then
return false
end
}
rspamd_config.FWD_MAILRU = {
- callback = function (task)
+ callback = function(task)
if not (task:has_from(1) and task:has_recipients(1)) then
return false
end
}
rspamd_config.FWD_SRS = {
- callback = function (task)
+ callback = function(task)
if not (task:has_from(1) and task:has_recipients(1)) then
return false
end
local envfrom = task:get_from(1)
local envrcpts = task:get_recipients(1)
-- Forwarding is only to a single recipient
- if #envrcpts > 1 then return false end
+ if #envrcpts > 1 then
+ return false
+ end
-- Get recipient and compute rewritten SRS address
local srs = '=' .. envrcpts[1].domain:lower() ..
'=' .. envrcpts[1].user:lower()
}
rspamd_config.FORWARDED = {
- callback = function (task)
+ callback = function(task)
local function normalize_addr(addr)
addr = string.match(addr, '^<?([^>]*)>?$') or addr
- local cap, _,domain = string.match(addr, '^([^%+][^%+]*)(%+[^@]*)@(.*)$')
+ local cap, _, domain = string.match(addr, '^([^%+][^%+]*)(%+[^@]*)@(.*)$')
if cap then
addr = string.format('%s@%s', cap, domain)
end
return addr
end
- if not task:has_recipients(1) or not task:has_recipients(2) then return false end
+ if not task:has_recipients(1) or not task:has_recipients(2) then
+ return false
+ end
local envrcpts = task:get_recipients(1)
-- Forwarding will only be for single recipient messages
- if #envrcpts > 1 then return false end
+ if #envrcpts > 1 then
+ return false
+ end
-- Get any other headers we might need
local has_list_unsub = task:has_header('List-Unsubscribe')
local to = task:get_recipients(2)
local fun = require "fun"
local E = {}
-local rcvd_cb_id = rspamd_config:register_symbol{
+local rcvd_cb_id = rspamd_config:register_symbol {
name = 'CHECK_RECEIVED',
type = 'callback',
score = 0.0,
local def = 'ZERO'
local received = task:get_received_headers()
local nreceived = fun.reduce(function(acc, rcvd)
- return acc + 1
- end, 0, fun.filter(function(h)
- return not h['flags']['artificial']
- end, received))
+ return acc + 1
+ end, 0, fun.filter(function(h)
+ return not h['flags']['artificial']
+ end, received))
- for k,v in pairs(cnts) do
+ for k, v in pairs(cnts) do
if nreceived >= tonumber(k) then
def = v
end
end
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCVD_COUNT_ZERO',
score = 0.0,
parent = rcvd_cb_id,
description = 'Message has no Received headers',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCVD_COUNT_ONE',
score = 0.0,
parent = rcvd_cb_id,
description = 'Message has one Received header',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCVD_COUNT_TWO',
score = 0.0,
parent = rcvd_cb_id,
description = 'Message has two Received headers',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCVD_COUNT_THREE',
score = 0.0,
parent = rcvd_cb_id,
description = 'Message has 3-5 Received headers',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCVD_COUNT_FIVE',
score = 0.0,
parent = rcvd_cb_id,
description = 'Message has 5-7 Received headers',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCVD_COUNT_SEVEN',
score = 0.0,
parent = rcvd_cb_id,
description = 'Message has 7-11 Received headers',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCVD_COUNT_TWELVE',
score = 0.0,
parent = rcvd_cb_id,
description = 'X-Priority check callback rule',
score = 0.0,
group = 'headers',
- callback = function (task)
- local cnts = {
+ callback = function(task)
+ local cnts = {
[1] = 'ONE',
[2] = 'TWO',
[3] = 'THREE',
}
local def = 'ZERO'
local xprio = task:get_header('X-Priority');
- if not xprio then return false end
- local _,_,x = xprio:find('^%s?(%d+)');
+ if not xprio then
+ return false
+ end
+ local _, _, x = xprio:find('^%s?(%d+)');
if (x) then
x = tonumber(x)
- for k,v in pairs(cnts) do
+ for k, v in pairs(cnts) do
if x >= tonumber(k) then
def = v
end
end
end
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'HAS_X_PRIO_ZERO',
score = 0.0,
parent = prio_cb_id,
description = 'Message has X-Priority header set to 0',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'HAS_X_PRIO_ONE',
score = 0.0,
parent = prio_cb_id,
description = 'Message has X-Priority header set to 1',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'HAS_X_PRIO_TWO',
score = 0.0,
parent = prio_cb_id,
description = 'Message has X-Priority header set to 2',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'HAS_X_PRIO_THREE',
score = 0.0,
parent = prio_cb_id,
description = 'Message has X-Priority header set to 3 or 4',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'HAS_X_PRIO_FIVE',
score = 0.0,
parent = prio_cb_id,
end
-- See if Reply-To matches From in some way
- local from = task:get_from{'mime', 'orig'}
+ local from = task:get_from { 'mime', 'orig' }
local from_h = get_raw_header(task, 'From')
if not (from and from[1]) then
return false
end
})
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'REPLYTO_UNPARSEABLE',
score = 1.0,
parent = check_replyto_id,
description = 'Reply-To header could not be parsed',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'HAS_REPLYTO',
score = 0.0,
parent = check_replyto_id,
description = 'Has Reply-To header',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'REPLYTO_EQ_FROM',
score = 0.0,
parent = check_replyto_id,
description = 'Reply-To header is identical to From header',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'REPLYTO_ADDR_EQ_FROM',
score = 0.0,
parent = check_replyto_id,
description = 'Reply-To header is identical to SMTP From',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'REPLYTO_DOM_EQ_FROM_DOM',
score = 0.0,
parent = check_replyto_id,
description = 'Reply-To domain matches the From domain',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'REPLYTO_DOM_NEQ_FROM_DOM',
score = 0.0,
parent = check_replyto_id,
description = 'Reply-To domain does not match the From domain',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'REPLYTO_DN_EQ_FROM_DN',
score = 0.0,
parent = check_replyto_id,
description = 'Reply-To display name matches From',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'REPLYTO_EMAIL_HAS_TITLE',
score = 2.0,
parent = check_replyto_id,
description = 'Reply-To header has title',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'REPLYTO_EQ_TO_ADDR',
score = 5.0,
parent = check_replyto_id,
rspamd_config:register_dependency('CHECK_REPLYTO', 'CHECK_FROM')
-local check_mime_id = rspamd_config:register_symbol{
+local check_mime_id = rspamd_config:register_symbol {
name = 'CHECK_MIME',
type = 'callback',
group = 'headers',
end
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'MISSING_MIME_VERSION',
score = 2.0,
parent = check_mime_id,
description = 'MIME-Version header is missing in MIME message',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'MIME_MA_MISSING_TEXT',
score = 2.0,
parent = check_mime_id,
description = 'MIME multipart/alternative missing text/plain part',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'MIME_MA_MISSING_HTML',
score = 1.0,
parent = check_mime_id,
-- Used to be called IS_LIST
rspamd_config.PREVIOUSLY_DELIVERED = {
callback = function(task)
- if not task:has_recipients(2) then return false end
+ if not task:has_recipients(2) then
+ return false
+ end
local to = task:get_recipients(2)
local rcvds = task:get_header_full('Received')
- if not rcvds then return false end
+ if not rcvds then
+ return false
+ end
for _, rcvd in ipairs(rcvds) do
- local _,_,addr = rcvd['decoded']:lower():find("%sfor%s<(.-)>")
+ local _, _, addr = rcvd['decoded']:lower():find("%sfor%s<(.-)>")
if addr then
for _, toa in ipairs(to) do
if toa and toa.addr:lower() == addr then
rspamd_config.BROKEN_CONTENT_TYPE = {
callback = function(task)
- return fun.any(function(p) return p:is_broken() end,
- task:get_parts())
+ return fun.any(function(p)
+ return p:is_broken()
+ end,
+ task:get_parts())
end,
score = 1.5,
group = 'headers',
}
rspamd_config.HEADER_RCONFIRM_MISMATCH = {
- callback = function (task)
+ callback = function(task)
local header_from = nil
local cread = task:get_header('X-Confirm-Reading-To')
if task:has_from('mime') then
- header_from = task:get_from('mime')[1]
+ header_from = task:get_from('mime')[1]
end
local header_cread = nil
if cread then
local headers_cread = util.parse_mail_address(cread, task:get_mempool())
- if headers_cread then header_cread = headers_cread[1] end
+ if headers_cread then
+ header_cread = headers_cread[1]
+ end
end
if header_from and header_cread then
}
rspamd_config.HEADER_FORGED_MDN = {
- callback = function (task)
+ callback = function(task)
local mdn = task:get_header('Disposition-Notification-To')
- if not mdn then return false end
+ if not mdn then
+ return false
+ end
local header_rp = nil
if task:has_from('smtp') then
-- Parse mail addr
local headers_mdn = util.parse_mail_address(mdn, task:get_mempool())
- if headers_mdn and not header_rp then return true end
- if header_rp and not headers_mdn then return false end
- if not headers_mdn and not header_rp then return false end
+ if headers_mdn and not header_rp then
+ return true
+ end
+ if header_rp and not headers_mdn then
+ return false
+ end
+ if not headers_mdn and not header_rp then
+ return false
+ end
local found_match = false
for _, h in ipairs(headers_mdn) do
local max_mult = 0.0
local res_tbl = {}
- for hdr,mult in pairs(headers_unique) do
+ for hdr, mult in pairs(headers_unique) do
local hc = task:get_header_count(hdr)
if hc > 1 then
end
if res > 0 then
- return true,max_mult,table.concat(res_tbl, ',')
+ return true, max_mult, table.concat(res_tbl, ',')
end
return false
callback = function(task)
local from = task:get_from('mime')
if from and from[2] then
- return true, 1.0, fun.totable(fun.map(function(a) return a.raw end, from))
+ return true, 1.0, fun.totable(fun.map(function(a)
+ return a.raw
+ end, from))
end
return false
end,
}
rspamd_config.MV_CASE = {
- callback = function (task)
+ callback = function(task)
return task:has_header('Mime-Version', true)
end,
description = 'Mime-Version .vs. MIME-Version',
group = 'headers'
}
-local check_from_id = rspamd_config:register_symbol{
+local check_from_id = rspamd_config:register_symbol {
name = 'CHECK_FROM',
type = 'callback',
score = 0.0,
if not (from[1]["flags"]["valid"]) then
task:insert_result('FROM_INVALID', 1.0)
end
- if (from[1].name == nil or from[1].name == '' ) then
+ if (from[1].name == nil or from[1].name == '') then
task:insert_result('FROM_NO_DN', 1.0)
elseif (from[1].name and
- util.strequal_caseless(from[1].name, from[1].addr)) then
+ util.strequal_caseless(from[1].name, from[1].addr)) then
task:insert_result('FROM_DN_EQ_ADDR', 1.0)
elseif (from[1].name and from[1].name ~= '') then
task:insert_result('FROM_HAS_DN', 1.0)
local match, match_end
match, match_end = n:find('^mrs?[%.%s]')
if match then
- task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end-1))
+ task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end - 1))
end
match, match_end = n:find('^dr[%.%s]')
if match then
- task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end-1))
+ task:insert_result('FROM_NAME_HAS_TITLE', 1.0, n:sub(match, match_end - 1))
end
-- Check for excess spaces
if n:find('%s%s') then
end
local to = task:get_recipients(2)
- if not (to and to[1] and #to == 1 and from and from[1]) then return false end
+ if not (to and to[1] and #to == 1 and from and from[1]) then
+ return false
+ end
-- Check if FROM == TO
if (util.strequal_caseless(to[1].addr, from[1].addr)) then
task:insert_result('TO_EQ_FROM', 1.0)
elseif (to[1].domain and from[1].domain and
- util.strequal_caseless(to[1].domain, from[1].domain))
+ util.strequal_caseless(to[1].domain, from[1].domain))
then
task:insert_result('TO_DOM_EQ_FROM_DOM', 1.0)
end
end
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'ENVFROM_INVALID',
score = 2.0,
group = 'headers',
type = 'virtual',
description = 'Envelope from does not have a valid format',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'FROM_INVALID',
score = 2.0,
group = 'headers',
type = 'virtual',
description = 'From header does not have a valid format',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'FROM_NO_DN',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'From header does not have a display name',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'FROM_DN_EQ_ADDR',
score = 1.0,
group = 'headers',
type = 'virtual',
description = 'From header display name is the same as the address',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'FROM_HAS_DN',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'From header has a display name',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'FROM_NAME_EXCESS_SPACE',
score = 1.0,
group = 'headers',
type = 'virtual',
description = 'From header display name contains excess whitespace',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'FROM_NAME_HAS_TITLE',
score = 1.0,
group = 'headers',
type = 'virtual',
description = 'From header display name has a title (Mr/Mrs/Dr)',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'FROM_EQ_ENVFROM',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'From address is the same as the envelope',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'FROM_NEQ_ENVFROM',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'From address is different to the envelope',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_EQ_FROM',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'To address matches the From address',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_DOM_EQ_FROM_DOM',
score = 0.0,
group = 'headers',
description = 'To domain is the same as the From domain',
}
-local check_to_cc_id = rspamd_config:register_symbol{
+local check_to_cc_id = rspamd_config:register_symbol {
name = 'CHECK_TO_CC',
type = 'callback',
score = 0.0,
[50] = 'GT_50'
}
local def = 'ZERO'
- if (not to) then return false end
+ if (not to) then
+ return false
+ end
-- Add symbol for recipient count
local nrcpt = #to
- for k,v in pairs(cnts) do
+ for k, v in pairs(cnts) do
if nrcpt >= tonumber(k) then
def = v
end
end
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCPT_COUNT_ZERO',
score = 0.0,
parent = check_to_cc_id,
description = 'No recipients',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCPT_COUNT_ONE',
score = 0.0,
parent = check_to_cc_id,
description = 'One recipient',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCPT_COUNT_TWO',
score = 0.0,
parent = check_to_cc_id,
description = 'Two recipients',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCPT_COUNT_THREE',
score = 0.0,
parent = check_to_cc_id,
description = '3-5 recipients',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCPT_COUNT_FIVE',
score = 0.0,
parent = check_to_cc_id,
description = '5-7 recipients',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCPT_COUNT_SEVEN',
score = 0.0,
parent = check_to_cc_id,
description = '7-11 recipients',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCPT_COUNT_TWELVE',
score = 0.0,
parent = check_to_cc_id,
description = '12-50 recipients',
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'RCPT_COUNT_GT_50',
score = 0.0,
parent = check_to_cc_id,
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_DN_RECIPIENTS',
score = 2.0,
group = 'headers',
type = 'virtual',
description = 'To header display name is "Recipients"',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_DN_NONE',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'None of the recipients have display names',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_DN_ALL',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'All the recipients have display names',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_DN_SOME',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'Some of the recipients have display names',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_DN_EQ_ADDR_ALL',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'All of the recipients have display names that are the same as their address',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_DN_EQ_ADDR_SOME',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'Some of the recipients have display names that are the same as their address',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_MATCH_ENVRCPT_ALL',
score = 0.0,
group = 'headers',
type = 'virtual',
description = 'All of the recipients match the envelope',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
name = 'TO_MATCH_ENVRCPT_SOME',
score = 0.0,
group = 'headers',
rspamd_config.CTYPE_MISSING_DISPOSITION = {
callback = function(task)
local parts = task:get_parts()
- if (not parts) or (parts and #parts < 1) then return false end
- for _,p in ipairs(parts) do
+ if (not parts) or (parts and #parts < 1) then
+ return false
+ end
+ for _, p in ipairs(parts) do
local ct = p:get_header('Content-Type')
if (ct and ct:lower():match('^application/octet%-stream') ~= nil) then
local cd = p:get_header('Content-Disposition')
local parent = p:get_parent()
if parent then
- local t,st = parent:get_type()
+ local t, st = parent:get_type()
if t == 'multipart' and st == 'encrypted' then
-- Special case
rspamd_config.CTYPE_MIXED_BOGUS = {
callback = function(task)
local ct = task:get_header('Content-Type')
- if (not ct) then return false end
+ if (not ct) then
+ return false
+ end
local parts = task:get_parts()
- if (not parts) then return false end
- if (not ct:lower():match('^multipart/mixed')) then return false end
+ if (not parts) then
+ return false
+ end
+ if (not ct:lower():match('^multipart/mixed')) then
+ return false
+ end
local found = false
-- Check each part and look for a part that isn't multipart/* or text/plain or text/html
local ntext_parts = 0
- for _,p in ipairs(parts) do
- local mtype,_ = p:get_type()
+ for _, p in ipairs(parts) do
+ local mtype, _ = p:get_type()
if mtype then
if mtype == 'text' and not p:is_attachment() then
ntext_parts = ntext_parts + 1
end
end
end
- if (not found) then return true end
+ if (not found) then
+ return true
+ end
return false
end,
description = 'multipart/mixed without non-textual part',
local function check_for_base64_text(part)
local ct = part:get_header('Content-Type')
- if (not ct) then return false end
+ if (not ct) then
+ return false
+ end
ct = ct:lower()
if (ct:match('^text')) then
-- Check encoding
return true
else
local parts = task:get_parts()
- if (not parts) then return false end
+ if (not parts) then
+ return false
+ end
-- Check each part and look for base64 encoded text parts
for _, part in ipairs(parts) do
if (check_for_base64_text(part)) then
rspamd_config.MIME_BASE64_TEXT_BOGUS = {
callback = function(task)
local parts = task:get_text_parts()
- if (not parts) then return false end
+ if (not parts) then
+ return false
+ end
-- Check each part and look for base64 encoded text parts
-- where the part does not have any 8bit characters within it
for _, part in ipairs(parts) do
}
rspamd_config.XM_CASE = {
- callback = function (task)
+ callback = function(task)
return task:has_header('X-mailer', true)
end,
description = 'X-mailer .vs. X-Mailer',
local function check_html_image(task, min, max)
local tp = task:get_text_parts()
- for _,p in ipairs(tp) do
+ for _, p in ipairs(tp) do
if p:is_html() then
local hc = p:get_html()
local len = p:get_length()
-
if hc and len >= min and len < max then
local images = hc:get_images()
if images then
- for _,i in ipairs(images) do
+ for _, i in ipairs(images) do
local tag = i['tag']
if tag then
if has_anchor_parent(tag) then
callback = function(task)
local tp = task:get_text_parts() -- get text parts in a message
- for _,p in ipairs(tp) do -- iterate over text parts array using `ipairs`
- if p:is_html() then -- if the current part is html part
+ for _, p in ipairs(tp) do
+ -- iterate over text parts array using `ipairs`
+ if p:is_html() then
+ -- if the current part is html part
local hc = p:get_html() -- we get HTML context
local len = p:get_length() -- and part's length
- if hc and len < 50 then -- if we have a part that has less than 50 bytes of text
+ if hc and len < 50 then
+ -- if we have a part that has less than 50 bytes of text
local images = hc:get_images() -- then we check for HTML images
- if images then -- if there are images
- for _,i in ipairs(images) do -- then iterate over images in the part
- if i['height'] + i['width'] >= 400 then -- if we have a large image
+ if images then
+ -- if there are images
+ for _, i in ipairs(images) do
+ -- then iterate over images in the part
+ if i['height'] + i['width'] >= 400 then
+ -- if we have a large image
local tag = i['tag']
if tag then
if not has_anchor_parent(tag) then
description = 'Message contains many suspicious messages'
}
-local vis_check_id = rspamd_config:register_symbol{
+local vis_check_id = rspamd_config:register_symbol {
name = 'HTML_VISIBLE_CHECKS',
type = 'callback',
group = 'html',
local normal_len = 0
local transp_len = 0
- for _,p in ipairs(tp) do -- iterate over text parts array using `ipairs`
+ for _, p in ipairs(tp) do
+ -- iterate over text parts array using `ipairs`
normal_len = normal_len + p:get_length()
- if p:is_html() and p:get_html() then -- if the current part is html part
+ if p:is_html() and p:get_html() then
+ -- if the current part is html part
local hc = p:get_html() -- we get HTML context
- hc:foreach_tag({'font', 'span', 'div', 'p', 'td'}, function(tag, clen, is_leaf)
+ hc:foreach_tag({ 'font', 'span', 'div', 'p', 'td' }, function(tag, clen, is_leaf)
local bl = tag:get_style()
if bl then
if not bl.visible and clen > 0 and is_leaf then
local tr = transp_len / (normal_len + transp_len)
if tr > transp_rate then
transp_rate = tr
- if not bl.color then bl.color = {0, 0, 0} end
- if not bl.bgcolor then bl.bgcolor = {0, 0, 0} end
+ if not bl.color then
+ bl.color = { 0, 0, 0 }
+ end
+ if not bl.bgcolor then
+ bl.bgcolor = { 0, 0, 0 }
+ end
arg = string.format('%s color #%x%x%x bgcolor #%x%x%x',
tag:get_type(),
bl.color[1], bl.color[2], bl.color[3],
end,
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = vis_check_id,
name = 'R_WHITE_ON_WHITE',
one_shot = true,
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = vis_check_id,
name = 'ZERO_FONT',
group = 'html'
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = vis_check_id,
name = 'MANY_INVISIBLE_PARTS',
local re = regexp_lib.create_cached('/^.*\\.css(?:[?#].*)?$/i')
local tp = task:get_text_parts() -- get text parts in a message
local ret = false
- for _,p in ipairs(tp) do -- iterate over text parts array using `ipairs`
- if p:is_html() and p:get_html() then -- if the current part is html part
+ for _, p in ipairs(tp) do
+ -- iterate over text parts array using `ipairs`
+ if p:is_html() and p:get_html() then
+ -- if the current part is html part
local hc = p:get_html() -- we get HTML context
- hc:foreach_tag({'link'}, function(tag)
+ hc:foreach_tag({ 'link' }, function(tag)
local bl = tag:get_extra()
if bl then
local s = tostring(bl)
local found_opts
local tp = task:get_text_parts() or {}
- for _,p in ipairs(tp) do
+ for _, p in ipairs(tp) do
if p:is_html() then
local hc = p:get_html()
- if (not hc) then return false end
+ if (not hc) then
+ return false
+ end
local found = false
- hc:foreach_tag('a', function (tag, _)
+ hc:foreach_tag('a', function(tag, _)
-- Skip this loop if we already have a match
- if (found) then return true end
+ if (found) then
+ return true
+ end
local c = tag:get_content()
if (c) then
- if (not https_re:match(c)) then return false end
+ if (not https_re:match(c)) then
+ return false
+ end
local u = tag:get_extra()
- if (not u) then return false end
+ if (not u) then
+ return false
+ end
local url_proto = u:get_protocol()
- if url_proto ~= 'http' then return false end
+ if url_proto ~= 'http' then
+ return false
+ end
-- Capture matches for http in href to https in visible part only
found = true
found_opts = u:get_host()
end)
if (found) then
- return true,1.0,found_opts
+ return true, 1.0, found_opts
end
return false
rspamd_config.HTTP_TO_IP = {
callback = function(task)
local tp = task:get_text_parts()
- if (not tp) then return false end
- for _,p in ipairs(tp) do
+ if (not tp) then
+ return false
+ end
+ for _, p in ipairs(tp) do
if p:is_html() then
local hc = p:get_html()
- if (not hc) then return false end
+ if (not hc) then
+ return false
+ end
local found = false
- hc:foreach_tag('a', function (tag, length)
- if (found) then return true end
+ hc:foreach_tag('a', function(tag, length)
+ if (found) then
+ return true
+ end
local u = tag:get_extra()
if (u) then
u = tostring(u):lower()
end
return false
end)
- if found then return true end
+ if found then
+ return true
+ end
return false
end
end
local rspamd_util = require "rspamd_util"
local function mid_check_func(task)
local mid = task:get_header('Message-ID')
- if not mid then return false end
+ if not mid then
+ return false
+ end
-- Check for 'bare' IP addresses in RHS
if mid:find("@%d+%.%d+%.%d+%.%d+>$") then
task:insert_result('MID_BARE_IP', 1.0)
local fd
if (from and from[1] and from[1].domain and from[1].domain ~= '') then
fd = from[1].domain:lower()
- local _,_,md = mid:find("@([^>]+)>?$")
+ local _, _, md = mid:find("@([^>]+)>?$")
-- See if all or part of the From address
-- can be found in the Message-ID
-- extract tld
fdtld = rspamd_util.get_tld(fd)
mdtld = rspamd_util.get_tld(md)
end
- if (mid:lower():find(from[1].addr:lower(),1,true)) then
+ if (mid:lower():find(from[1].addr:lower(), 1, true)) then
task:insert_result('MID_CONTAINS_FROM', 1.0)
elseif (md and fd == md:lower()) then
task:insert_result('MID_RHS_MATCH_FROM', 1.0)
local to = task:get_recipients(2)
if (to and to[1] and to[1].domain and to[1].domain ~= '') then
local td = to[1].domain:lower()
- local _,_,md = mid:find("@([^>]+)>?$")
+ local _, _, md = mid:find("@([^>]+)>?$")
-- Skip if from domain == to domain
if ((fd and fd ~= td) or not fd) then
-- See if all or part of the To address
-- can be found in the Message-ID
- if (mid:lower():find(to[1].addr:lower(),1,true)) then
+ if (mid:lower():find(to[1].addr:lower(), 1, true)) then
task:insert_result('MID_CONTAINS_TO', 1.0)
elseif (md and td == md:lower()) then
task:insert_result('MID_RHS_MATCH_TO', 1.0)
callback = function(task)
local mid = task:get_header('Message-ID')
- if not mid then return end
+ if not mid then
+ return
+ end
local mime_from = task:get_from('mime')
- local _,_,mid_realm = mid:find("@([a-z]+)>?$")
+ local _, _, mid_realm = mid:find("@([a-z]+)>?$")
if mid_realm and mime_from and mime_from[1] and mime_from[1].user then
if (mid_realm == mime_from[1].user) then
return true
score = (nd - 0.5)
end
task:insert_result('R_PARTS_DIFFER', score,
- string.format('%.1f%%', tostring(100.0 * nd)))
+ string.format('%.1f%%', tostring(100.0 * nd)))
end
end
end
return
end
- local dt = task:get_date({format = 'connect', gmt = true})
+ local dt = task:get_date({ format = 'connect', gmt = true })
local date_diff = dt - dm
if date_diff > 86400 then
-- Older than a day
- task:insert_result('DATE_IN_PAST', 1.0, tostring(math.floor(date_diff/3600)))
+ task:insert_result('DATE_IN_PAST', 1.0, tostring(math.floor(date_diff / 3600)))
elseif -date_diff > 7200 then
-- More than 2 hours in the future
- task:insert_result('DATE_IN_FUTURE', 1.0, tostring(math.floor(-date_diff/3600)))
+ task:insert_result('DATE_IN_FUTURE', 1.0, tostring(math.floor(-date_diff / 3600)))
end
end
})
parent = date_id,
})
-local obscured_id = rspamd_config:register_symbol{
+local obscured_id = rspamd_config:register_symbol {
callback = function(task)
- local susp_urls = task:get_urls_filtered({ 'obscured', 'zw_spaces'})
+ local susp_urls = task:get_urls_filtered({ 'obscured', 'zw_spaces' })
if susp_urls and susp_urls[1] then
local obs_flag = url_flags_tab.obscured
local zw_flag = url_flags_tab.zw_spaces
- for _,u in ipairs(susp_urls) do
+ for _, u in ipairs(susp_urls) do
local fl = u:get_flags_num()
if bit.band(fl, obs_flag) ~= 0 then
task:insert_result('R_SUSPICIOUS_URL', 1.0, u:get_host())
group = 'url'
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
name = 'ZERO_WIDTH_SPACE_URL',
score = 7.0,
parent = obscured_id,
}
-
rspamd_config.ENVFROM_PRVS = {
- callback = function (task)
+ callback = function(task)
--[[
Detect PRVS/BATV addresses to avoid FORGED_SENDER
https://en.wikipedia.org/wiki/Bounce_Address_Tag_Validation
local re_text = '^(?:(prvs|msprvs1)=([^=]+)=|btv1==[^=]+==)(.+@(.+))$'
local re = rspamd_regexp.create_cached(re_text)
local c = re:search(envfrom[1].addr:lower(), false, true)
- if not c then return false end
+ if not c then
+ return false
+ end
local ef = c[1][4]
-- See if it matches the From header
local from = task:get_from(2)
}
rspamd_config.ENVFROM_VERP = {
- callback = function (task)
+ callback = function(task)
if not (task:has_from(1) and task:has_recipients(1)) then
return false
end
local envfrom = task:get_from(1)
local envrcpts = task:get_recipients(1)
-- VERP only works for single recipient messages
- if #envrcpts > 1 then return false end
+ if #envrcpts > 1 then
+ return false
+ end
-- Get recipient and compute VERP address
local rcpt = envrcpts[1].addr:lower()
- local verp = rcpt:gsub('@','=')
+ local verp = rcpt:gsub('@', '=')
-- Get the user portion of the envfrom
local ef_user = envfrom[1].user:lower()
-- See if the VERP representation of the recipient appears in it
if ef_user:find(verp, 1, true)
- and not ef_user:find('+caf_=' .. verp, 1, true) -- Google Forwarding
- and not ef_user:find('^srs[01]=') -- SRS
+ and not ef_user:find('+caf_=' .. verp, 1, true) -- Google Forwarding
+ and not ef_user:find('^srs[01]=') -- SRS
then
return true
end
type = 'mime',
}
-local check_rcvd = rspamd_config:register_symbol{
+local check_rcvd = rspamd_config:register_symbol {
name = 'CHECK_RCVD',
group = 'headers',
- callback = function (task)
+ callback = function(task)
local rcvds = task:get_received_headers()
- if not rcvds or #rcvds == 0 then return false end
+ if not rcvds or #rcvds == 0 then
+ return false
+ end
local all_tls = fun.all(function(rc)
return rc.flags and rc.flags['ssl']
type = 'callback,mime',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_rcvd,
name = 'RCVD_TLS_ALL',
group = 'headers'
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_rcvd,
name = 'RCVD_TLS_LAST',
group = 'headers'
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_rcvd,
name = 'RCVD_NO_TLS_LAST',
group = 'headers'
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_rcvd,
name = 'RCVD_VIA_SMTP_AUTH',
}
rspamd_config.RCVD_HELO_USER = {
- callback = function (task)
+ callback = function(task)
-- Check HELO argument from MTA
local helo = task:get_helo()
if (helo and helo:lower():find('^user$')) then
end
-- Check Received headers
local rcvds = task:get_header_full('Received')
- if not rcvds then return false end
+ if not rcvds then
+ return false
+ end
for _, rcvd in ipairs(rcvds) do
local r = rcvd['decoded']:lower()
- if (r:find("^%s*from%suser%s")) then return true end
- if (r:find("helo[%s=]user[%s%)]")) then return true end
+ if (r:find("^%s*from%suser%s")) then
+ return true
+ end
+ if (r:find("helo[%s=]user[%s%)]")) then
+ return true
+ end
end
end,
description = 'HELO User spam pattern',
}
rspamd_config.URI_COUNT_ODD = {
- callback = function (task)
+ callback = function(task)
local ct = task:get_header('Content-Type')
if (ct and ct:lower():find('^multipart/alternative')) then
- local urls = task:get_urls_filtered(nil, {'subject', 'html_displayed', 'special'}) or {}
- local nurls = fun.foldl(function(acc, val) return acc + val:get_count() end, 0, urls)
+ local urls = task:get_urls_filtered(nil, { 'subject', 'html_displayed', 'special' }) or {}
+ local nurls = fun.foldl(function(acc, val)
+ return acc + val:get_count()
+ end, 0, urls)
if nurls % 2 == 1 then
return true, 1.0, tostring(nurls)
}
rspamd_config.HAS_ATTACHMENT = {
- callback = function (task)
+ callback = function(task)
local parts = task:get_parts()
if parts and #parts > 1 then
for _, p in ipairs(parts) do
local ff = task:get_symbol('FREEMAIL_FROM')
local frt_opts = frt[1]['options']
local ff_opts = ff[1]['options']
- return ( frt_opts and ff_opts and frt_opts[1] ~= ff_opts[1] )
+ return (frt_opts and ff_opts and frt_opts[1] ~= ff_opts[1])
end
rspamd_config:register_symbol({
local h1 = u:get_host()
local h2 = u:get_phished()
- if h2 then -- Due to changes of the phished flag in 2.8
+ if h2 then
+ -- Due to changes of the phished flag in 2.8
h2 = h2:get_host()
end
if h1 and h2 then
local urls = task:get_urls()
if urls then
- for _,u in ipairs(urls) do
+ for _, u in ipairs(urls) do
local flags = u:get_flags()
if flags.subject then
if flags.schemaless then
- return true,0.1,u:get_host()
+ return true, 0.1, u:get_host()
end
local subject = task:get_subject()
if subject then
if tostring(u) == subject then
- return true,1.0,u:get_host()
+ return true, 1.0, u:get_host()
end
end
- return true,0.25,u:get_host()
+ return true, 0.25, u:get_host()
end
end
end
description = 'Subject contains URL'
}
-local aliases_id = rspamd_config:register_symbol{
+local aliases_id = rspamd_config:register_symbol {
type = 'prefilter',
name = 'EMAIL_PLUS_ALIASES',
callback = function(task)
local function check_from(type)
if task:has_from(type) then
local addr = task:get_from(type)[1]
- local na,tags = lua_util.remove_email_aliases(addr)
+ local na, tags = lua_util.remove_email_aliases(addr)
if na then
task:set_from(type, addr, 'alias')
task:insert_result('TAGGED_FROM', 1.0, fun.totable(
- fun.filter(function(t) return t and #t > 0 end, tags)))
+ fun.filter(function(t)
+ return t and #t > 0
+ end, tags)))
end
end
end
local addrs = task:get_recipients(type)
for _, addr in ipairs(addrs) do
- local na,tags = lua_util.remove_email_aliases(addr)
+ local na, tags = lua_util.remove_email_aliases(addr)
if na then
modified = true
- fun.each(function(t) table.insert(all_tags, t) end,
- fun.filter(function(t) return t and #t > 0 end, tags))
+ fun.each(function(t)
+ table.insert(all_tags, t)
+ end,
+ fun.filter(function(t)
+ return t and #t > 0
+ end, tags))
end
end
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = aliases_id,
name = 'TAGGED_RCPT',
group = 'headers',
score = 0.0,
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = aliases_id,
name = 'TAGGED_FROM',
score = 0.0,
}
-local check_from_display_name = rspamd_config:register_symbol{
+local check_from_display_name = rspamd_config:register_symbol {
type = 'callback,mime',
name = 'FROM_DISPLAY_CALLBACK',
- callback = function (task)
+ callback = function(task)
local from = task:get_from(2)
- if not (from and from[1] and from[1].name) then return false end
+ if not (from and from[1] and from[1].name) then
+ return false
+ end
-- See if we can parse an email address from the name
local parsed = rspamd_parsers.parse_mail_address(from[1].name, task:get_mempool())
- if not parsed then return false end
- if not (parsed[1] and parsed[1]['addr']) then return false end
+ if not parsed then
+ return false
+ end
+ if not (parsed[1] and parsed[1]['addr']) then
+ return false
+ end
-- Make sure we did not mistake e.g. <something>@<name> for an email address
- if not parsed[1]['domain'] or not parsed[1]['domain']:find('%.') then return false end
+ if not parsed[1]['domain'] or not parsed[1]['domain']:find('%.') then
+ return false
+ end
-- See if the parsed domains differ
if not rspamd_util.strequal_caseless(from[1]['domain'], parsed[1]['domain']) then
-- See if the destination domain is the same as the spoof
group = 'headers',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_from_display_name,
name = 'SPOOF_DISPLAY_NAME',
score = 8.0,
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_from_display_name,
name = 'FROM_NEQ_DISPLAY_NAME',
}
rspamd_config.SPOOF_REPLYTO = {
- callback = function (task)
+ callback = function(task)
-- First check for a Reply-To header
local rt = task:get_header_full('Reply-To')
- if not rt or not rt[1] then return false end
+ if not rt or not rt[1] then
+ return false
+ end
-- Get From and To headers
rt = rt[1]['value']
local from = task:get_from(2)
local to = task:get_recipients(2)
- if not (from and from[1] and from[1].addr) then return false end
+ if not (from and from[1] and from[1].addr) then
+ return false
+ end
if (to and to[1] and to[1].addr) then
-- Handle common case for Web Contact forms of From = To
if rspamd_util.strequal_caseless(from[1].addr, to[1].addr) then
end
-- SMTP recipients must contain From domain
to = task:get_recipients(1)
- if not to then return false end
+ if not to then
+ return false
+ end
-- Try mitigate some possible FPs on mailing list posts
- if #to == 1 and rspamd_util.strequal_caseless(to[1].addr, from[1].addr) then return false end
+ if #to == 1 and rspamd_util.strequal_caseless(to[1].addr, from[1].addr) then
+ return false
+ end
local found_fromdom = false
for _, t in ipairs(to) do
if rspamd_util.strequal_caseless(t.domain, from[1].domain) then
break
end
end
- if not found_fromdom then return false end
+ if not found_fromdom then
+ return false
+ end
-- Parse Reply-To header
local parsed = ((rspamd_parsers.parse_mail_address(rt, task:get_mempool()) or E)[1] or E).domain
- if not parsed then return false end
+ if not parsed then
+ return false
+ end
-- Reply-To domain must be different to From domain
if not rspamd_util.strequal_caseless(parsed, from[1].domain) then
return true, from[1].domain, parsed
return false
end
local to = task:get_recipients('smtp')
- if not to then return false end
+ if not to then
+ return false
+ end
local found = false
- for _,r in ipairs(to) do
+ for _, r in ipairs(to) do
if rspamd_util.strequal_caseless(r['user'], 'info') then
found = true
end
end
- if found then return true end
+ if found then
+ return true
+ end
return false
end,
description = 'info@ From/To address with List-Unsubscribe headers',
callback = function(task)
local tp = task:get_text_parts() or {}
- for _,p in ipairs(tp) do
+ for _, p in ipairs(tp) do
local cte = p:get_mimepart():get_cte() or ''
if cte ~= '8bit' and p:has_8bit_raw() then
- local _,_,attrs = p:get_mimepart():get_type_full()
+ local _, _, attrs = p:get_mimepart():get_type_full()
local mul = 1.0
- local params = {cte}
+ local params = { cte }
if attrs then
if attrs.charset and attrs.charset:lower() == "utf-8" then
-- Penalise rule as people don't know that utf8 is surprisingly
end
end
- return true,mul,params
+ return true, mul, params
end
end
type = 'mime',
}
-
-local check_encrypted_name = rspamd_config:register_symbol{
+local check_encrypted_name = rspamd_config:register_symbol {
name = 'BOGUS_ENCRYPTED_AND_TEXT',
callback = function(task)
local parts = task:get_parts() or {}
local children = part:get_children() or {}
local text_kids = {}
- for _,cld in ipairs(children) do
+ for _, cld in ipairs(children) do
if cld:is_multipart() then
check_part(cld)
elseif cld:is_text() then
seen_text = true
text_kids[#text_kids + 1] = cld
else
- local type,subtype,_ = cld:get_type_full()
+ local type, subtype, _ = cld:get_type_full()
if type:lower() == 'application' then
if string.find(subtype:lower(), 'pkcs7%-mime') then
end
if seen_text and seen_encrypted then
-- Ensure that our seen text is not really part of pgp #3205
- for _,tp in ipairs(text_kids) do
- local t,_ = tp:get_type()
+ for _, tp in ipairs(text_kids) do
+ local t, _ = tp:get_type()
seen_text = false -- reset temporary
if t and t == 'text' then
seen_text = true
end
end
- for _,part in ipairs(parts) do
+ for _, part in ipairs(parts) do
check_part(part)
end
group = 'mime_types',
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_encrypted_name,
name = 'ENCRYPTED_PGP',
one_shot = true
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_encrypted_name,
name = 'ENCRYPTED_SMIME',
one_shot = true
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_encrypted_name,
name = 'SIGNED_PGP',
one_shot = true
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'virtual',
parent = check_encrypted_name,
name = 'SIGNED_SMIME',
-- X-Authenticated-Sender: accord.host-care.com: sales@cortaflex.si
rspamd_config.HAS_X_AS = {
- callback = function (task)
+ callback = function(task)
local xas = task:get_header('X-Authenticated-Sender')
- if not xas then return false end
- local _,_,auth = xas:find('[^:]+:%s(.+)$')
+ if not xas then
+ return false
+ end
+ local _, _, auth = xas:find('[^:]+:%s(.+)$')
if auth then
-- TODO: see if we can parse an e-mail address from auth
-- and see if it matches the from address or not
-- X-Get-Message-Sender-Via: accord.host-care.com: authenticated_id: sales@cortaflex.si
rspamd_config.HAS_X_GMSV = {
- callback = function (task)
+ callback = function(task)
local xgmsv = task:get_header('X-Get-Message-Sender-Via')
- if not xgmsv then return false end
- local _,_,auth = xgmsv:find('authenticated_id: (.+)$')
+ if not xgmsv then
+ return false
+ end
+ local _, _, auth = xgmsv:find('authenticated_id: (.+)$')
if auth then
-- TODO: see if we can parse an e-mail address from auth
-- and see if it matches the from address or not.
group = "compromised_hosts"
}
-local hidden_uri_re = rspamd_regexp.create_cached('/(?!\\/\\.well[-_]known\\/)(?:^\\.[A-Za-z0-9]|\\/'..
+local hidden_uri_re = rspamd_regexp.create_cached('/(?!\\/\\.well[-_]known\\/)(?:^\\.[A-Za-z0-9]|\\/' ..
'\\.[A-Za-z0-9]|\\/\\.\\.\\/)/i')
rspamd_config.URI_HIDDEN_PATH = {
- callback = function (task)
+ callback = function(task)
local urls = task:get_urls(false)
if (urls) then
- for _, url in ipairs(urls) do
- if (not (url:is_subject() and url:is_html_displayed())) then
- local path = url:get_path()
- if (hidden_uri_re:match(path)) then
- -- TODO: need url:is_schemeless() to improve this
- return true, 1.0, url:get_text()
- end
- end
+ for _, url in ipairs(urls) do
+ if (not (url:is_subject() and url:is_html_displayed())) then
+ local path = url:get_path()
+ if (hidden_uri_re:match(path)) then
+ -- TODO: need url:is_schemeless() to improve this
+ return true, 1.0, url:get_text()
+ end
end
+ end
end
end,
description = 'Message contains URI with a hidden path',
}
rspamd_config.FROM_SERVICE_ACCT = {
- callback = function (task)
+ callback = function(task)
local re = rspamd_regexp.create_cached('/^(?:www-data|anonymous|ftp|apache|nobody|guest|nginx|web|www)@/i');
-- From
local from = task:get_from(2)
if (from and from[1]) then
- if (re:match(from[1].addr)) then return true end
+ if (re:match(from[1].addr)) then
+ return true
+ end
end
-- Sender
local sender = task:get_header('Sender')
if sender then
local s = util.parse_mail_address(sender, task:get_mempool())
if (s and s[1]) then
- if (re:match(s[1].addr)) then return true end
+ if (re:match(s[1].addr)) then
+ return true
+ end
end
end
-- Reply-To
if replyto then
local rt = util.parse_mail_address(replyto, task:get_mempool())
if (rt and rt[1]) then
- if (re:match(rt[1].addr)) then return true end
+ if (re:match(rt[1].addr)) then
+ return true
+ end
end
end
end,
group = 'mua'
}
-- Forged OE/MSO boundary
-local suspicious_boundary_01C4 = 'Content-Type=/^\\s*multipart.+boundary="----=_NextPart_000_[A-Z\\d]{4}_01C4[\\dA-F]{4}\\.[A-Z\\d]{8}"[\\r\\n]*$/siX'
-local suspicious_boundary_01C4_date = 'Date=/^\\s*\\w\\w\\w,\\s+\\d+\\s+\\w\\w\\w 20(0[56789]|1\\d)/'
+local suspicious_boundary_01C4 = 'Content-Type=/^\\s*multipart.+boundary="----=_NextPart_000_[A-Z\\d]{4}_01C4[\\dA-F]{4}\\.[A-Z\\d]{8}"[\\r\\n]*$/siX'
+local suspicious_boundary_01C4_date = 'Date=/^\\s*\\w\\w\\w,\\s+\\d+\\s+\\w\\w\\w 20(0[56789]|1\\d)/'
reconf['SUSPICIOUS_BOUNDARY4'] = {
re = string.format('(%s) & (%s)', suspicious_boundary_01C4, suspicious_boundary_01C4_date),
score = 4.0,
-- Detect forged Mozilla Mail/Thunderbird/Seamonkey/Postbox headers
-- Mozilla based X-Mailer
-local user_agent_mozilla5 = 'User-Agent=/^\\s*Mozilla\\/5\\.0/H'
-local user_agent_thunderbird = 'User-Agent=/^\\s*(Thunderbird|Mozilla Thunderbird|Mozilla\\/.*Gecko\\/.*(Thunderbird|Betterbird|Icedove)\\/)/H'
-local user_agent_seamonkey = 'User-Agent=/^\\s*Mozilla\\/5\\.0\\s.+\\sSeaMonkey\\/\\d+\\.\\d+/H'
-local user_agent_postbox = [[User-Agent=/^\s*Mozilla\/5\.0\s\([^)]+\)\sGecko\/\d+\sPostboxApp\/\d+(?:\.\d+){2,3}$/H]]
-local user_agent_mozilla = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla5, user_agent_thunderbird, user_agent_seamonkey, user_agent_postbox)
+local user_agent_mozilla5 = 'User-Agent=/^\\s*Mozilla\\/5\\.0/H'
+local user_agent_thunderbird = 'User-Agent=/^\\s*(Thunderbird|Mozilla Thunderbird|Mozilla\\/.*Gecko\\/.*(Thunderbird|Betterbird|Icedove)\\/)/H'
+local user_agent_seamonkey = 'User-Agent=/^\\s*Mozilla\\/5\\.0\\s.+\\sSeaMonkey\\/\\d+\\.\\d+/H'
+local user_agent_postbox = [[User-Agent=/^\s*Mozilla\/5\.0\s\([^)]+\)\sGecko\/\d+\sPostboxApp\/\d+(?:\.\d+){2,3}$/H]]
+local user_agent_mozilla = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla5, user_agent_thunderbird,
+ user_agent_seamonkey, user_agent_postbox)
-- Mozilla based common Message-ID template
-local mozilla_msgid_common = 'Message-ID=/^\\s*<[\\dA-F]{8}\\.\\d{1,7}\\@([^>\\.]+\\.)+[^>\\.]+>$/H'
-local mozilla_msgid_common_sec = 'Message-ID=/^\\s*<[\\da-f]{8}-([\\da-f]{4}-){3}[\\da-f]{12}\\@([^>\\.]+\\.)+[^>\\.]+>$/H'
-local mozilla_msgid = 'Message-ID=/^\\s*<(3[3-9A-F]|[4-9A-F][\\dA-F])[\\dA-F]{6}\\.(\\d0){1,4}\\d\\@([^>\\.]+\\.)+[^>\\.]+>$/H'
+local mozilla_msgid_common = 'Message-ID=/^\\s*<[\\dA-F]{8}\\.\\d{1,7}\\@([^>\\.]+\\.)+[^>\\.]+>$/H'
+local mozilla_msgid_common_sec = 'Message-ID=/^\\s*<[\\da-f]{8}-([\\da-f]{4}-){3}[\\da-f]{12}\\@([^>\\.]+\\.)+[^>\\.]+>$/H'
+local mozilla_msgid = 'Message-ID=/^\\s*<(3[3-9A-F]|[4-9A-F][\\dA-F])[\\dA-F]{6}\\.(\\d0){1,4}\\d\\@([^>\\.]+\\.)+[^>\\.]+>$/H'
-- Summary rule for forged Mozilla Mail Message-ID header
reconf['FORGED_MUA_MOZILLA_MAIL_MSGID'] = {
- re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid, unusable_msgid),
+ re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
score = 4.0,
description = 'Message pretends to be send from Mozilla Mail but has forged Message-ID',
group = 'mua'
}
reconf['FORGED_MUA_MOZILLA_MAIL_MSGID_UNKNOWN'] = {
- re = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid, unusable_msgid),
+ re = string.format('(%s) & !(%s) & !(%s) & !(%s)', user_agent_mozilla, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
score = 2.5,
description = 'Message pretends to be send from Mozilla Mail but has forged Message-ID',
group = 'mua'
-- Summary rule for forged Thunderbird Message-ID header
reconf['FORGED_MUA_THUNDERBIRD_MSGID'] = {
- re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common, mozilla_msgid, unusable_msgid),
+ re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
score = 4.0,
description = 'Forged mail pretending to be from Mozilla Thunderbird but has forged Message-ID',
group = 'mua'
}
reconf['FORGED_MUA_THUNDERBIRD_MSGID_UNKNOWN'] = {
- re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common, mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid),
+ re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_thunderbird, mozilla_msgid_common,
+ mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid),
score = 2.5,
description = 'Forged mail pretending to be from Mozilla Thunderbird but has forged Message-ID',
group = 'mua'
}
-- Summary rule for forged Seamonkey Message-ID header
reconf['FORGED_MUA_SEAMONKEY_MSGID'] = {
- re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, mozilla_msgid, unusable_msgid),
+ re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
score = 4.0,
description = 'Forged mail pretending to be from Mozilla Seamonkey but has forged Message-ID',
group = 'mua'
}
reconf['FORGED_MUA_SEAMONKEY_MSGID_UNKNOWN'] = {
- re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common, mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid),
+ re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_seamonkey, mozilla_msgid_common,
+ mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid),
score = 2.5,
description = 'Forged mail pretending to be from Mozilla Seamonkey but has forged Message-ID',
group = 'mua'
}
-- Summary rule for forged Postbox Message-ID header
reconf['FORGED_MUA_POSTBOX_MSGID'] = {
- re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common, mozilla_msgid, unusable_msgid),
+ re = string.format('(%s) & (%s) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common, mozilla_msgid,
+ unusable_msgid),
score = 4.0,
description = 'Forged mail pretending to be from Postbox but has forged Message-ID',
group = 'mua'
}
reconf['FORGED_MUA_POSTBOX_MSGID_UNKNOWN'] = {
- re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common, mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid),
+ re = string.format('(%s) & !((%s) | (%s)) & !(%s) & !(%s)', user_agent_postbox, mozilla_msgid_common,
+ mozilla_msgid_common_sec, mozilla_msgid, unusable_msgid),
score = 2.5,
description = 'Forged mail pretending to be from Postbox but has forged Message-ID',
group = 'mua'
-- Empty delimiters between header names and header values
local function gen_check_header_delimiter_empty(header_name)
return function(task)
- for _,rh in ipairs(task:get_header_full(header_name) or {}) do
- if rh['empty_separator'] then return true end
+ for _, rh in ipairs(task:get_header_full(header_name) or {}) do
+ if rh['empty_separator'] then
+ return true
+ end
end
return false
end
group = 'headers'
}
-local MAIL_RU_Return_Path = 'Return-path=/^\\s*<.+\\@mail\\.ru>$/iX'
-local MAIL_RU_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@mail\\.ru>$/iX'
-local MAIL_RU_From = 'From=/\\@mail\\.ru>?$/iX'
-local MAIL_RU_Received = 'Received=/from mail\\.ru \\(/mH'
+local MAIL_RU_Return_Path = 'Return-path=/^\\s*<.+\\@mail\\.ru>$/iX'
+local MAIL_RU_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@mail\\.ru>$/iX'
+local MAIL_RU_From = 'From=/\\@mail\\.ru>?$/iX'
+local MAIL_RU_Received = 'Received=/from mail\\.ru \\(/mH'
reconf['FAKE_RECEIVED_mail_ru'] = {
re = string.format('(%s) & !(((%s) | (%s)) & (%s))',
group = 'headers'
}
-local GMAIL_COM_Return_Path = 'Return-path=/^\\s*<.+\\@gmail\\.com>$/iX'
-local GMAIL_COM_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@gmail\\.com>$/iX'
-local GMAIL_COM_From = 'From=/\\@gmail\\.com>?$/iX'
+local GMAIL_COM_Return_Path = 'Return-path=/^\\s*<.+\\@gmail\\.com>$/iX'
+local GMAIL_COM_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@gmail\\.com>$/iX'
+local GMAIL_COM_From = 'From=/\\@gmail\\.com>?$/iX'
-local UKR_NET_Return_Path = 'Return-path=/^\\s*<.+\\@ukr\\.net>$/iX'
-local UKR_NET_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@ukr\\.net>$/iX'
-local UKR_NET_From = 'From=/\\@ukr\\.net>?$/iX'
+local UKR_NET_Return_Path = 'Return-path=/^\\s*<.+\\@ukr\\.net>$/iX'
+local UKR_NET_X_Envelope_From = 'X-Envelope-From=/^\\s*<.+\\@ukr\\.net>$/iX'
+local UKR_NET_From = 'From=/\\@ukr\\.net>?$/iX'
-local RECEIVED_smtp_yandex_ru_1 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\((port=\\d+ )?helo=smtp\\.yandex\\.ru\\)/iX'
-local RECEIVED_smtp_yandex_ru_2 = 'Received=/from \\[UNAVAILABLE\\] \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX'
-local RECEIVED_smtp_yandex_ru_3 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX'
-local RECEIVED_smtp_yandex_ru_4 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\(account \\S+ HELO smtp\\.yandex\\.ru\\)/iX'
-local RECEIVED_smtp_yandex_ru_5 = 'Received=/from smtp\\.yandex\\.ru \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX'
-local RECEIVED_smtp_yandex_ru_6 = 'Received=/from smtp\\.yandex\\.ru \\(\\S+ \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX'
-local RECEIVED_smtp_yandex_ru_7 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\S+\\@\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX'
-local RECEIVED_smtp_yandex_ru_8 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX'
-local RECEIVED_smtp_yandex_ru_9 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] helo=smtp\\.yandex\\.ru\\)/iX'
+local RECEIVED_smtp_yandex_ru_1 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\((port=\\d+ )?helo=smtp\\.yandex\\.ru\\)/iX'
+local RECEIVED_smtp_yandex_ru_2 = 'Received=/from \\[UNAVAILABLE\\] \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX'
+local RECEIVED_smtp_yandex_ru_3 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]:\\d+ helo=smtp\\.yandex\\.ru\\)/iX'
+local RECEIVED_smtp_yandex_ru_4 = 'Received=/from \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] \\(account \\S+ HELO smtp\\.yandex\\.ru\\)/iX'
+local RECEIVED_smtp_yandex_ru_5 = 'Received=/from smtp\\.yandex\\.ru \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX'
+local RECEIVED_smtp_yandex_ru_6 = 'Received=/from smtp\\.yandex\\.ru \\(\\S+ \\[\\d+\\.\\d+\\.\\d+\\.\\d+\\]\\)/iX'
+local RECEIVED_smtp_yandex_ru_7 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\S+\\@\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX'
+local RECEIVED_smtp_yandex_ru_8 = 'Received=/from \\S+ \\(HELO smtp\\.yandex\\.ru\\) \\(\\d+\\.\\d+\\.\\d+\\.\\d+\\)/iX'
+local RECEIVED_smtp_yandex_ru_9 = 'Received=/from \\S+ \\(\\[\\d+\\.\\d+\\.\\d+\\.\\d+\\] helo=smtp\\.yandex\\.ru\\)/iX'
reconf['FAKE_RECEIVED_smtp_yandex_ru'] = {
- re = string.format('(((%s) & ((%s) | (%s))) | ((%s) & ((%s) | (%s))) '..
+ re = string.format('(((%s) & ((%s) | (%s))) | ((%s) & ((%s) | (%s))) ' ..
' | ((%s) & ((%s) | (%s)))) & (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s) | (%s)',
MAIL_RU_From, MAIL_RU_Return_Path, MAIL_RU_X_Envelope_From, GMAIL_COM_From,
GMAIL_COM_Return_Path, GMAIL_COM_X_Envelope_From, UKR_NET_From, UKR_NET_Return_Path,
local onion_uri_v2 = '/[a-z0-9]{16}\\.onion?/{url}i'
local onion_uri_v3 = '/[a-z0-9]{56}\\.onion?/{url}i'
reconf['HAS_ONION_URI'] = {
- re = string.format('(%s | %s)', onion_uri_v2, onion_uri_v3),
- description = 'Contains .onion hidden service URI',
- score = 0.0,
- group = 'url'
+ re = string.format('(%s | %s)', onion_uri_v2, onion_uri_v3),
+ description = 'Contains .onion hidden service URI',
+ score = 0.0,
+ group = 'url'
}
local my_victim = [[/(?:victim|prey)/{words}]]
check_data_images = function(task)
local tp = task:get_text_parts() or {}
- for _,p in ipairs(tp) do
+ for _, p in ipairs(tp) do
if p:is_html() then
local hc = p:get_html()
reconf['UNITEDINTERNET_SPAM'] = {
re = string.format('%s || %s',
- 'X-UI-Filterresults=/^junk:/H',
- 'X-UI-Out-Filterresults=/^junk:/H'),
+ 'X-UI-Filterresults=/^junk:/H',
+ 'X-UI-Out-Filterresults=/^junk:/H'),
score = 5.0,
description = "United Internet says this message is spam",
group = 'upstream_spam_filters'
-- This is main lua config file for rspamd
-require "global_functions" ()
+require "global_functions"()
config['regexp'] = {}
rspamd_maps = {} -- Global maps
dofile(local_conf .. '/local.d/rspamd.lua')
end
-local rmaps = rspamd_config:get_all_opt("lua_maps")
+local rmaps = rspamd_config:get_all_opt("lua_maps")
if rmaps and type(rmaps) == 'table' then
local rspamd_logger = require "rspamd_logger"
- for k,v in pairs(rmaps) do
- local status,map_or_err = pcall(function () return rspamd_config:add_map(v) end)
+ for k, v in pairs(rmaps) do
+ local status, map_or_err = pcall(function()
+ return rspamd_config:add_map(v)
+ end)
if not status then
rspamd_logger.errx(rspamd_config, "cannot add map %s: %s", k, map_or_err)
local function test_subject(task, check_function, rate)
local function normalize_linear(a, x)
- local f = a * x
- return true, (( f < 1 ) and f or 1), tostring(x)
+ local f = a * x
+ return true, ((f < 1) and f or 1), tostring(x)
end
local sbj = task:get_header('Subject')
local caps_test = function(sbj)
return util.is_uppercase(sbj)
end
- return test_subject(task, caps_test, 1.0/40.0)
+ return test_subject(task, caps_test, 1.0 / 40.0)
end,
score = 3.0,
group = 'subject',
local length_test = function(_, len)
return len > 200
end
- return test_subject(task, length_test, 1.0/400.0)
+ return test_subject(task, length_test, 1.0 / 400.0)
end,
score = 3.0,
group = 'subject',
if confighelp then
rspamd_config:add_example(nil, 'antivirus',
- "Check messages for viruses",
- [[
-antivirus {
- # multiple scanners could be checked, for each we create a configuration block with an arbitrary name
- clamav {
- # If set force this action if any virus is found (default unset: no action is forced)
- # action = "reject";
- # If set, then rejection message is set to this value (mention single quotes)
- # message = '${SCANNER}: virus found: "${VIRUS}"';
- # Scan mime_parts separately - otherwise the complete mail will be transferred to AV Scanner
- #scan_mime_parts = true;
- # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity)
- #scan_text_mime = false;
- #scan_image_mime = false;
- # If `max_size` is set, messages > n bytes in size are not scanned
- max_size = 20000000;
- # symbol to add (add it to metric if you want non-zero weight)
- symbol = "CLAM_VIRUS";
- # type of scanner: "clamav", "fprot", "sophos" or "savapi"
- type = "clamav";
- # For "savapi" you must also specify the following variable
- product_id = 12345;
- # You can enable logging for clean messages
- log_clean = true;
- # servers to query (if port is unspecified, scanner-specific default is used)
- # can be specified multiple times to pool servers
- # can be set to a path to a unix socket
- # Enable this in local.d/antivirus.conf
- servers = "127.0.0.1:3310";
- # if `patterns` is specified virus name will be matched against provided regexes and the related
- # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
- patterns {
- # symbol_name = "pattern";
- JUST_EICAR = "^Eicar-Test-Signature$";
+ "Check messages for viruses",
+ [[
+ antivirus {
+ # multiple scanners could be checked, for each we create a configuration block with an arbitrary name
+ clamav {
+ # If set force this action if any virus is found (default unset: no action is forced)
+ # action = "reject";
+ # If set, then rejection message is set to this value (mention single quotes)
+ # message = '${SCANNER}: virus found: "${VIRUS}"';
+ # Scan mime_parts separately - otherwise the complete mail will be transferred to AV Scanner
+ #scan_mime_parts = true;
+ # Scanning Text is suitable for some av scanner databases (e.g. Sanesecurity)
+ #scan_text_mime = false;
+ #scan_image_mime = false;
+ # If `max_size` is set, messages > n bytes in size are not scanned
+ max_size = 20000000;
+ # symbol to add (add it to metric if you want non-zero weight)
+ symbol = "CLAM_VIRUS";
+ # type of scanner: "clamav", "fprot", "sophos" or "savapi"
+ type = "clamav";
+ # For "savapi" you must also specify the following variable
+ product_id = 12345;
+ # You can enable logging for clean messages
+ log_clean = true;
+ # servers to query (if port is unspecified, scanner-specific default is used)
+ # can be specified multiple times to pool servers
+ # can be set to a path to a unix socket
+ # Enable this in local.d/antivirus.conf
+ servers = "127.0.0.1:3310";
+ # if `patterns` is specified virus name will be matched against provided regexes and the related
+ # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+ patterns {
+ # symbol_name = "pattern";
+ JUST_EICAR = "^Eicar-Test-Signature$";
+ }
+ # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+ whitelist = "/etc/rspamd/antivirus.wl";
+ # Replace content that exactly matches the following string to the EICAR pattern
+ # Useful for E2E testing when another party removes/blocks EICAR attachments
+ #eicar_fake_pattern = 'testpatterneicar';
}
- # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
- whitelist = "/etc/rspamd/antivirus.wl";
- # Replace content that exactly matches the following string to the EICAR pattern
- # Useful for E2E testing when another party removes/blocks EICAR attachments
- #eicar_fake_pattern = 'testpatterneicar';
}
-}
-]])
+ ]])
return
end
return nil
end
- if not opts.symbol then opts.symbol = sym:upper() end
+ if not opts.symbol then
+ opts.symbol = sym:upper()
+ end
local cfg = lua_antivirus[opts.type]
if not cfg then
-- WORKAROUND for deprecated attachments_only
if opts.attachments_only ~= nil then
opts.scan_mime_parts = opts.attachments_only
- rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. '..
+ rspamd_logger.warnx(rspamd_config, '%s [%s]: Using attachments_only is deprecated. ' ..
'Please use scan_mime_parts = %s instead', opts.symbol, opts.type, opts.attachments_only)
end
-- WORKAROUND for deprecated attachments_only
local rule = cfg.configure(opts)
- if not rule then return nil end
+ if not rule then
+ return nil
+ end
rule.type = opts.type
rule.symbol_fail = opts.symbol_fail
if clen == #opts.eicar_fake_pattern and content == opts.eicar_fake_pattern then
rspamd_logger.infox(task, 'found eicar fake replacement part in the part (filename="%s")',
- p:get_filename())
+ p:get_filename())
content = eicar_pattern
end
end
local has_valid = false
for k, m in pairs(opts) do
if type(m) == 'table' then
- if not m.type then m.type = k end
- if not m.name then m.name = k end
+ if not m.type then
+ m.type = k
+ end
+ if not m.name then
+ m.name = k
+ end
local cb = add_antivirus_rule(k, m)
if not cb then
local function parse_arc_header(hdr, target, is_aar)
-- Split elements by ';' and trim spaces
local arr = fun.totable(fun.map(
- function(val)
- return fun.totable(fun.map(lua_util.rspamd_str_trim,
- fun.filter(function(v) return v and #v > 0 end,
- lua_util.rspamd_str_split(val.decoded, ';')
- )
- ))
- end, hdr
+ function(val)
+ return fun.totable(fun.map(lua_util.rspamd_str_trim,
+ fun.filter(function(v)
+ return v and #v > 0
+ end,
+ lua_util.rspamd_str_split(val.decoded, ';')
+ )
+ ))
+ end, hdr
))
-- v[1] is the key and v[2] is the value
-- Now we have two tables in format:
-- [arc_header] -> [{arc_header1_elts}, {arc_header2_elts}...]
- for i,elts in ipairs(arr) do
- if not target[i] then target[i] = {} end
+ for i, elts in ipairs(arr) do
+ if not target[i] then
+ target[i] = {}
+ end
if not is_aar then
-- For normal ARC headers we split by kv pair, like k=v
fun.each(function(v)
else
-- For AAR we check special case of i=%d and pass everything else to
-- AAR specific parser
- for _,elt in ipairs(elts) do
+ for _, elt in ipairs(elts) do
if string.match(elt, "%s*i%s*=%s*%d+%s*") then
local pair = lua_util.rspamd_str_split(elt, '=')
fill_arc_header_table(pair, target[i])
local function arc_validate_seals(task, seals, sigs, seal_headers, sig_headers)
local fail_reason
- for i = 1,#seals do
+ for i = 1, #seals do
if (sigs[i].i or 0) ~= i then
fail_reason = string.format('bad i for signature: %d, expected %d; d=%s',
sigs[i].i, i, sigs[i].d)
rspamd_logger.infox(task, fail_reason)
task:insert_result(arc_symbols['invalid'], 1.0, fail_reason)
- return false,fail_reason
+ return false, fail_reason
end
if (seals[i].i or 0) ~= i then
fail_reason = string.format('bad i for seal: %d, expected %d; d=%s',
seals[i].i, i, seals[i].d)
rspamd_logger.infox(task, fail_reason)
- task:insert_result(arc_symbols['invalid'], 1.0,fail_reason)
- return false,fail_reason
+ task:insert_result(arc_symbols['invalid'], 1.0, fail_reason)
+ return false, fail_reason
end
if not seals[i].cv then
fail_reason = string.format('no cv on i=%d', i)
task:insert_result(arc_symbols['invalid'], 1.0, fail_reason)
- return false,fail_reason
+ return false, fail_reason
end
if i == 1 then
if seals[i].cv ~= 'none' then
fail_reason = 'cv is not "none" for i=1'
task:insert_result(arc_symbols['invalid'], 1.0, fail_reason)
- return false,fail_reason
+ return false, fail_reason
end
else
if seals[i].cv ~= 'pass' then
- fail_reason = string.format('cv is %s on i=%d', seals[i].cv, i)
+ fail_reason = string.format('cv is %s on i=%d', seals[i].cv, i)
task:insert_result(arc_symbols['reject'], 1.0, fail_reason)
- return true,fail_reason
+ return true, fail_reason
end
end
end
- return true,nil
+ return true, nil
end
local function arc_callback(task)
lua_util.debugm(N, task, 'got %s arc sections', #cbdata.seals)
-- Now check sanity of what we have
- local valid,validation_error = arc_validate_seals(task, cbdata.seals, cbdata.sigs,
+ local valid, validation_error = arc_validate_seals(task, cbdata.seals, cbdata.sigs,
arc_seal_headers, arc_sig_headers)
if not valid then
task:cache_set('arc-failure', validation_error)
end
local function gen_arc_seal_cb(index, sig)
- return function (_, res, err, domain)
+ return function(_, res, err, domain)
lua_util.debugm(N, task, 'checked arc seal: %s(%s), %s processed',
res, err, index)
else
task:cache_set(AR_TRUSTED_CACHE_KEY, cur_aar)
local seen_dmarc
- for _,ar in ipairs(cur_aar.ar) do
+ for _, ar in ipairs(cur_aar.ar) do
if ar.dmarc then
local dmarc_fwd = ar.dmarc
seen_dmarc = true
local function arc_signature_cb(_, res, err, domain)
lua_util.debugm(N, task, 'checked arc signature %s: %s(%s)',
- domain, res, err)
+ domain, res, err)
if not res then
cbdata.res = 'fail'
end
if cbdata.res == 'success' then
-- Verify seals
- for i,sig in ipairs(cbdata.seals) do
+ for i, sig in ipairs(cbdata.seals) do
local ret, lerr = dkim_verify(task, sig.header, gen_arc_seal_cb(i, sig), 'arc-seal')
if not ret then
cbdata.res = 'fail'
end
else
task:insert_result(arc_symbols['reject'], 1.0,
- rspamd_logger.slog('signature check failed: %s, %s', cbdata.res,
- cbdata.errors))
+ rspamd_logger.slog('signature check failed: %s, %s', cbdata.res,
+ cbdata.errors))
end
end
local processed = 0
local sig = cbdata.sigs[#cbdata.sigs] -- last AMS
- local ret,err = dkim_verify(task, sig.header, arc_signature_cb, 'arc-sign')
+ local ret, err = dkim_verify(task, sig.header, arc_signature_cb, 'arc-sign')
if not ret then
cbdata.res = 'fail'
else
processed = processed + 1
lua_util.debugm(N, task, 'processed arc signature %s[%s]: %s(%s), %s total',
- sig.d, sig.i, ret, err, #cbdata.seals)
+ sig.d, sig.i, ret, err, #cbdata.seals)
end
if processed == 0 then
task:insert_result(arc_symbols['reject'], 1.0,
- rspamd_logger.slog('cannot verify %s of %s signatures: %s',
- #arc_sig_headers - processed, #arc_sig_headers, cbdata.errors))
+ rspamd_logger.slog('cannot verify %s of %s signatures: %s',
+ #arc_sig_headers - processed, #arc_sig_headers, cbdata.errors))
end
end
end
if opts['symbols'] then
- for k,_ in pairs(arc_symbols) do
+ for k, _ in pairs(arc_symbols) do
if opts['symbols'][k] then
arc_symbols[k] = opts['symbols'][k]
end
end
end
-
local id = rspamd_config:register_symbol({
name = 'ARC_CHECK',
type = 'callback',
group = 'policies',
- groups = {'arc'},
+ groups = { 'arc' },
callback = arc_callback,
- augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)},
+ augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) },
})
rspamd_config:register_symbol({
name = 'ARC_CALLBACK', -- compatibility symbol
type = 'virtual',
score = -1.0,
group = 'policies',
- groups = {'arc'},
+ groups = { 'arc' },
})
rspamd_config:register_symbol({
name = arc_symbols['reject'],
type = 'virtual',
score = 2.0,
group = 'policies',
- groups = {'arc'},
+ groups = { 'arc' },
})
rspamd_config:register_symbol({
name = arc_symbols['invalid'],
type = 'virtual',
score = 1.0,
group = 'policies',
- groups = {'arc'},
+ groups = { 'arc' },
})
rspamd_config:register_symbol({
name = arc_symbols['dnsfail'],
type = 'virtual',
score = 0.0,
group = 'policies',
- groups = {'arc'},
+ groups = { 'arc' },
})
rspamd_config:register_symbol({
name = arc_symbols['na'],
type = 'virtual',
score = 0.0,
group = 'policies',
- groups = {'arc'},
+ groups = { 'arc' },
})
rspamd_config:register_dependency('ARC_CHECK', 'SPF_CHECK')
for i = 1, #arc_seals, 1 do
if arc_auth_results[i] then
local s = dkim_canonicalize('ARC-Authentication-Results',
- arc_auth_results[i].raw_header)
+ arc_auth_results[i].raw_header)
sha_ctx:update(s)
lua_util.debugm(N, task, 'update signature with header: %s', s)
end
if arc_sigs[i] then
local s = dkim_canonicalize('ARC-Message-Signature',
- arc_sigs[i].raw_header)
+ arc_sigs[i].raw_header)
sha_ctx:update(s)
lua_util.debugm(N, task, 'update signature with header: %s', s)
end
end
header = lua_util.fold_header(task,
- 'ARC-Message-Signature',
- header)
+ 'ARC-Message-Signature',
+ header)
cur_auth_results = string.format('i=%d; %s', cur_idx, cur_auth_results)
cur_auth_results = lua_util.fold_header(task,
cur_auth_results, ';')
local s = dkim_canonicalize('ARC-Authentication-Results',
- cur_auth_results)
+ cur_auth_results)
sha_ctx:update(s)
lua_util.debugm(N, task, 'update signature with header: %s', s)
s = dkim_canonicalize('ARC-Message-Signature', header)
local sig = rspamd_rsa.sign_memory(privkey, sha_ctx:bin())
cur_arc_seal = string.format('%s%s', cur_arc_seal,
- sig:base64(70, nl_type))
+ sig:base64(70, nl_type))
lua_mime.modify_headers(task, {
add = {
- ['ARC-Authentication-Results'] = {order = 1, value = cur_auth_results},
- ['ARC-Message-Signature'] = {order = 1, value = header},
- ['ARC-Seal'] = {order = 1, value = lua_util.fold_header(task,
- 'ARC-Seal', cur_arc_seal) }
+ ['ARC-Authentication-Results'] = { order = 1, value = cur_auth_results },
+ ['ARC-Message-Signature'] = { order = 1, value = header },
+ ['ARC-Seal'] = { order = 1, value = lua_util.fold_header(task,
+ 'ARC-Seal', cur_arc_seal) }
},
-- RFC requires a strict order for these headers to be inserted
- order = {'ARC-Authentication-Results', 'ARC-Message-Signature', 'ARC-Seal'},
+ order = { 'ARC-Authentication-Results', 'ARC-Message-Signature', 'ARC-Seal' },
})
task:insert_result(settings.sign_symbol, 1.0,
string.format('%s:s=%s:i=%d', params.domain, params.selector, cur_idx))
selector = cur_selector.selector
})
- local exists,err = rspamd_util.file_exists(cur_selector.key)
+ local exists, err = rspamd_util.file_exists(cur_selector.key)
if not exists then
if err and err == 'No such file or directory' then
lua_util.debugm(N, task, 'cannot read key from %s: %s', cur_selector.key, err)
redis_params = rspamd_parse_redis_server('arc')
if not redis_params then
- rspamd_logger.errx(rspamd_config, 'no servers are specified, '..
+ rspamd_logger.errx(rspamd_config, 'no servers are specified, ' ..
'but module is configured to load keys from redis, disable arc signing')
return
end
local sym_reg_tbl = {
name = settings['sign_symbol'],
callback = arc_signing_cb,
- groups = {"policies", "arc"},
+ groups = { "policies", "arc" },
flags = 'ignore_passthrough',
score = 0.0,
}
type = 'virtual',
score = -2.0,
group = 'policies',
- groups = {'arc'},
+ groups = { 'arc' },
})
end
sym_to_adjust = ar_settings.dmarc_symbols.quarantine
end
if sym_to_adjust and trusted_arc_ar and trusted_arc_ar.ar then
- for _,ar in ipairs(trusted_arc_ar.ar) do
+ for _, ar in ipairs(trusted_arc_ar.ar) do
if ar.dmarc then
local dmarc_fwd = ar.dmarc
if dmarc_fwd == 'pass' then
end
lua_util.debugm(N, task, 'got reply from %s when requesting %s: %s',
- serv, req_name, results[1])
+ serv, req_name, results[1])
local parts = rspamd_re:split(results[1])
-- "15169 | 8.8.8.0/24 | US | arin |" for 8.8.8.8
-- Configuration options
local configure_asn_module = function()
- local opts = rspamd_config:get_all_opt('asn')
+ local opts = rspamd_config:get_all_opt('asn')
if opts then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
options[k] = v
end
end
callback = asn_check,
priority = lua_util.symbols_priorities.high,
flags = 'empty,nostat',
- augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)},
+ augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) },
})
if options['symbol'] then
rspamd_config:register_symbol({
score = 0,
})
end
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
name = options['symbol_fail'],
parent = id,
type = 'virtual',
inline_content_limit = nil,
}
-local settings_schema = ts.shape{
+local settings_schema = ts.shape {
s3_bucket = ts.string,
s3_region = ts.string,
s3_host = ts.string,
local message_split = lua_mime.message_to_ucl(task)
if settings.inline_content_limit and settings.inline_content_limit > 0 then
- for i,part in ipairs(message_split.parts or {}) do
+ for i, part in ipairs(message_split.parts or {}) do
if part.content and #part.content >= settings.inline_content_limit then
local ref = convert_to_ref(task, nonce, queue_id, part, external_refs)
lua_util.debugm(N, task, "convert part number %s to a reference %s",
- i, ref)
+ i, ref)
end
end
end
local aws_host = string.format('%s.%s', settings.s3_bucket, settings.s3_host)
local function gen_s3_http_callback(path, what)
- return function (http_err, code, body, headers)
+ return function(http_err, code, body, headers)
if http_err then
if settings.fail_action then
timeout = settings.s3_timeout,
})
- for ref,part_content in pairs(external_refs) do
+ for ref, part_content in pairs(external_refs) do
local part_hdrs = lua_aws.aws_request_enrich({
region = settings.s3_region,
headers = {
end
settings = lua_util.override_defaults(settings, opts)
-local res,err = settings_schema:transform(settings)
+local res, err = settings_schema:transform(settings)
if not res then
rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err)
name = 'EXPORT_AWS_S3',
type = is_postfilter and 'postfilter' or 'idempotent',
callback = s3_aws_callback,
- augmentations = {string.format("timeout=%f", settings.s3_timeout)},
+ augmentations = { string.format("timeout=%f", settings.s3_timeout) },
priority = is_postfilter and lua_util.symbols_priorities.high or nil,
flags = 'empty,explicit_disable,ignore_passthrough,nostat',
})
\ No newline at end of file
local statfiles = cls.statfile
if statfiles[1] then
- for _,stf in ipairs(statfiles) do
+ for _, stf in ipairs(statfiles) do
if not stf.symbol then
- for k,v in pairs(stf) do
+ for k, v in pairs(stf) do
check_statfile_table(v, k)
end
else
end
end
else
- for stn,stf in pairs(statfiles) do
+ for stn, stf in pairs(statfiles) do
check_statfile_table(stf, stn)
end
end
if classifier then
if classifier[1] then
- for _,cls in ipairs(classifier) do
- if cls.bayes then cls = cls.bayes end
+ for _, cls in ipairs(classifier) do
+ if cls.bayes then
+ cls = cls.bayes
+ end
if cls.backend and cls.backend == 'redis' then
check_redis_classifier(cls, obj)
end
classifier = classifier.bayes
if classifier[1] then
- for _,cls in ipairs(classifier) do
+ for _, cls in ipairs(classifier) do
if cls.backend and cls.backend == 'redis' then
check_redis_classifier(cls, obj)
end
end
end
-
local opts = rspamd_config:get_all_opt(N)
if opts then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
settings[k] = v
end
end
if settings.cluster_nodes == 0 then
local neighbours = obj.neighbours or {}
local n_neighbours = 0
- for _,_ in pairs(neighbours) do n_neighbours = n_neighbours + 1 end
+ for _, _ in pairs(neighbours) do
+ n_neighbours = n_neighbours + 1
+ end
settings.cluster_nodes = n_neighbours
end
- -- Fill template
+-- Fill template
template.count = settings.count
template.threshold = settings.threshold
template.common_ttl = settings.common_ttl
template.expire_step = settings.interval
template.hostname = rspamd_util.get_hostname()
-for k,v in pairs(template) do
+for k, v in pairs(template) do
template[k] = tostring(v)
end
data[5]
}
logger.infox(rspamd_config,
- 'finished expiry %s: %s items checked, %s significant (%s %s), ' ..
- '%s insignificant (%s %s), %s common (%s discriminated), ' ..
- '%s infrequent (%s %s), %s mean, %s std',
- lutil.unpack(d))
+ 'finished expiry %s: %s items checked, %s significant (%s %s), ' ..
+ '%s insignificant (%s %s), %s common (%s discriminated), ' ..
+ '%s infrequent (%s %s), %s mean, %s std',
+ lutil.unpack(d))
if cycle then
- for i,cl in ipairs({'in ham', 'in spam', 'total'}) do
+ for i, cl in ipairs({ 'in ham', 'in spam', 'total' }) do
logger.infox(rspamd_config, 'tokens occurrences, %s: {%s}', cl, occ_distr[i])
end
end
end
end
lredis.exec_redis_script(cls.script,
- {ev_base = ev_base, is_write = true},
+ { ev_base = ev_base, is_write = true },
redis_step_cb,
- {'RS*_*', cls.expiry}
+ { 'RS*_*', cls.expiry }
)
end
-rspamd_config:add_on_load(function (_, ev_base, worker)
+rspamd_config:add_on_load(function(_, ev_base, worker)
-- Exit unless we're the first 'controller' worker
- if not worker:is_primary_controller() then return end
+ if not worker:is_primary_controller() then
+ return
+ end
local unique_redis_params = {}
-- Push redis script to all unique redis servers
- for _,cls in ipairs(settings.classifiers) do
+ for _, cls in ipairs(settings.classifiers) do
if not unique_redis_params[cls.redis_params.hash] then
unique_redis_params[cls.redis_params.hash] = cls.redis_params
end
end
- for h,rp in pairs(unique_redis_params) do
+ for h, rp in pairs(unique_redis_params) do
local script_id = lredis.add_redis_script(lutil.template(expiry_script,
template), rp)
- for _,cls in ipairs(settings.classifiers) do
+ for _, cls in ipairs(settings.classifiers) do
if cls.redis_params.hash == h then
cls.script = script_id
end
end
-- Expire tokens at regular intervals
- for _,cls in ipairs(settings.classifiers) do
+ for _, cls in ipairs(settings.classifiers) do
rspamd_config:add_periodic(ev_base,
settings['interval'],
- function ()
+ function()
expire_step(cls, ev_base, worker)
return true
end, true)
local function gen_bimi_grammar()
local lpeg = require "lpeg"
lpeg.locale(lpeg)
- local space = lpeg.space^0
- local name = lpeg.C(lpeg.alpha^1) * space
- local sep = (lpeg.S("\\;") * space) + (lpeg.space^1)
- local value = lpeg.C(lpeg.P(lpeg.graph - sep)^1)
- local pair = lpeg.Cg(name * "=" * space * value) * sep^-1
- local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset)
+ local space = lpeg.space ^ 0
+ local name = lpeg.C(lpeg.alpha ^ 1) * space
+ local sep = (lpeg.S("\\;") * space) + (lpeg.space ^ 1)
+ local value = lpeg.C(lpeg.P(lpeg.graph - sep) ^ 1)
+ local pair = lpeg.Cg(name * "=" * space * value) * sep ^ -1
+ local list = lpeg.Cf(lpeg.Ct("") * pair ^ 0, rawset)
local version = lpeg.P("v") * space * lpeg.P("=") * space * lpeg.P("BIMI1")
local record = version * sep * list
local content = rspamd_util.encode_base64(rspamd_util.decode_base64(bimi_content),
73, task:get_newlines_type())
lua_mime.modify_headers(task, {
- remove = {[hdr_name] = 0},
+ remove = { [hdr_name] = 0 },
add = {
[hdr_name] = {
order = 0,
}
}
})
- task:insert_result('BIMI_VALID', 1.0, {domain})
+ task:insert_result('BIMI_VALID', 1.0, { domain })
end
local function process_bimi_json(task, domain, redis_data)
local parser = ucl.parser()
- local _,err = parser:parse_string(redis_data)
+ local _, err = parser:parse_string(redis_data)
if err then
rspamd_logger.errx(task, "cannot parse BIMI result from Redis for %s: %s",
end
if is_sync then
local parser = ucl.parser()
- local _,err = parser:parse_string(body)
+ local _, err = parser:parse_string(body)
if err then
rspamd_logger.errx(task, "cannot parse BIMI result from helper for %s: %s",
upstream:fail()
else
lua_util.debugm(N, task, 'stored bimi image in Redis for domain %s; key=%s',
- domain, redis_key)
+ domain, redis_key)
end
end
- ret,_,upstream = lua_redis.redis_make_request(task,
+ ret, _, upstream = lua_redis.redis_make_request(task,
redis_params, -- connect params
redis_key, -- hash key
true, -- is write
redis_set_cb, --callback
'PSETEX', -- command
- {redis_key, tostring(settings.redis_min_expiry * 1000.0),
- ucl.to_format(d, "json-compact")})
+ { redis_key, tostring(settings.redis_min_expiry * 1000.0),
+ ucl.to_format(d, "json-compact") })
if not ret then
rspamd_logger.warnx(task, 'cannot make request to Redis when storing image; domain %s',
local serialised = ucl.to_format(request_data, 'json-compact')
lua_util.debugm(N, task, "send request to BIMI helper: %s",
- serialised)
+ serialised)
rspamd_http.request({
task = task,
mime_type = 'application/json',
local function check_bimi_vmc(task, domain, record)
local redis_key = string.format('%s%s', settings.redis_prefix,
- domain)
+ domain)
local ret, _, upstream
local function redis_cached_cb(err, data)
end
-- We first check Redis and then try to use helper
- ret,_,upstream = lua_redis.redis_make_request(task,
+ ret, _, upstream = lua_redis.redis_make_request(task,
redis_params, -- connect params
redis_key, -- hash key
false, -- is write
redis_cached_cb, --callback
'GET', -- command
- {redis_key})
+ { redis_key })
if not ret then
rspamd_logger.warnx(task, 'cannot make request to Redis; domain %s', domain)
local function check_bimi_dns(task, domain)
local resolve_name = string.format('default._bimi.%s', domain)
- local dns_cb = function (_, _, results, err)
+ local dns_cb = function(_, _, results, err)
if err then
lua_util.debugm(N, task, "cannot resolve bimi for %s: %s",
domain, err)
else
- for _,rec in ipairs(results) do
+ for _, rec in ipairs(results) do
local res = check_bimi_record(task, rec)
if res then
end
end
task:get_resolver():resolve_txt({
- task=task,
+ task = task,
name = resolve_name,
callback = dns_cb,
forced = true
local function bimi_callback(task)
local dmarc_domain_maybe = check_dmarc_policy(task)
- if not dmarc_domain_maybe then return end
+ if not dmarc_domain_maybe then
+ return
+ end
-- We can either check BIMI via DNS or check Redis cache
end
settings = lua_util.override_defaults(settings, opts)
-local res,err = settings_schema:transform(settings)
+local res, err = settings_schema:transform(settings)
if not res then
rspamd_logger.warnx(rspamd_config, 'plugin is misconfigured: %s', err)
name = 'BIMI_CHECK',
type = 'normal',
callback = bimi_callback,
- augmentations = {string.format("timeout=%f", settings.helper_timeout or
- redis_params.timeout or 0.0)}
+ augmentations = { string.format("timeout=%f", settings.helper_timeout or
+ redis_params.timeout or 0.0) }
})
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
name = 'BIMI_VALID',
type = 'virtual',
parent = id,
collect_garbage = false, -- Perform GC collection after sending the data
check_timeout = 10.0, -- Periodic timeout
timeout = 5.0,
- bayes_spam_symbols = {'BAYES_SPAM'},
- bayes_ham_symbols = {'BAYES_HAM'},
- ann_symbols_spam = {'NEURAL_SPAM'},
- ann_symbols_ham = {'NEURAL_HAM'},
- fuzzy_symbols = {'FUZZY_DENIED'},
- whitelist_symbols = {'WHITELIST_DKIM', 'WHITELIST_SPF_DKIM', 'WHITELIST_DMARC'},
- dkim_allow_symbols = {'R_DKIM_ALLOW'},
- dkim_reject_symbols = {'R_DKIM_REJECT'},
- dkim_dnsfail_symbols = {'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL'},
- dkim_na_symbols = {'R_DKIM_NA'},
- dmarc_allow_symbols = {'DMARC_POLICY_ALLOW'},
- dmarc_reject_symbols = {'DMARC_POLICY_REJECT'},
- dmarc_quarantine_symbols = {'DMARC_POLICY_QUARANTINE'},
- dmarc_softfail_symbols = {'DMARC_POLICY_SOFTFAIL'},
- dmarc_na_symbols = {'DMARC_NA'},
- spf_allow_symbols = {'R_SPF_ALLOW'},
- spf_reject_symbols = {'R_SPF_FAIL'},
- spf_dnsfail_symbols = {'R_SPF_DNSFAIL', 'R_SPF_PERMFAIL'},
- spf_neutral_symbols = {'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL'},
- spf_na_symbols = {'R_SPF_NA'},
+ bayes_spam_symbols = { 'BAYES_SPAM' },
+ bayes_ham_symbols = { 'BAYES_HAM' },
+ ann_symbols_spam = { 'NEURAL_SPAM' },
+ ann_symbols_ham = { 'NEURAL_HAM' },
+ fuzzy_symbols = { 'FUZZY_DENIED' },
+ whitelist_symbols = { 'WHITELIST_DKIM', 'WHITELIST_SPF_DKIM', 'WHITELIST_DMARC' },
+ dkim_allow_symbols = { 'R_DKIM_ALLOW' },
+ dkim_reject_symbols = { 'R_DKIM_REJECT' },
+ dkim_dnsfail_symbols = { 'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL' },
+ dkim_na_symbols = { 'R_DKIM_NA' },
+ dmarc_allow_symbols = { 'DMARC_POLICY_ALLOW' },
+ dmarc_reject_symbols = { 'DMARC_POLICY_REJECT' },
+ dmarc_quarantine_symbols = { 'DMARC_POLICY_QUARANTINE' },
+ dmarc_softfail_symbols = { 'DMARC_POLICY_SOFTFAIL' },
+ dmarc_na_symbols = { 'DMARC_NA' },
+ spf_allow_symbols = { 'R_SPF_ALLOW' },
+ spf_reject_symbols = { 'R_SPF_FAIL' },
+ spf_dnsfail_symbols = { 'R_SPF_DNSFAIL', 'R_SPF_PERMFAIL' },
+ spf_neutral_symbols = { 'R_DKIM_TEMPFAIL', 'R_DKIM_PERMFAIL' },
+ spf_na_symbols = { 'R_SPF_NA' },
stop_symbols = {},
ipmask = 19,
ipmask6 = 48,
}
--- @language SQL
-local clickhouse_schema = {[[
+local clickhouse_schema = { [[
CREATE TABLE IF NOT EXISTS rspamd
(
Date Date COMMENT 'Date (used for partitioning)',
PARTITION BY toMonday(Date)
ORDER BY TS
]],
-[[CREATE TABLE IF NOT EXISTS rspamd_version ( Version UInt32) ENGINE = TinyLog]],
-{[[INSERT INTO rspamd_version (Version) Values (${SCHEMA_VERSION})]], true},
+ [[CREATE TABLE IF NOT EXISTS rspamd_version ( Version UInt32) ENGINE = TinyLog]],
+ { [[INSERT INTO rspamd_version (Version) Values (${SCHEMA_VERSION})]], true },
}
-- This describes SQL queries to migrate between versions
'SettingsId',
}
- for _,v in ipairs(fields) do table.insert(res, v) end
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
end
local function clickhouse_attachments_row(res)
'Attachments.Digest',
}
- for _,v in ipairs(fields) do table.insert(res, v) end
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
end
local function clickhouse_urls_row(res)
'Urls.Url',
'Urls.Flags',
}
- for _,v in ipairs(fields) do table.insert(res, v) end
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
end
local function clickhouse_emails_row(res)
local fields = {
'Emails',
}
- for _,v in ipairs(fields) do table.insert(res, v) end
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
end
local function clickhouse_symbols_row(res)
'Symbols.Scores',
'Symbols.Options',
}
- for _,v in ipairs(fields) do table.insert(res, v) end
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
end
local function clickhouse_groups_row(res)
'Groups.Names',
'Groups.Scores',
}
- for _,v in ipairs(fields) do table.insert(res, v) end
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
end
local function clickhouse_asn_row(res)
'Country',
'IPNet',
}
- for _,v in ipairs(fields) do table.insert(res, v) end
+ for _, v in ipairs(fields) do
+ table.insert(res, v)
+ end
end
local function clickhouse_extra_columns(res)
- for _,v in ipairs(settings.extra_columns) do table.insert(res, v.name) end
+ for _, v in ipairs(settings.extra_columns) do
+ table.insert(res, v.name)
+ end
end
local function today(ts)
local function clickhouse_check_symbol(task, settings_field_name, fields_table,
field_name, value, value_negative)
- for _,s in ipairs(settings[settings_field_name] or {}) do
+ for _, s in ipairs(settings[settings_field_name] or {}) do
if task:has_symbol(s) then
if value_negative then
local sym = task:get_symbol(s)[1]
#gen_rows + #cust_rows, ip_addr, why)
local function gen_success_cb(what, how_many)
- return function (_, _)
+ return function(_, _)
rspamd_logger.messagex(log_object, "sent %s rows of %s to clickhouse server %s; started as %s",
how_many, what, ip_addr, why)
upstream:ok()
end
local function gen_fail_cb(what, how_many)
- return function (_, err)
+ return function(_, err)
rspamd_logger.errx(log_object, "cannot send %s rows of %s data to clickhouse server %s: %s; started as %s",
how_many, what, ip_addr, err, why)
upstream:fail()
string.format('INSERT INTO rspamd (%s)',
table.concat(fields, ',')))
- for k,crows in pairs(cust_rows) do
+ for k, crows in pairs(cust_rows) do
if #crows > 1 then
- send_data('custom data ('..k..')', crows,
+ send_data('custom data (' .. k .. ')', crows,
settings.custom_rules[k].first_row())
end
end
return
end
- for _,sym in ipairs(settings.stop_symbols) do
+ for _, sym in ipairs(settings.stop_symbols) do
if task:has_symbol(sym) then
rspamd_logger.infox(task, 'skip Clickhouse storage for message: symbol %s has fired', sym)
return
end
if settings.exceptions then
- local excepted,trace = settings.exceptions:process(task)
+ local excepted, trace = settings.exceptions:process(task)
if excepted then
rspamd_logger.infox(task, 'skipped Clickhouse storage for message: excepted (%s)',
trace)
local from_domain = ''
local from_user = ''
if task:has_from('smtp') then
- local from = task:get_from({'smtp','orig'})[1]
+ local from = task:get_from({ 'smtp', 'orig' })[1]
if from then
from_domain = from['domain']:lower()
local mime_domain = ''
local mime_user = ''
if task:has_from('mime') then
- local from = task:get_from({'mime','orig'})[1]
+ local from = task:get_from({ 'mime', 'orig' })[1]
if from then
mime_domain = from['domain']:lower()
mime_user = from['user']
local mime_recipients = {}
if task:has_recipients('mime') then
- local recipients = task:get_recipients({'mime','orig'})
+ local recipients = task:get_recipients({ 'mime', 'orig' })
for _, rcpt in ipairs(recipients) do
table.insert(mime_recipients, rcpt['user'] .. '@' .. rcpt['domain']:lower())
end
local ret
- ret = clickhouse_check_symbol(task,'bayes_spam_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'bayes_spam_symbols', fields,
'bayes', 'spam')
if not ret then
- clickhouse_check_symbol(task,'bayes_ham_symbols', fields,
+ clickhouse_check_symbol(task, 'bayes_ham_symbols', fields,
'bayes', 'ham')
end
- clickhouse_check_symbol(task,'ann_symbols_spam', fields,
+ clickhouse_check_symbol(task, 'ann_symbols_spam', fields,
'ann', 'spam')
if not ret then
- clickhouse_check_symbol(task,'ann_symbols_ham', fields,
+ clickhouse_check_symbol(task, 'ann_symbols_ham', fields,
'ann', 'ham')
end
- clickhouse_check_symbol(task,'whitelist_symbols', fields,
+ clickhouse_check_symbol(task, 'whitelist_symbols', fields,
'whitelist', 'blacklist', 'whitelist')
- clickhouse_check_symbol(task,'fuzzy_symbols', fields,
+ clickhouse_check_symbol(task, 'fuzzy_symbols', fields,
'fuzzy', 'deny')
-
- ret = clickhouse_check_symbol(task,'dkim_allow_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'dkim_allow_symbols', fields,
'dkim', 'allow')
if not ret then
- ret = clickhouse_check_symbol(task,'dkim_reject_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'dkim_reject_symbols', fields,
'dkim', 'reject')
end
if not ret then
- ret = clickhouse_check_symbol(task,'dkim_dnsfail_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'dkim_dnsfail_symbols', fields,
'dkim', 'dnsfail')
end
if not ret then
- clickhouse_check_symbol(task,'dkim_na_symbols', fields,
+ clickhouse_check_symbol(task, 'dkim_na_symbols', fields,
'dkim', 'na')
end
-
- ret = clickhouse_check_symbol(task,'dmarc_allow_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'dmarc_allow_symbols', fields,
'dmarc', 'allow')
if not ret then
- ret = clickhouse_check_symbol(task,'dmarc_reject_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'dmarc_reject_symbols', fields,
'dmarc', 'reject')
end
if not ret then
- ret = clickhouse_check_symbol(task,'dmarc_quarantine_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'dmarc_quarantine_symbols', fields,
'dmarc', 'quarantine')
end
if not ret then
- ret = clickhouse_check_symbol(task,'dmarc_softfail_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'dmarc_softfail_symbols', fields,
'dmarc', 'softfail')
end
if not ret then
- clickhouse_check_symbol(task,'dmarc_na_symbols', fields,
+ clickhouse_check_symbol(task, 'dmarc_na_symbols', fields,
'dmarc', 'na')
end
-
- ret = clickhouse_check_symbol(task,'spf_allow_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'spf_allow_symbols', fields,
'spf', 'allow')
if not ret then
- ret = clickhouse_check_symbol(task,'spf_reject_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'spf_reject_symbols', fields,
'spf', 'reject')
end
if not ret then
- ret = clickhouse_check_symbol(task,'spf_neutral_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'spf_neutral_symbols', fields,
'spf', 'neutral')
end
if not ret then
- ret = clickhouse_check_symbol(task,'spf_dnsfail_symbols', fields,
+ ret = clickhouse_check_symbol(task, 'spf_dnsfail_symbols', fields,
'spf', 'dnsfail')
end
if not ret then
- clickhouse_check_symbol(task,'spf_na_symbols', fields,
+ clickhouse_check_symbol(task, 'spf_na_symbols', fields,
'spf', 'na')
end
local nurls = 0
local task_urls = task:get_urls({
- content = true,
- images = true,
- emails = false,
- sort = true,
+ content = true,
+ images = true,
+ emails = false,
+ sort = true,
}) or {}
nurls = #task_urls
if #attachments_fnames > 0 then
table.insert(row, attachments_fnames)
- table.insert(row, attachments_ctypes)
- table.insert(row, attachments_lengths)
- table.insert(row, attachments_digests)
+ table.insert(row, attachments_ctypes)
+ table.insert(row, attachments_lengths)
+ table.insert(row, attachments_digests)
else
table.insert(row, {})
table.insert(row, {})
local urls_flags = {}
if settings.full_urls then
- for i,u in ipairs(task_urls) do
+ for i, u in ipairs(task_urls) do
urls_urls[i] = u:get_text()
urls_tlds[i] = u:get_tld() or u:get_host()
urls_flags[i] = u:get_flags_num()
idx_tbl = {}, -- indexed by host + flags, reference to an index in ord_tbl
__newindex = function(t, k, v)
local idx = getmetatable(t).idx_tbl
- local ord = getmetatable(t).ord_tbl
+ local ord = getmetatable(t).ord_tbl
local key = k:get_host() .. tostring(k:get_flags_num())
if idx[key] then
ord[idx[key]] = v -- replace
-- Extra index needed for making this unique
local urls_idx = {}
setmetatable(urls_idx, mt)
- for _,u in ipairs(task_urls) do
+ for _, u in ipairs(task_urls) do
if not urls_idx[u] then
urls_idx[u] = u
urls_urls[#urls_urls + 1] = u:get_host()
if task:has_urls(true) then
local emails = task:get_emails() or {}
local emails_formatted = {}
- for i,u in ipairs(emails) do
+ for i, u in ipairs(emails) do
emails_formatted[i] = string.format('%s@%s', u:get_user(), u:get_host())
end
table.insert(row, emails_formatted)
local scores_tab = {}
local options_tab = {}
- for _,s in ipairs(symbols) do
+ for _, s in ipairs(symbols) do
table.insert(syms_tab, s.name or '')
table.insert(scores_tab, s.score)
local groups = task:get_groups()
local groups_tab = {}
local gr_scores_tab = {}
- for gr,sc in pairs(groups) do
+ for gr, sc in pairs(groups) do
table.insert(groups_tab, gr)
table.insert(gr_scores_tab, sc)
end
-- Extra columns
if #settings.extra_columns > 0 then
- for _,col in ipairs(settings.extra_columns) do
+ for _, col in ipairs(settings.extra_columns) do
local elts = col.real_selector(task)
if elts then
end
-- Custom data
- for k,rule in pairs(settings.custom_rules) do
- if not custom_rows[k] then custom_rows[k] = {} end
+ for k, rule in pairs(settings.custom_rules) do
+ if not custom_rows[k] then
+ custom_rows[k] = {}
+ end
table.insert(custom_rows[k], lua_clickhouse.row_to_tsv(rule.get_row(task)))
end
local remove_partition_sql = "ALTER TABLE ${table_name} ${remove_method} PARTITION '${partition}'"
local remove_method = (settings.retention.method == 'drop') and 'DROP' or 'DETACH'
local sql_params = {
- ['table_name'] = table_name,
- ['remove_method'] = remove_method,
- ['partition'] = partition
+ ['table_name'] = table_name,
+ ['remove_method'] = remove_method,
+ ['partition'] = partition
}
local sql = lua_util.template(remove_partition_sql, sql_params)
local err, _ = lua_clickhouse.generic_sync(upstream, settings, ch_params, sql)
if err then
rspamd_logger.errx(rspamd_config,
- "cannot detach partition %s:%s from server %s: %s",
- table_name, partition,
- settings['server'], err)
+ "cannot detach partition %s:%s from server %s: %s",
+ table_name, partition,
+ settings['server'], err)
return
end
"GROUP BY partition, table " ..
"HAVING max(max_date) < toDate(now() - interval ${month} month)"
- local table_names = {'rspamd'}
+ local table_names = { 'rspamd' }
local tables = table.concat(table_names, "', '")
local sql_params = {
tables = tables,
- month = settings.retention.period_months,
+ month = settings.retention.period_months,
}
local sql = lua_util.template(partition_to_remove_sql, sql_params)
-
local ch_params = {
ev_base = ev_base,
config = cfg,
local err, rows = lua_clickhouse.select_sync(upstream, settings, ch_params, sql)
if err then
rspamd_logger.errx(rspamd_config,
- "cannot send data to clickhouse server %s: %s",
- settings['server'], err)
+ "cannot send data to clickhouse server %s: %s",
+ settings['server'], err)
else
fun.each(function(row)
do_remove_partition(ev_base, cfg, row.table, row.partition)
-- Process element and return nil if statement should be skipped
local function preprocess_schema_elt(v)
if type(v) == 'string' then
- return lua_util.template(v, {SCHEMA_VERSION = tostring(schema_version)})
+ return lua_util.template(v, { SCHEMA_VERSION = tostring(schema_version) })
elseif type(v) == 'table' then
-- Pair of statement + boolean
if initial == v[2] then
- return lua_util.template(v[1], {SCHEMA_VERSION = tostring(schema_version)})
+ return lua_util.template(v[1], { SCHEMA_VERSION = tostring(schema_version) })
else
rspamd_logger.debugm(N, rspamd_config, 'skip clickhouse schema element %s: schema already exists',
v)
-- Apply schema elements sequentially, users additions are concatenated to the tail
fun.each(upload_schema_elt,
- -- Also template schema version
- fun.filter(function(v) return v ~= nil end,
- fun.map(preprocess_schema_elt,
- fun.chain(clickhouse_schema, settings.schema_additions)
+ -- Also template schema version
+ fun.filter(function(v)
+ return v ~= nil
+ end,
+ fun.map(preprocess_schema_elt,
+ fun.chain(clickhouse_schema, settings.schema_additions)
+ )
)
- )
)
end
}
-- Apply migrations sequentially
local function migration_recursor(i)
- if i < schema_version then
+ if i < schema_version then
if migrations[i] then
-- We also need to apply statements sequentially
local function sql_recursor(j)
-- Apply the next statement
sql_recursor(j + 1)
end
- end ,
+ end,
function(_, err)
rspamd_logger.errx(rspamd_config,
"cannot apply migration %s: '%s' on clickhouse server %s: %s",
}
-- Apply migrations sequentially
local function columns_recursor(i)
- if i <= #settings.extra_columns then
+ if i <= #settings.extra_columns then
local col = settings.extra_columns[i]
local prev_column
if i == 1 then
col.name, col.type, prev_column)
-- Apply the next statement
columns_recursor(i + 1)
- end ,
+ end,
function(_, err)
rspamd_logger.errx(rspamd_config,
"cannot apply add column alter %s: '%s' on clickhouse server %s: %s",
local err, rows = lua_clickhouse.select_sync(upstream, settings, ch_params, sql)
if err then
rspamd_logger.errx(rspamd_config, "cannot check rspamd table in clickhouse server %s: %s",
- upstream:get_addr():to_string(true), err)
+ upstream:get_addr():to_string(true), err)
return
end
end
end
-
local function check_clickhouse_upstream(upstream, ev_base, cfg)
local ch_params = {
ev_base = ev_base,
config = cfg,
}
-- If we have some custom rules, we just send its schema to the upstream
- for k,rule in pairs(settings.custom_rules) do
+ for k, rule in pairs(settings.custom_rules) do
if rule.schema then
local sql = lua_util.template(rule.schema, settings)
local err, _ = lua_clickhouse.generic_sync(upstream, settings, ch_params, sql)
if err then
rspamd_logger.errx(rspamd_config, 'cannot send custom schema %s to clickhouse server %s: ' ..
- 'cannot make request (%s)',
+ 'cannot make request (%s)',
k, upstream:get_addr():to_string(true), err)
end
end
else
rspamd_logger.errx(rspamd_config,
"cannot get version on clickhouse server %s: %s",
- upstream:get_addr():to_string(true), err)
+ upstream:get_addr():to_string(true), err)
end
else
upload_clickhouse_schema(upstream, ev_base, cfg, false)
if opts.limit and not opts.limits then
settings.limits.max_rows = opts.limit
end
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
if k == 'custom_rules' then
if not v[1] then
- v = {v}
+ v = { v }
end
- for i,rule in ipairs(v) do
+ for i, rule in ipairs(v) do
if rule.schema and rule.first_row and rule.get_row then
local first_row, get_row
local loadstring = loadstring or load
-- Select traverse function depending on what we have
local iter_func = settings.extra_columns[1] and ipairs or pairs
- for col_name,col_data in iter_func(settings.extra_columns) do
+ for col_name, col_data in iter_func(settings.extra_columns) do
-- Array based extra columns
- if col_data.name then col_name = col_data.name end
+ if col_data.name then
+ col_name = col_data.name
+ end
if not col_data.selector or not col_data.type then
rspamd_logger.errx(rspamd_config, 'cannot add clickhouse extra row %s: no type or no selector',
col_name)
-- preserve strict order when doing altering
if need_sort then
rspamd_logger.infox(rspamd_config, 'sort extra columns as they are not configured as an array')
- table.sort(columns_transformed, function(c1, c2) return c1.name < c2.name end)
+ table.sort(columns_transformed, function(c1, c2)
+ return c1.name < c2.name
+ end)
end
settings.extra_columns = columns_transformed
end
type = 'idempotent',
callback = clickhouse_collect,
flags = 'empty,explicit_disable,ignore_passthrough',
- augmentations = {string.format("timeout=%f", settings.timeout)},
+ augmentations = { string.format("timeout=%f", settings.timeout) },
})
rspamd_config:register_finish_script(function(task)
if nrows > 0 then
if worker:is_primary_controller() then
local upstreams = settings.upstream:all_upstreams()
- for _,up in ipairs(upstreams) do
+ for _, up in ipairs(upstreams) do
check_clickhouse_upstream(up, ev_base, cfg)
end
score_mult = 0.1,
}
-local rule_schema = ts.shape{
+local rule_schema = ts.shape {
max_elts = ts.number + ts.string / tonumber,
expire = ts.number + ts.string / lua_util.parse_time_interval,
expire_overflow = ts.number + ts.string / lua_util.parse_time_interval,
'processed rule %s, selectors: source="%s", cluster="%s"; data: %s elts, %s score, %s elt score',
rule.name, source_selector, cluster_selector, cur_elts, total_score, element_score)
if final_score > 0.1 then
- task:insert_result(rule.symbol, final_score, {source_selector,
- tostring(size_score),
- tostring(cluster_score)})
+ task:insert_result(rule.symbol, final_score, { source_selector,
+ tostring(size_score),
+ tostring(cluster_score) })
end
end
end
lua_redis.exec_redis_script(query_cluster_id,
- {task = task, is_write = false, key = source_selector},
+ { task = task, is_write = false, key = source_selector },
redis_get_cb,
- {source_selector, cluster_selector})
+ { source_selector, cluster_selector })
end
local function clusterting_idempotent_cb(task, rule)
- if task:has_flag('skip') then return end
- if not rule.allow_local and lua_util.is_rspamc_or_controller(task) then return end
+ if task:has_flag('skip') then
+ return
+ end
+ if not rule.allow_local and lua_util.is_rspamc_or_controller(task) then
+ return
+ end
local verdict = lua_verdict.get_specific_verdict(N, task)
local score
end
lua_redis.exec_redis_script(update_cluster_id,
- {task = task, is_write = true, key = source_selector},
+ { task = task, is_write = true, key = source_selector },
redis_set_cb,
{
source_selector,
end
if opts['rules'] then
- for k,v in pairs(opts['rules']) do
+ for k, v in pairs(opts['rules']) do
local raw_rule = lua_util.override_defaults(default_rule, v)
- local rule,err = rule_schema:transform(raw_rule)
+ local rule, err = rule_schema:transform(raw_rule)
if not rule then
rspamd_logger.errx(rspamd_config, 'invalid clustering rule %s: %s',
k, err)
else
- if not rule.symbol then rule.symbol = k end
- if not rule.prefix then rule.prefix = k .. "_" end
+ if not rule.symbol then
+ rule.symbol = k
+ end
+ if not rule.prefix then
+ rule.prefix = k .. "_"
+ end
rule.source_selector = lua_selectors.create_selector_closure(rspamd_config,
rule.source_selector, '')
- rule.cluster_selector = lua_selectors.create_selector_closure(rspamd_config,
+ rule.cluster_selector = lua_selectors.create_selector_closure(rspamd_config,
rule.cluster_selector, '')
if rule.source_selector and rule.cluster_selector then
rule.name = k
query_cluster_id = lua_redis.add_redis_script(query_cluster_script, redis_params)
update_cluster_id = lua_redis.add_redis_script(update_cluster_script, redis_params)
local function callback_gen(f, rule)
- return function(task) return f(task, rule) end
+ return function(task)
+ return f(task, rule)
+ end
end
- for _,rule in ipairs(rules) do
- rspamd_config:register_symbol{
+ for _, rule in ipairs(rules) do
+ rspamd_config:register_symbol {
name = rule.symbol,
type = 'normal',
callback = callback_gen(clusterting_filter_cb, rule),
}
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
name = rule.symbol .. '_STORE',
type = 'idempotent',
flags = 'empty,explicit_disable,ignore_passthrough',
callback = callback_gen(clusterting_idempotent_cb, rule),
- augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
}
end
else
local rspamd_logger = require "rspamd_logger"
local dcc = require("lua_scanners").filter('dcc').dcc
-
if confighelp then
rspamd_config:add_example(nil, 'dcc',
- "Check messages for 'bulkiness' using DCC",
- [[
-dcc {
- socket = "/var/dcc/dccifd"; # Unix socket
- servers = "127.0.0.1:10045" # OR TCP upstreams
- timeout = 2s; # Timeout to wait for checks
- body_max = 999999; # Bulkness threshold for body
- fuz1_max = 999999; # Bulkness threshold for fuz1
- fuz2_max = 999999; # Bulkness threshold for fuz2
-}
-]])
+ "Check messages for 'bulkiness' using DCC",
+ [[
+ dcc {
+ socket = "/var/dcc/dccifd"; # Unix socket
+ servers = "127.0.0.1:10045" # OR TCP upstreams
+ timeout = 2s; # Timeout to wait for checks
+ body_max = 999999; # Bulkness threshold for body
+ fuz1_max = 999999; # Bulkness threshold for fuz1
+ fuz2_max = 999999; # Bulkness threshold for fuz2
+ }
+ ]])
return
end
-- WORKAROUND for deprecated host and port settings
if opts['host'] ~= nil and opts['port'] ~= nil then
opts['servers'] = opts['host'] .. ':' .. opts['port']
- rspamd_logger.warnx(rspamd_config, 'Using host and port parameters is deprecated. '..
- 'Please use servers = "%s:%s"; instead', opts['host'], opts['port'])
+ rspamd_logger.warnx(rspamd_config, 'Using host and port parameters is deprecated. ' ..
+ 'Please use servers = "%s:%s"; instead', opts['host'], opts['port'])
end
if opts['host'] ~= nil and not opts['port'] then
opts['socket'] = opts['host']
- rspamd_logger.warnx(rspamd_config, 'Using host parameters is deprecated. '..
- 'Please use socket = "%s"; instead', opts['host'])
+ rspamd_logger.warnx(rspamd_config, 'Using host parameters is deprecated. ' ..
+ 'Please use socket = "%s"; instead', opts['host'])
end
-- WORKAROUND for deprecated host and port settings
-if not opts.symbol_bulk then opts.symbol_bulk = symbol_bulk end
-if not opts.symbol then opts.symbol = symbol end
+if not opts.symbol_bulk then
+ opts.symbol_bulk = symbol_bulk
+end
+if not opts.symbol then
+ opts.symbol = symbol
+end
rule = dcc.configure(opts)
callback = check_dcc,
type = 'callback',
})
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual',
parent = id,
name = opts.symbol
}
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual',
parent = id,
name = opts.symbol_bulk
}
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual',
parent = id,
name = 'DCC_FAIL'
if settings.use_milter_headers then
lua_mime.modify_headers(task, {
add = {
- ['DKIM-Signature'] = {order = 1, value = hdr},
+ ['DKIM-Signature'] = { order = 1, value = hdr },
}
})
end
end
local function dkim_signing_cb(task)
- local ret,selectors = dkim_sign_tools.prepare_dkim_signing(N, task, settings)
+ local ret, selectors = dkim_sign_tools.prepare_dkim_signing(N, task, settings)
if not ret then
return
end
end
-local opts = rspamd_config:get_all_opt('dkim_signing')
-if not opts then return end
+local opts = rspamd_config:get_all_opt('dkim_signing')
+if not opts then
+ return
+end
dkim_sign_tools.process_signing_settings(N, settings, opts)
local sym_reg_tbl = {
name = settings['symbol'],
callback = dkim_signing_cb,
- groups = {"policies", "dkim"},
+ groups = { "policies", "dkim" },
flags = 'ignore_passthrough',
score = 0.0,
}
table.insert(reason, "No valid SPF")
end
-
local opts = ((task:get_symbol('DKIM_TRACE') or E)[1] or E).options
local dkim_results = {
pass = {},
fail = {},
}
-
if opts then
dkim_results.pass = {}
local dkim_violated
- for _,opt in ipairs(opts) do
+ for _, opt in ipairs(opts) do
local check_res = string.sub(opt, -1)
local domain = string.sub(opt, 1, -3):lower()
therefore cannot apply the advertised DMARC policy.
]]--
if spf_tmpfail or dkim_tmpfail then
- task:insert_result(settings.symbols['dnsfail'], 1.0, policy.domain..
+ task:insert_result(settings.symbols['dnsfail'], 1.0, policy.domain ..
' : ' .. 'SPF/DKIM temp error', policy.dmarc_policy)
else
-- We can now check the failed policy and maybe send report data elt
-- Prepare and send redis report element
local period = os.date('%Y%m%d',
- task:get_date({format = 'connect', gmt = false}))
+ task:get_date({ format = 'connect', gmt = false }))
-- Dmarc domain key must include dmarc domain, rua and period
local dmarc_domain_key = table.concat(
- {settings.reporting.redis_keys.report_prefix, dmarc_esld, policy.rua, period},
+ { settings.reporting.redis_keys.report_prefix, dmarc_esld, policy.rua, period },
settings.reporting.redis_keys.join_char)
local report_data = dmarc_common.dmarc_report(task, settings, {
- spf_ok = spf_ok and 'pass' or 'fail',
- dkim_ok = dkim_ok and 'pass' or 'fail',
- disposition = (disposition == "softfail") and "none" or disposition,
- sampled_out = sampled_out,
- domain = hdrfromdom,
- spf_domain = spf_domain,
- dkim_results = dkim_results,
- spf_result = spf_result
+ spf_ok = spf_ok and 'pass' or 'fail',
+ dkim_ok = dkim_ok and 'pass' or 'fail',
+ disposition = (disposition == "softfail") and "none" or disposition,
+ sampled_out = sampled_out,
+ domain = hdrfromdom,
+ spf_domain = spf_domain,
+ dkim_results = dkim_results,
+ spf_result = spf_result
})
-
- local idx_key = table.concat({settings.reporting.redis_keys.index_prefix, period},
+ local idx_key = table.concat({ settings.reporting.redis_keys.index_prefix, period },
settings.reporting.redis_keys.join_char)
if report_data then
lua_redis.exec_redis_script(take_report_id,
- {task = task, is_write = true},
+ { task = task, is_write = true },
dmarc_report_cb,
- {idx_key, dmarc_domain_key,
- tostring(settings.reporting.max_entries), tostring(settings.reporting.keys_expire)},
- {hdrfromdom, report_data})
+ { idx_key, dmarc_domain_key,
+ tostring(settings.reporting.max_entries), tostring(settings.reporting.keys_expire) },
+ { hdrfromdom, report_data })
end
end
end
return maybe_force_action(task, 'na')
elseif (from or E)[1] then
task:insert_result(settings.symbols['na'], 1.0, 'No domain in From header')
- return maybe_force_action(task,'na')
+ return maybe_force_action(task, 'na')
else
task:insert_result(settings.symbols['na'], 1.0, 'No From header')
- return maybe_force_action(task,'na')
+ return maybe_force_action(task, 'na')
end
-
local dns_checks_inflight = 0
local dmarc_domain_policy = {}
local dmarc_tld_policy = {}
policy_target = dmarc_tld_policy
end
- return function (_, _, results, err)
+ return function(_, _, results, err)
dns_checks_inflight = dns_checks_inflight - 1
if not seen_invalid then
else
local has_valid_policy = false
- for _,rec in ipairs(results) do
- local ret,results_or_err = dmarc_common.dmarc_check_record(task, rec, is_tld)
+ for _, rec in ipairs(results) do
+ local ret, results_or_err = dmarc_common.dmarc_check_record(task, rec, is_tld)
if not ret then
if results_or_err then
end
has_valid_policy = true
- for k,v in pairs(results_or_err) do
+ for k, v in pairs(results_or_err) do
policy_target[k] = v
end
end
local resolve_name = '_dmarc.' .. hfromdom
task:get_resolver():resolve_txt({
- task=task,
+ task = task,
name = resolve_name,
callback = gen_dmarc_cb(hfromdom, false),
forced = true
resolve_name = '_dmarc.' .. dmarc_domain
task:get_resolver():resolve_txt({
- task=task,
+ task = task,
name = resolve_name,
callback = gen_dmarc_cb(dmarc_domain, true),
forced = true
end
end
-
local opts = rspamd_config:get_all_opt('dmarc')
settings = lua_util.override_defaults(settings, opts)
})
end
-
if settings.reporting == true then
rspamd_logger.errx(rspamd_config, 'old style dmarc reporting is NO LONGER supported, please read the documentation')
elseif settings.reporting.enabled then
name = settings.symbols['allow'],
parent = id,
group = 'policies',
- groups = {'dmarc'},
+ groups = { 'dmarc' },
type = 'virtual'
})
rspamd_config:register_symbol({
name = settings.symbols['reject'],
parent = id,
group = 'policies',
- groups = {'dmarc'},
+ groups = { 'dmarc' },
type = 'virtual'
})
rspamd_config:register_symbol({
name = settings.symbols['quarantine'],
parent = id,
group = 'policies',
- groups = {'dmarc'},
+ groups = { 'dmarc' },
type = 'virtual'
})
rspamd_config:register_symbol({
name = settings.symbols['softfail'],
parent = id,
group = 'policies',
- groups = {'dmarc'},
+ groups = { 'dmarc' },
type = 'virtual'
})
rspamd_config:register_symbol({
name = settings.symbols['dnsfail'],
parent = id,
group = 'policies',
- groups = {'dmarc'},
+ groups = { 'dmarc' },
type = 'virtual'
})
rspamd_config:register_symbol({
name = settings.symbols['badpolicy'],
parent = id,
group = 'policies',
- groups = {'dmarc'},
+ groups = { 'dmarc' },
type = 'virtual'
})
rspamd_config:register_symbol({
name = settings.symbols['na'],
parent = id,
group = 'policies',
- groups = {'dmarc'},
+ groups = { 'dmarc' },
type = 'virtual'
})
local munging_opts = lua_util.override_defaults(munging_defaults, settings.munging)
- if not munging_opts.list_map then
+ if not munging_opts.list_map then
rspamd_logger.errx(rspamd_config, 'cannot enable DMARC munging with no list_map parameter')
return
munging_opts.list_map = lua_maps.map_add_from_ucl(munging_opts.list_map,
'set', 'DMARC munging map of the recipients addresses to munge')
- if not munging_opts.list_map then
+ if not munging_opts.list_map then
rspamd_logger.errx(rspamd_config, 'cannot enable DMARC munging with invalid list_map (invalid map)')
return
if munging_opts.munge_map_condition then
munging_opts.munge_map_condition = lua_maps_expressions.create(rspamd_config,
- munging_opts.munge_map_condition, N)
+ munging_opts.munge_map_condition, N)
end
-
rspamd_config:register_symbol({
name = 'DMARC_MUNGED',
type = 'normal',
flags = 'nostat',
score = 0,
group = 'policies',
- groups = {'dmarc'},
+ groups = { 'dmarc' },
callback = dmarc_common.gen_munging_callback(munging_opts, settings),
- augmentations = {lua_util.dns_timeout_augmentation(rspamd_config)},
+ augmentations = { lua_util.dns_timeout_augmentation(rspamd_config) },
})
rspamd_config:register_dependency('DMARC_MUNGED', 'DMARC_CHECK')
local function apply_dynamic_actions(_, acts)
fun.each(function(k, v)
- if type(v) == 'table' then
+ if type(v) == 'table' then
v['name'] = k
if not v['priority'] then
v['priority'] = settings.priority
settings.redis_key,
true,
redis_version_set_cb,
- 'HINCRBY', {settings.redis_key, 'v', '1'})
+ 'HINCRBY', { settings.redis_key, 'v', '1' })
end
end
fun.each(function(k, v)
cur_settings.data.scores[k] = v
end,
- fun.filter(function(k)
- if cur_settings.updates.symbols[k] then
- return false
- end
- return true
- end, recv['scores']))
+ fun.filter(function(k)
+ if cur_settings.updates.symbols[k] then
+ return false
+ end
+ return true
+ end, recv['scores']))
end
if recv['actions'] then
if not cur_settings.data.actions then
fun.each(function(k, v)
cur_settings.data.actions[k] = v
end,
- fun.filter(function(k)
- if cur_settings.updates.actions[k] then
- return false
- end
- return true
- end, recv['actions']))
+ fun.filter(function(k)
+ if cur_settings.updates.actions[k] then
+ return false
+ end
+ return true
+ end, recv['actions']))
end
end
local newdata = ucl.to_format(cur_settings.data, 'json-compact')
rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params,
settings.redis_key, true,
- redis_data_set_cb, 'HSET', {settings.redis_key, 'd', newdata})
+ redis_data_set_cb, 'HSET', { settings.redis_key, 'd', newdata })
end
local function check_dynamic_conf(cfg, ev_base)
rspamd_logger.errx(cfg, "cannot read dynamic conf from redis: %s", redis_err)
elseif data and type(data) == 'string' then
local parser = ucl.parser()
- local _,err = parser:parse_string(data)
+ local _, err = parser:parse_string(data)
if err then
rspamd_logger.errx(cfg, "cannot load dynamic conf from redis: %s", err)
if not cur_settings.version or (rver and rver > cur_settings.version) then
rspamd_logger.infox(cfg, "need to load fresh dynamic settings with version %s, local version is %s",
- rver, cur_settings.version)
+ rver, cur_settings.version)
cur_settings.version = rver
rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params,
settings.redis_key, false,
- redis_load_cb, 'HGET', {settings.redis_key, 'd'})
+ redis_load_cb, 'HGET', { settings.redis_key, 'd' })
elseif cur_settings.updates.has_updates then
-- Need to send our updates to Redis
update_dynamic_conf(cfg, ev_base)
rspamd_redis.redis_make_request_taskless(ev_base, cfg, redis_params,
settings.redis_key, false,
- redis_check_cb, 'HGET', {settings.redis_key, 'v'})
+ redis_check_cb, 'HGET', { settings.redis_key, 'v' })
end
local section = rspamd_config:get_all_opt("dynamic_conf")
return
end
- for k,v in pairs(section) do
+ for k, v in pairs(section) do
settings[k] = v
end
limit = 500,
index_pattern = 'rspamd-%Y.%m.%d',
template_file = rspamd_paths['SHAREDIR'] .. '/elastic/rspamd_template.json',
- kibana_file = rspamd_paths['SHAREDIR'] ..'/elastic/kibana.json',
+ kibana_file = rspamd_paths['SHAREDIR'] .. '/elastic/kibana.json',
key_prefix = 'elastic-',
expire = 3600,
timeout = 5.0,
}
local function read_file(path)
- local file = io.open(path, "rb")
- if not file then return nil end
- local content = file:read "*a"
- file:close()
- return content
+ local file = io.open(path, "rb")
+ if not file then
+ return nil
+ end
+ local content = file:read "*a"
+ file:close()
+ return content
end
local function elastic_send_data(task)
local es_index = os.date(settings['index_pattern'])
local tbl = {}
- for _,value in pairs(rows) do
+ for _, value in pairs(rows) do
if settings.elasticsearch_version >= 7 then
- table.insert(tbl, '{ "index" : { "_index" : "'..es_index..
- '","pipeline": "rspamd-geoip"} }')
+ table.insert(tbl, '{ "index" : { "_index" : "' .. es_index ..
+ '","pipeline": "rspamd-geoip"} }')
else
- table.insert(tbl, '{ "index" : { "_index" : "'..es_index..
- '", "_type" : "_doc" ,"pipeline": "rspamd-geoip"} }')
+ table.insert(tbl, '{ "index" : { "_index" : "' .. es_index ..
+ '", "_type" : "_doc" ,"pipeline": "rspamd-geoip"} }')
end
table.insert(tbl, ucl.to_format(value, 'json-compact'))
end
local upstream = settings.upstream:get_upstream_round_robin()
local ip_addr = upstream:get_addr():to_string(true)
- local push_url = connect_prefix .. ip_addr .. '/'..es_index..'/_bulk'
+ local push_url = connect_prefix .. ip_addr .. '/' .. es_index .. '/_bulk'
local bulk_json = table.concat(tbl, "\n")
local function http_callback(err, code, _, _)
local r = {}
local ip_addr = task:get_ip()
- if ip_addr and ip_addr:is_valid() then
+ if ip_addr and ip_addr:is_valid() then
r.is_local = ip_addr:is_local()
r.ip = tostring(ip_addr)
else
r.action = task:get_metric_action()
r.rspamd_server = HOSTNAME
if r.user ~= 'unknown' then
- r.direction = "Outbound"
+ r.direction = "Outbound"
end
local s = task:get_metric_score()[1]
- r.score = s
+ r.score = s
local rcpt = task:get_recipients('smtp')
if rcpt then
r.rcpt = 'unknown'
end
- local from = task:get_from{'smtp', 'orig'}
+ local from = task:get_from { 'smtp', 'orig' }
if ((from or E)[1] or E).addr then
r.from = from[1].addr
else
r.from = 'unknown'
end
- local mime_from = task:get_from{'mime', 'orig'}
+ local mime_from = task:get_from { 'mime', 'orig' }
if ((mime_from or E)[1] or E).addr then
r.mime_from = mime_from[1].addr
else
r.asn = {}
local pool = task:get_mempool()
r.asn.country = pool:get_variable("country") or 'unknown'
- r.asn.asn = pool:get_variable("asn") or 0
+ r.asn.asn = pool:get_variable("asn") or 0
r.asn.ipnet = pool:get_variable("ipnet") or 'unknown'
local function process_header(name)
end
local function elastic_collect(task)
- if not enabled then return end
- if task:has_flag('skip') then return end
- if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then return end
+ if not enabled then
+ return
+ end
+ if task:has_flag('skip') then
+ return
+ end
+ if not settings.allow_local and lua_util.is_rspamc_or_controller(task) then
+ return
+ end
- local row = {['rspamd_meta'] = get_general_metadata(task),
- ['@timestamp'] = tostring(util.get_time() * 1000)}
+ local row = { ['rspamd_meta'] = get_general_metadata(task),
+ ['@timestamp'] = tostring(util.get_time() * 1000) }
table.insert(rows, row)
nrows = nrows + 1
if nrows > settings['limit'] then
end
end
-
local opts = rspamd_config:get_all_opt('elastic')
local function check_elastic_server(cfg, ev_base, _)
local function http_callback(err, code, body, _)
if code == 200 then
local parser = ucl.parser()
- local res,ucl_err = parser:parse_string(body)
+ local res, ucl_err = parser:parse_string(body)
if not res then
rspamd_logger.infox(rspamd_config, 'failed to parse reply from %s: %s',
plugins_url, ucl_err)
return
end
local obj = parser:get_object()
- for node,value in pairs(obj['nodes']) do
+ for node, value in pairs(obj['nodes']) do
local plugin_found = false
- for _,plugin in pairs(value['plugins']) do
+ for _, plugin in pairs(value['plugins']) do
if plugin['name'] == 'ingest-geoip' then
plugin_found = true
lua_util.debugm(N, "ingest-geoip plugin has been found")
-- import ingest pipeline and kibana dashboard/visualization
local function initial_setup(cfg, ev_base, worker)
- if not worker:is_primary_controller() then return end
+ if not worker:is_primary_controller() then
+ return
+ end
local upstream = settings.upstream:get_upstream_round_robin()
local ip_addr = upstream:get_addr():to_string(true)
local kibana_mappings = read_file(settings['kibana_file'])
if kibana_mappings then
local parser = ucl.parser()
- local res,parser_err = parser:parse_string(kibana_mappings)
+ local res, parser_err = parser:parse_string(kibana_mappings)
if not res then
rspamd_logger.infox(rspamd_config, 'kibana template cannot be parsed: %s',
parser_err)
end
local obj = parser:get_object()
local tbl = {}
- for _,item in ipairs(obj) do
- table.insert(tbl, '{ "index" : { "_index" : ".kibana", "_type" : "doc" ,"_id": "'..
- item['_type'] .. ':' .. item["_id"]..'"} }')
+ for _, item in ipairs(obj) do
+ table.insert(tbl, '{ "index" : { "_index" : ".kibana", "_type" : "doc" ,"_id": "' ..
+ item['_type'] .. ':' .. item["_id"] .. '"} }')
table.insert(tbl, ucl.to_format(item['_source'], 'json-compact'))
end
table.insert(tbl, '') -- For last \n
- local kibana_url = connect_prefix .. ip_addr ..'/.kibana/_bulk'
+ local kibana_url = connect_prefix .. ip_addr .. '/.kibana/_bulk'
local function kibana_template_callback(err, code, body, _)
if code ~= 200 then
rspamd_logger.errx('cannot put template to %s: %s(%s) (%s)', kibana_url,
if enabled then
-- create ingest pipeline
- local geoip_url = connect_prefix .. ip_addr ..'/_ingest/pipeline/rspamd-geoip'
+ local geoip_url = connect_prefix .. ip_addr .. '/_ingest/pipeline/rspamd-geoip'
local function geoip_cb(err, code, body, _)
if code ~= 200 then
rspamd_logger.errx('cannot get data from %s: %s(%s) (%s)',
timeout = settings.timeout,
})
-- create template mappings if not exist
- local template_url = connect_prefix .. ip_addr ..'/_template/rspamd'
+ local template_url = connect_prefix .. ip_addr .. '/_template/rspamd'
local function http_template_put_callback(err, code, body, _)
if code ~= 200 then
rspamd_logger.errx('cannot put template to %s: %s(%s) (%s)',
redis_params = rspamd_redis.parse_redis_server('elastic')
if redis_params and opts then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
settings[k] = v
end
end
settings.upstream = upstream_list.create(rspamd_config,
- settings['server'] or settings['servers'], 9200)
+ settings['server'] or settings['servers'], 9200)
if not settings.upstream then
rspamd_logger.errx('cannot parse elastic address: %s',
- settings['server'] or settings['servers'])
+ settings['server'] or settings['servers'])
lua_util.disable_module(N, "config")
return
end
elastic_template = read_file(settings['template_file']);
if not elastic_template then
rspamd_logger.infox(rspamd_config, 'elastic unable to read %s, disabling module',
- settings['template_file'])
+ settings['template_file'])
lua_util.disable_module(N, "config")
return
end
type = 'idempotent',
callback = elastic_collect,
flags = 'empty,explicit_disable,ignore_passthrough',
- augmentations = {string.format("timeout=%f", settings.timeout)},
+ augmentations = { string.format("timeout=%f", settings.timeout) },
})
- rspamd_config:add_on_load(function(cfg, ev_base,worker)
+ rspamd_config:add_on_load(function(cfg, ev_base, worker)
if worker:is_scanner() then
check_elastic_server(cfg, ev_base, worker) -- check for elasticsearch requirements
initial_setup(cfg, ev_base, worker) -- import mappings pipeline and visualizations
rules = {},
}
-local config_schema = ts.shape{
+local config_schema = ts.shape {
enabled = ts.boolean:is_optional(),
rules = ts.map_of(
- ts.string, ts.one_of{
- ts.shape{
- priority = ts.number:is_optional(),
- strategy = 'authenticated',
- symbol = ts.string:is_optional(),
- user_map = lua_maps.map_schema:is_optional(),
- },
- ts.shape{
- count = ts.number,
- priority = ts.number:is_optional(),
- strategy = 'count',
- symbol = ts.string:is_optional(),
- },
- ts.shape{
- priority = ts.number:is_optional(),
- strategy = 'local',
- symbol = ts.string:is_optional(),
- },
- ts.shape{
- hostname_map = lua_maps.map_schema,
- priority = ts.number:is_optional(),
- strategy = 'hostname_map',
- symbol = ts.string:is_optional(),
- },
- ts.shape{
- ip_map = lua_maps.map_schema,
- priority = ts.number:is_optional(),
- strategy = 'ip_map',
- symbol = ts.string:is_optional(),
- },
- }
+ ts.string, ts.one_of {
+ ts.shape {
+ priority = ts.number:is_optional(),
+ strategy = 'authenticated',
+ symbol = ts.string:is_optional(),
+ user_map = lua_maps.map_schema:is_optional(),
+ },
+ ts.shape {
+ count = ts.number,
+ priority = ts.number:is_optional(),
+ strategy = 'count',
+ symbol = ts.string:is_optional(),
+ },
+ ts.shape {
+ priority = ts.number:is_optional(),
+ strategy = 'local',
+ symbol = ts.string:is_optional(),
+ },
+ ts.shape {
+ hostname_map = lua_maps.map_schema,
+ priority = ts.number:is_optional(),
+ strategy = 'hostname_map',
+ symbol = ts.string:is_optional(),
+ },
+ ts.shape {
+ ip_map = lua_maps.map_schema,
+ priority = ts.number:is_optional(),
+ strategy = 'ip_map',
+ symbol = ts.string:is_optional(),
+ },
+ }
),
}
if confighelp then
rspamd_config:add_example(nil, 'external_services',
- "Check messages using external services (e.g. OEM AS engines, DCC, Pyzor etc)",
- [[
-external_services {
- # multiple scanners could be checked, for each we create a configuration block with an arbitrary name
-
- oletools {
- # If set force this action if any virus is found (default unset: no action is forced)
- # action = "reject";
- # If set, then rejection message is set to this value (mention single quotes)
- # If `max_size` is set, messages > n bytes in size are not scanned
- # max_size = 20000000;
- # log_clean = true;
- # servers = "127.0.0.1:10050";
- # cache_expire = 86400;
- # scan_mime_parts = true;
- # extended = false;
- # if `patterns` is specified virus name will be matched against provided regexes and the related
- # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
- patterns {
- # symbol_name = "pattern";
- JUST_EICAR = "^Eicar-Test-Signature$";
+ "Check messages using external services (e.g. OEM AS engines, DCC, Pyzor etc)",
+ [[
+ external_services {
+ # multiple scanners could be checked, for each we create a configuration block with an arbitrary name
+
+ oletools {
+ # If set force this action if any virus is found (default unset: no action is forced)
+ # action = "reject";
+ # If set, then rejection message is set to this value (mention single quotes)
+ # If `max_size` is set, messages > n bytes in size are not scanned
+ # max_size = 20000000;
+ # log_clean = true;
+ # servers = "127.0.0.1:10050";
+ # cache_expire = 86400;
+ # scan_mime_parts = true;
+ # extended = false;
+ # if `patterns` is specified virus name will be matched against provided regexes and the related
+ # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+ patterns {
+ # symbol_name = "pattern";
+ JUST_EICAR = "^Eicar-Test-Signature$";
+ }
+ # mime-part regex matching in content-type or filename
+ mime_parts_filter_regex {
+ #GEN1 = "application\/octet-stream";
+ DOC2 = "application\/msword";
+ DOC3 = "application\/vnd\.ms-word.*";
+ XLS = "application\/vnd\.ms-excel.*";
+ PPT = "application\/vnd\.ms-powerpoint.*";
+ GEN2 = "application\/vnd\.openxmlformats-officedocument.*";
+ }
+ # Mime-Part filename extension matching (no regex)
+ mime_parts_filter_ext {
+ doc = "doc";
+ dot = "dot";
+ docx = "docx";
+ dotx = "dotx";
+ docm = "docm";
+ dotm = "dotm";
+ xls = "xls";
+ xlt = "xlt";
+ xla = "xla";
+ xlsx = "xlsx";
+ xltx = "xltx";
+ xlsm = "xlsm";
+ xltm = "xltm";
+ xlam = "xlam";
+ xlsb = "xlsb";
+ ppt = "ppt";
+ pot = "pot";
+ pps = "pps";
+ ppa = "ppa";
+ pptx = "pptx";
+ potx = "potx";
+ ppsx = "ppsx";
+ ppam = "ppam";
+ pptm = "pptm";
+ potm = "potm";
+ ppsm = "ppsm";
+ }
+ # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+ whitelist = "/etc/rspamd/antivirus.wl";
}
- # mime-part regex matching in content-type or filename
- mime_parts_filter_regex {
- #GEN1 = "application\/octet-stream";
- DOC2 = "application\/msword";
- DOC3 = "application\/vnd\.ms-word.*";
- XLS = "application\/vnd\.ms-excel.*";
- PPT = "application\/vnd\.ms-powerpoint.*";
- GEN2 = "application\/vnd\.openxmlformats-officedocument.*";
+ dcc {
+ # If set force this action if any virus is found (default unset: no action is forced)
+ # action = "reject";
+ # If set, then rejection message is set to this value (mention single quotes)
+ # If `max_size` is set, messages > n bytes in size are not scanned
+ max_size = 20000000;
+ #servers = "127.0.0.1:10045;
+ # if `patterns` is specified virus name will be matched against provided regexes and the related
+ # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
+ patterns {
+ # symbol_name = "pattern";
+ JUST_EICAR = "^Eicar-Test-Signature$";
+ }
+ # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
+ whitelist = "/etc/rspamd/antivirus.wl";
}
- # Mime-Part filename extension matching (no regex)
- mime_parts_filter_ext {
- doc = "doc";
- dot = "dot";
- docx = "docx";
- dotx = "dotx";
- docm = "docm";
- dotm = "dotm";
- xls = "xls";
- xlt = "xlt";
- xla = "xla";
- xlsx = "xlsx";
- xltx = "xltx";
- xlsm = "xlsm";
- xltm = "xltm";
- xlam = "xlam";
- xlsb = "xlsb";
- ppt = "ppt";
- pot = "pot";
- pps = "pps";
- ppa = "ppa";
- pptx = "pptx";
- potx = "potx";
- ppsx = "ppsx";
- ppam = "ppam";
- pptm = "pptm";
- potm = "potm";
- ppsm = "ppsm";
- }
- # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
- whitelist = "/etc/rspamd/antivirus.wl";
- }
- dcc {
- # If set force this action if any virus is found (default unset: no action is forced)
- # action = "reject";
- # If set, then rejection message is set to this value (mention single quotes)
- # If `max_size` is set, messages > n bytes in size are not scanned
- max_size = 20000000;
- #servers = "127.0.0.1:10045;
- # if `patterns` is specified virus name will be matched against provided regexes and the related
- # symbol will be yielded if a match is found. If no match is found, default symbol is yielded.
- patterns {
- # symbol_name = "pattern";
- JUST_EICAR = "^Eicar-Test-Signature$";
- }
- # `whitelist` points to a map of IP addresses. Mail from these addresses is not scanned.
- whitelist = "/etc/rspamd/antivirus.wl";
}
-}
-]])
+ ]])
return
end
-
local function add_scanner_rule(sym, opts)
if not opts.type then
rspamd_logger.errx(rspamd_config, 'unknown type for external scanner rule %s', sym)
if not rule then
rspamd_logger.errx(rspamd_config, 'cannot configure %s for %s',
- opts.type, rule.symbol or sym:upper())
+ opts.type, rule.symbol or sym:upper())
return nil
end
local has_valid = false
for k, m in pairs(opts) do
if type(m) == 'table' and m.servers then
- if not m.type then m.type = k end
- if not m.name then m.name = k end
+ if not m.type then
+ m.type = k
+ end
+ if not m.name then
+ m.name = k
+ end
local cb, nrule = add_scanner_rule(k, m)
if not cb then
end
if m.symbols then
local function reg_symbols(tbl)
- for _,sym in pairs(tbl) do
+ for _, sym in pairs(tbl) do
if type(sym) == 'string' then
rspamd_config:register_symbol({
type = 'virtual',
-- Add preloads if a module requires that
if type(m.preloads) == 'table' then
- for _,preload in ipairs(m.preloads) do
+ for _, preload in ipairs(m.preloads) do
rspamd_config:add_on_load(function(cfg, ev_base, worker)
preload(m, cfg, ev_base, worker)
end)
return 0
end
- local e, err = rspamd_expression.create(params.expr, {parse_atom, process_atom}, params.pool)
+ local e, err = rspamd_expression.create(params.expr, { parse_atom, process_atom }, params.pool)
if err then
rspamd_logger.errx(rspamd_config, 'Couldnt create expression [%1]: %2', params.expr, err)
return
if type(params.message) == 'string' then
-- process selector expressions in the message
local message = string.gsub(params.message, '(${(.-)})', process_message_selectors)
- task:set_pre_result{action = params.act, message = message, module = N, flags = flags}
+ task:set_pre_result { action = params.act, message = message, module = N, flags = flags }
else
- task:set_pre_result{action = params.act, module = N, flags = flags}
+ task:set_pre_result { action = params.act, module = N, flags = flags }
end
return true, params.act
end
end
if type(expr) == 'string' then
-- expr, act, pool, message, subject, raction, honor, limit, flags
- local cb, atoms = gen_cb{expr = expr,
- act = action,
- pool = rspamd_config:get_mempool(),
- message = message,
- subject = subject}
+ local cb, atoms = gen_cb { expr = expr,
+ act = action,
+ pool = rspamd_config:get_mempool(),
+ message = message,
+ subject = subject }
if cb and atoms then
local h = rspamd_cryptobox_hash.create()
h:update(expr)
if action and expr then
local flags = {}
- if sett.least then table.insert(flags, "least") end
- if sett.process_all then table.insert(flags, "process_all") end
+ if sett.least then
+ table.insert(flags, "least")
+ end
+ if sett.process_all then
+ table.insert(flags, "process_all")
+ end
local raction = lua_util.list_to_hash(sett.require_action)
local honor = lua_util.list_to_hash(sett.honor_action)
- local cb, atoms = gen_cb{expr = expr,
- act = action,
- pool = rspamd_config:get_mempool(),
- message = sett.message,
- subject = sett.subject,
- raction = raction,
- honor = honor,
- limit = sett.limit,
- flags = table.concat(flags, ',')}
+ local cb, atoms = gen_cb { expr = expr,
+ act = action,
+ pool = rspamd_config:get_mempool(),
+ message = sett.message,
+ subject = sett.subject,
+ raction = raction,
+ honor = honor,
+ limit = sett.limit,
+ flags = table.concat(flags, ',') }
if cb and atoms then
local t = {}
if (raction or honor) then
else
t.type = 'normal'
if not sett.least then
- t.augmentations = {'passthrough', 'important'}
+ t.augmentations = { 'passthrough', 'important' }
end
end
t.name = 'FORCE_ACTION_' .. name
local smtp_rcpts = task:get_recipients(1)
local smtp_from = task:get_from(1)
- if not smtp_rcpts then return end
- if #smtp_rcpts == 0 then return end
+ if not smtp_rcpts then
+ return
+ end
+ if #smtp_rcpts == 0 then
+ return
+ end
- local mime_rcpts = task:get_recipients({ 'mime', 'orig'})
+ local mime_rcpts = task:get_recipients({ 'mime', 'orig' })
if not mime_rcpts then
return
end
end
- for _,mime_rcpt in ipairs(mime_rcpts) do
+ for _, mime_rcpt in ipairs(mime_rcpts) do
if mime_rcpt.addr and mime_rcpt.addr ~= '' then
local addr = string.lower(mime_rcpt.addr)
- local dom = string.lower(mime_rcpt.domain)
+ local dom = string.lower(mime_rcpt.domain)
local matched_smtp_addr = smtp_rcpt_map[addr]
if matched_smtp_addr then
-- Direct match, go forward
local opts = {}
local seen_mime_unmatched = false
local seen_smtp_unmatched = false
- for _,mime_rcpt in ipairs(mime_rcpts) do
+ for _, mime_rcpt in ipairs(mime_rcpts) do
if not mime_rcpt.matched then
seen_mime_unmatched = true
table.insert(opts, 'm:' .. mime_rcpt.addr)
end
end
- for _,smtp_rcpt in ipairs(smtp_rcpts) do
+ for _, smtp_rcpt in ipairs(smtp_rcpts) do
if not smtp_rcpt.matched then
if not smtp_rcpt_domain_map[smtp_rcpt.domain:lower()]._seen_mime_domain then
seen_smtp_unmatched = true
end
-- Configuration
-local opts = rspamd_config:get_all_opt('forged_recipients')
+local opts = rspamd_config:get_all_opt('forged_recipients')
if opts then
if opts['symbol_rcpt'] or opts['symbol_sender'] then
local id = rspamd_config:register_symbol({
end
if opts['symbol_sender'] then
symbol_sender = opts['symbol_sender']
- rspamd_config:register_symbol({
+ rspamd_config:register_symbol({
name = symbol_sender,
type = 'virtual',
parent = id,
rspamd_logger.infox(cfg, 'saved data on %s(%s)', m.server, m.name)
end
end
- rspamd_http.request{
+ rspamd_http.request {
url = string.format('http://%s//update_v1/%s', m.server, m.name),
resolver = cfg:get_resolver(),
config = cfg,
rspamd_logger.errx(cfg, 'cannot load data: %s', err)
else
-- Here, we actually copy body once for each mirror
- fun.each(function(_, v) send_data_mirror(v, cfg, ev_base, body) end,
- settings.mirrors)
+ fun.each(function(_, v)
+ send_data_mirror(v, cfg, ev_base, body)
+ end,
+ settings.mirrors)
end
end
if settings.saved_cookie ~= tostring(body) then
settings.saved_cookie = tostring(body)
rspamd_logger.infox(cfg, 'received collection cookie %s',
- tostring(rspamd_util.encode_base32(settings.saved_cookie:sub(1, 6))))
+ tostring(rspamd_util.encode_base32(settings.saved_cookie:sub(1, 6))))
local sig = rspamd_cryptolib.sign_memory(settings.sign_keypair,
- settings.saved_cookie)
+ settings.saved_cookie)
if not sig then
rspamd_logger.info(cfg, 'cannot sign cookie')
else
- rspamd_http.request{
+ rspamd_http.request {
url = string.format('http://%s/data', settings.collect_server),
resolver = cfg:get_resolver(),
config = cfg,
end
end
rspamd_logger.infox(cfg, 'start fuzzy collection, next sync in %s seconds',
- settings.sync_time)
- rspamd_http.request{
+ settings.sync_time)
+ rspamd_http.request {
url = string.format('http://%s/cookie', settings.collect_server),
resolver = cfg:get_resolver(),
config = cfg,
local opts = rspamd_config:get_all_opt('fuzzy_collect')
if opts and type(opts) == 'table' then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
settings[k] = v
end
local sane_config = true
rspamd_config:add_on_load(function(_, ev_base, worker)
if worker:is_primary_controller() then
rspamd_config:add_periodic(ev_base, 0.0,
- function(cfg, _)
- return collect_fuzzy_hashes(cfg, ev_base)
- end)
+ function(cfg, _)
+ return collect_fuzzy_hashes(cfg, ev_base)
+ end)
end
end)
else
local body = task:get_rawbody()
- if not body then return nil end
+ if not body then
+ return nil
+ end
local len = body:len()
if len > settings['max_data_len'] then
if not t then
rspamd_logger.errx(task, 'not a valid number: %s', tm)
- return false,false
+ return false, false
end
if now - t < settings['timeout'] then
- return true,true
+ return true, true
else
-- We just set variable to pass when in post-filter stage
task:get_mempool():set_variable("grey_whitelisted", type)
- return true,false
+ return true, false
end
end
if settings.message_func then
task:set_pre_result(settings['action'],
- settings.message_func(task, end_time), N)
+ settings.message_func(task, end_time), N)
else
local message = settings['message']
if settings.report_time then
local greylisted_meta = false
if data then
- local end_time_body,end_time_meta
+ local end_time_body, end_time_meta
local now = rspamd_util.get_time()
if data[1] and type(data[1]) ~= 'userdata' then
local tm = tonumber(data[1]) or now
- ret_body,greylisted_body = check_time(task, data[1], 'body', now)
+ ret_body, greylisted_body = check_time(task, data[1], 'body', now)
if greylisted_body then
end_time_body = tm + settings['timeout']
task:get_mempool():set_variable("grey_greylisted_body",
if data[2] and type(data[2]) ~= 'userdata' then
if not ret_body or greylisted_body then
local tm = tonumber(data[2]) or now
- ret_meta,greylisted_meta = check_time(task, data[2], 'meta', now)
+ ret_meta, greylisted_meta = check_time(task, data[2], 'meta', now)
if greylisted_meta then
end_time_meta = tm + settings['timeout']
false, -- is write
redis_get_cb, --callback
'MGET', -- command
- {body_key, meta_key} -- arguments
+ { body_key, meta_key } -- arguments
)
if not ret then
rspamd_logger.errx(task, 'cannot make redis request to check results')
local ip = task:get_ip()
-- Don't do anything if pre-result has been already set
- if task:has_pre_result() then return end
+ if task:has_pre_result() then
+ return
+ end
-- Check whitelist_symbols
- for _,sym in ipairs(settings.whitelist_symbols) do
+ for _, sym in ipairs(settings.whitelist_symbols) do
if task:has_symbol(sym) then
rspamd_logger.infox(task, 'skip greylisting as we have found symbol %s', sym)
if action == 'greylist' then
local function redis_set_cb(err)
if err then
rspamd_logger.errx(task, 'got error %s when setting greylisting record on server %s',
- err, upstream:get_addr())
+ err, upstream:get_addr())
end
end
task:insert_result(settings['symbol'], 0.0, 'pass', is_whitelisted)
rspamd_logger.infox(task, 'greylisting pass (%s) until %s',
- is_whitelisted,
- rspamd_util.time_to_string(rspamd_util.get_time() + settings['expire']))
+ is_whitelisted,
+ rspamd_util.time_to_string(rspamd_util.get_time() + settings['expire']))
- if not settings.check_local and is_rspamc then return end
+ if not settings.check_local and is_rspamc then
+ return
+ end
- ret,conn,upstream = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- hash_key, -- hash key
- true, -- is write
- redis_set_cb, --callback
- 'EXPIRE', -- command
- {body_key, tostring(toint(settings['expire']))} -- arguments
+ ret, conn, upstream = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ hash_key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'EXPIRE', -- command
+ { body_key, tostring(toint(settings['expire'])) } -- arguments
)
-- Update greylisting record expire
if ret then
rspamd_logger.errx(task, 'got error while connecting to redis')
end
elseif do_greylisting or do_greylisting_required then
- if not settings.check_local and is_rspamc then return end
+ if not settings.check_local and is_rspamc then
+ return
+ end
local t = tostring(toint(rspamd_util.get_time()))
local end_time = rspamd_util.time_to_string(t + settings['timeout'])
rspamd_logger.infox(task, 'greylisted until "%s", new record', end_time)
greylist_message(task, end_time, 'new record')
-- Create new record
- ret,conn,upstream = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- hash_key, -- hash key
- true, -- is write
- redis_set_cb, --callback
- 'SETEX', -- command
- {body_key, tostring(toint(settings['expire'])), t} -- arguments
+ ret, conn, upstream = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ hash_key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'SETEX', -- command
+ { body_key, tostring(toint(settings['expire'])), t } -- arguments
)
if ret then
settings.message_func = assert(load(opts['message_func']))()
end
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
if k ~= 'message_func' then
settings[k] = v
end
end
whitelisted_ip = lua_map.rspamd_map_add(N, 'whitelisted_ip', 'radix',
- 'Greylist whitelist ip map')
+ 'Greylist whitelist ip map')
whitelist_domains_map = lua_map.rspamd_map_add(N, 'whitelist_domains_url',
- 'map', 'Greylist whitelist domains map')
+ 'map', 'Greylist whitelist domains map')
redis_params = lua_redis.parse_redis_server(N)
if not redis_params then
type = 'postfilter',
callback = greylist_set,
priority = lua_util.symbols_priorities.medium,
- augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)},
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
})
local id = rspamd_config:register_symbol({
name = 'GREYLIST_CHECK',
type = 'prefilter',
callback = greylist_check,
priority = lua_util.symbols_priorities.medium,
- augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
})
rspamd_config:register_symbol({
name = settings.symbol,
end
local function add_static_map(data)
- return rspamd_config:add_map{
+ return rspamd_config:add_map {
type = 'regexp_multi',
url = {
upstreams = 'static',
local function check_host_cb_mx(_, to_resolve, results, err)
if err and (err ~= 'requested record is not found' and err ~= 'no records with this name') then
- lua_util.debugm(N, task, 'error looking up %s: %s', to_resolve, err)
+ lua_util.debugm(N, task, 'error looking up %s: %s', to_resolve, err)
end
if not results then
task:insert_result('HFILTER_' .. symbol_suffix .. '_NORES_A_OR_MX', 1.0,
- to_resolve)
+ to_resolve)
else
- for _,mx in pairs(results) do
+ for _, mx in pairs(results) do
if mx['name'] then
local failed_mx_address = 0
-- Capture failed_mx_address
if failed_mx_address >= 2 then
task:insert_result('HFILTER_' .. symbol_suffix .. '_NORESOLVE_MX',
- 1.0, mx['name'])
+ 1.0, mx['name'])
end
end
task:get_resolver():resolve('a', {
- task=task,
+ task = task,
name = mx['name'],
callback = check_host_cb_mx_a
})
if not results then
failed_address = failed_address + 1
else
- for _,result in pairs(results) do
+ for _, result in pairs(results) do
table.insert(resolved_address, result:to_string())
end
end
if failed_address >= 2 then
-- No A or AAAA records
if eq_ip and eq_ip ~= '' then
- for _,result in pairs(resolved_address) do
+ for _, result in pairs(resolved_address) do
if result == eq_ip then
return true
end
if check_fqdn(host) then
if eq_host == '' or eq_host ~= host then
task:get_resolver():resolve('a', {
- task=task,
+ task = task,
name = host,
callback = check_host_cb_a
})
if parts then
local plain_text_part, html_text_part
- for _,p in ipairs(parts) do
+ for _, p in ipairs(parts) do
if p:is_html() then
html_text_part = p
else
local rel = url_len / plen
if rel > 0.8 then
local sc = (rel - 0.8) * 5.0
- if sc > 1.0 then sc = 1.0 end
+ if sc > 1.0 then
+ sc = 1.0
+ end
task:insert_result('HFILTER_URL_ONLY', sc, tostring(sc))
local lines = part:get_lines_count()
if lines > 0 and lines < 2 then
if not find_badip and not find_bareip then
-- Regexp check HELO (checks_hello)
local weights = checks_hello_map:get_key(helo)
- for _,weight in ipairs(weights or {}) do
+ for _, weight in ipairs(weights or {}) do
weight = tonumber(weight) or 0
if weight > weight_helo then
weight_helo = weight
end
-- Regexp check HELO (checks_hellohost)
weights = checks_hellohost_map:get_key(helo)
- for _,weight in ipairs(weights or {}) do
+ for _, weight in ipairs(weights or {}) do
weight = tonumber(weight) or 0
if weight > weight_helo then
weight_helo = weight
if hostname then
-- Check regexp HOSTNAME
local weights = checks_hellohost_map:get_key(hostname)
- for _,weight in ipairs(weights or {}) do
+ for _, weight in ipairs(weights or {}) do
weight = tonumber(weight) or 0
if weight > weight_hostname then
weight_hostname = weight
local from = task:get_from(1)
if from then
--FROM host check
- for _,fr in ipairs(from) do
+ for _, fr in ipairs(from) do
local fr_split = rspamd_str_split(fr['addr'], '@')
if #fr_split == 2 then
check_host(task, fr_split[2], 'FROMHOST', '', '')
if frombounce then
if count_rcpt > 1 then
task:insert_result('HFILTER_RCPT_BOUNCEMOREONE', 1.00,
- tostring(count_rcpt))
+ tostring(count_rcpt))
end
end
end
local opts = rspamd_config:get_all_opt('hfilter')
if opts then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
config[k] = v
end
end
local function append_t(t, a)
- for _,v in ipairs(a) do table.insert(t, v) end
+ for _, v in ipairs(a) do
+ table.insert(t, v)
+ end
end
if config['helo_enabled'] then
checks_hello_bareip_map = add_static_map(checks_hello_bareip)
--dumper(symbols_enabled)
if #symbols_enabled > 0 then
- local id = rspamd_config:register_symbol{
+ local id = rspamd_config:register_symbol {
name = 'HFILTER_CHECK',
callback = hfilter_callback,
type = 'callback',
- augmentations = {string.format("timeout=%f", timeout)},
+ augmentations = { string.format("timeout=%f", timeout) },
}
- for _,sym in ipairs(symbols_enabled) do
- rspamd_config:register_symbol{
+ for _, sym in ipairs(symbols_enabled) do
+ rspamd_config:register_symbol {
type = 'virtual',
score = 1.0,
parent = id,
return type(v) == 'table' and v.score
end, metric)
- fun.each(function(k, v) v.name = nil; tbl.symbols[k] = v; end, symbols)
- fun.each(function(k, v) tbl[k] = v end, others)
+ fun.each(function(k, v)
+ v.name = nil;
+ tbl.symbols[k] = v;
+ end, symbols)
+ fun.each(function(k, v)
+ tbl[k] = v
+ end, others)
-- Reset the original metric
tbl.default = nil
return
end
- local data = task:get_protocol_reply{'metrics', 'basic'}
+ local data = task:get_protocol_reply { 'metrics', 'basic' }
local prefix = settings.key_prefix .. hostname
if data then
end
local ret, conn, _ = lua_redis.rspamd_redis_make_request(task,
- redis_params, -- connect params
- nil, -- hash key
- true, -- is write
- redis_llen_cb, --callback
- 'LPUSH', -- command
- {prefix, json} -- arguments
+ redis_params, -- connect params
+ nil, -- hash key
+ true, -- is write
+ redis_llen_cb, --callback
+ 'LPUSH', -- command
+ { prefix, json } -- arguments
)
if ret then
- conn:add_cmd('LTRIM', {prefix, '0', string.format('%d', settings.nrows-1)})
+ conn:add_cmd('LTRIM', { prefix, '0', string.format('%d', settings.nrows - 1) })
if settings.expire and settings.expire > 0 then
- conn:add_cmd('EXPIRE', {prefix, string.format('%d', settings.expire)})
+ conn:add_cmd('EXPIRE', { prefix, string.format('%d', settings.expire) })
end
end
end
local function redis_ltrim_cb(err, _)
if err then
rspamd_logger.errx(task, 'got error %s when resetting history: %s',
- err)
+ err)
conn:send_error(504, '{"error": "' .. err .. '"}')
else
conn:send_string('{"success":true}')
end
end
lua_redis.rspamd_redis_make_request(task,
- redis_params, -- connect params
- nil, -- hash key
- true, -- is write
- redis_ltrim_cb, --callback
- 'LTRIM', -- command
- {prefix, '0', '0'} -- arguments
+ redis_params, -- connect params
+ nil, -- hash key
+ true, -- is write
+ redis_ltrim_cb, --callback
+ 'LTRIM', -- command
+ { prefix, '0', '0' } -- arguments
)
else
local function redis_lrange_cb(err, data)
if settings.compress then
local t1 = rspamd_util:get_ticks()
- data = fun.totable(fun.filter(function(e) return e ~= nil end,
- fun.map(function(e)
- local _,dec = rspamd_util.zstd_decompress(e)
- if dec then
- return dec
- end
- return nil
- end, data)))
+ data = fun.totable(fun.filter(function(e)
+ return e ~= nil
+ end,
+ fun.map(function(e)
+ local _, dec = rspamd_util.zstd_decompress(e)
+ if dec then
+ return dec
+ end
+ return nil
+ end, data)))
lua_util.debugm(N, task, 'decompress took %s ms',
(rspamd_util:get_ticks() - t1) * 1000.0)
collectgarbage()
-- Parse elements using ucl
local t1 = rspamd_util:get_ticks()
data = fun.totable(
- fun.map(function (_, obj) return obj end,
- fun.filter(function(res, obj)
- if res then
- return true
- end
- return false
+ fun.map(function(_, obj)
+ return obj
end,
- fun.map(function(elt)
- local parser = ucl.parser()
- local res,_ = parser:parse_text(elt)
-
- if res then
- return true, parser:get_object()
- else
- return false, nil
- end
- end, data))))
+ fun.filter(function(res, obj)
+ if res then
+ return true
+ end
+ return false
+ end,
+ fun.map(function(elt)
+ local parser = ucl.parser()
+ local res, _ = parser:parse_text(elt)
+
+ if res then
+ return true, parser:get_object()
+ else
+ return false, nil
+ end
+ end, data))))
lua_util.debugm(N, task, 'parse took %s ms',
(rspamd_util:get_ticks() - t1) * 1000.0)
collectgarbage()
collectgarbage()
else
rspamd_logger.errx(task, 'got error %s when getting history: %s',
- err)
+ err)
conn:send_error(504, '{"error": "' .. err .. '"}')
end
end
lua_redis.rspamd_redis_make_request(task,
- redis_params, -- connect params
- nil, -- hash key
- false, -- is write
- redis_lrange_cb, --callback
- 'LRANGE', -- command
- {prefix, string.format('%d', from), string.format('%d', to)}, -- arguments
- {opaque_data = true}
+ redis_params, -- connect params
+ nil, -- hash key
+ false, -- is write
+ redis_lrange_cb, --callback
+ 'LRANGE', -- command
+ { prefix, string.format('%d', from), string.format('%d', to) }, -- arguments
+ { opaque_data = true }
)
end
end
-local opts = rspamd_config:get_all_opt('history_redis')
+local opts = rspamd_config:get_all_opt('history_redis')
if opts then
settings = lua_util.override_defaults(settings, opts)
- local res,err = settings_schema:transform(settings)
+ local res, err = settings_schema:transform(settings)
if not res then
rspamd_logger.warnx(rspamd_config, '%s: plugin is misconfigured: %s', N, err)
type = 'idempotent',
callback = history_save,
flags = 'empty,explicit_disable,ignore_passthrough',
- augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)}
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) }
})
lua_redis.register_prefix(settings.key_prefix .. hostname, N,
"Redis history", {
local opts = rspamd_config:get_all_opt('dmarc')
if opts and opts['symbols'] then
- for k,_ in pairs(dmarc_symbols) do
+ for k, _ in pairs(dmarc_symbols) do
if opts['symbols'][k] then
dmarc_symbols[k] = opts['symbols'][k]
end
opts = rspamd_config:get_all_opt('dkim')
if opts then
- for k,_ in pairs(dkim_symbols) do
+ for k, _ in pairs(dkim_symbols) do
if opts[k] then
dkim_symbols[k] = opts[k]
end
opts = rspamd_config:get_all_opt('spf')
if opts then
- for k,_ in pairs(spf_symbols) do
+ for k, _ in pairs(spf_symbols) do
if opts[k] then
spf_symbols[k] = opts[k]
end
local p_obj = parser:get_object()
local results = p_obj['results']
if not results and p_obj['result'] then
- results = {{result = p_obj['result'], domain = 'unknown'}}
+ results = { { result = p_obj['result'], domain = 'unknown' } }
end
if results then
for _, obj in ipairs(results) do
- local dkim_domain = obj['domain'] or 'unknown'
+ local dkim_domain = obj['domain'] or 'unknown'
if obj['result'] == 'pass' or obj['result'] == 'allow' then
task:insert_result(dkim_symbols['symbol_allow'], 1.0, 'http header')
task:insert_result(dkim_symbols['symbol_trace'], 1.0,
- string.format('%s:%s', dkim_domain, dkim_trace.pass))
+ string.format('%s:%s', dkim_domain, dkim_trace.pass))
elseif obj['result'] == 'fail' or obj['result'] == 'reject' then
task:insert_result(dkim_symbols['symbol_deny'], 1.0, 'http header')
task:insert_result(dkim_symbols['symbol_trace'], 1.0,
- string.format('%s:%s', dkim_domain, dkim_trace.fail))
+ string.format('%s:%s', dkim_domain, dkim_trace.fail))
elseif obj['result'] == 'tempfail' or obj['result'] == 'softfail' then
task:insert_result(dkim_symbols['symbol_tempfail'], 1.0, 'http header')
task:insert_result(dkim_symbols['symbol_trace'], 1.0,
- string.format('%s:%s', dkim_domain, dkim_trace.temperror))
+ string.format('%s:%s', dkim_domain, dkim_trace.temperror))
elseif obj['result'] == 'permfail' then
task:insert_result(dkim_symbols['symbol_permfail'], 1.0, 'http header')
task:insert_result(dkim_symbols['symbol_trace'], 1.0,
- string.format('%s:%s', dkim_domain, dkim_trace.permerror))
+ string.format('%s:%s', dkim_domain, dkim_trace.permerror))
elseif obj['result'] == 'na' then
task:insert_result(dkim_symbols['symbol_na'], 1.0, 'http header')
end
-- not much elase we can check, Subjects can be changed in settings
return true
end
- else -- Mailman 3
+ else
+ -- Mailman 3
-- XXX not Mailman3 admin messages have this headers, but one
-- which don't usually have List-* headers examined below
if task:has_header('List-Administrivia') then
end
-- List-Archive and List-Post are optional, check other headers
- for _, h in ipairs({'List-Help', 'List-Subscribe', 'List-Unsubscribe'}) do
+ for _, h in ipairs({ 'List-Help', 'List-Subscribe', 'List-Unsubscribe' }) do
header = task:get_header(h)
if not (header and header:find('<mailto:', 1, true)) then
return false
end,
}
- for hname,hscore in pairs(common_list_headers) do
+ for hname, hscore in pairs(common_list_headers) do
if task:has_header(hname) then
if type(hscore) == 'number' then
score = score + hscore
elseif check_ml_cgp(task) then
task:insert_result(symbol, 1, 'cgp')
else
- if score > 2 then score = 2 end
+ if score > 2 then
+ score = 2
+ end
task:insert_result(symbol, 0.5 * score, 'generic')
end
end
-- Configuration
-local opts = rspamd_config:get_all_opt('maillist')
+local opts = rspamd_config:get_all_opt('maillist')
if opts then
if opts['symbol'] then
symbol = opts['symbol']
true, -- is write
redis_zrange_cb, --callback
'ZREMRANGEBYRANK', -- command
- {key, '0', tostring(-(settings.count) - 1)} -- arguments
+ { key, '0', tostring(-(settings.count) - 1) } -- arguments
)
end
end
true, -- is write
redis_card_cb, --callback
'ZCARD', -- command
- {key} -- arguments
+ { key } -- arguments
)
if ret and conn then
local stats = map:get_stats(true)
- for k,s in pairs(stats) do
+ for k, s in pairs(stats) do
if s > 0 then
- conn:add_cmd('ZINCRBY', {key, tostring(s), k})
+ conn:add_cmd('ZINCRBY', { key, tostring(s), k })
end
end
end
local opts = rspamd_config:get_all_opt(N)
if opts then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
settings[k] = v
end
end
redis_params = lua_redis.parse_redis_server(N, opts)
-- XXX, this is a poor approach as not all maps are defined here...
local tmaps = rspamd_config:get_maps()
-for _,m in ipairs(tmaps) do
+for _, m in ipairs(tmaps) do
if m:get_uri() ~= 'static' then
lua_redis.register_prefix(settings.prefix .. m:get_uri(), N,
'Maps stats data', {
end
if redis_params then
- rspamd_config:add_on_load(function (_, ev_base, worker)
+ rspamd_config:add_on_load(function(_, ev_base, worker)
local maps = rspamd_config:get_maps()
- for _,m in ipairs(maps) do
+ for _, m in ipairs(maps) do
rspamd_config:add_periodic(ev_base,
settings['interval'],
- function ()
+ function()
process_map(m, ev_base, worker)
return true
end, true)
local fuzzy = task:get_mempool():get_variable("fuzzy_hashes", "fstrings")
if fuzzy and #fuzzy > 0 then
local fz = {}
- for _,h in ipairs(fuzzy) do
+ for _, h in ipairs(fuzzy) do
table.insert(fz, h)
end
if not flatten then
meta.mail_to = table.concat(display_emails, ', ')
meta.our_message_id = rspamd_util.random_hex(12) .. '@rspamd'
meta.date = rspamd_util.time_to_string(rspamd_util.get_time())
- return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets}
+ return lua_util.template(rule.email_template or settings.email_template, meta), { mail_targets = mail_targets }
end,
json = function(task)
return ucl.to_format(get_general_metadata(task), 'json-compact')
local pushers = {
redis_pubsub = function(task, formatted, rule)
- local _,ret,upstream
+ local _, ret, upstream
local function redis_pub_cb(err)
if err then
rspamd_logger.errx(task, 'got error %s when publishing on server %s',
end
return true
end
- ret,_,upstream = rspamd_redis_make_request(task,
- redis_params, -- connect params
- nil, -- hash key
- true, -- is write
- redis_pub_cb, --callback
- 'PUBLISH', -- command
- {rule.channel, formatted} -- arguments
+ ret, _, upstream = rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ nil, -- hash key
+ true, -- is write
+ redis_pub_cb, --callback
+ 'PUBLISH', -- command
+ { rule.channel, formatted } -- arguments
)
if not ret then
rspamd_logger.errx(task, 'error connecting to redis')
end,
http = function(task, formatted, rule)
local function http_callback(err, code)
- local valid_status = {200, 201, 202, 204}
+ local valid_status = { 200, 201, 202, 204 }
if err then
rspamd_logger.errx(task, 'got error %s in http callback', err)
end
end
rspamd_http.request({
- task=task,
- url=rule.url,
- user=rule.user,
- password=rule.password,
- body=formatted,
- callback=http_callback,
- mime_type=rule.mime_type or settings.mime_type,
- headers=hdrs,
+ task = task,
+ url = rule.url,
+ user = rule.user,
+ password = rule.password,
+ body = formatted,
+ callback = http_callback,
+ mime_type = rule.mime_type or settings.mime_type,
+ headers = hdrs,
})
end,
send_mail = function(task, formatted, rule, extra)
}
local opts = rspamd_config:get_all_opt(N)
-if not opts then return end
+if not opts then
+ return
+end
local process_settings = {
select = function(val)
selectors.custom = assert(load(val))()
end
local function gen_exporter(rule)
- return function (task)
- if task:has_flag('skip') then return end
+ return function(task)
+ if task:has_flag('skip') then
+ return
+ end
local selector = rule.selector or 'default'
local selected = selectors[selector](task)
if selected then
type = 'idempotent',
callback = gen_exporter(r),
flags = 'empty,explicit_disable,ignore_passthrough',
- augmentations = {string.format("timeout=%f", r.timeout or 0.0)}
+ augmentations = { string.format("timeout=%f", r.timeout or 0.0) }
})
end
data = {
metrics_str, '\n',
},
- callback = (function (err)
+ callback = (function(err)
if err then
logger.errx('Push failed: %1', err)
return
return
end
-rspamd_config:add_on_load(function (_, ev_base, worker)
+rspamd_config:add_on_load(function(_, ev_base, worker)
-- Exit unless we're the first 'controller' worker
- if not worker:is_primary_controller() then return end
+ if not worker:is_primary_controller() then
+ return
+ end
-- Persist mempool variable to statefile on shutdown
pool = mempool.create()
- rspamd_config:register_finish_script(function ()
+ rspamd_config:register_finish_script(function()
local stamp = pool:get_variable(VAR_NAME, 'double')
if not stamp then
logger.warn('No last metric exporter push to persist to disk')
end
-- Push metrics at regular intervals
local function schedule_regular_push()
- rspamd_config:add_periodic(ev_base, settings['interval'], function ()
+ rspamd_config:add_periodic(ev_base, settings['interval'], function()
push_metrics()
return true
end)
end
-- Push metrics to backend and reschedule check
local function schedule_intermediate_push(when)
- rspamd_config:add_periodic(ev_base, when, function ()
+ rspamd_config:add_periodic(ev_base, when, function()
push_metrics()
schedule_regular_push()
return false
local header = task:get_header('Message-Id')
local das = task:get_symbol(settings['symbol_dkim_allow'])
if ((das or E)[1] or E).options then
- for _,dkim_domain in ipairs(das[1]['options']) do
+ for _, dkim_domain in ipairs(das[1]['options']) do
if dkim_domain then
local v = map:get_key(dkim_domain:match "[^:]+")
if v then
end
end
-local opts = rspamd_config:get_all_opt('mid')
+local opts = rspamd_config:get_all_opt('mid')
if opts then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
settings[k] = v
end
return
end
- map = rspamd_config:add_map{
+ map = rspamd_config:add_map {
url = opts.source,
description = "Message-IDs map",
type = 'map'
local settings_override = false
local function skip_wanted(hdr)
- if settings_override then return true end
+ if settings_override then
+ return true
+ end
-- Normal checks
local function match_extended_headers_rcpt()
local rcpts = task:get_recipients('smtp')
- if not rcpts then return false end
+ if not rcpts then
+ return false
+ end
local found
for _, r in ipairs(rcpts) do
found = false
found = true
end
end
- if found then break end
+ if found then
+ break
+ end
end
return found
end
if settings.skip_local and not settings.local_headers[hdr] then
local ip = task:get_ip()
- if (ip and ip:is_local()) then return true end
+ if (ip and ip:is_local()) then
+ return true
+ end
end
if settings.skip_authenticated and not settings.authenticated_headers[hdr] then
- if task:get_user() ~= nil then return true end
+ if task:get_user() ~= nil then
+ return true
+ end
end
if settings.skip_all then
routines['x-spamd-result'] = function()
local local_mod = settings.routines['x-spamd-result']
- if skip_wanted('x-spamd-result') then return end
+ if skip_wanted('x-spamd-result') then
+ return
+ end
if not common.symbols then
common.symbols = task:get_symbols_all()
end
local buf = {}
local verdict = string.format('default: %s [%.2f / %.2f]',
- --TODO: (common.metric_action == 'no action') and 'False' or 'True',
+ --TODO: (common.metric_action == 'no action') and 'False' or 'True',
(common.metric_action == 'reject') and 'True' or 'False',
common.metric_score[1], common.metric_score[2])
table.insert(buf, verdict)
for _, s in ipairs(common.symbols) do
local sym_str = string.format('%s(%.2f)[%s]',
- s.name, s.score, table.concat(s.options or {}, ','))
+ s.name, s.score, table.concat(s.options or {}, ','))
table.insert(buf, sym_str)
end
add_header('x-spamd-result', table.concat(buf, '; '), ';')
- local has_pr,action,message,module = task:has_pre_result()
+ local has_pr, action, message, module = task:has_pre_result()
if has_pr then
local pr_header = {}
end
routines['x-rspamd-queue-id'] = function()
- if skip_wanted('x-rspamd-queue-id') then return end
+ if skip_wanted('x-rspamd-queue-id') then
+ return
+ end
if common.queue_id ~= false then
common.queue_id = task:get_queue_id()
if not common.queue_id then
end
routines['remove-header'] = function()
- if skip_wanted('remove-header') then return end
+ if skip_wanted('remove-header') then
+ return
+ end
if settings.routines['remove-header'].header and settings.routines['remove-header'].remove then
remove[settings.routines['remove-header'].header] = settings.routines['remove-header'].remove
end
end
routines['remove-headers'] = function()
- if skip_wanted('remove-headers') then return end
+ if skip_wanted('remove-headers') then
+ return
+ end
for h, r in pairs(settings.routines['remove-headers'].headers) do
remove[h] = r
end
end
routines['add-headers'] = function()
- if skip_wanted('add-headers') then return end
+ if skip_wanted('add-headers') then
+ return
+ end
for h, r in pairs(settings.routines['add-headers'].headers) do
add[h] = r
remove[h] = settings.routines['add-headers'].remove
routines['x-rspamd-server'] = function()
local local_mod = settings.routines['x-rspamd-server']
- if skip_wanted('x-rspamd-server') then return end
+ if skip_wanted('x-rspamd-server') then
+ return
+ end
if local_mod.remove then
remove[local_mod.header] = local_mod.remove
end
routines['x-spamd-bar'] = function()
local local_mod = settings.routines['x-spamd-bar']
- if skip_wanted('x-rspamd-bar') then return end
+ if skip_wanted('x-rspamd-bar') then
+ return
+ end
if not common['metric_score'] then
common['metric_score'] = task:get_metric_score()
end
routines['x-spam-level'] = function()
local local_mod = settings.routines['x-spam-level']
- if skip_wanted('x-spam-level') then return end
+ if skip_wanted('x-spam-level') then
+ return
+ end
if not common['metric_score'] then
common['metric_score'] = task:get_metric_score()
end
routines['x-rspamd-action'] = function()
local local_mod = settings.routines['x-rspamd-action']
- if skip_wanted('x-rspamd-action') then return end
+ if skip_wanted('x-rspamd-action') then
+ return
+ end
if not common['metric_action'] then
common['metric_action'] = task:get_metric_action()
end
end
local function spam_header (class, name, value, remove_v)
- if skip_wanted(class) then return end
+ if skip_wanted(class) then
+ return
+ end
if not common['metric_action'] then
common['metric_action'] = task:get_metric_action()
end
routines['x-virus'] = function()
local local_mod = settings.routines['x-virus']
- if skip_wanted('x-virus') then return end
+ if skip_wanted('x-virus') then
+ return
+ end
if not common.symbols_hash then
if not common.symbols then
common.symbols = task:get_symbols_all()
end
routines['x-os-fingerprint'] = function()
- if skip_wanted('x-os-fingerprint') then return end
+ if skip_wanted('x-os-fingerprint') then
+ return
+ end
local local_mod = settings.routines['x-os-fingerprint']
- local os_string, link_type, uptime_min, distance =
- task:get_mempool():get_variable('os_fingerprint',
+ local os_string, link_type, uptime_min, distance = task:get_mempool():get_variable('os_fingerprint',
'string, string, double, double');
- if not os_string then return end
+ if not os_string then
+ return
+ end
local value = string.format('%s, (up: %i min), (distance %i, link: %s)',
- os_string, uptime_min, distance, link_type)
+ os_string, uptime_min, distance, link_type)
if local_mod.remove then
remove[local_mod.header] = local_mod.remove
end
routines['x-spam-status'] = function()
- if skip_wanted('x-spam-status') then return end
+ if skip_wanted('x-spam-status') then
+ return
+ end
if not common['metric_score'] then
common['metric_score'] = task:get_metric_score()
end
end
routines['authentication-results'] = function()
- if skip_wanted('authentication-results') then return end
+ if skip_wanted('authentication-results') then
+ return
+ end
local ar = require "lua_auth_results"
if settings.routines['authentication-results'].remove then
- remove[settings.routines['authentication-results'].header] =
- settings.routines['authentication-results'].remove
+ remove[settings.routines['authentication-results'].header] = settings.routines['authentication-results'].remove
end
local res = ar.gen_auth_results(task,
end
routines['stat-signature'] = function()
- if skip_wanted('stat-signature') then return end
+ if skip_wanted('stat-signature') then
+ return
+ end
if settings.routines['stat-signature'].remove then
- remove[settings.routines['stat-signature'].header] =
- settings.routines['stat-signature'].remove
+ remove[settings.routines['stat-signature'].header] = settings.routines['stat-signature'].remove
end
local res = task:get_mempool():get_variable("stat_signature")
if res then
local res = task:get_mempool():get_variable("fuzzy_hashes", "fstrings")
if res and #res > 0 then
- for _,h in ipairs(res) do
+ for _, h in ipairs(res) do
add_header('fuzzy-hashes', h)
end
end
end
end
- if not next(add) then add = nil end
- if not next(remove) then remove = nil end
+ if not next(add) then
+ add = nil
+ end
+ if not next(remove) then
+ remove = nil
+ end
if add or remove then
lua_mime.modify_headers(task, {
end
local config_schema = ts.shape({
- use = ts.array_of(ts.string) + ts.string / function(s) return {s} end,
+ use = ts.array_of(ts.string) + ts.string / function(s)
+ return { s }
+ end,
remove_upstream_spam_flag = ts.boolean:is_optional(),
extended_spam_headers = ts.boolean:is_optional(),
skip_local = ts.boolean:is_optional(),
})
local opts = rspamd_config:get_all_opt(N) or
- rspamd_config:get_all_opt('rmilter_headers')
+ rspamd_config:get_all_opt('rmilter_headers')
-if not opts then return end
+if not opts then
+ return
+end
-- Process config
do
- local res,err = config_schema:transform(opts)
+ local res, err = config_schema:transform(opts)
if not res then
logger.errx(rspamd_config, 'invalid config for %s: %s', N, err)
return
local ext = {}
for n = 1, 2 do
- ext[n] = #parts > n and string.lower(parts[#parts + 1 - n]) or nil
+ ext[n] = #parts > n and string.lower(parts[#parts + 1 - n]) or nil
end
- return ext[1],ext[2],parts
+ return ext[1], ext[2], parts
end
local function check_filename(fname, ct, is_archive, part, detected_ext, nfiles)
-- Decode hex encoded characters
fname = string.gsub(fname, '%%(%x%x)',
- function (hex) return string.char(tonumber(hex,16)) end )
+ function(hex)
+ return string.char(tonumber(hex, 16))
+ end)
-- Replace potentially bad characters with '?'
fname = fname:gsub('[^%s%g]', '?')
return
end
- local ext,ext2,parts = gen_extension(fname)
+ local ext, ext2, parts = gen_extension(fname)
-- ext is the last extension, LOWERCASED
-- ext2 is the one before last extension LOWERCASED
false, part, nil, 1)
end
- if not ext then return end
+ if not ext then
+ return
+ end
local function check_extension(badness_mult, badness_mult2)
- if not badness_mult and not badness_mult2 then return end
+ if not badness_mult and not badness_mult2 then
+ return
+ end
if #parts > 2 then
-- We need to ensure that next-to-last extension is an extension,
-- so we check for its length and if it is not a number or date
if user_settings.bad_extensions[1] then
-- Convert to a key-value map
extra_table = fun.tomap(
- fun.map(function(e) return e,1.0 end,
+ fun.map(function(e)
+ return e, 1.0
+ end,
user_settings.bad_extensions))
else
extra_table = user_settings.bad_extensions
if user_settings.bad_archive_extensions[1] then
-- Convert to a key-value map
extra_archive_table = fun.tomap(fun.map(
- function(e) return e,1.0 end,
+ function(e)
+ return e, 1.0
+ end,
user_settings.bad_archive_extensions))
else
extra_archive_table = user_settings.bad_archive_extensions
if mt and ct and ct ~= 'application/octet-stream' then
local found
local mult
- for _,v in ipairs(mt) do
+ for _, v in ipairs(mt) do
mult = v.mult
if ct == v.ct then
found = true
local parts = task:get_parts()
if parts then
- for _,p in ipairs(parts) do
- local mtype,subtype = p:get_type()
+ for _, p in ipairs(parts) do
+ local mtype, subtype = p:get_type()
if not mtype then
lua_util.debugm(N, task, "no content type for part: %s", p:get_id())
local nfiles = #fl
- for _,f in ipairs(fl) do
+ for _, f in ipairs(fl) do
if f['encrypted'] then
task:insert_result(settings['symbol_encrypted_archive'],
1.0, f['name'])
if nfiles == 1 and fl[1].name then
-- We check that extension of the file inside archive is
-- the same as double extension of the file
- local _,ext2 = gen_extension(filename)
+ local _, ext2 = gen_extension(filename)
if ext2 and #ext2 > 0 then
local enc_ext = gen_extension(fl[1].name)
if detected_type and detected_type.ct ~= ct then
local v_detected = map:get_key(detected_type.ct)
- if not v or v_detected and v_detected > v then v = v_detected end
+ if not v or v_detected and v_detected > v then
+ v = v_detected
+ end
detected_different = true
end
if v then
end
end
-local opts = rspamd_config:get_all_opt('mime_types')
+local opts = rspamd_config:get_all_opt('mime_types')
if opts then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
settings[k] = v
end
settings.filename_whitelist = lua_maps.rspamd_map_add('mime_types', 'filename_whitelist', 'regexp',
- 'filename whitelist')
+ 'filename whitelist')
local function change_extension_map_entry(ext, ct, mult)
if type(ct) == 'table' then
local tbl = {}
- for _,elt in ipairs(ct) do
+ for _, elt in ipairs(ct) do
table.insert(tbl, {
ct = elt,
mult = mult,
end
-- Transform extension_map
- for ext,ct in pairs(settings.extension_map) do
+ for ext, ct in pairs(settings.extension_map) do
change_extension_map_entry(ext, ct, 1.0)
end
-- Add all extensions
- for _,pair in ipairs(lua_mime_types.full_extensions_map) do
+ for _, pair in ipairs(lua_mime_types.full_extensions_map) do
local ext, ct = pair[1], pair[2]
if not settings.extension_map[ext] then
- change_extension_map_entry(ext, ct, settings.other_extensions_mult)
+ change_extension_map_entry(ext, ct, settings.other_extensions_mult)
end
end
local map_type = 'map'
- if settings['regexp'] then map_type = 'regexp' end
+ if settings['regexp'] then
+ map_type = 'regexp'
+ end
map = lua_maps.rspamd_map_add('mime_types', 'file', map_type,
- 'mime types map')
+ 'mime types map')
if map then
local id = rspamd_config:register_symbol({
name = 'MIME_TYPES_CALLBACK',
local number = {}
local digit = lpeg.R("09")
- number.integer =
- (lpeg.S("+-") ^ -1) *
- (digit ^ 1)
+ number.integer = (lpeg.S("+-") ^ -1) *
+ (digit ^ 1)
-- Matches: .6, .899, .9999873
- number.fractional =
- (lpeg.P(".") ) *
+ number.fractional = (lpeg.P(".")) *
(digit ^ 1)
-- Matches: 55.97, -90.8, .9
- number.decimal =
- (number.integer * -- Integer
- (number.fractional ^ -1)) + -- Fractional
+ number.decimal = (number.integer * -- Integer
+ (number.fractional ^ -1)) + -- Fractional
(lpeg.S("+-") * number.fractional) -- Completely fractional number
local sym_start = lpeg.R("az", "AZ") + lpeg.S("_")
local sym_elt = sym_start + lpeg.R("09")
- local symbol = sym_start * sym_elt ^0
+ local symbol = sym_start * sym_elt ^ 0
local symbol_cap = lpeg.Cg(symbol, 'symbol')
local score_cap = lpeg.Cg(number.decimal, 'score')
- local opts_cap = lpeg.Cg(lpeg.Ct(lpeg.C(symbol) * (lpeg.P(",") * lpeg.C(symbol))^0), 'opts')
+ local opts_cap = lpeg.Cg(lpeg.Ct(lpeg.C(symbol) * (lpeg.P(",") * lpeg.C(symbol)) ^ 0), 'opts')
local symscore_cap = (symbol_cap * lpeg.P(":") * score_cap)
local symscoreopt_cap = symscore_cap * lpeg.P(":") * opts_cap
local grammar = symscoreopt_cap + symscore_cap + symbol_cap + score_cap
opts = tbl.opts
end
- return true,sym,score,opts
+ return true, sym, score, opts
else
if p_ret ~= '' then
rspamd_logger.infox(rspamd_config, '%s: cannot parse string "%s"',
parse_rule.symbol, p_ret)
end
- return true,nil,1.0,{}
+ return true, nil, 1.0, {}
end
elseif type(p_ret) == 'boolean' then
- return p_ret,nil,1.0,{}
+ return p_ret, nil, 1.0, {}
end
- return false,nil,0.0,{}
+ return false, nil, 0.0, {}
end
-
local value_types = {
ip = {
- get_value = function(ip) return ip:to_string() end,
+ get_value = function(ip)
+ return ip:to_string()
+ end,
},
from = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
helo = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
header = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
rcpt = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
user = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
url = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
dnsbl = {
- get_value = function(ip) return ip:to_string() end,
+ get_value = function(ip)
+ return ip:to_string()
+ end,
},
filename = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
content = {
- get_value = function() return nil end,
+ get_value = function()
+ return nil
+ end,
},
hostname = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
asn = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
country = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
received = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
mempool = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
selector = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
symbol_options = {
- get_value = function(val) return val end,
+ get_value = function(val)
+ return val
+ end,
},
}
return nil
elseif string.find(filter, 'tld:regexp:') then
if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
+ local type, pat = string.match(filter, '(regexp:)(.+)')
if type and pat then
r['re_filter'] = rspamd_regexp.create_cached(pat)
end
end
elseif string.find(filter, 'full:regexp:') then
if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
+ local type, pat = string.match(filter, '(regexp:)(.+)')
if type and pat then
r['re_filter'] = rspamd_regexp.create_cached(pat)
end
end
elseif string.find(filter, 'regexp:') then
if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
+ local type, pat = string.match(filter, '(regexp:)(.+)')
if type and pat then
r['re_filter'] = rspamd_regexp.create_cached(pat)
end
if filter == 'email:addr' or filter == 'email' then
local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
if addr and addr[1] then
- return fun.totable(fun.map(function(a) return a.addr end, addr))
+ return fun.totable(fun.map(function(a)
+ return a.addr
+ end, addr))
end
elseif filter == 'email:user' then
local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
if addr and addr[1] then
- return fun.totable(fun.map(function(a) return a.user end, addr))
+ return fun.totable(fun.map(function(a)
+ return a.user
+ end, addr))
end
elseif filter == 'email:domain' then
local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
if addr and addr[1] then
- return fun.totable(fun.map(function(a) return a.domain end, addr))
+ return fun.totable(fun.map(function(a)
+ return a.domain
+ end, addr))
end
elseif filter == 'email:domain:tld' then
local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
if addr and addr[1] then
- return fun.totable(fun.map(function(a) return rspamd_util.get_tld(a.domain) end, addr))
+ return fun.totable(fun.map(function(a)
+ return rspamd_util.get_tld(a.domain)
+ end, addr))
end
elseif filter == 'email:name' then
local addr = rspamd_util.parse_mail_address(input, task:get_mempool(), 1024)
if addr and addr[1] then
- return fun.totable(fun.map(function(a) return a.name end, addr))
+ return fun.totable(fun.map(function(a)
+ return a.name
+ end, addr))
end
elseif filter == 'ip_addr' then
local ip_addr = rspamd_ip.from_string(input)
else
-- regexp case
if not rule['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
+ local type, pat = string.match(filter, '(regexp:)(.+)')
if type and pat then
rule['re_filter'] = rspamd_regexp.create_cached(pat)
end
return string.match(fn, '%.([^.]+)$')
elseif string.find(filter, 'regexp:') then
if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
+ local type, pat = string.match(filter, '(regexp:)(.+)')
if type and pat then
r['re_filter'] = rspamd_regexp.create_cached(pat)
end
local function apply_regexp_filter(task, filter, fn, r)
if string.find(filter, 'regexp:') then
if not r['re_filter'] then
- local type,pat = string.match(filter, '(regexp:)(.+)')
+ local type, pat = string.match(filter, '(regexp:)(.+)')
if type and pat then
r['re_filter'] = rspamd_regexp.create_cached(pat)
end
local function apply_content_filter(task, filter)
if filter == 'body' then
- return {task:get_rawbody()}
+ return { task:get_rawbody() }
elseif filter == 'full' then
- return {task:get_content()}
+ return { task:get_content() }
elseif filter == 'headers' then
- return {task:get_raw_headers()}
+ return { task:get_raw_headers() }
elseif filter == 'text' then
local ret = {}
- for _,p in ipairs(task:get_text_parts()) do
+ for _, p in ipairs(task:get_text_parts()) do
table.insert(ret, p:get_content())
end
return ret
elseif filter == 'rawtext' then
local ret = {}
- for _,p in ipairs(task:get_text_parts()) do
+ for _, p in ipairs(task:get_text_parts()) do
table.insert(ret, p:get_raw_content())
end
return ret
elseif filter == 'oneline' then
local ret = {}
- for _,p in ipairs(task:get_text_parts()) do
+ for _, p in ipairs(task:get_text_parts()) do
table.insert(ret, p:get_content_oneline())
end
return ret
cmd = 'HMGET'
end
- local srch = {key}
+ local srch = { key }
-- Insert all ips for some mask :(
if type(value) == 'userdata' and value.class == 'rspamd{ip}' then
maxbits = 32
minbits = 8
end
- for i=maxbits,minbits,-1 do
+ for i = maxbits, minbits, -1 do
local nip = value:apply_mask(i):tostring() .. "/" .. i
srch[#srch + 1] = nip
end
if ret then
if type(err_or_data) == 'table' then
- for _,elt in ipairs(err_or_data) do
+ for _, elt in ipairs(err_or_data) do
callback(elt)
end
else
end
elseif err_code ~= 404 then
rspamd_logger.infox(task, "map %s: get key returned error %s: %s",
- rule.symbol, err_code, err_or_data)
+ rule.symbol, err_code, err_or_data)
end
end
-- Here we need to spill this function into multiple queries
if type(results) == 'table' then
- for _,res in ipairs(results) do
+ for _, res in ipairs(results) do
ret = multimap_query_redis(res, task, value, callback)
if not ret then
end
local function insert_results(result, opt)
- local _,symbol,score,opts = parse_multimap_value(rule, result)
+ local _, symbol, score, opts = parse_multimap_value(rule, result)
local forced = false
if symbol then
if rule.symbols_set then
if type(opt) == 'table' then
table.insert(opt, result)
elseif type(opt) ~= nil then
- opt = {opt,result}
+ opt = { opt, result }
else
- opt = {result}
+ opt = { result }
end
else
forced = not rule.dynamic_symbols
local function rule_callback(result)
if result then
if type(result) == 'table' then
- for _,rs in ipairs(result) do
+ for _, rs in ipairs(result) do
if type(rs) ~= 'userdata' then
rule_callback(rs)
end
end
if type(value) == 'table' then
- fun.each(function(elt) match_element(r, elt, rule_callback) end, value)
+ fun.each(function(elt)
+ match_element(r, elt, rule_callback)
+ end, value)
else
match_element(r, value, rule_callback)
end
end
end, ls)
else
- fun.each(function(e) match_rule(r, e) end, ls)
+ fun.each(function(e)
+ match_rule(r, e)
+ end, ls)
end
end
end
local function match_addr(r, addr)
- match_list(r, addr, {'addr'})
+ match_list(r, addr, { 'addr' })
if not r.filter then
- match_list(r, addr, {'domain'})
- match_list(r, addr, {'user'})
+ match_list(r, addr, { 'domain' })
+ match_list(r, addr, { 'user' })
end
end
return
end
else
- if pos <= (total - (min_pos*-1)) then
+ if pos <= (total - (min_pos * -1)) then
return
end
end
end
if max_pos then
if max_pos < -1 then
- if (total - (max_pos*-1)) >= pos then
+ if (total - (max_pos * -1)) >= pos then
return
end
elseif max_pos > 0 then
local got_flags = h['flags']
if match_flags then
for _, flag in ipairs(match_flags) do
- if not got_flags[flag] then return end
+ if not got_flags[flag] then
+ return
+ end
end
end
if nmatch_flags then
for _, flag in ipairs(nmatch_flags) do
- if got_flags[flag] then return end
+ if got_flags[flag] then
+ return
+ end
end
end
end
if r['filter'] then
data = apply_content_filter(task, r['filter'], r)
else
- data = {task:get_content()}
+ data = { task:get_content() }
end
- for _,v in ipairs(data) do
+ for _, v in ipairs(data) do
match_rule(r, v)
end
end
if rule.expression and not rule.combined then
- local res,trace = rule['expression']:process_traced(task)
+ local res, trace = rule['expression']:process_traced(task)
if not res or res == 0 then
lua_util.debugm(N, task, 'condition is false for %s',
end
task:get_resolver():resolve_a({
- task= task,
+ task = task,
name = to_resolve,
callback = dns_cb,
forced = true
end,
header = function()
if type(rule['header']) == 'table' then
- for _,rh in ipairs(rule['header']) do
+ for _, rh in ipairs(rule['header']) do
local hv = task:get_header_full(rh)
- match_list(rule, hv, {'decoded'})
+ match_list(rule, hv, { 'decoded' })
end
else
local hv = task:get_header_full(rule['header'])
- match_list(rule, hv, {'decoded'})
+ match_list(rule, hv, { 'decoded' })
end
end,
rcpt = function()
if task:has_urls() then
local msg_urls = task:get_urls()
- for _,url in ipairs(msg_urls) do
+ for _, url in ipairs(msg_urls) do
match_url(rule, url)
end
end
return p:is_archive() and det_type == 'archive' and not rule.skip_archives
end
- for _,p in fun.iter(fun.filter(filter_parts, parts)) do
+ for _, p in fun.iter(fun.filter(filter_parts, parts)) do
if filter_archive(p) then
local fnames = p:get_archive():get_files(1000)
- for _,fn in ipairs(fnames) do
+ for _, fn in ipairs(fnames) do
match_filename(rule, fn)
end
end
if elts then
if type(elts) == 'table' then
- for _,elt in ipairs(elts) do
+ for _, elt in ipairs(elts) do
match_rule(rule, elt)
end
else
end
end,
combined = function()
- local ret,trace = rule.combined:process(task)
+ local ret, trace = rule.combined:process(task)
if ret and ret ~= 0 then
- for n,t in pairs(trace) do
+ for n, t in pairs(trace) do
insert_results(t.value, string.format("%s=%s",
n, t.matched))
end
end
end
-
local function gen_multimap_callback(rule)
return function(task)
multimap_callback(task, rule)
lua_util.debugm(N, rspamd_config, "loaded map object for rule %s", rule['symbol'])
local known_symbols = {}
rule.map_obj:foreach(function(key, value)
- local r,symbol,score,_ = parse_multimap_value(rule, value)
+ local r, symbol, score, _ = parse_multimap_value(rule, value)
if r and symbol and not known_symbols[symbol] then
lua_util.debugm(N, rspamd_config, "%s: adding new symbol %s (score = %s), triggered by %s",
rule.symbol, symbol, score, key)
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
name = value,
parent = rule.callback_id,
type = 'virtual',
if newrule['url'] and not newrule['map'] then
newrule['map'] = newrule['url']
end
- if not (newrule.map or newrule.rules) then
+ if not (newrule.map or newrule.rules) then
rspamd_logger.errx(rspamd_config, 'incomplete rule, missing map')
return nil
end
if type(newrule['flags']) == 'table' and newrule['flags'][1] then
newrule['flags'] = newrule['flags']
elseif type(newrule['flags']) == 'string' then
- newrule['flags'] = {newrule['flags']}
+ newrule['flags'] = { newrule['flags'] }
end
if type(newrule['nflags']) == 'table' and newrule['nflags'][1] then
newrule['nflags'] = newrule['nflags']
elseif type(newrule['nflags']) == 'string' then
- newrule['nflags'] = {newrule['nflags']}
+ newrule['nflags'] = { newrule['nflags'] }
end
local filter = newrule['filter'] or 'real_ip'
if filter == 'real_ip' or filter == 'from_ip' then
end
local expression = rspamd_expression.create(newrule['require_symbols'],
- {parse_atom, process_atom}, rspamd_config:get_mempool())
+ { parse_atom, process_atom }, rspamd_config:get_mempool())
if expression then
newrule['expression'] = expression
local opts = rspamd_config:get_all_opt(N)
if opts and type(opts) == 'table' then
redis_params = rspamd_parse_redis_server(N)
- for k,m in pairs(opts) do
+ for k, m in pairs(opts) do
if type(m) == 'table' and m['type'] then
local rule = add_multimap_rule(k, m)
if not rule then
- rspamd_logger.errx(rspamd_config, 'cannot add rule: "'..k..'"')
+ rspamd_logger.errx(rspamd_config, 'cannot add rule: "' .. k .. '"')
else
rspamd_logger.infox(rspamd_config, 'added multimap rule: %s (%s)',
k, rule.type)
local valid = false
local function check_results(mxes)
- if fun.all(function(_, elt) return elt.checked end, mxes) then
+ if fun.all(function(_, elt)
+ return elt.checked
+ end, mxes) then
-- Save cache
local key = settings.key_prefix .. mx_domain
local function redis_cache_cb(err)
task:insert_result(settings.symbol_bad_mx, 1.0)
end
local ret = rspamd_redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- true, -- is write
- redis_cache_cb, --callback
- 'SETEX', -- command
- {key, tostring(settings.expire_novalid), '0'} -- arguments
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_cache_cb, --callback
+ 'SETEX', -- command
+ { key, tostring(settings.expire_novalid), '0' } -- arguments
)
lua_util.debugm(N, task, "set redis cache key: %s; invalid MX", key)
if not ret then
local valid_mx = {}
fun.each(function(k)
table.insert(valid_mx, k)
- end, fun.filter(function (_, elt) return elt.working end, mxes))
+ end, fun.filter(function(_, elt)
+ return elt.working
+ end, mxes))
task:insert_result(settings.symbol_good_mx, 1.0, valid_mx)
- local value = table.concat(valid_mx, ';')
+ local value = table.concat(valid_mx, ';')
if mxes[mx_domain] and type(mxes[mx_domain]) == 'table' and mxes[mx_domain].mx_missing then
value = mx_miss_cache_prefix .. value
end
local ret = rspamd_redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- true, -- is write
- redis_cache_cb, --callback
- 'SETEX', -- command
- {key, tostring(settings.expire), value} -- arguments
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_cache_cb, --callback
+ 'SETEX', -- command
+ { key, tostring(settings.expire), value } -- arguments
)
lua_util.debugm(N, task, "set redis cache key: %s; %s", key, value)
if not ret then
end)
local max_mx_to_resolve = math.min(#results, settings.max_mx_a_records)
- lua_util.debugm(N, task,'check %s MX records (%d actually returned)',
+ lua_util.debugm(N, task, 'check %s MX records (%d actually returned)',
max_mx_to_resolve, #results)
- for i=1,max_mx_to_resolve do
+ for i = 1, max_mx_to_resolve do
local mx = results[i]
- mxes[mx.name] = {checked = false, working = false, ips = {}}
+ mxes[mx.name] = { checked = false, working = false, ips = {} }
local r = task:get_resolver()
-- XXX: maybe add ipv6?
r:resolve('a', {
local key = settings.key_prefix .. mx_domain
local ret = rspamd_redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- false, -- is write
- redis_cache_get_cb, --callback
- 'GET', -- command
- {key} -- arguments
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_cache_get_cb, --callback
+ 'GET', -- command
+ { key } -- arguments
)
if not ret then
type = 'normal',
callback = mx_check,
flags = 'empty',
- augmentations = {string.format("timeout=%f", settings.timeout + rspamd_config:get_dns_timeout() or 0.0)},
+ augmentations = { string.format("timeout=%f", settings.timeout + rspamd_config:get_dns_timeout() or 0.0) },
})
rspamd_config:register_symbol({
name = settings.symbol_no_mx,
})
if settings.exclude_domains then
- exclude_domains = rspamd_config:add_map{
+ exclude_domains = rspamd_config:add_map {
type = 'set',
description = 'Exclude specific domains from MX checks',
url = settings.exclude_domains,
local settings = neural_common.settings
-local redis_profile_schema = ts.shape{
+local redis_profile_schema = ts.shape {
digest = ts.string,
symbols = ts.array_of(ts.string),
version = ts.number,
true, -- is write
add_cb, --callback
'ZADD', -- command
- {set.prefix, tostring(rspamd_util.get_time()), profile_serialized}
+ { set.prefix, tostring(rspamd_util.get_time()), profile_serialized }
)
return profile
-- ANN filter function, used to insert scores based on the existing symbols
local function ann_scores_filter(task)
- for _,rule in pairs(settings.rules) do
+ for _, rule in pairs(settings.rules) do
local sid = task:get_settings_id() or -1
local ann
local profile
end
end
-
if learn_spam or learn_ham then
local learn_type
- if learn_spam then learn_type = 'spam' else learn_type = 'ham' end
+ if learn_spam then
+ learn_type = 'spam'
+ else
+ learn_type = 'ham'
+ end
local function vectors_len_cb(err, data)
if not err and type(data) == 'table' then
- local nspam,nham = data[1],data[2]
+ local nspam, nham = data[1], data[2]
if neural_common.can_push_train_vector(rule, task, learn_type, nspam, nham) then
local vec = neural_common.result_to_vector(task, set)
end
lua_redis.exec_redis_script(neural_common.redis_script_id.vectors_len,
- {task = task, is_write = false},
+ { task = task, is_write = false },
vectors_len_cb,
{
set.ann.redis_key,
-- Utility to extract and split saved training vectors to a table of tables
local function process_training_vectors(data)
return fun.totable(fun.map(function(tok)
- local _,str = rspamd_util.zstd_decompress(tok)
+ local _, str = rspamd_util.zstd_decompress(tok)
return fun.totable(fun.map(tonumber, lua_util.str_split(tostring(str), ';')))
end, data))
end
local function redis_ham_cb(err, data)
if err or type(data) ~= 'table' then
rspamd_logger.errx(rspamd_config, 'cannot get ham tokens for ANN %s from redis: %s',
- ann_key, err)
+ ann_key, err)
-- Unlock on error
lua_redis.redis_make_request_taskless(ev_base,
- rspamd_config,
- rule.redis,
- nil,
- true, -- is write
+ rspamd_config,
+ rule.redis,
+ nil,
+ true, -- is write
neural_common.gen_unlock_cb(rule, set, ann_key), --callback
- 'HDEL', -- command
- {ann_key, 'lock'}
+ 'HDEL', -- command
+ { ann_key, 'lock' }
)
else
-- Decompress and convert to numbers each training vector
ham_elts = process_training_vectors(data)
- neural_common.spawn_train({worker = worker, ev_base = ev_base,
- rule = rule, set = set, ann_key = ann_key, ham_vec = ham_elts,
- spam_vec = spam_elts})
+ neural_common.spawn_train({ worker = worker, ev_base = ev_base,
+ rule = rule, set = set, ann_key = ann_key, ham_vec = ham_elts,
+ spam_vec = spam_elts })
end
end
local function redis_spam_cb(err, data)
if err or type(data) ~= 'table' then
rspamd_logger.errx(rspamd_config, 'cannot get spam tokens for ANN %s from redis: %s',
- ann_key, err)
+ ann_key, err)
-- Unlock ANN on error
lua_redis.redis_make_request_taskless(ev_base,
- rspamd_config,
- rule.redis,
- nil,
- true, -- is write
+ rspamd_config,
+ rule.redis,
+ nil,
+ true, -- is write
neural_common.gen_unlock_cb(rule, set, ann_key), --callback
- 'HDEL', -- command
- {ann_key, 'lock'}
+ 'HDEL', -- command
+ { ann_key, 'lock' }
)
else
-- Decompress and convert to numbers each training vector
spam_elts = process_training_vectors(data)
-- Now get ham vectors...
lua_redis.redis_make_request_taskless(ev_base,
- rspamd_config,
- rule.redis,
- nil,
- false, -- is write
- redis_ham_cb, --callback
- 'SMEMBERS', -- command
- {ann_key .. '_ham_set'}
+ rspamd_config,
+ rule.redis,
+ nil,
+ false, -- is write
+ redis_ham_cb, --callback
+ 'SMEMBERS', -- command
+ { ann_key .. '_ham_set' }
)
end
end
local function redis_lock_cb(err, data)
if err then
rspamd_logger.errx(rspamd_config, 'cannot call lock script for ANN %s from redis: %s',
- ann_key, err)
+ ann_key, err)
elseif type(data) == 'number' and data == 1 then
-- ANN is locked, so we can extract SPAM and HAM vectors and spawn learning
lua_redis.redis_make_request_taskless(ev_base,
- rspamd_config,
- rule.redis,
- nil,
- false, -- is write
- redis_spam_cb, --callback
- 'SMEMBERS', -- command
- {ann_key .. '_spam_set'}
+ rspamd_config,
+ rule.redis,
+ nil,
+ false, -- is write
+ redis_spam_cb, --callback
+ 'SMEMBERS', -- command
+ { ann_key .. '_spam_set' }
)
rspamd_logger.infox(rspamd_config, 'lock ANN %s:%s (key name %s) for learning',
- rule.prefix, set.name, ann_key)
+ rule.prefix, set.name, ann_key)
else
local lock_tm = tonumber(data[1])
rspamd_logger.infox(rspamd_config, 'do not learn ANN %s:%s (key name %s), ' ..
-- This script returns either a boolean or a pair {'lock_time', 'hostname'} when
-- ANN is locked by another host (or a process, meh)
lua_redis.exec_redis_script(neural_common.redis_script_id.maybe_lock,
- {ev_base = ev_base, is_write = true},
- redis_lock_cb,
+ { ev_base = ev_base, is_write = true },
+ redis_lock_cb,
{
ann_key,
tostring(os.time()),
tostring(math.max(10.0, rule.watch_interval * 2)),
rspamd_util.get_hostname()
- })
+ })
end
-- This function loads new ann from Redis
else
if type(data) == 'table' then
if type(data[1]) == 'userdata' and data[1].cookie == text_cookie then
- local _err,ann_data = rspamd_util.zstd_decompress(data[1])
+ local _err, ann_data = rspamd_util.zstd_decompress(data[1])
local ann
if _err or not ann_data then
true, -- is write
rank_cb, --callback
'ZADD', -- command
- {set.prefix, tostring(rspamd_util.get_time()), profile_serialized}
+ { set.prefix, tostring(rspamd_util.get_time()), profile_serialized }
)
rspamd_logger.infox(rspamd_config,
'loaded ANN for %s:%s from %s; %s bytes compressed; version=%s',
local roc_thresholds = parser:get_object()
set.ann.roc_thresholds = roc_thresholds
rspamd_logger.infox(rspamd_config,
- 'loaded ROC thresholds for %s:%s; version=%s',
- rule.prefix, set.name, profile.version)
+ 'loaded ROC thresholds for %s:%s; version=%s',
+ rule.prefix, set.name, profile.version)
rspamd_logger.debugx("ROC thresholds: %s", roc_thresholds)
end
end
if set.ann and set.ann.ann and type(data[3]) == 'userdata' and data[3].cookie == text_cookie then
-- PCA table
- local _err,pca_data = rspamd_util.zstd_decompress(data[3])
+ local _err, pca_data = rspamd_util.zstd_decompress(data[3])
if pca_data then
if rule.max_inputs then
-- We can use PCA
false, -- is write
data_cb, --callback
'HMGET', -- command
- {ann_key, 'ann', 'roc_thresholds', 'pca'}, -- arguments
- {opaque_data = true}
+ { ann_key, 'ann', 'roc_thresholds', 'pca' }, -- arguments
+ { opaque_data = true }
)
end
local min_diff = math.huge
local sel_elt
- for _,elt in fun.iter(profiles) do
+ for _, elt in fun.iter(profiles) do
if elt and elt.symbols then
local dist = lua_util.distance_sorted(elt.symbols, my_symbols)
-- Check distance
ham = 0,
}
- for _,elt in fun.iter(profiles) do
+ for _, elt in fun.iter(profiles) do
if elt and elt.symbols then
local dist = lua_util.distance_sorted(elt.symbols, my_symbols)
-- Check distance
false, -- is write
redis_len_cb_gen(initiate_train, 'ham', true), --callback
'SCARD', -- command
- {ann_key .. '_ham_set'}
+ { ann_key .. '_ham_set' }
)
end
false, -- is write
redis_len_cb_gen(check_ham_len, 'spam', false), --callback
'SCARD', -- command
- {ann_key .. '_spam_set'}
+ { ann_key .. '_spam_set' }
)
end
end
local ucl = require "ucl"
local parser = ucl.parser()
- local res,ucl_err = parser:parse_string(element)
+ local res, ucl_err = parser:parse_string(element)
if not res then
rspamd_logger.warnx(rspamd_config, 'cannot parse ANN from redis: %s',
ucl_err)
return nil
else
local profile = parser:get_object()
- local checked,schema_err = redis_profile_schema:transform(profile)
+ local checked, schema_err = redis_profile_schema:transform(profile)
if not checked then
rspamd_logger.errx(rspamd_config, "cannot parse profile schema: %s", schema_err)
-- Function to check or load ANNs from Redis
local function check_anns(worker, cfg, ev_base, rule, process_callback, what)
- for _,set in pairs(rule.settings) do
+ for _, set in pairs(rule.settings) do
local function members_cb(err, data)
if err then
rspamd_logger.errx(cfg, 'cannot get ANNs list from redis: %s',
false, -- is write
members_cb, --callback
'ZREVRANGE', -- command
- {set.prefix, '0', tostring(settings.max_profiles)} -- arguments
+ { set.prefix, '0', tostring(settings.max_profiles) } -- arguments
)
end
end -- Cycle over all settings
-- Function to clean up old ANNs
local function cleanup_anns(rule, cfg, ev_base)
- for _,set in pairs(rule.settings) do
+ for _, set in pairs(rule.settings) do
local function invalidate_cb(err, data)
if err then
rspamd_logger.errx(cfg, 'cannot exec invalidate script in redis: %s',
err)
elseif type(data) == 'table' then
- for _,expired in ipairs(data) do
+ for _, expired in ipairs(data) do
local profile = load_ann_profile(expired)
rspamd_logger.infox(cfg, 'invalidated ANN for %s; redis key: %s; version=%s',
rule.prefix .. ':' .. set.name,
if type(set) == 'table' then
lua_redis.exec_redis_script(neural_common.redis_script_id.maybe_invalidate,
- {ev_base = ev_base, is_write = true},
+ { ev_base = ev_base, is_write = true },
invalidate_cb,
- {set.prefix, tostring(settings.max_profiles)})
+ { set.prefix, tostring(settings.max_profiles) })
end
end
end
return
end
- local verdict,score = lua_verdict.get_specific_verdict(N, task)
+ local verdict, score = lua_verdict.get_specific_verdict(N, task)
if verdict == 'passthrough' then
lua_util.debugm(N, task, 'ignore task as its verdict is %s(%s)',
return
end
- for _,rule in pairs(settings.rules) do
+ for _, rule in pairs(settings.rules) do
local set = neural_common.get_rule_settings(task, rule)
if set then
end
-- Check all rules
-for k,r in pairs(rules) do
+for k, r in pairs(rules) do
local rule_elt = lua_util.override_defaults(neural_common.default_options, r)
rule_elt['redis'] = neural_common.redis_params
rule_elt['anns'] = {} -- Store ANNs here
rule_elt.train.max_trains = rule_elt.train.max_train
end
- if not rule_elt.profile then rule_elt.profile = {} end
+ if not rule_elt.profile then
+ rule_elt.profile = {}
+ end
if rule_elt.max_inputs and not has_blas then
rspamd_logger.errx('cannot set max inputs to %s as BLAS is not compiled in',
rspamd_config:add_post_init(neural_common.process_rules_settings)
-- Add training scripts
-for _,rule in pairs(settings.rules) do
+for _, rule in pairs(settings.rules) do
neural_common.load_scripts(rule.redis)
-- This function will check ANNs in Redis when a worker is loaded
rspamd_config:add_on_load(function(cfg, ev_base, worker)
else
rspamd_logger.infox(task, 'source hostname has not been passed to Rspamd from MTA, ' ..
'but we could resolve source IP address PTR %s as "%s"',
- to_resolve, results[1])
+ to_resolve, results[1])
task:set_hostname(results[1])
if good_hosts then
- for _,gh in ipairs(good_hosts) do
+ for _, gh in ipairs(good_hosts) do
if string.find(results[1], gh) then
return
end
if nreceived <= 1 then
task:insert_result(symbol, 1)
- for _,h in ipairs(bad_hosts) do
+ for _, h in ipairs(bad_hosts) do
if string.find(results[1], h) then
task:insert_result(symbol_strict, 1, h)
end
if whitelist and task_ip and whitelist:get_key(task_ip) then
rspamd_logger.infox(task, 'whitelisted mail from %s',
- task_ip:to_string())
+ task_ip:to_string())
return
end
local hn = task:get_hostname()
-- Here we don't care about received
if (not hn) and task_ip and task_ip:is_valid() then
- task:get_resolver():resolve_ptr({task = task,
- name = task_ip:to_string(),
- callback = recv_dns_cb,
- forced = true
+ task:get_resolver():resolve_ptr({ task = task,
+ name = task_ip:to_string(),
+ callback = recv_dns_cb,
+ forced = true
})
return
end
local rhn = string.lower(r['real_hostname'])
-- Check for good hostname
if rhn and good_hosts then
- for _,gh in ipairs(good_hosts) do
+ for _, gh in ipairs(good_hosts) do
if string.find(rhn, gh) then
ret = false
break
-- Unresolved host
task:insert_result(symbol, 1)
- if not hn then return end
- for _,h in ipairs(bad_hosts) do
+ if not hn then
+ return
+ end
+ for _, h in ipairs(bad_hosts) do
if string.find(hn, h) then
task:insert_result(symbol_strict, 1, h)
return
callback = check_quantity_received,
})
- for n,v in pairs(opts) do
+ for n, v in pairs(opts) do
if n == 'symbol_strict' then
symbol_strict = v
elseif n == 'symbol_rdns' then
elseif n == 'whitelist' then
local lua_maps = require "lua_maps"
whitelist = lua_maps.map_add('once_received', 'whitelist', 'radix',
- 'once received whitelist')
+ 'once received whitelist')
elseif n == 'symbol_mx' then
symbol_mx = v
end
type = 'virtual',
parent = id
})
- rspamd_config:register_symbol({
+ rspamd_config:register_symbol({
name = symbol_strict,
type = 'virtual',
parent = id
if confighelp then
rspamd_config:add_example(nil, N,
- 'Detect remote OS via passive fingerprinting',
- [[
-p0f {
- # Enable module
- enabled = true
-
- # Path to the unix socket that p0f listens on
- socket = '/var/run/p0f.sock';
-
- # Connection timeout
- timeout = 5s;
-
- # If defined, insert symbol with lookup results
- symbol = 'P0F';
-
- # Patterns to match against results returned by p0f
- # Symbol will be yielded on OS string, link type or distance matches
- patterns = {
- WINDOWS = '^Windows.*';
- #DSL = '^DSL$';
- #DISTANCE10 = '^distance:10$';
+ 'Detect remote OS via passive fingerprinting',
+ [[
+ p0f {
+ # Enable module
+ enabled = true
+
+ # Path to the unix socket that p0f listens on
+ socket = '/var/run/p0f.sock';
+
+ # Connection timeout
+ timeout = 5s;
+
+ # If defined, insert symbol with lookup results
+ symbol = 'P0F';
+
+ # Patterns to match against results returned by p0f
+ # Symbol will be yielded on OS string, link type or distance matches
+ patterns = {
+ WINDOWS = '^Windows.*';
+ #DSL = '^DSL$';
+ #DISTANCE10 = '^distance:10$';
+ }
+
+ # Cache lifetime in seconds (default - 2 hours)
+ expire = 7200;
+
+ # Cache key prefix
+ prefix = 'p0f';
}
-
- # Cache lifetime in seconds (default - 2 hours)
- expire = 7200;
-
- # Cache key prefix
- prefix = 'p0f';
-}
-]])
+ ]])
return
end
priority = lua_util.symbols_priorities.medium,
flags = 'empty,nostat',
group = N,
- augmentations = {string.format("timeout=%f", rule.timeout or 0.0)},
+ augmentations = { string.format("timeout=%f", rule.timeout or 0.0) },
})
local generic_service_data = {}
local openphish_data = {}
-
local opts = rspamd_config:get_all_opt(N)
if not (opts and type(opts) == 'table') then
rspamd_logger.infox(rspamd_config, 'Module is unconfigured')
local query = url:get_query()
if path then
- for _,d in ipairs(elt) do
+ for _, d in ipairs(elt) do
if d['path'] == path then
found_path = true
data = d['data']
end
end
else
- for _,d in ipairs(elt) do
+ for _, d in ipairs(elt) do
if not d['path'] then
found_path = true
end
local function compose_dns_query(elts)
local cr = require "rspamd_cryptobox_hash"
local h = cr.create()
- for _,elt in ipairs(elts) do h:update(elt) end
+ for _, elt in ipairs(elts) do
+ h:update(elt)
+ end
return string.format("%s.%s", h:base32():sub(1, 32), dns_suffix)
end
local r = task:get_resolver()
end
end
- local to_resolve_hp = compose_dns_query({host, path})
+ local to_resolve_hp = compose_dns_query({ host, path })
rspamd_logger.debugm(N, task, 'try to resolve {%s, %s} -> %s',
host, path, to_resolve_hp)
r:resolve_txt({
task = task,
name = to_resolve_hp,
- callback = host_host_path_cb})
-
+ callback = host_host_path_cb })
if query then
local function host_host_path_query_cb(_, _, results, err)
end
end
- local to_resolve_hpq = compose_dns_query({host, path, query})
+ local to_resolve_hpq = compose_dns_query({ host, path, query })
rspamd_logger.debugm(N, task, 'try to resolve {%s, %s, %s} -> %s',
host, path, query, to_resolve_hpq)
r:resolve_txt({
task = task,
name = to_resolve_hpq,
- callback = host_host_path_query_cb})
+ callback = host_host_path_query_cb })
end
end
end
local urls = task:get_urls() or {}
- for _,url_iter in ipairs(urls) do
- local function do_loop_iter() -- to emulate continue
+ for _, url_iter in ipairs(urls) do
+ local function do_loop_iter()
+ -- to emulate continue
local url = url_iter
if generic_service_hash then
check_phishing_map(generic_service_data, url, generic_service_symbol)
purl = url:get_phished()
end
-
if not purl then
return
end
if dmarc_dom and tld == dmarc_dom then
lua_util.debugm(N, 'exclude phishing from %s -> %s by dmarc domain', tld,
- ptld)
+ ptld)
return
end
-- Now we can safely remove the last dot component if it is the same
- local b,_ = string.find(tld, '%.[^%.]+$')
- local b1,_ = string.find(ptld, '%.[^%.]+$')
+ local b, _ = string.find(tld, '%.[^%.]+$')
+ local b1, _ = string.find(ptld, '%.[^%.]+$')
- local stripped_tld,stripped_ptld = tld, ptld
+ local stripped_tld, stripped_ptld = tld, ptld
if b1 and b then
if string.sub(tld, b) == string.sub(ptld, b1) then
stripped_ptld = string.gsub(ptld, '%.[^%.]+$', '')
end
local weight = 1.0
- local spoofed,why = util.is_utf_spoofed(tld, ptld)
+ local spoofed, why = util.is_utf_spoofed(tld, ptld)
if spoofed then
lua_util.debugm(N, task, "confusable: %1 -> %2: %3", tld, ptld, why)
weight = 1.0
weight = util.tanh(3 * (1 - dist + 0.1))
elseif dist > 1 then
-- We also check if two labels are in the same ascii/non-ascii representation
- local a1, a2 = false,false
+ local a1, a2 = false, false
- if string.match(tld, '^[\001-\127]*$') then a1 = true end
- if string.match(ptld, '^[\001-\127]*$') then a2 = true end
+ if string.match(tld, '^[\001-\127]*$') then
+ a1 = true
+ end
+ if string.match(ptld, '^[\001-\127]*$') then
+ a2 = true
+ end
if a1 ~= a2 then
weight = 1
lua_util.debugm(N, task, "confusable: %1 -> %2: different characters",
- tld, ptld, why)
+ tld, ptld, why)
else
-- We have totally different strings in tld, so penalize it somehow
weight = 0.5
end
local function is_url_in_map(map, furl)
- for _,dn in ipairs({furl:get_tld(), furl:get_host()}) do
+ for _, dn in ipairs({ furl:get_tld(), furl:get_host() }) do
if map:get_key(dn) then
- return true,dn
+ return true, dn
end
end
return false
end
local function found_in_map(map, furl, sweight)
- if not furl then furl = url end
- if not sweight then sweight = weight end
+ if not furl then
+ furl = url
+ end
+ if not sweight then
+ sweight = weight
+ end
if #map > 0 then
- for _,rule in ipairs(map) do
- local found,dn = is_url_in_map(rule.map, furl)
+ for _, rule in ipairs(map) do
+ local found, dn = is_url_in_map(rule.map, furl)
if found then
task:insert_result(rule.symbol, sweight, string.format("%s->%s:%s", ptld, tld, dn))
return true
rspamd_logger.errx(rspamd_config, 'invalid exception table')
end
-
- for sym,map_data in pairs(xd) do
- local rmap = lua_maps.map_add_from_ucl (map_data, 'set',
- 'Phishing ' .. mapname .. ' map')
+ for sym, map_data in pairs(xd) do
+ local rmap = lua_maps.map_add_from_ucl(map_data, 'set',
+ 'Phishing ' .. mapname .. ' map')
if rmap then
rspamd_config:register_virtual_symbol(sym, 1, id)
- local rule = {symbol = sym, map = rmap}
+ local rule = { symbol = sym, map = rmap }
table.insert(phishmap, rule)
else
rspamd_logger.infox(rspamd_config, 'cannot add map for symbol: %s', sym)
local function rspamd_str_split_fun(s, sep, func)
local lpeg = require "lpeg"
sep = lpeg.P(sep)
- local elem = lpeg.P((1 - sep)^0 / func)
- local p = lpeg.P(elem * (sep * elem)^0)
+ local elem = lpeg.P((1 - sep) ^ 0 / func)
+ local p = lpeg.P(elem * (sep * elem) ^ 0)
return p:match(s)
end
if tbl[host] then
table.insert(tbl[host], elt)
else
- tbl[host] = {elt}
+ tbl[host] = { elt }
end
return true
generic_service_data = new_data
rspamd_logger.infox(generic_service_hash, "parsed %s elements from %s feed",
- nelts, generic_service_name)
+ nelts, generic_service_name)
pool:destroy()
end
local function openphish_elt_parser(cap)
if valid then
local parser = ucl.parser()
- local res,err = parser:parse_string(cap)
+ local res, err = parser:parse_string(cap)
if not res then
valid = false
rspamd_logger.warnx(openphish_hash, 'cannot parse openphish map: ' .. err)
if valid then
openphish_data = new_json_map
rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed",
- nelts)
+ nelts)
end
pool:destroy()
openphish_data = new_data
rspamd_logger.infox(openphish_hash, "parsed %s elements from openphish feed",
- nelts)
+ nelts)
pool:destroy()
end
if opts['generic_service_enabled'] then
generic_service_hash = rspamd_config:add_map({
- type = 'callback',
- url = generic_service_map,
- callback = generic_service_plain_cb,
- description = 'Generic feed'
- })
+ type = 'callback',
+ url = generic_service_map,
+ callback = generic_service_plain_cb,
+ description = 'Generic feed'
+ })
end
if opts['openphish_map'] then
})
else
openphish_hash = rspamd_config:add_map({
- type = 'callback',
- url = openphish_map,
- callback = openphish_json_cb,
- opaque_data = true,
- description = 'Open phishing premium feed map (see https://www.openphish.com for details)'
- })
+ type = 'callback',
+ url = openphish_map,
+ callback = openphish_json_cb,
+ opaque_data = true,
+ description = 'Open phishing premium feed map (see https://www.openphish.com for details)'
+ })
end
end
end
if opts['domains'] and type(opts['domains']) == 'string' then
domains = lua_maps.map_add_from_ucl(opts['domains'], 'set',
- 'Phishing domains')
+ 'Phishing domains')
end
phishing_map('phishing_exceptions', phishing_exceptions_maps, id)
phishing_map('exceptions', anchor_exceptions_maps, id)
-- Senders that are considered as bounce
local settings = {
bounce_senders = { 'postmaster', 'mailer-daemon', '', 'null', 'fetchmail-daemon', 'mdaemon' },
--- Do not check ratelimits for these recipients
+ -- Do not check ratelimits for these recipients
whitelisted_rcpts = { 'postmaster', 'mailer-daemon' },
prefix = 'RL',
ham_factor_rate = 1.01,
prefilter = true,
}
-
local bucket_check_script = "ratelimit_check.lua"
local bucket_check_id
-
local bucket_update_script = "ratelimit_update.lua"
local bucket_update_id
return string.format('Ratelimit "%s" exceeded', limit_type)
end
-
local function load_scripts(_, _)
bucket_check_id = lua_redis.load_redis_script_from_file(bucket_check_script, redis_params)
bucket_update_id = lua_redis.load_redis_script_from_file(bucket_update_script, redis_params)
if not limit_parser then
local digit = lpeg.R("09")
limit_parser = {}
- limit_parser.integer =
- (lpeg.S("+-") ^ -1) *
- (digit ^ 1)
- limit_parser.fractional =
- (lpeg.P(".") ) *
- (digit ^ 1)
- limit_parser.number =
- (limit_parser.integer *
- (limit_parser.fractional ^ -1)) +
- (lpeg.S("+-") * limit_parser.fractional)
+ limit_parser.integer = (lpeg.S("+-") ^ -1) *
+ (digit ^ 1)
+ limit_parser.fractional = (lpeg.P(".")) *
+ (digit ^ 1)
+ limit_parser.number = (limit_parser.integer *
+ (limit_parser.fractional ^ -1)) +
+ (lpeg.S("+-") * limit_parser.fractional)
limit_parser.time = lpeg.Cf(lpeg.Cc(1) *
- (limit_parser.number / tonumber) *
- ((lpeg.S("smhd") / parse_time_suffix) ^ -1),
- function (acc, val) return acc * val end)
+ (limit_parser.number / tonumber) *
+ ((lpeg.S("smhd") / parse_time_suffix) ^ -1),
+ function(acc, val)
+ return acc * val
+ end)
limit_parser.suffixed_number = lpeg.Cf(lpeg.Cc(1) *
- (limit_parser.number / tonumber) *
- ((lpeg.S("kmg") / parse_num_suffix) ^ -1),
- function (acc, val) return acc * val end)
+ (limit_parser.number / tonumber) *
+ ((lpeg.S("kmg") / parse_num_suffix) ^ -1),
+ function(acc, val)
+ return acc * val
+ end)
limit_parser.limit = lpeg.Ct(limit_parser.suffixed_number *
- (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) *
- limit_parser.time)
+ (lpeg.S(" ") ^ 0) * lpeg.S("/") * (lpeg.S(" ") ^ 0) *
+ limit_parser.time)
end
local t = lpeg.match(limit_parser.limit, lim)
end
local function str_to_rate(str)
- local divider,divisor = parse_string_limit(str, false)
+ local divider, divisor = parse_string_limit(str, false)
if not divisor then
rspamd_logger.errx(rspamd_config, 'bad rate string: %s', str)
return divisor / divider
end
-local bucket_schema = ts.shape{
+local bucket_schema = ts.shape {
burst = ts.number + ts.string / lua_util.dehumanize_number,
rate = ts.number + ts.string / str_to_rate,
skip_recipients = ts.boolean:is_optional(),
return nil
else
- local parsed_bucket,err = bucket_schema:transform(data)
+ local parsed_bucket, err = bucket_schema:transform(data)
if not parsed_bucket or err then
rspamd_logger.errx(rspamd_config, 'cannot parse bucket for %s: %s; original value: %s',
--- Check whether this addr is bounce
local function check_bounce(from)
- return fun.any(function(b) return b == from end, settings.bounce_senders)
+ return fun.any(function(b)
+ return b == from
+ end, settings.bounce_senders)
end
local keywords = {
['ip'] = {
['get_value'] = function(task)
local ip = task:get_ip()
- if ip and ip:is_valid() then return tostring(ip) end
+ if ip and ip:is_valid() then
+ return tostring(ip)
+ end
return nil
end,
},
['rip'] = {
['get_value'] = function(task)
local ip = task:get_ip()
- if ip and ip:is_valid() and not ip:is_local() then return tostring(ip) end
+ if ip and ip:is_valid() and not ip:is_local() then
+ return tostring(ip)
+ end
return nil
end,
},
if not ((from or E)[1] or E).user then
return '_'
end
- if check_bounce(from[1]['user']) then return '_' else return nil end
+ if check_bounce(from[1]['user']) then
+ return '_'
+ else
+ return nil
+ end
end,
},
['asn'] = {
local parts = task:get_parts() or E
local digests = {}
- for _,p in ipairs(parts) do
+ for _, p in ipairs(parts) do
if p:get_filename() then
table.insert(digests, p:get_digest())
end
local parts = task:get_parts() or E
local files = {}
- for _,p in ipairs(parts) do
+ for _, p in ipairs(parts) do
local fname = p:get_filename()
if fname then
table.insert(files, fname)
}
local function gen_rate_key(task, rtype, bucket)
- local key_t = {tostring(lua_util.round(100000.0 / bucket.burst))}
+ local key_t = { tostring(lua_util.round(100000.0 / bucket.burst)) }
local key_keywords = lua_util.str_split(rtype, '_')
local have_user = false
if keywords[v] and type(keywords[v]['get_value']) == 'function' then
ret = keywords[v]['get_value'](task)
end
- if not ret then return nil end
- if v == 'user' then have_user = true end
- if type(ret) ~= 'string' then ret = tostring(ret) end
+ if not ret then
+ return nil
+ end
+ if v == 'user' then
+ have_user = true
+ end
+ if type(ret) ~= 'string' then
+ ret = tostring(ret)
+ end
table.insert(key_t, ret)
end
local function make_prefix(redis_key, name, bucket)
local hash_len = 24
- if hash_len > #redis_key then hash_len = #redis_key end
+ if hash_len > #redis_key then
+ hash_len = #redis_key
+ end
local hash = settings.prefix ..
string.sub(rspamd_hash.create(redis_key):base32(), 1, hash_len)
-- Fill defaults
local function limit_to_prefixes(task, k, v, prefixes)
local n = 0
- for _,bucket in ipairs(v.buckets) do
+ for _, bucket in ipairs(v.buckets) do
if v.selector then
local selectors = lua_selectors.process_selectors(task, v.selector)
if selectors then
local function ratelimit_cb(task)
if not settings.allow_local and
- rspamd_lua_utils.is_rspamc_or_controller(task) then return end
+ rspamd_lua_utils.is_rspamc_or_controller(task) then
+ return
+ end
-- Get initial task data
local ip = task:get_from_ip()
local rcpts_user = {}
if rcpts then
fun.each(function(r)
- fun.each(function(type) table.insert(rcpts_user, r[type]) end, {'user', 'addr'})
+ fun.each(function(type)
+ table.insert(rcpts_user, r[type])
+ end, { 'user', 'addr' })
end, rcpts)
- if fun.any(function(r) return settings.whitelisted_rcpts:get_key(r) end, rcpts_user) then
+ if fun.any(function(r)
+ return settings.whitelisted_rcpts:get_key(r)
+ end, rcpts_user) then
rspamd_logger.infox(task, 'skip ratelimit for whitelisted recipient')
return
end
local prefixes = {}
local nprefixes = 0
- for k,v in pairs(settings.limits) do
+ for k, v in pairs(settings.limits) do
nprefixes = nprefixes + limit_to_prefixes(task, k, v, prefixes)
end
end
-- Don't do anything if pre-result has been already set
- if task:has_pre_result() then return end
+ if task:has_pre_result() then
+ return
+ end
- local _,nrcpt = task:has_recipients('smtp')
+ local _, nrcpt = task:has_recipients('smtp')
if not nrcpt or nrcpt <= 0 then
nrcpt = 1
end
now = lua_util.round(now * 1000.0) -- Get milliseconds
-- Now call check script for all defined prefixes
- for pr,value in pairs(prefixes) do
+ for pr, value in pairs(prefixes) do
local bucket = value.bucket
local rate = (bucket.rate) / 1000.0 -- Leak rate in messages/ms
local bincr = nrcpt
- if bucket.skip_recipients then bincr = 1 end
+ if bucket.skip_recipients then
+ bincr = 1
+ end
lua_util.debugm(N, task, "check limit %s:%s -> %s (%s/%s)",
value.name, pr, value.hash, bucket.burst, bucket.rate)
lua_redis.exec_redis_script(bucket_check_id,
- {key = value.hash, task = task, is_write = true},
+ { key = value.hash, task = task, is_write = true },
gen_check_cb(pr, bucket, value.name, value.hash),
- {value.hash, tostring(now), tostring(rate), tostring(bucket.burst),
- tostring(settings.expire), tostring(bincr)})
+ { value.hash, tostring(now), tostring(rate), tostring(bucket.burst),
+ tostring(settings.expire), tostring(bincr) })
end
end
end
lua_util.debugm(N, task, 'cleaned pending bucked for %s: %s', k, data)
end
end
- local _,nrcpt = task:has_recipients('smtp')
+ local _, nrcpt = task:has_recipients('smtp')
if not nrcpt or nrcpt <= 0 then
nrcpt = 1
end
local bincr = nrcpt
- if bucket.skip_recipients then bincr = 1 end
+ if bucket.skip_recipients then
+ bincr = 1
+ end
local now = task:get_timeval(true)
now = lua_util.round(now * 1000.0) -- Get milliseconds
lua_redis.exec_redis_script(bucket_cleanup_id,
- {key = v.hash, task = task, is_write = true},
+ { key = v.hash, task = task, is_write = true },
cleanup_cb,
- {v.hash, tostring(now), tostring(settings.expire), tostring(bincr)})
+ { v.hash, tostring(now), tostring(settings.expire), tostring(bincr) })
end
end
end
end
local verdict = lua_verdict.get_specific_verdict(N, task)
- local _,nrcpt = task:has_recipients('smtp')
+ local _, nrcpt = task:has_recipients('smtp')
if not nrcpt or nrcpt <= 0 then
nrcpt = 1
end
end
local bincr = nrcpt
- if bucket.skip_recipients then bincr = 1 end
+ if bucket.skip_recipients then
+ bincr = 1
+ end
lua_redis.exec_redis_script(bucket_update_id,
- {key = v.hash, task = task, is_write = true},
+ { key = v.hash, task = task, is_write = true },
update_bucket_cb,
- {v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst),
- tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult),
- tostring(settings.expire), tostring(bincr)})
+ { v.hash, tostring(now), tostring(mult_rate), tostring(mult_burst),
+ tostring(settings.max_rate_mult), tostring(settings.max_bucket_mult),
+ tostring(settings.expire), tostring(bincr) })
end
end
end
if type(lim) == 'table' and lim.bucket then
if lim.bucket[1] then
- for _,bucket in ipairs(lim.bucket) do
+ for _, bucket in ipairs(lim.bucket) do
local b = parse_limit(t, bucket)
if not b then
return
end
- buckets = {bucket}
+ buckets = { bucket }
end
settings.limits[t] = {
buckets = parse_limit(t, lim)
if buckets then
settings.limits[t] = {
- buckets = {buckets}
+ buckets = { buckets }
}
end
end
-- Display what's enabled
fun.each(function(s)
rspamd_logger.infox(rspamd_config, 'enabled ratelimit: %s', s)
- end, fun.map(function(n,d)
+ end, fun.map(function(n, d)
return string.format('%s [%s]', n,
table.concat(fun.totable(fun.map(function(v)
return string.format('symbol: %s, %s msgs burst, %s msgs/sec rate',
if type(wrcpts) == 'string' then
if string.find(wrcpts, ',') then
settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(
- lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts')
+ lua_util.rspamd_str_split(wrcpts, ','), 'set', 'Ratelimit whitelisted rcpts')
else
settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set',
- 'Ratelimit whitelisted rcpts')
+ 'Ratelimit whitelisted rcpts')
end
elseif type(opts['whitelisted_rcpts']) == 'table' then
settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(wrcpts, 'set',
- 'Ratelimit whitelisted rcpts')
+ 'Ratelimit whitelisted rcpts')
else
-- Stupid default...
settings.whitelisted_rcpts = lua_maps.rspamd_map_add_from_ucl(
if opts['whitelisted_ip'] then
settings.whitelisted_ip = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_ip', 'radix',
- 'Ratelimit whitelist ip map')
+ 'Ratelimit whitelist ip map')
end
if opts['whitelisted_user'] then
settings.whitelisted_user = lua_maps.rspamd_map_add('ratelimit', 'whitelisted_user', 'set',
- 'Ratelimit whitelist user map')
+ 'Ratelimit whitelist user map')
end
settings.custom_keywords = {}
if ret then
opts['custom_keywords'] = {}
if type(res_or_err) == 'table' then
- for k,hdl in pairs(res_or_err) do
+ for k, hdl in pairs(res_or_err) do
settings['custom_keywords'][k] = hdl
end
elseif type(res_or_err) == 'function' then
priority = lua_util.symbols_priorities.medium,
callback = ratelimit_cb,
flags = 'empty,nostat',
- augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)},
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
}
local id = rspamd_config:register_symbol(s)
-- Display what's enabled
fun.each(function(set, lim)
if type(lim.buckets) == 'table' then
- for _,b in ipairs(lim.buckets) do
+ for _, b in ipairs(lim.buckets) do
if b.symbol then
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual',
name = b.symbol,
score = 0.0,
end, settings.limits)
if settings.info_symbol then
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual',
name = settings.info_symbol,
score = 0.0,
}
end
if settings.symbol then
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual',
name = settings.symbol,
score = 0.0, -- Might be overridden if needed
name = 'RATELIMIT_UPDATE',
flags = 'explicit_disable,ignore_passthrough',
callback = ratelimit_update_cb,
- augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)},
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
}
end
end
local nmatch_flags = rbl.received_nflags
local function basic_received_check(rh)
- if not (rh.real_ip and rh.real_ip:is_valid()) then return false end
+ if not (rh.real_ip and rh.real_ip:is_valid()) then
+ return false
+ end
if ((rh.real_ip:get_version() == 6 and rbl.ipv6) or
(rh.real_ip:get_version() == 4 and rbl.ipv4)) and
((rbl.exclude_private_ips and not rh.real_ip:is_local()) or
end
local function positioned_received_check(rh, pos)
- if not rh or not basic_received_check(rh) then return false end
+ if not rh or not basic_received_check(rh) then
+ return false
+ end
local got_flags = rh.flags or E
if min_pos then
if min_pos < 0 then
return true
end
-
if not (max_pos or min_pos or match_flags or nmatch_flags) then
return basic_received_check
else
return
end
- for _,result in ipairs(results) do
+ for _, result in ipairs(results) do
local ipstr = result:to_string()
lua_util.debugm(N, task, '%s DNS result %s', to_resolve, ipstr)
local foundrc = false
-- Check return codes
if rbl.returnbits then
local ipnum = result:to_number()
- for s,bits in pairs(rbl.returnbits) do
- for _,check_bit in ipairs(bits) do
+ for s, bits in pairs(rbl.returnbits) do
+ for _, check_bit in ipairs(bits) do
if bit.band(ipnum, check_bit) == check_bit then
foundrc = true
insert_results(s)
end
elseif rbl.returncodes then
for s, codes in pairs(rbl.returncodes) do
- for _,v in ipairs(codes) do
+ for _, v in ipairs(codes) do
if string.find(ipstr, '^' .. v .. '$') then
foundrc = true
insert_results(s)
nreq.what[label] = true
end
- return true,nreq -- Duplicate
+ return true, nreq -- Duplicate
else
local nreq
n = processed,
orig = req_str,
resolve_ip = resolve_ip,
- what = {[label] = true},
+ what = { [label] = true },
}
requests_table[req] = nreq
end
n = to_resolve,
orig = req_str,
resolve_ip = resolve_ip,
- what = {[label] = true},
+ what = { [label] = true },
}
requests_table[req] = nreq
end
local function check_required_symbols(task, _)
if rule.require_symbols then
- return fun.all(function(sym) task:has_symbol(sym) end, rule.require_symbols)
+ return fun.all(function(sym)
+ task:has_symbol(sym)
+ end, rule.require_symbols)
end
return true
mime_from_domain = ((task:get_from('mime') or E)[1] or E).domain
if mime_from_domain then
local mime_from_domain_tld = rule.url_full_hostname and
- mime_from_domain or rspamd_util.get_tld(mime_from_domain)
+ mime_from_domain or rspamd_util.get_tld(mime_from_domain)
if rule.url_compose_map then
mime_from_domain = rule.url_compose_map:process_url(task, mime_from_domain_tld, mime_from_domain)
for _, d in ipairs(das[1].options) do
- local domain,result = d:match('^([^%:]*):([%+%-%~])$')
+ local domain, result = d:match('^([^%:]*):([%+%-%~])$')
-- We must ignore bad signatures, omg
if domain and result and result == '+' then
if mime_from_domain and mime_from_domain == domain_tld then
add_dns_request(task, domain_tld, true, false, requests_table,
- 'dkim', whitelist)
+ 'dkim', whitelist)
end
else
if rule.dkim_domainonly then
local urls = lua_util.extract_specific_urls(ex_params)
- for _,u in ipairs(urls) do
+ for _, u in ipairs(urls) do
local flags = u:get_flags_num()
if bit.band(flags, url_flag_bits.numeric) ~= 0 then
end
local function check_received(task, requests_table, whitelist)
- local received = fun.filter(function(h)
+ local received = fun .filter(function(h)
return not h['flags']['artificial']
end, task:get_received_headers()):totable()
local received_total = #received
local check_conditions = gen_check_rcvd_conditions(rule, received_total)
- for pos,rh in ipairs(received) do
+ for pos, rh in ipairs(received) do
if check_conditions(rh, pos) then
add_dns_request(task, rh.real_ip, false, true,
requests_table, 'received',
local res = selector(task)
if res and type(res) == 'table' then
- for _,r in ipairs(res) do
- add_dns_request(task, r, false, false, requests_table,
- selector_label, whitelist)
- end
+ for _, r in ipairs(res) do
+ add_dns_request(task, r, false, false, requests_table,
+ selector_label, whitelist)
+ end
elseif res then
add_dns_request(task, res, false, false,
- requests_table, selector_label, whitelist)
+ requests_table, selector_label, whitelist)
end
end
local ex_params = {
task = task,
limit = rule.requests_limit,
- filter = function(u) return u:get_protocol() == 'mailto' end,
+ filter = function(u)
+ return u:get_protocol() == 'mailto'
+ end,
need_emails = true,
prefix = 'rbl_email'
}
local emails = lua_util.extract_specific_urls(ex_params)
- for _,email in ipairs(emails) do
+ for _, email in ipairs(emails) do
local domain
if rule.emails_domainonly and not rule.url_full_hostname then
if rule.url_compose_map then
end
-- Execute functions pipeline
- for i,f in ipairs(pipeline) do
+ for i, f in ipairs(pipeline) do
if not f(task, dns_req, whitelist) then
lua_util.debugm(N, task,
"skip rbl check: %s; pipeline condition %s returned false",
local function gen_rbl_ip_dns_callback(orig_resolve_table_elt)
return function(_, _, results, err)
if not err then
- for _,dns_res in ipairs(results) do
+ for _, dns_res in ipairs(results) do
-- Check if we have rspamd{ip} userdata
if type(dns_res) == 'userdata' then
-- Add result as an actual RBL request
local label = next(orig_resolve_table_elt.what)
- local dup,nreq = add_dns_request(task, dns_res, false, true,
+ local dup, nreq = add_dns_request(task, dns_res, false, true,
resolved_req, label)
-- Add original name
if not dup then
end
end
- return callback_f,string.format('checks: %s', table.concat(description, ','))
+ return callback_f, string.format('checks: %s', table.concat(description, ','))
end
local function add_rbl(key, rbl, global_opts)
rbl.symbol = key:upper()
end
- local flags_tbl = {'no_squeeze'}
+ local flags_tbl = { 'no_squeeze' }
if rbl.is_whitelist then
flags_tbl[#flags_tbl + 1] = 'nice'
end
rbl.selectors = {}
if type(rbl.selector) ~= 'table' then
- rbl.selector = {['selector'] = rbl.selector}
+ rbl.selector = { ['selector'] = rbl.selector }
end
for selector_label, selector in pairs(rbl.selector) do
rbl.selector_flatten = true
end
local sel = selectors.create_selector_closure(rspamd_config, selector, '',
- rbl.selector_flatten)
+ rbl.selector_flatten)
if not sel then
rspamd_logger.errx('invalid selector for rbl rule %s: %s', key, selector)
rbl.symbol)
end
- local callback,description = gen_rbl_callback(rbl)
+ local callback, description = gen_rbl_callback(rbl)
if callback then
local id
if rbl.symbols_prefixes then
- id = rspamd_config:register_symbol{
+ id = rspamd_config:register_symbol {
type = 'callback',
callback = callback,
- groups = {'rbl'},
+ groups = { 'rbl' },
name = rbl.symbol .. '_CHECK',
flags = table.concat(flags_tbl, ',')
}
- for _,prefix in pairs(rbl.symbols_prefixes) do
+ for _, prefix in pairs(rbl.symbols_prefixes) do
-- For unknown results...
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual',
parent = id,
group = 'rbl',
rbl.symbol, rbl.is_whitelist, rbl.ignore_whitelist)
end
else
- id = rspamd_config:register_symbol{
+ id = rspamd_config:register_symbol {
type = 'callback',
callback = callback,
name = rbl.symbol,
- groups = {'rbl'},
+ groups = { 'rbl' },
group = 'rbl',
score = 0,
flags = table.concat(flags_tbl, ',')
end
end
-
rspamd_logger.infox(rspamd_config, 'added rbl rule %s: %s',
rbl.symbol, description)
lua_util.debugm(N, rspamd_config, 'rule dump for %s: %s',
end
if rbl.require_symbols then
- for _,dep in ipairs(rbl.require_symbols) do
+ for _, dep in ipairs(rbl.require_symbols) do
rspamd_config:register_dependency(check_sym, dep)
end
end
-- Failure symbol
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual',
flags = 'nostat',
name = rbl.symbol .. '_FAIL',
if s ~= rbl.symbol then
-- hack
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
type = 'virtual',
parent = id,
name = s,
end
if rbl.symbols_prefixes then
- for _,prefix in pairs(rbl.symbols_prefixes) do
+ for _, prefix in pairs(rbl.symbols_prefixes) do
process_specific_suffix(prefix .. '_' .. suffix)
end
else
end
if rbl.returncodes then
- for s,_ in pairs(rbl.returncodes) do
+ for s, _ in pairs(rbl.returncodes) do
process_return_code(s)
end
end
if rbl.returnbits then
- for s,_ in pairs(rbl.returnbits) do
+ for s, _ in pairs(rbl.returnbits) do
process_return_code(s)
end
end
opts.rbls = lua_util.override_defaults(opts.rbls, opts.rules)
end
-if(opts['local_exclude_ip_map'] ~= nil) then
+if (opts['local_exclude_ip_map'] ~= nil) then
local_exclusions = lua_maps.map_add(N, 'local_exclude_ip_map', 'radix',
- 'RBL exclusions map')
+ 'RBL exclusions map')
end
-- TODO: this code should be universal for all modules that use selectors to allow
-- maps usage from selectors registered for a specific module
if type(opts.attached_maps) == 'table' then
opts.attached_maps_processed = {}
- for i,map in ipairs(opts.attached_maps) do
+ for i, map in ipairs(opts.attached_maps) do
-- Store maps in the configuration table to keep lifetime track
opts.attached_maps_processed[i] = lua_maps.map_add_from_ucl(map)
if opts.attached_maps_processed[i] == nil then
end
end
-for key,rbl in pairs(opts.rbls) do
+for key, rbl in pairs(opts.rbls) do
if type(rbl) ~= 'table' or rbl.disabled == true or rbl.enabled == false then
rspamd_logger.infox(rspamd_config, 'disable rbl "%s"', key)
else
end
-- Propagate default options from opts to rule
if not rbl.ignore_defaults then
- for default_opt_key,_ in pairs(rbl_common.default_options) do
+ for default_opt_key, _ in pairs(rbl_common.default_options) do
local rbl_opt = default_opt_key:sub(#('default_') + 1)
if rbl[rbl_opt] == nil then
rbl[rbl_opt] = opts[default_opt_key]
rbl.requests_limit = rspamd_config:get_dns_max_requests()
end
- local res,err = rbl_common.rule_schema:transform(rbl)
+ local res, err = rbl_common.rule_schema:transform(rbl)
if not res then
rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED',
key, err)
else
res = rbl_common.convert_checks(res)
-- Aliases
- if res.return_codes then res.returncodes = res.return_codes end
- if res.return_bits then res.returnbits = res.return_bits end
+ if res.return_codes then
+ res.returncodes = res.return_codes
+ end
+ if res.return_bits then
+ res.returnbits = res.return_bits
+ end
if not res then
rspamd_logger.errx(rspamd_config, 'invalid config for %s: %s, RBL is DISABLED',
local ws = task:get_symbol(w)
if ws and ws[1] then
ws = ws[1]
- if not ws.options then ws.options = {} end
- for _,opt in ipairs(ws.options) do
- local elt,what = opt:match('^([^:]+):([^:]+)')
- lua_util.debugm(N, task,'found whitelist from %s: %s(%s)', w,
+ if not ws.options then
+ ws.options = {}
+ end
+ for _, opt in ipairs(ws.options) do
+ local elt, what = opt:match('^([^:]+):([^:]+)')
+ lua_util.debugm(N, task, 'found whitelist from %s: %s(%s)', w,
elt, what)
if elt and what then
whitelisted_elements[elt] = {
lua_util.debugm(N, task, "finished rbl processing")
end
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'callback',
callback = rbl_callback_white,
name = 'RBL_CALLBACK_WHITE',
flags = 'nice,empty,no_squeeze',
- groups = {'rbl'},
- augmentations = {string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0)},
+ groups = { 'rbl' },
+ augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) },
}
-rspamd_config:register_symbol{
+rspamd_config:register_symbol {
type = 'callback',
callback = rbl_callback_fin,
name = 'RBL_CALLBACK',
flags = 'empty,no_squeeze',
- groups = {'rbl'},
- augmentations = {string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0)},
+ groups = { 'rbl' },
+ augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() or 0.0) },
}
for _, w in ipairs(white_symbols) do
return real_rcpt_h == stored_rcpt
end
- if fun.any(filter_predicate, fun.map(function(rcpt) return rcpt.addr or '' end, rcpts)) then
+ if fun.any(filter_predicate, fun.map(function(rcpt)
+ return rcpt.addr or ''
+ end, rcpts)) then
lua_util.debugm(N, task, 'reply to %s validated', in_reply_to)
return true
end
local key = make_key(in_reply_to, settings.key_size, settings.key_prefix)
local ret = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- false, -- is write
- redis_get_cb, --callback
- 'GET', -- command
- {key} -- arguments
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_get_cb, --callback
+ 'GET', -- command
+ { key } -- arguments
)
if not ret then
local function replies_set(task)
local function redis_set_cb(err, _, addr)
- if err ~=nil then
+ if err ~= nil then
rspamd_logger.errx(task, 'redis_set_cb error when writing data to %s: %s', addr:get_addr(), err)
end
end
if sender then
local sender_hash = make_key(sender:lower(), 8)
lua_util.debugm(N, task, 'storing id: %s (%s), reply-to: %s (%s) for replies check',
- msg_id, key, sender, sender_hash)
+ msg_id, key, sender, sender_hash)
local ret = lua_redis.redis_make_request(task,
redis_params, -- connect params
key, -- hash key
true, -- is write
redis_set_cb, --callback
'PSETEX', -- command
- {key, tostring(math.floor(settings['expire'] * 1000)), sender_hash} -- arguments
+ { key, tostring(math.floor(settings['expire'] * 1000)), sender_hash } -- arguments
)
if not ret then
rspamd_logger.errx(task, "redis request wasn't scheduled")
local function replies_check_cookie(task)
local function cookie_matched(extra, ts)
- local dt = task:get_date{format = 'connect', gmt = true}
+ local dt = task:get_date { format = 'connect', gmt = true }
if dt < ts then
rspamd_logger.infox(task, 'ignore cookie as its date is in future')
extracted_cookie = irt
end
- local dec_cookie,ts = cr.decrypt_cookie(settings.cookie_key, extracted_cookie)
+ local dec_cookie, ts = cr.decrypt_cookie(settings.cookie_key, extracted_cookie)
if dec_cookie then
-- We have something that looks like a cookie
else
-- Cookies mode
-- Check key sanity:
- local pattern = {'^'}
- for i=1,32 do pattern[i + 1] = '[a-zA-Z0-9]' end
+ local pattern = { '^' }
+ for i = 1, 32 do
+ pattern[i + 1] = '[a-zA-Z0-9]'
+ end
pattern[34] = '$'
if not settings.cookie_key:match(table.concat(pattern, '')) then
rspamd_logger.errx(rspamd_config,
if rspamd_version('cmp', obj['min_version']) > 0 then
ret = false
rspamd_logger.errx(rspamd_config, 'updates require at least %s version of rspamd',
- obj['min_version'])
+ obj['min_version'])
end
end
if obj['max_version'] then
if rspamd_version('cmp', obj['max_version']) < 0 then
ret = false
rspamd_logger.errx(rspamd_config, 'updates require maximum %s version of rspamd',
- obj['max_version'])
+ obj['max_version'])
end
end
return function(data)
local parser = ucl.parser()
- local res,err = parser:parse_string(data)
+ local res, err = parser:parse_string(data)
if not res then
rspamd_logger.warnx(rspamd_config, 'cannot parse updates map: ' .. err)
end
rspamd_logger.infox(rspamd_config, 'loaded new rules with hash "%s"',
- h:hex())
+ h:hex())
end
end
end
if type(section.rules) ~= 'table' then
- section.rules = {section.rules}
+ section.rules = { section.rules }
end
fun.each(function(elt)
end
if to_apply.flags and type(to_apply.flags) == 'table' then
- for _,fl in ipairs(to_apply.flags) do
+ for _, fl in ipairs(to_apply.flags) do
task:set_flag(fl)
end
end
-- Add symbols, specified in the settings
if #to_apply.symbols > 0 then
-- Array like symbols
- for _,val in ipairs(to_apply.symbols) do
+ for _, val in ipairs(to_apply.symbols) do
task:insert_result(val, 1.0)
end
else
-- Object like symbols
- for k,v in pairs(to_apply.symbols) do
+ for k, v in pairs(to_apply.symbols) do
if type(v) == 'table' then
task:insert_result(k, v.score or 1.0, v.options or {})
elseif tonumber(v) then
if query_set then
local parser = ucl.parser()
- local res,err = parser:parse_text(query_set)
+ local res, err = parser:parse_text(query_set)
if res then
if settings_id then
rspamd_logger.warnx(task, "both settings-id '%s' and settings headers are presented, ignore settings-id; ",
local settings_obj = parser:get_object()
-- Treat as low priority
- return settings_obj,nil,1
+ return settings_obj, nil, 1
else
rspamd_logger.errx(task, 'Parse error: %s', err)
end
if query_maxscore then
if settings_id then
rspamd_logger.infox(task, "both settings id '%s' and maxscore '%s' headers are presented, merge them; " ..
- "settings id has priority",
- tostring(settings_id), tostring(query_maxscore))
+ "settings id has priority",
+ tostring(settings_id), tostring(query_maxscore))
end
-- We have score limits redefined by request
local ms = tonumber(tostring(query_maxscore))
if cached then
local elt = cached.settings
if elt['whitelist'] then
- elt['apply'] = {whitelist = true}
+ elt['apply'] = { whitelist = true }
end
if elt.apply then
if elt then
local input = elt.extract(task)
- if not input then return false end
+ if not input then
+ return false
+ end
if elt.check(input) then
matched[#matched + 1] = atom
if res and res > 0 then
if rule['whitelist'] then
- rule['apply'] = {whitelist = true}
+ rule['apply'] = { whitelist = true }
end
return rule
end
-- Check if we have override as query argument
- local query_apply,id_elt,priority = check_query_settings(task)
+ local query_apply, id_elt, priority = check_query_settings(task)
local function maybe_apply_query_settings()
if query_apply then
-- Match rules according their order
local applied = false
- for pri = max_pri,min_pri,-1 do
+ for pri = max_pri, min_pri, -1 do
if not applied and settings[pri] then
- for _,s in ipairs(settings[pri]) do
+ for _, s in ipairs(settings[pri]) do
local matched = {}
local result = check_specific_setting(s.rule, matched)
local function convert_to_table(chk_elt, out)
if type(chk_elt) == 'string' then
- return {out}
+ return { out }
end
return out
end
local function gen_settings_external_cb(name)
- return function (result, err_or_data, code, task)
+ return function(result, err_or_data, code, task)
if result then
local parser = ucl.parser()
- local res,ucl_err = parser:parse_text(err_or_data)
+ local res, ucl_err = parser:parse_text(err_or_data)
if not res then
rspamd_logger.warnx(task, 'cannot parse settings from the external map %s: %s',
name, ucl_err)
local out = {}
if type(ip) == "table" then
- for _,v in ipairs(ip) do
+ for _, v in ipairs(ip) do
table.insert(out, process_ip_condition(v))
end
elseif type(ip) == "string" then
local function process_email_condition(addr)
local out = {}
if type(addr) == "table" then
- for _,v in ipairs(addr) do
+ for _, v in ipairs(addr) do
table.insert(out, process_email_condition(v))
end
elseif type(addr) == "string" then
local function process_string_condition(addr)
local out = {}
if type(addr) == "table" then
- for _,v in ipairs(addr) do
+ for _, v in ipairs(addr) do
table.insert(out, process_string_condition(v))
end
elseif type(addr) == "string" then
-- Used to create a checking closure: if value matches expected somehow, return true
local function gen_check_closure(expected, check_func)
return function(value)
- if not value then return false end
+ if not value then
+ return false
+ end
if type(value) == 'function' then
value = value()
if value then
if not check_func then
- check_func = function(a, b) return a == b end
+ check_func = function(a, b)
+ return a == b
+ end
end
local ret
check = gen_check_closure(convert_to_table(elt.ip, ips_table), check_ip_setting),
extract = function(task)
local ip = task:get_from_ip()
- if ip and ip:is_valid() then return ip end
+ if ip and ip:is_valid() then
+ return ip
+ end
return nil
end,
}
check = gen_check_closure(ips_map, check_map_setting),
extract = function(task)
local ip = task:get_from_ip()
- if ip and ip:is_valid() then return ip end
+ if ip and ip:is_valid() then
+ return ip
+ end
return nil
end,
}
check_ip_setting),
extract = function(task)
local ip = task:get_client_ip()
- if ip:is_valid() then return ip end
+ if ip:is_valid() then
+ return ip
+ end
return nil
end,
}
check = gen_check_closure(ips_map, check_map_setting),
extract = function(task)
local ip = task:get_client_ip()
- if ip and ip:is_valid() then return ip end
+ if ip and ip:is_valid() then
+ return ip
+ end
return nil
end,
}
lua_util.debugm(N, rspamd_config, 'added authenticated condition to "%s"',
name)
checks.authenticated = {
- check = function(value) if value then return true end return false end,
+ check = function(value)
+ if value then
+ return true
+ end
+ return false
+ end,
extract = function(task)
return task:get_user()
end
lua_util.debugm(N, rspamd_config, 'added local condition to "%s"',
name)
checks['local'] = {
- check = function(value) if value then return true end return false end,
+ check = function(value)
+ if value then
+ return true
+ end
+ return false
+ end,
extract = function(task)
local ip = task:get_from_ip()
if not ip or not ip:is_valid() then
checks[full_key] = cond
-- Try numeric key
- for i=1,1000 do
+ for i = 1, 1000 do
local num_key = generic .. ':' .. tostring(i)
- if not checks[num_key] then
+ if not checks[num_key] then
checks[num_key] = cond
aliases[num_key] = true
break
if re then
local cond = {
check = function(values)
- return fun.any(function(c) return re:match(c) end, values)
+ return fun.any(function(c)
+ return re:match(c)
+ end, values)
end,
extract = extractor_func(k),
}
elseif type(v) == 'boolean' then
local cond = {
check = function(values)
- if #values == 0 then return (not v) end
+ if #values == 0 then
+ return (not v)
+ end
return v
end,
extract = extractor_func(k),
process_header_elt('request_header', function(hname)
return function(task)
local rh = task:get_request_header(hname)
- if rh then return {rh} end
+ if rh then
+ return { rh }
+ end
return {}
end
end)
return function(task)
local rh = task:get_header_full(hname)
if rh then
- return fun.totable(fun.map(function(h) return h.decoded end, rh))
+ return fun.totable(fun.map(function(h)
+ return h.decoded
+ end, rh))
end
return {}
end
-- Count checks and create Rspamd expression from a set of rules
local nchecks = 0
- for k,_ in pairs(checks) do
+ for k, _ in pairs(checks) do
if not aliases[k] then
nchecks = nchecks + 1
end
-- Here we get all keys and concatenate them with '&&'
local s = ' && '
-- By De Morgan laws
- if inverse then s = ' || ' end
+ if inverse then
+ s = ' || '
+ end
-- Exclude aliases and join all checks by key
local expr_str = table.concat(lua_util.keys(fun.filter(
- function(k, _) return not aliases[k] end,
+ function(k, _)
+ return not aliases[k]
+ end,
checks)), s)
if inverse then
rspamd_logger.errx(rspamd_config,
'use of undefined element "%s" when parsing settings expression, known checks: %s',
- atom, table.concat(fun.totable(fun.map(function(k, _) return k end, checks)), ','))
+ atom, table.concat(fun.totable(fun.map(function(k, _)
+ return k
+ end, checks)), ','))
return nil
end
-- In fact, it is useless and evil but who cares...
if elt.apply and elt.apply.symbols then
-- Register virtual symbols
- for k,v in pairs(elt.apply.symbols) do
+ for k, v in pairs(elt.apply.symbols) do
local rtb = {
type = 'virtual',
parent = module_sym_id,
settings_initialized = false
-- filter trash in the input
local ft = fun.filter(
- function(_, elt)
- if type(elt) == "table" then
- return true
- end
- return false
- end, tbl)
+ function(_, elt)
+ if type(elt) == "table" then
+ return true
+ end
+ return false
+ end, tbl)
-- clear all settings
max_pri = 0
local nrules = 0
- for k in pairs(settings) do settings[k]={} end
+ for k in pairs(settings) do
+ settings[k] = {}
+ end
-- fill new settings by priority
fun.for_each(function(k, v)
local pri = get_priority(v)
- if pri > max_pri then max_pri = pri end
+ if pri > max_pri then
+ max_pri = pri
+ end
if not settings[pri] then
settings[pri] = {}
end
local s = process_setting_elt(k, v)
if s then
- table.insert(settings[pri], {name = k, rule = s})
+ table.insert(settings[pri], { name = k, rule = s })
nrules = nrules + 1
end
end, ft)
-- sort settings with equal priorities in alphabetical order
- for pri,_ in pairs(settings) do
- table.sort(settings[pri], function(a,b) return a.name < b.name end)
+ for pri, _ in pairs(settings) do
+ table.sort(settings[pri], function(a, b)
+ return a.name < b.name
+ end)
end
settings_initialized = true
local function process_settings_map(map_text)
local parser = ucl.parser()
- local res,err = parser:parse_text(map_text)
+ local res, err = parser:parse_text(map_text)
if not res then
rspamd_logger.warnx(rspamd_config, 'cannot parse settings map: ' .. err)
for _, d in ipairs(data) do
if type(d) == 'string' then
local parser = ucl.parser()
- local res,ucl_err = parser:parse_text(d)
+ local res, ucl_err = parser:parse_text(d)
if not res then
rspamd_logger.warnx(rspamd_config, 'cannot parse settings from redis: %s',
- ucl_err)
+ ucl_err)
else
local obj = parser:get_object()
rspamd_logger.infox(task, "<%1> apply settings according to redis rule %2",
- task:get_message_id(), id)
+ task:get_message_id(), id)
apply_settings(task, obj, nil, 'redis')
break
end
if type(key) == 'table' then
keys = key
else
- keys = {key}
+ keys = { key }
end
key = keys[1]
- local ret,_,_ = rspamd_redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- false, -- is write
- redis_settings_cb, --callback
- 'MGET', -- command
- keys -- arguments
+ local ret, _, _ = rspamd_redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_settings_cb, --callback
+ 'MGET', -- command
+ keys -- arguments
)
if not ret then
rspamd_logger.errx(task, 'Redis MGET failed: %s', ret)
if redis_params then
local handlers = redis_section.handlers
- for id,h in pairs(handlers) do
- local chunk,err = load(h)
+ for id, h in pairs(handlers) do
+ local chunk, err = load(h)
if not chunk then
rspamd_logger.errx(rspamd_config, 'Cannot load handler from string: %s',
tostring(err))
else
- local res,func = pcall(chunk)
+ local res, func = pcall(chunk)
if not res then
rspamd_logger.errx(rspamd_config, 'Cannot add handler from string: %s',
- tostring(func))
+ tostring(func))
else
redis_key_handlers[id] = func
end
callback = gen_redis_callback(h, id),
priority = lua_util.symbols_priorities.top,
flags = 'empty,nostat',
- augmentations = {string.format("timeout=%f", redis_params.timeout or 0.0)},
+ augmentations = { string.format("timeout=%f", redis_params.timeout or 0.0) },
})
end, redis_key_handlers)
end
-- registered BEFORE settings plugin. Otherwise, we can have inconsistent settings expressions
fun.each(function(_, elt)
if elt.register_symbols then
- for k,v in pairs(elt.register_symbols) do
+ for k, v in pairs(elt.register_symbols) do
local rtb = {
type = 'virtual',
parent = module_sym_id,
elseif type(k) == 'string' then
rtb.name = k
if type(v) == 'table' then
- for kk,vv in pairs(v) do
+ for kk, vv in pairs(v) do
-- Enrich table wih extra values
rtb[kk] = vv
end
end
if elt.apply and elt.apply.symbols then
-- Register virtual symbols
- for k,v in pairs(elt.apply.symbols) do
+ for k, v in pairs(elt.apply.symbols) do
local rtb = {
type = 'virtual',
parent = module_sym_id,
end
end
end,
- -- Include only settings, exclude all maps
+ -- Include only settings, exclude all maps
fun.filter(
function(_, elt)
if type(elt) == "table" then
end, set_section)
)
- rspamd_config:add_post_init(function ()
+ rspamd_config:add_post_init(function()
process_settings_table(set_section, true, settings_map_pool, true)
end, 100)
end
local ffi
if type(jit) == 'table' then
ffi = require("ffi")
- ffi.cdef[[
+ ffi.cdef [[
int rspamd_re_cache_type_from_string (const char *str);
int rspamd_re_cache_process_ffi (void *ptask,
void *pre,
-- Check if an re is an ordinary re
local ordinary = true
- for _,h in ipairs(hdrs) do
+ for _, h in ipairs(hdrs) do
if h == 'ALL' or h == 'ALL:raw' then
ordinary = false
cur_rule['type'] = 'function'
end
fun.each(function(func)
- if func == 'addr' then
- cur_param['function'] = function(str)
- local addr_parsed = util.parse_mail_address(str)
- local ret = {}
- if addr_parsed then
- for _,elt in ipairs(addr_parsed) do
- if elt['addr'] then
- table.insert(ret, elt['addr'])
- end
+ if func == 'addr' then
+ cur_param['function'] = function(str)
+ local addr_parsed = util.parse_mail_address(str)
+ local ret = {}
+ if addr_parsed then
+ for _, elt in ipairs(addr_parsed) do
+ if elt['addr'] then
+ table.insert(ret, elt['addr'])
end
end
-
- return ret
end
- elseif func == 'name' then
- cur_param['function'] = function(str)
- local addr_parsed = util.parse_mail_address(str)
- local ret = {}
- if addr_parsed then
- for _,elt in ipairs(addr_parsed) do
- if elt['name'] then
- table.insert(ret, elt['name'])
- end
- end
- end
- return ret
- end
- elseif func == 'raw' then
- cur_param['raw'] = true
- elseif func == 'case' then
- cur_param['strong'] = true
- else
- rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2',
- func, cur_rule['symbol'])
+ return ret
end
- end, fun.tail(args))
-
- local function split_hdr_param(param, headers)
- for _,hh in ipairs(headers) do
- local nparam = {}
- for k,v in pairs(param) do
- if k ~= 'header' then
- nparam[k] = v
+ elseif func == 'name' then
+ cur_param['function'] = function(str)
+ local addr_parsed = util.parse_mail_address(str)
+ local ret = {}
+ if addr_parsed then
+ for _, elt in ipairs(addr_parsed) do
+ if elt['name'] then
+ table.insert(ret, elt['name'])
+ end
end
end
- nparam['header'] = hh
- table.insert(hdr_params, nparam)
+ return ret
end
- end
- -- Some header rules require splitting to check of multiple headers
- if cur_param['header'] == 'MESSAGEID' then
- -- Special case for spamassassin
- ordinary = false
- split_hdr_param(cur_param, {
- 'Message-ID',
- 'X-Message-ID',
- 'Resent-Message-ID'})
- elseif cur_param['header'] == 'ToCc' then
- ordinary = false
- split_hdr_param(cur_param, { 'To', 'Cc', 'Bcc' })
+ elseif func == 'raw' then
+ cur_param['raw'] = true
+ elseif func == 'case' then
+ cur_param['strong'] = true
else
- table.insert(hdr_params, cur_param)
+ rspamd_logger.warnx(rspamd_config, 'Function %1 is not supported in %2',
+ func, cur_rule['symbol'])
end
+ end, fun.tail(args))
+
+ local function split_hdr_param(param, headers)
+ for _, hh in ipairs(headers) do
+ local nparam = {}
+ for k, v in pairs(param) do
+ if k ~= 'header' then
+ nparam[k] = v
+ end
+ end
+
+ nparam['header'] = hh
+ table.insert(hdr_params, nparam)
+ end
+ end
+ -- Some header rules require splitting to check of multiple headers
+ if cur_param['header'] == 'MESSAGEID' then
+ -- Special case for spamassassin
+ ordinary = false
+ split_hdr_param(cur_param, {
+ 'Message-ID',
+ 'X-Message-ID',
+ 'Resent-Message-ID' })
+ elseif cur_param['header'] == 'ToCc' then
+ ordinary = false
+ split_hdr_param(cur_param, { 'To', 'Cc', 'Bcc' })
+ else
+ table.insert(hdr_params, cur_param)
+ end
end
cur_rule['ordinary'] = ordinary
end
end
-
local function freemail_search(input)
local res = 0
local function trie_callback(number, pos)
local function gen_eval_rule(arg)
local eval_funcs = {
- {'check_freemail_from', function(task)
- local from = task:get_from('mime')
- if from and from[1] then
- return freemail_search(string.lower(from[1]['addr']))
- end
- return 0
- end},
- {'check_freemail_replyto',
+ { 'check_freemail_from', function(task)
+ local from = task:get_from('mime')
+ if from and from[1] then
+ return freemail_search(string.lower(from[1]['addr']))
+ end
+ return 0
+ end },
+ { 'check_freemail_replyto',
function(task)
return freemail_search(task:get_header('Reply-To'))
end
},
- {'check_freemail_header',
+ { 'check_freemail_header',
function(task, remain)
-- Remain here contains one or two args: header and regexp to match
local larg = string.match(remain, "^%(%s*['\"]([^%s]+)['\"]%s*%)$")
local h
if larg == 'EnvelopeFrom' then
h = task:get_from('smtp')
- if h then h = h[1]['addr'] end
+ if h then
+ h = h[1]['addr']
+ end
else
h = task:get_header(larg)
end
},
{
'check_for_missing_to_header',
- function (task)
+ function(task)
local th = task:get_recipients('mime')
if not th or #th == 0 then
return 1
local rh_parsed = task:get_received_headers()
local rh_cnt = 0
- if rh_mime then rh_cnt = #rh_mime end
+ if rh_mime then
+ rh_cnt = #rh_mime
+ end
local parsed_cnt = 0
- if rh_parsed then parsed_cnt = #rh_parsed end
+ if rh_parsed then
+ parsed_cnt = #rh_parsed
+ end
return rh_cnt - parsed_cnt
end
},
{
'check_for_shifted_date',
- function (task, remain)
+ function(task, remain)
-- Remain here contains two args: start and end hours shift
local matches = internal_regexp['date_shift']:search(remain, true, true)
if matches and matches[1] then
end
-- Now get the difference between Date and message received date
- local dm = task:get_date { format = 'message', gmt = true}
- local dt = task:get_date { format = 'connect', gmt = true}
+ local dm = task:get_date { format = 'message', gmt = true }
+ local dt = task:get_date { format = 'connect', gmt = true }
local diff = dm - dt
if (max_diff == 0 and diff >= min_diff) or
if larg == 'mime_attachment' then
local parts = task:get_parts()
if parts then
- for _,p in ipairs(parts) do
+ for _, p in ipairs(parts) do
if p:get_filename() then
return 1
end
function(task)
local rcpt = task:get_recipients('mime')
if rcpt then
- for _,r in ipairs(rcpt) do
+ for _, r in ipairs(rcpt) do
if sa_lists['to_blacklist'][string.lower(r['addr'])] then
return 1
end
function(task)
local rcpt = task:get_recipients('mime')
if rcpt then
- for _,r in ipairs(rcpt) do
+ for _, r in ipairs(rcpt) do
if sa_lists['to_whitelist'][string.lower(r['addr'])] then
return 1
end
function(task, remain)
local tp = task:get_text_parts()
- for _,p in ipairs(tp) do
+ for _, p in ipairs(tp) do
if p:is_html() then
local hc = p:get_html()
}
}
- for _,f in ipairs(eval_funcs) do
+ for _, f in ipairs(eval_funcs) do
local pat = string.format('^%s', f[1])
- local first,last = string.find(arg, pat)
+ local first, last = string.find(arg, pat)
if first then
local func_arg = string.sub(arg, last + 1)
arg = elts[2]
lua_util.debugm(N, rspamd_config, 'trying to parse SA function %1 with args %2',
- elts[1], elts[2])
+ elts[1], elts[2])
local substitutions = {
- {'^exists:',
- function(task) -- filter
+ { '^exists:',
+ function(task)
+ -- filter
local hdrs_check
if arg == 'MESSAGEID' then
hdrs_check = {
elseif arg == 'ToCc' then
hdrs_check = { 'To', 'Cc', 'Bcc' }
else
- hdrs_check = {arg}
+ hdrs_check = { arg }
end
- for _,h in ipairs(hdrs_check) do
+ for _, h in ipairs(hdrs_check) do
if task:has_header(h) then
return 1
end
return 0
end,
},
- {'^eval:',
+ { '^eval:',
function(task)
local func = func_cache[arg]
if not func then
if not func then
rspamd_logger.errx(task, 'cannot find appropriate eval rule for function %1',
- arg)
+ arg)
else
return func(task)
end
},
}
- for _,s in ipairs(substitutions) do
+ for _, s in ipairs(substitutions) do
if string.find(line, s[1]) then
return s[2]
end
local valid_rule = false
local function insert_cur_rule()
- if cur_rule['type'] ~= 'meta' and cur_rule['publish'] then
- -- Create meta rule from this rule
- local nsym = '__fake' .. cur_rule['symbol']
- local nrule = {
- type = 'meta',
- symbol = cur_rule['symbol'],
- score = cur_rule['score'],
- meta = nsym,
- description = cur_rule['description'],
- }
- rules[nrule['symbol']] = nrule
- cur_rule['symbol'] = nsym
- end
- -- We have previous rule valid
- if not cur_rule['symbol'] then
- rspamd_logger.errx(rspamd_config, 'bad rule definition: %1', cur_rule)
- end
- rules[cur_rule['symbol']] = cur_rule
- cur_rule = {}
- valid_rule = false
+ if cur_rule['type'] ~= 'meta' and cur_rule['publish'] then
+ -- Create meta rule from this rule
+ local nsym = '__fake' .. cur_rule['symbol']
+ local nrule = {
+ type = 'meta',
+ symbol = cur_rule['symbol'],
+ score = cur_rule['score'],
+ meta = nsym,
+ description = cur_rule['description'],
+ }
+ rules[nrule['symbol']] = nrule
+ cur_rule['symbol'] = nsym
+ end
+ -- We have previous rule valid
+ if not cur_rule['symbol'] then
+ rspamd_logger.errx(rspamd_config, 'bad rule definition: %1', cur_rule)
+ end
+ rules[cur_rule['symbol']] = cur_rule
+ cur_rule = {}
+ valid_rule = false
end
local function parse_score(words)
local skip_to_endif = false
local if_nested = 0
for l in f:lines() do
- (function ()
- l = lua_util.rspamd_str_trim(l)
- -- Replace bla=~/re/ with bla =~ /re/ (#2372)
- l = l:gsub('([^%s])%s*([=!]~)%s*([^%s])', '%1 %2 %3')
+ (function()
+ l = lua_util.rspamd_str_trim(l)
+ -- Replace bla=~/re/ with bla =~ /re/ (#2372)
+ l = l:gsub('([^%s])%s*([=!]~)%s*([^%s])', '%1 %2 %3')
- if string.len(l) == 0 or string.sub(l, 1, 1) == '#' then
- return
- end
+ if string.len(l) == 0 or string.sub(l, 1, 1) == '#' then
+ return
+ end
- -- Unbalanced if/endif
- if if_nested < 0 then if_nested = 0 end
- if skip_to_endif then
- if string.match(l, '^endif') then
- if_nested = if_nested - 1
+ -- Unbalanced if/endif
+ if if_nested < 0 then
+ if_nested = 0
+ end
+ if skip_to_endif then
+ if string.match(l, '^endif') then
+ if_nested = if_nested - 1
- if if_nested == 0 then
+ if if_nested == 0 then
+ skip_to_endif = false
+ end
+ elseif string.match(l, '^if') then
+ if_nested = if_nested + 1
+ elseif string.match(l, '^else') then
+ -- Else counterpart for if
skip_to_endif = false
end
- elseif string.match(l, '^if') then
- if_nested = if_nested + 1
- elseif string.match(l, '^else') then
- -- Else counterpart for if
- skip_to_endif = false
- end
- return
- else
- if string.match(l, '^ifplugin') then
- local ls = split(l)
+ return
+ else
+ if string.match(l, '^ifplugin') then
+ local ls = split(l)
- if not fun.any(function(pl)
- if pl == ls[2] then return true end
+ if not fun.any(function(pl)
+ if pl == ls[2] then
+ return true
+ end
+ return false
+ end, known_plugins) then
+ skip_to_endif = true
+ end
+ if_nested = if_nested + 1
+ elseif string.match(l, '^if !plugin%(') then
+ local pname = string.match(l, '^if !plugin%(([A-Za-z:]+)%)')
+ if fun.any(function(pl)
+ if pl == pname then
+ return true
+ end
return false
- end, known_plugins) then
+ end, known_plugins) then
+ skip_to_endif = true
+ end
+ if_nested = if_nested + 1
+ elseif string.match(l, '^if') then
+ -- Unknown if
+ skip_to_endif = true
+ if_nested = if_nested + 1
+ elseif string.match(l, '^else') then
+ -- Else counterpart for if
skip_to_endif = true
+ elseif string.match(l, '^endif') then
+ if_nested = if_nested - 1
end
- if_nested = if_nested + 1
- elseif string.match(l, '^if !plugin%(') then
- local pname = string.match(l, '^if !plugin%(([A-Za-z:]+)%)')
- if fun.any(function(pl)
- if pl == pname then return true end
- return false
- end, known_plugins) then
- skip_to_endif = true
- end
- if_nested = if_nested + 1
- elseif string.match(l, '^if') then
- -- Unknown if
- skip_to_endif = true
- if_nested = if_nested + 1
- elseif string.match(l, '^else') then
- -- Else counterpart for if
- skip_to_endif = true
- elseif string.match(l, '^endif') then
- if_nested = if_nested - 1
- end
- end
-
- -- Skip comments
- local words = fun.totable(fun.take_while(
- function(w) return string.sub(w, 1, 1) ~= '#' end,
- fun.filter(function(w)
- return w ~= "" end,
- fun.iter(split(l)))))
-
- if words[1] == "header" or words[1] == 'mimeheader' then
- -- header SYMBOL Header ~= /regexp/
- if valid_rule then
- insert_cur_rule()
end
- if words[4] and (words[4] == '=~' or words[4] == '!~') then
- cur_rule['type'] = 'header'
- cur_rule['symbol'] = words[2]
- if words[4] == '!~' then
- cur_rule['not'] = true
+ -- Skip comments
+ local words = fun.totable(fun.take_while(
+ function(w)
+ return string.sub(w, 1, 1) ~= '#'
+ end,
+ fun.filter(function(w)
+ return w ~= ""
+ end,
+ fun.iter(split(l)))))
+
+ if words[1] == "header" or words[1] == 'mimeheader' then
+ -- header SYMBOL Header ~= /regexp/
+ if valid_rule then
+ insert_cur_rule()
end
+ if words[4] and (words[4] == '=~' or words[4] == '!~') then
+ cur_rule['type'] = 'header'
+ cur_rule['symbol'] = words[2]
- cur_rule['re_expr'] = words_to_re(words, 4)
- local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:')
- if unset_comp then
- -- We have optional part that needs to be processed
- local unset = string.match(string.sub(cur_rule['re_expr'], unset_comp),
- '%[if%-unset:%s*([^%]%s]+)]')
- cur_rule['unset'] = unset
- -- Cut it down
- cur_rule['re_expr'] = string.sub(cur_rule['re_expr'], 1, unset_comp - 1)
- end
+ if words[4] == '!~' then
+ cur_rule['not'] = true
+ end
- cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ cur_rule['re_expr'] = words_to_re(words, 4)
+ local unset_comp = string.find(cur_rule['re_expr'], '%s+%[if%-unset:')
+ if unset_comp then
+ -- We have optional part that needs to be processed
+ local unset = string.match(string.sub(cur_rule['re_expr'], unset_comp),
+ '%[if%-unset:%s*([^%]%s]+)]')
+ cur_rule['unset'] = unset
+ -- Cut it down
+ cur_rule['re_expr'] = string.sub(cur_rule['re_expr'], 1, unset_comp - 1)
+ end
- if not cur_rule['re'] then
- rspamd_logger.warnx(rspamd_config, "Cannot parse regexp '%1' for %2",
- cur_rule['re_expr'], cur_rule['symbol'])
- else
- cur_rule['re']:set_max_hits(1)
- handle_header_def(words[3], cur_rule)
- end
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
- if cur_rule['unset'] then
- cur_rule['ordinary'] = false
- end
+ if not cur_rule['re'] then
+ rspamd_logger.warnx(rspamd_config, "Cannot parse regexp '%1' for %2",
+ cur_rule['re_expr'], cur_rule['symbol'])
+ else
+ cur_rule['re']:set_max_hits(1)
+ handle_header_def(words[3], cur_rule)
+ end
- if words[1] == 'mimeheader' then
- cur_rule['mime'] = true
- else
- cur_rule['mime'] = false
- end
+ if cur_rule['unset'] then
+ cur_rule['ordinary'] = false
+ end
- if cur_rule['re'] and cur_rule['symbol'] and
- (cur_rule['header'] or cur_rule['function']) then
- valid_rule = true
- cur_rule['re']:set_max_hits(1)
- if cur_rule['header'] and cur_rule['ordinary'] then
- for _,h in ipairs(cur_rule['header']) do
- if type(h) == 'string' then
- if cur_rule['mime'] then
- rspamd_config:register_regexp({
- re = cur_rule['re'],
- type = 'mimeheader',
- header = h,
- pcre_only = is_pcre_only(cur_rule['symbol']),
- })
- else
- rspamd_config:register_regexp({
- re = cur_rule['re'],
- type = 'header',
- header = h,
- pcre_only = is_pcre_only(cur_rule['symbol']),
- })
- end
- else
- h['mime'] = cur_rule['mime']
- if cur_rule['mime'] then
- rspamd_config:register_regexp({
- re = cur_rule['re'],
- type = 'mimeheader',
- header = h['header'],
- pcre_only = is_pcre_only(cur_rule['symbol']),
- })
- else
- if h['raw'] then
+ if words[1] == 'mimeheader' then
+ cur_rule['mime'] = true
+ else
+ cur_rule['mime'] = false
+ end
+
+ if cur_rule['re'] and cur_rule['symbol'] and
+ (cur_rule['header'] or cur_rule['function']) then
+ valid_rule = true
+ cur_rule['re']:set_max_hits(1)
+ if cur_rule['header'] and cur_rule['ordinary'] then
+ for _, h in ipairs(cur_rule['header']) do
+ if type(h) == 'string' then
+ if cur_rule['mime'] then
rspamd_config:register_regexp({
re = cur_rule['re'],
- type = 'rawheader',
- header = h['header'],
+ type = 'mimeheader',
+ header = h,
pcre_only = is_pcre_only(cur_rule['symbol']),
})
else
rspamd_config:register_regexp({
re = cur_rule['re'],
type = 'header',
+ header = h,
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ end
+ else
+ h['mime'] = cur_rule['mime']
+ if cur_rule['mime'] then
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'mimeheader',
header = h['header'],
pcre_only = is_pcre_only(cur_rule['symbol']),
})
+ else
+ if h['raw'] then
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'rawheader',
+ header = h['header'],
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ else
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'header',
+ header = h['header'],
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ end
end
end
end
+ cur_rule['re']:set_limit(match_limit)
+ cur_rule['re']:set_max_hits(1)
end
- cur_rule['re']:set_limit(match_limit)
- cur_rule['re']:set_max_hits(1)
end
- end
- else
- -- Maybe we know the function and can convert it
- local args = words_to_re(words, 2)
- local func = maybe_parse_sa_function(args)
-
- if func then
- cur_rule['type'] = 'function'
- cur_rule['symbol'] = words[2]
- cur_rule['function'] = func
- valid_rule = true
else
- rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ -- Maybe we know the function and can convert it
+ local args = words_to_re(words, 2)
+ local func = maybe_parse_sa_function(args)
+
+ if func then
+ cur_rule['type'] = 'function'
+ cur_rule['symbol'] = words[2]
+ cur_rule['function'] = func
+ valid_rule = true
+ else
+ rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ end
end
- end
- elseif words[1] == "body" then
- -- body SYMBOL /regexp/
- if valid_rule then
- insert_cur_rule()
- end
-
- cur_rule['symbol'] = words[2]
- if words[3] and (string.sub(words[3], 1, 1) == '/'
- or string.sub(words[3], 1, 1) == 'm') then
- cur_rule['type'] = 'sabody'
- cur_rule['re_expr'] = words_to_re(words, 2)
- cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
- if cur_rule['re'] then
-
- rspamd_config:register_regexp({
- re = cur_rule['re'],
- type = 'sabody',
- pcre_only = is_pcre_only(cur_rule['symbol']),
- })
- valid_rule = true
- cur_rule['re']:set_limit(match_limit)
- cur_rule['re']:set_max_hits(1)
+ elseif words[1] == "body" then
+ -- body SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
end
- else
- -- might be function
- local args = words_to_re(words, 2)
- local func = maybe_parse_sa_function(args)
- if func then
- cur_rule['type'] = 'function'
- cur_rule['symbol'] = words[2]
- cur_rule['function'] = func
- valid_rule = true
+ cur_rule['symbol'] = words[2]
+ if words[3] and (string.sub(words[3], 1, 1) == '/'
+ or string.sub(words[3], 1, 1) == 'm') then
+ cur_rule['type'] = 'sabody'
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ if cur_rule['re'] then
+
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'sabody',
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ valid_rule = true
+ cur_rule['re']:set_limit(match_limit)
+ cur_rule['re']:set_max_hits(1)
+ end
else
- rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ -- might be function
+ local args = words_to_re(words, 2)
+ local func = maybe_parse_sa_function(args)
+
+ if func then
+ cur_rule['type'] = 'function'
+ cur_rule['symbol'] = words[2]
+ cur_rule['function'] = func
+ valid_rule = true
+ else
+ rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ end
end
- end
- elseif words[1] == "rawbody" then
- -- body SYMBOL /regexp/
- if valid_rule then
- insert_cur_rule()
- end
-
- cur_rule['symbol'] = words[2]
- if words[3] and (string.sub(words[3], 1, 1) == '/'
- or string.sub(words[3], 1, 1) == 'm') then
- cur_rule['type'] = 'sarawbody'
- cur_rule['re_expr'] = words_to_re(words, 2)
- cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
- if cur_rule['re'] then
-
- rspamd_config:register_regexp({
- re = cur_rule['re'],
- type = 'sarawbody',
- pcre_only = is_pcre_only(cur_rule['symbol']),
- })
- valid_rule = true
- cur_rule['re']:set_limit(match_limit)
- cur_rule['re']:set_max_hits(1)
+ elseif words[1] == "rawbody" then
+ -- body SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
end
- else
- -- might be function
- local args = words_to_re(words, 2)
- local func = maybe_parse_sa_function(args)
- if func then
- cur_rule['type'] = 'function'
- cur_rule['symbol'] = words[2]
- cur_rule['function'] = func
- valid_rule = true
+ cur_rule['symbol'] = words[2]
+ if words[3] and (string.sub(words[3], 1, 1) == '/'
+ or string.sub(words[3], 1, 1) == 'm') then
+ cur_rule['type'] = 'sarawbody'
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ if cur_rule['re'] then
+
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'sarawbody',
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ valid_rule = true
+ cur_rule['re']:set_limit(match_limit)
+ cur_rule['re']:set_max_hits(1)
+ end
else
- rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ -- might be function
+ local args = words_to_re(words, 2)
+ local func = maybe_parse_sa_function(args)
+
+ if func then
+ cur_rule['type'] = 'function'
+ cur_rule['symbol'] = words[2]
+ cur_rule['function'] = func
+ valid_rule = true
+ else
+ rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ end
+ end
+ elseif words[1] == "full" then
+ -- body SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
end
- end
- elseif words[1] == "full" then
- -- body SYMBOL /regexp/
- if valid_rule then
- insert_cur_rule()
- end
- cur_rule['symbol'] = words[2]
+ cur_rule['symbol'] = words[2]
- if words[3] and (string.sub(words[3], 1, 1) == '/'
- or string.sub(words[3], 1, 1) == 'm') then
- cur_rule['type'] = 'message'
+ if words[3] and (string.sub(words[3], 1, 1) == '/'
+ or string.sub(words[3], 1, 1) == 'm') then
+ cur_rule['type'] = 'message'
+ cur_rule['re_expr'] = words_to_re(words, 2)
+ cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
+ cur_rule['raw'] = true
+ if cur_rule['re'] then
+ valid_rule = true
+ rspamd_config:register_regexp({
+ re = cur_rule['re'],
+ type = 'body',
+ pcre_only = is_pcre_only(cur_rule['symbol']),
+ })
+ cur_rule['re']:set_limit(match_limit)
+ cur_rule['re']:set_max_hits(1)
+ end
+ else
+ -- might be function
+ local args = words_to_re(words, 2)
+ local func = maybe_parse_sa_function(args)
+
+ if func then
+ cur_rule['type'] = 'function'
+ cur_rule['symbol'] = words[2]
+ cur_rule['function'] = func
+ valid_rule = true
+ else
+ rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ end
+ end
+ elseif words[1] == "uri" then
+ -- uri SYMBOL /regexp/
+ if valid_rule then
+ insert_cur_rule()
+ end
+ cur_rule['type'] = 'uri'
+ cur_rule['symbol'] = words[2]
cur_rule['re_expr'] = words_to_re(words, 2)
cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
- cur_rule['raw'] = true
- if cur_rule['re'] then
+ if cur_rule['re'] and cur_rule['symbol'] then
valid_rule = true
rspamd_config:register_regexp({
re = cur_rule['re'],
- type = 'body',
+ type = 'url',
pcre_only = is_pcre_only(cur_rule['symbol']),
})
cur_rule['re']:set_limit(match_limit)
cur_rule['re']:set_max_hits(1)
end
- else
- -- might be function
- local args = words_to_re(words, 2)
- local func = maybe_parse_sa_function(args)
-
- if func then
- cur_rule['type'] = 'function'
- cur_rule['symbol'] = words[2]
- cur_rule['function'] = func
- valid_rule = true
- else
- rspamd_logger.infox(rspamd_config, 'unknown function %1', args)
+ elseif words[1] == "meta" then
+ -- meta SYMBOL expression
+ if valid_rule then
+ insert_cur_rule()
end
- end
- elseif words[1] == "uri" then
- -- uri SYMBOL /regexp/
- if valid_rule then
- insert_cur_rule()
- end
- cur_rule['type'] = 'uri'
- cur_rule['symbol'] = words[2]
- cur_rule['re_expr'] = words_to_re(words, 2)
- cur_rule['re'] = rspamd_regexp.create(cur_rule['re_expr'])
- if cur_rule['re'] and cur_rule['symbol'] then
- valid_rule = true
- rspamd_config:register_regexp({
- re = cur_rule['re'],
- type = 'url',
- pcre_only = is_pcre_only(cur_rule['symbol']),
- })
- cur_rule['re']:set_limit(match_limit)
- cur_rule['re']:set_max_hits(1)
- end
- elseif words[1] == "meta" then
- -- meta SYMBOL expression
- if valid_rule then
- insert_cur_rule()
- end
- cur_rule['type'] = 'meta'
- cur_rule['symbol'] = words[2]
- cur_rule['meta'] = words_to_re(words, 2)
- if cur_rule['meta'] and cur_rule['symbol']
- and cur_rule['meta'] ~= '0' then
+ cur_rule['type'] = 'meta'
+ cur_rule['symbol'] = words[2]
+ cur_rule['meta'] = words_to_re(words, 2)
+ if cur_rule['meta'] and cur_rule['symbol']
+ and cur_rule['meta'] ~= '0' then
valid_rule = true
- end
- elseif words[1] == "describe" and valid_rule then
- cur_rule['description'] = words_to_re(words, 2)
- elseif words[1] == "score" then
- scores[words[2]] = parse_score(words)
- elseif words[1] == 'freemail_domains' then
- fun.each(function(dom)
+ end
+ elseif words[1] == "describe" and valid_rule then
+ cur_rule['description'] = words_to_re(words, 2)
+ elseif words[1] == "score" then
+ scores[words[2]] = parse_score(words)
+ elseif words[1] == 'freemail_domains' then
+ fun.each(function(dom)
table.insert(freemail_domains, '@' .. dom)
end, fun.drop_n(1, words))
- elseif words[1] == 'blacklist_from' then
- sa_lists['from_blacklist'][words[2]] = 1
- sa_lists['elts'] = sa_lists['elts'] + 1
- elseif words[1] == 'whitelist_from' then
- sa_lists['from_whitelist'][words[2]] = 1
- sa_lists['elts'] = sa_lists['elts'] + 1
- elseif words[1] == 'whitelist_to' then
- sa_lists['to_whitelist'][words[2]] = 1
- sa_lists['elts'] = sa_lists['elts'] + 1
- elseif words[1] == 'blacklist_to' then
- sa_lists['to_blacklist'][words[2]] = 1
- sa_lists['elts'] = sa_lists['elts'] + 1
- elseif words[1] == 'tflags' then
- process_tflags(cur_rule, words)
- elseif words[1] == 'replace_tag' then
- process_replace(words, replace['tags'])
- elseif words[1] == 'replace_pre' then
- process_replace(words, replace['pre'])
- elseif words[1] == 'replace_inter' then
- process_replace(words, replace['inter'])
- elseif words[1] == 'replace_post' then
- process_replace(words, replace['post'])
- elseif words[1] == 'replace_rules' then
- fun.each(function(r) table.insert(replace['rules'], r) end,
- fun.drop_n(1, words))
- end
+ elseif words[1] == 'blacklist_from' then
+ sa_lists['from_blacklist'][words[2]] = 1
+ sa_lists['elts'] = sa_lists['elts'] + 1
+ elseif words[1] == 'whitelist_from' then
+ sa_lists['from_whitelist'][words[2]] = 1
+ sa_lists['elts'] = sa_lists['elts'] + 1
+ elseif words[1] == 'whitelist_to' then
+ sa_lists['to_whitelist'][words[2]] = 1
+ sa_lists['elts'] = sa_lists['elts'] + 1
+ elseif words[1] == 'blacklist_to' then
+ sa_lists['to_blacklist'][words[2]] = 1
+ sa_lists['elts'] = sa_lists['elts'] + 1
+ elseif words[1] == 'tflags' then
+ process_tflags(cur_rule, words)
+ elseif words[1] == 'replace_tag' then
+ process_replace(words, replace['tags'])
+ elseif words[1] == 'replace_pre' then
+ process_replace(words, replace['pre'])
+ elseif words[1] == 'replace_inter' then
+ process_replace(words, replace['inter'])
+ elseif words[1] == 'replace_post' then
+ process_replace(words, replace['post'])
+ elseif words[1] == 'replace_rules' then
+ fun.each(function(r)
+ table.insert(replace['rules'], r)
+ end,
+ fun.drop_n(1, words))
+ end
end)()
end
if valid_rule then
-- Now check all valid rules and add the according rspamd rules
local function calculate_score(sym, rule)
- if fun.all(function(c) return c == '_' end, fun.take_n(2, fun.iter(sym))) then
+ if fun.all(function(c)
+ return c == '_'
+ end, fun.take_n(2, fun.iter(sym))) then
return 0.0
end
end
res = res + re:matchn(data, lim, raw)
else
- if re:match(data, raw) then res = 1 end
+ if re:match(data, raw) then
+ res = 1
+ end
end
return res
local replacement = nil
local ret = s
fun.each(function(n, t)
- local ns,matches = string.gsub(s, string.format("<%s%s>", prefix, n), "")
+ local ns, matches = string.gsub(s, string.format("<%s%s>", prefix, n), "")
if matches > 0 then
replacement = t
ret = ns
end
end, tbl)
- return ret,replacement
+ return ret, replacement
end
local repl
- str,repl = check_specific_tag("pre ", str, replace['pre'])
+ str, repl = check_specific_tag("pre ", str, replace['pre'])
if repl then
pre = repl
end
- str,repl = check_specific_tag("inter ", str, replace['inter'])
+ str, repl = check_specific_tag("inter ", str, replace['inter'])
if repl then
inter = repl
end
- str,repl = check_specific_tag("post ", str, replace['post'])
+ str, repl = check_specific_tag("post ", str, replace['post'])
if repl then
post = repl
end
local s = replace_all_tags(str)
-
if str ~= s then
- return true,s
+ return true, s
end
- return false,str
+ return false, str
end
local function parse_atom(str)
end
local function gen_process_atom_cb(result_name, task)
- return function (atom)
+ return function(atom)
local atom_cb = atoms[atom]
if atom_cb then
-- Replace rule tags
local ntags = {}
local function rec_replace_tags(tag, tagv)
- if ntags[tag] then return ntags[tag] end
+ if ntags[tag] then
+ return ntags[tag]
+ end
fun.each(function(n, t)
if n ~= tag then
local s, matches = string.gsub(tagv, string.format("<%s>", n), t)
end
end, replace['tags'])
- if not ntags[tag] then ntags[tag] = tagv end
+ if not ntags[tag] then
+ ntags[tag] = tagv
+ end
return ntags[tag]
end
if not r['re'] then
rspamd_logger.errx(task, 're is missing for rule %1 (%2 header)', k,
- h['header'])
+ h['header'])
return 0
end
else
str = rh['decoded']
end
- if not str then return 0 end
+ if not str then
+ return 0
+ end
if h['function'] then
str = h['function'](str)
end, r['header'])
if #check == 0 then
- if r['not'] then return 1 end
+ if r['not'] then
+ return 1
+ end
return 0
end
end
atoms[k] = f
end,
- fun.filter(function(_, r)
- return r['type'] == 'header' and r['header']
- end,
- rules))
+ fun.filter(function(_, r)
+ return r['type'] == 'header' and r['header']
+ end,
+ rules))
-- Custom function rules
fun.each(function(k, r)
end
atoms[k] = f
end,
- fun.filter(function(_, r)
- return r['type'] == 'function' and r['function']
- end,
- rules))
+ fun.filter(function(_, r)
+ return r['type'] == 'function' and r['function']
+ end,
+ rules))
-- Parts rules
fun.each(function(k, r)
end
local t = 'mime'
- if r['raw'] then t = 'rawmime' end
+ if r['raw'] then
+ t = 'rawmime'
+ end
return process_regexp_opt(r.re, task, t)
end
end
atoms[k] = f
end,
- fun.filter(function(_, r)
- return r['type'] == 'part'
- end, rules))
+ fun.filter(function(_, r)
+ return r['type'] == 'part'
+ end, rules))
-- SA body rules
fun.each(function(k, r)
end
atoms[k] = f
end,
- fun.filter(function(_, r)
- return r['type'] == 'sabody' or r['type'] == 'message' or r['type'] == 'sarawbody'
- end, rules))
+ fun.filter(function(_, r)
+ return r['type'] == 'sabody' or r['type'] == 'message' or r['type'] == 'sarawbody'
+ end, rules))
-- URL rules
fun.each(function(k, r)
end
atoms[k] = f
end,
- fun.filter(function(_, r)
- return r['type'] == 'uri'
- end,
- rules))
+ fun.filter(function(_, r)
+ return r['type'] == 'uri'
+ end,
+ rules))
-- Meta rules
fun.each(function(k, r)
- local expression = nil
- -- Meta function callback
- -- Here are dragons!
- -- This function can be called from 2 DIFFERENT type of invocations:
- -- 1) Invocation from Rspamd itself where `res_name` will be nil
- -- 2) Invocation from other meta during expression:process_traced call
- -- So we need to distinguish that and return different stuff to be able to deal with atoms
- local meta_cb = function(task, res_name)
- lua_util.debugm(N, task, 'meta callback for %s; result name: %s', k, res_name)
- local cached = task:cache_get('sa_metas_processed')
-
- -- We avoid many task methods invocations here (likely)
- if not cached then
- cached = {}
- task:cache_set('sa_metas_processed', cached)
- end
-
- local already_processed = cached[k]
-
- -- Exclude elements that are named in the same way as the symbol itself
- local function exclude_sym_filter(sopt)
- return sopt ~= k
- end
+ local expression = nil
+ -- Meta function callback
+ -- Here are dragons!
+ -- This function can be called from 2 DIFFERENT type of invocations:
+ -- 1) Invocation from Rspamd itself where `res_name` will be nil
+ -- 2) Invocation from other meta during expression:process_traced call
+ -- So we need to distinguish that and return different stuff to be able to deal with atoms
+ local meta_cb = function(task, res_name)
+ lua_util.debugm(N, task, 'meta callback for %s; result name: %s', k, res_name)
+ local cached = task:cache_get('sa_metas_processed')
+
+ -- We avoid many task methods invocations here (likely)
+ if not cached then
+ cached = {}
+ task:cache_set('sa_metas_processed', cached)
+ end
- if not (already_processed and already_processed[res_name or 'default']) then
- -- Execute symbol
- local function exec_symbol(cur_res)
- local res,trace = expression:process_traced(gen_process_atom_cb(cur_res, task))
- lua_util.debugm(N, task, 'meta result for %s: %s; result name: %s', k, res, cur_res)
- if res > 0 then
- -- Symbol should be one shot to make it working properly
- task:insert_result_named(cur_res, k, res, fun.totable(fun.filter(exclude_sym_filter, trace)))
- end
+ local already_processed = cached[k]
- if not cached[k] then
- cached[k] = {}
- end
+ -- Exclude elements that are named in the same way as the symbol itself
+ local function exclude_sym_filter(sopt)
+ return sopt ~= k
+ end
- cached[k][cur_res] = res
+ if not (already_processed and already_processed[res_name or 'default']) then
+ -- Execute symbol
+ local function exec_symbol(cur_res)
+ local res, trace = expression:process_traced(gen_process_atom_cb(cur_res, task))
+ lua_util.debugm(N, task, 'meta result for %s: %s; result name: %s', k, res, cur_res)
+ if res > 0 then
+ -- Symbol should be one shot to make it working properly
+ task:insert_result_named(cur_res, k, res, fun.totable(fun.filter(exclude_sym_filter, trace)))
end
- if not res_name then
- -- Invoke for all named results
- local named_results = task:get_all_named_results()
- for _,cur_res in ipairs(named_results) do
- exec_symbol(cur_res)
- end
- else
- -- Invoked from another meta
- exec_symbol(res_name)
- return cached[k][res_name] or 0
+ if not cached[k] then
+ cached[k] = {}
end
- else
- -- We have cached the result
- local res = already_processed[res_name or 'default'] or 0
- lua_util.debugm(N, task, 'cached meta result for %s: %s; result name: %s',
- k, res, res_name)
- if res_name then
- return res
+ cached[k][cur_res] = res
+ end
+
+ if not res_name then
+ -- Invoke for all named results
+ local named_results = task:get_all_named_results()
+ for _, cur_res in ipairs(named_results) do
+ exec_symbol(cur_res)
end
+ else
+ -- Invoked from another meta
+ exec_symbol(res_name)
+ return cached[k][res_name] or 0
end
+ else
+ -- We have cached the result
+ local res = already_processed[res_name or 'default'] or 0
+ lua_util.debugm(N, task, 'cached meta result for %s: %s; result name: %s',
+ k, res, res_name)
- -- No return if invoked directly from Rspamd as we use task:insert_result_named directly
+ if res_name then
+ return res
+ end
end
- expression = rspamd_expression.create(r['meta'], parse_atom, rspamd_config:get_mempool())
- if not expression then
- rspamd_logger.errx(rspamd_config, 'Cannot parse expression ' .. r['meta'])
- else
+ -- No return if invoked directly from Rspamd as we use task:insert_result_named directly
+ end
- if r['score'] then
- rspamd_config:set_metric_symbol{
- name = k, score = r['score'],
- description = r['description'],
- priority = scores_priority,
- one_shot = true
- }
- scores_added[k] = 1
- rspamd_config:register_symbol{
- name = k,
- weight = calculate_score(k, r),
- callback = meta_cb
- }
- else
- -- Add 0 score to avoid issues
- rspamd_config:register_symbol{
- name = k,
- weight = calculate_score(k, r),
- callback = meta_cb,
- score = 0,
- }
- end
+ expression = rspamd_expression.create(r['meta'], parse_atom, rspamd_config:get_mempool())
+ if not expression then
+ rspamd_logger.errx(rspamd_config, 'Cannot parse expression ' .. r['meta'])
+ else
- r['expression'] = expression
+ if r['score'] then
+ rspamd_config:set_metric_symbol {
+ name = k, score = r['score'],
+ description = r['description'],
+ priority = scores_priority,
+ one_shot = true
+ }
+ scores_added[k] = 1
+ rspamd_config:register_symbol {
+ name = k,
+ weight = calculate_score(k, r),
+ callback = meta_cb
+ }
+ else
+ -- Add 0 score to avoid issues
+ rspamd_config:register_symbol {
+ name = k,
+ weight = calculate_score(k, r),
+ callback = meta_cb,
+ score = 0,
+ }
+ end
- if not atoms[k] then
- atoms[k] = meta_cb
- end
+ r['expression'] = expression
+
+ if not atoms[k] then
+ atoms[k] = meta_cb
end
- end,
- fun.filter(function(_, r)
+ end
+ end,
+ fun.filter(function(_, r)
return r['type'] == 'meta'
end,
- rules))
+ rules))
-- Check meta rules for foreign symbols and register dependencies
-- First direct dependencies:
fun.each(function(k, r)
- if r['expression'] then
- local expr_atoms = r['expression']:atoms()
-
- for _,a in ipairs(expr_atoms) do
- if not atoms[a] then
- local rspamd_symbol = replace_symbol(a)
- if not external_deps[k] then
- external_deps[k] = {}
- end
+ if r['expression'] then
+ local expr_atoms = r['expression']:atoms()
+
+ for _, a in ipairs(expr_atoms) do
+ if not atoms[a] then
+ local rspamd_symbol = replace_symbol(a)
+ if not external_deps[k] then
+ external_deps[k] = {}
+ end
- if not external_deps[k][rspamd_symbol] then
- rspamd_config:register_dependency(k, rspamd_symbol)
- external_deps[k][rspamd_symbol] = true
- lua_util.debugm(N, rspamd_config,
+ if not external_deps[k][rspamd_symbol] then
+ rspamd_config:register_dependency(k, rspamd_symbol)
+ external_deps[k][rspamd_symbol] = true
+ lua_util.debugm(N, rspamd_config,
'atom %1 is a direct foreign dependency, ' ..
- 'register dependency for %2 on %3',
+ 'register dependency for %2 on %3',
a, k, rspamd_symbol)
- end
end
end
end
- end,
- fun.filter(function(_, r)
- return r['type'] == 'meta'
- end,
- rules))
+ end
+ end,
+ fun.filter(function(_, r)
+ return r['type'] == 'meta'
+ end,
+ rules))
-- ... And then indirect ones ...
local nchanges
repeat
- nchanges = 0
+ nchanges = 0
fun.each(function(k, r)
if r['expression'] then
local expr_atoms = r['expression']:atoms()
- for _,a in ipairs(expr_atoms) do
+ for _, a in ipairs(expr_atoms) do
if type(external_deps[a]) == 'table' then
for dep in pairs(external_deps[a]) do
if not external_deps[k] then
rspamd_config:register_dependency(k, dep)
external_deps[k][dep] = true
lua_util.debugm(N, rspamd_config,
- 'atom %1 is an indirect foreign dependency, ' ..
- 'register dependency for %2 on %3',
- a, k, dep)
- nchanges = nchanges + 1
+ 'atom %1 is an indirect foreign dependency, ' ..
+ 'register dependency for %2 on %3',
+ a, k, dep)
+ nchanges = nchanges + 1
end
end
else
local rspamd_symbol, replaced_symbol = replace_symbol(a)
if replaced_symbol then
- external_deps[a] = {[rspamd_symbol] = true}
+ external_deps[a] = { [rspamd_symbol] = true }
else
external_deps[a] = {}
end
end
end
end,
- fun.filter(function(_, r)
- return r['type'] == 'meta'
- end,
- rules))
+ fun.filter(function(_, r)
+ return r['type'] == 'meta'
+ end,
+ rules))
until nchanges == 0
-- Set missing symbols
fun.each(function(key, score)
if not scores_added[key] then
rspamd_config:set_metric_symbol({
- name = key, score = score,
- priority = 2, flags = 'ignore'})
+ name = key, score = score,
+ priority = 2, flags = 'ignore' })
end
end, scores)
if freemail_domains then
freemail_trie = rspamd_trie.create(freemail_domains)
rspamd_logger.infox(rspamd_config, 'loaded %1 freemail domains definitions',
- #freemail_domains)
+ #freemail_domains)
end
rspamd_logger.infox(rspamd_config, 'loaded %1 blacklist/whitelist elements',
sa_lists['elts'])
if type(section) == "table" then
local keywords = {
- pcre_only = {'table', function(v) pcre_only_regexps = lua_util.list_to_hash(v) end},
- alpha = {'number', function(v) meta_score_alpha = tonumber(v) end},
- match_limit = {'number', function(v) match_limit = tonumber(v) end},
- scores_priority = {'number', function(v) scores_priority = tonumber(v) end},
+ pcre_only = { 'table', function(v)
+ pcre_only_regexps = lua_util.list_to_hash(v)
+ end },
+ alpha = { 'number', function(v)
+ meta_score_alpha = tonumber(v)
+ end },
+ match_limit = { 'number', function(v)
+ match_limit = tonumber(v)
+ end },
+ scores_priority = { 'number', function(v)
+ scores_priority = tonumber(v)
+ end },
}
for k, fn in pairs(section) do
if not files or #files == 0 then
rspamd_logger.errx(rspamd_config, "cannot find any files matching pattern %s", elt)
else
- for _,matched in ipairs(files) do
+ for _, matched in ipairs(files) do
local f = io.open(matched, "r")
if f then
rspamd_logger.infox(rspamd_config, 'loading SA rules from %s', matched)
if not files or #files == 0 then
rspamd_logger.errx(rspamd_config, "cannot find any files matching pattern %s", fn)
else
- for _,matched in ipairs(files) do
+ for _, matched in ipairs(files) do
local f = io.open(matched, "r")
if f then
rspamd_logger.infox(rspamd_config, 'loading SA rules from %s', matched)
local function do_action(rcpt)
if settings['learn_fuzzy'] then
rspamd_plugins.fuzzy_check.learn(task,
- settings['fuzzy_flag'],
- settings['fuzzy_weight'])
+ settings['fuzzy_flag'],
+ settings['fuzzy_weight'])
end
local act_flags = ''
if settings['learn_spam'] then
rspamd_logger.infox(task, 'spamtrap found: <%s>', rcpt)
local smtp_message
if settings.smtp_message then
- smtp_message = lua_util.template(settings.smtp_message, { rcpt = rcpt})
+ smtp_message = lua_util.template(settings.smtp_message, { rcpt = rcpt })
else
smtp_message = 'unknown error'
if settings.action == 'no action' then
smtp_message = 'message rejected'
end
end
- task:set_pre_result{action = settings.action,
- message = smtp_message,
- module = 'spamtrap',
- flags = act_flags}
+ task:set_pre_result { action = settings.action,
+ message = smtp_message,
+ module = 'spamtrap',
+ flags = act_flags }
end
return true
false, -- is write
gen_redis_spamtrap_cb(target), -- callback
'GET', -- command
- {key} -- arguments
+ { key } -- arguments
)
if not ret then
rspamd_logger.errx(task, "redis request wasn't scheduled")
end
-- Do not risk a FP by checking for more than one recipient
- if rcpts and (#rcpts == 1 or (#rcpts > 0 and settings.allow_multiple_rcpts)) then
- local targets = fun.map(function(r) return r['addr']:lower() end, rcpts)
+ if rcpts and (#rcpts == 1 or (#rcpts > 0 and settings.allow_multiple_rcpts)) then
+ local targets = fun.map(function(r)
+ return r['addr']:lower()
+ end, rcpts)
if use_redis then
fun.each(function(target)
local key = settings['key_prefix'] .. target
false, -- is write
gen_redis_spamtrap_cb(target), -- callback
'GET', -- command
- {key} -- arguments
+ { key } -- arguments
)
if not ret then
rspamd_logger.errx(task, "redis request wasn't scheduled")
return
end
-
local auth_and_local_conf = lua_util.config_check_local_or_authed(rspamd_config, 'spamtrap',
false, false)
check_local = auth_and_local_conf[1]
check_authed = auth_and_local_conf[2]
if opts then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
settings[k] = v
end
if settings['map'] then
- settings['map'] = rspamd_config:add_map{
+ settings['map'] = rspamd_config:add_map {
url = settings['map'],
description = string.format("Spamtrap map for %s", settings['symbol']),
type = "regexp"
redis_params = rspamd_parse_redis_server('spamtrap')
if not redis_params then
rspamd_logger.errx(
- rspamd_config, 'no redis servers are specified, disabling module')
+ rspamd_config, 'no redis servers are specified, disabling module')
return
end
use_redis = true;
local rh = task:get_received_headers() or {}
local found = false
- for i,hdr in ipairs(rh) do
+ for i, hdr in ipairs(rh) do
if hdr.real_ip and local_config.external_relay:get_key(hdr.real_ip) then
-- We can use the next header as a source of IP address
if rh[i + 1] then
local function policy_decode(res)
if res == rspamd_spf.policy.fail then
- return local_config.symbols.fail,'-'
+ return local_config.symbols.fail, '-'
elseif res == rspamd_spf.policy.pass then
- return local_config.symbols.allow,'+'
+ return local_config.symbols.allow, '+'
elseif res == rspamd_spf.policy.soft_fail then
- return local_config.symbols.softfail,'~'
+ return local_config.symbols.softfail, '~'
elseif res == rspamd_spf.policy.neutral then
- return local_config.symbols.neutral,'?'
+ return local_config.symbols.neutral, '?'
end
- return 'SPF_UNKNOWN','?'
+ return 'SPF_UNKNOWN', '?'
end
local function spf_resolved_cb(record, flags, err)
ip, flags, err, error_or_addr)
if result then
- local sym,code = policy_decode(flag_or_policy)
+ local sym, code = policy_decode(flag_or_policy)
local opt = string.format('%s%s', code, error_or_addr.str or '???')
if bit.band(flags, rspamd_spf.flags.cached) ~= 0 then
opt = opt .. ':c'
-- Register all symbols and init rspamd_spf library
rspamd_spf.config(local_config)
-local sym_id = rspamd_config:register_symbol{
+local sym_id = rspamd_config:register_symbol {
name = 'SPF_CHECK',
type = 'callback',
flags = 'fine,empty',
- groups = {'policies','spf'},
+ groups = { 'policies', 'spf' },
score = 0.0,
callback = spf_check_callback,
-- We can merely estimate timeout here, as it is possible to construct an SPF record that would cause
-- many DNS requests. But we won't like to set the maximum value for that all the time, as
-- the majority of requests will typically have 1-4 subrequests
- augmentations = {string.format("timeout=%f", rspamd_config:get_dns_timeout() * 4 or 0.0)},
+ augmentations = { string.format("timeout=%f", rspamd_config:get_dns_timeout() * 4 or 0.0) },
}
if local_config.whitelist then
local lua_maps = require "lua_maps"
local_config.external_relay = lua_maps.map_add_from_ucl(local_config.external_relay,
- "radix", "External IP SPF map")
+ "radix", "External IP SPF map")
end
-for _,sym in pairs(local_config.symbols) do
- rspamd_config:register_symbol{
+for _, sym in pairs(local_config.symbols) do
+ rspamd_config:register_symbol {
name = sym,
type = 'virtual',
parent = sym_id,
- groups = {'policies', 'spf'},
+ groups = { 'policies', 'spf' },
}
end
params = body_params
end
- return function (idx, pos)
+ return function(idx, pos)
local param = params[idx]
local pattern = patterns[idx]
local pattern_idx = pattern .. tostring(idx) .. type
if param['multi'] or not matched[pattern_idx] then
lua_util.debugm(N, task, "<%1> matched pattern %2 at pos %3",
- task:get_message_id(), pattern, pos)
+ task:get_message_id(), pattern, pos)
task:insert_result(param['symbol'], 1.0, type)
if not param['multi'] then
matched[pattern_idx] = true
local function process_single_pattern(pat, symbol, cf)
if pat then
local multi = false
- if cf['multi'] then multi = true end
+ if cf['multi'] then
+ multi = true
+ end
if cf['raw'] then
table.insert(raw_patterns, pat)
- table.insert(raw_params, {symbol=symbol, multi=multi})
+ table.insert(raw_params, { symbol = symbol, multi = multi })
elseif cf['body'] then
table.insert(body_patterns, pat)
- table.insert(body_params, {symbol=symbol, multi=multi})
+ table.insert(body_params, { symbol = symbol, multi = multi })
else
table.insert(mime_patterns, pat)
- table.insert(mime_params, {symbol=symbol, multi=multi})
+ table.insert(mime_params, { symbol = symbol, multi = multi })
end
end
end
else
if cf['binary'] then
rspamd_logger.errx(rspamd_config, 'binary trie patterns are not implemented yet: %1',
- cf['file'])
+ cf['file'])
else
for line in file:lines() do
local pat = string.match(line, '^([^#].*[^%s])%s*$')
local function process_trie_conf(symbol, cf)
if type(cf) ~= 'table' then
rspamd_logger.errx(rspamd_config, 'invalid value for symbol %1: "%2", expected table',
- symbol, cf)
+ symbol, cf)
return
end
end
end
-local opts = rspamd_config:get_all_opt("trie")
+local opts = rspamd_config:get_all_opt("trie")
if opts then
for sym, opt in pairs(opts) do
- process_trie_conf(sym, opt)
+ process_trie_conf(sym, opt)
end
if #raw_patterns > 0 then
raw_trie = rspamd_trie.create(raw_patterns)
rspamd_logger.infox(rspamd_config, 'registered raw search trie from %1 patterns', #raw_patterns)
- end
+ end
if #mime_patterns > 0 then
mime_trie = rspamd_trie.create(mime_patterns)
local function adjust_url(task, orig_url, redir_url)
if type(redir_url) == 'string' then
- redir_url = rspamd_url.create(task:get_mempool(), redir_url, {'redirect_target'})
+ redir_url = rspamd_url.create(task:get_mempool(), redir_url, { 'redirect_target' })
end
if redir_url then
rspamd_logger.errx(task, 'got error while getting top urls count: %s', err)
else
rspamd_logger.infox(task, 'trimmed url set to %s elements',
- settings.top_urls_count)
+ settings.top_urls_count)
end
end
if data then
if tonumber(data) > settings.top_urls_count * 2 then
local ret = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- true, -- is write
- redis_trim_cb, --callback
- 'ZREMRANGEBYRANK', -- command
- {settings.top_urls_key, '0',
- tostring(-(settings.top_urls_count + 1))} -- arguments
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_trim_cb, --callback
+ 'ZREMRANGEBYRANK', -- command
+ { settings.top_urls_key, '0',
+ tostring(-(settings.top_urls_count + 1)) } -- arguments
)
if not ret then
rspamd_logger.errx(task, 'cannot trim top urls set')
else
rspamd_logger.infox(task, 'need to trim urls set from %s to %s elements',
- data,
- settings.top_urls_count)
+ data,
+ settings.top_urls_count)
return
end
end
rspamd_logger.errx(task, 'got error while setting redirect keys: %s', err)
else
local ret = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- false, -- is write
- redis_card_cb, --callback
- 'ZCARD', -- command
- {settings.top_urls_key} -- arguments
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_card_cb, --callback
+ 'ZCARD', -- command
+ { settings.top_urls_key } -- arguments
)
if not ret then
rspamd_logger.errx(task, 'cannot make redis request to cache results')
-- Save url with prefix
str_url = string.format('^%s:%s', prefix, str_url)
end
- local ret,conn,_ = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- true, -- is write
- redis_set_cb, --callback
- 'SETEX', -- command
- {key, tostring(settings.expire), str_url} -- arguments
+ local ret, conn, _ = lua_redis.redis_make_request(task,
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_set_cb, --callback
+ 'SETEX', -- command
+ { key, tostring(settings.expire), str_url } -- arguments
)
if not ret then
rspamd_logger.errx(task, 'cannot make redis request to cache results')
else
- conn:add_cmd('ZINCRBY', {settings.top_urls_key, '1', str_url})
+ conn:add_cmd('ZINCRBY', { settings.top_urls_key, '1', str_url })
end
end
-- Reduce length of a string to a given length (16 by default)
local function maybe_trim_url(url, limit)
- if not limit then limit = 16 end
+ if not limit then
+ limit = 16
+ end
if #url > limit then
return string.sub(url, 1, limit) .. '...'
else
if ntries > settings.nested_limit then
-- We cannot resolve more, stop
rspamd_logger.debugm(N, task, 'cannot get more requests to resolve %s, stop on %s after %s attempts',
- orig_url, url, ntries)
+ orig_url, url, ntries)
cache_url(task, orig_url, url, key, 'nested')
task:insert_result(settings.redirector_symbol_nested, 1.0,
string.format('%s->%s:%d', maybe_trim_url(orig_url), maybe_trim_url(url), ntries))
local function http_callback(err, code, _, headers)
if err then
rspamd_logger.infox(task, 'found redirect error from %s to %s, err message: %s',
- orig_url, url, err)
+ orig_url, url, err)
cache_url(task, orig_url, url, key)
else
if code == 200 then
if orig_url == url then
rspamd_logger.infox(task, 'direct url %s, err code 200',
- url)
+ url)
else
rspamd_logger.infox(task, 'found redirect from %s to %s, err code 200',
- orig_url, url)
+ orig_url, url)
end
cache_url(task, orig_url, url, key)
redir_url = rspamd_url.create(task:get_mempool(), loc)
end
rspamd_logger.debugm(N, task, 'found redirect from %s to %s, err code %s',
- orig_url, loc, code)
+ orig_url, loc, code)
if redir_url then
if settings.redirectors_only then
resolve_cached(task, orig_url, redir_url, key, ntries + 1)
else
lua_util.debugm(N, task,
- "stop resolving redirects as %s is not a redirector", loc)
+ "stop resolving redirects as %s is not a redirector", loc)
cache_url(task, orig_url, redir_url, key)
end
else
end
else
rspamd_logger.debugm(N, task, 'found redirect error from %s to %s, err code: %s',
- orig_url, url, code)
+ orig_url, url, code)
cache_url(task, orig_url, url, key)
end
end
lua_util.debugm(N, task, 'select user agent %s', ua)
- rspamd_http.request{
+ rspamd_http.request {
headers = {
['User-Agent'] = ua,
},
if data ~= 'processing' then
-- Got cached result
rspamd_logger.debugm(N, task, 'found cached redirect from %s to %s',
- url, data)
+ url, data)
if data.sub(1, 1) == '^' then
-- Prefixed url stored
local prefix, new_url = data:match('^%^(%a+):(.+)$')
if ntries == 1 then
-- Reserve key in Redis that we are processing this redirection
local ret = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- true, -- is write
- redis_reserve_cb, --callback
- 'SET', -- command
- {key, 'processing', 'EX', tostring(settings.timeout * 2), 'NX'} -- arguments
+ redis_params, -- connect params
+ key, -- hash key
+ true, -- is write
+ redis_reserve_cb, --callback
+ 'SET', -- command
+ { key, 'processing', 'EX', tostring(settings.timeout * 2), 'NX' } -- arguments
)
if not ret then
rspamd_logger.errx(task, 'Couldn\'t schedule SET')
end
local ret = lua_redis.redis_make_request(task,
- redis_params, -- connect params
- key, -- hash key
- false, -- is write
- redis_get_cb, --callback
- 'GET', -- command
- {key} -- arguments
+ redis_params, -- connect params
+ key, -- hash key
+ false, -- is write
+ redis_get_cb, --callback
+ 'GET', -- command
+ { key } -- arguments
)
if not ret then
rspamd_logger.errx(task, 'cannot make redis request to check results')
})
if sp_urls then
- for _,u in ipairs(sp_urls) do
+ for _, u in ipairs(sp_urls) do
url_redirector_process_url(task, u)
end
end
end
-local opts = rspamd_config:get_all_opt('url_redirector')
+local opts = rspamd_config:get_all_opt('url_redirector')
if opts then
settings = lua_util.override_defaults(settings, opts)
redis_params = lua_redis.parse_redis_server('url_redirector', settings)
type = 'zlist',
})
end
- local id = rspamd_config:register_symbol{
+ local id = rspamd_config:register_symbol {
name = 'URL_REDIRECTOR_CHECK',
type = 'callback,prefilter',
priority = lua_util.symbols_priorities.medium,
callback = url_redirector_handler,
-- In fact, the real timeout is nested_limit * timeout...
- augmentations = {string.format("timeout=%f", settings.timeout)}
+ augmentations = { string.format("timeout=%f", settings.timeout) }
}
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
name = settings.redirector_symbol_nested,
type = 'virtual',
parent = id,
}
if settings.redirector_symbol then
- rspamd_config:register_symbol{
+ rspamd_config:register_symbol {
name = settings.redirector_symbol,
type = 'virtual',
parent = id,
local how = 'wl'
-- Can be overridden
- if rule.blacklist then how = 'bl' end
+ if rule.blacklist then
+ how = 'bl'
+ end
local function parse_val(val)
local how_override
-- Strict is 'special'
- if rule.strict then how_override = 'both' end
+ if rule.strict then
+ how_override = 'both'
+ end
if val then
lua_util.debugm(N, task, "found whitelist key: %s=%s", dom, val)
if val == '' then
- return (how_override or how),1.0
+ return (how_override or how), 1.0
elseif val:match('^bl:') then
- return (how_override or 'bl'),(tonumber(val:sub(4)) or 1.0)
+ return (how_override or 'bl'), (tonumber(val:sub(4)) or 1.0)
elseif val:match('^wl:') then
- return (how_override or 'wl'),(tonumber(val:sub(4)) or 1.0)
+ return (how_override or 'wl'), (tonumber(val:sub(4)) or 1.0)
elseif val:match('^both:') then
- return (how_override or 'both'),(tonumber(val:sub(6)) or 1.0)
+ return (how_override or 'both'), (tonumber(val:sub(6)) or 1.0)
else
- return (how_override or how),(tonumber(val) or 1.0)
+ return (how_override or how), (tonumber(val) or 1.0)
end
end
- return (how_override or how),1.0
+ return (how_override or how), 1.0
end
if rule['map'] then
local val = rule['map']:get_key(dom)
if val then
- how,mult = parse_val(val)
+ how, mult = parse_val(val)
if not domains[check] then
domains[check] = {}
end
domains[check] = {
- [dom] = {how, mult}
+ [dom] = { how, mult }
}
lua_util.debugm(N, task, "final result: %s: %s->%s",
dom, how, mult)
- return true,mult,how
+ return true, mult, how
end
elseif rule['maps'] then
- for _,v in pairs(rule['maps']) do
+ for _, v in pairs(rule['maps']) do
local map = v.map
if map then
local val = map:get_key(dom)
if val then
- how,mult = parse_val(val)
+ how, mult = parse_val(val)
if not domains[check] then
domains[check] = {}
end
domains[check] = {
- [dom] = {how, mult}
+ [dom] = { how, mult }
}
lua_util.debugm(N, task, "final result: %s: %s->%s",
dom, how, mult)
- return true,mult,how
+ return true, mult, how
end
end
end
end
domains[check] = {
- [dom] = {how, mult}
+ [dom] = { how, mult }
}
- return true, mult,how
+ return true, mult, how
end
end
- return false,0.0,how
+ return false, 0.0, how
end
local spf_violated = false
local dkim_opts = sym[1]['options']
if dkim_opts then
fun.each(function(val)
- if val[2] == '+' then
- local tld = rspamd_util.get_tld(val[1])
- find_domain(tld, 'dkim_success')
- elseif val[2] == '-' then
- local tld = rspamd_util.get_tld(val[1])
- find_domain(tld, 'dkim_fail')
- end
- end,
+ if val[2] == '+' then
+ local tld = rspamd_util.get_tld(val[1])
+ find_domain(tld, 'dkim_success')
+ elseif val[2] == '-' then
+ local tld = rspamd_util.get_tld(val[1])
+ find_domain(tld, 'dkim_fail')
+ end
+ end,
fun.map(function(s)
return lua_util.rspamd_str_split(s, ':')
end, dkim_opts))
end
end
-
local final_mult = 1.0
local found_wl, found_bl = false, false
local opts = {}
if rule.valid_dkim then
dkim_violated = true
- for dom,val in pairs(domains.dkim_success or E) do
+ for dom, val in pairs(domains.dkim_success or E) do
if val[1] == 'wl' or val[1] == 'both' then
-- We have valid and whitelisted signature
table.insert(opts, dom .. ':d:+')
end
-- Blacklist counterpart
- for dom,val in pairs(domains.dkim_fail or E) do
+ for dom, val in pairs(domains.dkim_fail or E) do
if val[1] == 'bl' or val[1] == 'both' then
-- We have valid and whitelisted signature
table.insert(opts, dom .. ':d:-')
found_wl = false
- for dom,val in pairs(domains.dmarc or E) do
+ for dom, val in pairs(domains.dmarc or E) do
check_domain_violation('D', dom, val,
(dmarc_violated or dkim_violated))
end
if rule.valid_spf then
found_wl = false
- for dom,val in pairs(domains.spf or E) do
+ for dom, val in pairs(domains.spf or E) do
check_domain_violation('s', dom, val,
(spf_violated or dkim_violated))
end
local configure_whitelist_module = function()
local opts = rspamd_config:get_all_opt('whitelist')
if opts then
- for k,v in pairs(opts) do
+ for k, v in pairs(opts) do
options[k] = v
end
fun.each(function(symbol, rule)
if rule['domains'] then
if type(rule['domains']) == 'string' then
- rule['map'] = rspamd_config:add_map{
+ rule['map'] = rspamd_config:add_map {
url = rule['domains'],
description = "Whitelist map for " .. symbol,
type = 'map'
if type(v) == 'table' then
return true
elseif type(v) == 'string' and not (string.match(v, '^https?://') or
- string.match(v, '^ftp://') or string.match(v, '^[./]')) then
+ string.match(v, '^ftp://') or string.match(v, '^[./]')) then
return true
end
if is_domains_list then
rule['domains'] = fun.tomap(fun.map(function(d)
if type(d) == 'table' then
- return d[1],d[2]
+ return d[1], d[2]
end
- return d,1.0
+ return d, 1.0
end, rule['domains']))
else
- rule['map'] = rspamd_config:add_map{
+ rule['map'] = rspamd_config:add_map {
url = rule['domains'],
description = "Whitelist map for " .. symbol,
type = 'map'
end
else
rspamd_logger.errx(rspamd_config, 'whitelist %s has bad "domains" value',
- symbol)
+ symbol)
return
end
projects / thirdparty / rspamd.git / commitdiff
