From: Vsevolod Stakhov Date: Mon, 30 Mar 2026 15:09:40 +0000 (+0100) Subject: [Feature] Add settings merge point and collect-then-merge flow X-Git-Tag: 4.0.1~6^2~10 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=ebd4cb1673ceeee1d057dbb812f5bfe0a8ce119d;p=thirdparty%2Frspamd.git [Feature] Add settings merge point and collect-then-merge flow Refactor settings.lua from first-match-wins to collect-then-merge: - Settings sources (rules, HTTP, redis, external maps) now collect layers via collect_settings_layer() instead of applying directly - New SETTINGS_APPLY prefilter (priority high=9, below top=10) merges collected layers and applies the result - Prefilter priority ordering ensures SETTINGS_APPLY runs after all collectors (including async redis callbacks) complete - Single-layer case applies directly without merge overhead - New task:merge_and_apply_settings() Lua method calls C merge --- diff --git a/src/lua/lua_task.c b/src/lua/lua_task.c index a18ee523f0..35b07462cd 100644 --- a/src/lua/lua_task.c +++ b/src/lua/lua_task.c @@ -26,6 +26,7 @@ #include "libserver/dkim.h" #include "libserver/task.h" #include "libserver/cfg_file_private.h" +#include "libserver/settings_merge.h" #include "libmime/scan_result_private.h" #include "libstat/stat_api.h" #include "libserver/maps/map_helpers.h" @@ -995,6 +996,15 @@ LUA_FUNCTION_DEF(task, set_settings); */ LUA_FUNCTION_DEF(task, set_settings_id); +/*** + * @method task:merge_and_apply_settings(layers) + * Merge multiple settings layers and apply the result. + * Each layer is a table: {level=N, name="str", settings_id=N, apply={...}} + * Layers are merged according to the settings merge rules. + * @param {table} layers array of layer tables + */ +LUA_FUNCTION_DEF(task, merge_and_apply_settings); + /*** * @method task:get_settings() * Gets users settings object for a task. The format of this object is described @@ -1428,6 +1438,7 @@ static const struct luaL_reg tasklib_m[] = { LUA_INTERFACE_DEF(task, lookup_settings), LUA_INTERFACE_DEF(task, get_settings_id), LUA_INTERFACE_DEF(task, set_settings_id), + LUA_INTERFACE_DEF(task, merge_and_apply_settings), LUA_INTERFACE_DEF(task, cache_get), LUA_INTERFACE_DEF(task, cache_set), LUA_INTERFACE_DEF(task, process_regexp), @@ -6354,6 +6365,133 @@ lua_task_set_settings(lua_State *L) return 0; } +static int +lua_task_merge_and_apply_settings(lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_task *task = lua_check_task(L, 1); + + if (task == NULL || !lua_istable(L, 2)) { + return luaL_error(L, "invalid arguments"); + } + + struct rspamd_settings_merge_ctx *ctx = + rspamd_settings_merge_ctx_create(task->task_pool, task->cfg); + + /* Iterate the layers table */ + for (lua_pushnil(L); lua_next(L, 2); lua_pop(L, 1)) { + if (!lua_istable(L, -1)) { + continue; + } + + lua_getfield(L, -1, "level"); + int level = lua_tointeger(L, -1); + lua_pop(L, 1); + + lua_getfield(L, -1, "name"); + const char *name = lua_tostring(L, -1); + lua_pop(L, 1); + + lua_getfield(L, -1, "settings_id"); + uint32_t settings_id = lua_tointeger(L, -1); + lua_pop(L, 1); + + lua_getfield(L, -1, "apply"); + if (lua_istable(L, -1)) { + ucl_object_t *apply_obj = ucl_object_lua_import(L, lua_gettop(L)); + if (apply_obj) { + rspamd_settings_merge_add_layer(ctx, + (enum rspamd_settings_layer) level, + name ? name : "unknown", + settings_id, + apply_obj); + ucl_object_unref(apply_obj); + } + } + lua_pop(L, 1); /* pop apply */ + } + + ucl_object_t *merged = rspamd_settings_merge_finalize(ctx); + + if (merged) { + /* Apply via set_settings path */ + if (task->settings) { + ucl_object_unref(task->settings); + } + task->settings = merged; + + /* Apply actions overrides */ + const ucl_object_t *act = ucl_object_lookup(task->settings, "actions"); + if (act && ucl_object_type(act) == UCL_OBJECT) { + struct rspamd_scan_result *mres = task->result; + ucl_object_iter_t it = NULL; + const ucl_object_t *cur; + + while ((cur = ucl_object_iterate(act, &it, true)) != NULL) { + const char *act_name = ucl_object_key(cur); + enum rspamd_action_type act_type; + + if (!rspamd_action_from_str(act_name, &act_type)) { + act_type = -1; + } + + for (unsigned int i = 0; i < mres->nactions; i++) { + struct rspamd_action_config *cur_act = &mres->actions_config[i]; + gboolean matched = FALSE; + + if (cur_act->action->action_type == METRIC_ACTION_CUSTOM && act_type == -1) { + matched = g_ascii_strcasecmp(act_name, cur_act->action->name) == 0; + } + else { + matched = cur_act->action->action_type == act_type; + } + + if (matched) { + if (ucl_object_type(cur) == UCL_NULL) { + cur_act->flags |= RSPAMD_ACTION_RESULT_DISABLED; + } + else { + double act_score = ucl_object_todouble(cur); + if (isnan(act_score)) { + cur_act->flags |= RSPAMD_ACTION_RESULT_NO_THRESHOLD; + } + else { + cur_act->cur_limit = act_score; + } + } + break; + } + } + } + } + + /* Apply variables */ + const ucl_object_t *vars = ucl_object_lookup(task->settings, "variables"); + if (vars && ucl_object_type(vars) == UCL_OBJECT) { + ucl_object_iter_t it = NULL; + const ucl_object_t *cur; + + while ((cur = ucl_object_iterate(vars, &it, true)) != NULL) { + if (ucl_object_type(cur) == UCL_STRING) { + rspamd_mempool_set_variable(task->task_pool, + ucl_object_key(cur), + rspamd_mempool_strdup(task->task_pool, + ucl_object_tostring(cur)), + NULL); + } + } + } + + rspamd_symcache_process_settings(task, task->cfg->cache); + lua_pushboolean(L, 1); + } + else { + lua_pushboolean(L, 0); + } + + return 1; +} + static int lua_task_set_milter_reply(lua_State *L) { diff --git a/src/plugins/lua/settings.lua b/src/plugins/lua/settings.lua index f8884fb0e4..90625a8501 100644 --- a/src/plugins/lua/settings.lua +++ b/src/plugins/lua/settings.lua @@ -41,21 +41,36 @@ local settings_initialized = false local max_pri = 0 local module_sym_id -- Main module symbol -local function apply_settings(task, to_apply, id, name) - local cached_name = task:cache_get('settings_name') - if cached_name then - rspamd_logger.infox(task, "replacing settings rule %s with %s (id=%s)", - cached_name, name, id) +-- Settings layer levels (must match enum rspamd_settings_layer in C) +local SETTINGS_LAYER = { + CONFIG = 0, + PROFILE = 1, + RULE = 2, + PER_USER = 3, + HTTP = 4, +} + +-- Collect a settings layer for later merging +local function collect_settings_layer(task, level, name, settings_id, to_apply) + local layers = task:cache_get('settings_layers') + if not layers then + layers = {} end - task:set_settings(to_apply) - task:cache_set('settings', to_apply) - task:cache_set('settings_name', name or 'unknown') + layers[#layers + 1] = { + level = level, + name = name or 'unknown', + settings_id = settings_id or 0, + apply = to_apply, + } - if id then - task:set_settings_id(id) - end + task:cache_set('settings_layers', layers) + lua_util.debugm(N, task, "collected settings layer %s (level=%s, id=%s)", + name, level, settings_id) +end +-- Apply Lua-side effects from merged settings (headers, flags, symbols, etc.) +local function apply_settings_side_effects(task, to_apply) if to_apply['add_headers'] or to_apply['remove_headers'] then local rep = { add_headers = to_apply['add_headers'] or {}, @@ -100,6 +115,15 @@ local function apply_settings(task, to_apply, id, name) task:append_message(message, category) end, to_apply.messages) end +end + +-- Legacy apply for single-layer case (backward compat) +local function apply_settings(task, to_apply, id, name) + collect_settings_layer(task, SETTINGS_LAYER.RULE, name, id, to_apply) + + if id then + task:set_settings_id(id) + end return true end @@ -317,12 +341,12 @@ local function check_settings(task) local function maybe_apply_query_settings() if query_apply then if id_elt then - apply_settings(task, query_apply, id_elt.id, id_elt.name) - rspamd_logger.infox(task, "applied settings id %s(%s); priority %s", + collect_settings_layer(task, SETTINGS_LAYER.PROFILE, id_elt.name, id_elt.id, query_apply) + rspamd_logger.infox(task, "collected settings id %s(%s); priority %s", id_elt.name, id_elt.id, priority_to_string(priority)) else - apply_settings(task, query_apply, nil, 'HTTP query') - rspamd_logger.infox(task, "applied settings from query; priority %s", + collect_settings_layer(task, SETTINGS_LAYER.HTTP, 'HTTP query', 0, query_apply) + rspamd_logger.infox(task, "collected settings from query; priority %s", priority_to_string(priority)) end end @@ -450,9 +474,9 @@ local function gen_settings_external_cb(name) name, ucl_err) else local obj = parser:get_object() - rspamd_logger.infox(task, "<%s> apply settings according to the external map %s", + rspamd_logger.infox(task, "<%s> collect settings from external map %s", name, task:get_message_id()) - apply_settings(task, obj, nil, 'external_map') + collect_settings_layer(task, SETTINGS_LAYER.RULE, 'external_map:' .. name, 0, obj) end else rspamd_logger.infox(task, "<%s> no settings returned from the external map %s: %s (code = %s)", @@ -1277,9 +1301,9 @@ local function gen_redis_callback(handler, id) ucl_err) else local obj = parser:get_object() - rspamd_logger.infox(task, "<%s> apply settings according to redis rule %s", + rspamd_logger.infox(task, "<%s> collect settings from redis rule %s", task:get_message_id(), id) - apply_settings(task, obj, nil, 'redis') + collect_settings_layer(task, SETTINGS_LAYER.PER_USER, 'redis:' .. tostring(id), 0, obj) break end end @@ -1362,6 +1386,61 @@ module_sym_id = rspamd_config:register_symbol({ flags = 'empty,nostat,explicit_disable,ignore_passthrough', }) +-- SETTINGS_APPLY runs after all settings collectors (SETTINGS_CHECK, REDIS_SETTINGS*) +-- have finished, merges collected layers, and applies the result. +-- Priority high (9) < top (10) ensures proper ordering via prefilter priority mechanism. +rspamd_config:register_symbol({ + name = 'SETTINGS_APPLY', + type = 'prefilter', + callback = function(task) + local layers = task:cache_get('settings_layers') + if not layers or #layers == 0 then + lua_util.debugm(N, task, "no settings layers collected, nothing to apply") + return + end + + if #layers == 1 then + -- Single layer: apply directly without merge overhead + local layer = layers[1] + lua_util.debugm(N, task, "single settings layer %s, applying directly", layer.name) + task:set_settings(layer.apply) + if layer.settings_id and layer.settings_id ~= 0 then + task:set_settings_id(layer.settings_id) + end + apply_settings_side_effects(task, layer.apply) + else + -- Multiple layers: merge via C infrastructure + rspamd_logger.infox(task, "merging %s settings layers", #layers) + + -- Also collect settings_id from the highest-priority layer that has one + local best_settings_id = nil + for _, layer in ipairs(layers) do + if layer.settings_id and layer.settings_id ~= 0 then + best_settings_id = layer.settings_id + end + end + + local merged = task:merge_and_apply_settings(layers) + if merged then + if best_settings_id then + task:set_settings_id(best_settings_id) + end + -- Apply Lua-side effects from the merged result + local merged_settings = task:get_settings() + if merged_settings then + apply_settings_side_effects(task, merged_settings) + end + else + rspamd_logger.warnx(task, "settings merge returned no result") + end + end + + task:cache_set('settings_applied', true) + end, + priority = lua_util.symbols_priorities.high, + flags = 'empty,nostat,explicit_disable,ignore_passthrough', +}) + local set_section = rspamd_config:get_all_opt("settings") if set_section and set_section[1] and type(set_section[1]) == "string" then