From: Vsevolod Stakhov Date: Fri, 27 Jun 2025 16:38:28 +0000 (+0100) Subject: [Project] Use re_cache scopes for maps X-Git-Tag: 3.13.0~47^2~12 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=b055748f7dcbf83fad5ca05160dbc5e24fdbaca4;p=thirdparty%2Frspamd.git [Project] Use re_cache scopes for maps --- diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c index 25ed72948d..23022b9f07 100644 --- a/src/libserver/re_cache.c +++ b/src/libserver/re_cache.c @@ -131,6 +131,7 @@ struct rspamd_re_cache { /* Intrusive linked list for scoped caches */ struct rspamd_re_cache *next, *prev; char *scope; + unsigned int flags; /* Cache flags (loaded state, etc.) */ #ifdef WITH_HYPERSCAN enum rspamd_hyperscan_status hyperscan_loaded; @@ -227,6 +228,7 @@ rspamd_re_cache_add_to_scope_list(struct rspamd_re_cache **cache_head, const cha g_free(new_cache->scope); } new_cache->scope = g_strdup(scope); + new_cache->flags = 0; /* New scopes start as unloaded */ /* Add to linked list */ if (*cache_head) { @@ -323,7 +325,8 @@ rspamd_re_cache_new(void) cache->re = g_ptr_array_new_full(256, rspamd_re_cache_elt_dtor); cache->selectors = kh_init(lua_selectors_hash); cache->next = cache->prev = NULL; - cache->scope = NULL; /* Default scope */ + cache->scope = NULL; /* Default scope */ + cache->flags = RSPAMD_RE_CACHE_FLAG_LOADED; /* Default scope is always loaded */ #ifdef WITH_HYPERSCAN cache->hyperscan_loaded = RSPAMD_HYPERSCAN_UNKNOWN; #endif @@ -634,7 +637,10 @@ void rspamd_re_cache_init_scoped(struct rspamd_re_cache *cache_head, DL_FOREACH(cache_head, cur) { - rspamd_re_cache_init(cur, cfg); + /* Only initialize loaded scopes */ + if (cur->flags & RSPAMD_RE_CACHE_FLAG_LOADED) { + rspamd_re_cache_init(cur, cfg); + } } } @@ -666,11 +672,16 @@ rspamd_re_cache_runtime_new(struct rspamd_re_cache *cache) g_assert(cache != NULL); /* - * Create runtime for all scopes in the chain. - * This ensures task has runtimes for all available scopes. + * Create runtime for all loaded scopes in the chain. + * This ensures task has runtimes for all available loaded scopes. */ DL_FOREACH(cache, cur) { + /* Skip unloaded scopes */ + if (!(cur->flags & RSPAMD_RE_CACHE_FLAG_LOADED)) { + continue; + } + rt = rspamd_re_cache_runtime_new_single(cur); if (rt) { if (rt_head) { @@ -3134,6 +3145,56 @@ unsigned int rspamd_re_cache_count_scopes(struct rspamd_re_cache *cache_head) return count; } +void rspamd_re_cache_set_flags(struct rspamd_re_cache *cache_head, const char *scope, unsigned int flags) +{ + struct rspamd_re_cache *target; + + if (!cache_head) { + return; + } + + target = rspamd_re_cache_find_by_scope(cache_head, scope); + if (target) { + target->flags |= flags; + } +} + +void rspamd_re_cache_clear_flags(struct rspamd_re_cache *cache_head, const char *scope, unsigned int flags) +{ + struct rspamd_re_cache *target; + + if (!cache_head) { + return; + } + + target = rspamd_re_cache_find_by_scope(cache_head, scope); + if (target) { + target->flags &= ~flags; + } +} + +unsigned int rspamd_re_cache_get_flags(struct rspamd_re_cache *cache_head, const char *scope) +{ + struct rspamd_re_cache *target; + + if (!cache_head) { + return 0; + } + + target = rspamd_re_cache_find_by_scope(cache_head, scope); + if (target) { + return target->flags; + } + + return 0; +} + +gboolean rspamd_re_cache_is_loaded(struct rspamd_re_cache *cache_head, const char *scope) +{ + unsigned int flags = rspamd_re_cache_get_flags(cache_head, scope); + return (flags & RSPAMD_RE_CACHE_FLAG_LOADED) != 0; +} + char **rspamd_re_cache_get_scope_names(struct rspamd_re_cache *cache_head, unsigned int *count_out) { struct rspamd_re_cache *cur; diff --git a/src/libserver/re_cache.h b/src/libserver/re_cache.h index b64c7a9ab7..fbd243723e 100644 --- a/src/libserver/re_cache.h +++ b/src/libserver/re_cache.h @@ -28,6 +28,9 @@ struct rspamd_re_runtime; struct rspamd_task; struct rspamd_config; +/* Re cache flags */ +#define RSPAMD_RE_CACHE_FLAG_LOADED (1U << 0) /* Scope is fully loaded and ready for use */ + enum rspamd_re_type { RSPAMD_RE_HEADER, RSPAMD_RE_RAWHEADER, @@ -306,6 +309,38 @@ unsigned int rspamd_re_cache_count_scopes(struct rspamd_re_cache *cache_head); */ char **rspamd_re_cache_get_scope_names(struct rspamd_re_cache *cache_head, unsigned int *count_out); +/** + * Set flags for a specific scope + * @param cache_head head of cache list + * @param scope scope name (NULL for default scope) + * @param flags flags to set + */ +void rspamd_re_cache_set_flags(struct rspamd_re_cache *cache_head, const char *scope, unsigned int flags); + +/** + * Clear flags for a specific scope + * @param cache_head head of cache list + * @param scope scope name (NULL for default scope) + * @param flags flags to clear + */ +void rspamd_re_cache_clear_flags(struct rspamd_re_cache *cache_head, const char *scope, unsigned int flags); + +/** + * Get flags for a specific scope + * @param cache_head head of cache list + * @param scope scope name (NULL for default scope) + * @return flags value + */ +unsigned int rspamd_re_cache_get_flags(struct rspamd_re_cache *cache_head, const char *scope); + +/** + * Check if a scope is loaded + * @param cache_head head of cache list + * @param scope scope name (NULL for default scope) + * @return TRUE if scope is loaded and ready for use + */ +gboolean rspamd_re_cache_is_loaded(struct rspamd_re_cache *cache_head, const char *scope); + #ifdef __cplusplus } #endif diff --git a/src/lua/lua_config.c b/src/lua/lua_config.c index 0c7f5d3405..cf13bffc2c 100644 --- a/src/lua/lua_config.c +++ b/src/lua/lua_config.c @@ -19,6 +19,7 @@ #include "src/libserver/composites/composites.h" #include "libserver/cfg_file_private.h" #include "libmime/lang_detection.h" +#include "libserver/re_cache.h" #include "lua/lua_map.h" #include "lua/lua_thread_pool.h" #include "utlist.h" @@ -634,6 +635,46 @@ LUA_FUNCTION_DEF(config, count_regexp_scopes); */ LUA_FUNCTION_DEF(config, list_regexp_scopes); +/*** + * @method rspamd_config:set_regexp_scope_flags(scope, flags) + * Sets flags for a regexp scope + * @param {string} scope scope name (can be nil for default scope) + * @param {number} flags flags to set + */ +LUA_FUNCTION_DEF(config, set_regexp_scope_flags); + +/*** + * @method rspamd_config:clear_regexp_scope_flags(scope, flags) + * Clears flags for a regexp scope + * @param {string} scope scope name (can be nil for default scope) + * @param {number} flags flags to clear + */ +LUA_FUNCTION_DEF(config, clear_regexp_scope_flags); + +/*** + * @method rspamd_config:get_regexp_scope_flags(scope) + * Gets flags for a regexp scope + * @param {string} scope scope name (can be nil for default scope) + * @return {number} current flags value + */ +LUA_FUNCTION_DEF(config, get_regexp_scope_flags); + +/*** + * @method rspamd_config:is_regexp_scope_loaded(scope) + * Checks if a regexp scope is loaded and ready for use + * @param {string} scope scope name (can be nil for default scope) + * @return {boolean} true if scope is loaded + */ +LUA_FUNCTION_DEF(config, is_regexp_scope_loaded); + +/*** + * @method rspamd_config:set_regexp_scope_loaded(scope, loaded) + * Sets the loaded state of a regexp scope + * @param {string} scope scope name (can be nil for default scope) + * @param {boolean} loaded whether scope should be marked as loaded (defaults to true) + */ +LUA_FUNCTION_DEF(config, set_regexp_scope_loaded); + /*** * @method rspamd_config:register_worker_script(worker_type, script) * Registers the following script for workers of a specified type. The exact type @@ -1000,6 +1041,11 @@ static const struct luaL_reg configlib_m[] = { LUA_INTERFACE_DEF(config, remove_regexp_scope), LUA_INTERFACE_DEF(config, count_regexp_scopes), LUA_INTERFACE_DEF(config, list_regexp_scopes), + LUA_INTERFACE_DEF(config, set_regexp_scope_flags), + LUA_INTERFACE_DEF(config, clear_regexp_scope_flags), + LUA_INTERFACE_DEF(config, get_regexp_scope_flags), + LUA_INTERFACE_DEF(config, is_regexp_scope_loaded), + LUA_INTERFACE_DEF(config, set_regexp_scope_loaded), LUA_INTERFACE_DEF(config, register_worker_script), LUA_INTERFACE_DEF(config, register_re_selector), LUA_INTERFACE_DEF(config, add_on_load), @@ -4866,6 +4912,10 @@ void luaopen_config(lua_State *L) rspamd_lua_new_class(L, rspamd_monitored_classname, monitoredlib_m); lua_pop(L, 1); + + /* Export constants */ + lua_pushinteger(L, RSPAMD_RE_CACHE_FLAG_LOADED); + lua_setglobal(L, "RSPAMD_RE_CACHE_FLAG_LOADED"); } void lua_call_finish_script(struct rspamd_config_cfg_lua_script *sc, @@ -5241,3 +5291,123 @@ lua_config_list_regexp_scopes(lua_State *L) return 1; } + +static int +lua_config_set_regexp_scope_flags(lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_config *cfg = lua_check_config(L, 1); + const char *scope = NULL; + unsigned int flags = 0; + + if (cfg) { + if (lua_type(L, 2) == LUA_TSTRING) { + scope = lua_tostring(L, 2); + } + flags = lua_tointeger(L, 3); + + rspamd_re_cache_set_flags(cfg->re_cache, scope, flags); + } + else { + return luaL_error(L, "invalid arguments"); + } + + return 0; +} + +static int +lua_config_clear_regexp_scope_flags(lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_config *cfg = lua_check_config(L, 1); + const char *scope = NULL; + unsigned int flags = 0; + + if (cfg) { + if (lua_type(L, 2) == LUA_TSTRING) { + scope = lua_tostring(L, 2); + } + flags = lua_tointeger(L, 3); + + rspamd_re_cache_clear_flags(cfg->re_cache, scope, flags); + } + else { + return luaL_error(L, "invalid arguments"); + } + + return 0; +} + +static int +lua_config_get_regexp_scope_flags(lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_config *cfg = lua_check_config(L, 1); + const char *scope = NULL; + + if (cfg) { + if (lua_type(L, 2) == LUA_TSTRING) { + scope = lua_tostring(L, 2); + } + + unsigned int flags = rspamd_re_cache_get_flags(cfg->re_cache, scope); + lua_pushinteger(L, flags); + } + else { + return luaL_error(L, "invalid arguments"); + } + + return 1; +} + +static int +lua_config_is_regexp_scope_loaded(lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_config *cfg = lua_check_config(L, 1); + const char *scope = NULL; + + if (cfg) { + if (lua_type(L, 2) == LUA_TSTRING) { + scope = lua_tostring(L, 2); + } + + gboolean loaded = rspamd_re_cache_is_loaded(cfg->re_cache, scope); + lua_pushboolean(L, loaded); + } + else { + return luaL_error(L, "invalid arguments"); + } + + return 1; +} + +static int +lua_config_set_regexp_scope_loaded(lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_config *cfg = lua_check_config(L, 1); + const char *scope = NULL; + gboolean loaded = TRUE; + + if (cfg) { + if (lua_type(L, 2) == LUA_TSTRING) { + scope = lua_tostring(L, 2); + } + if (lua_type(L, 3) == LUA_TBOOLEAN) { + loaded = lua_toboolean(L, 3); + } + + if (loaded) { + rspamd_re_cache_set_flags(cfg->re_cache, scope, RSPAMD_RE_CACHE_FLAG_LOADED); + } + else { + rspamd_re_cache_clear_flags(cfg->re_cache, scope, RSPAMD_RE_CACHE_FLAG_LOADED); + } + } + else { + return luaL_error(L, "invalid arguments"); + } + + return 0; +} diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 93d43f34b9..92a857fd52 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -202,6 +202,10 @@ local function process_sa_line(rule, line, map) end local rule_name = rule.symbol + local scope_name = rule.scope_name or rule_name + + -- All regexps for this SA-style rule are registered in a dedicated scope + -- This allows clean removal and replacement when the map is reloaded if words[1] == 'header' then -- header SYMBOL Header =~ /regexp/flags @@ -215,8 +219,8 @@ local function process_sa_line(rule, line, map) local re = parse_sa_regexp(atom_name, re_expr) if re then - -- Register regexp with cache - rspamd_config:register_regexp({ + -- Register regexp with cache in specific scope + rspamd_config:register_regexp_scoped(scope_name, { re = re, type = 'header', header = header_name, @@ -233,8 +237,8 @@ local function process_sa_line(rule, line, map) negate = negate }) - lua_util.debugm(N, rspamd_config, 'added SA header atom: %s for header %s', - atom_name, header_name) + lua_util.debugm(N, rspamd_config, 'added SA header atom: %s for header %s (scope: %s)', + atom_name, header_name, scope_name) end end elseif words[1] == 'body' then @@ -245,7 +249,7 @@ local function process_sa_line(rule, line, map) local re = parse_sa_regexp(atom_name, re_expr) if re then - rspamd_config:register_regexp({ + rspamd_config:register_regexp_scoped(scope_name, { re = re, type = 'sabody', pcre_only = false, @@ -256,7 +260,7 @@ local function process_sa_line(rule, line, map) sa_atoms[atom_name] = create_sa_atom_function(atom_name, re, 'body', {}) - lua_util.debugm(N, rspamd_config, 'added SA body atom: %s', atom_name) + lua_util.debugm(N, rspamd_config, 'added SA body atom: %s (scope: %s)', atom_name, scope_name) end end elseif words[1] == 'rawbody' then @@ -267,7 +271,7 @@ local function process_sa_line(rule, line, map) local re = parse_sa_regexp(atom_name, re_expr) if re then - rspamd_config:register_regexp({ + rspamd_config:register_regexp_scoped(scope_name, { re = re, type = 'sarawbody', pcre_only = false, @@ -278,7 +282,7 @@ local function process_sa_line(rule, line, map) sa_atoms[atom_name] = create_sa_atom_function(atom_name, re, 'rawbody', {}) - lua_util.debugm(N, rspamd_config, 'added SA rawbody atom: %s', atom_name) + lua_util.debugm(N, rspamd_config, 'added SA rawbody atom: %s (scope: %s)', atom_name, scope_name) end end elseif words[1] == 'uri' then @@ -289,7 +293,7 @@ local function process_sa_line(rule, line, map) local re = parse_sa_regexp(atom_name, re_expr) if re then - rspamd_config:register_regexp({ + rspamd_config:register_regexp_scoped(scope_name, { re = re, type = 'url', pcre_only = false, @@ -300,7 +304,7 @@ local function process_sa_line(rule, line, map) sa_atoms[atom_name] = create_sa_atom_function(atom_name, re, 'uri', {}) - lua_util.debugm(N, rspamd_config, 'added SA uri atom: %s', atom_name) + lua_util.debugm(N, rspamd_config, 'added SA uri atom: %s (scope: %s)', atom_name, scope_name) end end elseif words[1] == 'full' then @@ -311,7 +315,7 @@ local function process_sa_line(rule, line, map) local re = parse_sa_regexp(atom_name, re_expr) if re then - rspamd_config:register_regexp({ + rspamd_config:register_regexp_scoped(scope_name, { re = re, type = 'body', pcre_only = false, @@ -322,7 +326,7 @@ local function process_sa_line(rule, line, map) sa_atoms[atom_name] = create_sa_atom_function(atom_name, re, 'full', {}) - lua_util.debugm(N, rspamd_config, 'added SA full atom: %s', atom_name) + lua_util.debugm(N, rspamd_config, 'added SA full atom: %s (scope: %s)', atom_name, scope_name) end end elseif words[1] == 'meta' then @@ -1701,21 +1705,59 @@ local function add_multimap_rule(key, newrule) return nil end + -- Set scope name for this regexp_rules map + local scope_name = newrule.symbol + newrule.scope_name = scope_name + + -- Remove existing scope if it exists to ensure clean state + if rspamd_config:find_regexp_scope(scope_name) then + lua_util.debugm(N, rspamd_config, 'removing existing regexp scope: %s', scope_name) + rspamd_config:remove_regexp_scope(scope_name) + end + + -- Mark the scope as unloaded during map processing + -- The scope will be created automatically when first regexp is added + local first_line_processed = false + -- Create callback map with by_line processing newrule.map_obj = rspamd_config:add_map({ type = "callback", url = map_ucl.url or map_ucl.urls or map_ucl, description = newrule.description or 'SA-style multimap: ' .. newrule.symbol, callback = function(line, map) + -- Mark scope as unloaded on first line + if not first_line_processed then + first_line_processed = true + -- The scope will be created by process_sa_line when first regexp is added + -- We mark it as unloaded immediately after creation + rspamd_config:set_regexp_scope_loaded(scope_name, false) + lua_util.debugm(N, rspamd_config, 'marked regexp scope %s as unloaded during processing', scope_name) + end process_sa_line(newrule, line, map) end, by_line = true, -- Process line by line opaque_data = false, -- Use plain strings }) + -- Add on_load callback to mark scope as loaded when map processing is complete + if newrule.map_obj then + newrule.map_obj:on_load(function() + -- Mark scope as loaded when map processing is complete + -- Check if scope exists (it might not if map was empty) + if rspamd_config:find_regexp_scope(scope_name) then + rspamd_config:set_regexp_scope_loaded(scope_name, true) + lua_util.debugm(N, rspamd_config, 'marked regexp scope %s as loaded after map processing', scope_name) + else + lua_util.debugm(N, rspamd_config, 'regexp scope %s not created (empty map)', scope_name) + end + end) + end + if newrule.map_obj then -- Mark this rule as using SA functionality newrule.uses_sa = true + lua_util.debugm(N, rspamd_config, 'created regexp_rules map %s with scope: %s', + newrule.symbol, scope_name) ret = true else rspamd_logger.warnx(rspamd_config, 'Cannot add SA-style rule: map doesn\'t exists: %s',