From: Vsevolod Stakhov Date: Sat, 28 Jun 2025 12:32:17 +0000 (+0100) Subject: [Project] Implement scoped compilation X-Git-Tag: 3.13.0~47^2~11 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=40df8199246e0c4e07a05dd35e8af59a69c5c8bd;p=thirdparty%2Frspamd.git [Project] Implement scoped compilation --- diff --git a/src/hs_helper.c b/src/hs_helper.c index 26d57528ff..3bd2040f85 100644 --- a/src/hs_helper.c +++ b/src/hs_helper.c @@ -243,13 +243,122 @@ rspamd_hs_helper_cleanup_dir(struct hs_helper_ctx *ctx, gboolean forced) return ret; } -/* Bad hack, but who cares */ -static gboolean hack_global_forced; + +struct rspamd_hs_helper_compile_cbdata { + struct rspamd_worker *worker; + struct hs_helper_ctx *ctx; + unsigned int total_compiled; + unsigned int scopes_remaining; + gboolean forced; +}; + +static void +rspamd_rs_delayed_scoped_cb(EV_P_ ev_timer *w, int revents) +{ + struct rspamd_hs_helper_compile_cbdata *cbd = (struct rspamd_hs_helper_compile_cbdata *) w->data; + struct rspamd_worker *worker = cbd->worker; + struct hs_helper_ctx *ctx = cbd->ctx; + static struct rspamd_srv_command srv_cmd; + + memset(&srv_cmd, 0, sizeof(srv_cmd)); + srv_cmd.type = RSPAMD_SRV_HYPERSCAN_LOADED; + rspamd_strlcpy(srv_cmd.cmd.hs_loaded.cache_dir, ctx->hs_dir, + sizeof(srv_cmd.cmd.hs_loaded.cache_dir)); + srv_cmd.cmd.hs_loaded.forced = cbd->forced; + srv_cmd.cmd.hs_loaded.scope[0] = '\0'; /* NULL scope means all scopes */ + + rspamd_srv_send_command(worker, + ctx->event_loop, &srv_cmd, -1, NULL, NULL); + ev_timer_stop(EV_A_ w); + g_free(w); + g_free(cbd); + + ev_timer_again(EV_A_ & ctx->recompile_timer); +} + +static void +rspamd_rs_compile_scoped_cb(const char *scope, unsigned int ncompiled, GError *err, void *cbd) +{ + struct rspamd_hs_helper_compile_cbdata *compile_cbd = + (struct rspamd_hs_helper_compile_cbdata *) cbd; + struct rspamd_worker *worker = compile_cbd->worker; + struct hs_helper_ctx *ctx = compile_cbd->ctx; + static struct rspamd_srv_command srv_cmd; + + if (err != NULL) { + /* Failed to compile: log and continue */ + msg_err("cannot compile Hyperscan database for scope %s: %e", + scope ? scope : "default", err); + } + else { + if (ncompiled > 0) { + compile_cbd->total_compiled += ncompiled; + + /* Send notification for this specific scope */ + memset(&srv_cmd, 0, sizeof(srv_cmd)); + srv_cmd.type = RSPAMD_SRV_HYPERSCAN_LOADED; + rspamd_strlcpy(srv_cmd.cmd.hs_loaded.cache_dir, ctx->hs_dir, + sizeof(srv_cmd.cmd.hs_loaded.cache_dir)); + srv_cmd.cmd.hs_loaded.forced = compile_cbd->forced; + if (scope) { + rspamd_strlcpy(srv_cmd.cmd.hs_loaded.scope, scope, + sizeof(srv_cmd.cmd.hs_loaded.scope)); + } + else { + srv_cmd.cmd.hs_loaded.scope[0] = '\0'; + } + + rspamd_srv_send_command(worker, + ctx->event_loop, &srv_cmd, -1, NULL, NULL); + + msg_info("compiled %d regular expressions for scope %s", + ncompiled, scope ? scope : "default"); + } + } + + compile_cbd->scopes_remaining--; + + /* Check if all scopes are done */ + if (compile_cbd->scopes_remaining == 0) { + ev_timer *tm; + ev_tstamp when = 0.0; + + /* + * Do not send notification unless all other workers are started + * XXX: now we just sleep for 1 seconds to ensure that + */ + if (!ctx->loaded) { + when = 1.0; /* Postpone */ + ctx->loaded = TRUE; + msg_info("compiled %d total regular expressions to the hyperscan tree, " + "postpone final notification for %.0f seconds to avoid races", + compile_cbd->total_compiled, + when); + } + else { + msg_info("compiled %d total regular expressions to the hyperscan tree, " + "send final notification", + compile_cbd->total_compiled); + } + + tm = g_malloc0(sizeof(*tm)); + tm->data = (void *) compile_cbd; + ev_timer_init(tm, rspamd_rs_delayed_scoped_cb, when, 0); + ev_timer_start(ctx->event_loop, tm); + } +} + +struct rspamd_hs_helper_single_compile_cbdata { + struct rspamd_worker *worker; + gboolean forced; +}; static void rspamd_rs_delayed_cb(EV_P_ ev_timer *w, int revents) { - struct rspamd_worker *worker = (struct rspamd_worker *) w->data; + struct rspamd_hs_helper_single_compile_cbdata *cbd = + (struct rspamd_hs_helper_single_compile_cbdata *) w->data; + struct rspamd_worker *worker = cbd->worker; static struct rspamd_srv_command srv_cmd; struct hs_helper_ctx *ctx; @@ -258,13 +367,14 @@ rspamd_rs_delayed_cb(EV_P_ ev_timer *w, int revents) srv_cmd.type = RSPAMD_SRV_HYPERSCAN_LOADED; rspamd_strlcpy(srv_cmd.cmd.hs_loaded.cache_dir, ctx->hs_dir, sizeof(srv_cmd.cmd.hs_loaded.cache_dir)); - srv_cmd.cmd.hs_loaded.forced = hack_global_forced; - hack_global_forced = FALSE; + srv_cmd.cmd.hs_loaded.forced = cbd->forced; + srv_cmd.cmd.hs_loaded.scope[0] = '\0'; /* NULL scope means all scopes */ rspamd_srv_send_command(worker, ctx->event_loop, &srv_cmd, -1, NULL, NULL); ev_timer_stop(EV_A_ w); g_free(w); + g_free(cbd); ev_timer_again(EV_A_ & ctx->recompile_timer); } @@ -272,25 +382,23 @@ rspamd_rs_delayed_cb(EV_P_ ev_timer *w, int revents) static void rspamd_rs_compile_cb(unsigned int ncompiled, GError *err, void *cbd) { - struct rspamd_worker *worker = (struct rspamd_worker *) cbd; + struct rspamd_hs_helper_single_compile_cbdata *compile_cbd = + (struct rspamd_hs_helper_single_compile_cbdata *) cbd; + struct rspamd_worker *worker = compile_cbd->worker; ev_timer *tm; ev_tstamp when = 0.0; struct hs_helper_ctx *ctx; + struct rspamd_hs_helper_single_compile_cbdata *timer_cbd; ctx = (struct hs_helper_ctx *) worker->ctx; if (err != NULL) { /* Failed to compile: log and go out */ msg_err("cannot compile Hyperscan database: %e", err); - + g_free(compile_cbd); return; } - if (ncompiled > 0) { - /* Enforce update for other workers */ - hack_global_forced = TRUE; - } - /* * Do not send notification unless all other workers are started * XXX: now we just sleep for 1 seconds to ensure that @@ -309,10 +417,16 @@ rspamd_rs_compile_cb(unsigned int ncompiled, GError *err, void *cbd) ncompiled); } + timer_cbd = g_malloc0(sizeof(*timer_cbd)); + timer_cbd->worker = worker; + timer_cbd->forced = (ncompiled > 0) ? TRUE : compile_cbd->forced; + tm = g_malloc0(sizeof(*tm)); - tm->data = (void *) worker; + tm->data = (void *) timer_cbd; ev_timer_init(tm, rspamd_rs_delayed_cb, when, 0); ev_timer_start(ctx->event_loop, tm); + + g_free(compile_cbd); } static gboolean @@ -331,13 +445,80 @@ rspamd_rs_compile(struct hs_helper_ctx *ctx, struct rspamd_worker *worker, msg_warn("cannot cleanup cache dir '%s'", ctx->hs_dir); } - hack_global_forced = forced; /* killmeplease */ - rspamd_re_cache_compile_hyperscan(ctx->cfg->re_cache, - ctx->hs_dir, ctx->max_time, !forced, - ctx->event_loop, - rspamd_rs_compile_cb, - (void *) worker); + /* Check if we have any scopes */ + unsigned int scope_count = rspamd_re_cache_count_scopes(ctx->cfg->re_cache); + if (scope_count == 0) { + /* No additional scopes, just default scope - use standard compilation */ + struct rspamd_hs_helper_single_compile_cbdata *single_cbd = + g_malloc0(sizeof(*single_cbd)); + single_cbd->worker = worker; + single_cbd->forced = forced; + + rspamd_re_cache_compile_hyperscan(ctx->cfg->re_cache, + ctx->hs_dir, ctx->max_time, !forced, + ctx->event_loop, + rspamd_rs_compile_cb, + (void *) single_cbd); + return TRUE; + } + + /* Get all scope names */ + unsigned int names_count; + char **scope_names = rspamd_re_cache_get_scope_names(ctx->cfg->re_cache, &names_count); + + if (!scope_names || names_count == 0) { + /* Failed to get scope names, use standard compilation for default scope */ + struct rspamd_hs_helper_single_compile_cbdata *single_cbd = + g_malloc0(sizeof(*single_cbd)); + single_cbd->worker = worker; + single_cbd->forced = forced; + + rspamd_re_cache_compile_hyperscan(ctx->cfg->re_cache, + ctx->hs_dir, ctx->max_time, !forced, + ctx->event_loop, + rspamd_rs_compile_cb, + (void *) single_cbd); + return TRUE; + } + + /* Prepare compilation callback data */ + struct rspamd_hs_helper_compile_cbdata *compile_cbd = + g_malloc0(sizeof(*compile_cbd)); + compile_cbd->worker = worker; + compile_cbd->ctx = ctx; + compile_cbd->total_compiled = 0; + compile_cbd->scopes_remaining = names_count; + compile_cbd->forced = forced; + + /* Compile each scope */ + for (unsigned int i = 0; i < names_count; i++) { + const char *scope = strcmp(scope_names[i], "default") == 0 ? NULL : scope_names[i]; + struct rspamd_re_cache *scope_cache = rspamd_re_cache_find_scope(ctx->cfg->re_cache, scope); + + if (scope_cache && rspamd_re_cache_is_loaded(ctx->cfg->re_cache, scope)) { + rspamd_re_cache_compile_hyperscan_scoped_single(scope_cache, scope, + ctx->hs_dir, ctx->max_time, !forced, + ctx->event_loop, + rspamd_rs_compile_scoped_cb, + compile_cbd); + } + else { + /* Scope not loaded, skip it */ + compile_cbd->scopes_remaining--; + msg_debug("skipping unloaded scope: %s", scope ? scope : "default"); + + /* Check if we're done */ + if (compile_cbd->scopes_remaining == 0) { + /* No scopes to compile, send final notification immediately */ + ev_timer *tm = g_malloc0(sizeof(*tm)); + tm->data = (void *) compile_cbd; + ev_timer_init(tm, rspamd_rs_delayed_scoped_cb, 0.0, 0); + ev_timer_start(ctx->event_loop, tm); + } + } + } + g_strfreev(scope_names); return TRUE; } diff --git a/src/libserver/maps/map.c b/src/libserver/maps/map.c index ac82d39bb2..f0bf7ee716 100644 --- a/src/libserver/maps/map.c +++ b/src/libserver/maps/map.c @@ -26,6 +26,8 @@ #include "contrib/libev/ev.h" #include "contrib/uthash/utlist.h" +#include + #ifdef SYS_ZSTD #include "zstd.h" #else @@ -1858,6 +1860,7 @@ rspamd_map_read_http_cached_file(struct rspamd_map *map, g_atomic_int_set(&map->shared->loaded, 1); g_atomic_int_set(&map->shared->cached, 1); + rspamd_localtime(map->next_check, &tm); strftime(ncheck_buf, sizeof(ncheck_buf) - 1, "%Y-%m-%d %H:%M:%S", &tm); rspamd_localtime(htdata->last_modified, &tm); @@ -3350,3 +3353,58 @@ void rspamd_map_set_on_load_function(struct rspamd_map *map, rspamd_map_on_load_ map->on_load_ud_dtor = dtor; } } + +void rspamd_map_trigger_hyperscan_compilation(struct rspamd_map *map) +{ + /* Only trigger compilation in controller worker */ + if (!map->cfg || !map->cfg->cur_worker) { + return; + } + + struct rspamd_worker *worker = map->wrk; + if (!rspamd_worker_is_primary_controller(worker)) { + return; + } + + /* Check if we have any scopes that need compilation */ + if (!map->cfg->re_cache) { + return; + } + + unsigned int scope_count = rspamd_re_cache_count_scopes(map->cfg->re_cache); + if (scope_count == 0) { + return; + } + + /* Get scope names and compile those that are loaded */ + unsigned int names_count; + char **scope_names = rspamd_re_cache_get_scope_names(map->cfg->re_cache, &names_count); + + if (scope_names && names_count > 0) { + for (unsigned int i = 0; i < names_count; i++) { + const char *scope = strcmp(scope_names[i], "default") == 0 ? NULL : scope_names[i]; + + /* Only compile loaded scopes */ + if (rspamd_re_cache_is_loaded(map->cfg->re_cache, scope)) { + struct rspamd_re_cache *scope_cache = rspamd_re_cache_find_scope(map->cfg->re_cache, scope); + + if (scope_cache) { + msg_info_map("triggering hyperscan compilation for scope: %s after map update", + scope ? scope : "default"); + + /* Use default settings for compilation */ + rspamd_re_cache_compile_hyperscan_scoped_single(scope_cache, scope, + map->cfg->hs_cache_dir ? map->cfg->hs_cache_dir : RSPAMD_DBDIR "/", + 1.0, /* max_time */ + FALSE, /* silent */ + worker->ctx ? ((struct rspamd_abstract_worker_ctx *) worker->ctx)->event_loop : NULL, + NULL, /* callback */ + NULL); /* cbdata */ + } + } + } + + /* Clean up scope names */ + g_strfreev(scope_names); + } +} diff --git a/src/libserver/maps/map.h b/src/libserver/maps/map.h index b2ba53118f..27915e4c9a 100644 --- a/src/libserver/maps/map.h +++ b/src/libserver/maps/map.h @@ -161,6 +161,12 @@ void rspamd_map_traverse(struct rspamd_map *map, rspamd_map_traverse_cb cb, void rspamd_map_set_on_load_function(struct rspamd_map *map, rspamd_map_on_load_function cb, gpointer cbdata, GDestroyNotify dtor); +/** + * Trigger hyperscan compilation for regexp scopes that may have been updated + * @param map map that was updated + */ +void rspamd_map_trigger_hyperscan_compilation(struct rspamd_map *map); + #ifdef __cplusplus } #endif diff --git a/src/libserver/re_cache.c b/src/libserver/re_cache.c index 23022b9f07..ae1579b694 100644 --- a/src/libserver/re_cache.c +++ b/src/libserver/re_cache.c @@ -3234,3 +3234,161 @@ char **rspamd_re_cache_get_scope_names(struct rspamd_re_cache *cache_head, unsig *count_out = count; return names; } + +static gboolean +rspamd_re_cache_create_scope_lock(const char *cache_dir, const char *scope, int *lock_fd) +{ + char lock_path[PATH_MAX]; + pid_t myself = getpid(); + + if (!scope) { + scope = "default"; + } + + rspamd_snprintf(lock_path, sizeof(lock_path), "%s%c%s.scope.lock", + cache_dir, G_DIR_SEPARATOR, scope); + + *lock_fd = open(lock_path, O_WRONLY | O_CREAT | O_EXCL, 00600); + + if (*lock_fd == -1) { + if (errno == EEXIST || errno == EBUSY) { + /* Check if the lock is stale */ + int read_fd = open(lock_path, O_RDONLY); + if (read_fd != -1) { + pid_t lock_pid; + gssize r = read(read_fd, &lock_pid, sizeof(lock_pid)); + close(read_fd); + + if (r == sizeof(lock_pid)) { + /* Check if the process is still alive */ + if (lock_pid != myself && (kill(lock_pid, 0) == -1 && errno == ESRCH)) { + /* Stale lock, remove it */ + if (unlink(lock_path) == 0) { + /* Try to create lock again */ + *lock_fd = open(lock_path, O_WRONLY | O_CREAT | O_EXCL, 00600); + if (*lock_fd != -1) { + goto write_pid; + } + } + } + } + else { + /* Invalid lock file, remove it */ + if (unlink(lock_path) == 0) { + *lock_fd = open(lock_path, O_WRONLY | O_CREAT | O_EXCL, 00600); + if (*lock_fd != -1) { + goto write_pid; + } + } + } + } + } + return FALSE; + } + +write_pid: + /* Write our PID to the lock file */ + if (write(*lock_fd, &myself, sizeof(myself)) != sizeof(myself)) { + close(*lock_fd); + unlink(lock_path); + return FALSE; + } + + /* Lock the file */ + if (!rspamd_file_lock(*lock_fd, FALSE)) { + close(*lock_fd); + unlink(lock_path); + return FALSE; + } + + return TRUE; +} + +static void +rspamd_re_cache_remove_scope_lock(const char *cache_dir, const char *scope, int lock_fd) +{ + char lock_path[PATH_MAX]; + + if (!scope) { + scope = "default"; + } + + rspamd_snprintf(lock_path, sizeof(lock_path), "%s%c%s.scope.lock", + cache_dir, G_DIR_SEPARATOR, scope); + + if (lock_fd != -1) { + rspamd_file_unlock(lock_fd, FALSE); + close(lock_fd); + } + unlink(lock_path); +} + +#ifdef WITH_HYPERSCAN +struct rspamd_re_cache_hs_compile_scoped_cbdata { + struct rspamd_re_cache *cache; + const char *cache_dir; + const char *scope; + double max_time; + gboolean silent; + int lock_fd; + void (*cb)(const char *scope, unsigned int ncompiled, GError *err, void *cbd); + void *cbd; +}; + +static void +rspamd_re_cache_compile_scoped_cb(unsigned int ncompiled, GError *err, void *cbd) +{ + struct rspamd_re_cache_hs_compile_scoped_cbdata *scoped_cbd = + (struct rspamd_re_cache_hs_compile_scoped_cbdata *) cbd; + + /* Remove lock */ + rspamd_re_cache_remove_scope_lock(scoped_cbd->cache_dir, scoped_cbd->scope, + scoped_cbd->lock_fd); + + /* Call original callback */ + if (scoped_cbd->cb) { + scoped_cbd->cb(scoped_cbd->scope, ncompiled, err, scoped_cbd->cbd); + } + + g_free(scoped_cbd); +} + +int rspamd_re_cache_compile_hyperscan_scoped_single(struct rspamd_re_cache *cache, + const char *scope, + const char *cache_dir, + double max_time, + gboolean silent, + struct ev_loop *event_loop, + void (*cb)(const char *scope, unsigned int ncompiled, GError *err, void *cbd), + void *cbd) +{ + struct rspamd_re_cache_hs_compile_scoped_cbdata *scoped_cbd; + int lock_fd = -1; + + g_assert(cache != NULL); + g_assert(cache_dir != NULL); + + /* Try to acquire lock for this scope */ + if (!rspamd_re_cache_create_scope_lock(cache_dir, scope, &lock_fd)) { + /* Another process is compiling this scope */ + if (cb) { + cb(scope, 0, NULL, cbd); + } + return 0; + } + + /* Create callback data */ + scoped_cbd = g_malloc0(sizeof(*scoped_cbd)); + scoped_cbd->cache = cache; + scoped_cbd->cache_dir = cache_dir; + scoped_cbd->scope = scope; + scoped_cbd->max_time = max_time; + scoped_cbd->silent = silent; + scoped_cbd->lock_fd = lock_fd; + scoped_cbd->cb = cb; + scoped_cbd->cbd = cbd; + + return rspamd_re_cache_compile_hyperscan(cache, cache_dir, max_time, silent, + event_loop, rspamd_re_cache_compile_scoped_cb, scoped_cbd); +} +#endif diff --git a/src/libserver/re_cache.h b/src/libserver/re_cache.h index fbd243723e..eb236f197f 100644 --- a/src/libserver/re_cache.h +++ b/src/libserver/re_cache.h @@ -274,6 +274,18 @@ enum rspamd_hyperscan_status rspamd_re_cache_load_hyperscan_scoped( struct rspamd_re_cache *cache_head, const char *cache_dir, bool try_load); +/** + * Compile expressions to the hyperscan tree for a single scope with locking + */ +int rspamd_re_cache_compile_hyperscan_scoped_single(struct rspamd_re_cache *cache, + const char *scope, + const char *cache_dir, + double max_time, + gboolean silent, + struct ev_loop *event_loop, + void (*cb)(const char *scope, unsigned int ncompiled, GError *err, void *cbd), + void *cbd); + /** * Registers lua selector in the cache */ diff --git a/src/libserver/rspamd_control.c b/src/libserver/rspamd_control.c index 9e35cb5755..deab5064b1 100644 --- a/src/libserver/rspamd_control.c +++ b/src/libserver/rspamd_control.c @@ -1065,30 +1065,58 @@ rspamd_srv_handler(EV_P_ ev_io *w, int revents) case RSPAMD_SRV_HYPERSCAN_LOADED: #ifdef WITH_HYPERSCAN /* Load RE cache to provide it for new forks */ - if (rspamd_re_cache_is_hs_loaded(rspamd_main->cfg->re_cache) != RSPAMD_HYPERSCAN_LOADED_FULL || - cmd.cmd.hs_loaded.forced) { - rspamd_re_cache_load_hyperscan( + if (cmd.cmd.hs_loaded.scope[0] != '\0') { + /* Scoped loading */ + const char *scope = cmd.cmd.hs_loaded.scope; + msg_info_main("received scoped hyperscan cache loaded from %s for scope: %s", + cmd.cmd.hs_loaded.cache_dir, scope); + + /* Load specific scope */ + rspamd_re_cache_load_hyperscan_scoped( rspamd_main->cfg->re_cache, cmd.cmd.hs_loaded.cache_dir, false); - } - - /* After getting this notice, we can clean up old hyperscan files */ - - rspamd_hyperscan_notice_loaded(); - msg_info_main("received hyperscan cache loaded from %s", - cmd.cmd.hs_loaded.cache_dir); + /* Broadcast scoped command to all workers */ + memset(&wcmd, 0, sizeof(wcmd)); + wcmd.type = RSPAMD_CONTROL_HYPERSCAN_LOADED; + rspamd_strlcpy(wcmd.cmd.hs_loaded.cache_dir, + cmd.cmd.hs_loaded.cache_dir, + sizeof(wcmd.cmd.hs_loaded.cache_dir)); + rspamd_strlcpy(wcmd.cmd.hs_loaded.scope, + cmd.cmd.hs_loaded.scope, + sizeof(wcmd.cmd.hs_loaded.scope)); + wcmd.cmd.hs_loaded.forced = cmd.cmd.hs_loaded.forced; + rspamd_control_broadcast_cmd(rspamd_main, &wcmd, rfd, + rspamd_control_ignore_io_handler, NULL, worker->pid); + } + else { + /* Legacy full cache loading */ + if (rspamd_re_cache_is_hs_loaded(rspamd_main->cfg->re_cache) != RSPAMD_HYPERSCAN_LOADED_FULL || + cmd.cmd.hs_loaded.forced) { + rspamd_re_cache_load_hyperscan( + rspamd_main->cfg->re_cache, + cmd.cmd.hs_loaded.cache_dir, + false); + } - /* Broadcast command to all workers */ - memset(&wcmd, 0, sizeof(wcmd)); - wcmd.type = RSPAMD_CONTROL_HYPERSCAN_LOADED; - rspamd_strlcpy(wcmd.cmd.hs_loaded.cache_dir, - cmd.cmd.hs_loaded.cache_dir, - sizeof(wcmd.cmd.hs_loaded.cache_dir)); - wcmd.cmd.hs_loaded.forced = cmd.cmd.hs_loaded.forced; - rspamd_control_broadcast_cmd(rspamd_main, &wcmd, rfd, - rspamd_control_ignore_io_handler, NULL, worker->pid); + /* After getting this notice, we can clean up old hyperscan files */ + rspamd_hyperscan_notice_loaded(); + + msg_info_main("received hyperscan cache loaded from %s", + cmd.cmd.hs_loaded.cache_dir); + + /* Broadcast command to all workers */ + memset(&wcmd, 0, sizeof(wcmd)); + wcmd.type = RSPAMD_CONTROL_HYPERSCAN_LOADED; + rspamd_strlcpy(wcmd.cmd.hs_loaded.cache_dir, + cmd.cmd.hs_loaded.cache_dir, + sizeof(wcmd.cmd.hs_loaded.cache_dir)); + wcmd.cmd.hs_loaded.forced = cmd.cmd.hs_loaded.forced; + wcmd.cmd.hs_loaded.scope[0] = '\0'; /* Empty scope for legacy */ + rspamd_control_broadcast_cmd(rspamd_main, &wcmd, rfd, + rspamd_control_ignore_io_handler, NULL, worker->pid); + } #endif break; case RSPAMD_SRV_MONITORED_CHANGE: diff --git a/src/libserver/rspamd_control.h b/src/libserver/rspamd_control.h index a08ba7948a..92bdec85d5 100644 --- a/src/libserver/rspamd_control.h +++ b/src/libserver/rspamd_control.h @@ -74,6 +74,7 @@ struct rspamd_control_command { struct { gboolean forced; char cache_dir[CONTROL_PATHLEN]; + char scope[64]; /* Scope name, NULL means all scopes */ } hs_loaded; struct { char tag[32]; @@ -164,6 +165,7 @@ struct rspamd_srv_command { struct { gboolean forced; char cache_dir[CONTROL_PATHLEN]; + char scope[64]; /* Scope name, NULL means all scopes */ } hs_loaded; struct { char tag[32]; diff --git a/src/libserver/worker_util.c b/src/libserver/worker_util.c index 685ee9cd2d..fdcc5a4b3e 100644 --- a/src/libserver/worker_util.c +++ b/src/libserver/worker_util.c @@ -1908,14 +1908,27 @@ rspamd_worker_hyperscan_ready(struct rspamd_main *rspamd_main, memset(&rep, 0, sizeof(rep)); rep.type = RSPAMD_CONTROL_HYPERSCAN_LOADED; - if (rspamd_re_cache_is_hs_loaded(cache) != RSPAMD_HYPERSCAN_LOADED_FULL || - cmd->cmd.hs_loaded.forced) { + /* Check if this is a scoped notification */ + if (cmd->cmd.hs_loaded.scope[0] != '\0') { + /* Scoped hyperscan loading */ + const char *scope = cmd->cmd.hs_loaded.scope; - msg_info("loading hyperscan expressions after receiving compilation " - "notice: %s", - (rspamd_re_cache_is_hs_loaded(cache) != RSPAMD_HYPERSCAN_LOADED_FULL) ? "new db" : "forced update"); - rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan( - worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir, false); + msg_info("loading hyperscan expressions for scope '%s' after receiving compilation notice", scope); + + rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan_scoped( + cache, cmd->cmd.hs_loaded.cache_dir, false); + } + else { + /* Legacy/full cache loading */ + if (rspamd_re_cache_is_hs_loaded(cache) != RSPAMD_HYPERSCAN_LOADED_FULL || + cmd->cmd.hs_loaded.forced) { + + msg_info("loading hyperscan expressions after receiving compilation " + "notice: %s", + (rspamd_re_cache_is_hs_loaded(cache) != RSPAMD_HYPERSCAN_LOADED_FULL) ? "new db" : "forced update"); + rep.reply.hs_loaded.status = rspamd_re_cache_load_hyperscan( + worker->srv->cfg->re_cache, cmd->cmd.hs_loaded.cache_dir, false); + } } if (write(fd, &rep, sizeof(rep)) != sizeof(rep)) { @@ -2556,4 +2569,4 @@ rspamd_metrics_to_prometheus_string(const ucl_object_t *top) /* Must be finalized and freed by caller */ return output; -} \ No newline at end of file +} diff --git a/src/lua/lua_map.c b/src/lua/lua_map.c index 284aa46b6f..fa375cf63a 100644 --- a/src/lua/lua_map.c +++ b/src/lua/lua_map.c @@ -170,6 +170,13 @@ LUA_FUNCTION_DEF(map, get_data_digest); */ LUA_FUNCTION_DEF(map, get_nelts); +/*** + * @method map:trigger_hyperscan_compilation() + * Trigger hyperscan compilation for regexp scopes that may have been updated by this map + * This should be called after map loading is complete for maps that update regexp scopes + */ +LUA_FUNCTION_DEF(map, trigger_hyperscan_compilation); + static const struct luaL_reg maplib_m[] = { LUA_INTERFACE_DEF(map, get_key), LUA_INTERFACE_DEF(map, is_signed), @@ -183,6 +190,7 @@ static const struct luaL_reg maplib_m[] = { LUA_INTERFACE_DEF(map, on_load), LUA_INTERFACE_DEF(map, get_data_digest), LUA_INTERFACE_DEF(map, get_nelts), + LUA_INTERFACE_DEF(map, trigger_hyperscan_compilation), {"__tostring", rspamd_lua_class_tostring}, {NULL, NULL}}; @@ -1526,6 +1534,21 @@ lua_map_on_load(lua_State *L) return 0; } +static int +lua_map_trigger_hyperscan_compilation(lua_State *L) +{ + LUA_TRACE_POINT; + struct rspamd_lua_map *map = lua_check_map(L, 1); + + if (map == NULL) { + return luaL_error(L, "invalid arguments"); + } + + rspamd_map_trigger_hyperscan_compilation(map->map); + + return 0; +} + void luaopen_map(lua_State *L) { rspamd_lua_new_class(L, rspamd_map_classname, maplib_m); diff --git a/src/plugins/lua/multimap.lua b/src/plugins/lua/multimap.lua index 92a857fd52..4e73c414b9 100644 --- a/src/plugins/lua/multimap.lua +++ b/src/plugins/lua/multimap.lua @@ -1747,6 +1747,10 @@ local function add_multimap_rule(key, newrule) if rspamd_config:find_regexp_scope(scope_name) then rspamd_config:set_regexp_scope_loaded(scope_name, true) lua_util.debugm(N, rspamd_config, 'marked regexp scope %s as loaded after map processing', scope_name) + + -- Trigger hyperscan compilation for this updated scope + newrule.map_obj:trigger_hyperscan_compilation() + lua_util.debugm(N, rspamd_config, 'triggered hyperscan compilation for scope %s after map loading', scope_name) else lua_util.debugm(N, rspamd_config, 'regexp scope %s not created (empty map)', scope_name) end