From 9416ae7f2920d8dece1366572a7c0e494ff9ba70 Mon Sep 17 00:00:00 2001 From: Vsevolod Stakhov Date: Tue, 21 Oct 2025 10:56:57 +0100 Subject: [PATCH] [Feature] Use specialized pool types for long-lived and short-lived allocations Replace generic rspamd_mempool_new calls with specialized variants to optimize allocation strategies based on pool lifetime. Long-lived pools (RSPAMD_MEMPOOL_LONG_LIVED): - server_pool (main server context, rspamadm) - cfg_pool (configuration, 8MB) - Module contexts (fuzzy_check, fuzzy_backend) - Upstream context - Radix trees - Maps (hash, radix, regexp - 5 instances) - Static Lua pools (url, regexp) Short-lived pools (RSPAMD_MEMPOOL_SHORT_LIVED): - HTTP/controller sessions - Task pools (message processing) - Milter sessions - Proxy sessions (2 instances) - Lua parsers (HTML, email - 2 instances) - Lua mempool creation - Lua worker control Benefits: - Long-lived pools: larger initial page size, fewer reallocations - Short-lived pools: smaller initial size, faster allocation/deallocation - Better memory efficiency based on usage patterns --- src/controller.c | 3 +-- src/libserver/cfg_utils.cxx | 2 +- .../fuzzy_backend/fuzzy_backend_sqlite.c | 4 ++-- src/libserver/maps/map_helpers.c | 20 +++++++++---------- src/libserver/milter.c | 2 +- src/libserver/task.c | 6 ++++-- src/libutil/radix.c | 2 +- src/libutil/upstream.c | 4 ++-- src/lua/lua_mempool.c | 3 +-- src/lua/lua_parsers.c | 5 ++--- src/lua/lua_regexp.c | 4 ++-- src/lua/lua_url.c | 4 ++-- src/lua/lua_worker.c | 3 +-- src/plugins/fuzzy_check.c | 4 ++-- src/rspamadm/rspamadm.c | 4 ++-- src/rspamd.c | 4 ++-- src/rspamd_proxy.c | 5 ++--- 17 files changed, 38 insertions(+), 41 deletions(-) diff --git a/src/controller.c b/src/controller.c index 31d8bd9004..98ff12453d 100644 --- a/src/controller.c +++ b/src/controller.c @@ -3636,8 +3636,7 @@ rspamd_controller_accept_socket(EV_P_ ev_io *w, int revents) } session = g_malloc0(sizeof(struct rspamd_controller_session)); - session->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "csession", 0); + session->pool = rspamd_mempool_new_short_lived("csession"); session->ctx = ctx; session->cfg = ctx->cfg; session->lang_det = ctx->lang_det; diff --git a/src/libserver/cfg_utils.cxx b/src/libserver/cfg_utils.cxx index 3ae7dbdfc5..55ed5a18ba 100644 --- a/src/libserver/cfg_utils.cxx +++ b/src/libserver/cfg_utils.cxx @@ -226,7 +226,7 @@ rspamd_config_new(enum rspamd_config_init_flags flags) struct rspamd_config *cfg; rspamd_mempool_t *pool; - pool = rspamd_mempool_new(8 * 1024 * 1024, "cfg", 0); + pool = rspamd_mempool_new_long_lived(8 * 1024 * 1024, "cfg"); cfg = rspamd_mempool_alloc0_type(pool, struct rspamd_config); /* Allocate larger pool for cfg */ cfg->cfg_pool = pool; diff --git a/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.c b/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.c index 8ba6254465..f3c1690e06 100644 --- a/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.c +++ b/src/libserver/fuzzy_backend/fuzzy_backend_sqlite.c @@ -396,8 +396,8 @@ rspamd_fuzzy_backend_sqlite_open_db(const char *path, GError **err) bk = g_malloc0(sizeof(*bk)); bk->path = g_strdup(path); bk->expired = 0; - bk->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "fuzzy_backend", 0); + bk->pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + "fuzzy_backend"); bk->db = rspamd_sqlite3_open_or_create(bk->pool, bk->path, create_tables_sql, 1, err); diff --git a/src/libserver/maps/map_helpers.c b/src/libserver/maps/map_helpers.c index bd2a2f369b..7e7d701740 100644 --- a/src/libserver/maps/map_helpers.c +++ b/src/libserver/maps/map_helpers.c @@ -746,12 +746,12 @@ rspamd_map_helper_new_hash(struct rspamd_map *map) rspamd_mempool_t *pool; if (map) { - pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - map->tag, 0); + pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + map->tag); } else { - pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - NULL, 0); + pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + "map"); } htb = rspamd_mempool_alloc0_type(pool, struct rspamd_hash_map_helper); @@ -804,13 +804,13 @@ rspamd_map_helper_new_radix(struct rspamd_map *map) const char *name = "unnamed"; if (map) { - pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - map->tag, 0); + pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + map->tag); name = map->name; } else { - pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - NULL, 0); + pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + "radix_map"); } r = rspamd_mempool_alloc0_type(pool, struct rspamd_radix_map_helper); @@ -863,8 +863,8 @@ rspamd_map_helper_new_regexp(struct rspamd_map *map, struct rspamd_regexp_map_helper *re_map; rspamd_mempool_t *pool; - pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - map->tag, 0); + pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + map->tag); re_map = rspamd_mempool_alloc0_type(pool, struct rspamd_regexp_map_helper); re_map->pool = pool; diff --git a/src/libserver/milter.c b/src/libserver/milter.c index b811521afe..3f2c67f821 100644 --- a/src/libserver/milter.c +++ b/src/libserver/milter.c @@ -1398,7 +1398,7 @@ rspamd_milter_handle_socket(int fd, ev_tstamp timeout, priv->parser.buf = rspamd_fstring_sized_new(RSPAMD_MILTER_MESSAGE_CHUNK + 5); priv->event_loop = ev_base; priv->state = RSPAMD_MILTER_READ_MORE; - priv->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "milter", 0); + priv->pool = rspamd_mempool_new_short_lived("milter"); priv->discard_on_reject = milter_ctx->discard_on_reject; priv->quarantine_on_reject = milter_ctx->quarantine_on_reject; priv->ev.timeout = timeout; diff --git a/src/libserver/task.c b/src/libserver/task.c index b38c07e264..7e90d661b5 100644 --- a/src/libserver/task.c +++ b/src/libserver/task.c @@ -73,8 +73,10 @@ rspamd_task_new(struct rspamd_worker *worker, unsigned int flags = RSPAMD_TASK_FLAG_LEARN_AUTO; if (pool == NULL) { - task_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "task", debug_mem ? RSPAMD_MEMPOOL_DEBUG : 0); + task_pool = rspamd_mempool_new_(rspamd_mempool_suggest_size_(G_STRLOC), + "task", + RSPAMD_MEMPOOL_SHORT_LIVED | (debug_mem ? RSPAMD_MEMPOOL_DEBUG : 0), + G_STRLOC); flags |= RSPAMD_TASK_FLAG_OWN_POOL; } else { diff --git a/src/libutil/radix.c b/src/libutil/radix.c index bdd722b499..b9f09a1a0a 100644 --- a/src/libutil/radix.c +++ b/src/libutil/radix.c @@ -172,7 +172,7 @@ radix_create_compressed(const char *tree_name) return NULL; } - tree->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0); + tree->pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), "radix"); tree->size = 0; tree->duplicates = 0; tree->tree = btrie_init(tree->pool); diff --git a/src/libutil/upstream.c b/src/libutil/upstream.c index 1dd2d9f62e..85964c22a6 100644 --- a/src/libutil/upstream.c +++ b/src/libutil/upstream.c @@ -301,8 +301,8 @@ rspamd_upstreams_library_init(void) ctx = g_malloc0(sizeof(*ctx)); memcpy(&ctx->limits, &default_limits, sizeof(ctx->limits)); - ctx->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "upstreams", 0); + ctx->pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + "upstreams"); ctx->upstreams = g_queue_new(); REF_INIT_RETAIN(ctx, rspamd_upstream_ctx_dtor); diff --git a/src/lua/lua_mempool.c b/src/lua/lua_mempool.c index ddc9b25e9c..2c6c994380 100644 --- a/src/lua/lua_mempool.c +++ b/src/lua/lua_mempool.c @@ -159,8 +159,7 @@ static int lua_mempool_create(lua_State *L) { LUA_TRACE_POINT; - struct memory_pool_s *mempool = rspamd_mempool_new( - rspamd_mempool_suggest_size(), "lua", 0), + struct memory_pool_s *mempool = rspamd_mempool_new_short_lived("lua"), **pmempool; if (mempool) { diff --git a/src/lua/lua_parsers.c b/src/lua/lua_parsers.c index eb7fa6bf50..a3174d74f0 100644 --- a/src/lua/lua_parsers.c +++ b/src/lua/lua_parsers.c @@ -231,7 +231,7 @@ int lua_parsers_parse_html(lua_State *L) } if (start != NULL) { - pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), NULL, 0); + pool = rspamd_mempool_new_short_lived("lua_parser"); in = g_byte_array_sized_new(len); g_byte_array_append(in, start, len); @@ -327,8 +327,7 @@ int lua_parsers_parse_mail_address(lua_State *L) } } else { - pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "lua parsers", 0); + pool = rspamd_mempool_new_short_lived("lua_parsers"); own_pool = TRUE; } diff --git a/src/lua/lua_regexp.c b/src/lua/lua_regexp.c index 4a209057e9..bff19714dd 100644 --- a/src/lua/lua_regexp.c +++ b/src/lua/lua_regexp.c @@ -846,8 +846,8 @@ lua_load_regexp(lua_State *L) void luaopen_regexp(lua_State *L) { if (!regexp_static_pool) { - regexp_static_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "regexp_lua_pool", 0); + regexp_static_pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + "regexp_lua_pool"); } rspamd_lua_new_class(L, rspamd_regexp_classname, regexplib_m); diff --git a/src/lua/lua_url.c b/src/lua/lua_url.c index 8e62318ab6..1d163dce0b 100644 --- a/src/lua/lua_url.c +++ b/src/lua/lua_url.c @@ -831,8 +831,8 @@ static rspamd_mempool_t *static_lua_url_pool; RSPAMD_CONSTRUCTOR(rspamd_urls_static_pool_ctor) { - static_lua_url_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "static_lua_url", 0); + static_lua_url_pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + "static_lua_url"); } RSPAMD_DESTRUCTOR(rspamd_urls_static_pool_dtor) diff --git a/src/lua/lua_worker.c b/src/lua/lua_worker.c index b18e9e5151..358727714d 100644 --- a/src/lua/lua_worker.c +++ b/src/lua/lua_worker.c @@ -420,8 +420,7 @@ lua_worker_add_control_handler(lua_State *L) return luaL_error(L, "invalid command type: %s", cmd_name); } - rspamd_mempool_t *pool = rspamd_mempool_new( - rspamd_mempool_suggest_size(), "lua_control", 0); + rspamd_mempool_t *pool = rspamd_mempool_new_short_lived("lua_control"); cbd = rspamd_mempool_alloc0(pool, sizeof(*cbd)); cbd->pool = pool; cbd->event_loop = event_loop; diff --git a/src/plugins/fuzzy_check.c b/src/plugins/fuzzy_check.c index 0c2f5a617f..052ad092c4 100644 --- a/src/plugins/fuzzy_check.c +++ b/src/plugins/fuzzy_check.c @@ -2038,8 +2038,8 @@ int fuzzy_check_module_init(struct rspamd_config *cfg, struct module_ctx **ctx) fuzzy_module_ctx = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(struct fuzzy_ctx)); - fuzzy_module_ctx->fuzzy_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - NULL, 0); + fuzzy_module_ctx->fuzzy_pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + "fuzzy"); /* TODO: this should match rules count actually */ fuzzy_module_ctx->keypairs_cache = rspamd_keypair_cache_new(32); fuzzy_module_ctx->fuzzy_rules = g_ptr_array_new(); diff --git a/src/rspamadm/rspamadm.c b/src/rspamadm/rspamadm.c index e323880f3b..4a19aa667e 100644 --- a/src/rspamadm/rspamadm.c +++ b/src/rspamadm/rspamadm.c @@ -393,8 +393,8 @@ int main(int argc, char **argv, char **env) rspamd_main->cfg = cfg; rspamd_main->pid = getpid(); rspamd_main->type = process_quark; - rspamd_main->server_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "rspamadm", 0); + rspamd_main->server_pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + "rspamadm"); rspamadm_fill_internal_commands(all_commands); help_command.command_data = all_commands; diff --git a/src/rspamd.c b/src/rspamd.c index ba1ea1fb85..58e35b9822 100644 --- a/src/rspamd.c +++ b/src/rspamd.c @@ -1438,8 +1438,8 @@ int main(int argc, char **argv, char **env) rspamd_main = (struct rspamd_main *) g_malloc0(sizeof(struct rspamd_main)); - rspamd_main->server_pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "main", 0); + rspamd_main->server_pool = rspamd_mempool_new_long_lived(rspamd_mempool_suggest_size(), + "main"); rspamd_main->stat = rspamd_mempool_alloc0_shared_(rspamd_main->server_pool, sizeof(struct rspamd_stat), RSPAMD_ALIGNOF(struct rspamd_stat), diff --git a/src/rspamd_proxy.c b/src/rspamd_proxy.c index 5321c86772..aed9076e82 100644 --- a/src/rspamd_proxy.c +++ b/src/rspamd_proxy.c @@ -1513,7 +1513,7 @@ proxy_session_refresh(struct rspamd_proxy_session *session) session->client_addr = NULL; nsession->ctx = session->ctx; nsession->worker = session->worker; - nsession->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), "proxy", 0); + nsession->pool = rspamd_mempool_new_short_lived("proxy"); nsession->client_sock = session->client_sock; session->client_sock = -1; nsession->mirror_conns = g_ptr_array_sized_new(nsession->ctx->mirrors->len); @@ -2990,8 +2990,7 @@ proxy_accept_socket(EV_P_ ev_io *w, int revents) session->client_addr = addr; session->mirror_conns = g_ptr_array_sized_new(ctx->mirrors->len); - session->pool = rspamd_mempool_new(rspamd_mempool_suggest_size(), - "proxy", 0); + session->pool = rspamd_mempool_new_short_lived("proxy"); session->ctx = ctx; session->worker = worker; -- 2.47.3