From: Vsevolod Stakhov Date: Tue, 12 Apr 2022 20:31:56 +0000 (+0100) Subject: [Project] Some more adjustments in symbols registration X-Git-Tag: 3.3~293^2~35 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=3d8a5284b352cd0b080a2edcbf7373406f95489f;p=thirdparty%2Frspamd.git [Project] Some more adjustments in symbols registration --- diff --git a/src/libserver/symcache/symcache_c.cxx b/src/libserver/symcache/symcache_c.cxx index 7c204b04e6..8ba991f53d 100644 --- a/src/libserver/symcache/symcache_c.cxx +++ b/src/libserver/symcache/symcache_c.cxx @@ -54,3 +54,32 @@ rspamd_symcache_save (struct rspamd_symcache *cache) real_cache->save_items(); } + +gint +rspamd_symcache_add_symbol (struct rspamd_symcache *cache, + const gchar *name, + gint priority, + symbol_func_t func, + gpointer user_data, + enum rspamd_symbol_type type, + gint parent) +{ + auto *real_cache = C_API_SYMCACHE(cache); + + if (func) { + g_assert (parent == -1); + + return real_cache->add_symbol_with_callback(name, priority, func, user_data, type); + } + else { + return real_cache->add_virtual_symbol(name, parent, type); + } +} + +void +rspamd_symcache_set_peak_callback (struct rspamd_symcache *cache, gint cbref) +{ + auto *real_cache = C_API_SYMCACHE(cache); + + real_cache->set_peak_cb(cbref); +} \ No newline at end of file diff --git a/src/libserver/symcache/symcache_impl.cxx b/src/libserver/symcache/symcache_impl.cxx index 3da915ebe4..92cd6f3fb9 100644 --- a/src/libserver/symcache/symcache_impl.cxx +++ b/src/libserver/symcache/symcache_impl.cxx @@ -18,6 +18,7 @@ #include "unix-std.h" #include "libutil/cxx/locked_file.hxx" #include "fmt/core.h" +#include "contrib/t1ha/t1ha.h" #include @@ -556,6 +557,18 @@ auto symcache::add_symbol_with_callback(std::string_view name, auto real_type_pair = real_type_pair_maybe.value(); + if (real_type_pair.first != symcache_item_type::FILTER) { + real_type_pair.second |= SYMBOL_TYPE_NOSTAT; + } + if (real_type_pair.second & (SYMBOL_TYPE_GHOST|SYMBOL_TYPE_CALLBACK)) { + real_type_pair.second |= SYMBOL_TYPE_NOSTAT; + } + + if ((real_type_pair.second & SYMBOL_TYPE_FINE) && priority == 0) { + /* Adjust priority for negative weighted symbols */ + priority = 1; + } + std::string static_string_name; if (name.empty()) { @@ -570,14 +583,23 @@ auto symcache::add_symbol_with_callback(std::string_view name, return -1; } - auto item = cache_item::create_with_function(std::move(static_string_name), + auto id = items_by_id.size(); + + auto item = cache_item::create_with_function(static_pool, id, + std::move(static_string_name), priority, func, user_data, real_type_pair.first, real_type_pair.second); items_by_symbol[item->get_name()] = item; items_by_id.push_back(item); + used_items++; + + if (!(real_type_pair.second & SYMBOL_TYPE_NOSTAT)) { + cksum = t1ha(name.data(), name.size(), cksum); + stats_symbols_count ++; + } - return items_by_id.size(); + return id; } auto symcache::add_virtual_symbol(std::string_view name, int parent_id, enum rspamd_symbol_type flags_and_type) -> int @@ -602,12 +624,26 @@ auto symcache::add_virtual_symbol(std::string_view name, int parent_id, enum rsp return -1; } - auto item = cache_item::create_with_virtual(std::string{name}, + auto id = virtual_symbols.size(); + + auto item = cache_item::create_with_virtual(static_pool, + id, + std::string{name}, parent_id, real_type_pair.first, real_type_pair.second); items_by_symbol[item->get_name()] = item; virtual_symbols.push_back(item); - return virtual_symbols.size(); + return id; +} + +auto symcache::set_peak_cb(int cbref) -> void +{ + if (peak_cb != -1) { + luaL_unref(L, LUA_REGISTRYINDEX, peak_cb); + } + + peak_cb = cbref; + msg_info_cache("registered peak callback"); } @@ -727,6 +763,28 @@ auto cache_item::process_deps(const symcache &cache) -> void [](const auto &dep){ return !dep.item; }), std::end(deps)); } +auto cache_item::resolve_parent(const symcache &cache) -> bool +{ + auto log_tag = [&](){ return cache.log_tag(); }; + + if (is_virtual()) { + auto &virt = std::get(specific); + + if (virt.get_parent(cache)) { + msg_warn_cache("trying to resolve parent twice for %s", symbol.c_str()); + + return false; + } + + return virt.resolve_parent(cache); + } + else { + msg_warn_cache("trying to resolve a parent for non-virtual symbol %s", symbol.c_str()); + } + + return false; +} + auto virtual_item::get_parent(const symcache &cache) const -> const cache_item * { if (parent) { @@ -736,6 +794,23 @@ auto virtual_item::get_parent(const symcache &cache) const -> const cache_item * return cache.get_item_by_id(parent_id, false); } +auto virtual_item::resolve_parent(const symcache &cache) -> bool +{ + if (parent) { + return false; + } + + auto item_ptr = cache.get_item_by_id(parent_id, true); + + if (item_ptr) { + parent = const_cast(item_ptr)->getptr(); + + return true; + } + + return false; +} + auto item_type_from_c(enum rspamd_symbol_type type) -> tl::expected, std::string> { constexpr const auto trivial_types = SYMBOL_TYPE_CONNFILTER|SYMBOL_TYPE_PREFILTER diff --git a/src/libserver/symcache/symcache_internal.hxx b/src/libserver/symcache/symcache_internal.hxx index 674f4034d6..77f48de19a 100644 --- a/src/libserver/symcache/symcache_internal.hxx +++ b/src/libserver/symcache/symcache_internal.hxx @@ -149,6 +149,7 @@ public: explicit virtual_item(int _parent_id) : parent_id(_parent_id) {} auto get_parent(const symcache &cache) const -> const cache_item *; + auto resolve_parent(const symcache &cache) -> bool; }; struct cache_dependency { @@ -167,6 +168,8 @@ struct cache_item : std::enable_shared_from_this { struct rspamd_symcache_item_stat *st = nullptr; struct rspamd_counter_data *cd = nullptr; + /* Unique id - counter */ + int id; std::uint64_t last_count = 0; std::string symbol; symcache_item_type type; @@ -179,8 +182,6 @@ struct cache_item : std::enable_shared_from_this { int priority = 0; /* Topological order */ unsigned int order = 0; - /* Unique id - counter */ - int id = 0; int frequency_peaks = 0; /* Specific data for virtual and callback symbols */ @@ -208,15 +209,18 @@ public: * @param flags * @return */ - [[nodiscard]] static auto create_with_function(std::string &&name, + [[nodiscard]] static auto create_with_function(rspamd_mempool_t *pool, + int id, + std::string &&name, int priority, symbol_func_t func, void *user_data, symcache_item_type type, int flags) -> cache_item_ptr { - return std::shared_ptr(new cache_item(std::move(name), priority, - func, user_data, - type, flags)); + return std::shared_ptr(new cache_item(pool, + id, std::move(name), priority, + func, user_data, + type, flags)); } /** * Create a virtual item @@ -227,11 +231,13 @@ public: * @param flags * @return */ - [[nodiscard]] static auto create_with_virtual(std::string &&name, - int parent, - symcache_item_type type, - int flags) -> cache_item_ptr { - return std::shared_ptr(new cache_item(std::move(name), + [[nodiscard]] static auto create_with_virtual(rspamd_mempool_t *pool, + int id, + std::string &&name, + int parent, + symcache_item_type type, + int flags) -> cache_item_ptr { + return std::shared_ptr(new cache_item(pool, id, std::move(name), parent, type, flags)); } /** @@ -255,6 +261,7 @@ public: return flags & SYMBOL_TYPE_GHOST; } auto get_parent(const symcache &cache) const -> const cache_item *; + auto resolve_parent(const symcache &cache) -> bool; auto get_type() const -> auto { return type; } @@ -282,12 +289,15 @@ private: * @param _type * @param _flags */ - cache_item(std::string &&name, + cache_item(rspamd_mempool_t *pool, + int _id, + std::string &&name, int _priority, symbol_func_t func, void *user_data, symcache_item_type _type, - int _flags) : symbol(std::move(name)), + int _flags) : id(_id), + symbol(std::move(name)), type(_type), flags(_flags), priority(_priority), @@ -296,6 +306,8 @@ private: forbidden_ids.reset(); allowed_ids.reset(); exec_only_ids.reset(); + st = rspamd_mempool_alloc0_shared_type(pool, std::remove_pointer_t); + cd = rspamd_mempool_alloc0_shared_type(pool, std::remove_pointer_t); } /** * Constructor for a virtual symbol @@ -305,10 +317,13 @@ private: * @param _type * @param _flags */ - cache_item(std::string &&name, + cache_item(rspamd_mempool_t *pool, + int _id, + std::string &&name, int parent, symcache_item_type _type, - int _flags) : symbol(std::move(name)), + int _flags) : id(_id), + symbol(std::move(name)), type(_type), flags(_flags), specific(virtual_item{parent}) @@ -316,6 +331,8 @@ private: forbidden_ids.reset(); allowed_ids.reset(); exec_only_ids.reset(); + st = rspamd_mempool_alloc0_shared_type(pool, std::remove_pointer_t); + cd = rspamd_mempool_alloc0_shared_type(pool, std::remove_pointer_t); } }; @@ -476,6 +493,12 @@ public: */ auto add_virtual_symbol(std::string_view name, int parent_id, enum rspamd_symbol_type flags_and_type) -> int; + + /** + * Sets a lua callback to be called on peaks in execution time + * @param cbref + */ + auto set_peak_cb(int cbref) -> void; }; /* diff --git a/src/libutil/mem_pool.h b/src/libutil/mem_pool.h index 0ca1d3905a..a9f4fec987 100644 --- a/src/libutil/mem_pool.h +++ b/src/libutil/mem_pool.h @@ -248,11 +248,17 @@ void *rspamd_mempool_alloc_shared_ (rspamd_mempool_t *pool, gsize size, gsize al RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; #define rspamd_mempool_alloc_shared(pool, size) \ rspamd_mempool_alloc_shared_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC)) +#define rspamd_mempool_alloc_shared_type(pool, type) \ + (type *)(rspamd_mempool_alloc_shared_((pool), sizeof(type), \ + MAX(MIN_MEM_ALIGNMENT, RSPAMD_ALIGNOF(type)), (G_STRLOC))) void *rspamd_mempool_alloc0_shared_ (rspamd_mempool_t *pool, gsize size, gsize alignment, const gchar *loc) RSPAMD_ATTR_ALLOC_SIZE(2) RSPAMD_ATTR_ALLOC_ALIGN(MIN_MEM_ALIGNMENT) RSPAMD_ATTR_RETURNS_NONNUL; #define rspamd_mempool_alloc0_shared(pool, size) \ rspamd_mempool_alloc0_shared_((pool), (size), MIN_MEM_ALIGNMENT, (G_STRLOC)) +#define rspamd_mempool_alloc0_shared_type(pool, type) \ + (type *)(rspamd_mempool_alloc0_shared_((pool), sizeof(type), \ + MAX(MIN_MEM_ALIGNMENT, RSPAMD_ALIGNOF(type)), (G_STRLOC))) /** * Add destructor callback to pool