From: Vsevolod Stakhov Date: Sat, 23 Apr 2022 09:18:54 +0000 (+0100) Subject: [Project] Further split of the code X-Git-Tag: 3.3~293^2~19 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=00802c92400b4c0592c6c9ef1cbc13b9ca770e3c;p=thirdparty%2Frspamd.git [Project] Further split of the code --- diff --git a/src/libserver/CMakeLists.txt b/src/libserver/CMakeLists.txt index 17f5ca7511..d287f44c12 100644 --- a/src/libserver/CMakeLists.txt +++ b/src/libserver/CMakeLists.txt @@ -21,6 +21,8 @@ SET(LIBRSPAMDSERVERSRC ${CMAKE_CURRENT_SOURCE_DIR}/spf.c ${CMAKE_CURRENT_SOURCE_DIR}/ssl_util.c ${CMAKE_CURRENT_SOURCE_DIR}/symcache/symcache_impl.cxx + ${CMAKE_CURRENT_SOURCE_DIR}/symcache/symcache_item.cxx + ${CMAKE_CURRENT_SOURCE_DIR}/symcache/symcache_runtime.cxx ${CMAKE_CURRENT_SOURCE_DIR}/symcache/symcache_c.cxx ${CMAKE_CURRENT_SOURCE_DIR}/task.c ${CMAKE_CURRENT_SOURCE_DIR}/url.c diff --git a/src/libserver/symcache/symcache_impl.cxx b/src/libserver/symcache/symcache_impl.cxx index 48646de33f..e2032c063f 100644 --- a/src/libserver/symcache/symcache_impl.cxx +++ b/src/libserver/symcache/symcache_impl.cxx @@ -14,6 +14,7 @@ * limitations under the License. */ +#include "lua/lua_common.h" #include "symcache_internal.hxx" #include "symcache_item.hxx" #include "symcache_runtime.hxx" @@ -871,291 +872,11 @@ auto symcache::periodic_resort(struct ev_loop *ev_loop, double cur_time, double } } -auto cache_item::get_parent(const symcache &cache) const -> const cache_item * +symcache::~symcache() { - if (is_virtual()) { - const auto &virtual_sp = std::get(specific); - - return virtual_sp.get_parent(cache); - } - - return nullptr; -} - -auto cache_item::process_deps(const symcache &cache) -> void -{ - /* Allow logging macros to work */ - auto log_tag = [&]() { return cache.log_tag(); }; - - for (auto &dep: deps) { - msg_debug_cache ("process real dependency %s on %s", symbol.c_str(), dep.sym.c_str()); - auto *dit = cache.get_item_by_name_mut(dep.sym, true); - - if (dep.vid >= 0) { - /* Case of the virtual symbol that depends on another (maybe virtual) symbol */ - const auto *vdit = cache.get_item_by_name(dep.sym, false); - - if (!vdit) { - if (dit) { - msg_err_cache("cannot add dependency from %s on %s: no dependency symbol registered", - dep.sym.c_str(), dit->symbol.c_str()); - } - } - else { - msg_debug_cache("process virtual dependency %s(%d) on %s(%d)", symbol.c_str(), - dep.vid, vdit->symbol.c_str(), vdit->id); - - std::size_t nids = 0; - - /* Propagate ids */ - msg_debug_cache("check id propagation for dependency %s from %s", - symbol.c_str(), dit->symbol.c_str()); - - const auto *ids = dit->allowed_ids.get_ids(nids); - - if (nids > 0) { - msg_debug_cache("propagate allowed ids from %s to %s", - dit->symbol.c_str(), symbol.c_str()); - - allowed_ids.set_ids(ids, nids, cache.get_pool()); - } - - ids = dit->forbidden_ids.get_ids(nids); - - if (nids > 0) { - msg_debug_cache("propagate forbidden ids from %s to %s", - dit->symbol.c_str(), symbol.c_str()); - - forbidden_ids.set_ids(ids, nids, cache.get_pool()); - } - } - } - - if (dit != nullptr) { - if (!dit->is_filter()) { - /* - * Check sanity: - * - filters -> prefilter dependency is OK and always satisfied - * - postfilter -> (filter, prefilter) dep is ok - * - idempotent -> (any) dep is OK - * - * Otherwise, emit error - * However, even if everything is fine this dep is useless ¯\_(ツ)_/¯ - */ - auto ok_dep = false; - - if (dit->get_type() == type) { - ok_dep = true; - } - else if (type < dit->get_type()) { - ok_dep = true; - } - - if (!ok_dep) { - msg_err_cache ("cannot add dependency from %s on %s: invalid symbol types", - dep.sym.c_str(), symbol.c_str()); - - continue; - } - } - else { - if (dit->id == id) { - msg_err_cache ("cannot add dependency on self: %s -> %s " - "(resolved to %s)", - symbol.c_str(), dep.sym.c_str(), dit->symbol.c_str()); - } - else { - /* Create a reverse dep */ - dit->rdeps.emplace_back(getptr(), dep.sym, id, -1); - dep.item = dit->getptr(); - dep.id = dit->id; - - msg_debug_cache ("add dependency from %d on %d", id, - dit->id); - } - } - } - else if (dep.id >= 0) { - msg_err_cache ("cannot find dependency on symbol %s for symbol %s", - dep.sym.c_str(), symbol.c_str()); - - continue; - } - } - - // Remove empty deps - deps.erase(std::remove_if(std::begin(deps), std::end(deps), - [](const auto &dep) { return !dep.item; }), std::end(deps)); -} - -auto cache_item::resolve_parent(const symcache &cache) -> bool -{ - auto log_tag = [&]() { return cache.log_tag(); }; - - if (is_virtual()) { - auto &virt = std::get(specific); - - if (virt.get_parent(cache)) { - msg_warn_cache("trying to resolve parent twice for %s", symbol.c_str()); - - return false; - } - - return virt.resolve_parent(cache); - } - else { - msg_warn_cache("trying to resolve a parent for non-virtual symbol %s", symbol.c_str()); - } - - return false; -} - -auto cache_item::update_counters_check_peak(lua_State *L, - struct ev_loop *ev_loop, - double cur_time, - double last_resort) -> bool -{ - auto ret = false; - static const double decay_rate = 0.25; - - st->total_hits += st->hits; - g_atomic_int_set(&st->hits, 0); - - if (last_count > 0) { - auto cur_value = (st->total_hits - last_count) / - (cur_time - last_resort); - rspamd_set_counter_ema(&st->frequency_counter, - cur_value, decay_rate); - st->avg_frequency = st->frequency_counter.mean; - st->stddev_frequency = st->frequency_counter.stddev; - - auto cur_err = (st->avg_frequency - cur_value); - cur_err *= cur_err; - - if (st->frequency_counter.number > 10 && - cur_err > ::sqrt(st->stddev_frequency) * 3) { - frequency_peaks++; - ret = true; - } - } - - last_count = st->total_hits; - - if (cd->number > 0) { - if (!is_virtual()) { - st->avg_time = cd->mean; - rspamd_set_counter_ema(&st->time_counter, - st->avg_time, decay_rate); - st->avg_time = st->time_counter.mean; - memset(cd, 0, sizeof(*cd)); - } - } - - return ret; -} - -auto virtual_item::get_parent(const symcache &cache) const -> const cache_item * -{ - if (parent) { - return parent.get(); - } - - return cache.get_item_by_id(parent_id, false); -} - -auto virtual_item::resolve_parent(const symcache &cache) -> bool -{ - if (parent) { - return false; - } - - auto item_ptr = cache.get_item_by_id(parent_id, true); - - if (item_ptr) { - parent = const_cast(item_ptr)->getptr(); - - return true; - } - - return false; -} - -auto item_type_from_c(enum rspamd_symbol_type type) -> tl::expected, std::string> -{ - constexpr const auto trivial_types = SYMBOL_TYPE_CONNFILTER | SYMBOL_TYPE_PREFILTER - | SYMBOL_TYPE_POSTFILTER | SYMBOL_TYPE_IDEMPOTENT - | SYMBOL_TYPE_COMPOSITE | SYMBOL_TYPE_CLASSIFIER - | SYMBOL_TYPE_VIRTUAL; - - constexpr auto all_but_one_ty = [&](int type, int exclude_bit) -> auto { - return type & (trivial_types & ~exclude_bit); - }; - - if (type & trivial_types) { - auto check_trivial = [&](auto flag, - symcache_item_type ty) -> tl::expected, std::string> { - if (all_but_one_ty(type, flag)) { - return tl::make_unexpected(fmt::format("invalid flags for a symbol: {}", type)); - } - - return std::make_pair(ty, type & ~flag); - }; - if (type & SYMBOL_TYPE_CONNFILTER) { - return check_trivial(SYMBOL_TYPE_CONNFILTER, symcache_item_type::CONNFILTER); - } - else if (type & SYMBOL_TYPE_PREFILTER) { - return check_trivial(SYMBOL_TYPE_PREFILTER, symcache_item_type::PREFILTER); - } - else if (type & SYMBOL_TYPE_POSTFILTER) { - return check_trivial(SYMBOL_TYPE_POSTFILTER, symcache_item_type::POSTFILTER); - } - else if (type & SYMBOL_TYPE_IDEMPOTENT) { - return check_trivial(SYMBOL_TYPE_IDEMPOTENT, symcache_item_type::IDEMPOTENT); - } - else if (type & SYMBOL_TYPE_COMPOSITE) { - return check_trivial(SYMBOL_TYPE_COMPOSITE, symcache_item_type::COMPOSITE); - } - else if (type & SYMBOL_TYPE_CLASSIFIER) { - return check_trivial(SYMBOL_TYPE_CLASSIFIER, symcache_item_type::CLASSIFIER); - } - else if (type & SYMBOL_TYPE_VIRTUAL) { - return check_trivial(SYMBOL_TYPE_VIRTUAL, symcache_item_type::VIRTUAL); - } - - return tl::make_unexpected(fmt::format("internal error: impossible flags combination", type)); - } - - /* Maybe check other flags combination here? */ - return std::make_pair(symcache_item_type::FILTER, type); -} - -bool operator<(symcache_item_type lhs, symcache_item_type rhs) -{ - auto ret = false; - switch (lhs) { - case symcache_item_type::CONNFILTER: - break; - case symcache_item_type::PREFILTER: - if (rhs == symcache_item_type::CONNFILTER) { - ret = true; - } - break; - case symcache_item_type::FILTER: - if (rhs == symcache_item_type::CONNFILTER || rhs == symcache_item_type::PREFILTER) { - ret = true; - } - break; - case symcache_item_type::POSTFILTER: - if (rhs != symcache_item_type::IDEMPOTENT) { - ret = true; - } - break; - case symcache_item_type::IDEMPOTENT: - default: - break; + if (peak_cb != -1) { + luaL_unref(L, LUA_REGISTRYINDEX, peak_cb); } - - return ret; } } \ No newline at end of file diff --git a/src/libserver/symcache/symcache_internal.hxx b/src/libserver/symcache/symcache_internal.hxx index e094330fd2..a45ae88b9e 100644 --- a/src/libserver/symcache/symcache_internal.hxx +++ b/src/libserver/symcache/symcache_internal.hxx @@ -37,7 +37,6 @@ #include "contrib/robin-hood/robin_hood.h" #include "contrib/expected/expected.hpp" #include "cfg_file.h" -#include "lua/lua_common.h" #include "symcache_id_list.hxx" @@ -62,6 +61,8 @@ RSPAMD_LOG_FUNC, \ __VA_ARGS__) +struct lua_State; + namespace rspamd::symcache { /* Defined in symcache_impl.cxx */ @@ -168,11 +169,7 @@ public: delayed_deps = std::make_unique>(); } - virtual ~symcache() { - if (peak_cb != -1) { - luaL_unref(L, LUA_REGISTRYINDEX, peak_cb); - } - } + virtual ~symcache(); /** * Saves items on disk (if possible) diff --git a/src/libserver/symcache/symcache_item.cxx b/src/libserver/symcache/symcache_item.cxx new file mode 100644 index 0000000000..b25fc991ff --- /dev/null +++ b/src/libserver/symcache/symcache_item.cxx @@ -0,0 +1,310 @@ +/*- + * Copyright 2022 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "symcache_internal.hxx" +#include "symcache_item.hxx" +#include "fmt/core.h" + +namespace rspamd::symcache { + +auto cache_item::get_parent(const symcache &cache) const -> const cache_item * +{ + if (is_virtual()) { + const auto &virtual_sp = std::get(specific); + + return virtual_sp.get_parent(cache); + } + + return nullptr; +} + +auto cache_item::process_deps(const symcache &cache) -> void +{ + /* Allow logging macros to work */ + auto log_tag = [&]() { return cache.log_tag(); }; + + for (auto &dep: deps) { + msg_debug_cache ("process real dependency %s on %s", symbol.c_str(), dep.sym.c_str()); + auto *dit = cache.get_item_by_name_mut(dep.sym, true); + + if (dep.vid >= 0) { + /* Case of the virtual symbol that depends on another (maybe virtual) symbol */ + const auto *vdit = cache.get_item_by_name(dep.sym, false); + + if (!vdit) { + if (dit) { + msg_err_cache("cannot add dependency from %s on %s: no dependency symbol registered", + dep.sym.c_str(), dit->symbol.c_str()); + } + } + else { + msg_debug_cache("process virtual dependency %s(%d) on %s(%d)", symbol.c_str(), + dep.vid, vdit->symbol.c_str(), vdit->id); + + std::size_t nids = 0; + + /* Propagate ids */ + msg_debug_cache("check id propagation for dependency %s from %s", + symbol.c_str(), dit->symbol.c_str()); + + const auto *ids = dit->allowed_ids.get_ids(nids); + + if (nids > 0) { + msg_debug_cache("propagate allowed ids from %s to %s", + dit->symbol.c_str(), symbol.c_str()); + + allowed_ids.set_ids(ids, nids, cache.get_pool()); + } + + ids = dit->forbidden_ids.get_ids(nids); + + if (nids > 0) { + msg_debug_cache("propagate forbidden ids from %s to %s", + dit->symbol.c_str(), symbol.c_str()); + + forbidden_ids.set_ids(ids, nids, cache.get_pool()); + } + } + } + + if (dit != nullptr) { + if (!dit->is_filter()) { + /* + * Check sanity: + * - filters -> prefilter dependency is OK and always satisfied + * - postfilter -> (filter, prefilter) dep is ok + * - idempotent -> (any) dep is OK + * + * Otherwise, emit error + * However, even if everything is fine this dep is useless ¯\_(ツ)_/¯ + */ + auto ok_dep = false; + + if (dit->get_type() == type) { + ok_dep = true; + } + else if (type < dit->get_type()) { + ok_dep = true; + } + + if (!ok_dep) { + msg_err_cache ("cannot add dependency from %s on %s: invalid symbol types", + dep.sym.c_str(), symbol.c_str()); + + continue; + } + } + else { + if (dit->id == id) { + msg_err_cache ("cannot add dependency on self: %s -> %s " + "(resolved to %s)", + symbol.c_str(), dep.sym.c_str(), dit->symbol.c_str()); + } + else { + /* Create a reverse dep */ + dit->rdeps.emplace_back(getptr(), dep.sym, id, -1); + dep.item = dit->getptr(); + dep.id = dit->id; + + msg_debug_cache ("add dependency from %d on %d", id, + dit->id); + } + } + } + else if (dep.id >= 0) { + msg_err_cache ("cannot find dependency on symbol %s for symbol %s", + dep.sym.c_str(), symbol.c_str()); + + continue; + } + } + + // Remove empty deps + deps.erase(std::remove_if(std::begin(deps), std::end(deps), + [](const auto &dep) { return !dep.item; }), std::end(deps)); +} + +auto cache_item::resolve_parent(const symcache &cache) -> bool +{ + auto log_tag = [&]() { return cache.log_tag(); }; + + if (is_virtual()) { + auto &virt = std::get(specific); + + if (virt.get_parent(cache)) { + msg_warn_cache("trying to resolve parent twice for %s", symbol.c_str()); + + return false; + } + + return virt.resolve_parent(cache); + } + else { + msg_warn_cache("trying to resolve a parent for non-virtual symbol %s", symbol.c_str()); + } + + return false; +} + +auto cache_item::update_counters_check_peak(lua_State *L, + struct ev_loop *ev_loop, + double cur_time, + double last_resort) -> bool +{ + auto ret = false; + static const double decay_rate = 0.25; + + st->total_hits += st->hits; + g_atomic_int_set(&st->hits, 0); + + if (last_count > 0) { + auto cur_value = (st->total_hits - last_count) / + (cur_time - last_resort); + rspamd_set_counter_ema(&st->frequency_counter, + cur_value, decay_rate); + st->avg_frequency = st->frequency_counter.mean; + st->stddev_frequency = st->frequency_counter.stddev; + + auto cur_err = (st->avg_frequency - cur_value); + cur_err *= cur_err; + + if (st->frequency_counter.number > 10 && + cur_err > ::sqrt(st->stddev_frequency) * 3) { + frequency_peaks++; + ret = true; + } + } + + last_count = st->total_hits; + + if (cd->number > 0) { + if (!is_virtual()) { + st->avg_time = cd->mean; + rspamd_set_counter_ema(&st->time_counter, + st->avg_time, decay_rate); + st->avg_time = st->time_counter.mean; + memset(cd, 0, sizeof(*cd)); + } + } + + return ret; +} + +auto virtual_item::get_parent(const symcache &cache) const -> const cache_item * +{ + if (parent) { + return parent.get(); + } + + return cache.get_item_by_id(parent_id, false); +} + +auto virtual_item::resolve_parent(const symcache &cache) -> bool +{ + if (parent) { + return false; + } + + auto item_ptr = cache.get_item_by_id(parent_id, true); + + if (item_ptr) { + parent = const_cast(item_ptr)->getptr(); + + return true; + } + + return false; +} + +auto item_type_from_c(enum rspamd_symbol_type type) -> tl::expected, std::string> +{ + constexpr const auto trivial_types = SYMBOL_TYPE_CONNFILTER | SYMBOL_TYPE_PREFILTER + | SYMBOL_TYPE_POSTFILTER | SYMBOL_TYPE_IDEMPOTENT + | SYMBOL_TYPE_COMPOSITE | SYMBOL_TYPE_CLASSIFIER + | SYMBOL_TYPE_VIRTUAL; + + constexpr auto all_but_one_ty = [&](int type, int exclude_bit) -> auto { + return type & (trivial_types & ~exclude_bit); + }; + + if (type & trivial_types) { + auto check_trivial = [&](auto flag, + symcache_item_type ty) -> tl::expected, std::string> { + if (all_but_one_ty(type, flag)) { + return tl::make_unexpected(fmt::format("invalid flags for a symbol: {}", type)); + } + + return std::make_pair(ty, type & ~flag); + }; + if (type & SYMBOL_TYPE_CONNFILTER) { + return check_trivial(SYMBOL_TYPE_CONNFILTER, symcache_item_type::CONNFILTER); + } + else if (type & SYMBOL_TYPE_PREFILTER) { + return check_trivial(SYMBOL_TYPE_PREFILTER, symcache_item_type::PREFILTER); + } + else if (type & SYMBOL_TYPE_POSTFILTER) { + return check_trivial(SYMBOL_TYPE_POSTFILTER, symcache_item_type::POSTFILTER); + } + else if (type & SYMBOL_TYPE_IDEMPOTENT) { + return check_trivial(SYMBOL_TYPE_IDEMPOTENT, symcache_item_type::IDEMPOTENT); + } + else if (type & SYMBOL_TYPE_COMPOSITE) { + return check_trivial(SYMBOL_TYPE_COMPOSITE, symcache_item_type::COMPOSITE); + } + else if (type & SYMBOL_TYPE_CLASSIFIER) { + return check_trivial(SYMBOL_TYPE_CLASSIFIER, symcache_item_type::CLASSIFIER); + } + else if (type & SYMBOL_TYPE_VIRTUAL) { + return check_trivial(SYMBOL_TYPE_VIRTUAL, symcache_item_type::VIRTUAL); + } + + return tl::make_unexpected(fmt::format("internal error: impossible flags combination", type)); + } + + /* Maybe check other flags combination here? */ + return std::make_pair(symcache_item_type::FILTER, type); +} + +bool operator<(symcache_item_type lhs, symcache_item_type rhs) +{ + auto ret = false; + switch (lhs) { + case symcache_item_type::CONNFILTER: + break; + case symcache_item_type::PREFILTER: + if (rhs == symcache_item_type::CONNFILTER) { + ret = true; + } + break; + case symcache_item_type::FILTER: + if (rhs == symcache_item_type::CONNFILTER || rhs == symcache_item_type::PREFILTER) { + ret = true; + } + break; + case symcache_item_type::POSTFILTER: + if (rhs != symcache_item_type::IDEMPOTENT) { + ret = true; + } + break; + case symcache_item_type::IDEMPOTENT: + default: + break; + } + + return ret; +} + +} diff --git a/src/libserver/symcache/symcache_runtime.cxx b/src/libserver/symcache/symcache_runtime.cxx new file mode 100644 index 0000000000..a87dad2598 --- /dev/null +++ b/src/libserver/symcache/symcache_runtime.cxx @@ -0,0 +1,29 @@ +/*- + * Copyright 2022 Vsevolod Stakhov + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "symcache_internal.hxx" +#include "symcache_item.hxx" +#include "symcache_runtime.hxx" + +namespace rspamd::symcache { + +auto cache_savepoint::create_savepoint(struct rspamd_task *task, const symcache &cache) -> cache_savepoint * +{ + return nullptr; +} + +} + diff --git a/src/libserver/symcache/symcache_runtime.hxx b/src/libserver/symcache/symcache_runtime.hxx index bf3049ffee..3a7da615d8 100644 --- a/src/libserver/symcache/symcache_runtime.hxx +++ b/src/libserver/symcache/symcache_runtime.hxx @@ -59,8 +59,12 @@ struct cache_savepoint { order_generation_ptr order; /* Dynamically expanded as needed */ struct cache_dynamic_item dynamic_items[]; + +public: + static auto create_savepoint(struct rspamd_task *task, const symcache &cache) -> cache_savepoint *; }; + } #endif //RSPAMD_SYMCACHE_RUNTIME_HXX