From: Mike Stepanek (mstepane) Date: Tue, 3 Dec 2019 20:28:43 +0000 (+0000) Subject: Merge pull request #1840 in SNORT/snort3 from ~SMINUT/snort3:appid_memcap_rrt to... X-Git-Tag: 3.0.0-266~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9cb394c4000c3bb306f93f7d294e4769e2a1610b;p=thirdparty%2Fsnort3.git Merge pull request #1840 in SNORT/snort3 from ~SMINUT/snort3:appid_memcap_rrt to master Squashed commit of the following: commit a9c44d768cf79685c6d547ac5a18da01dd7851ca Author: Silviu Minut Date: Fri Nov 8 18:27:01 2019 -0500 appid: handle memcap during reload_config using RRT. --- diff --git a/src/network_inspectors/appid/CMakeLists.txt b/src/network_inspectors/appid/CMakeLists.txt index 5c2d73deb..917e3a337 100644 --- a/src/network_inspectors/appid/CMakeLists.txt +++ b/src/network_inspectors/appid/CMakeLists.txt @@ -193,6 +193,7 @@ set ( APPID_SOURCES appid_inspector.h appid_module.cc appid_module.h + appid_pegs.h appid_stats.cc appid_stats.h app_info_table.cc diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 6dc94b728..144307a3b 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -30,12 +30,14 @@ #include "log/messages.h" #include "main/analyzer_command.h" +#include "main/snort.h" #include "profiler/profiler.h" #include "utils/util.h" #include "app_info_table.h" #include "appid_debug.h" #include "appid_peg_counts.h" +#include "service_state.h" using namespace snort; using namespace std; @@ -193,7 +195,10 @@ static const PegInfo appid_pegs[] = { CountType::SUM, "ignored_packets", "count of packets ignored" }, { CountType::SUM, "total_sessions", "count of sessions created" }, { CountType::SUM, "appid_unknown", "count of sessions where appid could not be determined" }, - { CountType::END, nullptr, nullptr}, + { CountType::SUM, "service_cache_prunes", "number of times the service cache was pruned" }, + { CountType::SUM, "service_cache_adds", "number of times an entry was added to the service cache" }, + { CountType::SUM, "service_cache_removes", "number of times an item was removed from the service cache" }, + { CountType::END, nullptr, nullptr }, }; AppIdModule::AppIdModule() : @@ -275,10 +280,14 @@ bool AppIdModule::begin(const char* /*fqn*/, int, SnortConfig*) return true; } -bool AppIdModule::end(const char*, int, SnortConfig*) +bool AppIdModule::end(const char*, int, SnortConfig* sc) { assert(config); + appid_rrt.memcap = config->memcap; + if ( Snort::is_reloading() ) + sc->register_reload_resource_tuner(appid_rrt); + if ( !config->app_detector_dir ) { ParseWarning(WARN_CONF, @@ -312,3 +321,13 @@ void AppIdModule::show_dynamic_stats() { AppIdPegCounts::print(); } + +bool AppIdReloadTuner::tinit() +{ + return AppIdServiceState::initialize(memcap); +} + +bool AppIdReloadTuner::tune_resources(unsigned work_limit) +{ + return AppIdServiceState::prune(memcap, work_limit); +} diff --git a/src/network_inspectors/appid/appid_module.h b/src/network_inspectors/appid/appid_module.h index 27418a5e7..650822ae0 100644 --- a/src/network_inspectors/appid/appid_module.h +++ b/src/network_inspectors/appid/appid_module.h @@ -26,6 +26,7 @@ #include #include "appid_config.h" +#include "appid_pegs.h" #include "framework/module.h" extern THREAD_LOCAL snort::ProfileStats appid_perf_stats; @@ -35,13 +36,28 @@ extern Trace TRACE_NAME(appid_module); #define MOD_NAME "appid" #define MOD_HELP "application and service identification" -struct AppIdStats + +class AppIdReloadTuner : public snort::ReloadResourceTuner { - PegCount packets; - PegCount processed_packets; - PegCount ignored_packets; - PegCount total_sessions; - PegCount appid_unknown; +public: + AppIdReloadTuner() = default; + + bool tinit() override; + bool tune_packet_context() override + { + return tune_resources( max_work ); + } + bool tune_idle_context() override + { + return tune_resources( max_work_idle ); + } + + friend class AppIdModule; + +private: + size_t memcap = 0; + + bool tune_resources(unsigned work_limit); }; extern THREAD_LOCAL AppIdStats appid_stats; @@ -70,7 +86,7 @@ public: private: AppIdModuleConfig* config; + AppIdReloadTuner appid_rrt; }; #endif - diff --git a/src/network_inspectors/appid/appid_pegs.h b/src/network_inspectors/appid/appid_pegs.h new file mode 100644 index 000000000..c02d7772e --- /dev/null +++ b/src/network_inspectors/appid/appid_pegs.h @@ -0,0 +1,38 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// appid_pegs.h author Silviu Minut + +#ifndef APPID_PEGS_H +#define APPID_PEGS_H + +#include "framework/counts.h" + +struct AppIdStats +{ + PegCount packets; + PegCount processed_packets; + PegCount ignored_packets; + PegCount total_sessions; + PegCount appid_unknown; + PegCount service_cache_prunes; + PegCount service_cache_adds; + PegCount service_cache_removes; +}; + +#endif diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h index 0c3bb6f17..c6ea57b9f 100644 --- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h +++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h @@ -197,5 +197,11 @@ AppInfoTableEntry* AppInfoManager::get_app_info_entry(AppId, const AppInfoTable& return nullptr; } -#endif +bool AppIdReloadTuner::tinit() { return false; } +bool AppIdReloadTuner::tune_resources(unsigned int) +{ + return true; +} + +#endif diff --git a/src/network_inspectors/appid/service_state.cc b/src/network_inspectors/appid/service_state.cc index a85cfe0d0..723a94576 100644 --- a/src/network_inspectors/appid/service_state.cc +++ b/src/network_inspectors/appid/service_state.cc @@ -194,9 +194,17 @@ void ServiceDiscoveryState::update_service_incompatiable(const SfIp* ip) } -void AppIdServiceState::initialize(size_t memcap) +bool AppIdServiceState::initialize(size_t memcap) { - service_state_cache = new MapList(memcap); + if ( !service_state_cache ) + service_state_cache = new MapList(memcap); + else + { + bool have_work = memcap < service_state_cache->memcap; + service_state_cache->memcap = memcap; + return have_work; + } + return false; } void AppIdServiceState::clean() @@ -267,3 +275,11 @@ void AppIdServiceState::dump_stats() } #endif } + +bool AppIdServiceState::prune(size_t max_memory, size_t num_items) +{ + bool done = false; + if ( service_state_cache ) + done = service_state_cache->prune(max_memory, num_items); + return done; +} diff --git a/src/network_inspectors/appid/service_state.h b/src/network_inspectors/appid/service_state.h index 5fd1a6f1b..8ed9e4e29 100644 --- a/src/network_inspectors/appid/service_state.h +++ b/src/network_inspectors/appid/service_state.h @@ -30,6 +30,7 @@ #include "utils/cpp_macros.h" #include "utils/util.h" +#include "appid_pegs.h" #include "service_plugins/service_discovery.h" class ServiceDetector; @@ -149,7 +150,7 @@ private: class AppIdServiceState { public: - static void initialize(size_t memcap = 0); + static bool initialize(size_t memcap); static void clean(); static ServiceDiscoveryState* add(const snort::SfIp*, IpProtocol, uint16_t port, bool decrypted, bool do_touch = false); static ServiceDiscoveryState* get(const snort::SfIp*, IpProtocol, uint16_t port, bool decrypted, bool do_touch = false); @@ -157,6 +158,8 @@ public: static void check_reset(AppIdSession& asd, const snort::SfIp* ip, uint16_t port); static void dump_stats(); + + static bool prune(size_t max_memory = 0, size_t num_items = -1u); }; @@ -198,6 +201,8 @@ private: PADDING_GUARD_END +extern THREAD_LOCAL AppIdStats appid_stats; + class MapList { public: @@ -225,6 +230,7 @@ public: q.emplace_back(it); mem_used += sz; ss->qptr = --q.end(); // remember our place in the queue + appid_stats.service_cache_adds++; if ( mem_used > memcap ) remove( q.front() ); @@ -252,17 +258,34 @@ public: bool remove(Map_t::iterator it) { - if ( it != m.end() ) + if ( it != m.end() && !m.empty() ) { + assert( mem_used >= sz ); mem_used -= sz; q.erase(it->second->qptr); // remove from queue delete it->second; m.erase(it); // then from cache + appid_stats.service_cache_removes++; + return true; } return false; } + bool prune(size_t max_memory = 0, size_t num_items = -1u) + { + if ( max_memory == 0 ) + max_memory = memcap; + + size_t i=0; + while ( mem_used > max_memory && i++ < num_items ) + remove( q.front() ); + + appid_stats.service_cache_prunes++; + + return mem_used <= max_memory; + } + Map_t::iterator find(const Key_t& k) { return m.find(k); @@ -288,6 +311,8 @@ public: // how much memory we add when we put an SDS in the cache: static const size_t sz; + friend class AppIdServiceState; + private: Map_t m; Queue_t q; diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc index f2a02f2ce..0b2318d26 100644 --- a/src/network_inspectors/appid/test/appid_discovery_test.cc +++ b/src/network_inspectors/appid/test/appid_discovery_test.cc @@ -277,6 +277,13 @@ AppId check_session_for_AF_forecast(AppIdSession&, Packet*, AppidSessionDirectio return APP_ID_UNKNOWN; } +bool AppIdReloadTuner::tinit() { return false; } + +bool AppIdReloadTuner::tune_resources(unsigned int) +{ + return true; +} + TEST_GROUP(appid_discovery_tests) { void setup() override diff --git a/src/network_inspectors/appid/test/appid_mock_definitions.h b/src/network_inspectors/appid/test/appid_mock_definitions.h index 6f2449701..3716d8006 100644 --- a/src/network_inspectors/appid/test/appid_mock_definitions.h +++ b/src/network_inspectors/appid/test/appid_mock_definitions.h @@ -88,6 +88,18 @@ ServiceDiscoveryState* AppIdServiceState::add(SfIp const*, IpProtocol, unsigned return nullptr; } +bool AppIdServiceState::prune(size_t, size_t) +{ + return true; +} + +bool AppIdReloadTuner::tinit() { return false; } + +bool AppIdReloadTuner::tune_resources(unsigned int) +{ + return true; +} + void ServiceDiscoveryState::set_service_id_valid(ServiceDetector*) { } // Stubs for service_plugins/service_discovery.h