]> git.ipfire.org Git - thirdparty/snort3.git/commitdiff
Pull request #3775: appid: Adds logs for memory and pattern count
authorSreeja Athirkandathil Narayanan (sathirka) <sathirka@cisco.com>
Wed, 12 Apr 2023 17:46:20 +0000 (17:46 +0000)
committerSreeja Athirkandathil Narayanan (sathirka) <sathirka@cisco.com>
Wed, 12 Apr 2023 17:46:20 +0000 (17:46 +0000)
Merge in SNORT/snort3 from ~LCZARNIK/snort3:appid_logs to master

Squashed commit of the following:

commit b3115951ae0ed617ae5acb715e351c8789f8fba7
Author: Lukasz Czarnik <lczarnik@cisco.com>
Date:   Fri Mar 3 10:16:27 2023 -0500

    appid: log maxrss difference and pattern count during appid initialization and reload detectors

24 files changed:
src/network_inspectors/appid/appid_config.cc
src/network_inspectors/appid/appid_config.h
src/network_inspectors/appid/appid_inspector.cc
src/network_inspectors/appid/appid_module.cc
src/network_inspectors/appid/client_plugins/client_discovery.cc
src/network_inspectors/appid/client_plugins/client_discovery.h
src/network_inspectors/appid/client_plugins/eve_ca_patterns.cc
src/network_inspectors/appid/client_plugins/eve_ca_patterns.h
src/network_inspectors/appid/detector_plugins/detector_pattern.cc
src/network_inspectors/appid/detector_plugins/detector_pattern.h
src/network_inspectors/appid/detector_plugins/dns_patterns.cc
src/network_inspectors/appid/detector_plugins/dns_patterns.h
src/network_inspectors/appid/detector_plugins/http_url_patterns.cc
src/network_inspectors/appid/detector_plugins/http_url_patterns.h
src/network_inspectors/appid/detector_plugins/sip_patterns.cc
src/network_inspectors/appid/detector_plugins/sip_patterns.h
src/network_inspectors/appid/detector_plugins/ssh_patterns.cc
src/network_inspectors/appid/detector_plugins/ssh_patterns.h
src/network_inspectors/appid/detector_plugins/ssl_patterns.cc
src/network_inspectors/appid/detector_plugins/ssl_patterns.h
src/network_inspectors/appid/service_plugins/alpn_patterns.cc
src/network_inspectors/appid/service_plugins/alpn_patterns.h
src/network_inspectors/appid/service_plugins/service_discovery.cc
src/network_inspectors/appid/service_plugins/service_discovery.h

index f88c83805afdf046ff1a58da2d612da557460baf..6e5d4a4b69e3743e1cbdeee034270e931677501e 100644 (file)
@@ -157,6 +157,21 @@ void AppIdContext::show() const
     config.show();
 }
 
+unsigned OdpContext::get_pattern_count()
+{
+    return service_pattern_detector->get_pattern_count() +
+        client_pattern_detector->get_pattern_count() +
+        service_disco_mgr.get_pattern_count() +
+        client_disco_mgr.get_pattern_count() +
+        http_matchers.get_pattern_count() +
+        eve_ca_matchers.get_pattern_count() +
+        alpn_matchers.get_pattern_count() +
+        sip_matchers.get_pattern_count() +
+        ssl_matchers.get_pattern_count() +
+        ssh_matchers.get_pattern_count() +
+        dns_matchers.get_pattern_count();
+}
+
 OdpContext::OdpContext(const AppIdConfig& config, SnortConfig* sc)
 {
     app_info_mgr.init_appid_info_table(config, sc, *this);
index 76e845ef67760d6e47f4b36388b757decd6ff173..c8c69b45c903671eac66b2da6084f3894b18099a 100644 (file)
@@ -87,6 +87,7 @@ public:
     uint32_t first_decrypted_packet_debug = 0;
     bool log_eve_process_client_mappings = false;
     bool log_alpn_service_mappings = false;
+    bool log_memory_and_pattern_count = false;
 #endif
     bool log_stats = false;
     uint32_t app_stats_period = 300;
@@ -241,6 +242,7 @@ public:
         return alpn_matchers;
     }
 
+    unsigned get_pattern_count();
     void add_port_service_id(IpProtocol, uint16_t, AppId);
     void add_protocol_service_id(IpProtocol, AppId);
     AppId get_port_service_id(IpProtocol, uint16_t);
index 07eaf17c3e32e84a8b150aead138d2e7c6ea84c2..9d78e29a47177c07b83e66a9e093a635bbf6124c 100644 (file)
 #include "appid_inspector.h"
 
 #include <openssl/crypto.h>
+#include <sys/resource.h>
 
 #include "flow/flow.h"
+#include "log/messages.h"
 #include "main/analyzer_command.h"
 #include "managers/inspector_manager.h"
 #include "managers/module_manager.h"
@@ -116,9 +118,35 @@ bool AppIdInspector::configure(SnortConfig* sc)
 {
     assert(!ctxt);
 
+    struct rusage ru;
+    long prev_maxrss = -1;
+    #ifdef REG_TEST
+    if ( config->log_memory_and_pattern_count )
+    {
+    #endif
+        if ( getrusage(RUSAGE_SELF, &ru) == 0 )
+            prev_maxrss = ru.ru_maxrss;
+    #ifdef REG_TEST
+    }
+    #endif
+
     ctxt = new AppIdContext(const_cast<AppIdConfig&>(*config));
     ctxt->init_appid(sc, *this);
 
+    #ifdef REG_TEST
+    if ( config->log_memory_and_pattern_count )
+    {
+    #endif
+        if ( prev_maxrss == -1 or getrusage(RUSAGE_SELF, &ru) == -1 )
+            ErrorMessage("appid: fetching memory usage failed\n");
+        else
+            LogMessage("appid: MaxRss diff: %li\n", ru.ru_maxrss - prev_maxrss);
+
+        LogMessage("appid: patterns loaded: %u\n", ctxt->get_odp_ctxt().get_pattern_count());
+    #ifdef REG_TEST
+    }
+    #endif
+
     DataBus::subscribe_global(http_pub_key, HttpEventIds::REQUEST_HEADER,
         new HttpEventHandler(HttpEventHandler::REQUEST_EVENT, *this), *sc);
 
index 74dbccda623dacf8d23523a7a551db7c18c0e904..c50f180caf2375f147151fda701ce75f4a77c8ed 100644 (file)
@@ -27,6 +27,7 @@
 
 #include <climits>
 #include <lua.hpp>
+#include <sys/resource.h>
 
 #include "control/control.h"
 #include "host_tracker/host_cache.h"
@@ -73,6 +74,8 @@ static const Parameter s_params[] =
       "enable logging of encrypted visibility engine process to client mappings" },
     { "log_alpn_service_mappings", Parameter::PT_BOOL, nullptr, "false",
       "enable logging of alpn service mappings" },
+    { "log_memory_and_pattern_count", Parameter::PT_BOOL, nullptr, "false",
+      "enable logging of memory usage and pattern counts" },
 #endif
     { "memcap", Parameter::PT_INT, "1024:maxSZ", "1048576",
       "max size of the service cache before we start pruning the cache" },
@@ -382,6 +385,18 @@ static int reload_detectors(lua_State* L)
 
     ctrlcon->respond(".. reloading detectors\n");
 
+    struct rusage ru;
+    long prev_maxrss = -1;
+    #ifdef REG_TEST
+    if ( inspector->get_config().log_memory_and_pattern_count )
+    {
+    #endif
+        getrusage(RUSAGE_SELF, &ru);
+        prev_maxrss = ru.ru_maxrss;
+    #ifdef REG_TEST
+    }
+    #endif
+
     AppIdContext& ctxt = inspector->get_ctxt();
     OdpContext& old_odp_ctxt = ctxt.get_odp_ctxt();
     ServiceDiscovery::clear_ftp_service_state();
@@ -401,6 +416,18 @@ static int reload_detectors(lua_State* L)
 
     ctrlcon->respond("== swapping detectors configuration\n");
     ReloadTracker::update(ctrlcon, "swapping detectors configuration");
+
+    #ifdef REG_TEST
+    if ( inspector->get_config().log_memory_and_pattern_count )
+    {
+    #endif
+        getrusage(RUSAGE_SELF, &ru);
+        LogMessage("appid: MaxRss diff: %li\n", ru.ru_maxrss - prev_maxrss);
+        LogMessage("appid: patterns loaded: %u\n", odp_ctxt.get_pattern_count());
+    #ifdef REG_TEST
+    }
+    #endif
+
     main_broadcast_command(new ACOdpContextSwap(*inspector, old_odp_ctxt, ctrlcon), ctrlcon);
     return 0;
 }
@@ -486,6 +513,8 @@ bool AppIdModule::set(const char*, Value& v, SnortConfig*)
         config->log_eve_process_client_mappings = v.get_bool();
     else if (v.is("log_alpn_service_mappings") )
         config->log_alpn_service_mappings = v.get_bool();
+    else if (v.is("log_memory_and_pattern_count") )
+        config->log_memory_and_pattern_count = v.get_bool();
     else
 #endif
     if ( v.is("memcap") )
index 3c8730c94fd70bcd5e2a8f37ea2a4fce23182bca..c3a221fd79693dcd5eea7c59648deeb0948e9d2d 100644 (file)
@@ -92,6 +92,11 @@ void ClientDiscovery::reload_client_patterns()
     udp_patterns.reload();
 }
 
+unsigned ClientDiscovery::get_pattern_count()
+{
+    return tcp_pattern_count + udp_pattern_count;
+}
+
 /*
  * Callback function for string search
  *
index 9b80e604de94d59e46f78396d2b25bc1059b59db..3cbf54923060abb6bac8406903b2f256ab5bf3fc 100644 (file)
@@ -47,6 +47,7 @@ public:
 
     void finalize_client_patterns();
     void reload_client_patterns();
+    unsigned get_pattern_count();
     bool do_client_discovery(AppIdSession&, snort::Packet*,
         AppidSessionDirection direction, AppidChangeBits& change_bits);
 
index 1d2ec3688899653df65998b274c3c5612bc5e69c..86e0dab887023dbb30e9154f167c2557ec3a87b5 100644 (file)
@@ -125,3 +125,7 @@ void EveCaPatternMatchers::reload_patterns()
     eve_ca_pattern_matcher.reload();
 }
 
+unsigned EveCaPatternMatchers::get_pattern_count()
+{
+    return eve_ca_load_list.size();
+}
index f54be8a23ade4edf3c370ff71cc44b12176e56cc..06f77bbefdae761e5a2eaee4e54c7ea85d0b1948 100644 (file)
@@ -48,6 +48,7 @@ public:
     void add_eve_ca_pattern(AppId, const std::string&, uint8_t, const std::string&);
     void finalize_patterns();
     void reload_patterns();
+    unsigned get_pattern_count();
 
     const EveCaPatternList& get_eve_ca_load_list() const { return eve_ca_load_list; }
 
index 9ed0b6d32b77071afcd36fb1ef85d08234127b73..b371119bfc27e063a0e988256f839e49d87b1bb1 100644 (file)
@@ -328,6 +328,7 @@ void PatternServiceDetector::register_service_patterns()
             {
                 if (pattern->data && pattern->length)
                 {
+                    pattern_count++;
                     if (ps->proto == IpProtocol::TCP)
                     {
                         handler->register_tcp_pattern(this, pattern->data, pattern->length,
@@ -346,7 +347,10 @@ void PatternServiceDetector::register_service_patterns()
         else
         {
             for (Pattern* pattern = ps->pattern; pattern; pattern = pattern->next)
+            {
+                pattern_count++;
                 ps->count++;
+            }
         }
     }
 
@@ -428,6 +432,11 @@ void PatternServiceDetector::reload_service_port_patterns()
         udp_pattern_matcher->reload();
 }
 
+unsigned PatternServiceDetector::get_pattern_count()
+{
+    return pattern_count;
+}
+
 PatternServiceDetector::PatternServiceDetector(ServiceDiscovery* sd)
 {
     handler = sd;
@@ -612,6 +621,7 @@ void PatternClientDetector::register_client_patterns()
         {
             if (pattern->data && pattern->length)
             {
+                pattern_count++;
                 if (ps->proto == IpProtocol::TCP)
                 {
                     handler->register_tcp_pattern(this, pattern->data, pattern->length,
@@ -652,3 +662,7 @@ void PatternClientDetector::reload_client_port_patterns()
         udp_pattern_matcher->reload();
 }
 
+unsigned PatternClientDetector::get_pattern_count()
+{
+    return pattern_count;
+}
index df3d0e87591f11ebc1cc63d7f62dea7a40e1ade4..3ed4c8b7d94f6f43ad761e388b8b152bba12296c 100644 (file)
@@ -82,6 +82,7 @@ public:
     void insert_client_port_pattern(PortPatternNode*);
     void finalize_client_port_patterns(AppIdInspector&);
     void reload_client_port_patterns();
+    unsigned get_pattern_count();
 
     int validate(AppIdDiscoveryArgs&) override;
 
@@ -93,6 +94,7 @@ private:
     PatternService* service_port_pattern = nullptr;
     snort::SearchTool* tcp_pattern_matcher = nullptr;
     snort::SearchTool* udp_pattern_matcher = nullptr;
+    unsigned pattern_count = 0;
 };
 
 class PatternServiceDetector : public ServiceDetector
@@ -104,6 +106,7 @@ public:
     void insert_service_port_pattern(PortPatternNode*);
     void finalize_service_port_patterns(AppIdInspector&);
     void reload_service_port_patterns();
+    unsigned get_pattern_count();
 
     int validate(AppIdDiscoveryArgs&) override;
 
@@ -118,6 +121,7 @@ private:
     snort::SearchTool* udp_pattern_matcher = nullptr;
     snort::SearchTool* tcp_port_pattern_tree[65536] = { };
     snort::SearchTool* udp_port_pattern_tree[65536] = { };
+    unsigned pattern_count = 0;
 };
 
 #endif
index d7d8aeac944c2ec79390db1d6755c35e3ad24f3d..75ccc8d8e7beb3e90c388d4d9eeed9b41b3452f5 100644 (file)
@@ -52,6 +52,7 @@ void DnsPatternMatchers::finalize_patterns()
     /* Add patterns from Lua API */
     for (element = dns_host_pattern_list; element; element = element->next)
     {
+        pattern_count++;
         dns_host_matcher.add((char*)element->dpattern->pattern,
             element->dpattern->pattern_size, element->dpattern, true);
     }
@@ -64,6 +65,11 @@ void DnsPatternMatchers::reload_patterns()
     dns_host_matcher.reload();
 }
 
+unsigned DnsPatternMatchers::get_pattern_count()
+{
+    return pattern_count;
+}
+
 DnsPatternMatchers::~DnsPatternMatchers()
 {
     DnsHostPatternList* tmp_pattern;
index cd9380d87faf957a87b1c443a5bf33409b751bd8..149db652e6216e790b6329a8ef714d0fd84a757d 100644 (file)
@@ -51,11 +51,13 @@ public:
     void add_host_pattern(uint8_t*, size_t, uint8_t, AppId);
     void finalize_patterns();
     void reload_patterns();
+    unsigned get_pattern_count();
     int scan_hostname(const uint8_t*, size_t, AppId&, AppId&);
 
 private:
     DnsHostPatternList* dns_host_pattern_list = nullptr;
     snort::SearchTool dns_host_matcher = snort::SearchTool();
+    unsigned pattern_count = 0;
 };
 
 #endif
index 44d029112ac9cbc1cacb96af9bdd6029dedb092e..a21fe421d906ba357f6b4a9b6fbccaa5270c0f4e 100644 (file)
@@ -383,6 +383,7 @@ void HttpPatternMatchers::insert_chp_pattern(CHPListElement* chpa)
             tmp_chpa = tmp_chpa->next;
         tmp_chpa->next = chpa;
     }
+    chp_pattern_count++;
 }
 
 void HttpPatternMatchers::insert_http_pattern(enum httpPatternType pType,
@@ -748,6 +749,15 @@ void HttpPatternMatchers::reload_patterns()
         chp_matchers[i].reload();
 }
 
+unsigned HttpPatternMatchers::get_pattern_count()
+{
+    return chp_pattern_count +
+        client_agent_patterns.size() +
+        content_type_patterns.size() +
+        url_patterns.size() +
+        host_url_patterns.size();
+}
+
 static inline void free_matched_patterns(MatchedPatterns* mp)
 {
     while (mp)
index f9e13cd7e37d9b5273df63107e319e84490d8543..55e60a8137ba03bf97f6b48373b36ece9234c793 100644 (file)
@@ -284,6 +284,7 @@ public:
 
     int finalize_patterns();
     void reload_patterns();
+    unsigned get_pattern_count();
     void insert_chp_pattern(CHPListElement*);
     void insert_http_pattern(enum httpPatternType, DetectorHTTPPattern&);
     void remove_http_patterns_for_id(AppId);
@@ -327,6 +328,7 @@ private:
     snort::SearchTool chp_matchers[NUM_HTTP_FIELDS];
     tMlmpTree* host_url_matcher = nullptr;
     tMlmpTree* rtmp_host_url_matcher = nullptr;
+    unsigned chp_pattern_count = 0;
 
     void free_chp_app_elements();
     int add_mlmp_pattern(tMlmpTree* matcher, DetectorHTTPPattern& pattern );
index 17f7edd60573c753ec7de0540699ebb7d8ade640..61e2e4c860eb1e8f105185938f6789233dddb863 100644 (file)
@@ -123,6 +123,7 @@ void SipPatternMatchers::finalize_patterns(OdpContext& odp_ctxt)
     for ( pattern_node = sip_ua_list; pattern_node; pattern_node =
         pattern_node->next )
     {
+        pattern_count++;
         num_patterns = odp_ctxt.get_http_matchers().parse_multiple_http_patterns(
             (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0);
         patterns[num_patterns].pattern = nullptr;
@@ -133,6 +134,7 @@ void SipPatternMatchers::finalize_patterns(OdpContext& odp_ctxt)
     for ( pattern_node = sip_server_list; pattern_node; pattern_node =
         pattern_node->next )
     {
+        pattern_count++;
         num_patterns = odp_ctxt.get_http_matchers().parse_multiple_http_patterns(
             (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0);
         patterns[num_patterns].pattern = nullptr;
@@ -152,6 +154,11 @@ void SipPatternMatchers::reload_patterns()
     mlmp_reload_patterns(*sip_server_matcher);
 }
 
+unsigned SipPatternMatchers::get_pattern_count()
+{
+    return pattern_count;
+}
+
 int SipPatternMatchers::get_client_from_ua(const char* pattern, uint32_t pattern_len,
     AppId& client_id, char*& client_version)
 {
index 29af2db4457629899a280a40fb5bb7c83432c9fe..dc2e0546c56a6a1881f5a4b5498cef4c28b4f5bd 100644 (file)
@@ -49,6 +49,7 @@ public:
     int get_client_from_server(const char*, uint32_t, AppId&, char*&);
     void finalize_patterns(OdpContext&);
     void reload_patterns();
+    unsigned get_pattern_count();
 
 private:
     static const int PATTERN_PART_MAX = 10;
@@ -57,6 +58,7 @@ private:
     DetectorAppSipPattern* sip_ua_list = nullptr;
     tMlmpTree* sip_server_matcher = nullptr;
     DetectorAppSipPattern* sip_server_list = nullptr;
+    unsigned pattern_count = 0;
 };
 
 #endif
index 2a06d6748a99ad499052a8257662c2c9c3ed55c3..96b192b587bac80b2e891b43f6dfada175d469d6 100644 (file)
@@ -43,3 +43,8 @@ AppId SshPatternMatchers::get_appid(const std::string& pattern) const
 {
     return ssh_patterns.at(pattern);
 }
+
+unsigned SshPatternMatchers::get_pattern_count()
+{
+    return ssh_patterns.size();
+}
index a753d3853382aaaa88301dd6fbe426c1e0487636..3ada6cb7d49a2f8c51b5141b29819948f4b2bec6 100644 (file)
@@ -43,6 +43,7 @@ public:
     bool has_pattern(const std::string& pattern) const;
     bool empty() const;
     AppId get_appid(const std::string& pattern) const;
+    unsigned get_pattern_count();
 private:
     SshPatternTable ssh_patterns;
 };
index 27f3a042896f07c4c1cd40536bc9b8a6a69431af..486999a6f0d7034e767602e66bee818916ac0109 100644 (file)
@@ -28,7 +28,7 @@
 
 using namespace snort;
 
-static void create_matcher(SearchTool& matcher, SslPatternList* list, CnameCache& set)
+static void create_matcher(SearchTool& matcher, SslPatternList* list, CnameCache& set, unsigned& pattern_count)
 {
     size_t* pattern_index;
     size_t size = 0;
@@ -45,7 +45,7 @@ static void create_matcher(SearchTool& matcher, SslPatternList* list, CnameCache
             element->dpattern->pattern_size, element->dpattern, true);
         (*pattern_index)++;
     }
-
+    pattern_count = size;
     matcher.prep();
 }
 
@@ -187,7 +187,7 @@ void SslPatternMatchers::add_cert_pattern(uint8_t* pattern_str, size_t pattern_s
 
 void SslPatternMatchers::finalize_patterns()
 {
-    create_matcher(ssl_host_matcher, cert_pattern_list, cert_pattern_set);
+    create_matcher(ssl_host_matcher, cert_pattern_list, cert_pattern_set, pattern_count);
     cert_pattern_set.clear();
 }
 
@@ -196,6 +196,11 @@ void SslPatternMatchers::reload_patterns()
     ssl_host_matcher.reload();
 }
 
+unsigned SslPatternMatchers::get_pattern_count()
+{
+    return pattern_count;
+}
+
 bool SslPatternMatchers::scan_hostname(const uint8_t* hostname, size_t size, AppId& client_id, AppId& payload_id)
 {
     return scan_patterns(ssl_host_matcher, hostname, size, client_id, payload_id, false);
index aa2a27d8ba843df194f585a284887706bf1b0ca3..9988219aa9e1157991bfd41acfc71cf326cdb113 100644 (file)
@@ -71,6 +71,7 @@ public:
     void add_cert_pattern(uint8_t*, size_t, uint8_t, AppId, bool);
     void finalize_patterns();
     void reload_patterns();
+    unsigned get_pattern_count();
     bool scan_hostname(const uint8_t*, size_t, AppId&, AppId&);
     bool scan_cname(const uint8_t*, size_t, AppId&, AppId&);
 
@@ -78,6 +79,7 @@ private:
     SslPatternList* cert_pattern_list = nullptr;
     CnameCache cert_pattern_set;
     snort::SearchTool ssl_host_matcher = snort::SearchTool();
+    unsigned pattern_count = 0;
 };
 
 #endif
index 97ffd6ef952d73501a850ae379240a8976fdf03c..9945d0cc1eed2039af64d1baef43b3a6b58b98fb 100644 (file)
@@ -120,3 +120,7 @@ void AlpnPatternMatchers::reload_patterns()
     alpn_pattern_matcher.reload();
 }
 
+unsigned AlpnPatternMatchers::get_pattern_count()
+{
+    return alpn_load_list.size();
+}
index 415e88a6e00088a66cb376c70f062a2a73370e11..7ef6b53edff8bbfea919be82001d009179fce09a 100644 (file)
@@ -46,6 +46,7 @@ public:
     void add_alpn_pattern(AppId, const std::string&, const std::string&);
     void finalize_patterns();
     void reload_patterns();
+    unsigned get_pattern_count();
 
     const AlpnPatternList& get_alpn_load_list() const { return alpn_load_list; }
 
index 0a8aad80e6b099153b280252d69d3a8538d83671..847b800b09ee2a02ebdfac58978b98a3d6919f65 100644 (file)
@@ -158,6 +158,11 @@ void ServiceDiscovery::reload_service_patterns()
     udp_patterns.reload();
 }
 
+unsigned ServiceDiscovery::get_pattern_count()
+{
+    return tcp_pattern_count + udp_pattern_count;
+}
+
 int ServiceDiscovery::add_service_port(AppIdDetector* detector, const ServiceDetectorPort& pp)
 {
     ServiceDetector* service = static_cast<ServiceDetector*>(detector);
index e3aa961dcd7bfcdfe21161065e825b1f470dfa0c..258ca426cee9343ca76d14a03bfd60592b5b6a40 100644 (file)
@@ -68,6 +68,7 @@ public:
     void reload() override;
     void finalize_service_patterns();
     void reload_service_patterns();
+    unsigned get_pattern_count();
     int add_service_port(AppIdDetector*, const ServiceDetectorPort&) override;
 
     AppIdDetectorsIterator get_detector_iterator(IpProtocol);