Merge pull request #2458 in SNORT/snort3 from ~EBURMAI/snort3:decrypted_smtp to master

Squashed commit of the following:

commit 7a870446615e9ebd29ef3358104e60e79fa62621
Author: Eduard Burmai <eburmai@cisco.com>
Date:   Fri Sep 11 05:44:02 2020 -0400

    appid: detect SMTP after decryption

commit 09baaf304cc69cdbc4484ad763bdb8991709fe6b
Author: Eduard Burmai <eburmai@cisco.com>
Date:   Tue Sep 1 08:49:55 2020 -0400

    appid: update appid stats for decrypted flows

diff --git a/src/network_inspectors/appid/appid_app_descriptor.cc b/src/network_inspectors/appid/appid_app_descriptor.cc
index 34553769ab14d7c5e11adf656cf5f4cbd7b13ae8..2986c59cddd2be012468a35b15306fe1ea92c711 100644 (file)
--- a/src/network_inspectors/appid/appid_app_descriptor.cc
+++ b/src/network_inspectors/appid/appid_app_descriptor.cc
@@ -27,6 +27,14 @@ void ApplicationDescriptor::set_id(AppId app_id)
             update_stats(app_id);
         else if ( app_id == APP_ID_UNKNOWN )
             appid_stats.appid_unknown++;
+        else
+            return; // app_id == APP_ID_NONE
+
+        if ( overwritten_id > APP_ID_NONE )
+        {
+            update_stats(overwritten_id, false);
+            overwritten_id = APP_ID_NONE;
+        }
     }
 }
 
@@ -40,9 +48,9 @@ void ApplicationDescriptor::set_id(const Packet& p, AppIdSession& asd,
     }
 }
 
-void ServiceAppDescriptor::update_stats(AppId id)
+void ServiceAppDescriptor::update_stats(AppId id, bool increment)
 {
-    AppIdPegCounts::inc_service_count(id);
+    AppIdPegCounts::update_service_count(id, increment);
 }
 
 void ServiceAppDescriptor::set_port_service_id(AppId id)
@@ -51,7 +59,7 @@ void ServiceAppDescriptor::set_port_service_id(AppId id)
     {
         port_service_id = id;
         if ( id > APP_ID_NONE )
-            AppIdPegCounts::inc_service_count(id);
+            AppIdPegCounts::update_service_count(id, true);
     }
 }
 
@@ -77,12 +85,12 @@ void ClientAppDescriptor::update_user(AppId app_id, const char* username)
     }
 }
 
-void ClientAppDescriptor::update_stats(AppId id)
+void ClientAppDescriptor::update_stats(AppId id, bool increment)
 {
-    AppIdPegCounts::inc_client_count(id);
+    AppIdPegCounts::update_client_count(id, increment);
 }
 
-void PayloadAppDescriptor::update_stats(AppId id)
+void PayloadAppDescriptor::update_stats(AppId id, bool increment)
 {
-    AppIdPegCounts::inc_payload_count(id);
+    AppIdPegCounts::update_payload_count(id, increment);
 }

diff --git a/src/network_inspectors/appid/appid_app_descriptor.h b/src/network_inspectors/appid/appid_app_descriptor.h
index 648a9f13bfe276ecc8b16fc4fc04e8735991bad6..2ea26b7a887131ff4f6d1ae13aa72ba218fa4a23 100644 (file)
--- a/src/network_inspectors/appid/appid_app_descriptor.h
+++ b/src/network_inspectors/appid/appid_app_descriptor.h
@@ -59,7 +59,7 @@ public:
         set_version(version, change_bits);
     }
 
-    virtual void update_stats(AppId id) = 0;
+    virtual void update_stats(AppId id, bool increment = true) = 0;
 
     AppId get_id() const
     {
@@ -70,6 +70,11 @@ public:
 
     virtual void set_id(const snort::Packet& p, AppIdSession& asd, AppidSessionDirection dir, AppId app_id, AppidChangeBits& change_bits);
 
+    void set_overwritten_id(AppId app_id)
+    {
+        overwritten_id = app_id;
+    }
+
     const char* get_version() const
     {
         return my_version.empty() ? nullptr : my_version.c_str();
@@ -86,6 +91,7 @@ public:
 
 private:
     AppId my_id = APP_ID_NONE;
+    AppId overwritten_id = APP_ID_NONE;
     std::string my_version;
 };
 
@@ -129,7 +135,7 @@ public:
         }
     }
 
-    void update_stats(AppId id) override;
+    void update_stats(AppId id, bool increment = true) override;
 
     AppId get_port_service_id() const
     {
@@ -204,7 +210,7 @@ public:
         return my_username.empty() ? nullptr : my_username.c_str();
     }
 
-    void update_stats(AppId id) override;
+    void update_stats(AppId id, bool increment = true) override;
 
 private:
     std::string my_username;
@@ -221,7 +227,7 @@ public:
         ApplicationDescriptor::reset();
     }
 
-    void update_stats(AppId id) override;
+    void update_stats(AppId id, bool increment = true) override;
 };
 
 #endif

diff --git a/src/network_inspectors/appid/appid_peg_counts.cc b/src/network_inspectors/appid/appid_peg_counts.cc
index 229876159ad75280b6d5511183e1428ef4094f47..ec28957d8f03f34726f28fd637172a07529f66aa 100644 (file)
--- a/src/network_inspectors/appid/appid_peg_counts.cc
+++ b/src/network_inspectors/appid/appid_peg_counts.cc
@@ -82,24 +82,33 @@ void AppIdPegCounts::sum_stats()
         appid_dynamic_sum[SF_APPID_MAX].stats[j] += ptr[peg_num].stats[j];
 }
 
-void AppIdPegCounts::inc_service_count(AppId id)
+void AppIdPegCounts::update_service_count(AppId id, bool increment)
 {
-    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::SERVICE_DETECTS]++;
+    if (increment)
+        (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::SERVICE_DETECTS]++;
+    else
+        (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::SERVICE_DETECTS]--;
 }
 
-void AppIdPegCounts::inc_client_count(AppId id)
+void AppIdPegCounts::update_client_count(AppId id, bool increment)
 {
-    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::CLIENT_DETECTS]++;
+    if (increment)
+        (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::CLIENT_DETECTS]++;
+    else
+        (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::CLIENT_DETECTS]--;
 }
 
-void AppIdPegCounts::inc_user_count(AppId id)
+void AppIdPegCounts::update_payload_count(AppId id, bool increment)
 {
-    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::USER_DETECTS]++;
+    if (increment)
+        (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::PAYLOAD_DETECTS]++;
+    else
+        (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::PAYLOAD_DETECTS]--;
 }
 
-void AppIdPegCounts::inc_payload_count(AppId id)
+void AppIdPegCounts::inc_user_count(AppId id)
 {
-    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::PAYLOAD_DETECTS]++;
+    (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::USER_DETECTS]++;
 }
 
 void AppIdPegCounts::inc_misc_count(AppId id)

diff --git a/src/network_inspectors/appid/appid_peg_counts.h b/src/network_inspectors/appid/appid_peg_counts.h
index 8470dd6c7d7240a81beacc797ca8bc14e6477645..1f89557debfeaa4050f50bceb16d6141a225880b 100644 (file)
--- a/src/network_inspectors/appid/appid_peg_counts.h
+++ b/src/network_inspectors/appid/appid_peg_counts.h
@@ -80,10 +80,12 @@ public:
     static void init_pegs();
     static void cleanup_pegs();
     static void cleanup_peg_info();
-    static void inc_service_count(AppId id);
-    static void inc_client_count(AppId id);
+
+    static void update_service_count(AppId id, bool increment);
+    static void update_client_count(AppId id, bool increment);
+    static void update_payload_count(AppId id, bool increment);
+
     static void inc_user_count(AppId id);
-    static void inc_payload_count(AppId id);
     static void inc_misc_count(AppId id);
 
     static void inc_incompatible_count(AppId id)

diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc
index 24dff6e3c3b7ce90f1786538f16e40a2d64c474a..4af2d395292b42b4be62ba43bb20659985632f44 100644 (file)
--- a/src/network_inspectors/appid/appid_session.cc
+++ b/src/network_inspectors/appid/appid_session.cc
@@ -371,6 +371,19 @@ void AppIdSession::check_ssl_detection_restart(AppidChangeBits& change_bits)
         encrypted.client_id = pick_ss_client_app_id();
         encrypted.misc_id = pick_ss_misc_app_id();
         encrypted.referred_id = pick_ss_referred_payload_app_id();
+
+        // After decryption, new application ids might be detected
+        // overriding existing ones from the encrypted flow. Set overwritten id
+        // to update app statistics when new AppId is detected.
+        if (encrypted.service_id > APP_ID_NONE and client_inferred_service_id == APP_ID_NONE)
+            api.service.set_overwritten_id(encrypted.service_id);
+
+        if (encrypted.client_id > APP_ID_NONE)
+            api.client.set_overwritten_id(encrypted.client_id);
+
+        if (encrypted.payload_id > APP_ID_NONE)
+            api.payload.set_overwritten_id(encrypted.payload_id);
+
         reinit_session_data(change_bits);
         if (appidDebug->is_active())
             LogMessage("AppIdDbg %s SSL decryption is available, restarting app detection\n",

diff --git a/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h b/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h
index 437195bbcaba00551538db52bbbd6a6bf4868f19..8c0f94f1b7a11d3e2e1792b1ea6f3abc010cbfbb 100644 (file)
--- a/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h
+++ b/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h
@@ -103,8 +103,8 @@ AppIdContext ctxt(config);
 AppIdConfig::~AppIdConfig() = default;
 
 // Stubs for AppIdPegCounts
-void AppIdPegCounts::inc_payload_count(AppId) { }
-void AppIdPegCounts::inc_client_count(AppId) { }
+void AppIdPegCounts::update_payload_count(AppId, bool) { }
+void AppIdPegCounts::update_client_count(AppId, bool) { }
 
 THREAD_LOCAL AppIdStats appid_stats;
 

diff --git a/src/network_inspectors/appid/detector_plugins/detector_smtp.cc b/src/network_inspectors/appid/detector_plugins/detector_smtp.cc
index 597035064f22b0c4c1322c82ae755f5c6a2b1c35..7011512056c63f4188a79e4811665c07061dd7bd 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/detector_smtp.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_smtp.cc
@@ -364,7 +364,12 @@ SMTPDetectorData* SmtpClientDetector::get_common_data(AppIdSession& asd)
     {
         dd = (SMTPDetectorData*)snort_calloc(1, sizeof(*dd));
         data_add(asd, dd, &smtp_free_state);
-        dd->server.state = SMTP_SERVICE_STATE_CONNECTION;
+
+        if (asd.get_session_flags(APPID_SESSION_DECRYPTED))
+            dd->server.state = SMTP_SERVICE_STATE_HELO;
+        else
+            dd->server.state = SMTP_SERVICE_STATE_CONNECTION;
+
         dd->client.state = SMTP_CLIENT_STATE_HELO;
         dd->need_continue = 1;
         asd.set_session_flags(APPID_SESSION_CLIENT_GETS_SERVER_PACKETS);

diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
index b4775dbcccc9427100d37e76b07b4f0621594e4c..f19e887bf80a76b70f2ab01d37e147cc7eb9c5b8 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
+++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
@@ -171,10 +171,10 @@ AppIdHttpSession::~AppIdHttpSession()
 }
 
 // Stubs for AppIdPegCounts
-void AppIdPegCounts::inc_service_count(AppId) { }
-void AppIdPegCounts::inc_client_count(AppId) { }
+void AppIdPegCounts::update_service_count(AppId, bool) { }
+void AppIdPegCounts::update_client_count(AppId, bool) { }
 void AppIdPegCounts::inc_user_count(AppId) { }
-void AppIdPegCounts::inc_payload_count(AppId) { }
+void AppIdPegCounts::update_payload_count(AppId, bool) { }
 
 THREAD_LOCAL AppIdStats appid_stats;
 void AppIdModule::sum_stats(bool) { }
@@ -212,10 +212,10 @@ bool AppIdReloadTuner::tune_resources(unsigned int)
 }
 void ApplicationDescriptor::set_id(AppId){}
 void ServiceAppDescriptor::set_id(AppId, OdpContext&){}
-void ServiceAppDescriptor::update_stats(AppId){}
+void ServiceAppDescriptor::update_stats(AppId, bool){}
 void ClientAppDescriptor::update_user(AppId, const char*){}
-void ClientAppDescriptor::update_stats(AppId) {}
-void PayloadAppDescriptor::update_stats(AppId) {}
+void ClientAppDescriptor::update_stats(AppId, bool) {}
+void PayloadAppDescriptor::update_stats(AppId, bool) {}
 void ServiceDiscovery::initialize()
 { }
 

diff --git a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
index 51e235f32795f10a6835aa8e0dee61b391b091f8..8bbf129e85ddbfad07b9a80f4c0afb5f59ff8d82 100644 (file)
--- a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
+++ b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h
@@ -95,10 +95,10 @@ void AppIdDetector::add_payload(AppIdSession&, AppId){}
 void AppIdDetector::add_app(const snort::Packet&, AppIdSession&, AppidSessionDirection, AppId, AppId, const char*, AppidChangeBits&){}
 void ApplicationDescriptor::set_id(AppId){}
 void ServiceAppDescriptor::set_id(AppId, OdpContext&){}
-void ServiceAppDescriptor::update_stats(AppId){}
+void ServiceAppDescriptor::update_stats(AppId, bool){}
 void ClientAppDescriptor::update_user(AppId, const char*){}
-void ClientAppDescriptor::update_stats(AppId) {}
-void PayloadAppDescriptor::update_stats(AppId) {}
+void ClientAppDescriptor::update_stats(AppId, bool) {}
+void PayloadAppDescriptor::update_stats(AppId, bool) {}
 void AppIdDiscovery::add_pattern_data(AppIdDetector*, snort::SearchTool*, int,
         const uint8_t* const, unsigned, unsigned){}
 void AppIdDiscovery::register_detector(const std::string&, AppIdDetector*,  IpProtocol){}
@@ -185,9 +185,9 @@ void AppIdSession::free_flow_data()
 void* AppIdSession::get_flow_data(unsigned) const { return smb_data;}
 
 // Stubs for AppIdPegCounts
-void AppIdPegCounts::inc_service_count(AppId) { }
-void AppIdPegCounts::inc_client_count(AppId) { }
-void AppIdPegCounts::inc_payload_count(AppId) { }
+void AppIdPegCounts::update_service_count(AppId, bool) { }
+void AppIdPegCounts::update_client_count(AppId, bool) { }
+void AppIdPegCounts::update_payload_count(AppId, bool) { }
 
 THREAD_LOCAL AppIdStats appid_stats;
 void AppIdModule::show_dynamic_stats() { }

diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc
index 85774f5d69e6593c2feccc3de3033e2997ca3fbb..5f4d0b2098574a5620691073ddc25960e076c290 100644 (file)
--- a/src/network_inspectors/appid/test/appid_discovery_test.cc
+++ b/src/network_inspectors/appid/test/appid_discovery_test.cc
@@ -134,11 +134,11 @@ void ServiceAppDescriptor::set_id(AppId app_id, OdpContext& odp_ctxt)
     set_id(app_id);
     deferred = odp_ctxt.get_app_info_mgr().get_app_info_flags(app_id, APPINFO_FLAG_DEFER);
 }
-void ServiceAppDescriptor::update_stats(AppId){}
+void ServiceAppDescriptor::update_stats(AppId, bool){}
 void ServiceAppDescriptor::set_port_service_id(AppId){}
 void ClientAppDescriptor::update_user(AppId, const char*){}
-void ClientAppDescriptor::update_stats(AppId) {}
-void PayloadAppDescriptor::update_stats(AppId) {}
+void ClientAppDescriptor::update_stats(AppId, bool) {}
+void PayloadAppDescriptor::update_stats(AppId, bool) {}
 
 // Stubs for AppIdModule
 AppIdModule::AppIdModule(): Module("appid_mock", "appid_mock_help") {}

diff --git a/src/network_inspectors/appid/test/appid_mock_definitions.h b/src/network_inspectors/appid/test/appid_mock_definitions.h
index 05fd3426ab124d4778e0612e1a41b8db163c994f..ca4bb93eee2561c8b43b8f0a51042c6df99635fe 100644 (file)
--- a/src/network_inspectors/appid/test/appid_mock_definitions.h
+++ b/src/network_inspectors/appid/test/appid_mock_definitions.h
@@ -61,15 +61,15 @@ SearchTool::~SearchTool() { }
 
 void ApplicationDescriptor::set_id(AppId app_id){ my_id = app_id;}
 void ServiceAppDescriptor::set_id(AppId app_id, OdpContext&){ set_id(app_id); }
-void ServiceAppDescriptor::update_stats(AppId){}
+void ServiceAppDescriptor::update_stats(AppId, bool){}
 void ServiceAppDescriptor::set_port_service_id(AppId app_id){ port_service_id = app_id;}
 void ClientAppDescriptor::update_user(AppId app_id, const char* username)
 {
     my_username = username;
     my_user_id = app_id;
 }
-void ClientAppDescriptor::update_stats(AppId) {}
-void PayloadAppDescriptor::update_stats(AppId) {}
+void ClientAppDescriptor::update_stats(AppId, bool) {}
+void PayloadAppDescriptor::update_stats(AppId, bool) {}
 
 AppIdDiscovery::AppIdDiscovery() { }
 AppIdDiscovery::~AppIdDiscovery() { }

diff --git a/src/network_inspectors/appid/test/service_state_test.cc b/src/network_inspectors/appid/test/service_state_test.cc
index 7728fa263e523ac4cb3056fb7b2adbe45fe9687f..4bd4c99ce287e3c2c507c828b7452785a5c657a4 100644 (file)
--- a/src/network_inspectors/appid/test/service_state_test.cc
+++ b/src/network_inspectors/appid/test/service_state_test.cc
@@ -77,11 +77,11 @@ void AppIdDebug::activate(const Flow*, const AppIdSession*, bool) { active = tru
 void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { }
 void ApplicationDescriptor::set_id(AppId){}
 void ServiceAppDescriptor::set_id(AppId, OdpContext&){}
-void ServiceAppDescriptor::update_stats(AppId){}
+void ServiceAppDescriptor::update_stats(AppId, bool){}
 void ServiceAppDescriptor::set_port_service_id(AppId){}
 void ClientAppDescriptor::update_user(AppId, const char*){}
-void ClientAppDescriptor::update_stats(AppId) {}
-void PayloadAppDescriptor::update_stats(AppId) {}
+void ClientAppDescriptor::update_stats(AppId, bool) {}
+void PayloadAppDescriptor::update_stats(AppId, bool) {}
 AppIdConfig::~AppIdConfig() { }
 OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { }
 OdpContext::~OdpContext() { }