From: Shravan Rangarajuvenkata (shrarang) <shrarang@cisco.com>
Date: Tue, 3 Dec 2019 22:08:14 +0000 (+0000)
Subject: Merge pull request #1871 in SNORT/snort3 from ~SATHIRKA/snort3:host_cache_ssl to... 
X-Git-Tag: 3.0.0-266~1
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a010418cd682d10ced3fac6a705429030c706120;p=thirdparty%2Fsnort3.git

Merge pull request #1871 in SNORT/snort3 from ~SATHIRKA/snort3:host_cache_ssl to master

Squashed commit of the following:

commit 54ad92428c38323655e2b816d7eae3e7901a2b67
Author: Sreeja Athirkandathil Narayanan <sathirka@cisco.com>
Date:   Wed Nov 27 14:34:18 2019 -0500

    appid: Enabling host cache for unknown SSL flows
---

diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc
index ec847f556..bf2ddc3af 100644
--- a/src/network_inspectors/appid/app_info_table.cc
+++ b/src/network_inspectors/appid/app_info_table.cc
@@ -367,6 +367,20 @@ void AppInfoManager::load_appid_config(AppIdModuleConfig* config, const char* pa
                     config->is_host_port_app_cache_runtime = true;
                 }
             }
+            else if (!(strcasecmp(conf_key, "check_host_port_app_cache")))
+            {
+                if (!(strcasecmp(conf_val, "enabled")))
+                {
+                    config->check_host_port_app_cache = true;
+                }
+            }
+            else if (!(strcasecmp(conf_key, "check_host_cache_unknown_ssl")))
+            {
+                if (!(strcasecmp(conf_val, "enabled")))
+                {
+                    config->check_host_cache_unknown_ssl = true;
+                }
+            }
             else if (!(strcasecmp(conf_key, "allow_port_wildcard_host_cache")))
             {
                 if (!(strcasecmp(conf_val, "enabled")))
@@ -404,6 +418,48 @@ void AppInfoManager::load_appid_config(AppIdModuleConfig* config, const char* pa
                     LogMessage("AppId: allow_port_wildcard_host_cache enabled\n");
                 }
             }
+            else if (!(strcasecmp(conf_key, "ultrasurf_aggressiveness")))
+            {
+                int aggressiveness = atoi(conf_val);
+                LogMessage("AppId: ultrasurf_aggressiveness %d\n", aggressiveness);
+                if (aggressiveness >= 50)
+                {
+                    config->check_host_cache_unknown_ssl = true;
+                    set_app_info_flags(APP_ID_ULTRASURF, APPINFO_FLAG_DEFER);
+                    set_app_info_flags(APP_ID_ULTRASURF, APPINFO_FLAG_DEFER_PAYLOAD);
+                    config->max_tp_flow_depth = 25;
+                    LogMessage("AppId: check_host_cache_unknown_ssl enabled\n");
+                    LogMessage("AppId: defer_to_thirdparty %d\n", APP_ID_ULTRASURF);
+                    LogMessage("AppId: defer_payload_to_thirdparty %d\n", APP_ID_ULTRASURF);
+                    LogMessage("AppId: max_tp_flow_depth %d\n", config->max_tp_flow_depth);
+                }
+                if (aggressiveness >= 80)
+                {
+                    config->allow_port_wildcard_host_cache = true;
+                    LogMessage("AppId: allow_port_wildcard_host_cache enabled\n");
+                }
+            }
+            else if (!(strcasecmp(conf_key, "psiphon_aggressiveness")))
+            {
+                int aggressiveness = atoi(conf_val);
+                LogMessage("AppId: psiphon_aggressiveness %d\n", aggressiveness);
+                if (aggressiveness >= 50)
+                {
+                    config->check_host_cache_unknown_ssl = true;
+                    set_app_info_flags(APP_ID_PSIPHON, APPINFO_FLAG_DEFER);
+                    set_app_info_flags(APP_ID_PSIPHON, APPINFO_FLAG_DEFER_PAYLOAD);
+                    config->max_tp_flow_depth = 25;
+                    LogMessage("AppId: check_host_cache_unknown_ssl enabled\n");
+                    LogMessage("AppId: defer_to_thirdparty %d\n", APP_ID_PSIPHON);
+                    LogMessage("AppId: defer_payload_to_thirdparty %d\n", APP_ID_PSIPHON);
+                    LogMessage("AppId: max_tp_flow_depth %d\n", config->max_tp_flow_depth);
+                }
+                if (aggressiveness >= 80)
+                {
+                    config->allow_port_wildcard_host_cache = true;
+                    LogMessage("AppId: allow_port_wildcard_host_cache enabled\n");
+                }
+            }
             else if (!(strcasecmp(conf_key, "tp_allow_probes")))
             {
                 if (!(strcasecmp(conf_val, "enabled")))
diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h
index 5750b19f0..a29383567 100644
--- a/src/network_inspectors/appid/appid_config.h
+++ b/src/network_inspectors/appid/appid_config.h
@@ -93,6 +93,8 @@ public:
     bool chp_userid_disabled = false;
     bool http2_detection_enabled = false;
     bool is_host_port_app_cache_runtime = false;
+    bool check_host_port_app_cache = false;
+    bool check_host_cache_unknown_ssl = false;
     uint32_t ftp_userid_disabled = 0;
     uint32_t chp_body_collection_disabled = 0;
     uint32_t chp_body_collection_max = 0;
diff --git a/src/network_inspectors/appid/appid_detector.cc b/src/network_inspectors/appid/appid_detector.cc
index 37a3f70f0..a981971ca 100644
--- a/src/network_inspectors/appid/appid_detector.cc
+++ b/src/network_inspectors/appid/appid_detector.cc
@@ -88,15 +88,15 @@ void AppIdDetector::add_payload(AppIdSession& asd, AppId payload_id)
     asd.payload.set_id(payload_id);
 }
 
-void AppIdDetector::add_app(AppIdSession& asd, AppId service_id, AppId client_id,
-    const char* version, AppidChangeBits& change_bits)
+void AppIdDetector::add_app(const Packet& p, AppIdSession& asd, AppidSessionDirection dir, AppId service_id,
+    AppId client_id, const char* version, AppidChangeBits& change_bits)
 {
     if ( version )
         asd.client.set_version(version, change_bits);
 
     asd.set_client_detected();
     asd.client_inferred_service_id = service_id;
-    asd.client.set_id(client_id);
+    asd.client.set_id(p, asd, dir, client_id, change_bits);
 }
 
 const char* AppIdDetector::get_code_string(APPID_STATUS_CODE code) const
diff --git a/src/network_inspectors/appid/appid_detector.h b/src/network_inspectors/appid/appid_detector.h
index 9d302c8da..ceef1ca8b 100644
--- a/src/network_inspectors/appid/appid_detector.h
+++ b/src/network_inspectors/appid/appid_detector.h
@@ -122,7 +122,16 @@ public:
     virtual void add_info(AppIdSession&, const char*, AppidChangeBits&);
     virtual void add_user(AppIdSession&, const char*, AppId, bool);
     virtual void add_payload(AppIdSession&, AppId);
-    virtual void add_app(AppIdSession&, AppId, AppId, const char*, AppidChangeBits&);
+    virtual void add_app(AppIdSession& asd, AppId service_id, AppId client_id, const char* version, AppidChangeBits& change_bits)
+    {
+        if ( version )
+            asd.client.set_version(version, change_bits);
+
+        asd.set_client_detected();
+        asd.client_inferred_service_id = service_id;
+        asd.client.set_id(client_id);
+    }
+    virtual void add_app(const snort::Packet&, AppIdSession&, AppidSessionDirection, AppId, AppId, const char*, AppidChangeBits&);
     virtual void finalize_patterns() {}
     const char* get_code_string(APPID_STATUS_CODE) const;
 
diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index 168de593c..36c4c46e9 100644
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -945,6 +945,18 @@ bool AppIdDiscovery::do_host_port_based_discovery(Packet* p, AppIdSession& asd,
     return false;
 }
 
+static inline bool is_check_host_cache_valid(AppIdSession& asd, AppId service_id, AppId client_id, AppId payload_id, AppId misc_id)
+{
+    bool is_payload_client_misc_none = (payload_id <= APP_ID_NONE and client_id <= APP_ID_NONE and misc_id <= APP_ID_NONE);
+    bool is_appid_none = is_payload_client_misc_none and (service_id <= APP_ID_NONE or service_id == APP_ID_UNKNOWN_UI or
+        (asd.config->mod_config->recheck_for_portservice_appid and service_id == asd.service.get_port_service_id()));
+    bool is_ssl_none = asd.config->mod_config->check_host_cache_unknown_ssl and asd.get_session_flags(APPID_SESSION_SSL_SESSION) and
+                          (not(asd.tsession and asd.tsession->get_tls_host() and asd.tsession->get_tls_cname()));
+    if (is_appid_none or is_ssl_none or asd.config->mod_config->check_host_port_app_cache)
+        return true;
+    return false;
+}
+
 bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol protocol,
     AppidSessionDirection direction, AppId& service_id, AppId& client_id, AppId& payload_id,
     AppId& misc_id, AppidChangeBits& change_bits)
@@ -1043,10 +1055,7 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto
     misc_id =  asd.pick_misc_app_id();;
 
     bool is_http_tunnel = ((asd.payload.get_id() == APP_ID_HTTP_TUNNEL) || (asd.payload.get_id() == APP_ID_HTTP_SSL_TUNNEL)) ? true:false;
-    bool is_appid_none = (client_id <= APP_ID_NONE and payload_id <= APP_ID_NONE and misc_id <= APP_ID_NONE);
-    if ((is_appid_none and (service_id == APP_ID_UNKNOWN_UI or service_id <= APP_ID_NONE or
-        (asd.config->mod_config->recheck_for_portservice_appid and service_id == asd.service.get_port_service_id())))
-        or (is_http_tunnel))
+    if (is_check_host_cache_valid(asd, service_id, client_id, payload_id, misc_id) or (is_http_tunnel))
     {
         if(is_http_tunnel)
         {
diff --git a/src/network_inspectors/appid/appid_discovery.h b/src/network_inspectors/appid/appid_discovery.h
index 04e5b18ef..bd0354bf0 100644
--- a/src/network_inspectors/appid/appid_discovery.h
+++ b/src/network_inspectors/appid/appid_discovery.h
@@ -49,6 +49,7 @@ struct Packet;
 #define SCAN_HTTP_VIA_FLAG          (1<<0)
 #define SCAN_HTTP_USER_AGENT_FLAG   (1<<1)
 #define SCAN_HTTP_HOST_URL_FLAG     (1<<2)
+#define SCAN_SSL_CERTIFICATE_FLAG   (1<<3)
 #define SCAN_SSL_HOST_FLAG          (1<<4)
 #define SCAN_HOST_PORT_FLAG         (1<<5)
 #define SCAN_HTTP_VENDOR_FLAG       (1<<6)
diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc
index 40345561f..bb1b81fe6 100644
--- a/src/network_inspectors/appid/appid_session.cc
+++ b/src/network_inspectors/appid/appid_session.cc
@@ -430,7 +430,7 @@ void AppIdSession::examine_ssl_metadata(Packet* p, AppidChangeBits& change_bits)
         }
         scan_flags &= ~SCAN_SSL_HOST_FLAG;
     }
-    if ((tls_str = tsession->get_tls_cname()))
+    if ((scan_flags & SCAN_SSL_CERTIFICATE_FLAG) and (tls_str = tsession->get_tls_cname()))
     {
         size_t size = strlen(tls_str);
         if ((ret = ssl_scan_cname((const uint8_t*)tls_str, size,
@@ -441,7 +441,7 @@ void AppIdSession::examine_ssl_metadata(Packet* p, AppidChangeBits& change_bits)
             set_payload_appid_data(payload_id, change_bits);
             setSSLSquelch(p, ret, (ret == 1 ? payload_id : client_id));
         }
-        tsession->set_tls_cname(nullptr, 0);
+        scan_flags &= ~SCAN_SSL_CERTIFICATE_FLAG;
     }
     if ((tls_str = tsession->get_tls_org_unit()))
     {
diff --git a/src/network_inspectors/appid/appid_session_api.cc b/src/network_inspectors/appid/appid_session_api.cc
index 5215d2185..993ce994d 100644
--- a/src/network_inspectors/appid/appid_session_api.cc
+++ b/src/network_inspectors/appid/appid_session_api.cc
@@ -133,6 +133,9 @@ bool AppIdSessionApi::is_appid_inspecting_session()
         return true;
     }
 
+    if (asd->config->mod_config->check_host_port_app_cache)
+        return true;
+
     return false;
 }
 
diff --git a/src/network_inspectors/appid/application_ids.h b/src/network_inspectors/appid/application_ids.h
index 5945287a5..39b5dbbeb 100644
--- a/src/network_inspectors/appid/application_ids.h
+++ b/src/network_inspectors/appid/application_ids.h
@@ -1013,6 +1013,7 @@ enum ApplicationIds : AppId
     APP_ID_HTTP_SSL_TUNNEL                = 3860,
     APP_ID_FTP_ACTIVE                     = 4002,
     APP_ID_FTP_PASSIVE                    = 4003,
+    APP_ID_PSIPHON                        = 4075,
 #ifdef REG_TEST
     APP_ID_REGTEST                        = 10000,
     APP_ID_REGTEST1                       = 10001,
diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc
index 736d9f6cd..7983b81d4 100644
--- a/src/network_inspectors/appid/lua_detector_api.cc
+++ b/src/network_inspectors/appid/lua_detector_api.cc
@@ -914,7 +914,7 @@ static int client_add_application(lua_State* L)
     unsigned int service_id = lua_tonumber(L, 2);
     unsigned int productId = lua_tonumber(L, 4);
     const char* version = lua_tostring(L, 5);
-    ud->cd->add_app(*lsd->ldp.asd,
+    ud->cd->add_app(*lsd->ldp.pkt, *lsd->ldp.asd, lsd->ldp.dir,
         AppInfoManager::get_instance().get_appid_by_service_id(service_id),
         AppInfoManager::get_instance().get_appid_by_client_id(productId), version,
         *lsd->ldp.change_bits);
diff --git a/src/network_inspectors/appid/service_plugins/service_ssl.cc b/src/network_inspectors/appid/service_plugins/service_ssl.cc
index 15890eb5d..f97446254 100644
--- a/src/network_inspectors/appid/service_plugins/service_ssl.cc
+++ b/src/network_inspectors/appid/service_plugins/service_ssl.cc
@@ -807,8 +807,10 @@ success:
 
         /* TLS Common Name */
         if (ss->common_name)
+        {
             args.asd.tsession->set_tls_cname(ss->common_name, 0);
-
+            args.asd.scan_flags |= SCAN_SSL_CERTIFICATE_FLAG;
+        }
         /* TLS Org Unit */
         if (ss->org_name)
             args.asd.tsession->set_tls_org_unit(ss->org_name, 0);
diff --git a/src/network_inspectors/appid/test/appid_session_api_test.cc b/src/network_inspectors/appid/test/appid_session_api_test.cc
index 33dc2d9c0..3ecf9db5f 100644
--- a/src/network_inspectors/appid/test/appid_session_api_test.cc
+++ b/src/network_inspectors/appid/test/appid_session_api_test.cc
@@ -49,6 +49,13 @@ void NbdgmServiceDetector::AppIdFreeSMBData(FpSMBData* data)
 AppIdSession* mock_session = nullptr;
 AppIdSessionApi* appid_session_api = nullptr;
 
+//Stub for config
+AppIdConfig::AppIdConfig(AppIdModuleConfig* mod)
+{
+    this->mod_config = mod;
+    this->mod_config->check_host_port_app_cache = false;
+}
+
 TEST_GROUP(appid_session_api)
 {
     void setup() override
@@ -216,6 +223,12 @@ TEST(appid_session_api, is_appid_inspecting_session)
     mock_session->set_tp_app_id(APP_ID_SSH);
     val = appid_session_api->is_appid_inspecting_session();
     CHECK_TRUE(val);
+
+    // 4th if in is_appid_inspecting_session
+    mock_session->set_tp_app_id(APP_ID_NONE);
+    mock_session->config->mod_config->check_host_port_app_cache = true;
+    val = appid_session_api->is_appid_inspecting_session();
+    CHECK_TRUE(val);
 }
 
 TEST(appid_session_api, get_user_name)
@@ -360,6 +373,8 @@ int main(int argc, char** argv)
 {
     mock_init_appid_pegs();
     mock_session = new AppIdSession(IpProtocol::TCP, nullptr, 1492, appid_inspector);
+    AppIdModuleConfig *mod_config = new AppIdModuleConfig();
+    mock_session->config = new AppIdConfig(mod_config);
     int rc = CommandLineTestRunner::RunAllTests(argc, argv);
     mock_cleanup_appid_pegs();
     return rc;
diff --git a/src/network_inspectors/appid/tp_appid_utils.cc b/src/network_inspectors/appid/tp_appid_utils.cc
index 78abc01fd..8b0b53f58 100644
--- a/src/network_inspectors/appid/tp_appid_utils.cc
+++ b/src/network_inspectors/appid/tp_appid_utils.cc
@@ -512,6 +512,7 @@ static inline void process_ssl(AppIdSession& asd,
     AppId tmpAppId = APP_ID_NONE;
     int tmpConfidence = 0;
     const string* field = 0;
+    int reinspect_ssl_appid = 0;
 
     // if (tp_appid_module && asd.tpsession)
     tmpAppId = asd.tpsession->get_appid(tmpConfidence);
@@ -524,21 +525,25 @@ static inline void process_ssl(AppIdSession& asd,
     if (!asd.client.get_id())
         asd.set_client_appid_data(APP_ID_SSL_CLIENT, change_bits);
 
-    if ( (field=attribute_data.tls_host(false)) != nullptr )
+    reinspect_ssl_appid = check_ssl_appid_for_reinspect(tmpAppId);
+
+    if ((field=attribute_data.tls_host(false)) != nullptr)
     {
         asd.tsession->set_tls_host(field->c_str(), field->size(), change_bits);
-        if (check_ssl_appid_for_reinspect(tmpAppId))
+        if (reinspect_ssl_appid)
             asd.scan_flags |= SCAN_SSL_HOST_FLAG;
     }
 
-    if (check_ssl_appid_for_reinspect(tmpAppId))
+    if ((field=attribute_data.tls_cname()) != nullptr)
     {
-        if ( (field=attribute_data.tls_cname()) != nullptr )
-        {
-            asd.tsession->set_tls_cname(field->c_str(), field->size());
-        }
+        asd.tsession->set_tls_cname(field->c_str(), field->size());
+        if (reinspect_ssl_appid)
+            asd.scan_flags |= SCAN_SSL_CERTIFICATE_FLAG;
+    }
 
-        if ( (field=attribute_data.tls_org_unit()) != nullptr )
+    if (reinspect_ssl_appid)
+    {
+        if ((field=attribute_data.tls_org_unit()) != nullptr)
         {
             asd.tsession->set_tls_org_unit(field->c_str(), field->size());
         }