Merge pull request #1690 in SNORT/snort3 from ~KAMURTHI/snort3:BitTorrent-interval...

Squashed commit of the following:

commit fd2215d74037b7c5c4f5af1bd7613d24b9775539
Author: Kanimozhi Murthi <kamurthi@cisco.com>
Date:   Thu Jul 25 12:15:05 2019 -0400

    appid: support for bittorrent detection when UDP tracker packet arrives after the TCP resumed session has already started

diff --git a/src/flow/test/session_test.cc b/src/flow/test/session_test.cc
index df50b2d7cfd5cfde1bbdc88d66be9cc0294eaa7d..9020101f48897b0ebe935ee7ee46751baff27c47 100644 (file)
--- a/src/flow/test/session_test.cc
+++ b/src/flow/test/session_test.cc
@@ -34,7 +34,7 @@ class DummySession : public Session
 {
     public:
         DummySession(snort::Flow* f) : Session(f) { }
-        void clear() { }
+        void clear() override { }
         ~DummySession() { }
 };
 

diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc
index 3708ef24c8848ea1746488081196c897a458f93a..9992d0149bd8045276ffcf69c95c24a158777f0f 100644 (file)
--- a/src/network_inspectors/appid/app_info_table.cc
+++ b/src/network_inspectors/appid/app_info_table.cc
@@ -51,8 +51,10 @@ static AppInfoTable custom_app_info_table;
 static const char* CONF_SEPARATORS = "\t\n\r";
 static const int MIN_MAX_TP_FLOW_DEPTH = 1;
 static const int MAX_MAX_TP_FLOW_DEPTH = 1000000;
-static const int MIN_HOST_PORT_APP_CACHE_LOOKUP_DELAY = 0;
-static const int MAX_HOST_PORT_APP_CACHE_LOOKUP_DELAY = 1000000;
+static const int MIN_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL = 1;
+static const int MAX_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL = 1000000;
+static const int MIN_HOST_PORT_APP_CACHE_LOOKUP_RANGE = 1;
+static const int MAX_HOST_PORT_APP_CACHE_LOOKUP_RANGE = 1000000;
 static const char* APP_CONFIG_FILE = "appid.conf";
 static const char* USR_CONFIG_FILE = "userappid.conf";
 const char* APP_MAPPING_FILE = "appMapping.data";
@@ -327,19 +329,34 @@ void AppInfoManager::load_appid_config(AppIdModuleConfig* config, const char* pa
                     config->max_tp_flow_depth = max_tp_flow_depth;
                 }
             }
-            else if (!(strcasecmp(conf_key, "host_port_app_cache_lookup_delay")))
+            else if (!(strcasecmp(conf_key, "host_port_app_cache_lookup_interval")))
             {
-                int host_port_app_cache_lookup_delay = atoi(conf_val);
-                if (host_port_app_cache_lookup_delay < MIN_HOST_PORT_APP_CACHE_LOOKUP_DELAY
-                    || host_port_app_cache_lookup_delay > MAX_HOST_PORT_APP_CACHE_LOOKUP_DELAY)
+                int host_port_app_cache_lookup_interval = atoi(conf_val);
+                if (host_port_app_cache_lookup_interval < MIN_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL
+                    || host_port_app_cache_lookup_interval > MAX_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL )
                 {
                      ParseWarning(WARN_CONF,
-                        "AppId: invalid host_port_app_cache_lookup_delay %d, must be between %d and %d\n.",
-                        host_port_app_cache_lookup_delay, MIN_HOST_PORT_APP_CACHE_LOOKUP_DELAY, MAX_HOST_PORT_APP_CACHE_LOOKUP_DELAY);
+                        "AppId: invalid host_port_app_cache_lookup_interval %d, must be between %d and %d\n.",
+                        host_port_app_cache_lookup_interval, MIN_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL , MAX_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL);
                 }
                 else
                 {
-                     config->host_port_app_cache_lookup_delay = host_port_app_cache_lookup_delay;
+                     config->host_port_app_cache_lookup_interval = host_port_app_cache_lookup_interval;
+                }
+            }
+            else if (!(strcasecmp(conf_key, "host_port_app_cache_lookup_range")))
+            {
+                int host_port_app_cache_lookup_range = atoi(conf_val);
+                if (host_port_app_cache_lookup_range < MIN_HOST_PORT_APP_CACHE_LOOKUP_RANGE
+                    || host_port_app_cache_lookup_range > MAX_HOST_PORT_APP_CACHE_LOOKUP_RANGE)
+                {
+                     ParseWarning(WARN_CONF,
+                        "AppId: invalid host_port_app_cache_lookup_range %d, must be between %d and %d\n.",
+                        host_port_app_cache_lookup_range , MIN_HOST_PORT_APP_CACHE_LOOKUP_RANGE, MAX_HOST_PORT_APP_CACHE_LOOKUP_RANGE);
+                }
+                else
+                {
+                     config->host_port_app_cache_lookup_range = host_port_app_cache_lookup_range;
                 }
             }
             else if (!(strcasecmp(conf_key, "is_host_port_app_cache_runtime")))

diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h
index 0d4595046efc0824e5a8a22f63dda84eb1fd2eae..561c2b36f6e16f57dbb5f501b723913afbb2f0b7 100644 (file)
--- a/src/network_inspectors/appid/appid_config.h
+++ b/src/network_inspectors/appid/appid_config.h
@@ -99,7 +99,8 @@ public:
     uint32_t rtmp_max_packets = 15;
     uint32_t max_tp_flow_depth = 5;
     uint32_t tp_allow_probes = 0;
-    uint32_t host_port_app_cache_lookup_delay = 10;
+    uint32_t host_port_app_cache_lookup_interval = 10;
+    uint32_t host_port_app_cache_lookup_range = 100000;
     uint32_t http_response_version_enabled = 0;
 };
 

diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index ee81b0ce3591e764db210b54aa1647770b176a35..79dbfff1277405c26c55a98fa5769c01b85dc975 100644 (file)
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -148,10 +148,15 @@ void AppIdDiscovery::do_application_discovery(Packet* p, AppIdInspector& inspect
         return;
 
     AppId service_id = APP_ID_NONE;
+    AppId client_id = APP_ID_NONE;
+    AppId payload_id = APP_ID_NONE;
+    AppId misc_id = APP_ID_NONE;
     AppidChangeBits change_bits;
-    bool is_discovery_done = do_discovery(p, *asd, protocol, direction, service_id, change_bits);
+    bool is_discovery_done = do_discovery(p, *asd, protocol, direction, service_id, client_id,
+        payload_id, misc_id, change_bits);
 
-    do_post_discovery(p, *asd, direction, is_discovery_done, service_id, change_bits);
+    do_post_discovery(p, *asd, direction, is_discovery_done, service_id, client_id, payload_id,
+        misc_id, change_bits);
 }
 
 void AppIdDiscovery::publish_appid_event(AppidChangeBits& change_bits, snort::Flow* flow)
@@ -594,63 +599,6 @@ static uint64_t is_session_monitored(const Packet* p, AppidSessionDirection dir,
     return flow_flags;
 }
 
-static void lookup_appid_by_host_port(AppIdSession& asd, Packet* p, IpProtocol protocol,
-    AppidSessionDirection direction)
-{
-    HostPortVal* hv = nullptr;
-    uint16_t port = 0;
-    const SfIp* ip = nullptr;
-    asd.scan_flags |= SCAN_HOST_PORT_FLAG;
-    if (direction == APP_ID_FROM_INITIATOR)
-    {
-        ip = p->ptrs.ip_api.get_dst();
-        port = p->ptrs.dp;
-    }
-    else
-    {
-        ip = p->ptrs.ip_api.get_src();
-        port = p->ptrs.sp;
-    }
-    if ((hv = HostPortCache::find(ip, port, protocol)))
-    {
-        switch (hv->type)
-        {
-        case 1:
-            asd.client.set_id(hv->appId);
-            asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
-            break;
-        case 2:
-            asd.payload.set_id(hv->appId);
-            break;
-        default:
-            asd.service.set_id(hv->appId);
-            asd.sync_with_snort_protocol_id(hv->appId, p);
-            asd.service_disco_state = APPID_DISCO_STATE_FINISHED;
-            asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
-            asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED);
-#ifdef ENABLE_APPID_THIRD_PARTY
-            if (asd.tpsession)
-                asd.tpsession->reset();
-#endif
-            if ( asd.payload.get_id() == APP_ID_NONE)
-                asd.payload.set_id(APP_ID_UNKNOWN);
-        }
-    }
-    else if (asd.config->mod_config->is_host_port_app_cache_runtime)
-    {
-        auto ht = host_cache.find(*ip);
-        if ( ht )
-        {
-            AppId appid = ht->get_appid(port, protocol, true);
-            if ( appid > APP_ID_NONE )
-            {
-                asd.client.set_id(appid);
-                asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
-            }
-        }
-    }
-}
-
 bool AppIdDiscovery::handle_unmonitored_session(AppIdSession* asd, const Packet* p,
     IpProtocol protocol, AppidSessionDirection dir, AppIdInspector& inspector,
     uint64_t& flow_flags)
@@ -898,19 +846,99 @@ void AppIdDiscovery::do_port_based_discovery(Packet* p, AppIdSession& asd, IpPro
     asd.set_session_flags(APPID_SESSION_PORT_SERVICE_DONE);
 }
 
-bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol protocol,
-    AppidSessionDirection direction, AppId& service_id, AppidChangeBits& change_bits)
+bool AppIdDiscovery::do_host_port_based_discovery(Packet* p, AppIdSession& asd, IpProtocol protocol,
+    AppidSessionDirection direction)
 {
-    bool is_discovery_done = false;
+    if (asd.get_session_flags(APPID_SESSION_HOST_CACHE_MATCHED))
+        return false;
+
+    bool check_static = false;
+    bool check_dynamic = false;
 
-    // {host, port} based detection
     if (!(asd.scan_flags & SCAN_HOST_PORT_FLAG))
+        check_static = true;
+
+    if ((asd.session_packet_count % asd.config->mod_config->host_port_app_cache_lookup_interval == 0) and
+        (asd.session_packet_count <= asd.config->mod_config->host_port_app_cache_lookup_range) and
+        asd.config->mod_config->is_host_port_app_cache_runtime )
+        check_dynamic = true;
+
+    if (!(check_static || check_dynamic))
+        return false;
+
+    uint16_t port;
+    const SfIp* ip;
+
+    if (direction == APP_ID_FROM_INITIATOR)
+    {
+        ip = p->ptrs.ip_api.get_dst();
+        port = p->ptrs.dp;
+    }
+    else
     {
-        if ((asd.is_tp_processing_done() && asd.get_tp_app_id() <= APP_ID_NONE) ||
-            (asd.session_packet_count > asd.config->mod_config->host_port_app_cache_lookup_delay))
-            lookup_appid_by_host_port(asd, p, protocol, direction);
+        ip = p->ptrs.ip_api.get_src();
+        port = p->ptrs.sp;
     }
 
+    HostPortVal* hv = nullptr;
+
+    if (check_static and
+        (hv = HostPortCache::find(ip, port, protocol)))
+    {
+        asd.scan_flags |= SCAN_HOST_PORT_FLAG;
+        switch (hv->type)
+        {
+        case APP_ID_TYPE_CLIENT:
+            asd.client.set_id(hv->appId);
+            asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
+            break;
+        case APP_ID_TYPE_PAYLOAD:
+            asd.payload.set_id(hv->appId);
+            break;
+        default:
+            asd.service.set_id(hv->appId);
+            asd.sync_with_snort_protocol_id(hv->appId, p);
+            asd.service_disco_state = APPID_DISCO_STATE_FINISHED;
+            asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
+            asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED);
+#ifdef ENABLE_APPID_THIRD_PARTY
+            if (asd.tpsession)
+                asd.tpsession->reset();
+#endif
+            if ( asd.payload.get_id() == APP_ID_NONE)
+                asd.payload.set_id(APP_ID_UNKNOWN);
+        }
+        asd.set_session_flags(APPID_SESSION_HOST_CACHE_MATCHED);
+        return true;
+    }
+
+    if (!hv and check_dynamic)
+    {
+        auto ht = host_cache.find(*ip);
+        if (ht)
+        {
+            AppId appid = ht->get_appid(port, protocol, true);
+            if (appid > APP_ID_NONE)
+            {
+                // FIXIT-L: Make this more generic to support service and payload IDs
+                asd.client.set_id(appid);
+                asd.client_disco_state = APPID_DISCO_STATE_FINISHED;
+                asd.set_session_flags(APPID_SESSION_HOST_CACHE_MATCHED);
+            }
+        }
+
+        return true;
+    }
+
+    return false;
+}
+
+bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol protocol,
+    AppidSessionDirection direction, AppId& service_id, AppId& client_id, AppId& payload_id,
+    AppId& misc_id, AppidChangeBits& change_bits)
+{
+    bool is_discovery_done = false;
+
     asd.check_app_detection_restart(change_bits);
 
     if (protocol != IpProtocol::TCP and protocol != IpProtocol::UDP)
@@ -955,7 +983,6 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto
         if (asd.service.get_id() == APP_ID_DNS &&
             asd.config->mod_config->dns_host_reporting && dsession->get_host() )
         {
-            AppId client_id = APP_ID_NONE, payload_id = APP_ID_NONE;
             dns_host_scan_hostname((const uint8_t*)dsession->get_host(), dsession->get_host_len(),
                 &client_id, &payload_id);
             asd.set_client_appid_data(client_id, nullptr, change_bits);
@@ -1004,15 +1031,28 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto
         }
     }
 
+    payload_id = asd.pick_payload_app_id();
+    client_id = asd.pick_client_app_id();
+    misc_id =  asd.pick_misc_app_id();;
+
+    if ((service_id == APP_ID_UNKNOWN_UI or service_id <= APP_ID_NONE ) and
+        (client_id <= APP_ID_NONE and payload_id <= APP_ID_NONE and misc_id <= APP_ID_NONE))
+    {
+        if (do_host_port_based_discovery(p, asd, protocol, direction))
+        {
+            service_id = asd.pick_service_app_id();
+            client_id = asd.pick_client_app_id();
+            payload_id = asd.pick_payload_app_id();
+        }
+    }
+
     return is_discovery_done;
 }
 
 void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd,
     AppidSessionDirection direction, bool is_discovery_done, AppId service_id,
-    AppidChangeBits& change_bits)
+    AppId client_id, AppId payload_id, AppId misc_id, AppidChangeBits& change_bits)
 {
-    AppId payload_id = asd.pick_payload_app_id();
-
     if (service_id > APP_ID_NONE)
     {
         if (asd.get_session_flags(APPID_SESSION_DECRYPTED))
@@ -1067,8 +1107,7 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd,
         if ( asd.past_forecast != service_id and asd.past_forecast != APP_ID_UNKNOWN and
              asd.payload.get_id() == APP_ID_NONE )
         {
-            asd.past_forecast = check_session_for_AF_forecast(asd, p, direction,
-                (AppId)service_id);
+            asd.past_forecast = check_session_for_AF_forecast(asd, p, direction, service_id);
             if (asd.past_forecast != APP_ID_UNKNOWN)
                 payload_id = asd.pick_payload_app_id();
         }
@@ -1089,7 +1128,6 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd,
     }
 #endif
 
-    asd.set_application_ids(service_id, asd.pick_client_app_id(), payload_id,
-        asd.pick_misc_app_id(), change_bits);
+    asd.set_application_ids(service_id, client_id, payload_id, misc_id, change_bits);
     publish_appid_event(change_bits, p->flow);
 }

diff --git a/src/network_inspectors/appid/appid_discovery.h b/src/network_inspectors/appid/appid_discovery.h
index 4a0516aaa0025c3fdacb76194c82958937b6c636..3e7d8f5972a017d8f18351c1126f75f99f975852 100644 (file)
--- a/src/network_inspectors/appid/appid_discovery.h
+++ b/src/network_inspectors/appid/appid_discovery.h
@@ -135,12 +135,15 @@ private:
     static bool do_pre_discovery(snort::Packet* p, AppIdSession** p_asd, AppIdInspector& inspector,
         IpProtocol& protocol, AppidSessionDirection& direction);
     static bool do_discovery(snort::Packet* p, AppIdSession& asd, IpProtocol protocol,
-        AppidSessionDirection direction, AppId& service_id, AppidChangeBits& change_bits);
+        AppidSessionDirection direction, AppId& service_id, AppId& client_id, AppId& payload_id,
+        AppId& misc_id, AppidChangeBits& change_bits);
     static void do_post_discovery(snort::Packet* p, AppIdSession& asd,
-        AppidSessionDirection direction, bool is_discovery_done, AppId service_id,
-        AppidChangeBits& change_bits);
+        AppidSessionDirection direction, bool is_discovery_done, AppId service_id, AppId client_id,
+        AppId payload_id, AppId misc_id, AppidChangeBits& change_bits);
     static void do_port_based_discovery(snort::Packet* p, AppIdSession& asd, IpProtocol protocol,
         AppidSessionDirection direction);
+    static bool do_host_port_based_discovery(snort::Packet* p, AppIdSession& asd,
+        IpProtocol protocol, AppidSessionDirection direction);
     static bool handle_unmonitored_session(AppIdSession* asd, const snort::Packet* p,
         IpProtocol protocol, AppidSessionDirection dir, AppIdInspector& inspector,
         uint64_t& flow_flags);

diff --git a/src/network_inspectors/appid/appid_session_api.h b/src/network_inspectors/appid/appid_session_api.h
index 525a65f180176921a9a48fd231264bde9d9bce70..4b374502bf17fb53c7b7d56fec0607d23e486ac3 100644 (file)
--- a/src/network_inspectors/appid/appid_session_api.h
+++ b/src/network_inspectors/appid/appid_session_api.h
@@ -87,6 +87,7 @@ namespace snort
 #define APPID_SESSION_IGNORE_FLOW_IDED      (1ULL << 40)
 #define APPID_SESSION_OOO_CHECK_TP          (1ULL << 41)
 #define APPID_SESSION_PAYLOAD_SEEN          (1ULL << 42)
+#define APPID_SESSION_HOST_CACHE_MATCHED    (1ULL << 43)
 #define APPID_SESSION_IGNORE_ID_FLAGS \
     (APPID_SESSION_IGNORE_FLOW | \
     APPID_SESSION_NOT_A_SERVICE | \

diff --git a/src/network_inspectors/appid/application_ids.h b/src/network_inspectors/appid/application_ids.h
index 9acdaf569a578e72361b30c7ff38e0d68957799d..5945287a57a6e875eda9f2ca4ae2f1f12a1508e0 100644 (file)
--- a/src/network_inspectors/appid/application_ids.h
+++ b/src/network_inspectors/appid/application_ids.h
@@ -1021,6 +1021,14 @@ enum ApplicationIds : AppId
     APP_ID_UNKNOWN_UI                     = 65535   // UI renders this value as 'Unknown'
 };
 
+enum AppIdType
+{
+    APP_ID_TYPE_SERVICE,
+    APP_ID_TYPE_CLIENT,
+    APP_ID_TYPE_PAYLOAD,
+    APP_ID_TYPE_MAX
+};
+
 struct AppRegistryEntry
 {
     AppId appId;