From: Shravan Rangarajuvenkata (shrarang) Date: Mon, 12 Aug 2019 23:26:37 +0000 (-0400) Subject: Merge pull request #1690 in SNORT/snort3 from ~KAMURTHI/snort3:BitTorrent-interval... X-Git-Tag: 3.0.0-259~9 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0780c7b156dcb48672a01c84d94cc630b623e3bf;p=thirdparty%2Fsnort3.git Merge pull request #1690 in SNORT/snort3 from ~KAMURTHI/snort3:BitTorrent-interval-check to master Squashed commit of the following: commit fd2215d74037b7c5c4f5af1bd7613d24b9775539 Author: Kanimozhi Murthi Date: Thu Jul 25 12:15:05 2019 -0400 appid: support for bittorrent detection when UDP tracker packet arrives after the TCP resumed session has already started --- diff --git a/src/flow/test/session_test.cc b/src/flow/test/session_test.cc index df50b2d7c..9020101f4 100644 --- a/src/flow/test/session_test.cc +++ b/src/flow/test/session_test.cc @@ -34,7 +34,7 @@ class DummySession : public Session { public: DummySession(snort::Flow* f) : Session(f) { } - void clear() { } + void clear() override { } ~DummySession() { } }; diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc index 3708ef24c..9992d0149 100644 --- a/src/network_inspectors/appid/app_info_table.cc +++ b/src/network_inspectors/appid/app_info_table.cc @@ -51,8 +51,10 @@ static AppInfoTable custom_app_info_table; static const char* CONF_SEPARATORS = "\t\n\r"; static const int MIN_MAX_TP_FLOW_DEPTH = 1; static const int MAX_MAX_TP_FLOW_DEPTH = 1000000; -static const int MIN_HOST_PORT_APP_CACHE_LOOKUP_DELAY = 0; -static const int MAX_HOST_PORT_APP_CACHE_LOOKUP_DELAY = 1000000; +static const int MIN_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL = 1; +static const int MAX_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL = 1000000; +static const int MIN_HOST_PORT_APP_CACHE_LOOKUP_RANGE = 1; +static const int MAX_HOST_PORT_APP_CACHE_LOOKUP_RANGE = 1000000; static const char* APP_CONFIG_FILE = "appid.conf"; static const char* USR_CONFIG_FILE = "userappid.conf"; const char* APP_MAPPING_FILE = "appMapping.data"; @@ -327,19 +329,34 @@ void AppInfoManager::load_appid_config(AppIdModuleConfig* config, const char* pa config->max_tp_flow_depth = max_tp_flow_depth; } } - else if (!(strcasecmp(conf_key, "host_port_app_cache_lookup_delay"))) + else if (!(strcasecmp(conf_key, "host_port_app_cache_lookup_interval"))) { - int host_port_app_cache_lookup_delay = atoi(conf_val); - if (host_port_app_cache_lookup_delay < MIN_HOST_PORT_APP_CACHE_LOOKUP_DELAY - || host_port_app_cache_lookup_delay > MAX_HOST_PORT_APP_CACHE_LOOKUP_DELAY) + int host_port_app_cache_lookup_interval = atoi(conf_val); + if (host_port_app_cache_lookup_interval < MIN_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL + || host_port_app_cache_lookup_interval > MAX_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL ) { ParseWarning(WARN_CONF, - "AppId: invalid host_port_app_cache_lookup_delay %d, must be between %d and %d\n.", - host_port_app_cache_lookup_delay, MIN_HOST_PORT_APP_CACHE_LOOKUP_DELAY, MAX_HOST_PORT_APP_CACHE_LOOKUP_DELAY); + "AppId: invalid host_port_app_cache_lookup_interval %d, must be between %d and %d\n.", + host_port_app_cache_lookup_interval, MIN_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL , MAX_HOST_PORT_APP_CACHE_LOOKUP_INTERVAL); } else { - config->host_port_app_cache_lookup_delay = host_port_app_cache_lookup_delay; + config->host_port_app_cache_lookup_interval = host_port_app_cache_lookup_interval; + } + } + else if (!(strcasecmp(conf_key, "host_port_app_cache_lookup_range"))) + { + int host_port_app_cache_lookup_range = atoi(conf_val); + if (host_port_app_cache_lookup_range < MIN_HOST_PORT_APP_CACHE_LOOKUP_RANGE + || host_port_app_cache_lookup_range > MAX_HOST_PORT_APP_CACHE_LOOKUP_RANGE) + { + ParseWarning(WARN_CONF, + "AppId: invalid host_port_app_cache_lookup_range %d, must be between %d and %d\n.", + host_port_app_cache_lookup_range , MIN_HOST_PORT_APP_CACHE_LOOKUP_RANGE, MAX_HOST_PORT_APP_CACHE_LOOKUP_RANGE); + } + else + { + config->host_port_app_cache_lookup_range = host_port_app_cache_lookup_range; } } else if (!(strcasecmp(conf_key, "is_host_port_app_cache_runtime"))) diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h index 0d4595046..561c2b36f 100644 --- a/src/network_inspectors/appid/appid_config.h +++ b/src/network_inspectors/appid/appid_config.h @@ -99,7 +99,8 @@ public: uint32_t rtmp_max_packets = 15; uint32_t max_tp_flow_depth = 5; uint32_t tp_allow_probes = 0; - uint32_t host_port_app_cache_lookup_delay = 10; + uint32_t host_port_app_cache_lookup_interval = 10; + uint32_t host_port_app_cache_lookup_range = 100000; uint32_t http_response_version_enabled = 0; }; diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index ee81b0ce3..79dbfff12 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -148,10 +148,15 @@ void AppIdDiscovery::do_application_discovery(Packet* p, AppIdInspector& inspect return; AppId service_id = APP_ID_NONE; + AppId client_id = APP_ID_NONE; + AppId payload_id = APP_ID_NONE; + AppId misc_id = APP_ID_NONE; AppidChangeBits change_bits; - bool is_discovery_done = do_discovery(p, *asd, protocol, direction, service_id, change_bits); + bool is_discovery_done = do_discovery(p, *asd, protocol, direction, service_id, client_id, + payload_id, misc_id, change_bits); - do_post_discovery(p, *asd, direction, is_discovery_done, service_id, change_bits); + do_post_discovery(p, *asd, direction, is_discovery_done, service_id, client_id, payload_id, + misc_id, change_bits); } void AppIdDiscovery::publish_appid_event(AppidChangeBits& change_bits, snort::Flow* flow) @@ -594,63 +599,6 @@ static uint64_t is_session_monitored(const Packet* p, AppidSessionDirection dir, return flow_flags; } -static void lookup_appid_by_host_port(AppIdSession& asd, Packet* p, IpProtocol protocol, - AppidSessionDirection direction) -{ - HostPortVal* hv = nullptr; - uint16_t port = 0; - const SfIp* ip = nullptr; - asd.scan_flags |= SCAN_HOST_PORT_FLAG; - if (direction == APP_ID_FROM_INITIATOR) - { - ip = p->ptrs.ip_api.get_dst(); - port = p->ptrs.dp; - } - else - { - ip = p->ptrs.ip_api.get_src(); - port = p->ptrs.sp; - } - if ((hv = HostPortCache::find(ip, port, protocol))) - { - switch (hv->type) - { - case 1: - asd.client.set_id(hv->appId); - asd.client_disco_state = APPID_DISCO_STATE_FINISHED; - break; - case 2: - asd.payload.set_id(hv->appId); - break; - default: - asd.service.set_id(hv->appId); - asd.sync_with_snort_protocol_id(hv->appId, p); - asd.service_disco_state = APPID_DISCO_STATE_FINISHED; - asd.client_disco_state = APPID_DISCO_STATE_FINISHED; - asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED); -#ifdef ENABLE_APPID_THIRD_PARTY - if (asd.tpsession) - asd.tpsession->reset(); -#endif - if ( asd.payload.get_id() == APP_ID_NONE) - asd.payload.set_id(APP_ID_UNKNOWN); - } - } - else if (asd.config->mod_config->is_host_port_app_cache_runtime) - { - auto ht = host_cache.find(*ip); - if ( ht ) - { - AppId appid = ht->get_appid(port, protocol, true); - if ( appid > APP_ID_NONE ) - { - asd.client.set_id(appid); - asd.client_disco_state = APPID_DISCO_STATE_FINISHED; - } - } - } -} - bool AppIdDiscovery::handle_unmonitored_session(AppIdSession* asd, const Packet* p, IpProtocol protocol, AppidSessionDirection dir, AppIdInspector& inspector, uint64_t& flow_flags) @@ -898,19 +846,99 @@ void AppIdDiscovery::do_port_based_discovery(Packet* p, AppIdSession& asd, IpPro asd.set_session_flags(APPID_SESSION_PORT_SERVICE_DONE); } -bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol protocol, - AppidSessionDirection direction, AppId& service_id, AppidChangeBits& change_bits) +bool AppIdDiscovery::do_host_port_based_discovery(Packet* p, AppIdSession& asd, IpProtocol protocol, + AppidSessionDirection direction) { - bool is_discovery_done = false; + if (asd.get_session_flags(APPID_SESSION_HOST_CACHE_MATCHED)) + return false; + + bool check_static = false; + bool check_dynamic = false; - // {host, port} based detection if (!(asd.scan_flags & SCAN_HOST_PORT_FLAG)) + check_static = true; + + if ((asd.session_packet_count % asd.config->mod_config->host_port_app_cache_lookup_interval == 0) and + (asd.session_packet_count <= asd.config->mod_config->host_port_app_cache_lookup_range) and + asd.config->mod_config->is_host_port_app_cache_runtime ) + check_dynamic = true; + + if (!(check_static || check_dynamic)) + return false; + + uint16_t port; + const SfIp* ip; + + if (direction == APP_ID_FROM_INITIATOR) + { + ip = p->ptrs.ip_api.get_dst(); + port = p->ptrs.dp; + } + else { - if ((asd.is_tp_processing_done() && asd.get_tp_app_id() <= APP_ID_NONE) || - (asd.session_packet_count > asd.config->mod_config->host_port_app_cache_lookup_delay)) - lookup_appid_by_host_port(asd, p, protocol, direction); + ip = p->ptrs.ip_api.get_src(); + port = p->ptrs.sp; } + HostPortVal* hv = nullptr; + + if (check_static and + (hv = HostPortCache::find(ip, port, protocol))) + { + asd.scan_flags |= SCAN_HOST_PORT_FLAG; + switch (hv->type) + { + case APP_ID_TYPE_CLIENT: + asd.client.set_id(hv->appId); + asd.client_disco_state = APPID_DISCO_STATE_FINISHED; + break; + case APP_ID_TYPE_PAYLOAD: + asd.payload.set_id(hv->appId); + break; + default: + asd.service.set_id(hv->appId); + asd.sync_with_snort_protocol_id(hv->appId, p); + asd.service_disco_state = APPID_DISCO_STATE_FINISHED; + asd.client_disco_state = APPID_DISCO_STATE_FINISHED; + asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED); +#ifdef ENABLE_APPID_THIRD_PARTY + if (asd.tpsession) + asd.tpsession->reset(); +#endif + if ( asd.payload.get_id() == APP_ID_NONE) + asd.payload.set_id(APP_ID_UNKNOWN); + } + asd.set_session_flags(APPID_SESSION_HOST_CACHE_MATCHED); + return true; + } + + if (!hv and check_dynamic) + { + auto ht = host_cache.find(*ip); + if (ht) + { + AppId appid = ht->get_appid(port, protocol, true); + if (appid > APP_ID_NONE) + { + // FIXIT-L: Make this more generic to support service and payload IDs + asd.client.set_id(appid); + asd.client_disco_state = APPID_DISCO_STATE_FINISHED; + asd.set_session_flags(APPID_SESSION_HOST_CACHE_MATCHED); + } + } + + return true; + } + + return false; +} + +bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol protocol, + AppidSessionDirection direction, AppId& service_id, AppId& client_id, AppId& payload_id, + AppId& misc_id, AppidChangeBits& change_bits) +{ + bool is_discovery_done = false; + asd.check_app_detection_restart(change_bits); if (protocol != IpProtocol::TCP and protocol != IpProtocol::UDP) @@ -955,7 +983,6 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto if (asd.service.get_id() == APP_ID_DNS && asd.config->mod_config->dns_host_reporting && dsession->get_host() ) { - AppId client_id = APP_ID_NONE, payload_id = APP_ID_NONE; dns_host_scan_hostname((const uint8_t*)dsession->get_host(), dsession->get_host_len(), &client_id, &payload_id); asd.set_client_appid_data(client_id, nullptr, change_bits); @@ -1004,15 +1031,28 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto } } + payload_id = asd.pick_payload_app_id(); + client_id = asd.pick_client_app_id(); + misc_id = asd.pick_misc_app_id();; + + if ((service_id == APP_ID_UNKNOWN_UI or service_id <= APP_ID_NONE ) and + (client_id <= APP_ID_NONE and payload_id <= APP_ID_NONE and misc_id <= APP_ID_NONE)) + { + if (do_host_port_based_discovery(p, asd, protocol, direction)) + { + service_id = asd.pick_service_app_id(); + client_id = asd.pick_client_app_id(); + payload_id = asd.pick_payload_app_id(); + } + } + return is_discovery_done; } void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd, AppidSessionDirection direction, bool is_discovery_done, AppId service_id, - AppidChangeBits& change_bits) + AppId client_id, AppId payload_id, AppId misc_id, AppidChangeBits& change_bits) { - AppId payload_id = asd.pick_payload_app_id(); - if (service_id > APP_ID_NONE) { if (asd.get_session_flags(APPID_SESSION_DECRYPTED)) @@ -1067,8 +1107,7 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd, if ( asd.past_forecast != service_id and asd.past_forecast != APP_ID_UNKNOWN and asd.payload.get_id() == APP_ID_NONE ) { - asd.past_forecast = check_session_for_AF_forecast(asd, p, direction, - (AppId)service_id); + asd.past_forecast = check_session_for_AF_forecast(asd, p, direction, service_id); if (asd.past_forecast != APP_ID_UNKNOWN) payload_id = asd.pick_payload_app_id(); } @@ -1089,7 +1128,6 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd, } #endif - asd.set_application_ids(service_id, asd.pick_client_app_id(), payload_id, - asd.pick_misc_app_id(), change_bits); + asd.set_application_ids(service_id, client_id, payload_id, misc_id, change_bits); publish_appid_event(change_bits, p->flow); } diff --git a/src/network_inspectors/appid/appid_discovery.h b/src/network_inspectors/appid/appid_discovery.h index 4a0516aaa..3e7d8f597 100644 --- a/src/network_inspectors/appid/appid_discovery.h +++ b/src/network_inspectors/appid/appid_discovery.h @@ -135,12 +135,15 @@ private: static bool do_pre_discovery(snort::Packet* p, AppIdSession** p_asd, AppIdInspector& inspector, IpProtocol& protocol, AppidSessionDirection& direction); static bool do_discovery(snort::Packet* p, AppIdSession& asd, IpProtocol protocol, - AppidSessionDirection direction, AppId& service_id, AppidChangeBits& change_bits); + AppidSessionDirection direction, AppId& service_id, AppId& client_id, AppId& payload_id, + AppId& misc_id, AppidChangeBits& change_bits); static void do_post_discovery(snort::Packet* p, AppIdSession& asd, - AppidSessionDirection direction, bool is_discovery_done, AppId service_id, - AppidChangeBits& change_bits); + AppidSessionDirection direction, bool is_discovery_done, AppId service_id, AppId client_id, + AppId payload_id, AppId misc_id, AppidChangeBits& change_bits); static void do_port_based_discovery(snort::Packet* p, AppIdSession& asd, IpProtocol protocol, AppidSessionDirection direction); + static bool do_host_port_based_discovery(snort::Packet* p, AppIdSession& asd, + IpProtocol protocol, AppidSessionDirection direction); static bool handle_unmonitored_session(AppIdSession* asd, const snort::Packet* p, IpProtocol protocol, AppidSessionDirection dir, AppIdInspector& inspector, uint64_t& flow_flags); diff --git a/src/network_inspectors/appid/appid_session_api.h b/src/network_inspectors/appid/appid_session_api.h index 525a65f18..4b374502b 100644 --- a/src/network_inspectors/appid/appid_session_api.h +++ b/src/network_inspectors/appid/appid_session_api.h @@ -87,6 +87,7 @@ namespace snort #define APPID_SESSION_IGNORE_FLOW_IDED (1ULL << 40) #define APPID_SESSION_OOO_CHECK_TP (1ULL << 41) #define APPID_SESSION_PAYLOAD_SEEN (1ULL << 42) +#define APPID_SESSION_HOST_CACHE_MATCHED (1ULL << 43) #define APPID_SESSION_IGNORE_ID_FLAGS \ (APPID_SESSION_IGNORE_FLOW | \ APPID_SESSION_NOT_A_SERVICE | \ diff --git a/src/network_inspectors/appid/application_ids.h b/src/network_inspectors/appid/application_ids.h index 9acdaf569..5945287a5 100644 --- a/src/network_inspectors/appid/application_ids.h +++ b/src/network_inspectors/appid/application_ids.h @@ -1021,6 +1021,14 @@ enum ApplicationIds : AppId APP_ID_UNKNOWN_UI = 65535 // UI renders this value as 'Unknown' }; +enum AppIdType +{ + APP_ID_TYPE_SERVICE, + APP_ID_TYPE_CLIENT, + APP_ID_TYPE_PAYLOAD, + APP_ID_TYPE_MAX +}; + struct AppRegistryEntry { AppId appId;