From 514211db9334902b91a9f7d0415bf23dd0f7ab07 Mon Sep 17 00:00:00 2001 From: "Shravan Rangarajuvenkata (shrarang)" Date: Thu, 23 May 2019 10:59:25 -0400 Subject: [PATCH] Merge pull request #1608 in SNORT/snort3 from ~KAMURTHI/snort3:BitTorrent-Fix to master Squashed commit of the following: commit 4adad8bc5649000bb5d8ca10f933389d441ad20a Author: Kanimozhi Murthi Date: Thu May 9 16:16:46 2019 -0400 appid: support for dynamic host cache lookup-based app detection. --- src/host_tracker/host_cache.cc | 36 ++++++++++++ src/host_tracker/host_cache.h | 2 + src/host_tracker/host_tracker.h | 55 ++++++++++++++++++- src/host_tracker/test/host_cache_test.cc | 50 +++++++++++++++++ src/host_tracker/test/host_tracker_test.cc | 30 ++++++++++ .../appid/app_info_table.cc | 24 ++++++++ src/network_inspectors/appid/appid_config.h | 2 + .../appid/appid_discovery.cc | 18 +++++- src/network_inspectors/appid/appid_session.h | 6 +- .../appid/lua_detector_api.cc | 40 +++++++++++++- .../appid/test/appid_discovery_test.cc | 4 +- 11 files changed, 258 insertions(+), 9 deletions(-) diff --git a/src/host_tracker/host_cache.cc b/src/host_tracker/host_cache.cc index 6ee0e3766..17bc8d96d 100644 --- a/src/host_tracker/host_cache.cc +++ b/src/host_tracker/host_cache.cc @@ -64,5 +64,41 @@ bool host_cache_add_service(const SfIp& ipaddr, Protocol ipproto, Port port, con return ht->add_service(app_entry); } + +bool host_cache_add_app_mapping(const SfIp& ipaddr, Port port, Protocol proto, AppId appId) +{ + HostIpKey ipkey((const uint8_t*) ipaddr.get_ip6_ptr()); + std::shared_ptr ht; + + if (!host_cache.find(ipkey, ht)) + { + ht = std::make_shared (ipaddr); + + if (ht == nullptr) + { + return false; + } + ht->add_app_mapping(port, proto, appId); + host_cache.insert(ipkey, ht); + } + else + { + return ht->find_else_add_app_mapping(port, proto, appId); + } + + return true; } +AppId host_cache_find_app_mapping(const SfIp* ipaddr, Port port, Protocol proto) +{ + HostIpKey ipkey((const uint8_t*) ipaddr->get_ip6_ptr()); + std::shared_ptr ht; + + if (host_cache.find(ipkey, ht)) + { + return ht->find_app_mapping(port, proto); + } + + return APP_ID_NONE; +} +} diff --git a/src/host_tracker/host_cache.h b/src/host_tracker/host_cache.h index 7e262303c..7a89dad1d 100644 --- a/src/host_tracker/host_cache.h +++ b/src/host_tracker/host_cache.h @@ -69,6 +69,8 @@ namespace snort { // Insert a new service into host cache if it doesn't already exist. SO_PUBLIC bool host_cache_add_service(const SfIp&, Protocol, Port, const char* service); +bool host_cache_add_app_mapping(const SfIp&, Port, Protocol, AppId); +AppId host_cache_find_app_mapping(const SfIp* , Port, Protocol ); } #endif diff --git a/src/host_tracker/host_tracker.h b/src/host_tracker/host_tracker.h index a4217b3f5..ca665af67 100644 --- a/src/host_tracker/host_tracker.h +++ b/src/host_tracker/host_tracker.h @@ -29,16 +29,17 @@ #include #include #include - #include "framework/counts.h" +#include "main/snort_types.h" #include "main/thread.h" +#include "network_inspectors/appid/application_ids.h" +#include "protocols/protocol_ids.h" #include "sfip/sf_ip.h" #include "target_based/snort_protocols.h" // FIXIT-M For now this emulates the Snort++ attribute table. // Need to add in host_tracker.h data eventually. -typedef uint16_t Port; typedef uint16_t Protocol; typedef uint8_t Policy; @@ -72,6 +73,13 @@ struct HostApplicationEntry } }; +struct AppMapping +{ + Port port; + Protocol proto; + AppId appid; +}; + class HostTracker { private: @@ -80,6 +88,7 @@ private: // FIXIT-M do we need to use a host_id instead of SfIp as in sfrna? snort::SfIp ip_addr; + std::vector< AppMapping > app_mappings; // Policies to apply to this host. Policy stream_policy = 0; @@ -94,6 +103,11 @@ public: memset(&ip_addr, 0, sizeof(ip_addr)); } + HostTracker(const snort::SfIp& new_ip_addr) + { + std::memcpy(&ip_addr, &new_ip_addr, sizeof(ip_addr)); + } + snort::SfIp get_ip_addr() { std::lock_guard lck(host_tracker_lock); @@ -130,6 +144,43 @@ public: frag_policy = policy; } + void add_app_mapping(Port port, Protocol proto, AppId appid) + { + std::lock_guard lck(host_tracker_lock); + AppMapping app_map = {port, proto, appid}; + + app_mappings.push_back(app_map); + } + + AppId find_app_mapping(Port port, Protocol proto) + { + std::lock_guard lck(host_tracker_lock); + for (std::vector::iterator it=app_mappings.begin(); it!=app_mappings.end(); ++it) + { + if (it->port == port and it->proto ==proto) + { + return it->appid; + } + } + return APP_ID_NONE; + } + + bool find_else_add_app_mapping(Port port, Protocol proto, AppId appid) + { + std::lock_guard lck(host_tracker_lock); + for (std::vector::iterator it=app_mappings.begin(); it!=app_mappings.end(); ++it) + { + if (it->port == port and it->proto ==proto) + { + return false; + } + } + AppMapping app_map = {port, proto, appid}; + + app_mappings.push_back(app_map); + return true; + } + // Add host service data only if it doesn't already exist. Returns // false if entry exists already, and true if entry was added. bool add_service(const HostApplicationEntry& app_entry) diff --git a/src/host_tracker/test/host_cache_test.cc b/src/host_tracker/test/host_cache_test.cc index b41809255..db835d894 100644 --- a/src/host_tracker/test/host_cache_test.cc +++ b/src/host_tracker/test/host_cache_test.cc @@ -181,6 +181,56 @@ TEST(host_cache, host_cache_add_service_test) host_cache.clear(); // Free HostTracker objects } +TEST(host_cache, host_cache_app_mapping_test ) +{ + HostTracker* expected_ht = new HostTracker; + std::shared_ptr actual_ht; + uint8_t hk[16] = + { 0xde,0xad,0xbe,0xef,0xab,0xcd,0xef,0x01,0x23,0x34,0x56,0x78,0x90,0xab,0xcd,0xef }; + SfIp ip_addr1; + SfIp ip_addr2; + HostIpKey hkey(hk); + Port port1 = 4123; + Port port2 = 1827; + Protocol proto1 = 6; + Protocol proto2 = 7; + AppId appid1 = 61; + AppId appid2 = 62; + AppId appid3 = 63; + AppId ret; + bool add_ret; + + ip_addr1.pton(AF_INET6, "beef:dead:beef:abcd:ef01:2334:5678:90ab"); + ip_addr2.pton(AF_INET6, "beef:dead:beef:abcd:ef01:2334:5678:901b"); + + // Initialize cache with a HostTracker. + host_cache_add_host_tracker(expected_ht); + + add_ret = host_cache_add_app_mapping(ip_addr1, port1, proto1, appid1); + CHECK(true == add_ret); + + ret = host_cache_find_app_mapping(&ip_addr1, port1, proto1); + CHECK(61 == ret); + + ret = host_cache_find_app_mapping(&ip_addr1, port2, proto1); + CHECK(APP_ID_NONE == ret); + + ret = host_cache_find_app_mapping(&ip_addr2, port1, proto1); + CHECK(APP_ID_NONE == ret); + + add_ret = host_cache_add_app_mapping(ip_addr1, port2, proto1, appid2); + CHECK(true == add_ret); + ret = host_cache_find_app_mapping(&ip_addr1, port2, proto1); + CHECK(62 == ret); + + add_ret = host_cache_add_app_mapping(ip_addr1, port1, proto2, appid3); + CHECK(true == add_ret); + ret = host_cache_find_app_mapping(&ip_addr1, port1, proto2); + CHECK(63 == ret); + + host_cache.clear(); // Free HostTracker objects +} + int main(int argc, char** argv) { SfIp ip_addr1; diff --git a/src/host_tracker/test/host_tracker_test.cc b/src/host_tracker/test/host_tracker_test.cc index f47c69326..f7c7af435 100644 --- a/src/host_tracker/test/host_tracker_test.cc +++ b/src/host_tracker/test/host_tracker_test.cc @@ -83,6 +83,36 @@ TEST(host_tracker, policy_test) CHECK(expected_policy == actual_policy); } +TEST(host_tracker, app_mapping_test) +{ + HostTracker ht; + const uint16_t expected_ports = 4123; + const uint16_t port1 = 4122; + AppId actual_appid; + Protocol expected_proto1 = 6; + Protocol expected_proto2 = 17; + AppId appid1 = 61; + AppId appid2 = 62; + bool ret; + + ht.add_app_mapping(expected_ports, expected_proto1, appid1); + + actual_appid = ht.find_app_mapping(expected_ports, expected_proto1); + CHECK(61 == actual_appid); + + actual_appid = ht.find_app_mapping(expected_ports, expected_proto2); + CHECK(APP_ID_NONE == actual_appid); + + actual_appid = ht.find_app_mapping(port1, expected_proto2); + CHECK(APP_ID_NONE == actual_appid); + + ret = ht.find_else_add_app_mapping(port1, expected_proto1, appid2); + CHECK(true == ret); + + ret = ht.find_else_add_app_mapping(port1, expected_proto1, appid2); + CHECK(false == ret); +} + // Test HostTracker add and find service functions. TEST(host_tracker, add_find_service_test) { diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc index b12099d0c..3708ef24c 100644 --- a/src/network_inspectors/appid/app_info_table.cc +++ b/src/network_inspectors/appid/app_info_table.cc @@ -51,6 +51,8 @@ static AppInfoTable custom_app_info_table; static const char* CONF_SEPARATORS = "\t\n\r"; static const int MIN_MAX_TP_FLOW_DEPTH = 1; static const int MAX_MAX_TP_FLOW_DEPTH = 1000000; +static const int MIN_HOST_PORT_APP_CACHE_LOOKUP_DELAY = 0; +static const int MAX_HOST_PORT_APP_CACHE_LOOKUP_DELAY = 1000000; static const char* APP_CONFIG_FILE = "appid.conf"; static const char* USR_CONFIG_FILE = "userappid.conf"; const char* APP_MAPPING_FILE = "appMapping.data"; @@ -325,6 +327,28 @@ void AppInfoManager::load_appid_config(AppIdModuleConfig* config, const char* pa config->max_tp_flow_depth = max_tp_flow_depth; } } + else if (!(strcasecmp(conf_key, "host_port_app_cache_lookup_delay"))) + { + int host_port_app_cache_lookup_delay = atoi(conf_val); + if (host_port_app_cache_lookup_delay < MIN_HOST_PORT_APP_CACHE_LOOKUP_DELAY + || host_port_app_cache_lookup_delay > MAX_HOST_PORT_APP_CACHE_LOOKUP_DELAY) + { + ParseWarning(WARN_CONF, + "AppId: invalid host_port_app_cache_lookup_delay %d, must be between %d and %d\n.", + host_port_app_cache_lookup_delay, MIN_HOST_PORT_APP_CACHE_LOOKUP_DELAY, MAX_HOST_PORT_APP_CACHE_LOOKUP_DELAY); + } + else + { + config->host_port_app_cache_lookup_delay = host_port_app_cache_lookup_delay; + } + } + else if (!(strcasecmp(conf_key, "is_host_port_app_cache_runtime"))) + { + if (!(strcasecmp(conf_val, "enabled"))) + { + config->is_host_port_app_cache_runtime = true; + } + } else if (!(strcasecmp(conf_key, "tp_allow_probes"))) { if (!(strcasecmp(conf_val, "enabled"))) diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h index ca059ce36..2b53688da 100644 --- a/src/network_inspectors/appid/appid_config.h +++ b/src/network_inspectors/appid/appid_config.h @@ -92,12 +92,14 @@ public: bool mdns_user_reporting = true; bool chp_userid_disabled = false; bool http2_detection_enabled = false; + bool is_host_port_app_cache_runtime = false; uint32_t ftp_userid_disabled = 0; uint32_t chp_body_collection_disabled = 0; uint32_t chp_body_collection_max = 0; uint32_t rtmp_max_packets = 15; uint32_t max_tp_flow_depth = 5; uint32_t tp_allow_probes = 0; + uint32_t host_port_app_cache_lookup_delay = 10; uint32_t http_response_version_enabled = 0; }; diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index de56aa8d5..90d971910 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -24,6 +24,7 @@ #endif #include "appid_discovery.h" +#include "host_tracker/host_cache.h" #include "log/messages.h" #include "profiler/profiler.h" @@ -635,6 +636,15 @@ static void lookup_appid_by_host_port(AppIdSession& asd, Packet* p, IpProtocol p asd.payload.set_id(APP_ID_UNKNOWN); } } + else if (asd.config->mod_config->is_host_port_app_cache_runtime) + { + AppId appid = snort::host_cache_find_app_mapping(ip, port, (Protocol)protocol); + if (appid > APP_ID_NONE) + { + asd.client.set_id(appid); + asd.client_disco_state = APPID_DISCO_STATE_FINISHED; + } + } } bool AppIdDiscovery::handle_unmonitored_session(AppIdSession* asd, const Packet* p, @@ -851,8 +861,12 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto bool is_discovery_done = false; // {host, port} based detection - if ( !(asd.scan_flags & SCAN_HOST_PORT_FLAG) ) - lookup_appid_by_host_port(asd, p, protocol, direction); + if (!(asd.scan_flags & SCAN_HOST_PORT_FLAG)) + { + if ((asd.is_tp_processing_done() && asd.get_tp_app_id() <= APP_ID_NONE) || + (asd.session_packet_count > asd.config->mod_config->host_port_app_cache_lookup_delay)) + lookup_appid_by_host_port(asd, p, protocol, direction); + } asd.check_app_detection_restart(change_bits); diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h index 02ec271c2..8a1493a04 100644 --- a/src/network_inspectors/appid/appid_session.h +++ b/src/network_inspectors/appid/appid_session.h @@ -278,10 +278,10 @@ public: void clear_session_flags(uint64_t flags) { common.flags &= ~flags; } uint64_t get_session_flags(uint64_t flags) const { return (common.flags & flags); } void set_service_detected() { common.flags |= APPID_SESSION_SERVICE_DETECTED; } - bool is_service_detected() { return common.flags & APPID_SESSION_SERVICE_DETECTED; } + bool is_service_detected() { return ((common.flags & APPID_SESSION_SERVICE_DETECTED) == 0) ? false : true; } void set_client_detected() { common.flags |= APPID_SESSION_CLIENT_DETECTED; } - bool is_client_detected() { return common.flags & APPID_SESSION_CLIENT_DETECTED; } - bool is_decrypted() { return common.flags & APPID_SESSION_DECRYPTED; } + bool is_client_detected() { return ((common.flags & APPID_SESSION_CLIENT_DETECTED) == 0) ? false : true; } + bool is_decrypted() { return ((common.flags & APPID_SESSION_DECRYPTED) == 0) ? false : true; } void* get_flow_data(unsigned id); int add_flow_data(void* data, unsigned id, AppIdFreeFCN); diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc index 4f63cc9a9..9a0c134ab 100644 --- a/src/network_inspectors/appid/lua_detector_api.cc +++ b/src/network_inspectors/appid/lua_detector_api.cc @@ -24,7 +24,6 @@ #endif #include "lua_detector_api.h" - #include #include #include @@ -49,6 +48,7 @@ #include "lua_detector_util.h" #include "service_plugins/service_discovery.h" #include "service_plugins/service_ssl.h" +#include "host_tracker/host_cache.h" using namespace snort; @@ -1170,6 +1170,43 @@ static int detector_add_host_port_application(lua_State* L) return 0; } +static int detector_add_host_port_dynamic(lua_State* L) +{ + auto& ud = *UserData::check(L, DETECTOR, 1); + // Verify detector user data and that we are in packet context + ud->validate_lua_state(true); + + SfIp ip_addr; + int index = 1; + + uint8_t type = lua_tointeger(L, ++index); + if (type != 1) + { + return 0; + } + AppId appid = (AppId)lua_tointeger(L, ++index); + size_t ipaddr_size = 0; + const char* ip_str = lua_tolstring(L, ++index, &ipaddr_size); + if (!ip_str || !ipaddr_size || !convert_string_to_address(ip_str, &ip_addr)) + { + ErrorMessage("%s: Invalid IP address: %s\n",__func__, ip_str); + return 0; + } + + unsigned port = lua_tointeger(L, ++index); + unsigned proto = lua_tointeger(L, ++index); + if (proto > (unsigned)IpProtocol::RESERVED) + { + ErrorMessage("%s:Invalid protocol value %u\n",__func__, proto); + return 0; + } + + if (!(snort::host_cache_add_app_mapping(ip_addr, port, proto, appid))) + ErrorMessage("%s:Failed to add app mapping\n",__func__); + + return 0; +} + static int detector_add_content_type_pattern(lua_State* L) { auto& ud = *UserData::check(L, DETECTOR, 1); @@ -2351,6 +2388,7 @@ static const luaL_Reg detector_methods[] = { "addSipServer", detector_add_sip_server }, { "addSSLCnamePattern", detector_add_ssl_cname_pattern }, { "addHostPortApp", detector_add_host_port_application }, + { "addHostPortAppDynamic", detector_add_host_port_dynamic }, { "addDNSHostPattern", detector_add_dns_host_pattern }, /*Obsolete - new detectors should not use this API */ diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc index 073c4547c..60a1aa039 100644 --- a/src/network_inspectors/appid/test/appid_discovery_test.cc +++ b/src/network_inspectors/appid/test/appid_discovery_test.cc @@ -23,6 +23,7 @@ #define APPID_MOCK_INSPECTOR_H // avoiding mocked inspector +#include "host_tracker/host_cache.h" #include "network_inspectors/appid/appid_discovery.cc" #include "search_engines/search_tool.h" @@ -176,6 +177,7 @@ void AppIdSession::set_client_appid_data(AppId, char*, AppidChangeBits&) {} void AppIdSession::examine_rtmp_metadata(AppidChangeBits&) {} void AppIdSession::examine_ssl_metadata(Packet*, AppidChangeBits&) {} void AppIdSession::update_encrypted_app_id(AppId) {} +bool AppIdSession::is_tp_processing_done() const {return 0;} AppIdSession* AppIdSession::allocate_session(const Packet*, IpProtocol, AppidSessionDirection, AppIdInspector*) { @@ -211,7 +213,7 @@ ServiceDiscovery& ServiceDiscovery::get_instance() s_discovery_manager = new ServiceDiscovery(); return *s_discovery_manager; } - +AppId snort::host_cache_find_app_mapping(snort::SfIp const*, Port port , Protocol proto){ return 0; } // Stubs for ClientDiscovery ClientDiscovery::ClientDiscovery(){} ClientDiscovery::~ClientDiscovery() {} -- 2.47.3