From: Sreeja Athirkandathil Narayanan (sathirka) Date: Wed, 21 Sep 2022 03:20:20 +0000 (+0000) Subject: Pull request #3510: appid : A custom lua detector api to map ip and port to appids... X-Git-Tag: 3.1.42.0~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8a3958ff65fa70f9f10da506f74212d1e0826f27;p=thirdparty%2Fsnort3.git Pull request #3510: appid : A custom lua detector api to map ip and port to appids on the first packet. Merge in SNORT/snort3 from ~UMASHARM/snort3:POC_FirstPkt to master Squashed commit of the following: commit 7bc2782effcc61941091f0bce53640cc3c85c293 Author: Umang Sharma Date: Tue Jul 12 06:31:29 2022 -0400 appid: A custom lua detector api to map ip and port to appids on the first packet --- diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h index 4aaffcffd..b61f596ce 100644 --- a/src/network_inspectors/appid/appid_config.h +++ b/src/network_inspectors/appid/appid_config.h @@ -114,6 +114,7 @@ public: bool check_host_cache_unknown_ssl = false; bool ftp_userid_disabled = false; bool chp_body_collection_disabled = false; + bool need_reinspection = false; uint32_t chp_body_collection_max = 0; uint32_t rtmp_max_packets = 15; uint32_t max_tp_flow_depth = 5; @@ -161,6 +162,17 @@ public: return host_port_cache.add(sc, ip, port, proto, type, appid); } + bool host_first_pkt_add(const snort::SnortConfig* sc, const snort::SfIp* ip, uint16_t port, + IpProtocol proto, AppId protocol_appid, AppId client_appid, AppId web_appid, unsigned reinspect) + { + return first_pkt_cache.add_host(sc, ip, port, proto, protocol_appid, client_appid, web_appid, reinspect); + } + + HostAppIdsVal* host_first_pkt_find(const snort::SfIp* ip, uint16_t port, IpProtocol proto) + { + return first_pkt_cache.find_on_first_pkt(ip, port, proto, *this); + } + AppId length_cache_find(const LengthKey& key) { return length_cache.find(key); @@ -225,6 +237,7 @@ private: AppInfoManager app_info_mgr; ClientDiscovery client_disco_mgr; HostPortCache host_port_cache; + HostPortCache first_pkt_cache; LengthCache length_cache; DnsPatternMatchers dns_matchers; HttpPatternMatchers http_matchers; diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index 9a982fb75..0fe89b2bb 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -545,6 +545,108 @@ static inline bool is_check_host_cache_valid(AppIdSession& asd, AppId service_id return false; } +bool AppIdDiscovery::detect_on_first_pkt(Packet* p, AppIdSession& asd, + IpProtocol protocol, AppidSessionDirection direction, AppId& service_id, + AppId& client_id, AppId& payload_id) +{ + uint16_t port; + const SfIp* ip; + + if (direction == APP_ID_FROM_INITIATOR) + { + ip = p->ptrs.ip_api.get_dst(); + port = p->ptrs.dp; + } + else + { + ip = p->ptrs.ip_api.get_src(); + port = p->ptrs.sp; + } + + HostAppIdsVal* hv = nullptr; + hv = asd.get_odp_ctxt().host_first_pkt_find(ip, port, protocol); + if (hv) + { + uint32_t appids_found = 0; + const char *service_app_name = nullptr, *client_app_name = nullptr, *payload_app_name = nullptr; + FirstPktAppIdDiscovered appid_prefix = NO_APPID_FOUND; + + if (hv->client_appId) + { + client_id = hv->client_appId; + asd.set_client_id(client_id); + client_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(client_id); + appids_found++; + appid_prefix = CLIENT_APPID_FOUND; + } + if (hv->protocol_appId) + { + service_id = hv->protocol_appId; + asd.set_service_id(service_id, asd.get_odp_ctxt()); + asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED); + service_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id); + appids_found++; + appid_prefix = SERVICE_APPID_FOUND; + } + if (hv->web_appId) + { + payload_id = hv->web_appId; + asd.set_payload_id(payload_id); + payload_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(payload_id); + appids_found++; + if (appid_prefix == CLIENT_APPID_FOUND) + { + appid_prefix = CLIENT_PAYLOAD_APPID_FOUND; + } + else + { + appid_prefix = PAYLOAD_APPID_FOUND; + } + } + asd.get_odp_ctxt().need_reinspection = hv->reinspect; + + switch (appids_found) + { + case FIRST_PKT_CACHE_ONE_APPID_FOUND : + if (appid_prefix == PAYLOAD_APPID_FOUND) + { + service_id = payload_id; + asd.set_service_id(service_id, asd.get_odp_ctxt()); + asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED); + service_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id); + } + else if (appid_prefix == CLIENT_APPID_FOUND) + { + service_id = client_id; + asd.set_service_id(service_id, asd.get_odp_ctxt()); + asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED); + service_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id); + } + break; + case FIRST_PKT_CACHE_TWO_APPIDS_FOUND : + if (appid_prefix == CLIENT_PAYLOAD_APPID_FOUND) + { + service_id = client_id; + asd.set_service_id(service_id, asd.get_odp_ctxt()); + asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED); + service_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id); + } + break; + } + asd.set_session_flags(APPID_SESSION_FIRST_PKT_CACHE_MATCHED); + if (appidDebug->is_active()) + { + LogMessage("AppIdDbg %s Host cache match found on first packet, service: %s(%d), " + "client: %s(%d), payload: %s(%d), reinspect: %s \n", appidDebug->get_debug_session(), + (service_app_name ? service_app_name : ""), service_id, + (client_app_name ? client_app_name : ""), client_id, + (payload_app_name ? payload_app_name : ""), payload_id, (hv->reinspect ? "True" : "False")); + } + return true; + } + return false; +} + bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol protocol, IpProtocol outer_protocol, AppidSessionDirection direction, AppId& service_id, AppId& client_id, AppId& payload_id, AppId& misc_id, AppidChangeBits& change_bits, @@ -552,6 +654,22 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto { bool is_discovery_done = false; + if (asd.session_packet_count == 1) + { + detect_on_first_pkt(p, asd, protocol, direction, service_id, client_id, payload_id); + } + + if (asd.get_session_flags(APPID_SESSION_FIRST_PKT_CACHE_MATCHED) and !asd.get_odp_ctxt().need_reinspection) + { + is_discovery_done = true; + service_id = asd.pick_service_app_id(); + client_id = asd.pick_ss_client_app_id(); + payload_id = asd.pick_ss_payload_app_id(service_id); + asd.client_disco_state = APPID_DISCO_STATE_FINISHED; + asd.service_disco_state = APPID_DISCO_STATE_FINISHED; + return is_discovery_done; + } + asd.check_app_detection_restart(change_bits, tp_appid_ctxt); if (outer_protocol != IpProtocol::PROTO_NOT_SET) diff --git a/src/network_inspectors/appid/appid_discovery.h b/src/network_inspectors/appid/appid_discovery.h index c4fe30cfe..7bec32905 100644 --- a/src/network_inspectors/appid/appid_discovery.h +++ b/src/network_inspectors/appid/appid_discovery.h @@ -60,6 +60,18 @@ struct Packet; #define SCAN_HTTP_URI_FLAG (1<<9) #define SCAN_CERTVIZ_ENABLED_FLAG (1<<10) #define SCAN_SPOOFED_SNI_FLAG (1<<11) +#define FIRST_PKT_CACHE_ONE_APPID_FOUND 1 +#define FIRST_PKT_CACHE_TWO_APPIDS_FOUND 2 +#define FIRST_PKT_CACHE_ALL_APPIDS_FOUND 3 + +enum FirstPktAppIdDiscovered +{ + NO_APPID_FOUND = 0, + CLIENT_APPID_FOUND, + SERVICE_APPID_FOUND, + PAYLOAD_APPID_FOUND, + CLIENT_PAYLOAD_APPID_FOUND +}; class AppIdPatternMatchNode { @@ -150,6 +162,8 @@ private: AppidSessionDirection direction); static bool do_host_port_based_discovery(snort::Packet* p, AppIdSession& asd, IpProtocol protocol, AppidSessionDirection direction, ThirdPartyAppIdContext* tp_appid_ctxt); + static bool detect_on_first_pkt(snort::Packet* p, AppIdSession& asd, IpProtocol protocol, + AppidSessionDirection direction, AppId& service_id, AppId& client_id, AppId& payload_id); }; #endif diff --git a/src/network_inspectors/appid/appid_http_event_handler.cc b/src/network_inspectors/appid/appid_http_event_handler.cc index 8e95f51a3..242288dc6 100644 --- a/src/network_inspectors/appid/appid_http_event_handler.cc +++ b/src/network_inspectors/appid/appid_http_event_handler.cc @@ -70,6 +70,9 @@ void HttpEventHandler::handle(DataEvent& event, Flow* flow) if (!asd->get_session_flags(APPID_SESSION_DISCOVER_APP | APPID_SESSION_SPECIAL_MONITORED)) return; + if (asd->get_session_flags(APPID_SESSION_FIRST_PKT_CACHE_MATCHED) and !asd->get_odp_ctxt().need_reinspection) + return; + const uint8_t* header_start; int32_t header_length; HttpEvent* http_event = (HttpEvent*)&event; diff --git a/src/network_inspectors/appid/appid_session_api.h b/src/network_inspectors/appid/appid_session_api.h index 27dea5dcf..6f0c62aec 100644 --- a/src/network_inspectors/appid/appid_session_api.h +++ b/src/network_inspectors/appid/appid_session_api.h @@ -91,6 +91,7 @@ namespace snort #define APPID_SESSION_DECRYPT_MONITOR (1ULL << 42) #define APPID_SESSION_HTTP_TUNNEL (1ULL << 43) #define APPID_SESSION_OPPORTUNISTIC_TLS (1ULL << 44) +#define APPID_SESSION_FIRST_PKT_CACHE_MATCHED (1ULL << 45) #define APPID_SESSION_IGNORE_ID_FLAGS \ (APPID_SESSION_FUTURE_FLOW | \ APPID_SESSION_NOT_A_SERVICE | \ diff --git a/src/network_inspectors/appid/host_port_app_cache.cc b/src/network_inspectors/appid/host_port_app_cache.cc index 6c69946b0..94b1a43bd 100644 --- a/src/network_inspectors/appid/host_port_app_cache.cc +++ b/src/network_inspectors/appid/host_port_app_cache.cc @@ -73,6 +73,48 @@ bool HostPortCache::add(const SnortConfig* sc, const SfIp* ip, uint16_t port, Ip return true; } + +HostAppIdsVal* HostPortCache::find_on_first_pkt(const SfIp* ip, uint16_t port, IpProtocol protocol, + const OdpContext& odp_ctxt) +{ + HostPortKey hk; + + hk.ip = *ip; + hk.port = (odp_ctxt.allow_port_wildcard_host_cache)? 0 : port; + hk.proto = protocol; + + std::map::iterator it; + it = cache_first.find(hk); + if (it != cache_first.end()) + return &it->second; + else + return nullptr; +} + +bool HostPortCache::add_host(const SnortConfig* sc, const SfIp* ip, uint16_t port, IpProtocol proto, + AppId protocol_appId, AppId client_appId, AppId web_appId, unsigned reinspect) +{ + HostPortKey hk; + HostAppIdsVal hv; + + hk.ip = *ip; + AppIdInspector* inspector = + (AppIdInspector*)InspectorManager::get_inspector(MOD_NAME, false, sc); + assert(inspector); + const AppIdContext& ctxt = inspector->get_ctxt(); + hk.port = (ctxt.get_odp_ctxt().allow_port_wildcard_host_cache)? 0 : port; + hk.proto = proto; + + hv.protocol_appId = protocol_appId; + hv.client_appId = client_appId; + hv.web_appId = web_appId; + hv.reinspect = reinspect; + + cache_first[ hk ] = hv; + + return true; +} + void HostPortCache::dump() { for ( auto& kv : cache ) diff --git a/src/network_inspectors/appid/host_port_app_cache.h b/src/network_inspectors/appid/host_port_app_cache.h index 4a469372d..608e97dcd 100644 --- a/src/network_inspectors/appid/host_port_app_cache.h +++ b/src/network_inspectors/appid/host_port_app_cache.h @@ -65,21 +65,35 @@ struct HostPortVal unsigned type; }; +struct HostAppIdsVal +{ + AppId protocol_appId; + AppId client_appId; + AppId web_appId; + unsigned reinspect; +}; + class HostPortCache { public: HostPortVal* find(const snort::SfIp*, uint16_t port, IpProtocol, const OdpContext&); bool add(const snort::SnortConfig*, const snort::SfIp*, uint16_t port, IpProtocol, unsigned type, AppId); + + HostAppIdsVal* find_on_first_pkt(const snort::SfIp*, uint16_t port, IpProtocol, const OdpContext&); + bool add_host(const snort::SnortConfig*, const snort::SfIp*, uint16_t port, IpProtocol, + AppId, AppId, AppId, unsigned reinspect); void dump(); ~HostPortCache() { cache.clear(); + cache_first.clear(); } private: std::map cache; + std::map cache_first; }; #endif diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc index 6c28d3dcb..883fd53dc 100644 --- a/src/network_inspectors/appid/lua_detector_api.cc +++ b/src/network_inspectors/appid/lua_detector_api.cc @@ -1211,6 +1211,48 @@ static int detector_add_ssl_cname_pattern(lua_State* L) return 0; } +static int detector_add_host_first_pkt_application(lua_State* L) +{ + auto& ud = *UserData::check(L, DETECTOR, 1); + // Verify detector user data and that we are NOT in packet context + ud->validate_lua_state(false); + if (!init(L)) + return 0; + + SfIp ip_address; + int index = 1; + + /* Extract the three appIds and the reinspect flag */ + uint32_t protocol_appid = lua_tointeger(L, ++index); + uint32_t client_appid = lua_tointeger(L, ++index); + uint32_t web_appid = lua_tointeger(L, ++index); + unsigned reinspect = lua_tointeger(L, ++index); + + /* Extract IP, Port, protocol */ + size_t ipaddr_size = 0; + const char* ip_str = lua_tolstring(L, ++index, &ipaddr_size); + if (!ip_str or !ipaddr_size or !convert_string_to_address(ip_str, &ip_address)) + { + ErrorMessage("%s: Invalid IP address: %s\n",__func__, ip_str); + return 0; + } + + unsigned port = lua_tointeger(L, ++index); + IpProtocol proto; + if (toipprotocol(L, ++index, proto)) + return 0; + + lua_getglobal(L, LUA_STATE_GLOBAL_SC_ID); + const SnortConfig* sc = *static_cast(lua_touserdata(L, -1)); + lua_pop(L, 1); + + if (!ud->get_odp_ctxt().host_first_pkt_add( + sc, &ip_address, (uint16_t)port, proto, protocol_appid, client_appid, web_appid, reinspect)) + ErrorMessage("%s:Failed to backend call first pkt add\n",__func__); + + return 0; +} + static int detector_add_host_port_application(lua_State* L) { auto& ud = *UserData::check(L, DETECTOR, 1); @@ -2734,6 +2776,7 @@ static const luaL_Reg detector_methods[] = { "addSipServer", detector_add_sip_server }, { "addSSLCnamePattern", detector_add_ssl_cname_pattern }, { "addSSHPattern", detector_add_ssh_client_pattern}, + { "addHostFirstPktApp", detector_add_host_first_pkt_application }, { "addHostPortApp", detector_add_host_port_application }, { "addHostPortAppDynamic", detector_add_host_port_dynamic }, { "addDNSHostPattern", detector_add_dns_host_pattern }, diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc index 418ef9c34..fd99e95f7 100644 --- a/src/network_inspectors/appid/test/appid_discovery_test.cc +++ b/src/network_inspectors/appid/test/appid_discovery_test.cc @@ -280,6 +280,12 @@ HostPortVal* HostPortCache::find(const SfIp*, uint16_t, IpProtocol, const OdpCon { return nullptr; } + +HostAppIdsVal* HostPortCache::find_on_first_pkt(const SfIp*, uint16_t, IpProtocol, const OdpContext&) +{ + return nullptr; +} + void AppIdServiceState::check_reset(AppIdSession&, const SfIp*, uint16_t, int16_t, uint32_t) {} bool do_tp_discovery(ThirdPartyAppIdContext& , AppIdSession&, IpProtocol,