From: Bhumika Sachdeva (bsachdev) Date: Fri, 7 Feb 2025 14:21:49 +0000 (+0000) Subject: Pull request #4571: appid: Adding general AppID design to support shadow traffic... X-Git-Tag: 3.7.1.0~33 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=7b69515dd6c99f33f33bf1ff8d599f44e4a85537;p=thirdparty%2Fsnort3.git Pull request #4571: appid: Adding general AppID design to support shadow traffic and Encrypted DNS Support Merge in SNORT/snort3 from ~BSACHDEV/snort3:shadow_traffic_encrypted_dns to master Squashed commit of the following: commit e1e9f557a7fb265f71b210c5d35a7653260b744f Author: bsachdev Date: Tue Dec 10 10:02:46 2024 -0500 appid: Adding general appid support and encrypted dns --- diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc index 50711dc6c..b2caeb221 100644 --- a/src/network_inspectors/appid/app_info_table.cc +++ b/src/network_inspectors/appid/app_info_table.cc @@ -56,14 +56,14 @@ static const char* APP_CONFIG_FILE = "appid.conf"; static const char* USR_CONFIG_FILE = "userappid.conf"; const char* APP_MAPPING_FILE = "appMapping.data"; -AppInfoTableEntry::AppInfoTableEntry(AppId id, char* name) - : appId(id), serviceId(id), clientId(id), payloadId(id), app_name(name) +AppInfoTableEntry::AppInfoTableEntry(AppId id, char* name, uint32_t attr) + : appId(id), serviceId(id), clientId(id), payloadId(id), app_name(name), attributes(attr) { app_name_key = AppInfoManager::strdup_to_lower(name); } -AppInfoTableEntry::AppInfoTableEntry(AppId id, char* name, AppId sid, AppId cid, AppId pid) : - appId(id), serviceId(sid), clientId(cid), payloadId(pid), app_name(name) +AppInfoTableEntry::AppInfoTableEntry(AppId id, char* name, AppId sid, AppId cid, AppId pid, uint32_t attr) : + appId(id), serviceId(sid), clientId(cid), payloadId(pid), app_name(name), attributes(attr) { app_name_key = AppInfoManager::strdup_to_lower(name); } @@ -762,7 +762,7 @@ void AppInfoManager::init_appid_info_table(const AppIdConfig& config, while (fgets(buf, sizeof(buf), tableFile)) { AppId app_id; - uint32_t client_id, service_id, payload_id; + uint32_t client_id, service_id, payload_id, attributes = 0; char* app_name; char* context; @@ -809,15 +809,49 @@ void AppInfoManager::init_appid_info_table(const AppIdConfig& config, } payload_id = strtoul(token, nullptr, 10); - AppInfoTableEntry* entry = new AppInfoTableEntry(app_id, app_name, service_id, - client_id, payload_id); - /* snort service key, if it exists */ + const char* snort_service_key = strtok_r(nullptr, CONF_SEPARATORS, &context); + /* skipping 7th column app_snort_key*/ + strtok_r(nullptr, CONF_SEPARATORS, &context); + /* parsing 8th column attributes*/ token = strtok_r(nullptr, CONF_SEPARATORS, &context); + if (token) + { + char attr_token_buffer[MAX_TABLE_LINE_LEN]; + strncpy(attr_token_buffer, token, sizeof(attr_token_buffer) - 1); + attr_token_buffer[sizeof(attr_token_buffer) - 1] = '\0'; + char* attr_token; + char* attr_context; + attr_token = strtok_r(attr_token_buffer, ",", &attr_context); + while (attr_token) + { + if (strcmp(attr_token, "~") == 0) + { + attributes |= ATTR_EMPTY; + break; + } + if (strcmp(attr_token, "evasivevpn") == 0) + { + attributes |= ATTR_APPEVASIVEVPN; + } + else if (strcmp(attr_token, "encrypteddns") == 0) + { + attributes |= ATTR_APPENCRYPTEDDNS; + } + else if (strcmp(attr_token, "multihopproxy")== 0) + { + attributes |= ATTR_APPMULTIHOPPROXY; + } + attr_token = strtok_r(nullptr, ",", &attr_context); + } + } - // FIXIT-RC: Sometimes the token is "~". Should we ignore those? - if (token) - entry->snort_protocol_id = add_appid_protocol_reference(token, sc); + AppInfoTableEntry* entry = new AppInfoTableEntry(app_id, app_name, service_id, + client_id, payload_id, attributes); + + // FIXIT-RC: Sometimes the token is "~". Should we ignore those? + if (snort_service_key) + entry->snort_protocol_id = add_appid_protocol_reference(snort_service_key, sc); if (!add_entry_to_app_info_name_table(entry->app_name_key, entry)) delete entry; @@ -850,3 +884,12 @@ void AppInfoManager::init_appid_info_table(const AppIdConfig& config, } } +uint32_t AppInfoManager::getAttributeBits(AppId id) +{ + AppInfoTableEntry* entry = get_app_info_entry(id); + if (! entry) { + return 0; + } + return entry->attributes; +} + diff --git a/src/network_inspectors/appid/app_info_table.h b/src/network_inspectors/appid/app_info_table.h index 5ddbef5c3..8bd69b463 100644 --- a/src/network_inspectors/appid/app_info_table.h +++ b/src/network_inspectors/appid/app_info_table.h @@ -38,6 +38,11 @@ #define SF_APPID_CSD_MIN 1000000 #define SF_APPID_DYNAMIC_MIN 2000000 +#define ATTR_EMPTY 0 +#define ATTR_APPEVASIVEVPN (1<<0) +#define ATTR_APPMULTIHOPPROXY (1<<1) +#define ATTR_APPENCRYPTEDDNS (1<<2) + class AppIdConfig; class ClientDetector; class OdpContext; @@ -65,8 +70,8 @@ enum AppInfoFlags class AppInfoTableEntry { public: - AppInfoTableEntry(AppId id, char* name); - AppInfoTableEntry(AppId id, char* name, AppId sid, AppId cid, AppId pid); + AppInfoTableEntry(AppId id, char* name, uint32_t attr=0); + AppInfoTableEntry(AppId id, char* name, AppId sid, AppId cid, AppId pid, uint32_t attr=0); ~AppInfoTableEntry(); AppId appId; @@ -80,6 +85,7 @@ public: ServiceDetector* service_detector = nullptr; char* app_name = nullptr; char* app_name_key = nullptr; + uint32_t attributes = 0; }; typedef std::unordered_map AppInfoTable; @@ -138,6 +144,7 @@ public: void dump_app_info_table(); SnortProtocolId add_appid_protocol_reference(const char* protocol, snort::SnortConfig*); void dump_appid_configurations(const std::string&) const; + uint32_t getAttributeBits(AppId id); private: void load_odp_config(OdpContext&, const char* path); diff --git a/src/network_inspectors/appid/appid_api.cc b/src/network_inspectors/appid/appid_api.cc index 8d3420d99..a7e321624 100644 --- a/src/network_inspectors/appid/appid_api.cc +++ b/src/network_inspectors/appid/appid_api.cc @@ -287,3 +287,13 @@ void AppIdApi::reset_appid_cpu_profiler_stats() OdpContext& odp_ctxt = ctxt.get_odp_ctxt(); odp_ctxt.get_appid_cpu_profiler_mgr().cleanup_appid_cpu_profiler_table(); } + +void AppIdApi::update_shadow_traffic_status(bool status) +{ + AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME); + if (!inspector) + return; + const AppIdContext& ctxt = inspector->get_ctxt(); + OdpContext& odp_ctxt = ctxt.get_odp_ctxt(); + odp_ctxt.set_appid_shadow_traffic_status(status); +} diff --git a/src/network_inspectors/appid/appid_api.h b/src/network_inspectors/appid/appid_api.h index bdf56013b..5c5ce2537 100644 --- a/src/network_inspectors/appid/appid_api.h +++ b/src/network_inspectors/appid/appid_api.h @@ -54,6 +54,7 @@ public: bool is_inspection_needed(const Inspector& g) const; const char* get_appid_detector_directory() const; void reset_appid_cpu_profiler_stats(); + void update_shadow_traffic_status(bool status); bool is_service_http_type(AppId service_id) const { diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h index 3d89ba4ee..4c3cc703a 100644 --- a/src/network_inspectors/appid/appid_config.h +++ b/src/network_inspectors/appid/appid_config.h @@ -279,6 +279,16 @@ public: { return user_data_map; } + + void set_appid_shadow_traffic_status(bool status) + { + appid_shadow_traffic_status = status; + } + + bool get_appid_shadow_traffic_status() const + { + return appid_shadow_traffic_status; + } unsigned get_pattern_count(); void add_port_service_id(IpProtocol, uint16_t, AppId); @@ -315,6 +325,7 @@ private: uint32_t version; static uint32_t next_version; + bool appid_shadow_traffic_status = true; }; class OdpThreadContext diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index 61a23fcc2..821e9c314 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -890,5 +890,8 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd, if (PacketTracer::is_daq_activated()) populate_trace_data(asd); + if (is_discovery_done and asd.get_shadow_traffic_bits() == 0 ) + asd.process_shadow_traffic_appids(); + asd.publish_appid_event(change_bits, *p); } diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc index fcdfd2806..1ba7763ae 100644 --- a/src/network_inspectors/appid/appid_session.cc +++ b/src/network_inspectors/appid/appid_session.cc @@ -34,6 +34,7 @@ #include "protocols/packet.h" #include "protocols/tcp.h" #include "pub_sub/appid_events.h" +#include "pub_sub/shadowtraffic_aggregator.h" #include "stream/stream.h" #include "target_based/snort_protocols.h" #include "time/packet_time.h" @@ -155,6 +156,15 @@ AppIdSession::~AppIdSession() { api.asd->get_odp_ctxt().get_appid_cpu_profiler_mgr().check_appid_cpu_profiler_table_entry(api.asd, api.get_service_app_id(), api.get_client_app_id(), api.get_payload_app_id(), api.get_misc_app_id()); } + + if ((pkt_thread_odp_ctxt->get_version() == api.asd->get_odp_ctxt_version()) and api.asd->get_odp_ctxt().get_appid_shadow_traffic_status()) + { + if (get_shadow_traffic_publishing_appid() > APP_ID_NONE) + { + if (api.asd->appid_shadow_traffic_bits != 0) + api.asd->publish_shadow_traffic_event(api.asd->appid_shadow_traffic_bits, api.asd->flow); + } + } if (!in_expected_cache) { @@ -1197,6 +1207,36 @@ void AppIdSession::set_tp_payload_app_id(const Packet& p, AppidSessionDirection } } +void AppIdSession::publish_shadow_traffic_event(const uint32_t &shadow_traffic_bits, snort::Flow *) const +{ + if (shadow_traffic_bits == 0) + return; + + const char* app_name; + unsigned shadow_traffic_pub_id = 0; + std::string str_print; + + AppId publishing_appid = get_shadow_traffic_publishing_appid(); + app_name = api.asd->get_odp_ctxt().get_app_info_mgr().get_app_name(publishing_appid); + if (app_name == nullptr) + { + APPID_LOG(nullptr, TRACE_ERROR_LEVEL, "Appname is invalid, not publishing shadow traffic event without appname\n"); + return; + } + + shadow_traffic_pub_id = DataBus::get_id(shadowtraffic_pub_key); + + ShadowTrafficEvent shadow_event(shadow_traffic_bits, "", "", app_name); + DataBus::publish(shadow_traffic_pub_id, ShadowTrafficEventIds::SHADOWTRAFFIC_FLOW_DETECTED, shadow_event, flow); + + if (appidDebug and appidDebug->is_active()) + change_shadow_traffic_bits_to_string(shadow_traffic_bits, str_print); + + APPID_LOG(CURRENT_PACKET, TRACE_DEBUG_LEVEL, + "AppID: ShadowTraffic Published event for: %s, application_name: %s(%d)\n", + str_print.c_str(), app_name, publishing_appid); +} + void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, const Packet& p, bool is_httpx, uint32_t httpx_stream_index) { @@ -1252,3 +1292,46 @@ void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, const Packe else APPID_LOG(&p, TRACE_DEBUG_LEVEL, "Published event for changes: %s\n", str.c_str()); } + +void AppIdSession::check_shadow_traffic_bits(AppId id, uint32_t& shadow_bits, AppId& publishing_appid, bool& is_publishing_set) +{ + if (id > APP_ID_NONE) + { + uint32_t attributeBits = api.asd->get_odp_ctxt().get_app_info_mgr().getAttributeBits(id); + if (attributeBits & ATTR_APPENCRYPTEDDNS) + { + shadow_bits |= ShadowTraffic_Type_Encrypted_DNS; + if (!is_publishing_set) + { + publishing_appid = id; + is_publishing_set = true; + } + } + } +} + +void AppIdSession::process_shadow_traffic_appids() +{ + uint32_t shadow_bits = 0; + AppId publishing_appid = APP_ID_NONE; + bool is_publishing_set = false; + AppId service_id = api.get_service_app_id(); + AppId payload_id = api.get_payload_app_id(); + AppId client_id = api.get_client_app_id(); + AppId misc_id = api.get_misc_app_id(); + + if (service_id > 0) + check_shadow_traffic_bits(service_id, shadow_bits, publishing_appid, is_publishing_set); + if (payload_id > 0) + check_shadow_traffic_bits(payload_id, shadow_bits, publishing_appid, is_publishing_set); + if (client_id > 0) + check_shadow_traffic_bits(client_id, shadow_bits, publishing_appid, is_publishing_set); + if (misc_id > 0) + check_shadow_traffic_bits(misc_id, shadow_bits, publishing_appid, is_publishing_set); + + if (shadow_bits != 0) + { + set_shadow_traffic_bits(shadow_bits); + set_shadow_traffic_publishing_appid(publishing_appid); + } +} diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h index 827bf0bad..37b3c4404 100644 --- a/src/network_inspectors/appid/appid_session.h +++ b/src/network_inspectors/appid/appid_session.h @@ -40,6 +40,7 @@ #include "application_ids.h" #include "detector_plugins/http_url_patterns.h" #include "length_app_cache.h" +#include "pub_sub/shadowtraffic_aggregator.h" #include "service_state.h" namespace snort @@ -434,6 +435,9 @@ public: AppidChangeBits& change_bits); void publish_appid_event(AppidChangeBits&, const snort::Packet&, bool is_httpx = false, uint32_t httpx_stream_index = 0); + void publish_shadow_traffic_event(const uint32_t& shadow_traffic_bits,snort::Flow*)const; + void process_shadow_traffic_appids(); + void check_shadow_traffic_bits(AppId id, uint32_t& shadow_bits, AppId &publishing_appid, bool& is_publishing_set); bool need_to_delete_tp_conn(ThirdPartyAppIdContext*) const; @@ -759,6 +763,49 @@ public: return get_session_flags(APPID_SESSION_OPPORTUNISTIC_TLS) and !flow->flags.data_decrypted; } + void set_shadow_traffic_bits(uint32_t lv_bits) + { + appid_shadow_traffic_bits = lv_bits; + } + + uint32_t get_shadow_traffic_bits() + { + return appid_shadow_traffic_bits; + } + + void set_shadow_traffic_publishing_appid(AppId id) + { + shadow_traffic_appid = id; + } + + AppId get_shadow_traffic_publishing_appid() const + { + return shadow_traffic_appid; + } + + inline void change_shadow_traffic_bits_to_string (const uint32_t& st_bits,std::string& str) const + { + std::string tempStr; + + if (st_bits & ShadowTraffic_Type_Encrypted_DNS) { + tempStr.append("Encrypted_DNS "); + } + if (st_bits & ShadowTraffic_Type_Evasive_VPN) { + tempStr.append("Evasive_VPN "); + } + if (st_bits & ShadowTraffic_Type_Multihop_Proxy) { + tempStr.append("Multihop_Proxy "); + } + if (st_bits & ShadowTraffic_Type_Domain_Fronting) { + tempStr.append("Domain_Fronting "); + } + if (!tempStr.empty()) { + tempStr.pop_back(); + } + + str.append(tempStr); + } + private: uint16_t prev_httpx_raw_packet = 0; @@ -782,6 +829,8 @@ private: bool no_service_candidate = false; bool no_service_inspector = false; bool client_info_unpublished = false; + uint32_t appid_shadow_traffic_bits = 0; + AppId shadow_traffic_appid = APP_ID_NONE; }; #endif diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc index c5b5efdf4..bd176830f 100644 --- a/src/network_inspectors/appid/test/appid_discovery_test.cc +++ b/src/network_inspectors/appid/test/appid_discovery_test.cc @@ -217,6 +217,11 @@ const char* AppInfoManager::get_app_name(int32_t) return nullptr; } +uint32_t AppInfoManager::getAttributeBits(AppId) +{ + return 0; +} + // Stubs for AppIdSession void AppIdSession::sync_with_snort_protocol_id(AppId, Packet*) {} void AppIdSession::check_app_detection_restart(AppidChangeBits&, ThirdPartyAppIdContext*) {} @@ -227,6 +232,8 @@ void AppIdSession::update_encrypted_app_id(AppId) {} bool AppIdSession::is_tp_processing_done() const {return false;} AppId AppIdSession::pick_ss_payload_app_id(AppId) const { return get_payload_id(); } bool AppIdSession::need_to_delete_tp_conn(ThirdPartyAppIdContext*) const { return true; } +void AppIdSession::process_shadow_traffic_appids() {} + AppIdSession* AppIdSession::allocate_session(const Packet*, IpProtocol, AppidSessionDirection, AppIdInspector&, OdpContext&) {