Pull request #4571: appid: Adding general AppID design to support shadow traffic...

Merge in SNORT/snort3 from ~BSACHDEV/snort3:shadow_traffic_encrypted_dns to master

Squashed commit of the following:

commit e1e9f557a7fb265f71b210c5d35a7653260b744f
Author: bsachdev <bsachdev@cisco.com>
Date:   Tue Dec 10 10:02:46 2024 -0500

    appid: Adding general appid support and encrypted dns

diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc
index 50711dc6c7ff63f68e10fdd5c78881c1aaa12dec..b2caeb221e0ec5efdacded5c9cdabf5c10c88102 100644 (file)
--- a/src/network_inspectors/appid/app_info_table.cc
+++ b/src/network_inspectors/appid/app_info_table.cc
@@ -56,14 +56,14 @@ static const char* APP_CONFIG_FILE = "appid.conf";
 static const char* USR_CONFIG_FILE = "userappid.conf";
 const char* APP_MAPPING_FILE = "appMapping.data";
 
-AppInfoTableEntry::AppInfoTableEntry(AppId id, char* name)
-    : appId(id), serviceId(id), clientId(id), payloadId(id), app_name(name)
+AppInfoTableEntry::AppInfoTableEntry(AppId id, char* name, uint32_t attr)
+    : appId(id), serviceId(id), clientId(id), payloadId(id), app_name(name), attributes(attr)
 {
     app_name_key = AppInfoManager::strdup_to_lower(name);
 }
 
-AppInfoTableEntry::AppInfoTableEntry(AppId id, char* name, AppId sid, AppId cid, AppId pid) :
-    appId(id), serviceId(sid), clientId(cid), payloadId(pid), app_name(name)
+AppInfoTableEntry::AppInfoTableEntry(AppId id, char* name, AppId sid, AppId cid, AppId pid, uint32_t attr) :
+    appId(id), serviceId(sid), clientId(cid), payloadId(pid), app_name(name), attributes(attr)
 {
     app_name_key = AppInfoManager::strdup_to_lower(name);
 }
@@ -762,7 +762,7 @@ void AppInfoManager::init_appid_info_table(const AppIdConfig& config,
         while (fgets(buf, sizeof(buf), tableFile))
         {
             AppId app_id;
-            uint32_t client_id, service_id, payload_id;
+            uint32_t client_id, service_id, payload_id, attributes = 0;
             char* app_name;
             char* context;
 
@@ -809,15 +809,49 @@ void AppInfoManager::init_appid_info_table(const AppIdConfig& config,
             }
             payload_id = strtoul(token, nullptr, 10);
 
-            AppInfoTableEntry* entry = new AppInfoTableEntry(app_id, app_name, service_id,
-                client_id, payload_id);
-
             /* snort service key, if it exists */
+            const char* snort_service_key = strtok_r(nullptr, CONF_SEPARATORS, &context); 
+            /* skipping 7th column app_snort_key*/
+            strtok_r(nullptr, CONF_SEPARATORS, &context);
+            /* parsing 8th column attributes*/
             token = strtok_r(nullptr, CONF_SEPARATORS, &context);
+            if (token) 
+            { 
+                char attr_token_buffer[MAX_TABLE_LINE_LEN]; 
+                strncpy(attr_token_buffer, token, sizeof(attr_token_buffer) - 1); 
+                attr_token_buffer[sizeof(attr_token_buffer) - 1] = '\0';
+                char* attr_token; 
+                char* attr_context; 
+                attr_token = strtok_r(attr_token_buffer, ",", &attr_context); 
+                while (attr_token) 
+                { 
+                    if (strcmp(attr_token, "~") == 0) 
+                    { 
+                        attributes |= ATTR_EMPTY; 
+                        break; 
+                    } 
+                    if (strcmp(attr_token, "evasivevpn") == 0) 
+                    { 
+                        attributes |= ATTR_APPEVASIVEVPN; 
+                    } 
+                    else if (strcmp(attr_token, "encrypteddns") == 0) 
+                    { 
+                        attributes |= ATTR_APPENCRYPTEDDNS; 
+                    } 
+                    else if (strcmp(attr_token, "multihopproxy")== 0) 
+                    { 
+                        attributes |=  ATTR_APPMULTIHOPPROXY;  
+                    } 
+                attr_token = strtok_r(nullptr, ",", &attr_context); 
+                } 
+            }
 
-            // FIXIT-RC: Sometimes the token is "~". Should we ignore those?
-            if (token)
-                entry->snort_protocol_id = add_appid_protocol_reference(token, sc);
+            AppInfoTableEntry* entry = new AppInfoTableEntry(app_id, app_name, service_id,
+                client_id, payload_id, attributes);
+
+             // FIXIT-RC: Sometimes the token is "~". Should we ignore those?
+            if (snort_service_key) 
+                entry->snort_protocol_id = add_appid_protocol_reference(snort_service_key, sc);
 
             if (!add_entry_to_app_info_name_table(entry->app_name_key, entry))
                 delete entry;
@@ -850,3 +884,12 @@ void AppInfoManager::init_appid_info_table(const AppIdConfig& config,
     }
 }
 
+uint32_t AppInfoManager::getAttributeBits(AppId id) 
+{  
+    AppInfoTableEntry* entry = get_app_info_entry(id); 
+    if (! entry) { 
+        return 0;    
+    } 
+    return entry->attributes;  
+}
+

diff --git a/src/network_inspectors/appid/app_info_table.h b/src/network_inspectors/appid/app_info_table.h
index 5ddbef5c3792dfa62114156d59a4fc009fd5da9e..8bd69b4634e9ab9bf3b604296a6eae98a6c4b4b2 100644 (file)
--- a/src/network_inspectors/appid/app_info_table.h
+++ b/src/network_inspectors/appid/app_info_table.h
@@ -38,6 +38,11 @@
 #define SF_APPID_CSD_MIN        1000000
 #define SF_APPID_DYNAMIC_MIN    2000000
 
+#define ATTR_EMPTY 0 
+#define ATTR_APPEVASIVEVPN       (1<<0) 
+#define ATTR_APPMULTIHOPPROXY    (1<<1) 
+#define ATTR_APPENCRYPTEDDNS     (1<<2)
+
 class AppIdConfig;
 class ClientDetector;
 class OdpContext;
@@ -65,8 +70,8 @@ enum AppInfoFlags
 class AppInfoTableEntry
 {
 public:
-    AppInfoTableEntry(AppId id, char* name);
-    AppInfoTableEntry(AppId id, char* name, AppId sid, AppId cid, AppId pid);
+    AppInfoTableEntry(AppId id, char* name, uint32_t attr=0);
+    AppInfoTableEntry(AppId id, char* name, AppId sid, AppId cid, AppId pid, uint32_t attr=0);
     ~AppInfoTableEntry();
 
     AppId appId;
@@ -80,6 +85,7 @@ public:
     ServiceDetector* service_detector = nullptr;
     char* app_name = nullptr;
     char* app_name_key = nullptr;
+    uint32_t attributes = 0;
 };
 
 typedef std::unordered_map<AppId, AppInfoTableEntry*> AppInfoTable;
@@ -138,6 +144,7 @@ public:
     void dump_app_info_table();
     SnortProtocolId add_appid_protocol_reference(const char* protocol, snort::SnortConfig*);
     void dump_appid_configurations(const std::string&) const;
+    uint32_t getAttributeBits(AppId id);
 
 private:
     void load_odp_config(OdpContext&, const char* path);

diff --git a/src/network_inspectors/appid/appid_api.cc b/src/network_inspectors/appid/appid_api.cc
index 8d3420d99dc095ef0c50be0dc808b9c877d9699c..a7e321624fbaf810572bf8a0075ddd1a9b8c6ebe 100644 (file)
--- a/src/network_inspectors/appid/appid_api.cc
+++ b/src/network_inspectors/appid/appid_api.cc
@@ -287,3 +287,13 @@ void AppIdApi::reset_appid_cpu_profiler_stats()
     OdpContext& odp_ctxt = ctxt.get_odp_ctxt();
     odp_ctxt.get_appid_cpu_profiler_mgr().cleanup_appid_cpu_profiler_table();
 }
+
+void AppIdApi::update_shadow_traffic_status(bool status)
+{
+   AppIdInspector* inspector = (AppIdInspector*) InspectorManager::get_inspector(MOD_NAME);
+    if (!inspector)
+        return;
+    const AppIdContext& ctxt = inspector->get_ctxt();
+    OdpContext& odp_ctxt = ctxt.get_odp_ctxt();
+    odp_ctxt.set_appid_shadow_traffic_status(status);
+}

diff --git a/src/network_inspectors/appid/appid_api.h b/src/network_inspectors/appid/appid_api.h
index bdf56013ba4b1c2ad011dba633d34e14497204b1..5c5ce2537383a6dc4e36d129ec7e4e0bad8cad46 100644 (file)
--- a/src/network_inspectors/appid/appid_api.h
+++ b/src/network_inspectors/appid/appid_api.h
@@ -54,6 +54,7 @@ public:
     bool is_inspection_needed(const Inspector& g) const;
     const char* get_appid_detector_directory() const;
     void reset_appid_cpu_profiler_stats();
+    void update_shadow_traffic_status(bool status);
 
     bool is_service_http_type(AppId service_id) const
     {

diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h
index 3d89ba4eebf1fe5f3dc37761173e00061ea0692d..4c3cc703a1cc21b477dc781281c9d0ca6e55d93f 100644 (file)
--- a/src/network_inspectors/appid/appid_config.h
+++ b/src/network_inspectors/appid/appid_config.h
@@ -279,6 +279,16 @@ public:
     {
         return user_data_map;
     }
+
+    void set_appid_shadow_traffic_status(bool status)
+    { 
+        appid_shadow_traffic_status = status; 
+    }
+
+    bool get_appid_shadow_traffic_status() const
+    { 
+        return appid_shadow_traffic_status; 
+    }
  
     unsigned get_pattern_count();
     void add_port_service_id(IpProtocol, uint16_t, AppId);
@@ -315,6 +325,7 @@ private:
 
     uint32_t version;
     static uint32_t next_version;
+    bool appid_shadow_traffic_status = true;
 };
 
 class OdpThreadContext

diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index 61a23fcc2b30fefba89b2999a0a042eb1b2368c8..821e9c314b7ca421579558c4396dc568afa00ab8 100644 (file)
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -890,5 +890,8 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd,
     if (PacketTracer::is_daq_activated())
         populate_trace_data(asd);
 
+    if (is_discovery_done and asd.get_shadow_traffic_bits() == 0 )
+        asd.process_shadow_traffic_appids();
+
     asd.publish_appid_event(change_bits, *p);
 }

diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc
index fcdfd28068c83b1c6bc2606b49b17716b34509b1..1ba7763aede0ece9b24c7edaef9c4a8e1a4c1bfa 100644 (file)
--- a/src/network_inspectors/appid/appid_session.cc
+++ b/src/network_inspectors/appid/appid_session.cc
@@ -34,6 +34,7 @@
 #include "protocols/packet.h"
 #include "protocols/tcp.h"
 #include "pub_sub/appid_events.h"
+#include "pub_sub/shadowtraffic_aggregator.h"
 #include "stream/stream.h"
 #include "target_based/snort_protocols.h"
 #include "time/packet_time.h"
@@ -155,6 +156,15 @@ AppIdSession::~AppIdSession()
     {
         api.asd->get_odp_ctxt().get_appid_cpu_profiler_mgr().check_appid_cpu_profiler_table_entry(api.asd, api.get_service_app_id(), api.get_client_app_id(), api.get_payload_app_id(), api.get_misc_app_id());
     }
+ 
+    if ((pkt_thread_odp_ctxt->get_version() == api.asd->get_odp_ctxt_version()) and api.asd->get_odp_ctxt().get_appid_shadow_traffic_status())
+    { 
+        if (get_shadow_traffic_publishing_appid() > APP_ID_NONE)
+        {
+            if (api.asd->appid_shadow_traffic_bits != 0)
+                api.asd->publish_shadow_traffic_event(api.asd->appid_shadow_traffic_bits, api.asd->flow);
+        } 
+    }
 
     if (!in_expected_cache)
     {
@@ -1197,6 +1207,36 @@ void AppIdSession::set_tp_payload_app_id(const Packet& p, AppidSessionDirection
     }
 }
 
+void AppIdSession::publish_shadow_traffic_event(const uint32_t &shadow_traffic_bits, snort::Flow *) const
+{
+    if (shadow_traffic_bits == 0) 
+        return;
+     
+    const char* app_name;
+    unsigned shadow_traffic_pub_id = 0;
+    std::string str_print; 
+
+    AppId publishing_appid = get_shadow_traffic_publishing_appid();
+    app_name = api.asd->get_odp_ctxt().get_app_info_mgr().get_app_name(publishing_appid);
+    if (app_name == nullptr)
+    {
+        APPID_LOG(nullptr, TRACE_ERROR_LEVEL, "Appname is invalid, not publishing shadow traffic event without appname\n");
+        return;
+    }
+
+    shadow_traffic_pub_id = DataBus::get_id(shadowtraffic_pub_key);
+  
+    ShadowTrafficEvent shadow_event(shadow_traffic_bits, "", "", app_name);
+    DataBus::publish(shadow_traffic_pub_id, ShadowTrafficEventIds::SHADOWTRAFFIC_FLOW_DETECTED, shadow_event, flow); 
+
+    if (appidDebug and appidDebug->is_active())
+        change_shadow_traffic_bits_to_string(shadow_traffic_bits, str_print);
+    
+    APPID_LOG(CURRENT_PACKET, TRACE_DEBUG_LEVEL, 
+        "AppID: ShadowTraffic Published event for: %s, application_name: %s(%d)\n", 
+        str_print.c_str(), app_name, publishing_appid);
+} 
+
 void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, const Packet& p,
     bool is_httpx, uint32_t httpx_stream_index)
 {
@@ -1252,3 +1292,46 @@ void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, const Packe
     else
         APPID_LOG(&p, TRACE_DEBUG_LEVEL, "Published event for changes: %s\n",  str.c_str());
 }
+
+void AppIdSession::check_shadow_traffic_bits(AppId id, uint32_t& shadow_bits, AppId& publishing_appid, bool& is_publishing_set)
+{
+   if (id > APP_ID_NONE)
+   {
+        uint32_t attributeBits = api.asd->get_odp_ctxt().get_app_info_mgr().getAttributeBits(id); 
+        if (attributeBits & ATTR_APPENCRYPTEDDNS)
+        {
+            shadow_bits |= ShadowTraffic_Type_Encrypted_DNS;
+            if (!is_publishing_set)
+            {
+                publishing_appid = id;
+                is_publishing_set = true;
+            }
+        }
+    }
+}
+
+void AppIdSession::process_shadow_traffic_appids()
+{
+    uint32_t shadow_bits = 0;
+    AppId publishing_appid = APP_ID_NONE;
+    bool is_publishing_set = false;
+    AppId service_id = api.get_service_app_id();
+    AppId payload_id = api.get_payload_app_id();
+    AppId client_id = api.get_client_app_id();
+    AppId misc_id = api.get_misc_app_id();
+
+    if (service_id > 0)
+        check_shadow_traffic_bits(service_id, shadow_bits, publishing_appid, is_publishing_set);
+    if (payload_id > 0) 
+        check_shadow_traffic_bits(payload_id, shadow_bits, publishing_appid, is_publishing_set);
+    if (client_id > 0)
+        check_shadow_traffic_bits(client_id, shadow_bits, publishing_appid, is_publishing_set);
+    if (misc_id > 0) 
+        check_shadow_traffic_bits(misc_id, shadow_bits, publishing_appid, is_publishing_set); 
+
+    if (shadow_bits != 0)
+    {
+        set_shadow_traffic_bits(shadow_bits);
+        set_shadow_traffic_publishing_appid(publishing_appid);
+    } 
+}

diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h
index 827bf0badce0d462233ad669e45de8e23d94af95..37b3c440435126cb4c0f732498963bd8dea0de4a 100644 (file)
--- a/src/network_inspectors/appid/appid_session.h
+++ b/src/network_inspectors/appid/appid_session.h
@@ -40,6 +40,7 @@
 #include "application_ids.h"
 #include "detector_plugins/http_url_patterns.h"
 #include "length_app_cache.h"
+#include "pub_sub/shadowtraffic_aggregator.h"
 #include "service_state.h"
 
 namespace snort
@@ -434,6 +435,9 @@ public:
         AppidChangeBits& change_bits);
     void publish_appid_event(AppidChangeBits&, const snort::Packet&, bool is_httpx = false,
         uint32_t httpx_stream_index = 0);
+    void publish_shadow_traffic_event(const uint32_t& shadow_traffic_bits,snort::Flow*)const;
+    void process_shadow_traffic_appids();
+    void check_shadow_traffic_bits(AppId id, uint32_t& shadow_bits, AppId &publishing_appid, bool& is_publishing_set);
 
     bool need_to_delete_tp_conn(ThirdPartyAppIdContext*) const;
 
@@ -759,6 +763,49 @@ public:
         return get_session_flags(APPID_SESSION_OPPORTUNISTIC_TLS) and !flow->flags.data_decrypted;
     }
 
+    void set_shadow_traffic_bits(uint32_t lv_bits)
+    {
+       appid_shadow_traffic_bits = lv_bits;
+    }
+
+    uint32_t get_shadow_traffic_bits()
+    {
+        return appid_shadow_traffic_bits;
+    }
+
+    void set_shadow_traffic_publishing_appid(AppId id)
+    {
+       shadow_traffic_appid = id; 
+    }
+
+    AppId get_shadow_traffic_publishing_appid() const
+    {
+        return shadow_traffic_appid;
+    }
+    
+    inline void change_shadow_traffic_bits_to_string (const uint32_t& st_bits,std::string& str) const 
+    {  
+        std::string tempStr;
+
+        if (st_bits & ShadowTraffic_Type_Encrypted_DNS) {
+            tempStr.append("Encrypted_DNS ");
+        }
+        if (st_bits & ShadowTraffic_Type_Evasive_VPN) {
+            tempStr.append("Evasive_VPN ");
+        }
+        if (st_bits & ShadowTraffic_Type_Multihop_Proxy) {
+            tempStr.append("Multihop_Proxy ");
+        }
+        if (st_bits & ShadowTraffic_Type_Domain_Fronting) {
+            tempStr.append("Domain_Fronting ");
+        }
+        if (!tempStr.empty()) {
+            tempStr.pop_back();
+        }
+        
+        str.append(tempStr);     
+    } 
+
 private:
     uint16_t prev_httpx_raw_packet = 0;
 
@@ -782,6 +829,8 @@ private:
     bool no_service_candidate = false;
     bool no_service_inspector = false;
     bool client_info_unpublished = false;
+    uint32_t appid_shadow_traffic_bits = 0;
+    AppId shadow_traffic_appid = APP_ID_NONE;
 };
 
 #endif

diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc
index c5b5efdf4541a1d426dad4c2d5a1bac4423cecda..bd176830f236beade0020258069064fbc54990f9 100644 (file)
--- a/src/network_inspectors/appid/test/appid_discovery_test.cc
+++ b/src/network_inspectors/appid/test/appid_discovery_test.cc
@@ -217,6 +217,11 @@ const char* AppInfoManager::get_app_name(int32_t)
     return nullptr;
 }
 
+uint32_t AppInfoManager::getAttributeBits(AppId)
+{
+    return 0;
+} 
+
 // Stubs for AppIdSession
 void AppIdSession::sync_with_snort_protocol_id(AppId, Packet*) {}
 void AppIdSession::check_app_detection_restart(AppidChangeBits&, ThirdPartyAppIdContext*) {}
@@ -227,6 +232,8 @@ void AppIdSession::update_encrypted_app_id(AppId) {}
 bool AppIdSession::is_tp_processing_done() const {return false;}
 AppId AppIdSession::pick_ss_payload_app_id(AppId) const { return get_payload_id(); }
 bool AppIdSession::need_to_delete_tp_conn(ThirdPartyAppIdContext*) const { return true; }
+void AppIdSession::process_shadow_traffic_appids() {}
+
 AppIdSession* AppIdSession::allocate_session(const Packet*, IpProtocol,
     AppidSessionDirection, AppIdInspector&, OdpContext&)
 {