Pull request #3510: appid : A custom lua detector api to map ip and port to appids...

Merge in SNORT/snort3 from ~UMASHARM/snort3:POC_FirstPkt to master

Squashed commit of the following:

commit 7bc2782effcc61941091f0bce53640cc3c85c293
Author: Umang Sharma <umasharm@cisco.com>
Date:   Tue Jul 12 06:31:29 2022 -0400

    appid: A custom lua detector api to map ip and port to appids on the first packet

diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h
index 4aaffcffd25dfd7889843828e46e3351ac8a7b5b..b61f596cef2cf0218ebc8f13d28c516cd25c396c 100644 (file)
--- a/src/network_inspectors/appid/appid_config.h
+++ b/src/network_inspectors/appid/appid_config.h
@@ -114,6 +114,7 @@ public:
     bool check_host_cache_unknown_ssl = false;
     bool ftp_userid_disabled = false;
     bool chp_body_collection_disabled = false;
+    bool need_reinspection = false;
     uint32_t chp_body_collection_max = 0;
     uint32_t rtmp_max_packets = 15;
     uint32_t max_tp_flow_depth = 5;
@@ -161,6 +162,17 @@ public:
         return host_port_cache.add(sc, ip, port, proto, type, appid);
     }
 
+    bool host_first_pkt_add(const snort::SnortConfig* sc, const snort::SfIp* ip, uint16_t port,
+        IpProtocol proto, AppId protocol_appid, AppId client_appid, AppId web_appid, unsigned reinspect)
+    {
+        return first_pkt_cache.add_host(sc, ip, port, proto, protocol_appid, client_appid, web_appid, reinspect);
+    }
+
+    HostAppIdsVal* host_first_pkt_find(const snort::SfIp* ip, uint16_t port, IpProtocol proto)
+    {
+        return first_pkt_cache.find_on_first_pkt(ip, port, proto, *this);
+    }
+
     AppId length_cache_find(const LengthKey& key)
     {
         return length_cache.find(key);
@@ -225,6 +237,7 @@ private:
     AppInfoManager app_info_mgr;
     ClientDiscovery client_disco_mgr;
     HostPortCache host_port_cache;
+    HostPortCache first_pkt_cache;
     LengthCache length_cache;
     DnsPatternMatchers dns_matchers;
     HttpPatternMatchers http_matchers;

diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index 9a982fb7554b5a9a5c3984d23cd2f573c9f93130..0fe89b2bb05eca5138f25b9b761f615be83b3478 100644 (file)
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -545,6 +545,108 @@ static inline bool is_check_host_cache_valid(AppIdSession& asd, AppId service_id
     return false;
 }
 
+bool AppIdDiscovery::detect_on_first_pkt(Packet* p, AppIdSession& asd,
+    IpProtocol protocol, AppidSessionDirection direction, AppId& service_id,
+    AppId& client_id, AppId& payload_id)
+{ 
+    uint16_t port;
+    const SfIp* ip;
+
+    if (direction == APP_ID_FROM_INITIATOR)
+    {
+        ip = p->ptrs.ip_api.get_dst();
+        port = p->ptrs.dp;
+    }
+    else
+    {
+        ip = p->ptrs.ip_api.get_src();
+        port = p->ptrs.sp;
+    }
+
+    HostAppIdsVal* hv = nullptr;
+    hv = asd.get_odp_ctxt().host_first_pkt_find(ip, port, protocol);
+    if (hv)
+    {
+        uint32_t appids_found = 0;
+        const char *service_app_name = nullptr, *client_app_name = nullptr, *payload_app_name = nullptr;
+        FirstPktAppIdDiscovered appid_prefix = NO_APPID_FOUND;
+
+        if (hv->client_appId)
+        {
+            client_id = hv->client_appId;
+            asd.set_client_id(client_id);
+            client_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(client_id);
+            appids_found++;
+            appid_prefix = CLIENT_APPID_FOUND;
+        } 
+        if (hv->protocol_appId) 
+        {
+            service_id = hv->protocol_appId;
+            asd.set_service_id(service_id, asd.get_odp_ctxt());
+            asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED);
+            service_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id);
+            appids_found++;
+            appid_prefix = SERVICE_APPID_FOUND;
+        }
+        if (hv->web_appId) 
+        {
+            payload_id = hv->web_appId;
+            asd.set_payload_id(payload_id);
+            payload_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(payload_id);
+            appids_found++;
+            if (appid_prefix == CLIENT_APPID_FOUND)
+            {
+                appid_prefix = CLIENT_PAYLOAD_APPID_FOUND;
+            } 
+            else 
+            {
+                appid_prefix = PAYLOAD_APPID_FOUND;
+            }
+        }
+        asd.get_odp_ctxt().need_reinspection = hv->reinspect;  
+
+        switch (appids_found) 
+        {
+        case FIRST_PKT_CACHE_ONE_APPID_FOUND :
+            if (appid_prefix == PAYLOAD_APPID_FOUND) 
+            {
+                service_id = payload_id;
+                asd.set_service_id(service_id, asd.get_odp_ctxt());
+                asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED);
+                service_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id);
+            } 
+            else if (appid_prefix == CLIENT_APPID_FOUND) 
+            {    
+                service_id = client_id;
+                asd.set_service_id(service_id, asd.get_odp_ctxt());
+                asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED);
+                service_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id);
+            } 
+            break;
+        case FIRST_PKT_CACHE_TWO_APPIDS_FOUND :
+            if (appid_prefix == CLIENT_PAYLOAD_APPID_FOUND) 
+            {
+                service_id = client_id;
+                asd.set_service_id(service_id, asd.get_odp_ctxt());
+                asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED);
+                service_app_name = asd.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id);
+            } 
+            break;
+        }
+        asd.set_session_flags(APPID_SESSION_FIRST_PKT_CACHE_MATCHED);
+        if (appidDebug->is_active())
+        {
+            LogMessage("AppIdDbg %s Host cache match found on first packet, service: %s(%d), "
+                "client: %s(%d), payload: %s(%d), reinspect: %s \n", appidDebug->get_debug_session(),
+                (service_app_name ? service_app_name : ""), service_id,
+                (client_app_name ? client_app_name : ""), client_id,
+                (payload_app_name ? payload_app_name : ""), payload_id, (hv->reinspect ? "True" : "False"));
+        }
+        return true;
+    }
+    return false;
+}
+
 bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol protocol,
     IpProtocol outer_protocol, AppidSessionDirection direction, AppId& service_id,
     AppId& client_id, AppId& payload_id, AppId& misc_id, AppidChangeBits& change_bits,
@@ -552,6 +654,22 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto
 {
     bool is_discovery_done = false;
 
+    if (asd.session_packet_count == 1) 
+    {
+        detect_on_first_pkt(p, asd, protocol, direction, service_id, client_id, payload_id);
+    } 
+
+    if (asd.get_session_flags(APPID_SESSION_FIRST_PKT_CACHE_MATCHED) and !asd.get_odp_ctxt().need_reinspection) 
+    {
+       is_discovery_done = true;
+       service_id = asd.pick_service_app_id();
+       client_id = asd.pick_ss_client_app_id();
+       payload_id = asd.pick_ss_payload_app_id(service_id);
+       asd.client_disco_state = APPID_DISCO_STATE_FINISHED; 
+       asd.service_disco_state = APPID_DISCO_STATE_FINISHED;
+       return is_discovery_done;
+    }  
+
     asd.check_app_detection_restart(change_bits, tp_appid_ctxt);
 
     if (outer_protocol != IpProtocol::PROTO_NOT_SET)

diff --git a/src/network_inspectors/appid/appid_discovery.h b/src/network_inspectors/appid/appid_discovery.h
index c4fe30cfe957826486d0b76d094142202a2adf06..7bec32905e9fdb13803790e2cb37d791f6488fec 100644 (file)
--- a/src/network_inspectors/appid/appid_discovery.h
+++ b/src/network_inspectors/appid/appid_discovery.h
@@ -60,6 +60,18 @@ struct Packet;
 #define SCAN_HTTP_URI_FLAG          (1<<9)
 #define SCAN_CERTVIZ_ENABLED_FLAG   (1<<10)
 #define SCAN_SPOOFED_SNI_FLAG       (1<<11)
+#define FIRST_PKT_CACHE_ONE_APPID_FOUND 1
+#define FIRST_PKT_CACHE_TWO_APPIDS_FOUND 2
+#define FIRST_PKT_CACHE_ALL_APPIDS_FOUND 3
+
+enum FirstPktAppIdDiscovered
+{
+    NO_APPID_FOUND = 0,
+    CLIENT_APPID_FOUND,
+    SERVICE_APPID_FOUND,
+    PAYLOAD_APPID_FOUND,
+    CLIENT_PAYLOAD_APPID_FOUND
+};
 
 class AppIdPatternMatchNode
 {
@@ -150,6 +162,8 @@ private:
         AppidSessionDirection direction);
     static bool do_host_port_based_discovery(snort::Packet* p, AppIdSession& asd,
         IpProtocol protocol, AppidSessionDirection direction, ThirdPartyAppIdContext* tp_appid_ctxt);
+    static bool detect_on_first_pkt(snort::Packet* p, AppIdSession& asd, IpProtocol protocol,
+        AppidSessionDirection direction, AppId& service_id, AppId& client_id, AppId& payload_id);
 };
 #endif
 

diff --git a/src/network_inspectors/appid/appid_http_event_handler.cc b/src/network_inspectors/appid/appid_http_event_handler.cc
index 8e95f51a3ba602bfaa832f6d7650f6c23aea826e..242288dc684dedd55a365ccfddc0e823a56ec9b3 100644 (file)
--- a/src/network_inspectors/appid/appid_http_event_handler.cc
+++ b/src/network_inspectors/appid/appid_http_event_handler.cc
@@ -70,6 +70,9 @@ void HttpEventHandler::handle(DataEvent& event, Flow* flow)
     if (!asd->get_session_flags(APPID_SESSION_DISCOVER_APP | APPID_SESSION_SPECIAL_MONITORED))
         return;
 
+    if (asd->get_session_flags(APPID_SESSION_FIRST_PKT_CACHE_MATCHED) and !asd->get_odp_ctxt().need_reinspection) 
+        return;
+
     const uint8_t* header_start;
     int32_t header_length;
     HttpEvent* http_event = (HttpEvent*)&event;

diff --git a/src/network_inspectors/appid/appid_session_api.h b/src/network_inspectors/appid/appid_session_api.h
index 27dea5dcfa17215b434bd33e62270625fcc732c4..6f0c62aec54eee7a45fe6f785b4f69a41d9e4a23 100644 (file)
--- a/src/network_inspectors/appid/appid_session_api.h
+++ b/src/network_inspectors/appid/appid_session_api.h
@@ -91,6 +91,7 @@ namespace snort
 #define APPID_SESSION_DECRYPT_MONITOR       (1ULL << 42)
 #define APPID_SESSION_HTTP_TUNNEL           (1ULL << 43)
 #define APPID_SESSION_OPPORTUNISTIC_TLS     (1ULL << 44)
+#define APPID_SESSION_FIRST_PKT_CACHE_MATCHED    (1ULL << 45)
 #define APPID_SESSION_IGNORE_ID_FLAGS \
     (APPID_SESSION_FUTURE_FLOW | \
     APPID_SESSION_NOT_A_SERVICE | \

diff --git a/src/network_inspectors/appid/host_port_app_cache.cc b/src/network_inspectors/appid/host_port_app_cache.cc
index 6c69946b04c26ccd731a8c79e0c177aa38b4024a..94b1a43bdb741cc746ab144a819986be85d59492 100644 (file)
--- a/src/network_inspectors/appid/host_port_app_cache.cc
+++ b/src/network_inspectors/appid/host_port_app_cache.cc
@@ -73,6 +73,48 @@ bool HostPortCache::add(const SnortConfig* sc, const SfIp* ip, uint16_t port, Ip
     return true;
 }
 
+
+HostAppIdsVal* HostPortCache::find_on_first_pkt(const SfIp* ip, uint16_t port, IpProtocol protocol,
+    const OdpContext& odp_ctxt)
+{
+    HostPortKey hk;
+
+    hk.ip = *ip;
+    hk.port = (odp_ctxt.allow_port_wildcard_host_cache)? 0 : port;
+    hk.proto = protocol;
+
+    std::map<HostPortKey, HostAppIdsVal>::iterator it;
+    it = cache_first.find(hk);
+    if (it != cache_first.end())
+        return &it->second;
+    else
+        return nullptr;
+}
+
+bool HostPortCache::add_host(const SnortConfig* sc, const SfIp* ip, uint16_t port, IpProtocol proto,
+    AppId protocol_appId, AppId client_appId, AppId web_appId, unsigned reinspect)
+{
+    HostPortKey hk;
+    HostAppIdsVal hv;
+
+    hk.ip = *ip;
+    AppIdInspector* inspector =
+        (AppIdInspector*)InspectorManager::get_inspector(MOD_NAME, false, sc);
+    assert(inspector);
+    const AppIdContext& ctxt = inspector->get_ctxt();
+    hk.port = (ctxt.get_odp_ctxt().allow_port_wildcard_host_cache)? 0 : port;
+    hk.proto = proto;
+
+    hv.protocol_appId = protocol_appId;
+    hv.client_appId = client_appId;
+    hv.web_appId = web_appId;
+    hv.reinspect = reinspect;
+
+    cache_first[ hk ] = hv;
+
+    return true;
+}
+
 void HostPortCache::dump()
 {
     for ( auto& kv : cache )

diff --git a/src/network_inspectors/appid/host_port_app_cache.h b/src/network_inspectors/appid/host_port_app_cache.h
index 4a469372dc53868769eeaf83c5240be485723fb9..608e97dcd22154fd06008ad945d65f52b9b49cdc 100644 (file)
--- a/src/network_inspectors/appid/host_port_app_cache.h
+++ b/src/network_inspectors/appid/host_port_app_cache.h
@@ -65,21 +65,35 @@ struct HostPortVal
     unsigned type;
 };
 
+struct HostAppIdsVal
+{
+    AppId protocol_appId;
+    AppId client_appId;
+    AppId web_appId;
+    unsigned reinspect;
+};
+
 class HostPortCache
 {
 public:
     HostPortVal* find(const snort::SfIp*, uint16_t port, IpProtocol, const OdpContext&);
     bool add(const snort::SnortConfig*, const snort::SfIp*, uint16_t port, IpProtocol,
         unsigned type, AppId);
+    
+    HostAppIdsVal* find_on_first_pkt(const snort::SfIp*, uint16_t port, IpProtocol, const OdpContext&);
+    bool add_host(const snort::SnortConfig*, const snort::SfIp*, uint16_t port, IpProtocol,
+        AppId, AppId, AppId, unsigned reinspect);
     void dump();
 
     ~HostPortCache()
     {
         cache.clear();
+        cache_first.clear();
     }
 
 private:
     std::map<HostPortKey, HostPortVal> cache;
+    std::map<HostPortKey, HostAppIdsVal> cache_first;
 };
 
 #endif

diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc
index 6c28d3dcbdaf7573fda2e7d29293b463d062b512..883fd53dcf09cbc0d3a95ebc50afacc39825538c 100644 (file)
--- a/src/network_inspectors/appid/lua_detector_api.cc
+++ b/src/network_inspectors/appid/lua_detector_api.cc
@@ -1211,6 +1211,48 @@ static int detector_add_ssl_cname_pattern(lua_State* L)
     return 0;
 }
 
+static int detector_add_host_first_pkt_application(lua_State* L)
+{
+    auto& ud = *UserData<LuaObject>::check(L, DETECTOR, 1);
+    // Verify detector user data and that we are NOT in packet context
+    ud->validate_lua_state(false);
+    if (!init(L))
+        return 0;
+
+    SfIp ip_address;
+    int index = 1;
+
+    /* Extract the three appIds and the reinspect flag */
+    uint32_t protocol_appid = lua_tointeger(L, ++index);
+    uint32_t client_appid = lua_tointeger(L, ++index);
+    uint32_t web_appid = lua_tointeger(L, ++index);
+    unsigned reinspect = lua_tointeger(L, ++index);
+
+    /* Extract IP, Port, protocol */
+    size_t ipaddr_size = 0;
+    const char* ip_str = lua_tolstring(L, ++index, &ipaddr_size);
+    if (!ip_str or !ipaddr_size or !convert_string_to_address(ip_str, &ip_address))
+    {
+        ErrorMessage("%s: Invalid IP address: %s\n",__func__, ip_str);
+        return 0;
+    }
+
+    unsigned port = lua_tointeger(L, ++index);
+    IpProtocol proto;
+    if (toipprotocol(L, ++index, proto))
+        return 0;
+
+    lua_getglobal(L, LUA_STATE_GLOBAL_SC_ID);
+    const SnortConfig* sc = *static_cast<const SnortConfig**>(lua_touserdata(L, -1));
+    lua_pop(L, 1);
+
+    if (!ud->get_odp_ctxt().host_first_pkt_add(
+        sc, &ip_address, (uint16_t)port, proto, protocol_appid, client_appid, web_appid, reinspect))
+        ErrorMessage("%s:Failed to backend call first pkt add\n",__func__);
+
+    return 0;
+}
+
 static int detector_add_host_port_application(lua_State* L)
 {
     auto& ud = *UserData<LuaObject>::check(L, DETECTOR, 1);
@@ -2734,6 +2776,7 @@ static const luaL_Reg detector_methods[] =
     { "addSipServer",             detector_add_sip_server },
     { "addSSLCnamePattern",       detector_add_ssl_cname_pattern },
     { "addSSHPattern",            detector_add_ssh_client_pattern},
+    { "addHostFirstPktApp",       detector_add_host_first_pkt_application },
     { "addHostPortApp",           detector_add_host_port_application },
     { "addHostPortAppDynamic",    detector_add_host_port_dynamic },
     { "addDNSHostPattern",        detector_add_dns_host_pattern },

diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc
index 418ef9c3447682b09ec71453e2e052f5e13e9af3..fd99e95f7554268cfb5b5f0417299d014aa9dcf7 100644 (file)
--- a/src/network_inspectors/appid/test/appid_discovery_test.cc
+++ b/src/network_inspectors/appid/test/appid_discovery_test.cc
@@ -280,6 +280,12 @@ HostPortVal* HostPortCache::find(const SfIp*, uint16_t, IpProtocol, const OdpCon
 {
     return nullptr;
 }
+
+HostAppIdsVal* HostPortCache::find_on_first_pkt(const SfIp*, uint16_t, IpProtocol, const OdpContext&)
+{
+    return nullptr;
+}
+
 void AppIdServiceState::check_reset(AppIdSession&, const SfIp*, uint16_t,
     int16_t, uint32_t) {}
 bool do_tp_discovery(ThirdPartyAppIdContext& , AppIdSession&, IpProtocol,