Pull request #3898: appid: SSL regex patterns

Merge in SNORT/snort3 from ~LCZARNIK/snort3:regex_ssl to master

Squashed commit of the following:

commit b75fe307c9e2f091dcdd2bd5ad669e8b22d95df5
Author: Lukasz Czarnik <lczarnik@cisco.com>
Date:   Tue Jul 4 08:02:45 2023 -0400

    appid: SSL regex pattern implementation

diff --git a/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h b/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h
index 5588a52f082b031b28e3c2a6884bcedbff34cf94..1370d6e63f27f54f1fb1234da8ac7111fb6886a3 100644 (file)
--- a/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h
+++ b/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h
@@ -28,10 +28,10 @@ void WarningMessage(const char*,...) { }
 // Stubs for search_tool.cc
 SearchTool::SearchTool(bool) { }
 SearchTool::~SearchTool() = default;
-void SearchTool::add(const char*, unsigned, int, bool) { }
-void SearchTool::add(const char*, unsigned, void*, bool) { }
-void SearchTool::add(const uint8_t*, unsigned, int, bool) { }
-void SearchTool::add(const uint8_t*, unsigned, void*, bool) { }
+void SearchTool::add(const char*, unsigned, int, bool, bool) { }
+void SearchTool::add(const char*, unsigned, void*, bool, bool) { }
+void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) { }
+void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) { }
 void SearchTool::prep() { }
 void SearchTool::reload() { }
 }

diff --git a/src/network_inspectors/appid/detector_plugins/ssl_patterns.cc b/src/network_inspectors/appid/detector_plugins/ssl_patterns.cc
index b8ae25dc130118dd71a4b352d1bdb82838556266..fdadbc43a27e21a616543e9d1ad5b83fd6624521 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/ssl_patterns.cc
+++ b/src/network_inspectors/appid/detector_plugins/ssl_patterns.cc
@@ -42,7 +42,7 @@ static void create_matcher(SearchTool& matcher, SslPatternList* list, CnameCache
             continue;
 
         matcher.add(element->dpattern->pattern,
-            element->dpattern->pattern_size, element->dpattern, true);
+            element->dpattern->pattern_size, element->dpattern, true, element->dpattern->is_literal);
         (*pattern_index)++;
     }
     pattern_count = size;
@@ -81,12 +81,33 @@ static int cname_pattern_match(void* id, void*, int match_end_pos, void* data, v
     }
     return 0;
 }
+/*  
+Only patterns that match end of the payload AND
+(match the start of the payload
+or match after '.'
+or patterns starting with '.')
+are considered a match. */
+inline static bool ssl_pattern_validate_match(const MatchedSslPatterns * const mp, const uint8_t* data, int data_size)
+{
+    return mp->match_start_pos + mp->mpattern->pattern_size == data_size and
+            (mp->match_start_pos == 0 or
+            data[mp->match_start_pos-1] == '.' or
+            *mp->mpattern->pattern == '.');
+}
+
+inline static bool is_perfect_literal_match(const MatchedSslPatterns * const mp, int data_size)
+{
+    return mp->mpattern->is_literal and
+            (mp->match_start_pos + mp->mpattern->pattern_size == data_size) and
+            mp->match_start_pos == 0;
+
+}
 
 static bool scan_patterns(SearchTool& matcher, const uint8_t* data, size_t size,
     AppId& client_id, AppId& payload_id, bool is_cname_search)
 {
     MatchedSslPatterns* mp = nullptr;
-    SslPattern* best_match;
+    SslPattern* best_match = nullptr;
 
     if (is_cname_search)
         matcher.find_all((const char*)data, size, cname_pattern_match, false, &mp);
@@ -95,26 +116,30 @@ static bool scan_patterns(SearchTool& matcher, const uint8_t* data, size_t size,
 
     if (!mp)
         return false;
+    
+    MatchedSslPatterns* tmp = mp;
 
-    best_match = nullptr;
-    while (mp)
+    while (tmp)
     {
-        /*  Only patterns that match end of the payload AND
-            (match the start of the payload
-            or match after '.'
-            or patterns starting with '.'
-            ) are considered a match. */
-        if (mp->match_start_pos + mp->mpattern->pattern_size == (int)size and
-            (mp->match_start_pos == 0 or
-            data[mp->match_start_pos-1] == '.' or
-            *mp->mpattern->pattern == '.'))
+        if (!tmp->mpattern->is_literal or ssl_pattern_validate_match(tmp, data, (int)size))
         {
-            if (!best_match ||
-                mp->mpattern->pattern_size > best_match->pattern_size)
+            if(is_perfect_literal_match(tmp, (int)size))
+            {
+                best_match = tmp->mpattern;
+                break;
+            }
+
+            if (!best_match or
+                    tmp->mpattern->pattern_size > best_match->pattern_size)
             {
-                best_match = mp->mpattern;
+                best_match = tmp->mpattern;
             }
         }
+        tmp = tmp->next;
+    }
+
+    while (mp)
+    {
         MatchedSslPatterns* tmpMp = mp;
         mp = mp->next;
         snort_free(tmpMp);
@@ -159,7 +184,7 @@ static void free_patterns(SslPatternList*& list)
 }
 
 static void add_pattern(SslPatternList*& list, uint8_t* pattern_str, size_t
-    pattern_size, uint8_t type, AppId app_id, bool is_cname, CnameCache& set)
+    pattern_size, uint8_t type, AppId app_id, bool is_cname, bool is_literal, CnameCache& set)
 {
     SslPatternList* new_ssl_pattern;
 
@@ -170,6 +195,7 @@ static void add_pattern(SslPatternList*& list, uint8_t* pattern_str, size_t
     new_ssl_pattern->dpattern->pattern = pattern_str;
     new_ssl_pattern->dpattern->pattern_size = pattern_size;
     new_ssl_pattern->dpattern->is_cname = is_cname;
+    new_ssl_pattern->dpattern->is_literal = is_literal;
 
     new_ssl_pattern->next = list;
     list = new_ssl_pattern;
@@ -183,9 +209,9 @@ SslPatternMatchers::~SslPatternMatchers()
     free_patterns(cert_pattern_list);
 }
 
-void SslPatternMatchers::add_cert_pattern(uint8_t* pattern_str, size_t pattern_size, uint8_t type, AppId app_id, bool is_cname)
+void SslPatternMatchers::add_cert_pattern(uint8_t* pattern_str, size_t pattern_size, uint8_t type, AppId app_id, bool is_cname, bool is_literal)
 {
-    add_pattern(cert_pattern_list, pattern_str, pattern_size, type, app_id, is_cname, cert_pattern_set);
+    add_pattern(cert_pattern_list, pattern_str, pattern_size, type, app_id, is_cname, is_literal, cert_pattern_set);
 }
 
 void SslPatternMatchers::finalize_patterns()

diff --git a/src/network_inspectors/appid/detector_plugins/ssl_patterns.h b/src/network_inspectors/appid/detector_plugins/ssl_patterns.h
index 2590fede30f6a4241a79003ff1e71dfd89c3ebcc..92a046175e89bda5d50fa7c815defa139e9f75ff 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/ssl_patterns.h
+++ b/src/network_inspectors/appid/detector_plugins/ssl_patterns.h
@@ -33,6 +33,7 @@ struct SslPattern
     uint8_t* pattern;
     int pattern_size;
     bool is_cname;
+    bool is_literal; // is not regex pattern
 
     bool operator==(const SslPattern& v) const
     {
@@ -68,7 +69,7 @@ class SslPatternMatchers
 {
 public:
     ~SslPatternMatchers();
-    void add_cert_pattern(uint8_t*, size_t, uint8_t, AppId, bool);
+    void add_cert_pattern(uint8_t*, size_t, uint8_t, AppId, bool, bool = true);
     void finalize_patterns();
     void reload_patterns();
     unsigned get_pattern_count();

diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
index b82fda8b01fa972f8651e5810790f651413909a9..40863e12cf9742bc7d0722aa29482d15794db077 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
+++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
@@ -53,10 +53,10 @@ class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; }
 // Stubs for search_tool.cc
 SearchTool::~SearchTool() = default;
 // LCOV_EXCL_START
-void SearchTool::add(const char*, unsigned, int, bool) { }
-void SearchTool::add(const char*, unsigned, void*, bool) { }
-void SearchTool::add(const uint8_t*, unsigned, int, bool) { }
-void SearchTool::add(const uint8_t*, unsigned, void*, bool) { }
+void SearchTool::add(const char*, unsigned, int, bool, bool) { }
+void SearchTool::add(const char*, unsigned, void*, bool, bool) { }
+void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) { }
+void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) { }
 // LCOV_EXCL_STOP
 void SearchTool::prep() { }
 

diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc
index 81288e14545f2087cdfb7ccfe74eaac3579dc589..889b8b20e5c543de990d73ee09f8e02ccd45a0cb 100644 (file)
--- a/src/network_inspectors/appid/lua_detector_api.cc
+++ b/src/network_inspectors/appid/lua_detector_api.cc
@@ -28,10 +28,14 @@
 #include <pcre.h>
 #include <unordered_map>
 
+#include "detection/fp_config.h"
+#include "framework/mpse.h"
 #include "host_tracker/cache_allocator.cc"
 #include "host_tracker/host_cache.h"
 #include "log/messages.h"
+#include "main/snort_config.h"
 #include "main/snort_types.h"
+#include "managers/mpse_manager.h"
 #include "profiler/profiler.h"
 #include "protocols/packet.h"
 #include "trace/trace_api.h"
@@ -1259,6 +1263,42 @@ static int detector_add_ssl_cert_pattern(lua_State* L)
     return 0;
 }
 
+static int detector_add_ssl_cert_regex_pattern(lua_State* L)
+{
+    auto& ud = *UserData<LuaObject>::check(L, DETECTOR, 1);
+    // Verify detector user data and that we are NOT in packet context
+    ud->validate_lua_state(false);
+    if (!init(L))
+        return 0;
+
+    const FastPatternConfig* const fp = SnortConfig::get_conf()->fast_pattern_config;
+    if (!MpseManager::is_regex_capable(fp->get_search_api())){
+        ErrorMessage("appid: Regex patterns require usage of regex capable search engine like hyperscan in %s\n", 
+            ud->get_detector()->get_name().c_str());
+            return 0;
+    }
+
+    int index = 1;
+
+    uint8_t type = lua_tointeger(L, ++index);
+    AppId app_id = (AppId)lua_tointeger(L, ++index);
+    size_t pattern_size = 0;
+    const char* tmp_string = lua_tolstring(L, ++index, &pattern_size);
+    if (!tmp_string or !pattern_size)
+    {
+        ErrorMessage("appid: Invalid SSL Host regex pattern string in %s.\n", 
+            ud->get_detector()->get_name().c_str());
+        return 0;
+    }
+
+    uint8_t* pattern_str = (uint8_t*)snort_strdup(tmp_string);
+    ud->get_odp_ctxt().get_ssl_matchers().add_cert_pattern(pattern_str, pattern_size, type, app_id,
+        false, false);
+    ud->get_odp_ctxt().get_app_info_mgr().set_app_info_active(app_id);
+
+    return 0;
+}
+
 static int detector_add_ssl_cname_pattern(lua_State* L)
 {
     auto& ud = *UserData<LuaObject>::check(L, DETECTOR, 1);
@@ -1289,6 +1329,43 @@ static int detector_add_ssl_cname_pattern(lua_State* L)
     return 0;
 }
 
+static int detector_add_ssl_cname_regex_pattern(lua_State* L)
+{
+    auto& ud = *UserData<LuaObject>::check(L, DETECTOR, 1);
+    // Verify detector user data and that we are NOT in packet context
+    ud->validate_lua_state(false);
+    if (!init(L))
+        return 0;
+
+    const FastPatternConfig* const fp = SnortConfig::get_conf()->fast_pattern_config;
+    if (!MpseManager::is_regex_capable(fp->get_search_api())){
+        ErrorMessage("appid: Regex patterns require usage of regex capable search engine like hyperscan in %s\n", 
+            ud->get_detector()->get_name().c_str());
+            return 0;
+    }
+
+    int index = 1;
+
+    uint8_t type = lua_tointeger(L, ++index);
+    AppId app_id = (AppId)lua_tointeger(L, ++index);
+
+    size_t pattern_size = 0;
+    const char* tmp_string = lua_tolstring(L, ++index, &pattern_size);
+    if (!tmp_string or !pattern_size)
+    {
+        ErrorMessage("appid: Invalid SSL CN regex pattern string in %s.\n",
+            ud->get_detector()->get_name().c_str());
+        return 0;
+    }
+
+    uint8_t* pattern_str = (uint8_t*)snort_strdup(tmp_string);
+    ud->get_odp_ctxt().get_ssl_matchers().add_cert_pattern(pattern_str, pattern_size, type, app_id,
+        true, false);
+    ud->get_odp_ctxt().get_app_info_mgr().set_app_info_active(app_id);
+
+    return 0;
+}
+
 // for Lua this looks something like: addDNSHostPattern(<appId>, '<pattern string>')
 static int detector_add_dns_host_pattern(lua_State* L)
 {
@@ -3092,6 +3169,8 @@ static const luaL_Reg detector_methods[] =
     { "addContentTypePattern",    detector_add_content_type_pattern },
     { "addSSLCertPattern",        detector_add_ssl_cert_pattern },
     { "addSSLCnamePattern",       detector_add_ssl_cname_pattern },
+    { "addSSLCertRegexPattern",   detector_add_ssl_cert_regex_pattern },
+    { "addSSLCnameRegexPattern",  detector_add_ssl_cname_regex_pattern },
     { "addSipUserAgent",          detector_add_sip_user_agent },
     { "addSipServer",             detector_add_sip_server },
     { "addSSHPattern",            detector_add_ssh_client_pattern},

diff --git a/src/network_inspectors/appid/service_plugins/test/service_alpn_patterns_mock.h b/src/network_inspectors/appid/service_plugins/test/service_alpn_patterns_mock.h
index ffb01bf41ddc5a7273f878ecd67fffcf4dd5d4d7..cd0638813989c8e4cdaa84c9ea9ab76852dfd19b 100644 (file)
--- a/src/network_inspectors/appid/service_plugins/test/service_alpn_patterns_mock.h
+++ b/src/network_inspectors/appid/service_plugins/test/service_alpn_patterns_mock.h
@@ -28,10 +28,10 @@ void WarningMessage(const char*,...) { }
 // Stubs for search_tool.cc
 SearchTool::SearchTool(bool) { }
 SearchTool::~SearchTool() = default;
-void SearchTool::add(const char*, unsigned, int, bool) { }
-void SearchTool::add(const char*, unsigned, void*, bool) { }
-void SearchTool::add(const uint8_t*, unsigned, int, bool) { }
-void SearchTool::add(const uint8_t*, unsigned, void*, bool) { }
+void SearchTool::add(const char*, unsigned, int, bool, bool) { }
+void SearchTool::add(const char*, unsigned, void*, bool, bool) { }
+void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) { }
+void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) { }
 void SearchTool::prep() { }
 void SearchTool::reload() { }
 }

diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc
index 9910b18719d49f1ea4d8e8b2fcb2a23df06edece..33442be8608a8af1ec4732887917982eef80e2f8 100644 (file)
--- a/src/network_inspectors/appid/test/appid_discovery_test.cc
+++ b/src/network_inspectors/appid/test/appid_discovery_test.cc
@@ -103,10 +103,10 @@ time_t packet_time() { return std::time(nullptr); }
 // Stubs for search_tool
 SearchTool::SearchTool(bool) {}
 SearchTool::~SearchTool() = default;
-void SearchTool::add(const char*, unsigned, int, bool) {}
-void SearchTool::add(const char*, unsigned, void*, bool) {}
-void SearchTool::add(const uint8_t*, unsigned, int, bool) {}
-void SearchTool::add(const uint8_t*, unsigned, void*, bool) {}
+void SearchTool::add(const char*, unsigned, int, bool, bool) {}
+void SearchTool::add(const char*, unsigned, void*, bool, bool) {}
+void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) {}
+void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) {}
 
 // Mocks for ip
 namespace ip

diff --git a/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc b/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc
index 99ccde8eebb165a5558e2ec5fea8581dd36776b2..c6da0e684e182252aa7c6dc464b11b08e004d219 100644 (file)
--- a/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc
+++ b/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc
@@ -72,7 +72,7 @@ namespace snort
         s_prep_data.clear();
     }
 
-    void SearchTool::add(const char* s, unsigned n, void*, bool)
+    void SearchTool::add(const char* s, unsigned n, void*, bool, bool)
     {
         s_count++;
         s_data.append(s, n);

diff --git a/src/network_inspectors/rna/test/ua_fp_stubs.cc b/src/network_inspectors/rna/test/ua_fp_stubs.cc
index 1f15f6da1509d9d395bc2ee6287fab6ed0b7eca2..732479fd0c3713bde8b598414c15b41c01442066 100644 (file)
--- a/src/network_inspectors/rna/test/ua_fp_stubs.cc
+++ b/src/network_inspectors/rna/test/ua_fp_stubs.cc
@@ -26,9 +26,9 @@
 
 namespace snort
 {
-void SearchTool::add(const char*, unsigned, int, bool) { }
-void SearchTool::add(const uint8_t*, unsigned, int, bool) { }
-void SearchTool::add(const uint8_t*, unsigned, void*, bool) { }
+void SearchTool::add(const char*, unsigned, int, bool, bool) { }
+void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) { }
+void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) { }
 
 void SearchTool::reload() { }
 

diff --git a/src/search_engines/search_tool.cc b/src/search_engines/search_tool.cc
index 74181250c1828c18056d3fcc285faf0d54de35ac..613b8ae70276b190693f03482eea585d2267686a 100644 (file)
--- a/src/search_engines/search_tool.cc
+++ b/src/search_engines/search_tool.cc
@@ -54,18 +54,18 @@ SearchTool::~SearchTool()
     delete mpsegrp;
 }
 
-void SearchTool::add(const char* pat, unsigned len, int id, bool no_case)
-{ add((const uint8_t*)pat, len, id, no_case); }
+void SearchTool::add(const char* pat, unsigned len, int id, bool no_case, bool literal)
+{ add((const uint8_t*)pat, len, id, no_case, literal); }
 
-void SearchTool::add(const char* pat, unsigned len, void* id, bool no_case)
-{ add((const uint8_t*)pat, len, id, no_case); }
+void SearchTool::add(const char* pat, unsigned len, void* id, bool no_case, bool literal)
+{ add((const uint8_t*)pat, len, id, no_case, literal); }
 
-void SearchTool::add(const uint8_t* pat, unsigned len, int id, bool no_case)
-{ add(pat, len, (void*)(long)id, no_case); }
+void SearchTool::add(const uint8_t* pat, unsigned len, int id, bool no_case, bool literal)
+{ add(pat, len, (void*)(long)id, no_case, literal); }
 
-void SearchTool::add(const uint8_t* pat, unsigned len, void* id, bool no_case)
+void SearchTool::add(const uint8_t* pat, unsigned len, void* id, bool no_case, bool literal)
 {
-    Mpse::PatternDescriptor desc(no_case, false, true, multi_match);
+    Mpse::PatternDescriptor desc(no_case, false, literal, multi_match);
 
     if ( mpsegrp->normal_mpse )
         mpsegrp->normal_mpse->add_pattern(pat, len, desc, id);

diff --git a/src/search_engines/search_tool.h b/src/search_engines/search_tool.h
index 9c4a2da04fa5812f6307e320162cc813126cea63..1fb6a7dcee9b8645f783b375292736225587ecd8 100644 (file)
--- a/src/search_engines/search_tool.h
+++ b/src/search_engines/search_tool.h
@@ -40,11 +40,11 @@ public:
     SearchTool(bool multi_match = true);
     ~SearchTool();
 
-    void add(const char* pattern, unsigned len, int s_id, bool no_case = true);
-    void add(const char* pattern, unsigned len, void* s_context, bool no_case = true);
+    void add(const char* pattern, unsigned len, int s_id, bool no_case = true, bool literal = true);
+    void add(const char* pattern, unsigned len, void* s_context, bool no_case = true, bool literal = true);
 
-    void add(const uint8_t* pattern, unsigned len, int s_id, bool no_case = true);
-    void add(const uint8_t* pattern, unsigned len, void* s_context, bool no_case = true);
+    void add(const uint8_t* pattern, unsigned len, int s_id, bool no_case = true, bool literal = true);
+    void add(const uint8_t* pattern, unsigned len, void* s_context, bool no_case = true, bool literal = true);
 
     void prep();
     void reload();