From: Lukasz Czarnik -X (lczarnik - SOFTSERVE INC at Cisco) Date: Thu, 27 Jul 2023 16:35:27 +0000 (+0000) Subject: Pull request #3898: appid: SSL regex patterns X-Git-Tag: 3.1.67.0~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ba362ad4e523a3fcaf3a2dfb69b00b1e25d5aafc;p=thirdparty%2Fsnort3.git Pull request #3898: appid: SSL regex patterns Merge in SNORT/snort3 from ~LCZARNIK/snort3:regex_ssl to master Squashed commit of the following: commit b75fe307c9e2f091dcdd2bd5ad669e8b22d95df5 Author: Lukasz Czarnik Date: Tue Jul 4 08:02:45 2023 -0400 appid: SSL regex pattern implementation --- diff --git a/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h b/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h index 5588a52f0..1370d6e63 100644 --- a/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h +++ b/src/network_inspectors/appid/client_plugins/test/client_plugins_mock.h @@ -28,10 +28,10 @@ void WarningMessage(const char*,...) { } // Stubs for search_tool.cc SearchTool::SearchTool(bool) { } SearchTool::~SearchTool() = default; -void SearchTool::add(const char*, unsigned, int, bool) { } -void SearchTool::add(const char*, unsigned, void*, bool) { } -void SearchTool::add(const uint8_t*, unsigned, int, bool) { } -void SearchTool::add(const uint8_t*, unsigned, void*, bool) { } +void SearchTool::add(const char*, unsigned, int, bool, bool) { } +void SearchTool::add(const char*, unsigned, void*, bool, bool) { } +void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) { } +void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) { } void SearchTool::prep() { } void SearchTool::reload() { } } diff --git a/src/network_inspectors/appid/detector_plugins/ssl_patterns.cc b/src/network_inspectors/appid/detector_plugins/ssl_patterns.cc index b8ae25dc1..fdadbc43a 100644 --- a/src/network_inspectors/appid/detector_plugins/ssl_patterns.cc +++ b/src/network_inspectors/appid/detector_plugins/ssl_patterns.cc @@ -42,7 +42,7 @@ static void create_matcher(SearchTool& matcher, SslPatternList* list, CnameCache continue; matcher.add(element->dpattern->pattern, - element->dpattern->pattern_size, element->dpattern, true); + element->dpattern->pattern_size, element->dpattern, true, element->dpattern->is_literal); (*pattern_index)++; } pattern_count = size; @@ -81,12 +81,33 @@ static int cname_pattern_match(void* id, void*, int match_end_pos, void* data, v } return 0; } +/* +Only patterns that match end of the payload AND +(match the start of the payload +or match after '.' +or patterns starting with '.') +are considered a match. */ +inline static bool ssl_pattern_validate_match(const MatchedSslPatterns * const mp, const uint8_t* data, int data_size) +{ + return mp->match_start_pos + mp->mpattern->pattern_size == data_size and + (mp->match_start_pos == 0 or + data[mp->match_start_pos-1] == '.' or + *mp->mpattern->pattern == '.'); +} + +inline static bool is_perfect_literal_match(const MatchedSslPatterns * const mp, int data_size) +{ + return mp->mpattern->is_literal and + (mp->match_start_pos + mp->mpattern->pattern_size == data_size) and + mp->match_start_pos == 0; + +} static bool scan_patterns(SearchTool& matcher, const uint8_t* data, size_t size, AppId& client_id, AppId& payload_id, bool is_cname_search) { MatchedSslPatterns* mp = nullptr; - SslPattern* best_match; + SslPattern* best_match = nullptr; if (is_cname_search) matcher.find_all((const char*)data, size, cname_pattern_match, false, &mp); @@ -95,26 +116,30 @@ static bool scan_patterns(SearchTool& matcher, const uint8_t* data, size_t size, if (!mp) return false; + + MatchedSslPatterns* tmp = mp; - best_match = nullptr; - while (mp) + while (tmp) { - /* Only patterns that match end of the payload AND - (match the start of the payload - or match after '.' - or patterns starting with '.' - ) are considered a match. */ - if (mp->match_start_pos + mp->mpattern->pattern_size == (int)size and - (mp->match_start_pos == 0 or - data[mp->match_start_pos-1] == '.' or - *mp->mpattern->pattern == '.')) + if (!tmp->mpattern->is_literal or ssl_pattern_validate_match(tmp, data, (int)size)) { - if (!best_match || - mp->mpattern->pattern_size > best_match->pattern_size) + if(is_perfect_literal_match(tmp, (int)size)) + { + best_match = tmp->mpattern; + break; + } + + if (!best_match or + tmp->mpattern->pattern_size > best_match->pattern_size) { - best_match = mp->mpattern; + best_match = tmp->mpattern; } } + tmp = tmp->next; + } + + while (mp) + { MatchedSslPatterns* tmpMp = mp; mp = mp->next; snort_free(tmpMp); @@ -159,7 +184,7 @@ static void free_patterns(SslPatternList*& list) } static void add_pattern(SslPatternList*& list, uint8_t* pattern_str, size_t - pattern_size, uint8_t type, AppId app_id, bool is_cname, CnameCache& set) + pattern_size, uint8_t type, AppId app_id, bool is_cname, bool is_literal, CnameCache& set) { SslPatternList* new_ssl_pattern; @@ -170,6 +195,7 @@ static void add_pattern(SslPatternList*& list, uint8_t* pattern_str, size_t new_ssl_pattern->dpattern->pattern = pattern_str; new_ssl_pattern->dpattern->pattern_size = pattern_size; new_ssl_pattern->dpattern->is_cname = is_cname; + new_ssl_pattern->dpattern->is_literal = is_literal; new_ssl_pattern->next = list; list = new_ssl_pattern; @@ -183,9 +209,9 @@ SslPatternMatchers::~SslPatternMatchers() free_patterns(cert_pattern_list); } -void SslPatternMatchers::add_cert_pattern(uint8_t* pattern_str, size_t pattern_size, uint8_t type, AppId app_id, bool is_cname) +void SslPatternMatchers::add_cert_pattern(uint8_t* pattern_str, size_t pattern_size, uint8_t type, AppId app_id, bool is_cname, bool is_literal) { - add_pattern(cert_pattern_list, pattern_str, pattern_size, type, app_id, is_cname, cert_pattern_set); + add_pattern(cert_pattern_list, pattern_str, pattern_size, type, app_id, is_cname, is_literal, cert_pattern_set); } void SslPatternMatchers::finalize_patterns() diff --git a/src/network_inspectors/appid/detector_plugins/ssl_patterns.h b/src/network_inspectors/appid/detector_plugins/ssl_patterns.h index 2590fede3..92a046175 100644 --- a/src/network_inspectors/appid/detector_plugins/ssl_patterns.h +++ b/src/network_inspectors/appid/detector_plugins/ssl_patterns.h @@ -33,6 +33,7 @@ struct SslPattern uint8_t* pattern; int pattern_size; bool is_cname; + bool is_literal; // is not regex pattern bool operator==(const SslPattern& v) const { @@ -68,7 +69,7 @@ class SslPatternMatchers { public: ~SslPatternMatchers(); - void add_cert_pattern(uint8_t*, size_t, uint8_t, AppId, bool); + void add_cert_pattern(uint8_t*, size_t, uint8_t, AppId, bool, bool = true); void finalize_patterns(); void reload_patterns(); unsigned get_pattern_count(); diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h index b82fda8b0..40863e12c 100644 --- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h +++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h @@ -53,10 +53,10 @@ class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; } // Stubs for search_tool.cc SearchTool::~SearchTool() = default; // LCOV_EXCL_START -void SearchTool::add(const char*, unsigned, int, bool) { } -void SearchTool::add(const char*, unsigned, void*, bool) { } -void SearchTool::add(const uint8_t*, unsigned, int, bool) { } -void SearchTool::add(const uint8_t*, unsigned, void*, bool) { } +void SearchTool::add(const char*, unsigned, int, bool, bool) { } +void SearchTool::add(const char*, unsigned, void*, bool, bool) { } +void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) { } +void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) { } // LCOV_EXCL_STOP void SearchTool::prep() { } diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc index 81288e145..889b8b20e 100644 --- a/src/network_inspectors/appid/lua_detector_api.cc +++ b/src/network_inspectors/appid/lua_detector_api.cc @@ -28,10 +28,14 @@ #include #include +#include "detection/fp_config.h" +#include "framework/mpse.h" #include "host_tracker/cache_allocator.cc" #include "host_tracker/host_cache.h" #include "log/messages.h" +#include "main/snort_config.h" #include "main/snort_types.h" +#include "managers/mpse_manager.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "trace/trace_api.h" @@ -1259,6 +1263,42 @@ static int detector_add_ssl_cert_pattern(lua_State* L) return 0; } +static int detector_add_ssl_cert_regex_pattern(lua_State* L) +{ + auto& ud = *UserData::check(L, DETECTOR, 1); + // Verify detector user data and that we are NOT in packet context + ud->validate_lua_state(false); + if (!init(L)) + return 0; + + const FastPatternConfig* const fp = SnortConfig::get_conf()->fast_pattern_config; + if (!MpseManager::is_regex_capable(fp->get_search_api())){ + ErrorMessage("appid: Regex patterns require usage of regex capable search engine like hyperscan in %s\n", + ud->get_detector()->get_name().c_str()); + return 0; + } + + int index = 1; + + uint8_t type = lua_tointeger(L, ++index); + AppId app_id = (AppId)lua_tointeger(L, ++index); + size_t pattern_size = 0; + const char* tmp_string = lua_tolstring(L, ++index, &pattern_size); + if (!tmp_string or !pattern_size) + { + ErrorMessage("appid: Invalid SSL Host regex pattern string in %s.\n", + ud->get_detector()->get_name().c_str()); + return 0; + } + + uint8_t* pattern_str = (uint8_t*)snort_strdup(tmp_string); + ud->get_odp_ctxt().get_ssl_matchers().add_cert_pattern(pattern_str, pattern_size, type, app_id, + false, false); + ud->get_odp_ctxt().get_app_info_mgr().set_app_info_active(app_id); + + return 0; +} + static int detector_add_ssl_cname_pattern(lua_State* L) { auto& ud = *UserData::check(L, DETECTOR, 1); @@ -1289,6 +1329,43 @@ static int detector_add_ssl_cname_pattern(lua_State* L) return 0; } +static int detector_add_ssl_cname_regex_pattern(lua_State* L) +{ + auto& ud = *UserData::check(L, DETECTOR, 1); + // Verify detector user data and that we are NOT in packet context + ud->validate_lua_state(false); + if (!init(L)) + return 0; + + const FastPatternConfig* const fp = SnortConfig::get_conf()->fast_pattern_config; + if (!MpseManager::is_regex_capable(fp->get_search_api())){ + ErrorMessage("appid: Regex patterns require usage of regex capable search engine like hyperscan in %s\n", + ud->get_detector()->get_name().c_str()); + return 0; + } + + int index = 1; + + uint8_t type = lua_tointeger(L, ++index); + AppId app_id = (AppId)lua_tointeger(L, ++index); + + size_t pattern_size = 0; + const char* tmp_string = lua_tolstring(L, ++index, &pattern_size); + if (!tmp_string or !pattern_size) + { + ErrorMessage("appid: Invalid SSL CN regex pattern string in %s.\n", + ud->get_detector()->get_name().c_str()); + return 0; + } + + uint8_t* pattern_str = (uint8_t*)snort_strdup(tmp_string); + ud->get_odp_ctxt().get_ssl_matchers().add_cert_pattern(pattern_str, pattern_size, type, app_id, + true, false); + ud->get_odp_ctxt().get_app_info_mgr().set_app_info_active(app_id); + + return 0; +} + // for Lua this looks something like: addDNSHostPattern(, '') static int detector_add_dns_host_pattern(lua_State* L) { @@ -3092,6 +3169,8 @@ static const luaL_Reg detector_methods[] = { "addContentTypePattern", detector_add_content_type_pattern }, { "addSSLCertPattern", detector_add_ssl_cert_pattern }, { "addSSLCnamePattern", detector_add_ssl_cname_pattern }, + { "addSSLCertRegexPattern", detector_add_ssl_cert_regex_pattern }, + { "addSSLCnameRegexPattern", detector_add_ssl_cname_regex_pattern }, { "addSipUserAgent", detector_add_sip_user_agent }, { "addSipServer", detector_add_sip_server }, { "addSSHPattern", detector_add_ssh_client_pattern}, diff --git a/src/network_inspectors/appid/service_plugins/test/service_alpn_patterns_mock.h b/src/network_inspectors/appid/service_plugins/test/service_alpn_patterns_mock.h index ffb01bf41..cd0638813 100644 --- a/src/network_inspectors/appid/service_plugins/test/service_alpn_patterns_mock.h +++ b/src/network_inspectors/appid/service_plugins/test/service_alpn_patterns_mock.h @@ -28,10 +28,10 @@ void WarningMessage(const char*,...) { } // Stubs for search_tool.cc SearchTool::SearchTool(bool) { } SearchTool::~SearchTool() = default; -void SearchTool::add(const char*, unsigned, int, bool) { } -void SearchTool::add(const char*, unsigned, void*, bool) { } -void SearchTool::add(const uint8_t*, unsigned, int, bool) { } -void SearchTool::add(const uint8_t*, unsigned, void*, bool) { } +void SearchTool::add(const char*, unsigned, int, bool, bool) { } +void SearchTool::add(const char*, unsigned, void*, bool, bool) { } +void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) { } +void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) { } void SearchTool::prep() { } void SearchTool::reload() { } } diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc index 9910b1871..33442be86 100644 --- a/src/network_inspectors/appid/test/appid_discovery_test.cc +++ b/src/network_inspectors/appid/test/appid_discovery_test.cc @@ -103,10 +103,10 @@ time_t packet_time() { return std::time(nullptr); } // Stubs for search_tool SearchTool::SearchTool(bool) {} SearchTool::~SearchTool() = default; -void SearchTool::add(const char*, unsigned, int, bool) {} -void SearchTool::add(const char*, unsigned, void*, bool) {} -void SearchTool::add(const uint8_t*, unsigned, int, bool) {} -void SearchTool::add(const uint8_t*, unsigned, void*, bool) {} +void SearchTool::add(const char*, unsigned, int, bool, bool) {} +void SearchTool::add(const char*, unsigned, void*, bool, bool) {} +void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) {} +void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) {} // Mocks for ip namespace ip diff --git a/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc b/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc index 99ccde8ee..c6da0e684 100644 --- a/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc +++ b/src/network_inspectors/rna/test/rna_ua_fp_processor_test.cc @@ -72,7 +72,7 @@ namespace snort s_prep_data.clear(); } - void SearchTool::add(const char* s, unsigned n, void*, bool) + void SearchTool::add(const char* s, unsigned n, void*, bool, bool) { s_count++; s_data.append(s, n); diff --git a/src/network_inspectors/rna/test/ua_fp_stubs.cc b/src/network_inspectors/rna/test/ua_fp_stubs.cc index 1f15f6da1..732479fd0 100644 --- a/src/network_inspectors/rna/test/ua_fp_stubs.cc +++ b/src/network_inspectors/rna/test/ua_fp_stubs.cc @@ -26,9 +26,9 @@ namespace snort { -void SearchTool::add(const char*, unsigned, int, bool) { } -void SearchTool::add(const uint8_t*, unsigned, int, bool) { } -void SearchTool::add(const uint8_t*, unsigned, void*, bool) { } +void SearchTool::add(const char*, unsigned, int, bool, bool) { } +void SearchTool::add(const uint8_t*, unsigned, int, bool, bool) { } +void SearchTool::add(const uint8_t*, unsigned, void*, bool, bool) { } void SearchTool::reload() { } diff --git a/src/search_engines/search_tool.cc b/src/search_engines/search_tool.cc index 74181250c..613b8ae70 100644 --- a/src/search_engines/search_tool.cc +++ b/src/search_engines/search_tool.cc @@ -54,18 +54,18 @@ SearchTool::~SearchTool() delete mpsegrp; } -void SearchTool::add(const char* pat, unsigned len, int id, bool no_case) -{ add((const uint8_t*)pat, len, id, no_case); } +void SearchTool::add(const char* pat, unsigned len, int id, bool no_case, bool literal) +{ add((const uint8_t*)pat, len, id, no_case, literal); } -void SearchTool::add(const char* pat, unsigned len, void* id, bool no_case) -{ add((const uint8_t*)pat, len, id, no_case); } +void SearchTool::add(const char* pat, unsigned len, void* id, bool no_case, bool literal) +{ add((const uint8_t*)pat, len, id, no_case, literal); } -void SearchTool::add(const uint8_t* pat, unsigned len, int id, bool no_case) -{ add(pat, len, (void*)(long)id, no_case); } +void SearchTool::add(const uint8_t* pat, unsigned len, int id, bool no_case, bool literal) +{ add(pat, len, (void*)(long)id, no_case, literal); } -void SearchTool::add(const uint8_t* pat, unsigned len, void* id, bool no_case) +void SearchTool::add(const uint8_t* pat, unsigned len, void* id, bool no_case, bool literal) { - Mpse::PatternDescriptor desc(no_case, false, true, multi_match); + Mpse::PatternDescriptor desc(no_case, false, literal, multi_match); if ( mpsegrp->normal_mpse ) mpsegrp->normal_mpse->add_pattern(pat, len, desc, id); diff --git a/src/search_engines/search_tool.h b/src/search_engines/search_tool.h index 9c4a2da04..1fb6a7dce 100644 --- a/src/search_engines/search_tool.h +++ b/src/search_engines/search_tool.h @@ -40,11 +40,11 @@ public: SearchTool(bool multi_match = true); ~SearchTool(); - void add(const char* pattern, unsigned len, int s_id, bool no_case = true); - void add(const char* pattern, unsigned len, void* s_context, bool no_case = true); + void add(const char* pattern, unsigned len, int s_id, bool no_case = true, bool literal = true); + void add(const char* pattern, unsigned len, void* s_context, bool no_case = true, bool literal = true); - void add(const uint8_t* pattern, unsigned len, int s_id, bool no_case = true); - void add(const uint8_t* pattern, unsigned len, void* s_context, bool no_case = true); + void add(const uint8_t* pattern, unsigned len, int s_id, bool no_case = true, bool literal = true); + void add(const uint8_t* pattern, unsigned len, void* s_context, bool no_case = true, bool literal = true); void prep(); void reload();