From: Lukasz Czarnik -X (lczarnik - SOFTSERVE INC at Cisco) Date: Fri, 26 Apr 2024 12:55:18 +0000 (+0000) Subject: Pull request #4287: appid: add http url regex patterns X-Git-Tag: 3.2.1.0~17 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9fa15413bceb155abfcb4a4c52776071ea317acf;p=thirdparty%2Fsnort3.git Pull request #4287: appid: add http url regex patterns Merge in SNORT/snort3 from ~LCZARNIK/snort3:http_regex to master Squashed commit of the following: commit 1eabb9424b2930e9f74a90b473acee322362e8f6 Author: Lukasz Czarnik Date: Mon Apr 15 14:50:37 2024 -0400 appid: add http url regex patterns --- diff --git a/src/network_inspectors/appid/appid_utils/sf_mlmp.cc b/src/network_inspectors/appid/appid_utils/sf_mlmp.cc index c82f2a848..441eefdf4 100644 --- a/src/network_inspectors/appid/appid_utils/sf_mlmp.cc +++ b/src/network_inspectors/appid/appid_utils/sf_mlmp.cc @@ -76,7 +76,7 @@ struct tMatchedPatternList }; static int compareMlmpPatterns(const void* p1, const void* p2); -static int createTreesRecusively(tMlmpTree* root); +static int createTreesRecursively(tMlmpTree* root); static void destroyTreesRecursively(tMlmpTree* root); static int addPatternRecursively(tMlmpTree* root, const tMlmpPattern* inputPatternList, void* metaData, uint32_t level); @@ -108,7 +108,7 @@ int mlmpProcessPatterns(tMlmpTree* root) { int rvalue; - rvalue = createTreesRecusively(root); + rvalue = createTreesRecursively(root); if (rvalue) destroyTreesRecursively(root); return rvalue; @@ -204,7 +204,7 @@ static int compareMlmpPatterns(const void* p1, const void* p2) /*pattern trees are not freed on error because in case of error, caller should call detroyTreesRecursively. */ -static int createTreesRecusively(tMlmpTree* rootNode) +static int createTreesRecursively(tMlmpTree* rootNode) { SearchTool* patternMatcher; tPatternPrimaryNode* primaryPatternNode; @@ -220,7 +220,7 @@ static int createTreesRecusively(tMlmpTree* rootNode) /*recursion into next lower level */ if (primaryPatternNode->nextLevelMatcher) { - if (createTreesRecusively(primaryPatternNode->nextLevelMatcher)) + if (createTreesRecursively(primaryPatternNode->nextLevelMatcher)) return -1; } @@ -229,7 +229,7 @@ static int createTreesRecusively(tMlmpTree* rootNode) ddPatternNode = ddPatternNode->nextPattern) { patternMatcher->add(ddPatternNode->pattern.pattern, - ddPatternNode->pattern.patternSize, ddPatternNode, true); + ddPatternNode->pattern.patternSize, ddPatternNode, true, ddPatternNode->pattern.is_literal); } } @@ -295,32 +295,33 @@ static tPatternNode* patternSelector(const tMatchedPatternList* patternMatchList tmpList; tmpList = tmpList->next) { - if (tmpList->patternNode->patternId != patternId) + const tPatternNode& node = *tmpList->patternNode; + if (node.patternId != patternId) { /*first pattern */ /*skip incomplete pattern */ - if (tmpList->patternNode->partNum != 1) + if (node.partNum != 1) continue; /*new pattern started */ - patternId = tmpList->patternNode->patternId; + patternId = node.patternId; currentPrimaryNode = tmpList->patternNode; partNum = 0; patternSize = 0; } - if (tmpList->patternNode->partNum == (partNum+1)) + if (node.partNum == (partNum+1)) { partNum++; - patternSize += tmpList->patternNode->pattern.patternSize; + patternSize += node.pattern.patternSize; } - if (tmpList->patternNode->partTotal != partNum) + if (node.partTotal != partNum) continue; /*backward compatibility */ - if ((tmpList->patternNode->partTotal == 1) + if (node.pattern.is_literal && (node.partTotal == 1) && domain && !match_is_domain_pattern(tmpList, payload)) continue; @@ -485,6 +486,7 @@ static int addPatternRecursively(tMlmpTree* rootNode, const tMlmpPattern* inputP tmpPrimaryNode->patternNode.pattern.pattern = patterns->pattern; tmpPrimaryNode->patternNode.pattern.patternSize = patterns->patternSize; tmpPrimaryNode->patternNode.pattern.level = patterns->level; + tmpPrimaryNode->patternNode.pattern.is_literal = patterns->is_literal; tmpPrimaryNode->patternNode.partNum = 1; tmpPrimaryNode->patternNode.partTotal = partTotal; tmpPrimaryNode->patternNode.patternId = patternId; @@ -512,6 +514,7 @@ static int addPatternRecursively(tMlmpTree* rootNode, const tMlmpPattern* inputP newNode->pattern.pattern = patterns->pattern; newNode->pattern.patternSize = patterns->patternSize; newNode->pattern.level = patterns->level; + newNode->pattern.is_literal = patterns->is_literal; newNode->partNum = partNum; newNode->partTotal = partTotal; newNode->patternId = patternId; diff --git a/src/network_inspectors/appid/appid_utils/sf_mlmp.h b/src/network_inspectors/appid/appid_utils/sf_mlmp.h index 295e3b8b7..bc3104630 100644 --- a/src/network_inspectors/appid/appid_utils/sf_mlmp.h +++ b/src/network_inspectors/appid/appid_utils/sf_mlmp.h @@ -41,6 +41,7 @@ struct tMlmpPattern /**level of pattern. It should start from 0.*/ uint32_t level; + bool is_literal; }; struct tMlmpTree; diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc b/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc index bab88b974..f8b939388 100644 --- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc +++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc @@ -491,7 +491,7 @@ int HttpPatternMatchers::add_mlmp_pattern(tMlmpTree* matcher, DetectorHTTPPatter tMlmpPattern patterns[PATTERN_PART_MAX]; int num_patterns = parse_multiple_http_patterns((const char*)pattern.pattern, patterns, - PATTERN_PART_MAX, 0); + PATTERN_PART_MAX, 0, true); patterns[num_patterns].pattern = nullptr; return mlmpAddPattern(matcher, patterns, detector); } @@ -531,10 +531,10 @@ int HttpPatternMatchers::add_mlmp_pattern(tMlmpTree* matcher, DetectorAppUrlPatt tMlmpPattern patterns[PATTERN_PART_MAX]; int num_patterns = parse_multiple_http_patterns((const char*)pattern.patterns.host.pattern, - patterns, PATTERN_PART_MAX, 0); + patterns, PATTERN_PART_MAX, 0, pattern.is_literal); if (pattern.patterns.path.pattern) num_patterns += parse_multiple_http_patterns((const char*)pattern.patterns.path.pattern, - patterns + num_patterns, PATTERN_PART_MAX - num_patterns, 1); + patterns + num_patterns, PATTERN_PART_MAX - num_patterns, 1, pattern.is_literal); patterns[num_patterns].pattern = nullptr; return mlmpAddPattern(matcher, patterns, detector); @@ -1655,7 +1655,7 @@ void HttpPatternMatchers::get_server_vendor_version(const char* data, int len, c } uint32_t HttpPatternMatchers::parse_multiple_http_patterns(const char* pattern, - tMlmpPattern* parts, uint32_t numPartLimit, int level) + tMlmpPattern* parts, uint32_t numPartLimit, int level, bool is_literal) { uint32_t partNum = 0; @@ -1679,6 +1679,7 @@ uint32_t HttpPatternMatchers::parse_multiple_http_patterns(const char* pattern, tmp = nullptr; } parts[partNum].level = level; + parts[partNum].is_literal = is_literal; if ( !parts[partNum].pattern ) { diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h index c42f1fde2..8fe6b1b0a 100644 --- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h +++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h @@ -76,6 +76,7 @@ struct DetectorAppUrlPattern } patterns; UrlUserData userData; + bool is_literal; }; // These values are used in Lua code as raw numbers. Do NOT reassign new values. @@ -309,7 +310,7 @@ public: void get_server_vendor_version(const char*, int, char**, char**, AppIdServiceSubtype**); void identify_user_agent(const char*, int, AppId&, AppId&, char**); uint32_t parse_multiple_http_patterns(const char* pattern, tMlmpPattern*, - uint32_t numPartLimit, int level); + uint32_t numPartLimit, int level, bool is_literal); private: DetectorHTTPPatterns client_agent_patterns; diff --git a/src/network_inspectors/appid/detector_plugins/sip_patterns.cc b/src/network_inspectors/appid/detector_plugins/sip_patterns.cc index 188a126de..f9de52cb5 100644 --- a/src/network_inspectors/appid/detector_plugins/sip_patterns.cc +++ b/src/network_inspectors/appid/detector_plugins/sip_patterns.cc @@ -125,7 +125,7 @@ void SipPatternMatchers::finalize_patterns(OdpContext& odp_ctxt) { pattern_count++; num_patterns = odp_ctxt.get_http_matchers().parse_multiple_http_patterns( - (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0); + (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0, true); patterns[num_patterns].pattern = nullptr; mlmpAddPattern(sip_ua_matcher, patterns, pattern_node); @@ -136,7 +136,7 @@ void SipPatternMatchers::finalize_patterns(OdpContext& odp_ctxt) { pattern_count++; num_patterns = odp_ctxt.get_http_matchers().parse_multiple_http_patterns( - (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0); + (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0, true); patterns[num_patterns].pattern = nullptr; mlmpAddPattern(sip_server_matcher, patterns, pattern_node); diff --git a/src/network_inspectors/appid/detector_plugins/sip_patterns.h b/src/network_inspectors/appid/detector_plugins/sip_patterns.h index dc2219219..ebbd90a90 100644 --- a/src/network_inspectors/appid/detector_plugins/sip_patterns.h +++ b/src/network_inspectors/appid/detector_plugins/sip_patterns.h @@ -53,7 +53,7 @@ public: private: static const int PATTERN_PART_MAX = 10; - tMlmpPattern patterns[PATTERN_PART_MAX] = { { nullptr, 0, 0 } }; + tMlmpPattern patterns[PATTERN_PART_MAX] = { { nullptr, 0, 0, true } }; tMlmpTree* sip_ua_matcher = nullptr; DetectorAppSipPattern* sip_ua_list = nullptr; tMlmpTree* sip_server_matcher = nullptr; diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc index 23e0105a1..dadad937c 100644 --- a/src/network_inspectors/appid/lua_detector_api.cc +++ b/src/network_inspectors/appid/lua_detector_api.cc @@ -2414,6 +2414,107 @@ static int detector_add_url_application(lua_State* L) pattern->patterns.path.patternSize = (int)path_pattern_size; pattern->patterns.scheme.pattern = schemePattern; pattern->patterns.scheme.patternSize = (int)schemePatternSize; + pattern->is_literal = true; + ud->get_odp_ctxt().get_http_matchers().insert_url_pattern(pattern); + + app_info_manager.set_app_info_active(pattern->userData.service_id); + app_info_manager.set_app_info_active(pattern->userData.client_id); + app_info_manager.set_app_info_active(pattern->userData.payload_id); + app_info_manager.set_app_info_active(appId); + + return 0; +} + +static int detector_add_url_application_regex(lua_State* L) +{ + // Verify detector user data and that we are NOT in packet context + auto& ud = *UserData::check(L, DETECTOR, 1); + ud->validate_lua_state(false); + if (!init(L)) + return 0; + + const FastPatternConfig* const fp = SnortConfig::get_conf()->fast_pattern_config; + if (!MpseManager::is_regex_capable(fp->get_search_api())){ + appid_log(nullptr, TRACE_WARNING_LEVEL, "WARNING: appid: Regex patterns require usage of " + "regex capable search engine like hyperscan in %s\n", ud->get_detector()->get_name().c_str()); + return 0; + } + + + int index = 1; + + uint32_t service_id = lua_tointeger(L, ++index); + uint32_t client_id = lua_tointeger(L, ++index); + lua_tointeger(L, ++index); //client_id_type + uint32_t payload_id = lua_tointeger(L, ++index); + lua_tointeger(L, ++index); // payload_type + + /* Verify that host pattern is a valid string */ + size_t host_pattern_size = 0; + uint8_t* host_pattern = nullptr; + const char* tmp_string = lua_tolstring(L, ++index, &host_pattern_size); + if (!tmp_string or !host_pattern_size) + { + appid_log(nullptr, TRACE_ERROR_LEVEL, "appid: Invalid host regex pattern string: service_id %u; " + "client_id %u; payload_id %u.\n", service_id, client_id, payload_id); + return 0; + } + else + host_pattern = (uint8_t*)snort_strdup(tmp_string); + + /* Verify that path pattern is a valid string */ + size_t path_pattern_size = 0; + uint8_t* path_pattern = nullptr; + tmp_string = lua_tolstring(L, ++index, &path_pattern_size); + if (!tmp_string or !path_pattern_size) + { + appid_log(nullptr, TRACE_ERROR_LEVEL, "appid: Invalid path regex pattern string: service_id %u; " + "client_id %u; payload %u.\n", service_id, client_id, payload_id); + snort_free(host_pattern); + return 0; + } + else + path_pattern = (uint8_t*)snort_strdup(tmp_string); + + /* Verify that scheme pattern is a valid string */ + size_t schemePatternSize; + uint8_t* schemePattern = nullptr; + tmp_string = lua_tolstring(L, ++index, &schemePatternSize); + if (!tmp_string or !schemePatternSize) + { + appid_log(nullptr, TRACE_ERROR_LEVEL, "appid: Invalid scheme regex pattern string: service_id %u; " + "client_id %u; payload %u.\n", service_id, client_id, payload_id); + snort_free(path_pattern); + snort_free(host_pattern); + return 0; + } + else + schemePattern = (uint8_t*)snort_strdup(tmp_string); + + /* Verify that query pattern is a valid string */ + size_t query_pattern_size; + uint8_t* query_pattern = nullptr; + tmp_string = lua_tolstring(L, ++index, &query_pattern_size); + if (tmp_string and query_pattern_size) + query_pattern = (uint8_t*)snort_strdup(tmp_string); + + uint32_t appId = lua_tointeger(L, ++index); + AppInfoManager& app_info_manager = ud->get_odp_ctxt().get_app_info_mgr(); + DetectorAppUrlPattern* pattern = + (DetectorAppUrlPattern*)snort_calloc(sizeof(DetectorAppUrlPattern)); + pattern->userData.service_id = app_info_manager.get_appid_by_service_id(service_id); + pattern->userData.client_id = app_info_manager.get_appid_by_client_id(client_id); + pattern->userData.payload_id = app_info_manager.get_appid_by_payload_id(payload_id); + pattern->userData.appId = appId; + pattern->userData.query.pattern = query_pattern; + pattern->userData.query.patternSize = query_pattern_size; + pattern->patterns.host.pattern = host_pattern; + pattern->patterns.host.patternSize = (int)host_pattern_size; + pattern->patterns.path.pattern = path_pattern; + pattern->patterns.path.patternSize = (int)path_pattern_size; + pattern->patterns.scheme.pattern = schemePattern; + pattern->patterns.scheme.patternSize = (int)schemePatternSize; + pattern->is_literal = false; ud->get_odp_ctxt().get_http_matchers().insert_url_pattern(pattern); app_info_manager.set_app_info_active(pattern->userData.service_id); @@ -2503,6 +2604,7 @@ static int detector_add_rtmp_url(lua_State* L) pattern->patterns.path.patternSize = (int)path_pattern_size; pattern->patterns.scheme.pattern = schemePattern; pattern->patterns.scheme.patternSize = (int)schemePatternSize; + pattern->is_literal = true; ud->get_odp_ctxt().get_http_matchers().insert_rtmp_url_pattern(pattern); AppInfoManager& app_info_manager = ud->get_odp_ctxt().get_app_info_mgr(); @@ -2752,6 +2854,7 @@ static int add_url_pattern(lua_State* L) pattern->patterns.path.patternSize = (int)path_pattern_size; pattern->patterns.scheme.pattern = schemePattern; pattern->patterns.scheme.patternSize = (int)schemePatternSize; + pattern->is_literal = true; ud->get_odp_ctxt().get_http_matchers().insert_app_url_pattern(pattern); AppInfoManager& app_info_manager = ud->get_odp_ctxt().get_app_info_mgr(); @@ -3243,6 +3346,7 @@ static const luaL_Reg detector_methods[] = { "cLog", detector_log_snort_message}, { "addHttpPattern", detector_add_http_pattern }, { "addAppUrl", detector_add_url_application }, + { "addAppUrlRegex", detector_add_url_application_regex }, { "addRTMPUrl", detector_add_rtmp_url }, { "addContentTypePattern", detector_add_content_type_pattern }, { "addSSLCertPattern", detector_add_ssl_cert_pattern },