Pull request #4287: appid: add http url regex patterns

Merge in SNORT/snort3 from ~LCZARNIK/snort3:http_regex to master

Squashed commit of the following:

commit 1eabb9424b2930e9f74a90b473acee322362e8f6
Author: Lukasz Czarnik <lczarnik@cisco.com>
Date:   Mon Apr 15 14:50:37 2024 -0400

    appid: add http url regex patterns

diff --git a/src/network_inspectors/appid/appid_utils/sf_mlmp.cc b/src/network_inspectors/appid/appid_utils/sf_mlmp.cc
index c82f2a848648b7730bf48c8885d7427f2def386a..441eefdf462eb5307776498e47591d0a78c24499 100644 (file)
--- a/src/network_inspectors/appid/appid_utils/sf_mlmp.cc
+++ b/src/network_inspectors/appid/appid_utils/sf_mlmp.cc
@@ -76,7 +76,7 @@ struct tMatchedPatternList
 };
 
 static int compareMlmpPatterns(const void* p1, const void* p2);
-static int createTreesRecusively(tMlmpTree* root);
+static int createTreesRecursively(tMlmpTree* root);
 static void destroyTreesRecursively(tMlmpTree* root);
 static int addPatternRecursively(tMlmpTree* root, const tMlmpPattern* inputPatternList,
     void* metaData, uint32_t level);
@@ -108,7 +108,7 @@ int mlmpProcessPatterns(tMlmpTree* root)
 {
     int rvalue;
 
-    rvalue = createTreesRecusively(root);
+    rvalue = createTreesRecursively(root);
     if (rvalue)
         destroyTreesRecursively(root);
     return rvalue;
@@ -204,7 +204,7 @@ static int compareMlmpPatterns(const void* p1, const void* p2)
 
 /*pattern trees are not freed on error because in case of error, caller should call
    detroyTreesRecursively. */
-static int createTreesRecusively(tMlmpTree* rootNode)
+static int createTreesRecursively(tMlmpTree* rootNode)
 {
     SearchTool* patternMatcher;
     tPatternPrimaryNode* primaryPatternNode;
@@ -220,7 +220,7 @@ static int createTreesRecusively(tMlmpTree* rootNode)
         /*recursion into next lower level */
         if (primaryPatternNode->nextLevelMatcher)
         {
-            if (createTreesRecusively(primaryPatternNode->nextLevelMatcher))
+            if (createTreesRecursively(primaryPatternNode->nextLevelMatcher))
                 return -1;
         }
 
@@ -229,7 +229,7 @@ static int createTreesRecusively(tMlmpTree* rootNode)
             ddPatternNode = ddPatternNode->nextPattern)
         {
             patternMatcher->add(ddPatternNode->pattern.pattern,
-                ddPatternNode->pattern.patternSize, ddPatternNode, true);
+                ddPatternNode->pattern.patternSize, ddPatternNode, true, ddPatternNode->pattern.is_literal);
         }
     }
 
@@ -295,32 +295,33 @@ static tPatternNode* patternSelector(const tMatchedPatternList* patternMatchList
         tmpList;
         tmpList = tmpList->next)
     {
-        if (tmpList->patternNode->patternId != patternId)
+        const tPatternNode& node = *tmpList->patternNode;
+        if (node.patternId != patternId)
         {
             /*first pattern */
 
             /*skip incomplete pattern */
-            if (tmpList->patternNode->partNum != 1)
+            if (node.partNum != 1)
                 continue;
 
             /*new pattern started */
-            patternId = tmpList->patternNode->patternId;
+            patternId = node.patternId;
             currentPrimaryNode = tmpList->patternNode;
             partNum = 0;
             patternSize = 0;
         }
 
-        if (tmpList->patternNode->partNum == (partNum+1))
+        if (node.partNum == (partNum+1))
         {
             partNum++;
-            patternSize += tmpList->patternNode->pattern.patternSize;
+            patternSize += node.pattern.patternSize;
         }
 
-        if (tmpList->patternNode->partTotal != partNum)
+        if (node.partTotal != partNum)
             continue;
 
         /*backward compatibility */
-        if ((tmpList->patternNode->partTotal == 1)
+        if (node.pattern.is_literal && (node.partTotal == 1)
             && domain && !match_is_domain_pattern(tmpList, payload))
             continue;
 
@@ -485,6 +486,7 @@ static int addPatternRecursively(tMlmpTree* rootNode, const tMlmpPattern* inputP
         tmpPrimaryNode->patternNode.pattern.pattern = patterns->pattern;
         tmpPrimaryNode->patternNode.pattern.patternSize = patterns->patternSize;
         tmpPrimaryNode->patternNode.pattern.level = patterns->level;
+        tmpPrimaryNode->patternNode.pattern.is_literal = patterns->is_literal;
         tmpPrimaryNode->patternNode.partNum = 1;
         tmpPrimaryNode->patternNode.partTotal = partTotal;
         tmpPrimaryNode->patternNode.patternId = patternId;
@@ -512,6 +514,7 @@ static int addPatternRecursively(tMlmpTree* rootNode, const tMlmpPattern* inputP
             newNode->pattern.pattern = patterns->pattern;
             newNode->pattern.patternSize = patterns->patternSize;
             newNode->pattern.level = patterns->level;
+            newNode->pattern.is_literal = patterns->is_literal;
             newNode->partNum = partNum;
             newNode->partTotal = partTotal;
             newNode->patternId = patternId;

diff --git a/src/network_inspectors/appid/appid_utils/sf_mlmp.h b/src/network_inspectors/appid/appid_utils/sf_mlmp.h
index 295e3b8b7965b2eaa6aa47be33b3b7f29cda27eb..bc3104630a5a7ffa09cba2d66ac2eec4fba3b34f 100644 (file)
--- a/src/network_inspectors/appid/appid_utils/sf_mlmp.h
+++ b/src/network_inspectors/appid/appid_utils/sf_mlmp.h
@@ -41,6 +41,7 @@ struct tMlmpPattern
 
     /**level of pattern. It should start from 0.*/
     uint32_t level;
+    bool is_literal;
 };
 
 struct tMlmpTree;

diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc b/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc
index bab88b97404380b74cac5a931f2a261e4f71ba70..f8b9393889ed24e99becc65404b156c8c6f4daf1 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc
+++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc
@@ -491,7 +491,7 @@ int HttpPatternMatchers::add_mlmp_pattern(tMlmpTree* matcher, DetectorHTTPPatter
 
     tMlmpPattern patterns[PATTERN_PART_MAX];
     int num_patterns = parse_multiple_http_patterns((const char*)pattern.pattern, patterns,
-        PATTERN_PART_MAX, 0);
+        PATTERN_PART_MAX, 0, true);
     patterns[num_patterns].pattern = nullptr;
     return mlmpAddPattern(matcher, patterns, detector);
 }
@@ -531,10 +531,10 @@ int HttpPatternMatchers::add_mlmp_pattern(tMlmpTree* matcher, DetectorAppUrlPatt
 
     tMlmpPattern patterns[PATTERN_PART_MAX];
     int num_patterns = parse_multiple_http_patterns((const char*)pattern.patterns.host.pattern,
-        patterns, PATTERN_PART_MAX, 0);
+        patterns, PATTERN_PART_MAX, 0, pattern.is_literal);
     if (pattern.patterns.path.pattern)
         num_patterns += parse_multiple_http_patterns((const char*)pattern.patterns.path.pattern,
-            patterns + num_patterns, PATTERN_PART_MAX - num_patterns, 1);
+            patterns + num_patterns, PATTERN_PART_MAX - num_patterns, 1, pattern.is_literal);
 
     patterns[num_patterns].pattern = nullptr;
     return mlmpAddPattern(matcher, patterns, detector);
@@ -1655,7 +1655,7 @@ void HttpPatternMatchers::get_server_vendor_version(const char* data, int len, c
 }
 
 uint32_t HttpPatternMatchers::parse_multiple_http_patterns(const char* pattern,
-    tMlmpPattern* parts, uint32_t numPartLimit, int level)
+    tMlmpPattern* parts, uint32_t numPartLimit, int level, bool is_literal)
 {
     uint32_t partNum = 0;
 
@@ -1679,6 +1679,7 @@ uint32_t HttpPatternMatchers::parse_multiple_http_patterns(const char* pattern,
             tmp = nullptr;
         }
         parts[partNum].level = level;
+        parts[partNum].is_literal = is_literal;
 
         if ( !parts[partNum].pattern )
         {

diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
index c42f1fde2ed3e05d715a6f19ae62fc2f0ba7d0ce..8fe6b1b0ad2d9aa1aa11eab6e8f795248ded2d11 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
+++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
@@ -76,6 +76,7 @@ struct DetectorAppUrlPattern
     } patterns;
 
     UrlUserData userData;
+    bool is_literal;
 };
 
 // These values are used in Lua code as raw numbers. Do NOT reassign new values.
@@ -309,7 +310,7 @@ public:
     void get_server_vendor_version(const char*, int, char**, char**, AppIdServiceSubtype**);
     void identify_user_agent(const char*, int, AppId&, AppId&, char**);
     uint32_t parse_multiple_http_patterns(const char* pattern, tMlmpPattern*,
-        uint32_t numPartLimit, int level);
+        uint32_t numPartLimit, int level, bool is_literal);
 
 private:
     DetectorHTTPPatterns client_agent_patterns;

diff --git a/src/network_inspectors/appid/detector_plugins/sip_patterns.cc b/src/network_inspectors/appid/detector_plugins/sip_patterns.cc
index 188a126de97cdddb81debd92c11c57d255e1dbda..f9de52cb5ef172b650cebb54f02edeeaab4d4b6b 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/sip_patterns.cc
+++ b/src/network_inspectors/appid/detector_plugins/sip_patterns.cc
@@ -125,7 +125,7 @@ void SipPatternMatchers::finalize_patterns(OdpContext& odp_ctxt)
     {
         pattern_count++;
         num_patterns = odp_ctxt.get_http_matchers().parse_multiple_http_patterns(
-            (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0);
+            (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0, true);
         patterns[num_patterns].pattern = nullptr;
 
         mlmpAddPattern(sip_ua_matcher, patterns, pattern_node);
@@ -136,7 +136,7 @@ void SipPatternMatchers::finalize_patterns(OdpContext& odp_ctxt)
     {
         pattern_count++;
         num_patterns = odp_ctxt.get_http_matchers().parse_multiple_http_patterns(
-            (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0);
+            (const char*)pattern_node->pattern.pattern, patterns, PATTERN_PART_MAX, 0, true);
         patterns[num_patterns].pattern = nullptr;
 
         mlmpAddPattern(sip_server_matcher, patterns, pattern_node);

diff --git a/src/network_inspectors/appid/detector_plugins/sip_patterns.h b/src/network_inspectors/appid/detector_plugins/sip_patterns.h
index dc221921970929cf4dabbecf5d2287a9d41ebc69..ebbd90a903350ea2cd0edae1696d01ec0f73f294 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/sip_patterns.h
+++ b/src/network_inspectors/appid/detector_plugins/sip_patterns.h
@@ -53,7 +53,7 @@ public:
 
 private:
     static const int PATTERN_PART_MAX = 10;
-    tMlmpPattern patterns[PATTERN_PART_MAX] = { { nullptr, 0, 0 } };
+    tMlmpPattern patterns[PATTERN_PART_MAX] = { { nullptr, 0, 0, true } };
     tMlmpTree* sip_ua_matcher = nullptr;
     DetectorAppSipPattern* sip_ua_list = nullptr;
     tMlmpTree* sip_server_matcher = nullptr;

diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc
index 23e0105a18f3bfd05a0f914faa7f7356bb6b6dca..dadad937c5a66126a8591cca27d3de34ae7b8e48 100644 (file)
--- a/src/network_inspectors/appid/lua_detector_api.cc
+++ b/src/network_inspectors/appid/lua_detector_api.cc
@@ -2414,6 +2414,107 @@ static int detector_add_url_application(lua_State* L)
     pattern->patterns.path.patternSize  = (int)path_pattern_size;
     pattern->patterns.scheme.pattern    = schemePattern;
     pattern->patterns.scheme.patternSize = (int)schemePatternSize;
+    pattern->is_literal = true;
+    ud->get_odp_ctxt().get_http_matchers().insert_url_pattern(pattern);
+
+    app_info_manager.set_app_info_active(pattern->userData.service_id);
+    app_info_manager.set_app_info_active(pattern->userData.client_id);
+    app_info_manager.set_app_info_active(pattern->userData.payload_id);
+    app_info_manager.set_app_info_active(appId);
+
+    return 0;
+}
+
+static int detector_add_url_application_regex(lua_State* L)
+{
+    // Verify detector user data and that we are NOT in packet context
+    auto& ud = *UserData<LuaObject>::check(L, DETECTOR, 1);
+    ud->validate_lua_state(false);
+    if (!init(L))
+        return 0;
+
+    const FastPatternConfig* const fp = SnortConfig::get_conf()->fast_pattern_config;
+    if (!MpseManager::is_regex_capable(fp->get_search_api())){
+        appid_log(nullptr, TRACE_WARNING_LEVEL, "WARNING: appid: Regex patterns require usage of "
+            "regex capable search engine like hyperscan in %s\n", ud->get_detector()->get_name().c_str());
+            return 0;
+    }
+
+
+    int index = 1;
+
+    uint32_t service_id      = lua_tointeger(L, ++index);
+    uint32_t client_id       = lua_tointeger(L, ++index);
+    lua_tointeger(L, ++index); //client_id_type
+    uint32_t payload_id      = lua_tointeger(L, ++index);
+    lua_tointeger(L, ++index); // payload_type
+
+    /* Verify that host pattern is a valid string */
+    size_t host_pattern_size = 0;
+    uint8_t* host_pattern = nullptr;
+    const char* tmp_string = lua_tolstring(L, ++index, &host_pattern_size);
+    if (!tmp_string or !host_pattern_size)
+    {
+        appid_log(nullptr, TRACE_ERROR_LEVEL, "appid: Invalid host regex pattern string: service_id %u; "
+            "client_id %u; payload_id %u.\n", service_id, client_id, payload_id);
+        return 0;
+    }
+    else
+        host_pattern = (uint8_t*)snort_strdup(tmp_string);
+
+    /* Verify that path pattern is a valid string */
+    size_t path_pattern_size = 0;
+    uint8_t* path_pattern = nullptr;
+    tmp_string = lua_tolstring(L, ++index, &path_pattern_size);
+    if (!tmp_string or !path_pattern_size)
+    {
+        appid_log(nullptr, TRACE_ERROR_LEVEL, "appid: Invalid path regex pattern string: service_id %u; "
+            "client_id %u; payload %u.\n", service_id, client_id, payload_id);
+        snort_free(host_pattern);
+        return 0;
+    }
+    else
+        path_pattern = (uint8_t*)snort_strdup(tmp_string);
+
+    /* Verify that scheme pattern is a valid string */
+    size_t schemePatternSize;
+    uint8_t* schemePattern = nullptr;
+    tmp_string = lua_tolstring(L, ++index, &schemePatternSize);
+    if (!tmp_string or !schemePatternSize)
+    {
+        appid_log(nullptr, TRACE_ERROR_LEVEL, "appid: Invalid scheme regex pattern string: service_id %u; "
+            "client_id %u; payload %u.\n", service_id, client_id, payload_id);
+        snort_free(path_pattern);
+        snort_free(host_pattern);
+        return 0;
+    }
+    else
+        schemePattern = (uint8_t*)snort_strdup(tmp_string);
+
+    /* Verify that query pattern is a valid string */
+    size_t query_pattern_size;
+    uint8_t* query_pattern = nullptr;
+    tmp_string = lua_tolstring(L, ++index, &query_pattern_size);
+    if (tmp_string and query_pattern_size)
+        query_pattern = (uint8_t*)snort_strdup(tmp_string);
+
+    uint32_t appId = lua_tointeger(L, ++index);
+    AppInfoManager& app_info_manager = ud->get_odp_ctxt().get_app_info_mgr();
+    DetectorAppUrlPattern* pattern =
+        (DetectorAppUrlPattern*)snort_calloc(sizeof(DetectorAppUrlPattern));
+    pattern->userData.service_id        = app_info_manager.get_appid_by_service_id(service_id);
+    pattern->userData.client_id        = app_info_manager.get_appid_by_client_id(client_id);
+    pattern->userData.payload_id           = app_info_manager.get_appid_by_payload_id(payload_id);
+    pattern->userData.appId             = appId;
+    pattern->userData.query.pattern     = query_pattern;
+    pattern->userData.query.patternSize = query_pattern_size;
+    pattern->patterns.host.pattern      = host_pattern;
+    pattern->patterns.host.patternSize  = (int)host_pattern_size;
+    pattern->patterns.path.pattern      = path_pattern;
+    pattern->patterns.path.patternSize  = (int)path_pattern_size;
+    pattern->patterns.scheme.pattern    = schemePattern;
+    pattern->patterns.scheme.patternSize = (int)schemePatternSize;
+    pattern->is_literal = false;
     ud->get_odp_ctxt().get_http_matchers().insert_url_pattern(pattern);
 
     app_info_manager.set_app_info_active(pattern->userData.service_id);
@@ -2503,6 +2604,7 @@ static int detector_add_rtmp_url(lua_State* L)
     pattern->patterns.path.patternSize  = (int)path_pattern_size;
     pattern->patterns.scheme.pattern    = schemePattern;
     pattern->patterns.scheme.patternSize = (int)schemePatternSize;
+    pattern->is_literal = true;
     ud->get_odp_ctxt().get_http_matchers().insert_rtmp_url_pattern(pattern);
 
     AppInfoManager& app_info_manager = ud->get_odp_ctxt().get_app_info_mgr();
@@ -2752,6 +2854,7 @@ static int add_url_pattern(lua_State* L)
     pattern->patterns.path.patternSize  = (int)path_pattern_size;
     pattern->patterns.scheme.pattern    = schemePattern;
     pattern->patterns.scheme.patternSize = (int)schemePatternSize;
+    pattern->is_literal = true;
     ud->get_odp_ctxt().get_http_matchers().insert_app_url_pattern(pattern);
 
     AppInfoManager& app_info_manager = ud->get_odp_ctxt().get_app_info_mgr();
@@ -3243,6 +3346,7 @@ static const luaL_Reg detector_methods[] =
     { "cLog",                     detector_log_snort_message},
     { "addHttpPattern",           detector_add_http_pattern },
     { "addAppUrl",                detector_add_url_application },
+    { "addAppUrlRegex",           detector_add_url_application_regex },
     { "addRTMPUrl",               detector_add_rtmp_url },
     { "addContentTypePattern",    detector_add_content_type_pattern },
     { "addSSLCertPattern",        detector_add_ssl_cert_pattern },