]> git.ipfire.org Git - thirdparty/snort3.git/commitdiff
Merge pull request #1119 in SNORT/snort3 from appid_http_ut to master
authorMike Stepanek (mstepane) <mstepane@cisco.com>
Fri, 2 Mar 2018 20:23:45 +0000 (15:23 -0500)
committerMike Stepanek (mstepane) <mstepane@cisco.com>
Fri, 2 Mar 2018 20:23:45 +0000 (15:23 -0500)
Squashed commit of the following:

commit 90db53493af707c95856912e46f0e7a31494dd7f
Author: Masud Hasan <mashasan@cisco.com>
Date:   Mon Feb 26 01:09:12 2018 -0500

    appid: unit-tests for http detector plugins

src/network_inspectors/appid/detector_plugins/http_url_patterns.cc
src/network_inspectors/appid/detector_plugins/http_url_patterns.h
src/network_inspectors/appid/detector_plugins/test/CMakeLists.txt
src/network_inspectors/appid/detector_plugins/test/Makefile.am
src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h [new file with mode: 0644]
src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc [new file with mode: 0644]

index bf0b2021fd81d1af7b8cdac678a23ebcb0477d16..d2255bd08d69535594fb0ef22a2507b03c1bb221 100644 (file)
@@ -1577,16 +1577,28 @@ bool HttpPatternMatchers::get_appid_from_url(char* host, const char* url, char**
         url_len = strlen(url);
     }
 
-    // FIXIT-M - This logic that fiddles with host is very dicey, should refactor to improve this
+    int host_len;
     if (!host)
     {
-        temp_host = host = snort_strdup(url);
-        host  = strchr(host, '/');
+        host = (char *)strchr(url, '/');
         if (host != nullptr)
-            *host = '\0';
-        host = temp_host;
+            host_len = host - url;
+        else
+            host_len = url_len;
+        if (host_len > 0)
+        {
+            temp_host = snort_strndup(url, host_len);
+            if (!temp_host)
+            {
+                host_len = 0;
+                host = nullptr;
+            }
+            else
+                host = temp_host;
+        }
     }
-    int host_len = strlen(host);
+    else
+        host_len = strlen(host);
 
     const char* path = nullptr;
     int path_len = 0;
index 689dd51ee3f7f044fb2ae5bac23fc331fcca1900..3db813dd584c2f28f2708e9acc428862d6482650 100644 (file)
@@ -121,6 +121,8 @@ struct DetectorHTTPPattern
 };
 typedef std::vector<DetectorHTTPPattern> DetectorHTTPPatterns;
 
+// CHP (Complex HTTP Pattern) uses more than one HTTP pattern
+// to do appid detection and/or perform other actions
 #define CHP_APPID_BITS_FOR_INSTANCE  7
 #define CHP_APPID_INSTANCE_MAX (1 << CHP_APPID_BITS_FOR_INSTANCE)
 #define CHP_APPIDINSTANCE_TO_ID(_appIdInstance) \
index ee1502c836e8eda81e8c46c71ea6fbf363e510d9..bd64bc15154983866c4fb2280925b0be369691bc 100644 (file)
@@ -5,6 +5,8 @@ set (
 
 add_cpputest(detector_smtp_test ${SMTP_TEST_LIBS})
 
+add_cpputest(http_url_patterns_test)
+
 include_directories ( appid PRIVATE ${APPID_INCLUDE_DIR} )
 
 
index 73233f16acadc32d105f6631d1a6f62f1392b928..c2ef0f29d08b87491681ca939fc42f0d21fcfe2a 100644 (file)
@@ -2,7 +2,8 @@
 AM_DEFAULT_SOURCE_EXT = .cc
 
 check_PROGRAMS = \
-detector_smtp_test
+detector_smtp_test \
+http_url_patterns_test
 
 TESTS = $(check_PROGRAMS)
 
@@ -12,3 +13,9 @@ detector_smtp_test_LDADD = \
 ../../../../utils/libutils.a \
 @CPPUTEST_LDFLAGS@
 
+http_url_patterns_test_CPPFLAGS = -I$(top_srcdir)/src/network_inspectors/appid @AM_CPPFLAGS@ @CPPUTEST_CPPFLAGS@
+
+http_url_patterns_test_LDADD = \
+../../../../utils/libutils.a \
+@CPPUTEST_LDFLAGS@
+
diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h
new file mode 100644 (file)
index 0000000..a0fbb0a
--- /dev/null
@@ -0,0 +1,178 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2018-2018 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// detector_plugins_mock.h author Masud Hasan <mashasan@cisco.com>
+
+#ifndef DETECTOR_PLUGINS_MOCK_H
+#define DETECTOR_PLUGINS_MOCK_H
+
+// Stubs for messages
+void ErrorMessage(const char*,...) {}
+void WarningMessage(const char*,...) {}
+void LogMessage(const char*,...) {}
+void ParseWarning(WarningGroup, const char*, ...) {}
+#ifdef DEBUG_MSGS
+void Debug::print(const char*, int, uint64_t, const char*, ...) {}
+#endif
+
+// Stubs for packet
+Packet::Packet(bool) { }
+Packet::~Packet() { }
+
+// Stubs for inspectors
+unsigned AppIdSession::inspector_id = 0;
+Inspector::Inspector() {}
+Inspector::~Inspector() {}
+bool Inspector::likes(Packet*) { return true; }
+bool Inspector::get_buf(const char*, Packet*, InspectionBuffer&) { return true; }
+class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; }
+class AppIdInspector : public Inspector
+{
+public:
+    AppIdInspector(AppIdModule& ) {}
+    ~AppIdInspector() {}
+    void eval(Packet*) {}
+    bool configure(SnortConfig*) { return true; }
+    void show(SnortConfig*) {}
+    void tinit() {}
+    void tterm() {}
+};
+
+// Stubs for modules, config
+AppIdModuleConfig::AppIdModuleConfig() {}
+AppIdModuleConfig::~AppIdModuleConfig() {}
+AppIdModule::AppIdModule()
+    : Module("a", "b") {}
+AppIdModule::~AppIdModule() {}
+bool AppIdModule::begin(const char*, int, SnortConfig*)
+{
+    return false;
+}
+bool AppIdModule::set(const char*, Value&, SnortConfig*)
+{
+    return false;
+}
+bool AppIdModule::end(const char*, int, SnortConfig*)
+{
+    return false;
+}
+const PegInfo* AppIdModule::get_pegs() const
+{
+    return nullptr;
+}
+PegCount* AppIdModule::get_counts() const
+{
+    return nullptr;
+}
+ProfileStats* AppIdModule::get_profile() const
+{
+    return nullptr;
+}
+void show_stats(PegCount*, const PegInfo*, unsigned, const char*) {}
+void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*) {}
+void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*, FILE*) {}
+
+// Stubs for appid sessions
+FlowData::FlowData(unsigned, Inspector*) {}
+FlowData::~FlowData() = default;
+AppIdSession::AppIdSession(IpProtocol, const SfIp*, uint16_t, AppIdInspector& inspector)
+    : FlowData(inspector_id, (Inspector*)&inspector), inspector(inspector) {}
+AppIdSession::~AppIdSession() {}
+AppIdHttpSession::AppIdHttpSession(AppIdSession& session)
+    : asd(session) {}
+AppIdHttpSession::~AppIdHttpSession() {}
+
+// Stubs for AppIdPegCounts
+void AppIdPegCounts::inc_disco_peg(enum DiscoveryPegs) {}
+void AppIdPegCounts::inc_service_count(AppId) {}
+void AppIdPegCounts::inc_client_count(AppId) {}
+void AppIdPegCounts::inc_user_count(AppId) {}
+void AppIdPegCounts::inc_payload_count(AppId) {}
+PegCount AppIdPegCounts::get_disco_peg(enum DiscoveryPegs)
+{
+    return 0;
+}
+
+// Stubs for search_tool.cc
+SearchTool::SearchTool(const char*, bool) {}
+SearchTool::~SearchTool() {}
+void SearchTool::add(const char*, unsigned, int, bool) {}
+void SearchTool::add(const char*, unsigned, void*, bool) {}
+void SearchTool::add(const uint8_t*, unsigned, int, bool) {}
+void SearchTool::add(const uint8_t*, unsigned, void*, bool) {}
+void SearchTool::prep() {}
+static bool test_find_all_done = false;
+static bool test_find_all_enabled = false;
+static MatchedPatterns* mock_mp = nullptr;
+int SearchTool::find_all(const char*, unsigned, MpseMatch, bool, void* mp_arg)
+{
+    test_find_all_done = true;
+    if (test_find_all_enabled)
+        memcpy(mp_arg, &mock_mp, sizeof(MatchedPatterns*));
+    return 0;
+}
+
+// Stubs for appid_session.cc
+static bool test_service_strstr_enabled = false;
+const uint8_t* service_strstr(const uint8_t* p, unsigned,
+    const uint8_t*, unsigned)
+{
+    if (test_service_strstr_enabled)
+        return p;
+    return nullptr;
+}
+
+// Stubs for appid_http_session.cc
+static bool test_field_offset_set_done = false;
+void AppIdHttpSession::set_field_offset(HttpFieldIds, uint16_t)
+{
+    test_field_offset_set_done = true;
+}
+void AppIdHttpSession::set_field_end_offset(HttpFieldIds, uint16_t) {}
+
+// Stubs for app_info_table.cc
+AppInfoTableEntry* AppInfoManager::get_app_info_entry(int)
+{
+    return nullptr;
+}
+AppInfoTableEntry* AppInfoManager::get_app_info_entry(AppId, const AppInfoTable&)
+{
+    return nullptr;
+}
+
+// Stubs for util.cc
+char* snort_strndup(const char* src, size_t dst_size)
+{
+    char* dup = (char*)snort_calloc(dst_size + 1);
+    if ( SnortStrncpy(dup, src, dst_size + 1) == SNORT_STRNCPY_ERROR )
+    {
+        snort_free(dup);
+        return nullptr;
+    }
+    return dup;
+}
+char* snort_strdup(const char* str)
+{
+    assert(str);
+    size_t n = strlen(str) + 1;
+    char* p = (char*)snort_alloc(n);
+    memcpy(p, str, n);
+    return p;
+}
+
+#endif
+
diff --git a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc
new file mode 100644 (file)
index 0000000..6dd3dc5
--- /dev/null
@@ -0,0 +1,677 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2018-2018 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_url_patterns_test.cc author Masud Hasan <mashasan@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "network_inspectors/appid/detector_plugins/http_url_patterns.cc"
+
+#include "protocols/protocol_ids.h"
+#include "framework/module.cc"
+#include "network_inspectors/appid/appid_utils/sf_multi_mpse.h"
+#include "network_inspectors/appid/appid_utils/sf_mlmp.cc"
+#include "utils/util_cstring.cc"
+#include "detector_plugins_mock.h"
+
+#include <CppUTest/CommandLineTestRunner.h>
+#include <CppUTest/TestHarness.h>
+#include <CppUTestExt/MockSupport.h>
+
+static HttpPatternMatchers* hm = nullptr;
+static Packet pkt;
+static const SfIp* sfip = nullptr;
+static AppIdModule appid_mod;
+static AppIdInspector appid_inspector( appid_mod );
+static AppIdSession session(IpProtocol::IP, sfip, 0, appid_inspector);
+static AppIdHttpSession hsession(session);
+static ChpMatchDescriptor cmd;
+static MatchedCHPAction mchp;
+static CHPAction chpa;
+static char* version = nullptr;
+static char* user = nullptr;
+static char* my_action_data = (char*)"0";
+static const char* my_chp_data = (const char*)"chp_data";
+static int total_found;
+static AppIdModuleConfig mod_config;
+static AppId service_id = APP_ID_NONE;
+static AppId client_id = APP_ID_NONE;
+static DetectorHTTPPattern mpattern;
+static const char* my_buffer[MAX_HTTP_FIELD_ID] = { nullptr };
+static uint16_t my_length[MAX_HTTP_FIELD_ID] = { 0 };
+static CHPAction my_match;
+static void* my_chp_rewritten = nullptr;
+
+TEST_GROUP(http_url_patterns_tests)
+{
+    void setup() override
+    {
+        hm = new HttpPatternMatchers();
+    }
+
+    void teardown() override
+    {
+        delete hm;
+        delete sfip;
+    }
+};
+
+TEST(http_url_patterns_tests, http_field_pattern_match)
+{
+    FieldPatternData fpd;
+    FieldPattern fp;
+
+    // verify service_strstr getting called
+    fpd.payload = (const uint8_t*)"Google";
+    fpd.length = 6;
+    test_service_strstr_enabled = false;
+    test_field_offset_set_done = false;
+    CHECK_EQUAL(1, http_field_pattern_match(&fp, nullptr, 0, &fpd, nullptr));
+    CHECK_EQUAL(false, test_field_offset_set_done);
+    test_service_strstr_enabled = true;
+    CHECK_EQUAL(1, http_field_pattern_match(&fp, nullptr, 0, &fpd, nullptr));
+    CHECK_EQUAL(true, test_field_offset_set_done);
+}
+
+TEST(http_url_patterns_tests, match_query_elements)
+{
+    // null check
+    CHECK_EQUAL(0, match_query_elements(nullptr, nullptr, nullptr, 0));
+
+    // pattern matched
+    char appVersion[10];
+    const char pattern1[] = "6.2.3-1515 & Official Build";
+    tMlpPattern packetData;
+    packetData.pattern = (const uint8_t*)pattern1;
+    packetData.patternSize = sizeof(pattern1) - 1;
+    const char pattern2[] = "6.2.3";
+    tMlpPattern userPattern;
+    userPattern.pattern = (const uint8_t*)pattern2;
+    userPattern.patternSize = sizeof(pattern2) - 1;
+    CHECK_EQUAL(sizeof(pattern2), match_query_elements(&packetData, &userPattern, appVersion, 10));
+}
+
+TEST(http_url_patterns_tests, chp_add_candidate_to_tally)
+{
+    CHPMatchTally match_tally;
+    CHPApp chpapp;
+    CHPMatchCandidate chc;
+
+    // verify pattern countdown
+    chc.chpapp = &chpapp;
+    chc.key_pattern_countdown = 1;
+    chc.key_pattern_length_sum = 0;
+    match_tally.push_back(chc);
+    chp_add_candidate_to_tally(match_tally, &chpapp);
+    CHECK_EQUAL(match_tally[0].key_pattern_countdown, 0);
+}
+
+TEST(http_url_patterns_tests, get_http_offsets)
+{
+    // field_offset is set for small payload
+    test_field_offset_set_done = false;
+    pkt.data = (const uint8_t*)"Go";
+    pkt.dsize = 2;
+    hm->get_http_offsets(&pkt, &hsession);
+    CHECK_EQUAL(true, test_field_offset_set_done);
+
+    // find_all is not called for bigger payload when sevice_strstr returns nullptr
+    test_service_strstr_enabled = false;
+    test_find_all_done = false;
+    pkt.data = (const uint8_t*)"GET http://www.w3.org HTTP/1.1";
+    pkt.dsize = strlen((const char*)pkt.data);
+    hm->get_http_offsets(&pkt, &hsession);
+    CHECK_EQUAL(false, test_find_all_done);
+
+    // find_all is called for bigger payload when sevice_strstr returns something
+    test_service_strstr_enabled = true;
+    hm->get_http_offsets(&pkt, &hsession);
+    CHECK_EQUAL(true, test_find_all_done);
+}
+
+TEST(http_url_patterns_tests, rewrite_chp_exist)
+{
+    // don't insert a string that is already present
+    my_buffer[REQ_AGENT_FID] = (const char*)"existing data";
+    my_match.action_data = (char*)"exist";
+    my_match.psize = 0;
+    rewrite_chp(my_buffer[REQ_AGENT_FID], my_length[REQ_AGENT_FID], 0, my_match.psize,
+            my_match.action_data, (const char**)&my_chp_rewritten, 1);
+    CHECK((char*)my_chp_rewritten == nullptr);
+}
+
+TEST(http_url_patterns_tests, rewrite_chp_insert)
+{
+    // insert a string in my_chp_rewritten
+    my_buffer[REQ_AGENT_FID] = (const char*)"existing data";
+    my_match.action_data = (char*)"new";
+    rewrite_chp(my_buffer[REQ_AGENT_FID], my_length[REQ_AGENT_FID], 0, my_match.psize,
+            my_match.action_data, (const char**)&my_chp_rewritten, 1);
+    STRCMP_EQUAL((const char*)my_chp_rewritten, (const char*)my_match.action_data);
+    snort_free(my_chp_rewritten);
+    my_chp_rewritten = nullptr;
+}
+
+TEST(http_url_patterns_tests, rewrite_chp_same)
+{
+    // don't replace if they are same
+    my_chp_rewritten = nullptr;
+    my_buffer[REQ_AGENT_FID] = (const char*)"some data";
+    my_match.action_data = (char*)"some data";
+    rewrite_chp(my_buffer[REQ_AGENT_FID], my_length[REQ_AGENT_FID], 0, my_match.psize,
+            my_match.action_data, (const char**)&my_chp_rewritten, 0);
+    CHECK((char *)my_chp_rewritten == nullptr);
+}
+
+TEST(http_url_patterns_tests, rewrite_chp_replace_null)
+{
+    // replace null aciton data in my_chp_rewritten
+    my_chp_rewritten = nullptr;
+    my_buffer[REQ_AGENT_FID] = (const char*)"existing data";
+    my_match.action_data = nullptr;
+    my_match.psize = 0;
+    rewrite_chp(my_buffer[REQ_AGENT_FID], strlen(my_buffer[REQ_AGENT_FID]), 0, my_match.psize,
+            my_match.action_data, (const char**)&my_chp_rewritten, 0);
+    STRCMP_EQUAL((const char*)my_chp_rewritten, my_buffer[REQ_AGENT_FID]);
+    snort_free(my_chp_rewritten);
+    my_chp_rewritten = nullptr;
+}
+
+TEST(http_url_patterns_tests, rewrite_chp_replace_non_null)
+{
+    // replace non-null aciton data in my_chp_rewritten
+    my_chp_rewritten = nullptr;
+    my_buffer[REQ_AGENT_FID] = (const char*)"existing data";
+    my_match.action_data = (char*)"new data";
+    my_match.psize = 1;
+    rewrite_chp(my_buffer[REQ_AGENT_FID], 1, 0, my_match.psize,
+            my_match.action_data, (const char**)&my_chp_rewritten, 0);
+    STRCMP_EQUAL((const char*)my_chp_rewritten, (const char*)my_match.action_data);
+    snort_free(my_chp_rewritten);
+    my_chp_rewritten = nullptr;
+}
+
+TEST(http_url_patterns_tests, normalize_userid)
+{
+    // no change
+    char uid1[] = "abcd_ID";
+    normalize_userid(uid1);
+    STRCMP_EQUAL((const char*)uid1, (const char*)"abcd_ID");
+
+    // % is replaced with alpha id
+    char uid2[] = "%abcd_ID";
+    normalize_userid(uid2);
+    CHECK(strchr(uid2, '%') == nullptr);
+
+    // % is replaced with numeric id
+    char uid3[] = "%1234";
+    normalize_userid(uid3);
+    CHECK(strchr(uid3, '%') == nullptr);
+}
+
+TEST(http_url_patterns_tests, scan_header_x_working_with)
+{
+    // appid is APP_ID_ASPROXY
+    char *version = snort_strdup("456");
+    const char* data = "ASProxy/123";
+    CHECK(hm->scan_header_x_working_with(data, (uint32_t)strlen(data), (char**)&version) == APP_ID_ASPROXY);
+    STRCMP_EQUAL(version, "123");
+    snort_free(version);
+    version = nullptr;
+
+    // appid is APP_ID_NONE
+    const char* data2 = "Not ASProxy format";
+    CHECK(hm->scan_header_x_working_with(data2, (uint32_t)strlen(data2), (char**)&version) == APP_ID_NONE);
+    CHECK(version == nullptr);
+}
+
+
+TEST(http_url_patterns_tests, scan_chp_defer)
+{
+    // testing DEFER_TO_SIMPLE_DETECT
+    test_find_all_done = false;
+    chpa.appIdInstance = APP_ID_NONE;
+    chpa.action = DEFER_TO_SIMPLE_DETECT;
+    mchp.mpattern = &chpa;
+    cmd.chp_matches[RSP_BODY_FID].push_back(mchp);
+    cmd.cur_ptype = RSP_BODY_FID;
+    mod_config.safe_search_enabled = false;
+    CHECK(hm->scan_chp(cmd, &version, &user, &total_found, &hsession, (const AppIdModuleConfig*)&mod_config) == APP_ID_NONE);
+    CHECK_EQUAL(true, test_find_all_done);
+}
+
+TEST(http_url_patterns_tests, scan_chp_alt_appid)
+{
+    // testing ALTERNATE_APPID
+    test_find_all_done = false;
+    chpa.action_data = my_action_data;
+    chpa.appIdInstance = APP_ID_NONE;
+    chpa.action = ALTERNATE_APPID;
+    mchp.mpattern = &chpa;
+    cmd.chp_matches[RSP_BODY_FID].push_back(mchp);
+    cmd.cur_ptype = RSP_BODY_FID;
+    mod_config.safe_search_enabled = false;
+    CHECK(hm->scan_chp(cmd, &version, &user, &total_found, &hsession, (const AppIdModuleConfig*)&mod_config) == APP_ID_NONE);
+    CHECK_EQUAL(true, test_find_all_done);
+}
+
+
+TEST(http_url_patterns_tests, scan_chp_extract_user)
+{
+    // testing EXTRACT_USER
+    test_find_all_done = false;
+    chpa.action_data = my_action_data;
+    chpa.appIdInstance = APP_ID_NONE;
+    chpa.action = EXTRACT_USER;
+    chpa.psize = 1;
+    mchp.mpattern = &chpa;
+    mchp.start_match_pos = 0;
+    cmd.cur_ptype = RSP_BODY_FID;
+    mod_config.safe_search_enabled = false;
+    cmd.chp_matches[RSP_BODY_FID].push_back(mchp);
+    cmd.buffer[RSP_BODY_FID] = (const char*)"userid\n\rpassword";
+    cmd.length[RSP_BODY_FID] = strlen(cmd.buffer[RSP_BODY_FID]);
+    CHECK(hm->scan_chp(cmd, &version, &user, &total_found, &hsession, (const AppIdModuleConfig*)&mod_config) == APP_ID_NONE);
+    CHECK_EQUAL(true, test_find_all_done);
+    snort_free(user);
+    user = nullptr;
+}
+
+TEST(http_url_patterns_tests, scan_chp_rewrite_field)
+{
+    // testing REWRITE_FIELD
+    test_find_all_done = false;
+    cmd.cur_ptype = RSP_BODY_FID;
+    mod_config.safe_search_enabled = false;
+    chpa.action_data = my_action_data;
+    chpa.appIdInstance = APP_ID_NONE;
+    chpa.action = REWRITE_FIELD;
+    chpa.psize = 1;
+    mchp.mpattern = &chpa;
+    mchp.start_match_pos = 0;
+    cmd.chp_matches[RSP_BODY_FID].push_back(mchp);
+    cmd.buffer[RSP_BODY_FID] = my_chp_data;
+    cmd.length[RSP_BODY_FID] = strlen(cmd.buffer[RSP_BODY_FID]);
+    CHECK(hm->scan_chp(cmd, &version, &user, &total_found, &hsession, (const AppIdModuleConfig*)&mod_config) == APP_ID_NONE);
+    CHECK_EQUAL(true, test_find_all_done);
+    snort_free(const_cast<char*>(cmd.chp_rewritten[RSP_BODY_FID]));
+    cmd.chp_rewritten[RSP_BODY_FID] = nullptr;
+}
+
+TEST(http_url_patterns_tests, scan_chp_insert_without_action)
+{
+    // testing INSERT_FIELD without action_data
+    test_find_all_done = false;
+    cmd.cur_ptype = RSP_BODY_FID;
+    mod_config.safe_search_enabled = false;
+    chpa.action_data = nullptr;
+    chpa.appIdInstance = APP_ID_NONE;
+    chpa.action = INSERT_FIELD;
+    chpa.psize = 1;
+    mchp.mpattern = &chpa;
+    mchp.start_match_pos = 0;
+    cmd.chp_matches[RSP_BODY_FID].push_back(mchp);
+    cmd.buffer[RSP_BODY_FID] = my_chp_data;
+    cmd.length[RSP_BODY_FID] = strlen(cmd.buffer[RSP_BODY_FID]);
+    CHECK(hm->scan_chp(cmd, &version, &user, &total_found, &hsession, (const AppIdModuleConfig*)&mod_config) == APP_ID_NONE);
+    CHECK_EQUAL(true, test_find_all_done);
+    snort_free(const_cast<char*>(cmd.chp_rewritten[RSP_BODY_FID]));
+    cmd.chp_rewritten[RSP_BODY_FID] = nullptr;
+}
+
+TEST(http_url_patterns_tests, scan_chp_insert_with_action)
+{
+    // testing INSERT_FIELD with action_data
+    test_find_all_done = false;
+    cmd.cur_ptype = RSP_BODY_FID;
+    mod_config.safe_search_enabled = false;
+    chpa.action_data = my_action_data;
+    chpa.appIdInstance = APP_ID_NONE;
+    chpa.action = INSERT_FIELD;
+    chpa.psize = 1;
+    mchp.mpattern = &chpa;
+    mchp.start_match_pos = 0;
+    cmd.chp_matches[RSP_BODY_FID].push_back(mchp);
+    cmd.buffer[RSP_BODY_FID] = my_chp_data;
+    cmd.length[RSP_BODY_FID] = strlen(cmd.buffer[RSP_BODY_FID]);
+    CHECK(hm->scan_chp(cmd, &version, &user, &total_found, &hsession, (const AppIdModuleConfig*)&mod_config) == APP_ID_NONE);
+    CHECK_EQUAL(true, test_find_all_done);
+    snort_free(const_cast<char*>(cmd.chp_rewritten[RSP_BODY_FID]));
+    cmd.chp_rewritten[RSP_BODY_FID] = nullptr;
+}
+
+TEST(http_url_patterns_tests, scan_chp_hold_and_default)
+{
+    // testing HOLD_FLOW
+    test_find_all_done = false;
+    chpa.action_data = my_action_data;
+    chpa.appIdInstance = APP_ID_NONE;
+    chpa.action = HOLD_FLOW;
+    mchp.mpattern = &chpa;
+    cmd.chp_matches[RSP_BODY_FID].push_back(mchp);
+    cmd.cur_ptype = RSP_BODY_FID;
+    mod_config.safe_search_enabled = false;
+    chpa.psize = 1;
+    mchp.start_match_pos = 0;
+    cmd.buffer[RSP_BODY_FID] = my_chp_data;
+    cmd.length[RSP_BODY_FID] = strlen(cmd.buffer[RSP_BODY_FID]);
+    CHECK(hm->scan_chp(cmd, &version, &user, &total_found, &hsession, (const AppIdModuleConfig*)&mod_config) == APP_ID_NONE);
+    CHECK_EQUAL(true, test_find_all_done);
+
+    // testing FUTURE_APPID_SESSION_SIP (default action)
+    test_find_all_done = false;
+    chpa.action_data = my_action_data;
+    chpa.appIdInstance = APP_ID_NONE;
+    chpa.action = FUTURE_APPID_SESSION_SIP;
+    mchp.mpattern = &chpa;
+    cmd.chp_matches[RSP_BODY_FID].push_back(mchp);
+    CHECK(hm->scan_chp(cmd, &version, &user, &total_found, &hsession, (const AppIdModuleConfig*)&mod_config) == APP_ID_NONE);
+    CHECK_EQUAL(true, test_find_all_done);
+}
+
+TEST(http_url_patterns_tests, insert_and_process_pattern)
+{
+    DetectorHTTPPattern url_pat, payload_pat;
+    DetectorAppUrlPattern* au_pat = (DetectorAppUrlPattern*)snort_calloc(sizeof(DetectorAppUrlPattern));
+
+    // adding to host_payload_patterns
+    payload_pat.init(nullptr, 6, SINGLE, APP_ID_HTTP, APP_ID_NONE, APP_ID_GOOGLE, APP_ID_NONE); // null pattern test
+    payload_pat.init((const uint8_t*)"Google", 6, (DHPSequence)10, APP_ID_HTTP, APP_ID_NONE, APP_ID_GOOGLE, APP_ID_NONE); // high seq test
+    payload_pat.init((const uint8_t*)"Google", 6, SINGLE, APP_ID_HTTP, APP_ID_NONE, APP_ID_GOOGLE, APP_ID_NONE);
+    snort_free(const_cast<uint8_t*>(payload_pat.pattern));
+    payload_pat.pattern = nullptr;
+    hm->insert_http_pattern(HTTP_PAYLOAD, payload_pat);
+
+    // adding to url_patterns
+    url_pat.pattern = nullptr;
+    hm->insert_http_pattern(HTTP_URL, url_pat);
+
+    // adding to app_url_patterns
+    au_pat->userData.query.pattern = nullptr;
+    au_pat->patterns.host.pattern = nullptr;
+    au_pat->patterns.path.pattern = nullptr;
+    au_pat->patterns.scheme.pattern = nullptr;
+    hm->insert_app_url_pattern(au_pat);
+
+    // The above insert operations would generate errors
+    // 1. by ~HttpPatternMatchers if there is allocation/deallocation mismatch
+    // 2. by CHECK if there is any leaks
+    CHECK(true);
+}
+
+
+TEST(http_url_patterns_tests, identify_user_agent_firefox)
+{
+    test_find_all_enabled = true;
+
+    // null buffPtr in continue_buffer_scan for APP_ID_FIREFOX
+    mpattern.client_id = APP_ID_FIREFOX;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns)); // freed by identify_user_agent
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 100; // exceeding size
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Firefox/57.1", 12, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    // invalid version format following after_match_pos for APP_ID_FIREFOX
+    mpattern.client_id = APP_ID_FIREFOX;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 7;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Firefox/;", 9, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_chrome)
+{
+    test_find_all_enabled = true;
+
+    // null buffPtr in continue_buffer_scan for APP_ID_CHROME
+    mpattern.client_id = APP_ID_CHROME;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 100; // exceeding size
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Chrome/64.0", 11, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    // invalid version format following after_match_pos for APP_ID_CHROME
+    mpattern.client_id = APP_ID_CHROME;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 6;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Chrome/;", 8, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_android)
+{
+    // invalid version format following after_match_pos for APP_ID_ANDROID_BROWSER
+    test_find_all_enabled = true;
+    mpattern.client_id = APP_ID_ANDROID_BROWSER;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 7;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Android/;", 9, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_bittorrent)
+{
+    test_find_all_enabled = true;
+
+    // null buffPtr in continue_buffer_scan for APP_ID_BITTORRENT
+    mpattern.client_id = APP_ID_BITTORRENT;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 100; // exceeding size
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("BitTorrent/64.0", 15, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    // invalid version format following after_match_pos for APP_ID_BITTORRENT
+    mpattern.client_id = APP_ID_BITTORRENT;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns)); // freed by identify_user_agent
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 10;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("BitTorrent/;", 12, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_google_desktop)
+{
+    test_find_all_enabled = true;
+
+    // exceeding after_match_pos for APP_ID_GOOGLE_DESKTOP
+    mpattern.client_id = APP_ID_GOOGLE_DESKTOP;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns)); // freed by identify_user_agent
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 100; // exceeding size
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("GoogleDesktop/12", 16, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    // no tab/space/slash format for APP_ID_GOOGLE_DESKTOP
+    mpattern.client_id = APP_ID_GOOGLE_DESKTOP;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 13;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("GoogleDesktopVersion12", 22, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    // invalid version format following after_match_pos for APP_ID_GOOGLE_DESKTOP
+    mpattern.client_id = APP_ID_GOOGLE_DESKTOP;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 13;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("GoogleDesktop/;", 15, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_wget)
+{
+    // exceeding after_match_pos for APP_ID_WGET
+    test_find_all_enabled = true;
+    mpattern.client_id = APP_ID_WGET;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 100; // exceeding size
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Wget/12", 7, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_fake_version_appid)
+{
+    // exceeding after_match_pos for FAKE_VERSION_APP_ID
+    test_find_all_enabled = true;
+    mpattern.client_id = FAKE_VERSION_APP_ID;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns)); // freed by identify_user_agent
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 100; // exceeding size
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Fake/12", 7, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_blackberry)
+{
+    test_find_all_enabled = true;
+
+    // null buffPtr in continue_buffer_scan for APP_ID_BLACKBERRY_BROWSER
+    mpattern.client_id = APP_ID_BLACKBERRY_BROWSER;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns)); // freed by identify_user_agent
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 100;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Blackberry", 10, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    // invalid version format following after_match_pos for APP_ID_BLACKBERRY_BROWSER
+    mpattern.client_id = APP_ID_BLACKBERRY_BROWSER;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 10;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Blackberry/;", 12, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_http)
+{
+    // APP_ID_HTTP
+    test_find_all_enabled = true;
+    mpattern.client_id = APP_ID_HTTP;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 4;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("HTTP/2.1", 8, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_mobiledirect)
+{
+    // mobileDetect
+    test_find_all_enabled = true;
+    mpattern.client_id = APP_ID_SAFARI_MOBILE_DUMMY;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns));
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 11;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("SafariMobile/2.1", 16, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    snort_free(version);
+    version = nullptr;
+    test_find_all_enabled = false;
+}
+
+TEST(http_url_patterns_tests, identify_user_agent_skypedirect)
+{
+    // skypeDetect
+    test_find_all_enabled = true;
+    mpattern.client_id = APP_ID_SKYPE;
+    mock_mp = (MatchedPatterns*)snort_calloc(sizeof(MatchedPatterns)); // freed by identify_user_agent
+    mock_mp->mpattern = &mpattern;
+    mock_mp->after_match_pos = 5;
+    mock_mp->next = nullptr;
+    hm->identify_user_agent("Skype/2.1", 9, service_id, client_id, &version);
+    STRCMP_EQUAL(version, "");
+    CHECK_EQUAL(service_id, APP_ID_SKYPE_AUTH);
+    CHECK_EQUAL(client_id, APP_ID_SKYPE);
+    snort_free(version);
+    version = nullptr;
+    test_find_all_enabled = false;
+}
+
+int main(int argc, char** argv)
+{
+    int return_value = CommandLineTestRunner::RunAllTests(argc, argv);
+    return return_value;
+}