Merge pull request #1003 in SNORT/snort3 from appid_lua_detectors to master

Squashed commit of the following:

commit d6a01cdeb7716a02aeb45007cba39d928d223675
Author: davis mcpherson <davmcphe.cisco.com>
Date:   Tue Aug 29 08:05:04 2017 -0400

    hard code use of 'ac_full' as search method for search engine for appid

    select 'ac_full' as search method for appid for now as it is the only one that supports 'find_all', also force enable dfa flag in search engine, improve encapsulation of appid pattern match service class

diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index 2715472d1b890d1a6fdfe7a3a164fad933db625c..425aa8892f26c736ce218cb48f9176f3cd5ac78f 100644 (file)
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -49,19 +49,16 @@
 
 AppIdDiscovery::AppIdDiscovery()
 {
-    tcp_patterns = new SearchTool;
-    udp_patterns = new SearchTool;
+    tcp_patterns = new SearchTool("ac_full", true);
+    udp_patterns = new SearchTool("ac_full", true);
 }
 
 AppIdDiscovery::~AppIdDiscovery()
 {
-    AppIdPatternMatchNode* pd = pattern_data_list;
-    while (pd)
-    {
-        pattern_data_list = pd->next;
-        snort_free(pd);
-        pd = pattern_data_list;
-    }
+    for (auto pd : pattern_data )
+        delete pd;
+
+    pattern_data.clear();
 
     delete tcp_patterns;
     delete udp_patterns;
@@ -105,13 +102,8 @@ void AppIdDiscovery::register_detector(std::string name, AppIdDetector* cd,  IpP
 void AppIdDiscovery::add_pattern_data(AppIdDetector* detector, SearchTool* st, int position, const
     uint8_t* const pattern, unsigned size, unsigned nocase)
 {
-    AppIdPatternMatchNode* pd =
-        (AppIdPatternMatchNode*)snort_calloc(sizeof(AppIdPatternMatchNode));
-    pd->service = detector;
-    pd->pattern_start_pos = position;
-    pd->size = size;
-    pd->next = pattern_data_list;
-    pattern_data_list = pd;
+    AppIdPatternMatchNode* pd = new AppIdPatternMatchNode(detector, position, size);
+    pattern_data.push_back(pd);
     st->add((const char*)pattern, size, pd, nocase);
 }
 

diff --git a/src/network_inspectors/appid/appid_discovery.h b/src/network_inspectors/appid/appid_discovery.h
index 19712cf38abcc8f00823b5ad96ecb7dff8ba52bc..391c7f764b1145b713bcac0569bb090d92462d04 100644 (file)
--- a/src/network_inspectors/appid/appid_discovery.h
+++ b/src/network_inspectors/appid/appid_discovery.h
@@ -22,8 +22,9 @@
 #ifndef APPID_DISCOVERY_H
 #define APPID_DISCOVERY_H
 
-#include <string>
 #include <map>
+#include <string>
+#include <vector>
 
 #include "protocols/protocol_ids.h"
 #include "search_engines/search_tool.h"
@@ -44,12 +45,25 @@ struct Packet;
 #define SCAN_HTTP_XWORKINGWITH_FLAG (1<<7)
 #define SCAN_HTTP_CONTENT_TYPE_FLAG (1<<8)
 
-struct AppIdPatternMatchNode
+class AppIdPatternMatchNode
 {
-    AppIdPatternMatchNode* next = nullptr;
-    int pattern_start_pos = 0;
-    unsigned size = 0;
-    AppIdDetector* service = nullptr;
+public:
+    AppIdPatternMatchNode(AppIdDetector* detector, int start, unsigned len)
+        : service(detector), pattern_start_pos(start), size(len)
+    {}
+    ~AppIdPatternMatchNode() {}
+
+    bool valid_match(int end_position)
+    {
+        if ( pattern_start_pos >= 0 && pattern_start_pos != (end_position - (int)size) )
+            return false;
+        else
+            return true;
+    }
+
+    AppIdDetector* service;
+    int pattern_start_pos;
+    unsigned size;
 };
 
 struct ServiceMatch
@@ -101,7 +115,7 @@ protected:
     int tcp_pattern_count = 0;
     SearchTool* udp_patterns = nullptr;
     int udp_pattern_count = 0;
-    AppIdPatternMatchNode* pattern_data_list = nullptr;
+    std::vector<AppIdPatternMatchNode*> pattern_data;
 };
 #endif
 

diff --git a/src/network_inspectors/appid/appid_utils/sf_mlmp.cc b/src/network_inspectors/appid/appid_utils/sf_mlmp.cc
index 60cd63035eb9fe103883630ec5ed72db88bc9b62..a746b3e8331c36560d7024502a22ca8bb9210060 100644 (file)
--- a/src/network_inspectors/appid/appid_utils/sf_mlmp.cc
+++ b/src/network_inspectors/appid/appid_utils/sf_mlmp.cc
@@ -211,7 +211,7 @@ static int createTreesRecusively(tMlmpTree* rootNode)
     tPatternNode* ddPatternNode;
 
     /* set up the MPSE for url patterns */
-    patternMatcher = rootNode->patternTree = new SearchTool;
+    patternMatcher = rootNode->patternTree = new SearchTool("ac_full", true);
 
     for (primaryPatternNode = rootNode->patternList;
         primaryPatternNode;

diff --git a/src/network_inspectors/appid/appid_utils/sf_multi_mpse.cc b/src/network_inspectors/appid/appid_utils/sf_multi_mpse.cc
index a96e7d8da4450a0966fd94a638e768b11658a089..ab3098df9df8368f9b1dd91bd3a9b37ce3225fc9 100644 (file)
--- a/src/network_inspectors/appid/appid_utils/sf_multi_mpse.cc
+++ b/src/network_inspectors/appid/appid_utils/sf_multi_mpse.cc
@@ -184,7 +184,7 @@ static int createTreesRecusively(void* root)
     tPatternList* patternNode;
 
     /* set up the MPSE for url patterns */
-    if (!(patternMatcher = rootNode->patternTree = new SearchTool))
+    if (!(patternMatcher = rootNode->patternTree = new SearchTool("ac_full", true)))
         return -1;
 
     for (patternNode = rootNode->patternList;

diff --git a/src/network_inspectors/appid/client_plugins/client_discovery.cc b/src/network_inspectors/appid/client_plugins/client_discovery.cc
index c58059d1526ac1047e7c1dc270ac947df0d1a494..08b87c3037f39eb1879f547b4cc51f196093934b 100644 (file)
--- a/src/network_inspectors/appid/client_plugins/client_discovery.cc
+++ b/src/network_inspectors/appid/client_plugins/client_discovery.cc
@@ -17,7 +17,7 @@
 // 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 //--------------------------------------------------------------------------
 
-// client_app_bit.cc author Sourcefire Inc.
+// client_discovery.cc author Sourcefire Inc.
 
 #ifdef HAVE_CONFIG_H
 #include "config.h"
@@ -128,35 +128,33 @@ static int pattern_match(void* id, void* /*unused_tree*/, int match_end_pos, voi
 {
     ClientAppMatch** matches = (ClientAppMatch**)data;
     AppIdPatternMatchNode* pd = (AppIdPatternMatchNode*)id;
-    ClientAppMatch* cam;
-
-    // Ignore matches that don't start at the expected position.
-    if ( pd->pattern_start_pos >= 0 && pd->pattern_start_pos != (match_end_pos - (int)pd->size))
-        return 0;
 
-    for (cam = *matches; cam; cam = cam->next)
+    if ( pd->valid_match(match_end_pos) )
     {
-        if (cam->detector == pd->service)
-            break;
-    }
+        ClientAppMatch* cam;
 
-    if (cam)
-        cam->count++;
-    else
-    {
-        if (match_free_list)
-        {
-            cam = match_free_list;
-            match_free_list = cam->next;
-            memset(cam, 0, sizeof(*cam));
-        }
+        for (cam = *matches; cam; cam = cam->next)
+            if (cam->detector == pd->service)
+                break;
+
+        if (cam)
+            cam->count++;
         else
-            cam = (ClientAppMatch*)snort_calloc(sizeof(ClientAppMatch));
+        {
+            if (match_free_list)
+            {
+                cam = match_free_list;
+                match_free_list = cam->next;
+                memset(cam, 0, sizeof(*cam));
+            }
+            else
+                cam = (ClientAppMatch*)snort_calloc(sizeof(ClientAppMatch));
 
-        cam->count = 1;
-        cam->detector =  static_cast<const ClientDetector*>(pd->service);
-        cam->next = *matches;
-        *matches = cam;
+            cam->count = 1;
+            cam->detector =  static_cast<const ClientDetector*>(pd->service);
+            cam->next = *matches;
+            *matches = cam;
+        }
     }
 
     return 0;
@@ -231,10 +229,9 @@ ClientAppMatch* ClientDiscovery::find_detector_candidates(const Packet* pkt, IpP
     else
         patterns = ClientDiscovery::get_instance().udp_patterns;
 
-    if (!patterns)
-        return nullptr;
+    if ( patterns )
+        patterns->find_all((char*)pkt->data, pkt->dsize, &pattern_match, false, (void*)&match_list);
 
-    patterns->find_all((char*)pkt->data, pkt->dsize, &pattern_match, false, (void*)&match_list);
     return match_list;
 }
 

diff --git a/src/network_inspectors/appid/client_plugins/client_discovery.h b/src/network_inspectors/appid/client_plugins/client_discovery.h
index 0e31f1e13acd08f956ba6daeec44af9b2a87a286..01daff61cbbae3d3972924bafc97387808b0d30c 100644 (file)
--- a/src/network_inspectors/appid/client_plugins/client_discovery.h
+++ b/src/network_inspectors/appid/client_plugins/client_discovery.h
@@ -17,7 +17,7 @@
 // 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 //--------------------------------------------------------------------------
 
-// client_app_config.h author Sourcefire Inc.
+// client_discovery.h author Sourcefire Inc.
 
 #ifndef CLIENT_DISCOVERY_H
 #define CLIENT_DISCOVERY_H

diff --git a/src/network_inspectors/appid/detector_plugins/detector_dns.cc b/src/network_inspectors/appid/detector_plugins/detector_dns.cc
index 889f5340829375d5fbbf97a02f64b631afc0d637..cdb55dd2e4137ef688c17ee031797cecf2ae058a 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/detector_dns.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_dns.cc
@@ -184,7 +184,7 @@ static int dns_host_detector_create_matcher(DetectorDNSHostPattern* list)
     if (serviceDnsConfig.dns_host_host_matcher)
         delete serviceDnsConfig.dns_host_host_matcher;
 
-    serviceDnsConfig.dns_host_host_matcher = new SearchTool;
+    serviceDnsConfig.dns_host_host_matcher = new SearchTool("ac_full", true);
     if (!serviceDnsConfig.dns_host_host_matcher)
         return 0;
 

diff --git a/src/network_inspectors/appid/detector_plugins/detector_imap.cc b/src/network_inspectors/appid/detector_plugins/detector_imap.cc
index 1ebce6fd3baacae1597d33acb7a4313dc1dec7c9..e1902b49deebe5cfa7b7c4e6adf40d52278b13b2 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/detector_imap.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_imap.cc
@@ -516,7 +516,7 @@ ImapClientDetector::~ImapClientDetector()
 void ImapClientDetector::do_custom_init()
 {
     unsigned index = 0;
-    cmd_matcher = new SearchTool;
+    cmd_matcher = new SearchTool("ac_full", true);
 
     if ( tcp_patterns.size() )
         for (auto& pat : tcp_patterns)

diff --git a/src/network_inspectors/appid/detector_plugins/detector_pattern.cc b/src/network_inspectors/appid/detector_plugins/detector_pattern.cc
index c2885bd98c3ccec7f0f7fc39825ed7e94b54b87c..50e99681f9d37c156bbae79e7dac771e5b51a755 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/detector_pattern.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_pattern.cc
@@ -122,7 +122,7 @@ static void register_pattern(SearchTool** patterns, Pattern* pattern)
 {
     if (!*patterns)
     {
-        *patterns = new SearchTool;
+        *patterns = new SearchTool("ac_full", true);
         if (!*patterns)
         {
             ErrorMessage("Error initializing the pattern table\n");

diff --git a/src/network_inspectors/appid/detector_plugins/detector_pop3.cc b/src/network_inspectors/appid/detector_plugins/detector_pop3.cc
index 6ad92ce8cae42c8b617596a622e07ac9fa3a2f52..07c6a076d7248d0cd0046fde041e9350635a49cd 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/detector_pop3.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_pop3.cc
@@ -209,7 +209,7 @@ Pop3ClientDetector::~Pop3ClientDetector()
 void Pop3ClientDetector::do_custom_init()
 {
     unsigned index = 0;
-    cmd_matcher = new SearchTool;
+    cmd_matcher = new SearchTool("ac_full", true);
 
     if ( tcp_patterns.size() )
         for (auto& pat : tcp_patterns)

diff --git a/src/network_inspectors/appid/detector_plugins/detector_sip.cc b/src/network_inspectors/appid/detector_plugins/detector_sip.cc
index 9ff0ea6eaa121b019339c6947f79661536102e3b..9b53c0b6f1feb105ca9bb94819c8d2e179d7ba4a 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/detector_sip.cc
+++ b/src/network_inspectors/appid/detector_plugins/detector_sip.cc
@@ -548,14 +548,15 @@ void SipEventHandler::client_handler(SipEvent& sip_event, AppIdSession* asd)
         return;
 
 success:
-    //client detection successful
-    client->add_app(asd, APP_ID_SIP, ClientAppId, clientVersion);
-    appid_stats.sip_clients++;
+    if( !asd->is_client_detected() )
+    {
+        //client detection successful
+        client->add_app(asd, APP_ID_SIP, ClientAppId, clientVersion);
+        appid_stats.sip_clients++;
+    }
 
     if ( fd->user_name.size() )
         client->add_user(asd, fd->user_name.c_str(), APP_ID_SIP, true);
-
-    asd->set_client_detected();
 }
 
 void SipEventHandler::service_handler(SipEvent& sip_event, AppIdSession* asd)

diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc b/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc
index 6c66bf68921a6e545a117c8e542d671f53df64e6..971acea8ec321bceecdfbffaf626b044fa5e4063 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc
+++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.cc
@@ -666,7 +666,7 @@ int HttpPatternMatchers::process_host_patterns(DetectorHTTPPatterns patterns)
 int HttpPatternMatchers::process_chp_list(CHPListElement* chplist)
 {
     for (size_t i = 0; i <= MAX_PATTERN_TYPE; i++)
-        chp_matchers[i] = new SearchTool;
+        chp_matchers[i] = new SearchTool("ac_full", true);
 
     for (CHPListElement* chpe = chplist; chpe; chpe = chpe->next)
         chp_matchers[chpe->chp_action.ptype]->add(chpe->chp_action.pattern,
@@ -707,7 +707,7 @@ static FieldPattern http_field_patterns[] =
 
 static SearchTool* process_http_field_patterns(FieldPattern* patternList, size_t patternListCount)
 {
-    SearchTool* patternMatcher = new SearchTool;
+    SearchTool* patternMatcher = new SearchTool("ac_full", true);
 
     for (size_t i=0; i < patternListCount; i++)
         patternMatcher->add( (char*)patternList[i].data, patternList[i].length,

diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
index 252b75b1033a5d4bc78df8eb986967fa3f9eeb10..45171d9db4c9c8505474d94486973e2930f766d6 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
+++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
@@ -292,7 +292,10 @@ struct HostUrlDetectorPattern
 class HttpPatternMatchers
 {
 public:
-    HttpPatternMatchers() { }
+    HttpPatternMatchers()
+        : url_matcher("ac_full", true), client_agent_matcher("ac_full", true), via_matcher("ac_full", true),
+          content_type_matcher("ac_full", true)
+    { }
     ~HttpPatternMatchers();
 
     static HttpPatternMatchers* get_instance();

diff --git a/src/network_inspectors/appid/service_plugins/service_discovery.cc b/src/network_inspectors/appid/service_plugins/service_discovery.cc
index e4bd1abce6e06b0ec31ba59ccb39a9a1977c40b4..6a84145963b9a8a994f8fe0efca31c2b61489365 100644 (file)
--- a/src/network_inspectors/appid/service_plugins/service_discovery.cc
+++ b/src/network_inspectors/appid/service_plugins/service_discovery.cc
@@ -210,27 +210,28 @@ static int pattern_match(void* id, void*, int match_end_pos, void* data, void*)
 {
     ServiceMatch** matches = (ServiceMatch**)data;
     AppIdPatternMatchNode* pd = (AppIdPatternMatchNode*)id;
-    ServiceMatch* sm;
 
-    // Ignore matches that don't start at the expected position.
-    if ( pd->pattern_start_pos >= 0 && pd->pattern_start_pos != (match_end_pos - (int)pd->size) )
-        return 0;
+    if ( pd->valid_match(match_end_pos) )
+    {
+        ServiceMatch* sm;
 
-    for (sm = *matches; sm; sm = sm->next)
-        if (sm->service == (ServiceDetector*)pd->service)
-            break;
+        for (sm = *matches; sm; sm = sm->next)
+            if (sm->service == (ServiceDetector*)pd->service)
+                break;
 
-    if (sm)
-        sm->count++;
-    else
-    {
-        sm = (ServiceMatch*)snort_calloc(sizeof(ServiceMatch));
-        sm->count++;
-        sm->service = static_cast<ServiceDetector*>(pd->service);
-        sm->size = pd->size;
-        sm->next = *matches;
-        *matches = sm;
+        if (sm)
+            sm->count++;
+        else
+        {
+            sm = (ServiceMatch*)snort_calloc(sizeof(ServiceMatch));
+            sm->count++;
+            sm->service = static_cast<ServiceDetector*>(pd->service);
+            sm->size = pd->size;
+            sm->next = *matches;
+            *matches = sm;
+        }
     }
+
     return 0;
 }
 

diff --git a/src/network_inspectors/appid/service_plugins/service_mdns.cc b/src/network_inspectors/appid/service_plugins/service_mdns.cc
index ccc4f484be1edb0513d322a66a84e9828a20f0a8..a0dd8534599b5067a31e5d6e936745dd6cb22325 100644 (file)
--- a/src/network_inspectors/appid/service_plugins/service_mdns.cc
+++ b/src/network_inspectors/appid/service_plugins/service_mdns.cc
@@ -104,7 +104,7 @@ MdnsServiceDetector::MdnsServiceDetector(ServiceDiscovery* sd)
         { 5353, IpProtocol::UDP, false },
     };
 
-    matcher = new SearchTool;
+    matcher = new SearchTool("ac_full", true);
     for (unsigned i = 0; i < sizeof(patterns) / sizeof(*patterns); i++)
         matcher->add((char*)patterns[i].pattern, patterns[i].length, &patterns[i]);
     matcher->prep();

diff --git a/src/network_inspectors/appid/service_plugins/service_ssl.cc b/src/network_inspectors/appid/service_plugins/service_ssl.cc
index 0b671f874d4f8cbd8102851124265aaba2bf1e8e..9421c503b839e03e912a5d6ffc277dcd2894abc6 100644 (file)
--- a/src/network_inspectors/appid/service_plugins/service_ssl.cc
+++ b/src/network_inspectors/appid/service_plugins/service_ssl.cc
@@ -239,7 +239,7 @@ static int ssl_detector_create_matcher(SearchTool** matcher, DetectorSSLCertPatt
     if (*matcher)
         delete *matcher;
 
-    if (!(*matcher = new SearchTool))
+    if (!(*matcher = new SearchTool("ac_full", true)))
         return 0;
 
     patternIndex = &size;

diff --git a/src/search_engines/search_tool.cc b/src/search_engines/search_tool.cc
index 4a78d4aaf019397ff412f8c7986bd6ff4a437e20..46efacab82940d7b35d5003987508f87ae8ee4d8 100644 (file)
--- a/src/search_engines/search_tool.cc
+++ b/src/search_engines/search_tool.cc
@@ -27,11 +27,13 @@
 
 #include "managers/mpse_manager.h"
 
-SearchTool::SearchTool(const char* method)
+SearchTool::SearchTool(const char* method, bool dfa)
 {
     mpse = MpseManager::get_search_engine(method);
     assert(mpse);
     max_len = 0;
+    if( dfa )
+        mpse->set_opt(1);
 }
 
 SearchTool::~SearchTool()

diff --git a/src/search_engines/search_tool.h b/src/search_engines/search_tool.h
index 1c87ec82975998c989a0fb883c1d8facdc898701..a1f56b4560d43029efcf5de3ad6050321c545f1f 100644 (file)
--- a/src/search_engines/search_tool.h
+++ b/src/search_engines/search_tool.h
@@ -25,7 +25,7 @@
 class SO_PUBLIC SearchTool
 {
 public:
-    SearchTool(const char* method = nullptr);
+    SearchTool(const char* method = nullptr, bool dfa = false);
     ~SearchTool();
 
     void add(const char* pattern, unsigned len, int s_id, bool no_case = true);