Merge pull request #1303 in SNORT/snort3 from xhash_alternatives to master

Squashed commit of the following:

commit 2a80681d630c8ef65435733247f7539e42441b59
Author: Masud Hasan <mashasan@cisco.com>
Date:   Fri Jul 6 15:26:11 2018 -0400

    appid: Replacing xhash with alternative data structures

diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc
index c4045e38dcb1b5fb2fe6e38e9329270876519d6b..45fc82cb856089a7a7b12d6e8273f559a6a557de 100644 (file)
--- a/src/network_inspectors/appid/appid_discovery.cc
+++ b/src/network_inspectors/appid/appid_discovery.cc
@@ -885,7 +885,7 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto
         asd.length_sequence.sequence_cnt++;
         asd.length_sequence.sequence[index].direction = direction;
         asd.length_sequence.sequence[index].length    = p->dsize;
-        AppId id = find_length_app_cache(&asd.length_sequence);
+        AppId id = find_length_app_cache(asd.length_sequence);
         if (id > APP_ID_NONE)
         {
             asd.service.set_port_service_id(id);

diff --git a/src/network_inspectors/appid/appid_utils/network_set.cc b/src/network_inspectors/appid/appid_utils/network_set.cc
index 0cd3321d090b9c8635d2c4ae4676a59a19636ccf..8a0d3b9473e6ea183c7f66fc684d2f9a8eb37d6d 100644 (file)
--- a/src/network_inspectors/appid/appid_utils/network_set.cc
+++ b/src/network_inspectors/appid/appid_utils/network_set.cc
@@ -39,23 +39,7 @@ int NetworkSetManager::create(NetworkSet** network_set)
 
     NetworkSet* tmp = (NetworkSet*)snort_calloc(sizeof(NetworkSet));
     sflist_init(&tmp->networks);
-    tmp->ids = xhash_new(64, sizeof(unsigned), 0, 0, 0, nullptr, nullptr, 1);
-    if (tmp->ids == nullptr)
-    {
-        ErrorMessage("NetworkSet:Out of memory (wanted %zu bytes)", sizeof(NetworkSet));
-        destroy(tmp);
-        return -1;
-    }
-
     sflist_init(&tmp->networks6);
-    tmp->ids6 = xhash_new(64, sizeof(unsigned), 0, 0, 0, nullptr, nullptr, 1);
-    if (tmp->ids6 == nullptr)
-    {
-        ErrorMessage("NetworkSet:Out of memory (wanted %zu bytes)", sizeof(NetworkSet));
-        destroy(tmp);
-        return -1;
-    }
-
     *network_set = tmp;
     return 0;
 }
@@ -71,14 +55,14 @@ int NetworkSetManager::destroy(NetworkSet* network_set)
         network_set->pnetwork = nullptr;
     }
     sflist_static_free_all(&network_set->networks, &snort_free);
-    xhash_delete(network_set->ids);
+    network_set->ids.clear();
     if (network_set->pnetwork6)
     {
         snort_free(network_set->pnetwork6);
         network_set->pnetwork6 = nullptr;
     }
     sflist_static_free_all(&network_set->networks6, &snort_free);
-    xhash_delete(network_set->ids6);
+    network_set->ids6.clear();
     snort_free(network_set);
 
     return 0;
@@ -126,11 +110,9 @@ int NetworkSetManager::add_network_range_ex(NetworkSet* network_set, uint32_t ra
     }
 
     sflist_add_tail(&network_set->networks, (void*)network);
-    int rval = xhash_add(network_set->ids, &network->info.id, &network->info.id);
-    if (rval != XHASH_OK && rval != XHASH_INTABLE)
+    if (network_set->ids.insert(network->info.id).second == false)
     {
-        ErrorMessage("NetworkSet:Out of memory");
-        snort_free(network);
+        ErrorMessage("NetworkSet: Failed to add id %u\n", network->info.id);
         return -1;
     }
 
@@ -186,11 +168,9 @@ int NetworkSetManager::add_network_range6(NetworkSet* network_set, NSIPv6Addr* r
     }
 
     sflist_add_tail(&network_set->networks6, (void*)network);
-    int rval = xhash_add(network_set->ids6, &network->info.id, &network->info.id);
-    if (rval != XHASH_OK && rval != XHASH_INTABLE)
+    if (network_set->ids6.insert(network->info.id).second == false)
     {
-        ErrorMessage("NetworkSet:Out of memory");
-        snort_free(network);
+        ErrorMessage("NetworkSet: Failed to add IPv6 id %u\n", network->info.id);
         return -1;
     }
 
@@ -943,8 +923,6 @@ int NetworkSetManager::reduce_network_set6(SF_LIST* networks)
 
 int NetworkSetManager::reduce(NetworkSet* network_set)
 {
-    XHashNode* hnode;
-    unsigned id;
     int rval;
     SF_LIST ordered_networks;
     Network* network;
@@ -955,11 +933,8 @@ int NetworkSetManager::reduce(NetworkSet* network_set)
     if (!network_set)
         return -1;
 
-    for (hnode = xhash_gfindfirst(network_set->ids);
-        hnode;
-        hnode=xhash_gfindnext(network_set->ids))
+    for (auto& id : network_set->ids)
     {
-        id = *(unsigned*)(hnode->data);
         if ((rval = order_by_netmask(&ordered_networks, &network_set->networks, id)) != 0)
         {
             sflist_free_all(&ordered_networks, &snort_free);
@@ -1019,10 +994,8 @@ int NetworkSetManager::reduce(NetworkSet* network_set)
         }
     }
 
-    for (hnode = xhash_gfindfirst(network_set->ids6); hnode; hnode=xhash_gfindnext(
-            network_set->ids6))
+    for (auto& id : network_set->ids6)
     {
-        id = *(unsigned*)(hnode->data);
         if ((rval = order_by_netmask(&ordered_networks, &network_set->networks6, id)) != 0)
         {
             sflist_free_all(&ordered_networks, &snort_free);

diff --git a/src/network_inspectors/appid/appid_utils/network_set.h b/src/network_inspectors/appid/appid_utils/network_set.h
index e032fc28ca96d0de49dbd0c9da7d599f660dcf6e..4d49e7c03d0ce50a6318597d238636eafd2523ba 100644 (file)
--- a/src/network_inspectors/appid/appid_utils/network_set.h
+++ b/src/network_inspectors/appid/appid_utils/network_set.h
@@ -25,8 +25,8 @@
 #ifdef USE_RNA_CONFIG
 
 #include <cstdio>
+#include <unordered_set>
 
-#include "hash/xhash.h"
 #include "protocols/ipv6.h"
 #include "utils/sflsq.h"
 
@@ -102,11 +102,11 @@ struct NetworkSet
 {
     NetworkSet* next;
     SF_LIST networks;
-    XHash* ids;
+    std::unordered_set<unsigned> ids;
     Network** pnetwork;
     unsigned count;
     SF_LIST networks6;
-    XHash* ids6;
+    std::unordered_set<unsigned> ids6;
     Network6** pnetwork6;
     unsigned count6;
 };

diff --git a/src/network_inspectors/appid/length_app_cache.cc b/src/network_inspectors/appid/length_app_cache.cc
index 4ba2162ef1d8ba61f5807887bcee6049c033a2d5..8a759dd8e3375f81c1faa7420259f96301f31bb9 100644 (file)
--- a/src/network_inspectors/appid/length_app_cache.cc
+++ b/src/network_inspectors/appid/length_app_cache.cc
@@ -25,50 +25,39 @@
 
 #include "length_app_cache.h"
 
-#include "application_ids.h"
-#include "hash/xhash.h"
+#include <map>
+
 #include "log/messages.h"
 #include "main/thread.h"
 
-using namespace snort;
+#include "application_ids.h"
 
-#define HASH_NUM_ROWS (1024)
+using namespace snort;
 
-static XHash* lengthCache = nullptr;
+static std::map<LengthKey, AppId>* length_cache = nullptr;
 
 void init_length_app_cache()
 {
-    if (!(lengthCache = xhash_new(HASH_NUM_ROWS, sizeof(LengthKey), sizeof(AppId),
-            0, 0, nullptr, nullptr, 0)))
-    {
-        ErrorMessage("lengthAppCache: Failed to allocate length cache!");
-    }
+    length_cache = new std::map<LengthKey, AppId>;
 }
 
 void free_length_app_cache()
 {
-    if (lengthCache)
-    {
-        xhash_delete(lengthCache);
-        lengthCache = nullptr;
-    }
+    delete length_cache;
+    length_cache = nullptr;
 }
 
-AppId find_length_app_cache(const LengthKey* key)
+AppId find_length_app_cache(const LengthKey& key)
 {
-    AppId* val = (AppId*)xhash_find(lengthCache, (void*)key);
-    if (val == nullptr)
+    auto entry = length_cache->find(key);
+    if (entry == length_cache->end())
         return APP_ID_NONE;    /* no match */
     else
-        return *val;           /* match found */
+        return entry->second;  /* match found */
 }
 
-bool add_length_app_cache(const LengthKey* key, AppId val)
+bool add_length_app_cache(const LengthKey& key, AppId val)
 {
-    if (xhash_add(lengthCache, (void*)key, (void*)&val))
-    {
-        return false;
-    }
-    return true;
+    return (length_cache->insert(std::make_pair(key, val))).second == true;
 }
 

diff --git a/src/network_inspectors/appid/length_app_cache.h b/src/network_inspectors/appid/length_app_cache.h
index 05c3d5c864c89d0afdacd553583898696d3db02d..5261927e67f467c4c3cb3fa01791c75e52a2d215 100644 (file)
--- a/src/network_inspectors/appid/length_app_cache.h
+++ b/src/network_inspectors/appid/length_app_cache.h
@@ -44,14 +44,43 @@ struct LengthKey
     IpProtocol proto = IpProtocol::PROTO_NOT_SET;  // IpProtocol::TCP or IpProtocol::UDP
     uint8_t sequence_cnt = 0;                      // num valid entries in sequence
     LengthSequenceEntry sequence[LENGTH_SEQUENCE_CNT_MAX];
+
+    // Used by map where LengthKey object is the key
+    bool operator<(const LengthKey& right) const
+    {
+        if (proto < right.proto)
+            return true;
+        else if (right.proto < proto)
+            return false;
+
+        if (sequence_cnt < right.sequence_cnt)
+            return true;
+        else if (right.sequence_cnt < sequence_cnt)
+            return false;
+
+        for (uint8_t i = 0; i < LENGTH_SEQUENCE_CNT_MAX; ++i)
+        {
+            if (sequence[i].direction < right.sequence[i].direction)
+                return true;
+            else if (right.sequence[i].direction < sequence[i].direction)
+                return false;
+
+            if (sequence[i].length < right.sequence[i].length)
+                return true;
+            else if (right.sequence[i].length < sequence[i].length)
+                return false;
+        }
+
+        return false;
+    }
 };
 
 #pragma pack()
 
 void init_length_app_cache();
 void free_length_app_cache();
-AppId find_length_app_cache(const LengthKey*);
-bool add_length_app_cache(const LengthKey*, AppId);
+AppId find_length_app_cache(const LengthKey&);
+bool add_length_app_cache(const LengthKey&, AppId);
 
 #endif
 

diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc
index 1c0b01dc8e20064c8c9b92b8b77c2c8fadc458ef..e556db646db0934e88b80921ae5c0676dd91913f 100644 (file)
--- a/src/network_inspectors/appid/lua_detector_api.cc
+++ b/src/network_inspectors/appid/lua_detector_api.cc
@@ -27,27 +27,28 @@
 
 #include <lua.hpp>
 #include <pcre.h>
+#include <unordered_map>
+
+#include "log/messages.h"
+#include "main/snort_types.h"
+#include "profiler/profiler.h"
+#include "protocols/packet.h"
 
 #include "app_forecast.h"
 #include "app_info_table.h"
 #include "appid_inspector.h"
+#include "client_plugins/client_discovery.h"
+#include "detector_plugins/detector_dns.h"
+#include "detector_plugins/detector_http.h"
+#include "detector_plugins/detector_pattern.h"
+#include "detector_plugins/detector_sip.h"
+#include "detector_plugins/http_url_patterns.h"
 #include "host_port_app_cache.h"
 #include "lua_detector_flow_api.h"
 #include "lua_detector_module.h"
 #include "lua_detector_util.h"
-#include "client_plugins/client_discovery.h"
 #include "service_plugins/service_discovery.h"
 #include "service_plugins/service_ssl.h"
-#include "detector_plugins/detector_dns.h"
-#include "detector_plugins/detector_http.h"
-#include "detector_plugins/http_url_patterns.h"
-#include "detector_plugins/detector_sip.h"
-#include "detector_plugins/detector_pattern.h"
-#include "hash/xhash.h"
-#include "log/messages.h"
-#include "main/snort_types.h"
-#include "profiler/profiler.h"
-#include "protocols/packet.h"
 
 using namespace snort;
 
@@ -67,30 +68,24 @@ ProfileStats luaDetectorsPerfStats;
 ProfileStats luaCiscoPerfStats;
 ProfileStats luaCustomPerfStats;
 
-static XHash* CHP_glossary = nullptr;      // keep track of http multipatterns here
+static std::unordered_map<AppId, CHPApp*>* CHP_glossary = nullptr; // tracks http multipatterns
 
-static int free_chp_data(void* /* key */, void* data)
+void init_chp_glossary()
 {
-    if (data)
-        snort_free(data);
-    return 0;
+    CHP_glossary = new std::unordered_map<AppId, CHPApp*>;
 }
 
-int init_chp_glossary()
+void free_chp_glossary()
 {
-    if (!(CHP_glossary = xhash_new(1024, sizeof(AppId), 0, 0, 0, nullptr, &free_chp_data, 0)))
+    if (!CHP_glossary)
+        return;
+
+    for (auto& entry : *CHP_glossary)
     {
-        ErrorMessage("Config: failed to allocate memory for an sfxhash.");
-        return 0;
+        if (entry.second)
+            snort_free(entry.second);
     }
-    else
-        return 1;
-}
-
-void free_chp_glossary()
-{
-    if (CHP_glossary)
-        xhash_delete(CHP_glossary);
+    delete CHP_glossary;
     CHP_glossary = nullptr;
 }
 
@@ -1201,7 +1196,7 @@ static int create_chp_application(AppId appIdInstance, unsigned app_type_flags,
     new_app->app_type_flags = app_type_flags;
     new_app->num_matches = num_matches;
 
-    if (xhash_add(CHP_glossary, &(new_app->appIdInstance), new_app))
+    if (CHP_glossary->insert(std::make_pair(appIdInstance, new_app)).second == false)
     {
         ErrorMessage("LuaDetectorApi:Failed to add CHP for appId %d, instance %d",
             CHP_APPIDINSTANCE_TO_ID(appIdInstance), CHP_APPIDINSTANCE_TO_INSTANCE(appIdInstance));
@@ -1227,7 +1222,7 @@ static int detector_chp_create_application(lua_State* L)
     int num_matches = lua_tointeger(L, ++index);
 
     // We only want one of these for each appId.
-    if (xhash_find(CHP_glossary, &appIdInstance))
+    if (CHP_glossary->find(appIdInstance) != CHP_glossary->end())
     {
         ErrorMessage(
             "LuaDetectorApi:Attempt to add more than one CHP for appId %d - use CHPMultiCreateApp",
@@ -1301,12 +1296,9 @@ static inline int get_chp_action_data(lua_State* L, int index, char** action_dat
 static int add_chp_pattern_action(AppId appIdInstance, int isKeyPattern, HttpFieldIds patternType,
     size_t patternSize, char* patternData, ActionType actionType, char* optionalActionData)
 {
-    CHPListElement* chpa;
-    CHPApp* chpapp;
-    AppInfoManager& app_info_mgr = AppInfoManager::get_instance();
-
     //find the CHP App for this
-    if (!(chpapp = (decltype(chpapp))xhash_find(CHP_glossary, &appIdInstance)))
+    auto chp_entry = CHP_glossary->find(appIdInstance);
+    if (chp_entry == CHP_glossary->end() or !chp_entry->second)
     {
         ErrorMessage(
             "LuaDetectorApi:Invalid attempt to add a CHP action for unknown appId %d, instance %d. - pattern:\"%s\" - action \"%s\"\n",
@@ -1318,6 +1310,9 @@ static int add_chp_pattern_action(AppId appIdInstance, int isKeyPattern, HttpFie
         return 0;
     }
 
+    CHPApp* chpapp = chp_entry->second;
+    AppInfoManager& app_info_mgr = AppInfoManager::get_instance();
+
     if (isKeyPattern)
     {
         chpapp->key_pattern_count++;
@@ -1365,7 +1360,7 @@ static int add_chp_pattern_action(AppId appIdInstance, int isKeyPattern, HttpFie
     else if (actionType != ALTERNATE_APPID && actionType != DEFER_TO_SIMPLE_DETECT)
         chpapp->ptype_req_counts[patternType]++;
 
-    chpa = (CHPListElement*)snort_calloc(sizeof(CHPListElement));
+    CHPListElement* chpa = (CHPListElement*)snort_calloc(sizeof(CHPListElement));
     chpa->chp_action.appIdInstance = appIdInstance;
     chpa->chp_action.precedence = precedence;
     chpa->chp_action.key_pattern = isKeyPattern;
@@ -1457,7 +1452,7 @@ static int detector_create_chp_multi_application(lua_State* L)
     for (instance=0; instance < CHP_APPID_INSTANCE_MAX; instance++ )
     {
         appIdInstance = (appId << CHP_APPID_BITS_FOR_INSTANCE) + instance;
-        if ( xhash_find(CHP_glossary, &appIdInstance) )
+        if (CHP_glossary->find(appIdInstance) != CHP_glossary->end())
             continue;
         break;
     }
@@ -1660,7 +1655,7 @@ static int detector_add_length_app_cache(lua_State* L)
         str_ptr++;
     }
 
-    if ( !add_length_app_cache(&length_sequence, appId) )
+    if ( !add_length_app_cache(length_sequence, appId) )
     {
         ErrorMessage("LuaDetectorApi:Could not add entry to cache!");
         lua_pushnumber(L, -1);

diff --git a/src/network_inspectors/appid/lua_detector_api.h b/src/network_inspectors/appid/lua_detector_api.h
index d4dafdbcbe9a581b910a93134aae6f5b71e9c5da..cabe60533db73deff4dd9e62367e4a2f6e4f2517 100644 (file)
--- a/src/network_inspectors/appid/lua_detector_api.h
+++ b/src/network_inspectors/appid/lua_detector_api.h
@@ -137,7 +137,7 @@ public:
 };
 
 int register_detector(lua_State*);
-int init_chp_glossary();
+void init_chp_glossary();
 int init(lua_State*, int result=0);
 void free_chp_glossary();