From: Mike Stepanek (mstepane) Date: Mon, 9 Jul 2018 17:42:14 +0000 (-0400) Subject: Merge pull request #1303 in SNORT/snort3 from xhash_alternatives to master X-Git-Tag: 3.0.0-246~25 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=a82f3e46c10d805beea8737ab8369b6ec7252b65;p=thirdparty%2Fsnort3.git Merge pull request #1303 in SNORT/snort3 from xhash_alternatives to master Squashed commit of the following: commit 2a80681d630c8ef65435733247f7539e42441b59 Author: Masud Hasan Date: Fri Jul 6 15:26:11 2018 -0400 appid: Replacing xhash with alternative data structures --- diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index c4045e38d..45fc82cb8 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -885,7 +885,7 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto asd.length_sequence.sequence_cnt++; asd.length_sequence.sequence[index].direction = direction; asd.length_sequence.sequence[index].length = p->dsize; - AppId id = find_length_app_cache(&asd.length_sequence); + AppId id = find_length_app_cache(asd.length_sequence); if (id > APP_ID_NONE) { asd.service.set_port_service_id(id); diff --git a/src/network_inspectors/appid/appid_utils/network_set.cc b/src/network_inspectors/appid/appid_utils/network_set.cc index 0cd3321d0..8a0d3b947 100644 --- a/src/network_inspectors/appid/appid_utils/network_set.cc +++ b/src/network_inspectors/appid/appid_utils/network_set.cc @@ -39,23 +39,7 @@ int NetworkSetManager::create(NetworkSet** network_set) NetworkSet* tmp = (NetworkSet*)snort_calloc(sizeof(NetworkSet)); sflist_init(&tmp->networks); - tmp->ids = xhash_new(64, sizeof(unsigned), 0, 0, 0, nullptr, nullptr, 1); - if (tmp->ids == nullptr) - { - ErrorMessage("NetworkSet:Out of memory (wanted %zu bytes)", sizeof(NetworkSet)); - destroy(tmp); - return -1; - } - sflist_init(&tmp->networks6); - tmp->ids6 = xhash_new(64, sizeof(unsigned), 0, 0, 0, nullptr, nullptr, 1); - if (tmp->ids6 == nullptr) - { - ErrorMessage("NetworkSet:Out of memory (wanted %zu bytes)", sizeof(NetworkSet)); - destroy(tmp); - return -1; - } - *network_set = tmp; return 0; } @@ -71,14 +55,14 @@ int NetworkSetManager::destroy(NetworkSet* network_set) network_set->pnetwork = nullptr; } sflist_static_free_all(&network_set->networks, &snort_free); - xhash_delete(network_set->ids); + network_set->ids.clear(); if (network_set->pnetwork6) { snort_free(network_set->pnetwork6); network_set->pnetwork6 = nullptr; } sflist_static_free_all(&network_set->networks6, &snort_free); - xhash_delete(network_set->ids6); + network_set->ids6.clear(); snort_free(network_set); return 0; @@ -126,11 +110,9 @@ int NetworkSetManager::add_network_range_ex(NetworkSet* network_set, uint32_t ra } sflist_add_tail(&network_set->networks, (void*)network); - int rval = xhash_add(network_set->ids, &network->info.id, &network->info.id); - if (rval != XHASH_OK && rval != XHASH_INTABLE) + if (network_set->ids.insert(network->info.id).second == false) { - ErrorMessage("NetworkSet:Out of memory"); - snort_free(network); + ErrorMessage("NetworkSet: Failed to add id %u\n", network->info.id); return -1; } @@ -186,11 +168,9 @@ int NetworkSetManager::add_network_range6(NetworkSet* network_set, NSIPv6Addr* r } sflist_add_tail(&network_set->networks6, (void*)network); - int rval = xhash_add(network_set->ids6, &network->info.id, &network->info.id); - if (rval != XHASH_OK && rval != XHASH_INTABLE) + if (network_set->ids6.insert(network->info.id).second == false) { - ErrorMessage("NetworkSet:Out of memory"); - snort_free(network); + ErrorMessage("NetworkSet: Failed to add IPv6 id %u\n", network->info.id); return -1; } @@ -943,8 +923,6 @@ int NetworkSetManager::reduce_network_set6(SF_LIST* networks) int NetworkSetManager::reduce(NetworkSet* network_set) { - XHashNode* hnode; - unsigned id; int rval; SF_LIST ordered_networks; Network* network; @@ -955,11 +933,8 @@ int NetworkSetManager::reduce(NetworkSet* network_set) if (!network_set) return -1; - for (hnode = xhash_gfindfirst(network_set->ids); - hnode; - hnode=xhash_gfindnext(network_set->ids)) + for (auto& id : network_set->ids) { - id = *(unsigned*)(hnode->data); if ((rval = order_by_netmask(&ordered_networks, &network_set->networks, id)) != 0) { sflist_free_all(&ordered_networks, &snort_free); @@ -1019,10 +994,8 @@ int NetworkSetManager::reduce(NetworkSet* network_set) } } - for (hnode = xhash_gfindfirst(network_set->ids6); hnode; hnode=xhash_gfindnext( - network_set->ids6)) + for (auto& id : network_set->ids6) { - id = *(unsigned*)(hnode->data); if ((rval = order_by_netmask(&ordered_networks, &network_set->networks6, id)) != 0) { sflist_free_all(&ordered_networks, &snort_free); diff --git a/src/network_inspectors/appid/appid_utils/network_set.h b/src/network_inspectors/appid/appid_utils/network_set.h index e032fc28c..4d49e7c03 100644 --- a/src/network_inspectors/appid/appid_utils/network_set.h +++ b/src/network_inspectors/appid/appid_utils/network_set.h @@ -25,8 +25,8 @@ #ifdef USE_RNA_CONFIG #include +#include -#include "hash/xhash.h" #include "protocols/ipv6.h" #include "utils/sflsq.h" @@ -102,11 +102,11 @@ struct NetworkSet { NetworkSet* next; SF_LIST networks; - XHash* ids; + std::unordered_set ids; Network** pnetwork; unsigned count; SF_LIST networks6; - XHash* ids6; + std::unordered_set ids6; Network6** pnetwork6; unsigned count6; }; diff --git a/src/network_inspectors/appid/length_app_cache.cc b/src/network_inspectors/appid/length_app_cache.cc index 4ba2162ef..8a759dd8e 100644 --- a/src/network_inspectors/appid/length_app_cache.cc +++ b/src/network_inspectors/appid/length_app_cache.cc @@ -25,50 +25,39 @@ #include "length_app_cache.h" -#include "application_ids.h" -#include "hash/xhash.h" +#include + #include "log/messages.h" #include "main/thread.h" -using namespace snort; +#include "application_ids.h" -#define HASH_NUM_ROWS (1024) +using namespace snort; -static XHash* lengthCache = nullptr; +static std::map* length_cache = nullptr; void init_length_app_cache() { - if (!(lengthCache = xhash_new(HASH_NUM_ROWS, sizeof(LengthKey), sizeof(AppId), - 0, 0, nullptr, nullptr, 0))) - { - ErrorMessage("lengthAppCache: Failed to allocate length cache!"); - } + length_cache = new std::map; } void free_length_app_cache() { - if (lengthCache) - { - xhash_delete(lengthCache); - lengthCache = nullptr; - } + delete length_cache; + length_cache = nullptr; } -AppId find_length_app_cache(const LengthKey* key) +AppId find_length_app_cache(const LengthKey& key) { - AppId* val = (AppId*)xhash_find(lengthCache, (void*)key); - if (val == nullptr) + auto entry = length_cache->find(key); + if (entry == length_cache->end()) return APP_ID_NONE; /* no match */ else - return *val; /* match found */ + return entry->second; /* match found */ } -bool add_length_app_cache(const LengthKey* key, AppId val) +bool add_length_app_cache(const LengthKey& key, AppId val) { - if (xhash_add(lengthCache, (void*)key, (void*)&val)) - { - return false; - } - return true; + return (length_cache->insert(std::make_pair(key, val))).second == true; } diff --git a/src/network_inspectors/appid/length_app_cache.h b/src/network_inspectors/appid/length_app_cache.h index 05c3d5c86..5261927e6 100644 --- a/src/network_inspectors/appid/length_app_cache.h +++ b/src/network_inspectors/appid/length_app_cache.h @@ -44,14 +44,43 @@ struct LengthKey IpProtocol proto = IpProtocol::PROTO_NOT_SET; // IpProtocol::TCP or IpProtocol::UDP uint8_t sequence_cnt = 0; // num valid entries in sequence LengthSequenceEntry sequence[LENGTH_SEQUENCE_CNT_MAX]; + + // Used by map where LengthKey object is the key + bool operator<(const LengthKey& right) const + { + if (proto < right.proto) + return true; + else if (right.proto < proto) + return false; + + if (sequence_cnt < right.sequence_cnt) + return true; + else if (right.sequence_cnt < sequence_cnt) + return false; + + for (uint8_t i = 0; i < LENGTH_SEQUENCE_CNT_MAX; ++i) + { + if (sequence[i].direction < right.sequence[i].direction) + return true; + else if (right.sequence[i].direction < sequence[i].direction) + return false; + + if (sequence[i].length < right.sequence[i].length) + return true; + else if (right.sequence[i].length < sequence[i].length) + return false; + } + + return false; + } }; #pragma pack() void init_length_app_cache(); void free_length_app_cache(); -AppId find_length_app_cache(const LengthKey*); -bool add_length_app_cache(const LengthKey*, AppId); +AppId find_length_app_cache(const LengthKey&); +bool add_length_app_cache(const LengthKey&, AppId); #endif diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc index 1c0b01dc8..e556db646 100644 --- a/src/network_inspectors/appid/lua_detector_api.cc +++ b/src/network_inspectors/appid/lua_detector_api.cc @@ -27,27 +27,28 @@ #include #include +#include + +#include "log/messages.h" +#include "main/snort_types.h" +#include "profiler/profiler.h" +#include "protocols/packet.h" #include "app_forecast.h" #include "app_info_table.h" #include "appid_inspector.h" +#include "client_plugins/client_discovery.h" +#include "detector_plugins/detector_dns.h" +#include "detector_plugins/detector_http.h" +#include "detector_plugins/detector_pattern.h" +#include "detector_plugins/detector_sip.h" +#include "detector_plugins/http_url_patterns.h" #include "host_port_app_cache.h" #include "lua_detector_flow_api.h" #include "lua_detector_module.h" #include "lua_detector_util.h" -#include "client_plugins/client_discovery.h" #include "service_plugins/service_discovery.h" #include "service_plugins/service_ssl.h" -#include "detector_plugins/detector_dns.h" -#include "detector_plugins/detector_http.h" -#include "detector_plugins/http_url_patterns.h" -#include "detector_plugins/detector_sip.h" -#include "detector_plugins/detector_pattern.h" -#include "hash/xhash.h" -#include "log/messages.h" -#include "main/snort_types.h" -#include "profiler/profiler.h" -#include "protocols/packet.h" using namespace snort; @@ -67,30 +68,24 @@ ProfileStats luaDetectorsPerfStats; ProfileStats luaCiscoPerfStats; ProfileStats luaCustomPerfStats; -static XHash* CHP_glossary = nullptr; // keep track of http multipatterns here +static std::unordered_map* CHP_glossary = nullptr; // tracks http multipatterns -static int free_chp_data(void* /* key */, void* data) +void init_chp_glossary() { - if (data) - snort_free(data); - return 0; + CHP_glossary = new std::unordered_map; } -int init_chp_glossary() +void free_chp_glossary() { - if (!(CHP_glossary = xhash_new(1024, sizeof(AppId), 0, 0, 0, nullptr, &free_chp_data, 0))) + if (!CHP_glossary) + return; + + for (auto& entry : *CHP_glossary) { - ErrorMessage("Config: failed to allocate memory for an sfxhash."); - return 0; + if (entry.second) + snort_free(entry.second); } - else - return 1; -} - -void free_chp_glossary() -{ - if (CHP_glossary) - xhash_delete(CHP_glossary); + delete CHP_glossary; CHP_glossary = nullptr; } @@ -1201,7 +1196,7 @@ static int create_chp_application(AppId appIdInstance, unsigned app_type_flags, new_app->app_type_flags = app_type_flags; new_app->num_matches = num_matches; - if (xhash_add(CHP_glossary, &(new_app->appIdInstance), new_app)) + if (CHP_glossary->insert(std::make_pair(appIdInstance, new_app)).second == false) { ErrorMessage("LuaDetectorApi:Failed to add CHP for appId %d, instance %d", CHP_APPIDINSTANCE_TO_ID(appIdInstance), CHP_APPIDINSTANCE_TO_INSTANCE(appIdInstance)); @@ -1227,7 +1222,7 @@ static int detector_chp_create_application(lua_State* L) int num_matches = lua_tointeger(L, ++index); // We only want one of these for each appId. - if (xhash_find(CHP_glossary, &appIdInstance)) + if (CHP_glossary->find(appIdInstance) != CHP_glossary->end()) { ErrorMessage( "LuaDetectorApi:Attempt to add more than one CHP for appId %d - use CHPMultiCreateApp", @@ -1301,12 +1296,9 @@ static inline int get_chp_action_data(lua_State* L, int index, char** action_dat static int add_chp_pattern_action(AppId appIdInstance, int isKeyPattern, HttpFieldIds patternType, size_t patternSize, char* patternData, ActionType actionType, char* optionalActionData) { - CHPListElement* chpa; - CHPApp* chpapp; - AppInfoManager& app_info_mgr = AppInfoManager::get_instance(); - //find the CHP App for this - if (!(chpapp = (decltype(chpapp))xhash_find(CHP_glossary, &appIdInstance))) + auto chp_entry = CHP_glossary->find(appIdInstance); + if (chp_entry == CHP_glossary->end() or !chp_entry->second) { ErrorMessage( "LuaDetectorApi:Invalid attempt to add a CHP action for unknown appId %d, instance %d. - pattern:\"%s\" - action \"%s\"\n", @@ -1318,6 +1310,9 @@ static int add_chp_pattern_action(AppId appIdInstance, int isKeyPattern, HttpFie return 0; } + CHPApp* chpapp = chp_entry->second; + AppInfoManager& app_info_mgr = AppInfoManager::get_instance(); + if (isKeyPattern) { chpapp->key_pattern_count++; @@ -1365,7 +1360,7 @@ static int add_chp_pattern_action(AppId appIdInstance, int isKeyPattern, HttpFie else if (actionType != ALTERNATE_APPID && actionType != DEFER_TO_SIMPLE_DETECT) chpapp->ptype_req_counts[patternType]++; - chpa = (CHPListElement*)snort_calloc(sizeof(CHPListElement)); + CHPListElement* chpa = (CHPListElement*)snort_calloc(sizeof(CHPListElement)); chpa->chp_action.appIdInstance = appIdInstance; chpa->chp_action.precedence = precedence; chpa->chp_action.key_pattern = isKeyPattern; @@ -1457,7 +1452,7 @@ static int detector_create_chp_multi_application(lua_State* L) for (instance=0; instance < CHP_APPID_INSTANCE_MAX; instance++ ) { appIdInstance = (appId << CHP_APPID_BITS_FOR_INSTANCE) + instance; - if ( xhash_find(CHP_glossary, &appIdInstance) ) + if (CHP_glossary->find(appIdInstance) != CHP_glossary->end()) continue; break; } @@ -1660,7 +1655,7 @@ static int detector_add_length_app_cache(lua_State* L) str_ptr++; } - if ( !add_length_app_cache(&length_sequence, appId) ) + if ( !add_length_app_cache(length_sequence, appId) ) { ErrorMessage("LuaDetectorApi:Could not add entry to cache!"); lua_pushnumber(L, -1); diff --git a/src/network_inspectors/appid/lua_detector_api.h b/src/network_inspectors/appid/lua_detector_api.h index d4dafdbcb..cabe60533 100644 --- a/src/network_inspectors/appid/lua_detector_api.h +++ b/src/network_inspectors/appid/lua_detector_api.h @@ -137,7 +137,7 @@ public: }; int register_detector(lua_State*); -int init_chp_glossary(); +void init_chp_glossary(); int init(lua_State*, int result=0); void free_chp_glossary();