From: Russ Combs (rucombs) Date: Wed, 26 Feb 2020 20:54:36 +0000 (+0000) Subject: Merge pull request #2009 in SNORT/snort3 from ~DAVMCPHE/snort3:lru_cache_for_hash... X-Git-Tag: 3.0.0-269~34 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bb26ceaaed7ca78c25ff5b8aa6f0b338fb9ecc1a;p=thirdparty%2Fsnort3.git Merge pull request #2009 in SNORT/snort3 from ~DAVMCPHE/snort3:lru_cache_for_hash to master Squashed commit of the following: commit 6f962204d41c0b1007992483f739db508e5d9c0d Author: davis mcpherson Date: Fri Jan 24 09:11:21 2020 -0500 xhash/zhash: refactor duplicated code into a common base class, xhash/zhash will subclass this new base class utils: create memory allocation class based on sfmemcap functionality xhash: refactor XHash and HashFnc to eliminate c-style callbacks and simplify ctor options xhash: rename hashfcn.[cc|h] to hash_keys.[cc|h] zhash: refactor to use hash_lru_cache and hash_key_operations classes zhash: make zhash a subclass of xhash...eliminate duplicate code utils: add unit tests for MemCapAllocator class hash: add unit tests for new HashLruCache class - (PR review comments --- diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc index c6b573f83..6fd3b26a2 100644 --- a/src/detection/detection_options.cc +++ b/src/detection/detection_options.cc @@ -37,7 +37,8 @@ #include "filters/detection_filter.h" #include "framework/cursor.h" -#include "hash/hashfcn.h" +#include "hash/hash_defs.h" +#include "hash/hash_key_operations.h" #include "hash/xhash.h" #include "ips_options/extract.h" #include "ips_options/ips_flowbits.h" @@ -79,93 +80,82 @@ struct detection_option_key_t static inline bool operator==(const struct timeval& a, const struct timeval& b) { return a.tv_sec == b.tv_sec && a.tv_usec == b.tv_usec; } -static uint32_t detection_option_hash_func(HashFnc*, const unsigned char* k, int) +class DetectionOptionHashKeyOps : public HashKeyOperations { - const detection_option_key_t* key = (const detection_option_key_t*)k; +public: + DetectionOptionHashKeyOps(int rows) + : HashKeyOperations(rows) + { } - if ( key->option_type != RULE_OPTION_TYPE_LEAF_NODE ) + unsigned do_hash(const unsigned char* k, int) override { - IpsOption* opt = (IpsOption*)key->option_data; - return opt->hash(); + const detection_option_key_t* key = (const detection_option_key_t*)k; + + if ( key->option_type != RULE_OPTION_TYPE_LEAF_NODE ) + { + IpsOption* opt = (IpsOption*)key->option_data; + return opt->hash(); + } + return 0; } - return 0; -} -static bool detection_option_key_compare_func(const void* k1, const void* k2, size_t) -{ - const detection_option_key_t* key1 = (const detection_option_key_t*)k1; - const detection_option_key_t* key2 = (const detection_option_key_t*)k2; + bool key_compare(const void* k1, const void* k2, size_t) override + { + const detection_option_key_t* key1 = (const detection_option_key_t*)k1; + const detection_option_key_t* key2 = (const detection_option_key_t*)k2; - assert(key1 && key2); + assert(key1 && key2); - if ( key1->option_type != key2->option_type ) - return false; + if ( key1->option_type != key2->option_type ) + return false; - if ( key1->option_type != RULE_OPTION_TYPE_LEAF_NODE ) - { - IpsOption* opt1 = (IpsOption*)key1->option_data; - IpsOption* opt2 = (IpsOption*)key2->option_data; + if ( key1->option_type != RULE_OPTION_TYPE_LEAF_NODE ) + { + IpsOption* opt1 = (IpsOption*)key1->option_data; + IpsOption* opt2 = (IpsOption*)key2->option_data; - if ( *opt1 == *opt2 ) - return true; + if ( *opt1 == *opt2 ) + return true; + } + return false; } - return false; -} +}; -static int detection_hash_free_func(void* option_key, void*) +class DetectionOptionHash : public XHash { - detection_option_key_t* key = (detection_option_key_t*)option_key; +public: - if ( key->option_type != RULE_OPTION_TYPE_LEAF_NODE ) + DetectionOptionHash(int rows, int key_len) + : XHash(rows, key_len) { - IpsOption* opt = (IpsOption*)key->option_data; - IpsManager::delete_option(opt); + initialize(new DetectionOptionHashKeyOps(nrows)); } - return 0; -} - -static XHash* DetectionHashTableNew() -{ - XHash* doht = new XHash(HASH_RULE_OPTIONS, sizeof(detection_option_key_t), - 0, 0, false, nullptr, detection_hash_free_func, true); - - doht->set_key_opcodes(detection_option_hash_func, detection_option_key_compare_func); - - return doht; -} - -void DetectionHashTableFree(XHash* doht) -{ - delete doht; -} - -void* add_detection_option(SnortConfig* sc, option_type_t type, void* option_data) -{ - if ( !sc->detection_option_hash_table ) - sc->detection_option_hash_table = DetectionHashTableNew(); - detection_option_key_t key; - key.option_type = type; - key.option_data = option_data; + ~DetectionOptionHash() override + { + delete_hash_table(); + } - if ( void* p = sc->detection_option_hash_table->get_user_data(&key) ) - return p; + void free_user_data(HashNode* hnode) override + { + detection_option_key_t* key = (detection_option_key_t*)hnode->key; - sc->detection_option_hash_table->insert(&key, option_data); - return nullptr; -} + if ( key->option_type != RULE_OPTION_TYPE_LEAF_NODE ) + { + IpsOption* opt = (IpsOption*)key->option_data; + IpsManager::delete_option(opt); + } + } +}; static uint32_t detection_option_tree_hash(detection_option_tree_node_t* node) { - uint32_t a,b,c; - int i; - - if (!node) - return 0; + assert(node); + uint32_t a, b, c; a = b = c = 0; - for (i=0; inum_children; i++) + for ( int i = 0; i < node->num_children; i++) { #if (defined(__ia64) || defined(__amd64) || defined(_LP64)) { @@ -198,19 +188,6 @@ static uint32_t detection_option_tree_hash(detection_option_tree_node_t* node) return c; } -static uint32_t detection_option_tree_hash_func(HashFnc*, const unsigned char* k, int) -{ - const detection_option_key_t* key = (const detection_option_key_t*)k; - detection_option_tree_node_t* node; - - if (!key || !key->option_data) - return 0; - - node = (detection_option_tree_node_t*)key->option_data; - - return detection_option_tree_hash(node); -} - static bool detection_option_tree_compare( const detection_option_tree_node_t* r, const detection_option_tree_node_t* l) { @@ -232,40 +209,94 @@ static bool detection_option_tree_compare( return true; } -static bool detection_option_tree_compare_func(const void* k1, const void* k2, size_t) +void free_detection_option_tree(detection_option_tree_node_t* node) +{ + for (int i = 0; i < node->num_children; i++) + free_detection_option_tree(node->children[i]); + + snort_free(node->children); + snort_free(node->state); + snort_free(node); +} + +class DetectionOptionTreeHashKeyOps : public HashKeyOperations { - const detection_option_key_t* key_r = (const detection_option_key_t*)k1; - const detection_option_key_t* key_l = (const detection_option_key_t*)k2; +public: + DetectionOptionTreeHashKeyOps(int rows) + : HashKeyOperations(rows) + { } - if ( !key_r or !key_l ) - return false; + unsigned do_hash(const unsigned char* k, int) override + { + assert(k); + const detection_option_key_t* key = (const detection_option_key_t*)k; + if ( !key->option_data ) + return 0; - const detection_option_tree_node_t* r = (const detection_option_tree_node_t*)key_r->option_data; - const detection_option_tree_node_t* l = (const detection_option_tree_node_t*)key_l->option_data; + detection_option_tree_node_t* node = (detection_option_tree_node_t*)key->option_data; - return detection_option_tree_compare(r, l); -} + return detection_option_tree_hash(node); + } + + bool key_compare(const void* k1, const void* k2, size_t) override + { + assert(k1 && k2); + + const detection_option_key_t* key_r = (const detection_option_key_t*)k1; + const detection_option_key_t* key_l = (const detection_option_key_t*)k2; + + const detection_option_tree_node_t* r = (const detection_option_tree_node_t*)key_r->option_data; + const detection_option_tree_node_t* l = (const detection_option_tree_node_t*)key_l->option_data; + + return detection_option_tree_compare(r, l); + } +}; + +class DetectionOptionTreeHash : public XHash +{ +public: + DetectionOptionTreeHash(int rows, int key_len) + : XHash(rows, key_len) + { + initialize(new DetectionOptionTreeHashKeyOps(nrows)); + } + + ~DetectionOptionTreeHash() override + { + delete_hash_table(); + } -static int detection_option_tree_free_func(void*, void* data) + void free_user_data(HashNode* hnode) override + { + free_detection_option_tree((detection_option_tree_node_t*)hnode->data); + } + +}; + +static DetectionOptionHash* DetectionHashTableNew() { - detection_option_tree_node_t* node = (detection_option_tree_node_t*)data; - free_detection_option_tree(node); - return 0; + return new DetectionOptionHash(HASH_RULE_OPTIONS, sizeof(detection_option_key_t)); } -void DetectionTreeHashTableFree(XHash* dtht) +static DetectionOptionTreeHash* DetectionTreeHashTableNew() { - delete dtht; + return new DetectionOptionTreeHash(HASH_RULE_TREE, sizeof(detection_option_key_t)); } -static XHash* DetectionTreeHashTableNew() +void* add_detection_option(SnortConfig* sc, option_type_t type, void* option_data) { - XHash* dtht = new XHash(HASH_RULE_TREE, sizeof(detection_option_key_t), - 0, 0, false, nullptr, detection_option_tree_free_func, true); + if ( !sc->detection_option_hash_table ) + sc->detection_option_hash_table = DetectionHashTableNew(); - dtht->set_key_opcodes(detection_option_tree_hash_func, detection_option_tree_compare_func); + detection_option_key_t key; + key.option_type = type; + key.option_data = option_data; + + if ( void* p = sc->detection_option_hash_table->get_user_data(&key) ) + return p; - return dtht; + sc->detection_option_hash_table->insert(&key, option_data); + return nullptr; } void print_option_tree(detection_option_tree_node_t* node, int level) @@ -527,7 +558,7 @@ int detection_option_node_evaluate( // Back up byte_extract vars so they don't get overwritten between rules trace_log(detection, TRACE_RULE_VARS, "Rule options variables: \n"); - for ( int i = 0; i < NUM_IPS_OPTIONS_VARS; ++i ) + for ( unsigned i = 0; i < NUM_IPS_OPTIONS_VARS; ++i ) { GetVarValueByIndex(&(tmp_byte_extract_vars[i]), (int8_t)i); trace_logf_wo_name(detection, TRACE_RULE_VARS, "var[%d]=%d ", i, @@ -552,7 +583,7 @@ int detection_option_node_evaluate( detection_option_tree_node_t* child_node = node->children[i]; dot_node_state_t* child_state = child_node->state + get_instance_id(); - for ( int j = 0; j < NUM_IPS_OPTIONS_VARS; ++j ) + for ( unsigned j = 0; j < NUM_IPS_OPTIONS_VARS; ++j ) SetVarValueByIndex(tmp_byte_extract_vars[j], (int8_t)j); if ( loop_count > 0 ) @@ -758,7 +789,7 @@ static void detection_option_node_update_otn_stats(detection_option_tree_node_t* if ( node->num_children ) { - for ( int i=0; i < node->num_children; ++i ) + for ( int i = 0; i < node->num_children; ++i ) detection_option_node_update_otn_stats(node->children[i], &local_stats, checks, timeouts, suspends); } @@ -829,16 +860,3 @@ detection_option_tree_node_t* new_node(option_type_t type, void* data) return p; } - -void free_detection_option_tree(detection_option_tree_node_t* node) -{ - int i; - for (i=0; inum_children; i++) - { - free_detection_option_tree(node->children[i]); - } - snort_free(node->children); - snort_free(node->state); - snort_free(node); -} - diff --git a/src/detection/detection_options.h b/src/detection/detection_options.h index 6f6fdf4a9..f5523cfaa 100644 --- a/src/detection/detection_options.h +++ b/src/detection/detection_options.h @@ -35,15 +35,16 @@ #include "detection/rule_option_types.h" #include "time/clock_defs.h" - #include "main/snort_debug.h" + extern Trace TRACE_NAME(detection); namespace snort { +class HashNode; +class XHash; struct Packet; struct SnortConfig; -class XHash; } struct RuleLatencyState; @@ -123,9 +124,6 @@ void* add_detection_option_tree(struct snort::SnortConfig*, detection_option_tre int detection_option_node_evaluate( detection_option_tree_node_t*, detection_option_eval_data_t&, const class Cursor&); -void DetectionHashTableFree(snort::XHash*); -void DetectionTreeHashTableFree(snort::XHash*); - void print_option_tree(detection_option_tree_node_t*, int level); void detection_option_tree_update_otn_stats(snort::XHash*); diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc index 9f9fde9f0..0d933ff0f 100644 --- a/src/detection/fp_create.cc +++ b/src/detection/fp_create.cc @@ -37,6 +37,7 @@ #include "framework/mpse.h" #include "framework/mpse_batch.h" #include "hash/ghash.h" +#include "hash/xhash.h" #include "log/messages.h" #include "main/snort.h" #include "main/snort_config.h" @@ -1660,8 +1661,8 @@ void fpDeleteFastPacketDetection(SnortConfig* sc) return; /* Cleanup the detection option tree */ - DetectionHashTableFree(sc->detection_option_hash_table); - DetectionTreeHashTableFree(sc->detection_option_tree_hash_table); + delete sc->detection_option_hash_table; + delete sc->detection_option_tree_hash_table; fpFreeRuleMaps(sc); ServicePortGroupMapFree(sc->spgmmTable); diff --git a/src/detection/signature.cc b/src/detection/signature.cc index bed991f8a..5d6f8fb4a 100644 --- a/src/detection/signature.cc +++ b/src/detection/signature.cc @@ -26,6 +26,7 @@ #include "signature.h" +#include "hash/hash_defs.h" #include "hash/ghash.h" #include "log/messages.h" #include "main/snort_config.h" @@ -232,10 +233,10 @@ void OtnLookupAdd(GHash* otn_map, OptTreeNode* otn) key.sid = otn->sigInfo.sid; int status = otn_map->insert(&key, otn); - if ( status == GHASH_OK ) + if ( status == HASH_OK ) return; - assert(status == GHASH_INTABLE); + assert(status == HASH_INTABLE); ParseError("duplicate rule with same gid (%u) and sid (%u)", key.gid, key.sid); } diff --git a/src/detection/tag.cc b/src/detection/tag.cc index afa2c3b0d..200d31b57 100644 --- a/src/detection/tag.cc +++ b/src/detection/tag.cc @@ -26,6 +26,7 @@ #include "tag.h" #include "events/event.h" +#include "hash/hash_defs.h" #include "hash/xhash.h" #include "log/messages.h" #include "main/snort_config.h" @@ -102,11 +103,6 @@ struct TagNode }; /* G L O B A L S **************************************************/ -static THREAD_LOCAL XHash* host_tag_cache_ptr = nullptr; - -// FIXIT-M utilize Flow instead of separate cache -static THREAD_LOCAL XHash* ssn_tag_cache_ptr = nullptr; - static THREAD_LOCAL uint32_t last_prune_time = 0; static THREAD_LOCAL uint32_t tag_alloc_faults = 0; static THREAD_LOCAL uint32_t tag_memory_usage = 0; @@ -119,11 +115,10 @@ static THREAD_LOCAL unsigned s_sessions = 0; // (consecutive) sessions to be captured. static const unsigned s_max_sessions = 1; + /* P R O T O T Y P E S ********************************************/ static TagNode* TagAlloc(XHash*); static void TagFree(XHash*, TagNode*); -static int TagFreeSessionNodeFunc(void* key, void* data); -static int TagFreeHostNodeFunc(void* key, void* data); static int PruneTagCache(uint32_t, int); static int PruneTime(XHash* tree, uint32_t thetime); static void TagSession(Packet*, TagData*, uint32_t, uint16_t, void*); @@ -131,6 +126,56 @@ static void TagHost(Packet*, TagData*, uint32_t, uint16_t, void*); static void AddTagNode(Packet*, TagData*, int, uint32_t, uint16_t, void*); static inline void SwapTag(TagNode*); +class TagSessionCache : public XHash +{ +public: + TagSessionCache(int rows, int key_len) + : XHash(rows, key_len) + { + initialize(); + anr_enabled = false; + recycle_nodes = false; + } + + ~TagSessionCache() override + { + delete_hash_table(); + } + + void free_user_data(HashNode* hnode) override + { + TagFree(this, (TagNode*)hnode->data); + } +}; + +class TagHostCache : public XHash +{ +public: + TagHostCache(int rows, int key_len) + : XHash(rows, key_len) + { + initialize(); + anr_enabled = false; + recycle_nodes = false; + } + + ~TagHostCache() override + { + delete_hash_table(); + } + + void free_user_data(HashNode* hnode) override + { + TagFree(this, (TagNode*)hnode->data); + } +}; + +static THREAD_LOCAL TagHostCache* host_tag_cache = nullptr; + +// FIXIT-M utilize Flow instead of separate cache +static THREAD_LOCAL TagSessionCache* ssn_tag_cache = nullptr; + + /**Calculated memory needed per node insertion into respective cache. Its includes * memory needed for allocating TagNode, HashNode and key size. * @@ -141,9 +186,9 @@ static inline void SwapTag(TagNode*); */ static inline unsigned int memory_per_node(XHash* hash) { - if ( hash == ssn_tag_cache_ptr ) + if ( hash == ssn_tag_cache ) return sizeof(tTagFlowKey) + sizeof(HashNode) + sizeof(TagNode); - else if ( hash == host_tag_cache_ptr ) + else if ( hash == host_tag_cache ) return sizeof(SfIp) + sizeof(HashNode) + sizeof(TagNode); return 0; @@ -218,28 +263,6 @@ static void TagFree( tag_memory_usage -= memory_per_node(hash); } -/**Callback from session tag cache to free user data. - * @param key - pointer to key to session tag - * @param data - pointer to user data, to be freed. - * @returns 0 - */ -static int TagFreeSessionNodeFunc(void*, void* data) -{ - TagFree(ssn_tag_cache_ptr, (TagNode*)data); - return 0; -} - -/**Callback from host tag cache to free user data. - * @param key - pointer to key to session tag - * @param data - pointer to user data, to be freed. - * @returns 0 - */ -static int TagFreeHostNodeFunc(void*, void* data) -{ - TagFree(host_tag_cache_ptr, (TagNode*)data); - return 0; -} - /** * swap the sips and dips, dp's and sp's * @@ -263,17 +286,14 @@ void InitTag() { unsigned int hashTableSize = TAG_MEMCAP/sizeof(TagNode); - ssn_tag_cache_ptr = new XHash(hashTableSize, sizeof(tTagFlowKey), 0, 0, - false, nullptr, TagFreeSessionNodeFunc, false); - - host_tag_cache_ptr = new XHash(hashTableSize, sizeof(SfIp), 0, 0, false, - nullptr, TagFreeHostNodeFunc, false); + ssn_tag_cache = new TagSessionCache(hashTableSize, sizeof(tTagFlowKey)); + host_tag_cache = new TagHostCache(hashTableSize, sizeof(SfIp)); } void CleanupTag() { - delete ssn_tag_cache_ptr; - delete host_tag_cache_ptr; + delete ssn_tag_cache; + delete host_tag_cache; } static void TagSession(Packet* p, TagData* tag, uint32_t time, uint16_t event_id, void* log_list) @@ -323,11 +343,11 @@ static void AddTagNode(Packet* p, TagData* tag, int mode, uint32_t now, } if (mode == TAG_SESSION) { - tag_cache_ptr = ssn_tag_cache_ptr; + tag_cache_ptr = ssn_tag_cache; } else { - tag_cache_ptr = host_tag_cache_ptr; + tag_cache_ptr = host_tag_cache; } idx = TagAlloc(tag_cache_ptr); @@ -416,7 +436,7 @@ int CheckTagList(Packet* p, Event& event, void** log_list) char create_event = 1; /* check for active tags */ - if (!host_tag_cache_ptr->get_node_count() && !ssn_tag_cache_ptr->get_node_count()) + if (!host_tag_cache->get_num_nodes() && !ssn_tag_cache->get_num_nodes()) { return 0; } @@ -432,7 +452,7 @@ int CheckTagList(Packet* p, Event& event, void** log_list) idx.key.dp = p->ptrs.dp; /* check for session tags... */ - returned = (TagNode*)ssn_tag_cache_ptr->get_user_data(&idx); + returned = (TagNode*)ssn_tag_cache->get_user_data(&idx); if (returned == nullptr) { @@ -441,11 +461,11 @@ int CheckTagList(Packet* p, Event& event, void** log_list) idx.key.dp = p->ptrs.sp; idx.key.sp = p->ptrs.dp; - returned = (TagNode*)ssn_tag_cache_ptr->get_user_data(&idx); + returned = (TagNode*)ssn_tag_cache->get_user_data(&idx); if (returned == nullptr) { - returned = (TagNode*)host_tag_cache_ptr->get_user_data(&idx); + returned = (TagNode*)host_tag_cache->get_user_data(&idx); if (returned == nullptr) { @@ -455,22 +475,22 @@ int CheckTagList(Packet* p, Event& event, void** log_list) */ idx.key.sip = *p->ptrs.ip_api.get_src(); - returned = (TagNode*)host_tag_cache_ptr->get_user_data(&idx); + returned = (TagNode*)host_tag_cache->get_user_data(&idx); } if (returned != nullptr) { - taglist = host_tag_cache_ptr; + taglist = host_tag_cache; } } else { - taglist = ssn_tag_cache_ptr; + taglist = ssn_tag_cache; } } else { - taglist = ssn_tag_cache_ptr; + taglist = ssn_tag_cache; } if (returned != nullptr) @@ -533,7 +553,7 @@ int CheckTagList(Packet* p, Event& event, void** log_list) if ( !returned->metric ) { - if (taglist->release_node(returned) != HASH_OK) + if (taglist->release_node(&returned->key) != HASH_OK) { LogMessage("WARNING: failed to remove tagNode from hash.\n"); } @@ -558,35 +578,26 @@ static int PruneTagCache(uint32_t thetime, int mustdie) if (mustdie == 0) { - if (ssn_tag_cache_ptr->get_node_count() != 0) - pruned = PruneTime(ssn_tag_cache_ptr, thetime); + if (ssn_tag_cache->get_num_nodes() != 0) + pruned = PruneTime(ssn_tag_cache, thetime); - if (host_tag_cache_ptr->get_node_count() != 0) - pruned += PruneTime(host_tag_cache_ptr, thetime); + if (host_tag_cache->get_num_nodes() != 0) + pruned += PruneTime(host_tag_cache, thetime); } else { while (pruned < mustdie && - (ssn_tag_cache_ptr->get_node_count() > 0 || host_tag_cache_ptr->get_node_count() > 0)) + (ssn_tag_cache->get_num_nodes() > 0 || host_tag_cache->get_num_nodes() > 0)) { - TagNode* lru_node; + if ( ssn_tag_cache->delete_lru_node() ) + ++pruned; + else + LogMessage("WARNING: failed to remove tagNode from ssn hash.\n"); - if ((lru_node = (TagNode*)ssn_tag_cache_ptr->get_lru_user_data()) != nullptr) - { - if (ssn_tag_cache_ptr->release_node(lru_node) != HASH_OK) - { - LogMessage("WARNING: failed to remove tagNode from hash.\n"); - } - pruned++; - } - if ((lru_node = (TagNode*)host_tag_cache_ptr->get_lru_user_data()) != nullptr) - { - if (host_tag_cache_ptr->release_node(lru_node) != HASH_OK) - { - LogMessage("WARNING: failed to remove tagNode from hash.\n"); - } - pruned++; - } + if ( host_tag_cache->delete_lru_node() ) + ++pruned; + else + LogMessage("WARNING: failed to remove tagNode from host hash.\n"); } } @@ -602,7 +613,7 @@ static int PruneTime(XHash* tree, uint32_t thetime) { if ((lru_node->last_access + TAG_PRUNE_QUANTUM) < thetime) { - if (tree->release_node(lru_node) != HASH_OK) + if (tree->release_node(&lru_node->key) != HASH_OK) { LogMessage("WARNING: failed to remove tagNode from hash.\n"); } diff --git a/src/file_api/file_cache.cc b/src/file_api/file_cache.cc index 1f435900a..357e3874f 100644 --- a/src/file_api/file_cache.cc +++ b/src/file_api/file_cache.cc @@ -23,6 +23,7 @@ #include "file_cache.h" +#include "hash/hash_defs.h" #include "hash/xhash.h" #include "log/messages.h" #include "main/snort_config.h" @@ -37,35 +38,39 @@ using namespace snort; -static int file_cache_anr_free_func(void*, void* data) +class ExpectedFileCache : public XHash { - FileCache::FileNode* node = (FileCache::FileNode*)data; +public: + ExpectedFileCache(unsigned rows, unsigned key_len, unsigned datasize) + : XHash(rows, key_len, datasize, 0) + { } - if (!node) - return 0; - - struct timeval now; - packet_gettimeofday(&now); + ~ExpectedFileCache() override + { + delete_hash_table(); + } - // only recycle expired nodes - if (timercmp(&node->cache_expire_time, &now, <)) + bool is_node_recovery_ok(HashNode* hnode) override { - delete node->file; - return 0; + FileCache::FileNode* node = (FileCache::FileNode*)hnode->data; + if ( !node ) + return true; + + struct timeval now; + packet_gettimeofday(&now); + if ( timercmp(&node->cache_expire_time, &now, <) ) + return true; + else + return false; } - else - return 1; -} -static int file_cache_free_func(void*, void* data) -{ - FileCache::FileNode* node = (FileCache::FileNode*)data; - if (node) + void free_user_data(HashNode* hnode) override { - delete node->file; + FileCache::FileNode* node = (FileCache::FileNode*)hnode->data; + if ( node ) + delete node->file; } - return 0; -} +}; // Return the time in ms since we started waiting for pending file lookup. static int64_t time_elapsed_ms(struct timeval* now, struct timeval* expire_time, int64_t lookup_timeout) @@ -79,8 +84,7 @@ static int64_t time_elapsed_ms(struct timeval* now, struct timeval* expire_time, FileCache::FileCache(int64_t max_files_cached) { max_files = max_files_cached; - fileHash = new XHash(max_files, sizeof(FileHashKey), sizeof(FileNode), - 0, true, file_cache_anr_free_func, file_cache_free_func, true); + fileHash = new ExpectedFileCache(max_files, sizeof(FileHashKey), sizeof(FileNode)); fileHash->set_max_nodes(max_files); } @@ -155,18 +159,15 @@ FileContext* FileCache::find(const FileHashKey& hashKey, int64_t timeout) { std::lock_guard lock(cache_mutex); - if (!fileHash->get_node_count()) - { + if ( !fileHash->get_num_nodes() ) return nullptr; - } HashNode* hash_node = fileHash->find_node(&hashKey); - - if (!hash_node) + if ( !hash_node ) return nullptr; FileNode* node = (FileNode*)hash_node->data; - if (!node) + if ( !node ) { fileHash->release_node(hash_node); return nullptr; @@ -175,7 +176,7 @@ FileContext* FileCache::find(const FileHashKey& hashKey, int64_t timeout) struct timeval now; packet_gettimeofday(&now); - if (timercmp(&node->cache_expire_time, &now, <)) + if ( timercmp(&node->cache_expire_time, &now, <) ) { fileHash->release_node(hash_node); return nullptr; diff --git a/src/file_api/file_cache.h b/src/file_api/file_cache.h index 1ee25d89d..4c56cf10d 100644 --- a/src/file_api/file_cache.h +++ b/src/file_api/file_cache.h @@ -28,10 +28,7 @@ #include "file_config.h" -namespace snort -{ -class XHash; -} +class ExpectedFileCache; class FileCache { @@ -74,7 +71,7 @@ private: int store_verdict(snort::Flow*, snort::FileInfo*, int64_t timeout); /* The hash table of expected files */ - snort::XHash* fileHash = nullptr; + ExpectedFileCache* fileHash = nullptr; int64_t block_timeout = DEFAULT_FILE_BLOCK_TIMEOUT; int64_t lookup_timeout = DEFAULT_FILE_LOOKUP_TIMEOUT; int64_t max_files = DEFAULT_MAX_FILES_CACHED; diff --git a/src/file_api/file_identifier.cc b/src/file_api/file_identifier.cc index 2e7143ff5..5796f101b 100644 --- a/src/file_api/file_identifier.cc +++ b/src/file_api/file_identifier.cc @@ -32,6 +32,7 @@ #include #include +#include "hash/ghash.h" #include "log/messages.h" #include "utils/util.h" diff --git a/src/file_api/file_identifier.h b/src/file_api/file_identifier.h index c698db991..81c621ae3 100644 --- a/src/file_api/file_identifier.h +++ b/src/file_api/file_identifier.h @@ -29,10 +29,13 @@ #include #include -#include "hash/ghash.h" - #include "file_lib.h" +namespace snort +{ +class GHash; +} + #define MAX_BRANCH (UINT8_MAX + 1) enum IdNodeState diff --git a/src/filters/sfrf.cc b/src/filters/sfrf.cc index a985f6cae..6e47a7ef5 100644 --- a/src/filters/sfrf.cc +++ b/src/filters/sfrf.cc @@ -29,6 +29,7 @@ #include "main/thread.h" #include "detection/rules.h" #include "hash/ghash.h" +#include "hash/hash_defs.h" #include "hash/xhash.h" #include "sfip/sf_ip.h" #include "sfip/sf_ipvar.h" @@ -146,14 +147,12 @@ static void SFRF_New(unsigned nbytes) /* Calc max ip nodes for this memory */ if ( nbytes < SFRF_BYTES ) - { nbytes = SFRF_BYTES; - } + nrows = nbytes / (SFRF_BYTES); /* Create global hash table for all of the IP Nodes */ - rf_hash = new XHash(nrows, sizeof(tSFRFTrackingNodeKey), - sizeof(tSFRFTrackingNode), nbytes, true, nullptr, nullptr, true); + rf_hash = new XHash(nrows, sizeof(tSFRFTrackingNodeKey), sizeof(tSFRFTrackingNode), nbytes); } void SFRF_Delete() @@ -168,7 +167,7 @@ void SFRF_Delete() void SFRF_Flush() { if ( rf_hash ) - rf_hash->clear(); + rf_hash->clear_hash(); } static void SFRF_ConfigNodeFree(void* item) @@ -782,30 +781,24 @@ static tSFRFTrackingNode* _getSFRFTrackingNode(const SfIp* ip, unsigned tid, tim key.policyId = get_ips_policy()->policy_id; key.padding = 0; - /* - * Check for any Permanent sid objects for this gid or add this one ... - */ - HashNode* hnode = rf_hash->get_node((void*)&key); - if ( !hnode ) + // Check for any Permanent sid objects for this gid or add this one ... + if ( rf_hash->insert(&key, nullptr) == HASH_NOMEM ) { // xhash_get_node fails to insert only if rf_hash is full. rate_filter_stats.xhash_nomem_peg++; return dynNode; } - if ( hnode->data ) + dynNode = (tSFRFTrackingNode*)rf_hash->get_user_data(); + if ( dynNode->filterState == FS_NEW ) { - dynNode = (tSFRFTrackingNode*)hnode->data; - - if ( dynNode->filterState == FS_NEW ) - { - // first time initialization - dynNode->tstart = curTime; + // first time initialization + dynNode->tstart = curTime; #ifdef SFRF_OVER_RATE - dynNode->tlast = curTime; + dynNode->tlast = curTime; #endif - dynNode->filterState = FS_OFF; - } + dynNode->filterState = FS_OFF; } + return dynNode; } diff --git a/src/filters/sfthd.cc b/src/filters/sfthd.cc index b83ba0fa5..6eaaa5d2d 100644 --- a/src/filters/sfthd.cc +++ b/src/filters/sfthd.cc @@ -36,6 +36,7 @@ #include #include "hash/ghash.h" +#include "hash/hash_defs.h" #include "hash/xhash.h" #include "main/thread.h" #include "sfip/sf_ipvar.h" @@ -60,7 +61,7 @@ XHash* sfthd_new_hash(unsigned nbytes, size_t key, size_t data) nbytes = size; nrows = nbytes / size; - return new XHash(nrows, key, data, nbytes, true, nullptr, nullptr, true); + return new XHash(nrows, key, data, nbytes); } /*! @@ -891,7 +892,7 @@ int sfthd_test_local( if (status == HASH_INTABLE) { /* Already in the table */ - sfthd_ip_node = (THD_IP_NODE*)local_hash->get_cnode()->data; + sfthd_ip_node = (THD_IP_NODE*)local_hash->get_user_data(); /* Increment the event count */ sfthd_ip_node->count++; @@ -989,7 +990,7 @@ static inline int sfthd_test_global( if (status == HASH_INTABLE) { /* Already in the table */ - sfthd_ip_node = (THD_IP_NODE*)global_hash->get_cnode()->data; + sfthd_ip_node = (THD_IP_NODE*)global_hash->get_user_data(); /* Increment the event count */ sfthd_ip_node->count++; diff --git a/src/flow/expect_cache.cc b/src/flow/expect_cache.cc index 69873cc46..d63b6c6a4 100644 --- a/src/flow/expect_cache.cc +++ b/src/flow/expect_cache.cc @@ -134,7 +134,7 @@ void ExpectCache::prune() for (unsigned i = 0; i < MAX_PRUNE; ++i ) { - ExpectNode* node = (ExpectNode*)hash_table->first(); + ExpectNode* node = (ExpectNode*)hash_table->lru_first(); if ( !node || now <= node->expires ) break; @@ -147,7 +147,7 @@ void ExpectCache::prune() ExpectNode* ExpectCache::find_node_by_packet(Packet* p, FlowKey &key) { - if (!hash_table->get_count()) + if (!hash_table->get_num_nodes()) return nullptr; const SfIp* srcIP = p->ptrs.ip_api.get_src(); @@ -171,7 +171,7 @@ ExpectNode* ExpectCache::find_node_by_packet(Packet* p, FlowKey &key) */ // FIXIT-P X This should be optimized to only do full matches when full keys // are present, likewise for partial keys. - ExpectNode* node = (ExpectNode*) hash_table->find(&key); + ExpectNode* node = (ExpectNode*) hash_table->get_user_data(&key); if (!node) { // FIXIT-M X This logic could fail if IPs were equal because the original key @@ -192,12 +192,12 @@ ExpectNode* ExpectCache::find_node_by_packet(Packet* p, FlowKey &key) port2 = key.port_h; key.port_h = 0; } - node = (ExpectNode*) hash_table->find(&key); + node = (ExpectNode*) hash_table->get_user_data(&key); if (!node) { key.port_l = port1; key.port_h = port2; - node = (ExpectNode*) hash_table->find(&key); + node = (ExpectNode*) hash_table->get_user_data(&key); if (!node) return nullptr; } @@ -206,7 +206,7 @@ ExpectNode* ExpectCache::find_node_by_packet(Packet* p, FlowKey &key) { if (node->head) node->clear(free_list); - hash_table->release(&key); + hash_table->release_node(&key); return nullptr; } /* Make sure the packet direction is correct */ @@ -257,7 +257,7 @@ bool ExpectCache::process_expected(ExpectNode* node, FlowKey& key, Packet* p, Fl lws->ssn_state.snort_protocol_id = node->snort_protocol_id; if (!node->count) - hash_table->release(&key); + hash_table->release_node(&key); return ignoring; } @@ -270,11 +270,9 @@ ExpectCache::ExpectCache(uint32_t max) { // -size forces use of abs(size) ie w/o bumping up hash_table = new ZHash(-MAX_HASH, sizeof(FlowKey)); - hash_table->set_key_opcodes(FlowKey::hash, FlowKey::is_equal); - nodes = new ExpectNode[max]; for (unsigned i = 0; i < max; ++i) - hash_table->push(nodes+i); + hash_table->push(nodes + i); /* Preallocate a pool of ExpectFlows big enough to handle the worst case requirement (max number of nodes * max flows per node) and add them all @@ -332,54 +330,44 @@ int ExpectCache::add_flow(const Packet *ctrlPkt, PktType type, IpProtocol ip_pro uint16_t vlanId = (ctrlPkt->proto_bits & PROTO_BIT__VLAN) ? layer::get_vlan_layer(ctrlPkt)->vid() : 0; uint32_t mplsId = (ctrlPkt->proto_bits & PROTO_BIT__MPLS) ? ctrlPkt->ptrs.mplsHdr.label : 0; uint16_t addressSpaceId = ctrlPkt->pkth->address_space_id; - FlowKey key; bool reversed_key = key.init(type, ip_proto, cliIP, cliPort, srvIP, srvPort, vlanId, mplsId, addressSpaceId); bool new_node = false; - ExpectNode* node = (ExpectNode*) hash_table->get(&key, &new_node); - if (!node) + ExpectNode* node = (ExpectNode*) hash_table->get_user_data(&key); + if ( !node ) { prune(); - node = (ExpectNode*) hash_table->get(&key, &new_node); - /* The flow free list should never be empty if there was a node - to be (re-)used unless we managed to leak some. Check just - in case. Maybe assert instead? */ - if (!node || !free_list) - { - ++overflows; - return -1; - } + node = (ExpectNode*) hash_table->get(&key); + assert(node); + new_node = true; } - - /* If the node is past its expiration date, whack it and reuse it. */ - if (!new_node && packet_time() > node->expires) + else if ( packet_time() > node->expires ) { + // node is past its expiration date, whack it and reuse it. node->clear(free_list); new_node = true; } ExpectFlow* last = nullptr; - if (!new_node) + if ( !new_node ) { - // Requests will be rejected if the snort_protocol_id doesn't - // match what has already been set. - if (node->snort_protocol_id != snort_protocol_id) + // reject if the snort_protocol_id doesn't match + if ( node->snort_protocol_id != snort_protocol_id ) { - if (node->snort_protocol_id && snort_protocol_id) + if ( node->snort_protocol_id && snort_protocol_id ) return -1; node->snort_protocol_id = snort_protocol_id; } last = node->tail; - if (last) + if ( last ) { FlowData* lfd = last->data; - - while (lfd) + while ( lfd ) { - if (lfd->get_id() == fd->get_id()) + if ( lfd->get_id() == fd->get_id() ) { last = nullptr; break; @@ -403,9 +391,9 @@ int ExpectCache::add_flow(const Packet *ctrlPkt, PktType type, IpProtocol ip_pro } bool new_expect_flow = false; - if (!last) + if ( !last ) { - if (node->count >= MAX_LIST) + if ( node->count >= MAX_LIST ) { // fail when maxed out ++overflows; @@ -414,7 +402,7 @@ int ExpectCache::add_flow(const Packet *ctrlPkt, PktType type, IpProtocol ip_pro last = free_list; free_list = free_list->next; - if (!node->tail) + if ( !node->tail ) node->head = last; else node->tail->next = last; @@ -427,7 +415,7 @@ int ExpectCache::add_flow(const Packet *ctrlPkt, PktType type, IpProtocol ip_pro last->add_flow_data(fd); node->expires = packet_time() + MAX_WAIT; ++expects; - if (new_expect_flow) + if ( new_expect_flow ) { // chain all expected flows created by this packet packet_expect_flows->emplace_back(last); diff --git a/src/flow/flow_cache.cc b/src/flow/flow_cache.cc index b7ef59dc9..4054422f9 100644 --- a/src/flow/flow_cache.cc +++ b/src/flow/flow_cache.cc @@ -53,11 +53,8 @@ static const unsigned ALL_FLOWS = 3; FlowCache::FlowCache(const FlowCacheConfig& cfg) : config(cfg) { hash_table = new ZHash(config.max_flows, sizeof(FlowKey)); - hash_table->set_key_opcodes(FlowKey::hash, FlowKey::is_equal); - uni_flows = new FlowUniList; uni_ip_flows = new FlowUniList; - flags = 0x0; assert(prune_stats.get_total() == 0); @@ -79,12 +76,12 @@ void FlowCache::push(Flow* flow) unsigned FlowCache::get_count() { - return hash_table ? hash_table->get_count() : 0; + return hash_table ? hash_table->get_num_nodes() : 0; } Flow* FlowCache::find(const FlowKey* key) { - Flow* flow = (Flow*)hash_table->find(key); + Flow* flow = (Flow*)hash_table->get_user_data(key); if ( flow ) { @@ -119,7 +116,6 @@ Flow* FlowCache::allocate(const FlowKey* key) { time_t timestamp = packet_time(); Flow* flow = (Flow*)hash_table->get(key); - if ( !flow ) { if ( flows_allocated < config.max_flows ) @@ -160,7 +156,7 @@ void FlowCache::remove(Flow* flow) // FIXIT-M This check is added for offload case where both Flow::reset // and Flow::retire try remove the flow from hash. Flow::reset should // just mark the flow as pending instead of trying to remove it. - if ( hash_table->release(flow->key) ) + if ( hash_table->release_node(flow->key) ) memory::MemoryCap::update_deallocations(config.proto[to_utype(flow->key->pkt_type)].cap_weight); } @@ -184,7 +180,7 @@ unsigned FlowCache::prune_stale(uint32_t thetime, const Flow* save_me) ActiveSuspendContext act_susp; unsigned pruned = 0; - auto flow = static_cast(hash_table->first()); + auto flow = static_cast(hash_table->lru_first()); while ( flow and pruned <= cleanup_flows ) { @@ -196,7 +192,7 @@ unsigned FlowCache::prune_stale(uint32_t thetime, const Flow* save_me) if ( hash_table->get_count() == 1 ) break; - hash_table->touch(); + hash_table->lru_touch(); } #else // Reached the current flow. This *should* be the newest flow @@ -213,7 +209,7 @@ unsigned FlowCache::prune_stale(uint32_t thetime, const Flow* save_me) release(flow, PruneReason::IDLE); ++pruned; - flow = static_cast(hash_table->first()); + flow = static_cast(hash_table->lru_first()); } return pruned; @@ -262,11 +258,11 @@ unsigned FlowCache::prune_excess(const Flow* save_me) // initially skip offloads but if that doesn't work the hash table is iterated from the // beginning again. prune offloads at that point. - unsigned ignore_offloads = hash_table->get_count(); + unsigned ignore_offloads = hash_table->get_num_nodes(); - while ( hash_table->get_count() > max_cap and hash_table->get_count() > blocks ) + while ( hash_table->get_num_nodes() > max_cap and hash_table->get_num_nodes() > blocks ) { - auto flow = static_cast(hash_table->first()); + auto flow = static_cast(hash_table->lru_first()); assert(flow); // holds true because hash_table->get_count() > 0 if ( (save_me and flow == save_me) or flow->was_blocked() or @@ -279,8 +275,7 @@ unsigned FlowCache::prune_excess(const Flow* save_me) // FIXIT-M we should update last_data_seen upon touch to ensure // the hash_table LRU list remains sorted by time - if ( !hash_table->touch() ) - break; + hash_table->lru_touch(); } else { @@ -292,7 +287,7 @@ unsigned FlowCache::prune_excess(const Flow* save_me) --ignore_offloads; } - if (!pruned and hash_table->get_count() > max_cap) + if (!pruned and hash_table->get_num_nodes() > max_cap) { prune_one(PruneReason::EXCESS, true); ++pruned; @@ -304,11 +299,11 @@ unsigned FlowCache::prune_excess(const Flow* save_me) bool FlowCache::prune_one(PruneReason reason, bool do_cleanup) { // so we don't prune the current flow (assume current == MRU) - if ( hash_table->get_count() <= 1 ) + if ( hash_table->get_num_nodes() <= 1 ) return false; // ZHash returns in LRU order, which is updated per packet via find --> move_to_front call - auto flow = static_cast(hash_table->first()); + auto flow = static_cast(hash_table->lru_first()); assert(flow); flow->ssn_state.session_flags |= SSNFLAG_PRUNED; @@ -322,10 +317,10 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime) ActiveSuspendContext act_susp; unsigned retired = 0; - auto flow = static_cast(hash_table->current()); + auto flow = static_cast(hash_table->lru_current()); if ( !flow ) - flow = static_cast(hash_table->first()); + flow = static_cast(hash_table->lru_first()); while ( flow and (retired < num_flows) ) { @@ -340,7 +335,7 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime) if ( HighAvailabilityManager::in_standby(flow) or flow->is_suspended() ) { - flow = static_cast(hash_table->next()); + flow = static_cast(hash_table->lru_next()); continue; } @@ -349,7 +344,7 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime) ++retired; - flow = static_cast(hash_table->current()); + flow = static_cast(hash_table->lru_current()); } return retired; @@ -357,22 +352,20 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime) unsigned FlowCache::delete_active_flows(unsigned mode, unsigned num_to_delete, unsigned &deleted) { - unsigned flows_to_check = hash_table->get_count(); + unsigned flows_to_check = hash_table->get_num_nodes(); while ( num_to_delete && flows_to_check-- ) { - auto flow = static_cast(hash_table->first()); + auto flow = static_cast(hash_table->lru_first()); assert(flow); if ( (mode == ALLOWED_FLOWS_ONLY and (flow->was_blocked() || flow->is_suspended())) or (mode == OFFLOADED_FLOWS_TOO and flow->was_blocked()) ) { - if ( !hash_table->touch() ) - break; - + hash_table->lru_touch(); continue; } // we have a winner... - hash_table->remove(flow->key); + hash_table->remove(); if ( flow->next ) unlink_uni(flow); @@ -427,7 +420,7 @@ unsigned FlowCache::purge() unsigned retired = 0; - while ( auto flow = static_cast(hash_table->first()) ) + while ( auto flow = static_cast(hash_table->lru_first()) ) { retire(flow); ++retired; diff --git a/src/flow/flow_key.cc b/src/flow/flow_key.cc index 72575eb1d..b0c10fa87 100644 --- a/src/flow/flow_key.cc +++ b/src/flow/flow_key.cc @@ -24,7 +24,7 @@ #include "flow/flow_key.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "main/snort_config.h" #include "protocols/icmp4.h" #include "protocols/icmp6.h" @@ -297,46 +297,10 @@ bool FlowKey::init( // hash foo //------------------------------------------------------------------------- -uint32_t FlowKey::hash(HashFnc* hf, const unsigned char* p, int) -{ - uint32_t a, b, c; - a = b = c = hf->hardener; - - const uint32_t* d = (const uint32_t*)p; - - a += d[0]; // IPv6 lo[0] - b += d[1]; // IPv6 lo[1] - c += d[2]; // IPv6 lo[2] - - mix(a, b, c); - - a += d[3]; // IPv6 lo[3] - b += d[4]; // IPv6 hi[0] - c += d[5]; // IPv6 hi[1] - - mix(a, b, c); - - a += d[6]; // IPv6 hi[2] - b += d[7]; // IPv6 hi[3] - c += d[8]; // mpls label - - mix(a, b, c); - - a += d[9]; // port lo & port hi - b += d[10]; // vlan tag, address space id - c += d[11]; // ip_proto, pkt_type, version, and 8 bits of zeroed pad - - finalize(a, b, c); - - return c; -} - bool FlowKey::is_equal(const void* s1, const void* s2, size_t) { - const uint64_t* a,* b; - - a = (const uint64_t*)s1; - b = (const uint64_t*)s2; + const uint64_t* a = (const uint64_t*)s1; + const uint64_t* b = (const uint64_t*)s2; if (*a - *b) return false; /* Compares IPv4 lo/hi Compares IPv6 low[0,1] */ @@ -370,3 +334,43 @@ bool FlowKey::is_equal(const void* s1, const void* s2, size_t) return true; } +unsigned FlowHashKeyOps::do_hash(const unsigned char* k, int) +{ + uint32_t a, b, c; + a = b = c = hardener; + + const uint32_t* d = (const uint32_t*)k; + + a += d[0]; // IPv6 lo[0] + b += d[1]; // IPv6 lo[1] + c += d[2]; // IPv6 lo[2] + + mix(a, b, c); + + a += d[3]; // IPv6 lo[3] + b += d[4]; // IPv6 hi[0] + c += d[5]; // IPv6 hi[1] + + mix(a, b, c); + + a += d[6]; // IPv6 hi[2] + b += d[7]; // IPv6 hi[3] + c += d[8]; // mpls label + + mix(a, b, c); + + a += d[9]; // port lo & port hi + b += d[10]; // vlan tag, address space id + c += d[11]; // ip_proto, pkt_type, version, and 8 bits of zeroed pad + + finalize(a, b, c); + + return c; +} + +bool FlowHashKeyOps::key_compare(const void* k1, const void* k2, size_t len) +{ + return FlowKey::is_equal(k1, k2, len); +} + + diff --git a/src/flow/flow_key.h b/src/flow/flow_key.h index 4d9fb4ad8..c85129dc0 100644 --- a/src/flow/flow_key.h +++ b/src/flow/flow_key.h @@ -26,14 +26,27 @@ #include #include "framework/decode_data.h" +#include "hash/hash_key_operations.h" #include "utils/cpp_macros.h" -struct HashFnc; +class HashKeyOperations; namespace snort { struct SfIp; +class FlowHashKeyOps : public HashKeyOperations +{ +public: + FlowHashKeyOps(int rows) + : HashKeyOperations(rows) + { } + + unsigned do_hash(const unsigned char* k, int len) override; + bool key_compare(const void* k1, const void* k2, size_t) override; +}; + + PADDING_GUARD_BEGIN struct SO_PUBLIC FlowKey { @@ -70,8 +83,7 @@ struct SO_PUBLIC FlowKey void init_address_space(uint16_t); // If this data structure changes size, compare must be updated! - static uint32_t hash(HashFnc*, const unsigned char* d, int); - static bool is_equal(const void* s1, const void* s2, size_t); + static bool is_equal(const void* k1, const void* k2, size_t); private: bool init4( diff --git a/src/flow/test/CMakeLists.txt b/src/flow/test/CMakeLists.txt index 2cb88afba..df99e24f7 100644 --- a/src/flow/test/CMakeLists.txt +++ b/src/flow/test/CMakeLists.txt @@ -13,9 +13,11 @@ add_cpputest( flow_cache_test ../flow_cache.cc ../flow_control.cc ../flow_key.cc - ../../hash/zhash.cc - ../../hash/hashfcn.cc + ../../hash/hash_key_operations.cc + ../../hash/hash_lru_cache.cc ../../hash/primetable.cc + ../../hash/xhash.cc + ../../hash/zhash.cc ) add_cpputest( session_test ) diff --git a/src/framework/ips_option.cc b/src/framework/ips_option.cc index 44f0763ff..7f6fc3ec5 100644 --- a/src/framework/ips_option.cc +++ b/src/framework/ips_option.cc @@ -25,7 +25,7 @@ #include -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" using namespace snort; diff --git a/src/hash/CMakeLists.txt b/src/hash/CMakeLists.txt index 8cbdd87e2..029a77ef9 100644 --- a/src/hash/CMakeLists.txt +++ b/src/hash/CMakeLists.txt @@ -1,23 +1,25 @@ set (HASH_INCLUDES + ghash.h hashes.h hash_defs.h - ghash.h - xhash.h - hashfcn.h + hash_key_operations.h lru_cache_shared.h + xhash.h ) add_library( hash OBJECT ${HASH_INCLUDES} + ghash.cc hashes.cc + hash_lru_cache.cc + hash_lru_cache.h + hash_key_operations.cc lru_cache_shared.cc - ghash.cc - hashfcn.cc - primetable.cc - primetable.h - xhash.cc - zhash.cc + primetable.cc + primetable.h + xhash.cc + zhash.cc zhash.h ) diff --git a/src/hash/ghash.cc b/src/hash/ghash.cc index 80e264bb4..d2261ed49 100644 --- a/src/hash/ghash.cc +++ b/src/hash/ghash.cc @@ -62,6 +62,7 @@ #include "utils/util.h" +#include "hash_defs.h" #include "primetable.h" namespace snort @@ -75,7 +76,7 @@ GHash::GHash(int nrows_, unsigned keysize, bool userkey, gHashFree userfree) else nrows = -nrows_; - hashfcn = hashfcn_new(nrows); + hashfcn = new HashKeyOperations(nrows); table = (GHashNode**)snort_calloc(nrows, sizeof(GHashNode*)); for ( int i = 0; i < nrows; i++ ) table[i] = nullptr; @@ -87,8 +88,6 @@ GHash::GHash(int nrows_, unsigned keysize, bool userkey, gHashFree userfree) GHash::~GHash() { - hashfcn_free(hashfcn); - for (int i = 0; i < nrows; i++) for ( GHashNode* node = table[i]; node; ) { @@ -105,6 +104,7 @@ GHash::~GHash() } snort_free(table); + delete hashfcn; } // set key length, hashkey, and index parameters required to find/add/remove a node @@ -112,7 +112,7 @@ GHash::~GHash() void GHash::set_node_parameters(const void* const key) { klen = ( keysize > 0 ) ? keysize : strlen((const char*)key) + 1; - hashkey = hashfcn->hash_fcn(hashfcn, (const unsigned char*)key, klen); + hashkey = hashfcn->do_hash((const unsigned char*)key, klen); index = hashkey % nrows; } @@ -130,7 +130,7 @@ GHashNode* GHash::find_node(const void* const key) } else { - if ( hashfcn->keycmp_fcn(hnode->key, key, keysize) ) + if ( hashfcn->key_compare(hnode->key, key, keysize) ) return hnode; } } @@ -145,7 +145,7 @@ int GHash::insert(const void* const key, void* const data) if ( GHashNode* hnode = find_node(key) ) { cnode = hnode; - return GHASH_INTABLE; + return HASH_INTABLE; } GHashNode* hnode = (GHashNode*)snort_calloc(sizeof(GHashNode)); @@ -177,7 +177,7 @@ int GHash::insert(const void* const key, void* const data) count++; - return GHASH_OK; + return HASH_OK; } void* GHash::find(const void* const key) @@ -219,7 +219,7 @@ int GHash::free_node(unsigned index, GHashNode* hnode) snort_free(hnode); count--; - return GHASH_OK; + return HASH_OK; } int GHash::remove(const void* const key) @@ -229,7 +229,7 @@ int GHash::remove(const void* const key) if ( GHashNode* hnode = find_node(key) ) return free_node(index, hnode); else - return GHASH_NOT_FOUND; + return HASH_NOT_FOUND; } void GHash::next() @@ -272,9 +272,10 @@ GHashNode* GHash::find_next() return n; } -void GHash::set_key_opcodes(hash_func hash_fcn, keycmp_func keycmp_fcn) +void GHash::set_hashkey_ops(HashKeyOperations* hk) { - hashfcn_set_keyops(hashfcn, hash_fcn, keycmp_fcn); + delete hashfcn; + hashfcn = hk; } } diff --git a/src/hash/ghash.h b/src/hash/ghash.h index 17b874e74..ca210e69f 100644 --- a/src/hash/ghash.h +++ b/src/hash/ghash.h @@ -24,22 +24,17 @@ // generic hash table - stores and maps key + data pairs +#include "hash_key_operations.h" #include "main/snort_types.h" -#include "hashfcn.h" - namespace snort { -#define GHASH_NOT_FOUND (-1) -#define GHASH_OK 0 -#define GHASH_INTABLE 1 - struct GHashNode { struct GHashNode* next; struct GHashNode* prev; - const void* key; /* Copy of, or Pointer to, the Users key */ - void* data; /* The users data, this is never copied! */ + const void* key; + void* data; }; typedef void (* gHashFree)(void*); @@ -55,7 +50,7 @@ public: void* find(const void* const key); GHashNode* find_first(); GHashNode* find_next(); - void set_key_opcodes(hash_func, keycmp_func); + void set_hashkey_ops(HashKeyOperations*); unsigned get_count() const { return count; } @@ -70,7 +65,7 @@ private: bool userkey; // user owns the key */ gHashFree userfree; int nrows; // # rows int the hash table use a prime number 211, 9871 - HashFnc* hashfcn; + HashKeyOperations* hashfcn; GHashNode** table; // array of node ptr's unsigned count; // total # nodes in table int crow; // findfirst/next row in table diff --git a/src/hash/hash_defs.h b/src/hash/hash_defs.h index f66f16495..e7ab1d21c 100644 --- a/src/hash/hash_defs.h +++ b/src/hash/hash_defs.h @@ -20,30 +20,28 @@ #ifndef HASH_DEFS_H #define HASH_DEFS_H +#include "hash_key_operations.h" #include "main/snort_types.h" #include "utils/sfmemcap.h" -#include "hashfcn.h" - namespace snort { -#define HASH_NOMEM (-2) -#define HASH_ERR (-1) +#define HASH_NOMEM (-2) +#define HASH_NOT_FOUND (-1) #define HASH_OK 0 #define HASH_INTABLE 1 #define HASH_PENDING 2 -struct HashNode +class HashNode { - struct HashNode* gnext; // global node list - used for aging nodes - struct HashNode* gprev; - struct HashNode* next; // row node list - struct HashNode* prev; - int rindex; // row index of table this node belongs to. - void* key; // Pointer to the key. - void* data; // Pointer to the users data, this is not copied ! +public: + HashNode* gnext; // lru or free node list + HashNode* gprev; + HashNode* next; // hash row node list + HashNode* prev; + void* key; + void* data; + int rindex; }; - -typedef int (* Hash_FREE_FCN)(void* key, void* data); } #endif diff --git a/src/hash/hashfcn.cc b/src/hash/hash_key_operations.cc similarity index 65% rename from src/hash/hashfcn.cc rename to src/hash/hash_key_operations.cc index 12b73a495..72700c32d 100644 --- a/src/hash/hashfcn.cc +++ b/src/hash/hash_key_operations.cc @@ -17,22 +17,11 @@ // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. //-------------------------------------------------------------------------- -/* - hashfcn.c - - Each hash table allocates it's own GHash struct, using the number of - rows in the hash table to modulo the random values. - - Updates: - - 8/31/2006 - man - changed to use sfprimetable.c -*/ - #ifdef HAVE_CONFIG_H #include "config.h" #endif -#include "hashfcn.h" +#include "hash_key_operations.h" #include @@ -43,77 +32,48 @@ using namespace snort; -static bool hashfcn_key_compare(const void* k1, const void* k2, size_t len) -{ - if ( memcmp(k1, k2, len ) == 0 ) - return true; - else - return false; -} - -HashFnc* hashfcn_new(int m) +HashKeyOperations::HashKeyOperations(int rows) { - HashFnc* p; - static int one = 1; + static bool one = true; if ( one ) /* one time init */ { srand( (unsigned)time(nullptr) ); - one = 0; + one = false; } - p = (HashFnc*)snort_calloc(sizeof(*p)); - if ( SnortConfig::static_hash() ) { - p->seed = 3193; - p->scale = 719; - p->hardener = 133824503; + seed = 3193; + scale = 719; + hardener = 133824503; } else { - p->seed = nearest_prime( (rand() % m) + 3191); - p->scale = nearest_prime( (rand() % m) + 709); - p->hardener = ((unsigned) rand() * rand()) + 133824503; + seed = nearest_prime( (rand() % rows) + 3191); + scale = nearest_prime( (rand() % rows) + 709); + hardener = ((unsigned) rand() * rand()) + 133824503; } - - p->hash_fcn = &hashfcn_hash; - p->keycmp_fcn = &hashfcn_key_compare; - - return p; -} - -void hashfcn_free(HashFnc* p) -{ - if ( p ) - snort_free(p); } -unsigned hashfcn_hash(HashFnc* p, const unsigned char* d, int n) +unsigned HashKeyOperations::do_hash(const unsigned char* key, int len) { - unsigned hash = p->seed; - while ( n ) + unsigned hash = seed; + while ( len ) { - hash *= p->scale; - hash += *d++; - n--; + hash *= scale; + hash += *key++; + len--; } - return hash ^ p->hardener; + return hash ^ hardener; } -/** - * Make hashfcn use a separate set of opcodes for the backend. - * - * @param h hashfcn ptr - * @param hash_fcn user specified hash function - * @param keycmp_fcn user specified key comparison function - */ -void hashfcn_set_keyops(HashFnc* h, hash_func hash_fcn, keycmp_func keycmp_fcn) +bool HashKeyOperations::key_compare(const void* key1, const void* key2, size_t len) { - assert(h && hash_fcn && keycmp_fcn); - - h->hash_fcn = hash_fcn; - h->keycmp_fcn = keycmp_fcn; + if ( memcmp(key1, key2, len ) == 0 ) + return true; + else + return false; } namespace snort diff --git a/src/hash/hashfcn.h b/src/hash/hash_key_operations.h similarity index 83% rename from src/hash/hashfcn.h rename to src/hash/hash_key_operations.h index ad5663404..b48ab11fb 100644 --- a/src/hash/hashfcn.h +++ b/src/hash/hash_key_operations.h @@ -17,8 +17,8 @@ // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. //-------------------------------------------------------------------------- -#ifndef HASHFCN_H -#define HASHFCN_H +#ifndef HASH_KEY_OPERATIONS_H +#define HASH_KEY_OPERATIONS_H #include "main/snort_types.h" @@ -65,26 +65,22 @@ static inline int hash_nearest_power_of_2(int nrows) return nrows; } -} - -struct HashFnc; +class HashKeyOperations +{ +public: + HashKeyOperations(int rows); + virtual ~HashKeyOperations() + { } -typedef uint32_t (* hash_func)(HashFnc*, const unsigned char* d, int n); -typedef bool (* keycmp_func)(const void* s1, const void* s2, size_t n); + virtual unsigned do_hash(const unsigned char* key, int len); + virtual bool key_compare(const void* key1, const void* key2, size_t len); -struct HashFnc -{ +protected: unsigned seed; unsigned scale; unsigned hardener; - hash_func hash_fcn; - keycmp_func keycmp_fcn; }; - -HashFnc* hashfcn_new(int nrows); -void hashfcn_free(HashFnc*); -unsigned hashfcn_hash(HashFnc*, const unsigned char* d, int n); -void hashfcn_set_keyops(HashFnc*, hash_func, keycmp_func); +} #endif diff --git a/src/hash/hash_lru_cache.cc b/src/hash/hash_lru_cache.cc new file mode 100644 index 000000000..b155e59c0 --- /dev/null +++ b/src/hash/hash_lru_cache.cc @@ -0,0 +1,87 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// hash_lru_cache.cc author davis mcpherson davmcphe@cisco.com + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "hash_lru_cache.h" + +#include +#include "utils/util.h" + +using namespace snort; + +HashLruCache::HashLruCache() +{ + head = nullptr; + tail = nullptr; +} + +void HashLruCache::insert(HashNode* hnode) +{ + if ( head ) + { + hnode->gprev = nullptr; + hnode->gnext = head; + head->gprev = hnode; + head = hnode; + } + else + { + hnode->gprev = nullptr; + hnode->gnext = nullptr; + head = hnode; + tail = hnode; + } +} + +void HashLruCache::touch(HashNode* hnode) +{ + if ( hnode == cursor ) + cursor = hnode->gprev; + + if ( hnode != head ) + { + remove_node(hnode); + insert(hnode); + } +} + +void HashLruCache::remove_node(HashNode* hnode) +{ + if ( cursor == hnode ) + cursor = hnode->gprev; + + if ( head == hnode ) + { + head = head->gnext; + if ( head ) + head->gprev = nullptr; + } + + if ( hnode->gprev ) + hnode->gprev->gnext = hnode->gnext; + if ( hnode->gnext ) + hnode->gnext->gprev = hnode->gprev; + + if ( tail == hnode ) + tail = hnode->gprev; +} diff --git a/src/hash/hash_lru_cache.h b/src/hash/hash_lru_cache.h new file mode 100644 index 000000000..6436c3910 --- /dev/null +++ b/src/hash/hash_lru_cache.h @@ -0,0 +1,73 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// hash_lru_cache.h author davis mcpherson davmcphe@cisco.com + +#ifndef HASH_LRU_CACHE_H +#define HASH_LRU_CACHE_H + +#include "hash_defs.h" + +class HashLruCache +{ +public: + HashLruCache(); + + void insert(snort::HashNode*); + void touch(snort::HashNode*); + void remove_node(snort::HashNode*); + + snort::HashNode* get_lru_node() + { + cursor = tail; + return cursor; + } + + snort::HashNode* get_next_lru_node() + { + if ( cursor ) + cursor = cursor->gprev; + return cursor; + } + + snort::HashNode* get_current_node() + { return cursor; } + + void* get_mru_user_data() + { return ( head ) ? head->data : nullptr; } + + void* get_lru_user_data() + { return ( tail ) ? tail->data : nullptr; } + + snort::HashNode* remove_lru_node() + { + snort::HashNode* hnode = tail; + if ( hnode ) + remove_node(hnode); + + return hnode; + } + +private: + snort::HashNode* head = nullptr; + snort::HashNode* tail = nullptr; + snort::HashNode* cursor = nullptr; +}; + +#endif + diff --git a/src/hash/test/CMakeLists.txt b/src/hash/test/CMakeLists.txt index 39896fa9b..7346627d1 100644 --- a/src/hash/test/CMakeLists.txt +++ b/src/hash/test/CMakeLists.txt @@ -2,17 +2,33 @@ add_cpputest( lru_cache_shared_test SOURCES ../lru_cache_shared.cc ) +add_cpputest( hash_lru_cache_test + SOURCES ../hash_lru_cache.cc +) + add_cpputest( xhash_test SOURCES - ../xhash.cc - ../hashfcn.cc + ../hash_key_operations.cc + ../hash_lru_cache.cc ../primetable.cc + ../xhash.cc ../../utils/sfmemcap.cc ) add_cpputest( ghash_test SOURCES ../ghash.cc - ../hashfcn.cc + ../hash_key_operations.cc + ../hash_lru_cache.cc ../primetable.cc ) + +add_cpputest( zhash_test + SOURCES + ../hash_key_operations.cc + ../hash_lru_cache.cc + ../primetable.cc + ../xhash.cc + ../zhash.cc + ../../utils/sfmemcap.cc +) \ No newline at end of file diff --git a/src/hash/test/ghash_test.cc b/src/hash/test/ghash_test.cc index c1175a9cc..fba122d44 100644 --- a/src/hash/test/ghash_test.cc +++ b/src/hash/test/ghash_test.cc @@ -25,6 +25,7 @@ #include "hash/ghash.h" +#include "hash/hash_defs.h" #include "main/snort_config.h" #include "utils/util.h" @@ -119,7 +120,7 @@ TEST(ghash, collision_test) snprintf(str, sizeof(str), "KeyWord%d",1); str[sizeof(str) - 1] = '\0'; i = t->insert(str, (void *)(str + (1))); - CHECK(i == GHASH_INTABLE); + CHECK(i == HASH_INTABLE); // find those nodes for (i=num-1; i>=0; i--) @@ -192,7 +193,7 @@ TEST(ghash, userfree_test) CHECK(t->find(str) == nullptr); // try to remove a node that is not in the table - CHECK(t->remove( str) == GHASH_NOT_FOUND); + CHECK(t->remove( str) == HASH_NOT_FOUND); for ( GHashNode* n = t->find_first(); n; n = t->find_next() ) { diff --git a/src/hash/test/hash_lru_cache_test.cc b/src/hash/test/hash_lru_cache_test.cc new file mode 100644 index 000000000..5a2877476 --- /dev/null +++ b/src/hash/test/hash_lru_cache_test.cc @@ -0,0 +1,140 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// hash_lru_cache_test.cc author davis mcpherson +// unit tests for the HashLruCache class + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "../hash_lru_cache.h" + +#include "../../utils/util.h" + +#include +#include + +using namespace snort; + +HashLruCache* lru = nullptr; + +TEST_GROUP(hash_lru_cache) +{ + void setup() override + { + lru = new HashLruCache; + CHECK(lru); + } + + void teardown() override + { + unsigned node_id = 1; + HashNode* node = lru->remove_lru_node(); + while( node ) + { + CHECK(*(unsigned*)node->data == node_id); + unsigned* data = (unsigned *)lru->get_mru_user_data(); + if ( data ) + CHECK(*data == 5); + data = (unsigned *)lru->get_lru_user_data(); + if ( data ) + CHECK(*data == node_id + 1); + ++node_id; + delete (unsigned*)node->data; + snort_free(node); + node = lru->remove_lru_node(); + } + + delete lru; + } +}; + +TEST(hash_lru_cache, create_hash_lru_cache_test) +{ + CHECK(!lru->get_next_lru_node()); + CHECK(!lru->get_current_node()); + CHECK(!lru->get_mru_user_data()); + CHECK(!lru->get_lru_user_data()); + CHECK(!lru->remove_lru_node()); +} + +TEST(hash_lru_cache, hash_lru_cache_insert_test) +{ + HashNode* node; + for (unsigned i = 0; i < 5; i++) + { + node = (HashNode*)snort_calloc(sizeof(HashNode)); + unsigned* data = new unsigned; + *data = i + 1; + node->data = data; + lru->insert(node); + CHECK(*((unsigned *)lru->get_mru_user_data()) == i + 1); + CHECK(*((unsigned *)lru->get_lru_user_data()) == 1); + } + + for (unsigned i = 0; i < 5; i++) + { + node = lru->get_lru_node(); + CHECK(node); + CHECK(*((unsigned*)node->data) == i + 1); + lru->touch(node); + CHECK(*((unsigned *)lru->get_mru_user_data()) == i + 1); + CHECK(*((unsigned *)lru->get_lru_user_data()) == ((i + 1) % 5) + 1); + } + + node = lru->get_lru_node(); + CHECK(node); + CHECK(*((unsigned*)node->data) == 1); +} + +TEST(hash_lru_cache, hash_lru_cache_browse_test) +{ + HashNode* node; + for (unsigned i = 0; i < 5; i++) + { + node = (HashNode*)snort_calloc(sizeof(HashNode)); + unsigned* data = new unsigned; + *data = i + 1; + node->data = data; + lru->insert(node); + } + + for (unsigned i = 0; i < 5; i++) + { + node = lru->get_lru_node(); + CHECK(node); + CHECK(*((unsigned*)node->data) == (i + 1)); + lru->touch(node); + } + + node = lru->get_lru_node(); + unsigned i = 1; + while( node ) + { + CHECK(*(unsigned*)node->data == i); + CHECK(*(unsigned*)lru->get_current_node()->data == i); + node = lru->get_next_lru_node(); + ++i; + } +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +} diff --git a/src/hash/test/xhash_test.cc b/src/hash/test/xhash_test.cc index a52a4d92d..96002f4e8 100644 --- a/src/hash/test/xhash_test.cc +++ b/src/hash/test/xhash_test.cc @@ -25,6 +25,7 @@ #include "hash/xhash.h" +#include "hash/hash_defs.h" #include "main/snort_config.h" #include "utils/util.h" @@ -56,9 +57,8 @@ TEST_GROUP(xhash) // Test create a hash table, add nodes, find and delete. TEST(xhash, create_xhash_test) { - XHash* test_table = new XHash(4, sizeof(struct xhash_test_key), - 0, 0, false, nullptr, nullptr, false); - CHECK(test_table->get_keysize() == sizeof(struct xhash_test_key)); + XHash* test_table = new XHash(4, sizeof(struct xhash_test_key), 0, 0); + CHECK(test_table); void* data = test_table->get_mru_user_data(); CHECK(data == nullptr); @@ -84,39 +84,40 @@ TEST(xhash, create_xhash_test) // Create a free node in xhash and verifies if xhash_free_anr_lru() deletes it TEST(xhash, free_anr_lru_delete_free_node_test) { - XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), - 1, 1040, false, nullptr, nullptr, true); + XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), 1, 1040); + CHECK(test_table); + xhash_test_key xtk; xtk.key = 10; int ret = test_table->insert(&xtk, nullptr); CHECK(ret == HASH_OK); - HashNode* xnode = test_table->get_node(&xtk); - CHECK(xnode != nullptr); + HashNode* xnode = test_table->find_node(&xtk); + CHECK(xnode); ret = test_table->release_node(xnode); CHECK(ret == HASH_OK); - ret = test_table->delete_anr_or_lru_node(); + ret = test_table->delete_lru_node(); CHECK(ret == HASH_OK); HashNode* xhnode = test_table->find_node(&xtk); - CHECK(xhnode == nullptr); + CHECK(!xhnode); delete test_table; } -// No free node is available, verifies if xhash_free_anr_lru() deletes the last node +// No free node is available, verifies the LRU node is deleted TEST(xhash, free_anr_lru_delete_tail_node_test) { - XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), - 1, 1040, false, nullptr, nullptr, true); + XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), 1, 1040); + CHECK(test_table); + xhash_test_key xtk; int ret = test_table->insert(&xtk, nullptr); CHECK(ret == HASH_OK); - ret = test_table->delete_anr_or_lru_node(); - CHECK(ret == HASH_OK); + CHECK(test_table->delete_lru_node()); HashNode* xhnode = test_table->find_node(&xtk); CHECK(xhnode == nullptr); @@ -124,20 +125,6 @@ TEST(xhash, free_anr_lru_delete_tail_node_test) delete test_table; } -// No free node is available [recycle is not enabled], verifies if last node is deleted -TEST(xhash, free_anr_lru_usr_free_delete_tail_node_test) -{ - XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), - 1, 1040, false, nullptr, nullptr, false); - xhash_test_key xtk; - int ret = test_table->insert(&xtk, nullptr); - CHECK(ret == HASH_OK); - - ret = test_table->delete_anr_or_lru_node(); - CHECK(ret == HASH_OK); - delete test_table; -} - int main(int argc, char** argv) { return CommandLineTestRunner::RunAllTests(argc, argv); diff --git a/src/hash/test/zhash_test.cc b/src/hash/test/zhash_test.cc new file mode 100644 index 000000000..93e4817ac --- /dev/null +++ b/src/hash/test/zhash_test.cc @@ -0,0 +1,149 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// zhash_test.cc author davis mcpherson +// unit tests for the HashLruCache class + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include + +#include "../zhash.h" +#include "../hash_key_operations.h" + +#include "flow/flow_key.h" +#include "main/snort_config.h" + +#include +#include + +using namespace snort; + +namespace snort +{ +unsigned FlowHashKeyOps::do_hash(const unsigned char* k, int len) +{ + unsigned hash = seed; + while ( len ) + { + hash *= scale; + hash += *k++; + len--; + } + return hash ^ hardener; +} + +bool FlowHashKeyOps::key_compare(const void* k1, const void* k2, size_t len) +{ + if ( memcmp(k1, k2, len ) == 0 ) + return true; + else + return false; +} +} + +// Stubs whose sole purpose is to make the test code link +static SnortConfig my_config; +THREAD_LOCAL SnortConfig *snort_conf = &my_config; + +SnortConfig::SnortConfig(const SnortConfig* const) +{ snort_conf->run_flags = 0;} // run_flags is used indirectly from HashFnc class by calling SnortConfig::static_hash() + +SnortConfig::~SnortConfig() = default; + +SnortConfig* SnortConfig::get_conf() +{ return snort_conf; } + +const unsigned ZHASH_ROWS = 1000; +const unsigned ZHASH_KEY_SIZE = 100; +const unsigned MAX_ZHASH_NODES = 100; +char key_buf[ZHASH_KEY_SIZE]; + +ZHash* zh = nullptr; + +TEST_GROUP(zhash) +{ + void setup() override + { + zh = new ZHash(ZHASH_ROWS, ZHASH_KEY_SIZE); + CHECK(zh); + + memset(key_buf, '\0', ZHASH_KEY_SIZE); + } + + void teardown() override + { + delete zh; + } +}; + +TEST(zhash, create_zhash_test) +{ + for (unsigned i = 0; i < MAX_ZHASH_NODES; i++ ) + { + unsigned* data; + data = (unsigned*)snort_calloc(sizeof(unsigned)); + *data = 0; + zh->push(data); + } + + std::string key_prefix = "foo"; + for (unsigned i = 0; i < MAX_ZHASH_NODES; i++ ) + { + std::string key; + key = key_prefix + std::to_string(i + 1); + memcpy(key_buf, key.c_str(), key.size()); + unsigned* data = (unsigned*)zh->get(key_buf); + CHECK(*data == 0); + *data = i + 1; + } + + unsigned nodes_walked = 0; + unsigned* data = (unsigned*)zh->lru_first(); + while ( data ) + { + CHECK(*data == ++nodes_walked); + data = (unsigned*)zh->lru_next(); + } + + CHECK(nodes_walked == MAX_ZHASH_NODES); + + data = (unsigned*)zh->lru_first(); + CHECK(*data == 1); + data = (unsigned*)zh->remove(); + CHECK(*data == 1); + snort_free(data); + data = (unsigned*)zh->lru_current(); + CHECK(*data == 2); + data = (unsigned*)zh->lru_first(); + CHECK(*data == 2); + + for (unsigned i = 1; i < MAX_ZHASH_NODES; i++ ) + { + data = (unsigned*)zh->remove(); + CHECK(*data == (i + 1)); + snort_free(data); + } +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +} diff --git a/src/hash/xhash.cc b/src/hash/xhash.cc index bcca7c412..962e49e81 100644 --- a/src/hash/xhash.cc +++ b/src/hash/xhash.cc @@ -95,209 +95,259 @@ #include #include "utils/util.h" +#include "hash_defs.h" +#include "hash_key_operations.h" +#include "hash_lru_cache.h" + using namespace snort; namespace snort { -XHash::XHash(int nrows_, int keysize_, int datasize_, unsigned long maxmem, - bool anr_enabled, Hash_FREE_FCN anr_free, Hash_FREE_FCN usr_free, - bool recycle_nodes) - : keysize(keysize_), datasize(datasize_), recycle_nodes(recycle_nodes), - anr_enabled(anr_enabled), anr_free(anr_free), usr_free(usr_free) +void XHash::initialize(HashKeyOperations* hk_ops) +{ + hashkey_ops = hk_ops; + table = (HashNode**)snort_calloc(sizeof(HashNode*) * nrows); + lru_cache = new HashLruCache(); + mem_allocator = new MemCapAllocator(mem_cap, sizeof(HashNode) + keysize + datasize); +} + +void XHash::initialize() +{ + initialize(new HashKeyOperations(nrows)); +} + +void XHash::set_number_of_rows (int rows) { - // adjust rows to be power of 2 - if ( nrows_ > 0 ) - nrows = hash_nearest_power_of_2(nrows_); + if ( rows > 0 ) + nrows = hash_nearest_power_of_2 (rows); else - nrows = -nrows_; // if negative use as is + nrows = -rows; +} - table = (HashNode**)snort_calloc(sizeof(HashNode*) * nrows); - hashfcn = hashfcn_new(nrows); - sfmemcap_init(&mc, maxmem); +XHash::XHash(int rows, int keysize) + : keysize(keysize) - for ( unsigned i = 0; i < nrows; i++ ) - table[i] = nullptr; +{ + set_number_of_rows(rows); +} - mem_allocated_per_entry = sizeof(HashNode) + keysize + datasize + sizeof(long); +XHash::XHash(int rows, int keysize, int datasize, unsigned long memcap) + : keysize(keysize), datasize(datasize), mem_cap(memcap) +{ + set_number_of_rows(rows); + initialize(); } XHash::~XHash() { - if ( hashfcn ) - hashfcn_free(hashfcn); - - for (unsigned i = 0; i < nrows; i++) + if ( table ) { - for ( HashNode* node = table[i]; node; ) - { - HashNode* onode = node; - node = node->next; - - if ( usr_free ) - usr_free(onode->key, onode->data); + for (unsigned i = 0; i < nrows; i++) + for (HashNode* node = table[i]; node;) + { + HashNode* xnode; + xnode = node; + node = node->next; + mem_allocator->free(xnode); + } - sfmemcap_free(&mc, onode); - } + snort_free(table); } - snort_free(table); + purge_free_list(); + delete hashkey_ops; + delete lru_cache; + delete mem_allocator; } -void XHash::clear() +void XHash::delete_hash_table() { + assert( table ); + for (unsigned i = 0; i < nrows; i++) - { - HashNode* n = table[i]; - while ( n ) + for (HashNode* node = table[i]; node;) { - HashNode* tmp; - tmp = n; - n = n->next; - release_node(tmp); + HashNode* xnode; + xnode = node; + node = node->next; + free_user_data(xnode); + mem_allocator->free(xnode); } - } - max_nodes = 0; - crow = 0; - cnode = nullptr; - count = 0; - ghead = nullptr; - gtail = nullptr; - anr_count = 0; - anr_tries = 0; - find_success = 0; - find_fail = 0; + snort_free(table); + table = nullptr; } -void XHash::save_free_node(HashNode* hnode) +void XHash::initialize_node(HashNode *hnode, const void *key, void *data, int index) { - if ( fhead ) + hnode->key = (char*) (hnode) + sizeof(HashNode); + memcpy(hnode->key, key, keysize); + if ( datasize ) { - hnode->gprev = nullptr; - hnode->gnext = fhead; - fhead->gprev = hnode; - fhead = hnode; + hnode->data = (char*) (hnode) + sizeof(HashNode) + keysize; + if ( data ) + memcpy (hnode->data, data, datasize); } else - { - hnode->gprev = nullptr; - hnode->gnext = nullptr; - fhead = hnode; - ftail = hnode; - } + hnode->data = data; + + hnode->rindex = index; + link_node(hnode); + lru_cache->insert(hnode); } -HashNode* XHash::get_free_node() +HashNode* XHash::allocate_node(const void* key, void* data, int index) { - HashNode* node = fhead; + // use a free one if available... + HashNode* hnode = get_free_node(); - if ( fhead ) - { - fhead = fhead->gnext; - if ( fhead ) - fhead->gprev = nullptr; + // if no free nodes, try to allocate a new one... + if ( !hnode && ((max_nodes == 0) || (num_nodes < max_nodes)) ) + hnode = (HashNode*)mem_allocator->allocate(); - if ( ftail == node ) - ftail = nullptr; + // if still no node then try to reuse one... + if ( !hnode && anr_enabled ) + hnode = release_lru_node(); + + if ( hnode ) + { + initialize_node(hnode, key, data, index); + ++num_nodes; } - return node; + return hnode; } -void XHash::purge_free_list() +int XHash::insert(const void* key, void* data) { - HashNode* cur = fhead; - while ( cur ) + assert(key); + + int index = 0; + HashNode* hnode = find_node_row(key, index); + if ( hnode ) { - HashNode* next = cur->gnext; - sfmemcap_free(&mc, (void*)cur); - cur = next; + cursor = hnode; + return HASH_INTABLE; } - fhead = nullptr; - ftail = nullptr; + hnode = allocate_node(key, data, index); + cursor = hnode; + return ( hnode ) ? HASH_OK : HASH_NOMEM; } -void XHash::glink_node(HashNode* hnode) +HashNode* XHash::find_node(const void* key) { - if ( ghead ) - { - hnode->gprev = nullptr; - hnode->gnext = ghead; - ghead->gprev = hnode; - ghead = hnode; - } - else - { - hnode->gprev = nullptr; - hnode->gnext = nullptr; - ghead = hnode; - gtail = hnode; - } + assert(key); + + int rindex = 0; + return find_node_row(key, rindex); } -void XHash::gunlink_node(HashNode* hnode) +HashNode* XHash::find_first_node() { - if ( gnode == hnode ) - gnode = hnode->gnext; - - if ( ghead == hnode ) + for ( crow = 0; crow < nrows; crow++ ) { - ghead = ghead->gnext; - if ( ghead ) - ghead->gprev = nullptr; + cursor = table[crow]; + if ( cursor ) + { + HashNode* n = cursor; + update_cursor(); + return n; + } } - if ( hnode->gprev ) - hnode->gprev->gnext = hnode->gnext; - if ( hnode->gnext ) - hnode->gnext->gprev = hnode->gprev; + return nullptr; +} - if ( gtail == hnode ) - gtail = hnode->gprev; +HashNode* XHash::find_next_node() +{ + HashNode* n = cursor; + if ( !n ) + return nullptr; + + update_cursor(); + return n; } -void XHash::gmove_to_front(HashNode* hnode) +void* XHash::get_user_data() { - if ( hnode != ghead ) + if ( cursor ) + return cursor->data; + else + return nullptr; +} + +void XHash::update_cursor() +{ + if ( !cursor ) + return; + + cursor = cursor->next; + if ( cursor ) + return; + + for ( crow++; crow < nrows; crow++ ) { - gunlink_node(hnode); - glink_node(hnode); + cursor = table[crow]; + if ( cursor ) + return; } } -HashNode* XHash::gfind_next() +void* XHash::get_user_data(const void* key) { - HashNode* n = gnode; - if ( n ) - gnode = n->gnext; - return n; + assert(key); + + int rindex = 0; + HashNode* hnode = find_node_row(key, rindex); + return ( hnode ) ? hnode->data : nullptr; } -HashNode* XHash::gfind_first() +void XHash::release() { - if ( ghead ) - gnode = ghead->gnext; - else - gnode = nullptr; - return ghead; + HashNode* node = lru_cache->get_current_node(); + assert(node); + release_node(node); } -void* XHash::get_mru_user_data() +int XHash::release_node(HashNode* hnode) { - if ( ghead ) - return ghead->data; + assert(hnode); + + free_user_data(hnode); + unlink_node(hnode); + lru_cache->remove_node(hnode); + num_nodes--; + + if ( recycle_nodes ) + { + save_free_node(hnode); + ++stats.release_recycles; + } else - return nullptr; + { + mem_allocator->free(hnode); + ++stats.release_deletes; + } + + return HASH_OK; } -void* XHash::get_lru_user_data() +int XHash::release_node(const void* key) { - if ( gtail ) - return gtail->data; - else - return nullptr; + assert(key); + + unsigned hashkey = hashkey_ops->do_hash((const unsigned char*)key, keysize); + + unsigned index = hashkey & (nrows - 1); + for (HashNode* hnode = table[index]; hnode; hnode = hnode->next) + { + if ( hashkey_ops->key_compare(hnode->key, key, keysize) ) + return release_node(hnode); + } + + return HASH_NOT_FOUND; } void XHash::link_node(HashNode* hnode) @@ -333,327 +383,172 @@ void XHash::unlink_node(HashNode* hnode) } } -void XHash::move_to_front(HashNode* n) +void XHash::move_to_front(HashNode* node) { - if ( table[n->rindex] != n ) + if ( table[node->rindex] != node ) { - unlink_node(n); - link_node(n); + unlink_node(node); + link_node(node); } - if (n == gnode) - gnode = n->gnext; - gmove_to_front(n); + lru_cache->touch(node); } -/* - * Allocate a new hash node, uses Auto Node Recovery if needed and enabled. - * - * The oldest node is the one with the longest time since it was last touched, - * and does not have any direct indication of how long the node has been around. - * We don't monitor the actual time since last being touched, instead we use a - * splayed global list of node pointers. As nodes are accessed they are splayed - * to the front of the list. The oldest node is just the tail node. - * - */ -HashNode* XHash::allocate_node() +HashNode* XHash::find_node_row(const void* key, int& rindex) { - // use previously allocated node if there is a free one... - HashNode* hnode = get_free_node(); - if ( !hnode ) - { - if ( (max_nodes == 0) || (count < max_nodes) ) - hnode = (HashNode*)sfmemcap_alloc(&mc, - sizeof(HashNode) + keysize + datasize); + unsigned hashkey = hashkey_ops->do_hash((const unsigned char*)key, keysize); - if ( !hnode && anr_enabled && gtail ) - { - /* Find the oldest node the users willing to let go. */ - for (hnode = gtail; hnode; hnode = hnode->gprev ) - { - if ( anr_free ) - { - anr_tries++; - if ( anr_free(hnode->key, hnode->data) ) - continue; // don't recycle this one... - } - - gunlink_node(hnode); - unlink_node(hnode); - count--; - anr_count++; - break; - } - } - } - - return hnode; -} - -HashNode* XHash::find_node_row(const void* key, int* rindex) -{ - unsigned hashkey = hashfcn->hash_fcn(hashfcn, (const unsigned char*)key, keysize); - - // Modulus is slow. masking since table size is a power of 2. - int index = hashkey & (nrows - 1); - *rindex = index; - - for (HashNode* hnode = table[index]; hnode; hnode = hnode->next ) + /* Modulus is slow. Switched to a table size that is a power of 2. */ + rindex = hashkey & (nrows - 1); + for (HashNode* hnode = table[rindex]; hnode; hnode = hnode->next ) { - if ( hashfcn->keycmp_fcn(hnode->key, key, keysize) ) + if ( hashkey_ops->key_compare(hnode->key, key, keysize) ) { - if ( splay > 0 ) - move_to_front(hnode); - - find_success++; + move_to_front(hnode); return hnode; } } - find_fail++; return nullptr; } -int XHash::insert(const void* key, void* data) +void XHash::save_free_node(HashNode* hnode) { - assert(key); - - int index = 0; - - /* Enforce uniqueness: Check for the key in the table */ - HashNode* hnode = find_node_row(key, &index); - if ( hnode ) + if ( fhead ) { - cnode = hnode; - return HASH_INTABLE; + hnode->gprev = nullptr; + hnode->gnext = fhead; + fhead->gprev = hnode; + fhead = hnode; } - - hnode = allocate_node(); - if ( !hnode ) - return HASH_NOMEM; - - hnode->key = (char*)hnode + sizeof(HashNode); - memcpy(hnode->key, key, keysize); - hnode->rindex = index; - - if ( datasize ) + else { - hnode->data = (char*)hnode + sizeof(HashNode) + keysize; - if ( data ) - memcpy(hnode->data, data, datasize); + hnode->gprev = nullptr; + hnode->gnext = nullptr; + fhead = hnode; } - else - hnode->data = data; - - link_node (hnode); - glink_node(hnode); - count++; - - return HASH_OK; } -HashNode* XHash::get_node(const void* key) +HashNode* XHash::get_free_node() { - assert(key); - - int index = 0; - - // Enforce uniqueness: Check for the key in the table - HashNode* hnode = find_node_row( key, &index); - if ( hnode ) + HashNode* node = fhead; + if ( fhead ) { - cnode = hnode; - return hnode; + fhead = fhead->gnext; + if ( fhead ) + fhead->gprev = nullptr; } - hnode = allocate_node(); - if ( !hnode ) - return nullptr; - - hnode->key = (char*)hnode + sizeof(HashNode); - memcpy(hnode->key, key, keysize); - hnode->rindex = index; - - if ( datasize ) - hnode->data = (char*)hnode + sizeof(HashNode) + keysize; - else - hnode->data = nullptr; - - link_node(hnode); - glink_node(hnode); - count++; - - return hnode; -} - -HashNode* XHash::get_node_with_prune(const void* key, bool* prune_performed) -{ - assert(key); - - size_t mem_after_alloc = mc.memused + mem_allocated_per_entry; - bool over_capacity = (mc.memcap < mem_after_alloc); - - if ( over_capacity ) - *prune_performed = (delete_anr_or_lru_node() == HASH_OK); - - HashNode* hnode = nullptr; - if ( *prune_performed or !over_capacity ) - hnode = get_node(key); - - return hnode; + return node; } -HashNode* XHash::find_node(const void* key) +bool XHash::delete_free_node() { - assert(key); - - int rindex = 0; - return find_node_row(key, &rindex); + HashNode* hnode = get_free_node(); + if ( hnode ) + { + mem_allocator->free(hnode); + return true; + } + return false; } -void* XHash::get_user_data(void* key) +void XHash::purge_free_list() { - assert(key); - - int rindex = 0; - HashNode* hnode = find_node_row(key, &rindex); - if ( hnode ) - return hnode->data; + HashNode* cur = fhead; + while ( cur ) + { + HashNode* next = cur->gnext; + mem_allocator->free(cur); + cur = next; + } - return nullptr; + fhead = nullptr; } -int XHash::release_node(HashNode* hnode) +void XHash::clear_hash() { - assert(hnode); - - unlink_node(hnode); - gunlink_node(hnode); - count--; - - if ( usr_free ) - usr_free(hnode->key, hnode->data); - - if ( recycle_nodes ) - save_free_node(hnode); - else - sfmemcap_free(&mc, hnode); + for (unsigned i = 0; i < nrows; i++) + for (HashNode* node = table[i]; node;) + { + HashNode* xnode = node; + node = node->next; + release_node(xnode); + } - return HASH_OK; + max_nodes = 0; + num_nodes = 0; + crow = 0; + cursor = nullptr; } -int XHash::release_node(void* key) +void* XHash::get_mru_user_data() { - assert(key); - - unsigned hashkey = hashfcn->hash_fcn(hashfcn, (unsigned char*)key, keysize); - - unsigned index = hashkey & (nrows - 1); - for ( HashNode* hnode = table[index]; hnode; hnode = hnode->next ) - { - if ( hashfcn->keycmp_fcn(hnode->key, key, keysize) ) - return release_node(hnode); - } - - return HASH_ERR; + return lru_cache->get_mru_user_data(); } -int XHash::delete_free_node() +void* XHash::get_lru_user_data() { - HashNode* fn = get_free_node(); - if (fn) - { - sfmemcap_free(&mc, fn); - return HASH_OK; - } - return HASH_ERR; + return lru_cache->get_lru_user_data(); } -int XHash::delete_anr_or_lru_node() +HashNode* XHash::release_lru_node() { - if ( fhead ) + HashNode* hnode = lru_cache->get_lru_node(); + while ( hnode ) { - if (delete_free_node() == HASH_OK) - return HASH_OK; - } - - if ( gtail ) - { - if ( release_node(gtail) == HASH_OK ) + if ( is_node_recovery_ok(hnode) ) { - if ( fhead ) - { - if ( delete_free_node() == HASH_OK ) - return HASH_OK; - } - else if ( !recycle_nodes ) - return HASH_OK; + lru_cache->remove_node(hnode); + free_user_data(hnode); + unlink_node(hnode); + --num_nodes; + ++stats.memcap_prunes; + break; } + else + hnode = lru_cache->get_next_lru_node (); } - return HASH_ERR; + return hnode; } -int XHash::free_over_allocations(unsigned work_limit, unsigned* num_freed) +bool XHash::delete_lru_node() { - - while (mc.memcap < mc.memused and work_limit--) + if ( HashNode* hnode = lru_cache->remove_lru_node() ) { - if (delete_anr_or_lru_node() != HASH_OK) - return HASH_ERR; - - ++*num_freed; + unlink_node(hnode); // remove from the hash table + free_user_data(hnode); + mem_allocator->free(hnode); + --num_nodes; + return true; } - return (mc.memcap >= mc.memused) ? HASH_OK : HASH_PENDING; + return false; } -void XHash::update_cnode() +bool XHash::delete_a_node() { - if ( !cnode ) - return; + if ( delete_free_node() ) + return true; - cnode = cnode->next; - if ( cnode ) - return; + if ( delete_lru_node() ) + return true; - for ( crow++; crow < nrows; crow++ ) - { - cnode = table[crow]; - if ( cnode ) - return; - } + return false; } -HashNode* XHash::find_first_node() +int XHash::tune_memory_resources(unsigned work_limit, unsigned& num_freed) { - for ( crow = 0; crow < nrows; crow++ ) + while ( work_limit-- and mem_allocator->is_over_capacity() ) { - cnode = table[crow]; - if ( cnode ) - { - HashNode* n = cnode; - update_cnode(); - return n; - } - } - - return nullptr; -} - -HashNode* XHash::find_next_node() -{ - HashNode* n = cnode; - if ( !n ) - return nullptr; - - update_cnode(); + if ( !delete_a_node() ) + break; - return n; -} + ++stats.memcap_deletes; + ++num_freed; + } -void XHash::set_key_opcodes(hash_func hash_fcn, keycmp_func keycmp_fcn) -{ - hashfcn_set_keyops(hashfcn, hash_fcn, keycmp_fcn); + return ( mem_allocator->is_over_capacity() ) ? HASH_PENDING : HASH_OK; } } // namespace snort diff --git a/src/hash/xhash.h b/src/hash/xhash.h index d9ea5f7df..2d0e2f36c 100644 --- a/src/hash/xhash.h +++ b/src/hash/xhash.h @@ -25,121 +25,113 @@ // generic hash table - stores and maps key + data pairs // (supports memcap and automatic memory recovery when out of memory) -#include "utils/sfmemcap.h" +#include "framework/counts.h" #include "main/snort_types.h" +#include "utils/memcap_allocator.h" -#include "hash_defs.h" -#include "hashfcn.h" +class HashLruCache; namespace snort { +class HashKeyOperations; +class HashNode; + +struct XHashStats +{ + PegCount nodes_created = 0; + PegCount memcap_prunes = 0; + PegCount memcap_deletes = 0; + PegCount release_recycles = 0; + PegCount release_deletes = 0; +}; + class SO_PUBLIC XHash { public: - XHash(int nrows_, int keysize_, int datasize_, unsigned long memcap, - bool anr_enabled, Hash_FREE_FCN, Hash_FREE_FCN, bool recycle_nodes); - ~XHash(); - - int free_over_allocations(unsigned work_limit, unsigned* num_freed); - void clear(); + XHash(int rows, int keysize); + XHash(int rows, int keysize, int datasize, unsigned long memcap); + virtual ~XHash(); int insert(const void* key, void* data); - HashNode* get_node(const void* key); - HashNode* get_node_with_prune(const void* key, bool* prune_performed); - int release_node(void* key); - int release_node(HashNode* node); - int delete_anr_or_lru_node(); HashNode* find_node(const void* key); HashNode* find_first_node(); HashNode* find_next_node(); - void* get_user_data(void* key); + void* get_user_data(); + void* get_user_data(const void* key); + void release(); + int release_node(const void* key); + int release_node(HashNode* node); void* get_mru_user_data(); void* get_lru_user_data(); - void set_key_opcodes(hash_func, keycmp_func); + bool delete_lru_node(); + void clear_hash(); - // Set the maximum nodes used in this hash table. - // Specifying 0 is unlimited (or otherwise limited by memcap). + // set max hash nodes, 0 == no limit void set_max_nodes(int max) { max_nodes = max; } - unsigned get_node_count() - { return count; } + unsigned get_num_nodes() + { return num_nodes; } - unsigned get_anr_count() - { return anr_count; } + void set_memcap(unsigned long memcap) + { mem_allocator->set_mem_capacity(memcap); } - unsigned get_total_finds() - { return find_success + find_fail; } + unsigned long get_memcap() + { return mem_allocator->get_mem_capacity(); } - unsigned get_find_fails() - { return find_fail; } + unsigned long get_mem_used() + { return mem_allocator->get_mem_allocated(); } - unsigned get_find_successes() - { return find_success; } + const XHashStats& get_stats() const + { return stats; } - void set_memcap(unsigned long new_memcap) - { mc.memcap = new_memcap; } + virtual int tune_memory_resources(unsigned work_limit, unsigned& num_freed); - unsigned long get_memcap() - { return mc.memcap; } +protected: + void initialize(HashKeyOperations*); + void initialize(); - unsigned long get_mem_used() - { return mc.memused; } + void initialize_node (HashNode*, const void* key, void* data, int index); + HashNode* allocate_node(const void* key, void* data, int index); + HashNode* find_node_row(const void* key, int& rindex); + void link_node(HashNode*); + void unlink_node(HashNode*); + bool delete_a_node(); + void save_free_node(HashNode*); + HashNode* get_free_node(); + void delete_hash_table(); + + virtual bool is_node_recovery_ok(HashNode*) + { return true; } - const HashNode* get_cnode () const - { return cnode; } + virtual void free_user_data(HashNode*) + { } - int get_keysize () const - { return keysize; } + MemCapAllocator* mem_allocator = nullptr; + HashLruCache* lru_cache = nullptr; + unsigned nrows = 0; + unsigned keysize = 0; + unsigned num_nodes = 0; + bool recycle_nodes = true; + bool anr_enabled = true; private: + HashNode** table = nullptr; + HashKeyOperations* hashkey_ops = nullptr; + HashNode* cursor = nullptr; + HashNode* fhead = nullptr; + unsigned datasize = 0; + unsigned long mem_cap = 0; + unsigned max_nodes = 0; + unsigned crow = 0; + XHashStats stats; + + void set_number_of_rows(int nrows); + void move_to_front(HashNode*); + bool delete_free_node(); + HashNode* release_lru_node(); + void update_cursor(); void purge_free_list(); - void save_free_node(HashNode* hnode); - HashNode* get_free_node(); - void glink_node(HashNode* hnode); - void gunlink_node(HashNode* hnode); - void gmove_to_front(HashNode* hnode); - HashNode* gfind_first(); - HashNode* gfind_next(); - void link_node(HashNode* hnode); - void unlink_node(HashNode* hnode); - void move_to_front(HashNode* n); - HashNode* allocate_node(); - HashNode* find_node_row(const void* key, int* rindex); - void update_cnode(); - int delete_free_node(); - - HashFnc* hashfcn = nullptr; // hash function - int keysize = 0; // bytes in key, if <= 0 -> keys are strings - FIXIT-H does negative keysize work? - int datasize = 0; // bytes in key, if == 0 -> user data - unsigned mem_allocated_per_entry = 0; - HashNode** table = nullptr; // array of node ptr's */ - unsigned nrows = 0; // # rows int the hash table use a prime number 211, 9871 - unsigned count = 0; // total # nodes in table - unsigned crow = 0; // findfirst/next row in table - HashNode* cnode = nullptr; // find_[first|next] node ptr - int splay = 1; // whether to splay nodes with same hash bucket - unsigned max_nodes = 0; // maximum # of nodes within a hash - MEMCAP mc; - unsigned find_fail = 0; - unsigned find_success = 0; - - HashNode* ghead = nullptr; // global - root of all nodes allocated in table - HashNode* gtail = nullptr; - HashNode* gnode = nullptr; // gfirst/gnext node ptr */ - HashNode* fhead = nullptr; // list of free nodes, which are recycled - HashNode* ftail = nullptr; - bool recycle_nodes = false; // recycle nodes... - - // Automatic Node Recover (ANR): When number of nodes in hash is equal - // to max_nodes, remove the least recently used nodes and use it for - // the new node. anr_tries indicates # of ANR tries.*/ - unsigned anr_tries = 0; - unsigned anr_count = 0; // # ANR ops performed - bool anr_enabled = false; // false = anr disable, true = anr enabled - - Hash_FREE_FCN anr_free = nullptr; - Hash_FREE_FCN usr_free = nullptr; }; } // namespace snort diff --git a/src/hash/zhash.cc b/src/hash/zhash.cc index f2b87d3ac..d806d7a3b 100644 --- a/src/hash/zhash.cc +++ b/src/hash/zhash.cc @@ -28,233 +28,65 @@ #include #include +#include "flow/flow_key.h" + #include "hash_defs.h" +#include "hash_key_operations.h" +#include "hash_lru_cache.h" using namespace snort; //------------------------------------------------------------------------- -// private stuff +// public stuff //------------------------------------------------------------------------- -static inline HashNode* s_node_alloc(int keysize) -{ - auto node = static_cast( - ::operator new(sizeof(HashNode) + keysize)); - - *node = {}; - return node; -} - -static inline void s_node_free(HashNode* node) -{ ::operator delete(node); } - -void ZHash::delete_free_list() -{ - if ( !fhead ) - return; - - HashNode* cur = fhead; - - while ( cur ) - { - fhead = cur->gnext; - s_node_free(cur); - cur = fhead; - } -} - -void ZHash::save_free_node(HashNode* node) -{ - if ( fhead ) - { - node->gprev = nullptr; - node->gnext = fhead; - fhead->gprev = node; - fhead = node; - } - else - { - node->gprev = nullptr; - node->gnext = nullptr; - fhead = node; - } -} - -HashNode* ZHash::get_free_node() -{ - HashNode* node = fhead; - - if ( fhead ) - { - fhead = fhead->gnext; - - if ( fhead ) - fhead->gprev = nullptr; - } - - return node; -} - -void ZHash::glink_node(HashNode* node) -{ - if ( ghead ) - { - node->gprev = nullptr; - node->gnext = ghead; - ghead->gprev = node; - ghead = node; - } - else - { - node->gprev = nullptr; - node->gnext = nullptr; - ghead = node; - gtail = node; - } -} - -void ZHash::gunlink_node(HashNode* node) -{ - if ( cursor == node ) - cursor = node->gprev; - - if ( ghead == node ) - { - ghead = ghead->gnext; - if ( ghead ) - ghead->gprev = nullptr; - } - - if ( node->gprev ) - node->gprev->gnext = node->gnext; - - if ( node->gnext ) - node->gnext->gprev = node->gprev; - - if ( gtail == node ) - gtail = node->gprev; -} - -void ZHash::link_node(HashNode* node) -{ - if ( table[node->rindex] ) // UNINITUSE - { - node->prev = nullptr; - node->next = table[node->rindex]; - table[node->rindex]->prev = node; - table[node->rindex] = node; - } - else - { - node->prev = nullptr; - node->next = nullptr; - table[node->rindex] = node; // UNINITUSE - } -} -void ZHash::unlink_node(HashNode* node) +ZHash::ZHash(int rows, int key_len) + : XHash(rows, key_len) { - if ( node->prev ) - { - node->prev->next = node->next; - if ( node->next ) - node->next->prev = node->prev; - } - else if ( table[node->rindex] ) - { - table[node->rindex] = table[node->rindex]->next; - - if ( table[node->rindex] ) - table[node->rindex]->prev = nullptr; - } + initialize(new FlowHashKeyOps(nrows)); + anr_enabled = false; } -void ZHash::move_to_front(HashNode* node) +void* ZHash::get(const void* key) { - // move to front of row list - if ( table[node->rindex] != node ) - { - unlink_node(node); - link_node(node); - } - - // move to front of global list - if ( node != ghead ) - { - gunlink_node(node); - glink_node(node); - } -} + assert(key); -HashNode* ZHash::find_node_row(const void* key, int& row) -{ - unsigned hashkey = hashfcn->hash_fcn( - hashfcn, (const unsigned char*)key, keysize); - - // Modulus is slow; use a table size that is a power of 2. - int index = hashkey & (nrows - 1); - row = index; - - for ( HashNode* node = table[index]; node; node = node->next ) // UNINITUSE - { - if ( hashfcn->keycmp_fcn(node->key, key, keysize) ) - { - move_to_front(node); - find_success++; - return node; - } - } - - find_fail++; - return nullptr; -} - -//------------------------------------------------------------------------- -// public stuff -//------------------------------------------------------------------------- - -ZHash::ZHash(int rows, int keysz) - : keysize(keysz) -{ - // adjust rows to be power of 2 - if ( rows > 0 ) - nrows = hash_nearest_power_of_2(rows); - else - nrows = -rows; // if negative use as is + int index; + HashNode* node = find_node_row(key, index); + if ( node ) + return node->data; - table = new HashNode*[nrows](); - hashfcn = hashfcn_new(nrows); + node = get_free_node(); + if ( !node ) + return nullptr; - fhead = cursor = nullptr; - ghead = gtail = nullptr; - count = find_success = find_fail = 0; + memcpy(node->key, key, keysize); + node->rindex = index; + link_node(node); + lru_cache->insert(node); + num_nodes++; + return node->data; } -ZHash::~ZHash() +void* ZHash::remove() { - hashfcn_free(hashfcn); - - for ( unsigned i = 0; i < nrows; ++i ) - { - for ( HashNode* node = table[i]; node; ) - { - HashNode* onode = node; - node = node->next; - s_node_free(onode); - } - } + HashNode* node = lru_cache->get_current_node(); + assert(node); + void* pv = node->data; - delete[] table; - delete_free_list(); + unlink_node(node); + lru_cache->remove_node(node); + num_nodes--; + mem_allocator->free(node); + return pv; } void* ZHash::push(void* p) { - auto node = s_node_alloc(keysize); - + auto node = (HashNode*)mem_allocator->allocate(); node->key = (char*)node + sizeof(HashNode); node->data = p; - save_free_node(node); return node->key; } @@ -262,137 +94,36 @@ void* ZHash::push(void* p) void* ZHash::pop() { HashNode* node = get_free_node(); - if ( !node ) return nullptr; void* pv = node->data; - s_node_free(node); + mem_allocator->free(node); return pv; } -void* ZHash::get(const void* key, bool *new_node) +void* ZHash::lru_first() { - int row; - HashNode* node = find_node_row(key, row); - - if ( node ) - return node->data; - - node = get_free_node(); - - if ( !node ) - return nullptr; - - memcpy(node->key, key, keysize); - - node->rindex = row; - link_node (node); - glink_node(node); - - count++; - - if (new_node) - *new_node = true; - - return node->data; -} - -void* ZHash::find(const void* key) -{ - int row; - HashNode* node = find_node_row(key, row); - - if ( node ) - return node->data; - - return nullptr; + HashNode* node = lru_cache->get_lru_node(); + return node ? node->data : nullptr; } -void* ZHash::first() +void* ZHash::lru_next() { - cursor = gtail; - return cursor ? cursor->data : nullptr; + HashNode* node = lru_cache->get_next_lru_node(); + return node ? node->data : nullptr; } -void* ZHash::next() +void* ZHash::lru_current() { - if ( !cursor ) - return nullptr; - - cursor = cursor->gprev; - return cursor ? cursor->data : nullptr; -} - -void* ZHash::current() -{ - return cursor ? cursor->data : nullptr; -} - -bool ZHash::touch() -{ - HashNode* node = cursor; - - if ( !node ) - return false; - - cursor = cursor->gprev; - - if ( node != ghead ) - { - gunlink_node(node); - glink_node(node); - return true; - } - return false; -} - -bool ZHash::move_to_free_list(HashNode* node) -{ - if ( !node ) - return false; - - unlink_node(node); - gunlink_node(node); - count--; - save_free_node(node); - - return true; -} - -bool ZHash::release() -{ - HashNode* node = cursor; - cursor = nullptr; - return move_to_free_list(node); -} - -bool ZHash::release(const void* key) -{ - int row; - HashNode* node = find_node_row(key, row); - return move_to_free_list(node); -} - -void* ZHash::remove(const void* key) -{ - void* pv = nullptr; - int row; - HashNode* node = find_node_row(key, row); - if ( node ) - { - unlink_node(node); - gunlink_node(node); - count--; - pv = node->data; - s_node_free(node); - } - - return pv; + HashNode* node = lru_cache->get_current_node(); + return node ? node->data : nullptr; } -void ZHash::set_key_opcodes(hash_func hash_fcn, keycmp_func keycmp_fcn) +void ZHash::lru_touch() { - hashfcn_set_keyops(hashfcn, hash_fcn, keycmp_fcn); + HashNode* node = lru_cache->get_current_node(); + assert(node); + lru_cache->touch(node); } diff --git a/src/hash/zhash.h b/src/hash/zhash.h index 27584532b..f7c2370bc 100644 --- a/src/hash/zhash.h +++ b/src/hash/zhash.h @@ -22,18 +22,12 @@ #include -#include "hashfcn.h" +#include "hash/xhash.h" -namespace snort -{ -struct HashNode; -} - -class ZHash +class ZHash : public snort::XHash { public: ZHash(int nrows, int keysize); - ~ZHash(); ZHash(const ZHash&) = delete; ZHash& operator=(const ZHash&) = delete; @@ -41,51 +35,13 @@ public: void* push(void* p); void* pop(); - void* first(); - void* next(); - void* current(); - bool touch(); - - void* find(const void* key); - void* get(const void* key, bool *new_node = nullptr); - bool release(const void* key); - bool release(); - void* remove(const void* key); - void set_key_opcodes(hash_func, keycmp_func); - - inline unsigned get_count() - { return count; } - -private: - snort::HashNode* get_free_node(); - snort::HashNode* find_node_row(const void*, int&); - - void glink_node(snort::HashNode*); - void gunlink_node(snort::HashNode*); - - void link_node(snort::HashNode*); - void unlink_node(snort::HashNode*); - - void delete_free_list(); - void save_free_node(snort::HashNode*); - - bool move_to_free_list(snort::HashNode*); - void move_to_front(snort::HashNode*); - -private: - HashFnc* hashfcn; - int keysize; - unsigned nrows; - unsigned count; - - unsigned find_fail; - unsigned find_success; + void* get(const void* key); + void* remove(); - snort::HashNode** table; - snort::HashNode* ghead; - snort::HashNode* gtail; - snort::HashNode* fhead; - snort::HashNode* cursor; + void* lru_first(); + void* lru_next(); + void* lru_current(); + void lru_touch(); }; #endif diff --git a/src/ips_options/ips_ack.cc b/src/ips_options/ips_ack.cc index 3aa0c676e..14e382481 100644 --- a/src/ips_options/ips_ack.cc +++ b/src/ips_options/ips_ack.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler_defs.h" #include "protocols/packet.h" #include "protocols/tcp.h" diff --git a/src/ips_options/ips_asn1.cc b/src/ips_options/ips_asn1.cc index c10ef9f96..cb067e213 100644 --- a/src/ips_options/ips_asn1.cc +++ b/src/ips_options/ips_asn1.cc @@ -56,7 +56,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_base64.cc b/src/ips_options/ips_base64.cc index d1155bcd7..ad9b8e74b 100644 --- a/src/ips_options/ips_base64.cc +++ b/src/ips_options/ips_base64.cc @@ -25,7 +25,7 @@ #include "detection/detection_util.h" #include "detection/treenodes.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" diff --git a/src/ips_options/ips_ber_data.cc b/src/ips_options/ips_ber_data.cc index 2a24f7808..06b521bee 100644 --- a/src/ips_options/ips_ber_data.cc +++ b/src/ips_options/ips_ber_data.cc @@ -24,7 +24,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "utils/util_ber.h" diff --git a/src/ips_options/ips_ber_skip.cc b/src/ips_options/ips_ber_skip.cc index 9cdae7563..4d4765cde 100644 --- a/src/ips_options/ips_ber_skip.cc +++ b/src/ips_options/ips_ber_skip.cc @@ -24,7 +24,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "utils/util_ber.h" diff --git a/src/ips_options/ips_bufferlen.cc b/src/ips_options/ips_bufferlen.cc index a347f7ed3..0aa305372 100644 --- a/src/ips_options/ips_bufferlen.cc +++ b/src/ips_options/ips_bufferlen.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" using namespace snort; diff --git a/src/ips_options/ips_byte_extract.cc b/src/ips_options/ips_byte_extract.cc index c55bbb3c4..9213600c8 100644 --- a/src/ips_options/ips_byte_extract.cc +++ b/src/ips_options/ips_byte_extract.cc @@ -27,7 +27,7 @@ #include "framework/endianness.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/ips_options/ips_byte_jump.cc b/src/ips_options/ips_byte_jump.cc index 65d58a1f2..ea672a92c 100644 --- a/src/ips_options/ips_byte_jump.cc +++ b/src/ips_options/ips_byte_jump.cc @@ -81,7 +81,7 @@ #include "framework/endianness.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_byte_math.cc b/src/ips_options/ips_byte_math.cc index 434495141..00ce2dd1e 100644 --- a/src/ips_options/ips_byte_math.cc +++ b/src/ips_options/ips_byte_math.cc @@ -27,7 +27,7 @@ #include "framework/endianness.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_byte_test.cc b/src/ips_options/ips_byte_test.cc index 5ba9bd3db..2ad0b406e 100644 --- a/src/ips_options/ips_byte_test.cc +++ b/src/ips_options/ips_byte_test.cc @@ -97,7 +97,7 @@ #include "framework/endianness.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_content.cc b/src/ips_options/ips_content.cc index 3a39ecfc3..0830bbdec 100644 --- a/src/ips_options/ips_content.cc +++ b/src/ips_options/ips_content.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "helpers/literal_search.h" #include "log/messages.h" #include "parser/parse_utils.h" diff --git a/src/ips_options/ips_cvs.cc b/src/ips_options/ips_cvs.cc index d09b2adce..b17c8f893 100644 --- a/src/ips_options/ips_cvs.cc +++ b/src/ips_options/ips_cvs.cc @@ -41,7 +41,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_dsize.cc b/src/ips_options/ips_dsize.cc index 52f0f7e7f..14621adf8 100644 --- a/src/ips_options/ips_dsize.cc +++ b/src/ips_options/ips_dsize.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_file_type.cc b/src/ips_options/ips_file_type.cc index 61f762aac..7797fcd5f 100644 --- a/src/ips_options/ips_file_type.cc +++ b/src/ips_options/ips_file_type.cc @@ -29,7 +29,7 @@ #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" using namespace snort; diff --git a/src/ips_options/ips_flags.cc b/src/ips_options/ips_flags.cc index bb83a5cbd..b0453a830 100644 --- a/src/ips_options/ips_flags.cc +++ b/src/ips_options/ips_flags.cc @@ -26,7 +26,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_flow.cc b/src/ips_options/ips_flow.cc index ac71613c2..789c44bbd 100644 --- a/src/ips_options/ips_flow.cc +++ b/src/ips_options/ips_flow.cc @@ -27,7 +27,7 @@ #include "detection/treenodes.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_flowbits.cc b/src/ips_options/ips_flowbits.cc index 4205b3264..5b6336faf 100644 --- a/src/ips_options/ips_flowbits.cc +++ b/src/ips_options/ips_flowbits.cc @@ -46,7 +46,8 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "hash/ghash.h" -#include "hash/hashfcn.h" +#include "hash/hash_defs.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "parser/mstring.h" #include "protocols/packet.h" @@ -618,7 +619,7 @@ static FLOWBITS_OBJECT* getFlowBitItem(char* flowbitName, FLOWBITS_OP* flowbits, } int hstatus = flowbit_state->flowbits_hash->insert(flowbitName, flowbits_item); - if (hstatus != GHASH_OK) + if (hstatus != HASH_OK) ParseError("Could not add flowbits key (%s) to hash.",flowbitName); } flowbits_item->toggle = flowbit_state->flowbits_toggle; @@ -811,7 +812,7 @@ static FLOWBITS_GRP* getFlowBitGroup(char* groupName, FlowBitState* flowbit_stat // new group defined, add (bitop set later once we know size) flowbits_grp = (FLOWBITS_GRP*)snort_calloc(sizeof(*flowbits_grp)); int hstatus = flowbit_state->flowbits_grp_hash->insert(groupName, flowbits_grp); - if (hstatus != GHASH_OK) + if (hstatus != HASH_OK) ParseAbort("Could not add flowbits group (%s) to hash.\n",groupName); flowbit_state->flowbits_grp_count++; diff --git a/src/ips_options/ips_fragbits.cc b/src/ips_options/ips_fragbits.cc index 10ae82cee..48a147d66 100644 --- a/src/ips_options/ips_fragbits.cc +++ b/src/ips_options/ips_fragbits.cc @@ -52,7 +52,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_fragoffset.cc b/src/ips_options/ips_fragoffset.cc index 614237daa..e9b6f3e76 100644 --- a/src/ips_options/ips_fragoffset.cc +++ b/src/ips_options/ips_fragoffset.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_hash.cc b/src/ips_options/ips_hash.cc index db237d187..b3b6f741a 100644 --- a/src/ips_options/ips_hash.cc +++ b/src/ips_options/ips_hash.cc @@ -27,7 +27,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "hash/hashes.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "parser/parse_utils.h" #include "profiler/profiler.h" diff --git a/src/ips_options/ips_icmp_id.cc b/src/ips_options/ips_icmp_id.cc index e7296e4ad..a17ad7246 100644 --- a/src/ips_options/ips_icmp_id.cc +++ b/src/ips_options/ips_icmp_id.cc @@ -46,7 +46,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/icmp4.h" #include "protocols/icmp6.h" diff --git a/src/ips_options/ips_icmp_seq.cc b/src/ips_options/ips_icmp_seq.cc index 4f56193e9..d16d372f8 100644 --- a/src/ips_options/ips_icmp_seq.cc +++ b/src/ips_options/ips_icmp_seq.cc @@ -46,7 +46,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/icmp4.h" #include "protocols/icmp6.h" diff --git a/src/ips_options/ips_icode.cc b/src/ips_options/ips_icode.cc index e1b00daf0..ac2f3b471 100644 --- a/src/ips_options/ips_icode.cc +++ b/src/ips_options/ips_icode.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/icmp4.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_id.cc b/src/ips_options/ips_id.cc index 32a72e292..09fcb78e5 100644 --- a/src/ips_options/ips_id.cc +++ b/src/ips_options/ips_id.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_ip_proto.cc b/src/ips_options/ips_ip_proto.cc index d37f8eced..c263d595e 100644 --- a/src/ips_options/ips_ip_proto.cc +++ b/src/ips_options/ips_ip_proto.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_ipopts.cc b/src/ips_options/ips_ipopts.cc index 4c80b430d..48c0480d3 100644 --- a/src/ips_options/ips_ipopts.cc +++ b/src/ips_options/ips_ipopts.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/ipv4_options.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_isdataat.cc b/src/ips_options/ips_isdataat.cc index 0de1051a5..f39124891 100644 --- a/src/ips_options/ips_isdataat.cc +++ b/src/ips_options/ips_isdataat.cc @@ -42,7 +42,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "parser/mstring.h" #include "profiler/profiler.h" diff --git a/src/ips_options/ips_itype.cc b/src/ips_options/ips_itype.cc index 327b3cece..eb65fa551 100644 --- a/src/ips_options/ips_itype.cc +++ b/src/ips_options/ips_itype.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/icmp4.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_luajit.cc b/src/ips_options/ips_luajit.cc index 5018984db..07f66e870 100644 --- a/src/ips_options/ips_luajit.cc +++ b/src/ips_options/ips_luajit.cc @@ -24,7 +24,7 @@ #include "framework/cursor.h" #include "framework/decode_data.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "helpers/chunk.h" #include "lua/lua.h" #include "log/messages.h" diff --git a/src/ips_options/ips_pcre.cc b/src/ips_options/ips_pcre.cc index 5da5a8e3d..56f498996 100644 --- a/src/ips_options/ips_pcre.cc +++ b/src/ips_options/ips_pcre.cc @@ -31,7 +31,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/parameter.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "helpers/scratch_allocator.h" #include "log/messages.h" #include "main/snort_config.h" diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc index 74be74f70..6159e5e23 100644 --- a/src/ips_options/ips_regex.cc +++ b/src/ips_options/ips_regex.cc @@ -32,7 +32,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "helpers/hyper_scratch_allocator.h" #include "log/messages.h" #include "main/snort_config.h" diff --git a/src/ips_options/ips_replace.cc b/src/ips_options/ips_replace.cc index 8c10b64d9..50071f2ee 100644 --- a/src/ips_options/ips_replace.cc +++ b/src/ips_options/ips_replace.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "main/snort_config.h" #include "main/thread_config.h" diff --git a/src/ips_options/ips_rpc.cc b/src/ips_options/ips_rpc.cc index 7fe436a2e..7c3f27eb9 100644 --- a/src/ips_options/ips_rpc.cc +++ b/src/ips_options/ips_rpc.cc @@ -26,7 +26,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_sd_pattern.cc b/src/ips_options/ips_sd_pattern.cc index 83f68b705..3a09e2ef9 100644 --- a/src/ips_options/ips_sd_pattern.cc +++ b/src/ips_options/ips_sd_pattern.cc @@ -31,7 +31,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "helpers/hyper_scratch_allocator.h" #include "log/messages.h" #include "log/obfuscator.h" diff --git a/src/ips_options/ips_seq.cc b/src/ips_options/ips_seq.cc index d3d145895..c70e8dbfa 100644 --- a/src/ips_options/ips_seq.cc +++ b/src/ips_options/ips_seq.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "protocols/tcp.h" diff --git a/src/ips_options/ips_session.cc b/src/ips_options/ips_session.cc index c86362ab4..598edbe46 100644 --- a/src/ips_options/ips_session.cc +++ b/src/ips_options/ips_session.cc @@ -49,7 +49,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "main/snort_config.h" #include "profiler/profiler.h" diff --git a/src/ips_options/ips_so.cc b/src/ips_options/ips_so.cc index e32932b99..ce7b452a3 100644 --- a/src/ips_options/ips_so.cc +++ b/src/ips_options/ips_so.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/so_rule.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "managers/so_manager.h" #include "profiler/profiler.h" diff --git a/src/ips_options/ips_tos.cc b/src/ips_options/ips_tos.cc index ddf32df9c..075eaa813 100644 --- a/src/ips_options/ips_tos.cc +++ b/src/ips_options/ips_tos.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_ttl.cc b/src/ips_options/ips_ttl.cc index 93d7acb0e..c190aaa6e 100644 --- a/src/ips_options/ips_ttl.cc +++ b/src/ips_options/ips_ttl.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_window.cc b/src/ips_options/ips_window.cc index e0e140848..8b1711377 100644 --- a/src/ips_options/ips_window.cc +++ b/src/ips_options/ips_window.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "protocols/tcp.h" diff --git a/src/main/snort_config.h b/src/main/snort_config.h index d379f902a..9edad61b9 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -147,8 +147,8 @@ struct VarNode; namespace snort { -class ProtocolReference; class GHash; +class ProtocolReference; class XHash; struct ProfilerConfig; struct SnortConfig; diff --git a/src/mime/file_mime_process.cc b/src/mime/file_mime_process.cc index 7c962b619..4cfcc4603 100644 --- a/src/mime/file_mime_process.cc +++ b/src/mime/file_mime_process.cc @@ -28,7 +28,7 @@ #include "detection/detection_engine.h" #include "file_api/file_flows.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "search_engines/search_tool.h" #include "utils/util_cstring.h" diff --git a/src/network_inspectors/appid/ips_appid_option.cc b/src/network_inspectors/appid/ips_appid_option.cc index ad8f530e3..77a53b064 100644 --- a/src/network_inspectors/appid/ips_appid_option.cc +++ b/src/network_inspectors/appid/ips_appid_option.cc @@ -27,7 +27,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "utils/util.h" diff --git a/src/network_inspectors/perf_monitor/flow_ip_tracker.cc b/src/network_inspectors/perf_monitor/flow_ip_tracker.cc index 85c5eb02c..9a4d0c5ac 100644 --- a/src/network_inspectors/perf_monitor/flow_ip_tracker.cc +++ b/src/network_inspectors/perf_monitor/flow_ip_tracker.cc @@ -24,6 +24,7 @@ #include "flow_ip_tracker.h" +#include "hash/hash_defs.h" #include "log/messages.h" #include "protocols/packet.h" @@ -46,9 +47,8 @@ FlowStateValue* FlowIPTracker::find_stats(const SfIp* src_addr, const SfIp* dst_ { FlowStateKey key; FlowStateValue* value = nullptr; - bool prune_required = false; - if (src_addr->less_than(*dst_addr)) + if ( src_addr->less_than(*dst_addr) ) { key.ipA = *src_addr; key.ipB = *dst_addr; @@ -62,21 +62,12 @@ FlowStateValue* FlowIPTracker::find_stats(const SfIp* src_addr, const SfIp* dst_ } value = (FlowStateValue*)ip_map->get_user_data(&key); - if (!value) + if ( !value ) { - HashNode* node = ip_map->get_node_with_prune(&key, &prune_required); - - if (!node) + if ( ip_map->insert(&key, nullptr) != HASH_OK ) return nullptr; - - if (prune_required) - { - ++pmstats.total_frees; - ++pmstats.alloc_prunes; - } - - memset(node->data, 0, sizeof(FlowStateValue)); - value = (FlowStateValue*)node->data; + value = (FlowStateValue*)ip_map->get_user_data(); + memset(value, 0, sizeof(FlowStateValue)); } return value; @@ -88,8 +79,8 @@ bool FlowIPTracker::initialize(size_t new_memcap) if ( !ip_map ) { - ip_map = new XHash(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), sizeof(FlowStateValue), - new_memcap, true, nullptr, nullptr, true); + ip_map = new XHash(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), + sizeof(FlowStateValue), new_memcap); } else { @@ -140,23 +131,25 @@ FlowIPTracker::FlowIPTracker(PerfConfig* perf) : PerfTracker(perf, TRACKER_NAME) formatter->finalize_fields(); memcap = perf->flowip_memcap; - ip_map = new XHash(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), sizeof(FlowStateValue), - memcap, true, nullptr, nullptr, true); + ip_map = new XHash(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), sizeof(FlowStateValue), memcap); } FlowIPTracker::~FlowIPTracker() { + const XHashStats& stats = ip_map->get_stats(); + pmstats.flow_tracker_creates = stats.nodes_created; + pmstats.flow_tracker_total_deletes = stats.memcap_deletes; + pmstats.flow_tracker_prunes = stats.memcap_prunes; + delete ip_map; } void FlowIPTracker::reset() -{ - ip_map->clear(); -} +{ ip_map->clear_hash(); } void FlowIPTracker::update(Packet* p) { - if (p->has_ip() && !p->is_rebuilt()) + if ( p->has_ip() && !p->is_rebuilt() ) { FlowType type = SFS_TYPE_OTHER; int swapped; @@ -171,12 +164,12 @@ void FlowIPTracker::update(Packet* p) type = SFS_TYPE_UDP; FlowStateValue* value = find_stats(src_addr, dst_addr, &swapped); - if (!value) + if ( !value ) return; TrafficStats* stats = &value->traffic_stats[type]; - if (!swapped) + if ( !swapped ) { stats->packets_a_to_b++; stats->bytes_a_to_b += len; @@ -214,11 +207,10 @@ int FlowIPTracker::update_state(const SfIp* src_addr, const SfIp* dst_addr, Flow int swapped; FlowStateValue* value = find_stats(src_addr, dst_addr, &swapped); - if (!value) + if ( !value ) return 1; value->state_changes[state]++; - return 0; } diff --git a/src/network_inspectors/perf_monitor/perf_monitor.cc b/src/network_inspectors/perf_monitor/perf_monitor.cc index 9df381e3b..8c038418f 100644 --- a/src/network_inspectors/perf_monitor/perf_monitor.cc +++ b/src/network_inspectors/perf_monitor/perf_monitor.cc @@ -29,6 +29,7 @@ #endif #include "framework/data_bus.h" +#include "hash/hash_defs.h" #include "hash/xhash.h" #include "log/messages.h" #include "managers/inspector_manager.h" @@ -270,9 +271,8 @@ bool PerfMonReloadTuner::tune_resources(unsigned work_limit) if (flow_ip_tracker) { unsigned num_freed = 0; - int result = flow_ip_tracker->get_ip_map()->free_over_allocations(work_limit, &num_freed); - pmstats.total_frees += num_freed; - pmstats.reload_frees += num_freed; + int result = flow_ip_tracker->get_ip_map()->tune_memory_resources(work_limit, num_freed); + pmstats.flow_tracker_reload_deletes += num_freed; return (result == HASH_OK); } else diff --git a/src/network_inspectors/perf_monitor/perf_pegs.h b/src/network_inspectors/perf_monitor/perf_pegs.h index 26628b198..8d503970d 100644 --- a/src/network_inspectors/perf_monitor/perf_pegs.h +++ b/src/network_inspectors/perf_monitor/perf_pegs.h @@ -27,20 +27,21 @@ static const PegInfo perf_module_pegs[] = { - { CountType::SUM, "packets", "total packets processed by performance monitor" }, - { CountType::SUM, "total_frees", "total flows pruned or freed by performance monitor" }, - { CountType::SUM, "reload_frees", "flows freed on reload with changed memcap" }, - { CountType::SUM, "alloc_prunes", "flows pruned on allocation of IP flows" }, + { CountType::SUM, "flow_tracker_creates", "total number of flow trackers created" }, + { CountType::SUM, "flow_tracker_total_deletes", "flow trackers deleted to stay below memcap limit" }, + { CountType::SUM, "flow_tracker_reload_deletes", "flow trackers deleted due to memcap change on config reload" }, + { CountType::SUM, "flow_tracker_prunes", "flow trackers pruned for reuse by new flows" }, { CountType::END, nullptr, nullptr }, }; struct PerfPegStats { PegCount total_packets; - PegCount total_frees; - PegCount reload_frees; - PegCount alloc_prunes; + PegCount flow_tracker_creates; + PegCount flow_tracker_total_deletes; + PegCount flow_tracker_reload_deletes; + PegCount flow_tracker_prunes; }; #endif diff --git a/src/network_inspectors/port_scan/ps_detect.cc b/src/network_inspectors/port_scan/ps_detect.cc index 4ee4800b3..693bb7a9f 100644 --- a/src/network_inspectors/port_scan/ps_detect.cc +++ b/src/network_inspectors/port_scan/ps_detect.cc @@ -33,6 +33,7 @@ #include "ps_detect.h" +#include "hash/hash_defs.h" #include "hash/xhash.h" #include "log/messages.h" #include "protocols/icmp4.h" @@ -57,7 +58,32 @@ struct PS_HASH_KEY }; PADDING_GUARD_END -static THREAD_LOCAL XHash* portscan_hash = nullptr; +class PortScanCache : public XHash +{ +public: + PortScanCache(unsigned rows, unsigned key_len, unsigned datasize, unsigned memcap) + : XHash(rows, key_len, datasize, memcap) + { } + + bool is_node_recovery_ok(HashNode* hnode) override + { + PS_TRACKER* tracker = (PS_TRACKER*)hnode->data; + + if ( !tracker->priority_node ) + return true; + + /* + ** Cycle through the protos to see if it's past the time. + ** We only get here if we ARE a priority node. + */ + if ( tracker->proto.window >= packet_time() ) + return false; + + return true; + } +}; + +static THREAD_LOCAL PortScanCache* portscan_hash = nullptr; extern THREAD_LOCAL PsPegStats spstats; PS_PKT::PS_PKT(Packet* p) @@ -84,32 +110,6 @@ PortscanConfig::~PortscanConfig() ipset_free(watch_ip); } -/* -** This function is passed into the hash algorithm, so that -** we only reuse nodes that aren't priority nodes. We have to make -** sure that we only track so many priority nodes, otherwise we could -** have all priority nodes and not be able to allocate more. -*/ -static int ps_tracker_free(void* key, void* data) -{ - if (!key || !data) - return 0; - - PS_TRACKER* tracker = (PS_TRACKER*)data; - - if (!tracker->priority_node) - return 0; - - /* - ** Cycle through the protos to see if it's past the time. - ** We only get here if we ARE a priority node. - */ - if (tracker->proto.window >= packet_time()) - return 1; - - return 0; -} - void ps_cleanup() { if ( portscan_hash ) @@ -132,8 +132,8 @@ bool ps_init_hash(unsigned long memcap) } int rows = memcap / ps_node_size(); - portscan_hash = new XHash(rows, sizeof(PS_HASH_KEY), sizeof(PS_TRACKER), - memcap, true, ps_tracker_free, nullptr, true); + portscan_hash = new PortScanCache(rows, sizeof(PS_HASH_KEY), sizeof(PS_TRACKER), + memcap); return false; } @@ -144,7 +144,7 @@ bool ps_prune_hash(unsigned work_limit) return true; unsigned num_pruned = 0; - int result = portscan_hash->free_over_allocations(work_limit, &num_pruned); + int result = portscan_hash->tune_memory_resources(work_limit, num_pruned); spstats.reload_prunes += num_pruned; return result != HASH_PENDING; } @@ -152,7 +152,7 @@ bool ps_prune_hash(unsigned work_limit) void ps_reset() { if ( portscan_hash ) - portscan_hash->clear(); + portscan_hash->clear_hash(); } // Check scanner and scanned ips to see if we can filter them out. @@ -297,12 +297,12 @@ static PS_TRACKER* ps_tracker_get(PS_HASH_KEY* key) if ( ht ) return ht; - auto prev_count = portscan_hash->get_node_count(); + auto prev_count = portscan_hash->get_num_nodes(); if ( portscan_hash->insert((void*)key, nullptr) != HASH_OK ) return nullptr; ++spstats.trackers; - if ( prev_count == portscan_hash->get_node_count() ) + if ( prev_count == portscan_hash->get_num_nodes() ) ++spstats.alloc_prunes; ht = (PS_TRACKER*)portscan_hash->get_mru_user_data(); diff --git a/src/parser/parser.cc b/src/parser/parser.cc index 8dd5ef27e..05a7b299c 100644 --- a/src/parser/parser.cc +++ b/src/parser/parser.cc @@ -34,7 +34,8 @@ #include "filters/detection_filter.h" #include "filters/rate_filter.h" #include "filters/sfthreshold.h" -#include "hash/hashfcn.h" +#include "hash/ghash.h" +#include "hash/hash_key_operations.h" #include "hash/xhash.h" #include "helpers/directory.h" #include "log/messages.h" @@ -64,6 +65,62 @@ static struct rule_index_map_t* ruleIndexMap = nullptr; static std::string s_aux_rules; +class RuleTreeHashKeyOps : public HashKeyOperations +{ +public: + RuleTreeHashKeyOps(int rows) + : HashKeyOperations(rows) + { } + + unsigned do_hash(const unsigned char* k, int) override + { + uint32_t a,b,c; + const RuleTreeNodeKey* rtnk = (const RuleTreeNodeKey*)k; + RuleTreeNode* rtn = rtnk->rtn; + + a = rtn->action; + b = rtn->flags; + c = (uint32_t)(uintptr_t)rtn->listhead; + + mix(a,b,c); + + a += (uint32_t)(uintptr_t)rtn->src_portobject; + b += (uint32_t)(uintptr_t)rtn->dst_portobject; + c += (uint32_t)(uintptr_t)rtnk->policyId; + + finalize(a,b,c); + + return c; + } + + bool key_compare(const void* k1, const void* k2, size_t) override + { + assert(k1 && k2); + + const RuleTreeNodeKey* rtnk1 = (const RuleTreeNodeKey*)k1; + const RuleTreeNodeKey* rtnk2 = (const RuleTreeNodeKey*)k2; + + if (rtnk1->policyId != rtnk2->policyId) + return false; + + if (same_headers(rtnk1->rtn, rtnk2->rtn)) + return true; + + return false; + } +}; + +class RuleTreeCache : public XHash +{ +public: + RuleTreeCache(int rows, int key_len) + : XHash(rows, key_len) + { + initialize(new RuleTreeHashKeyOps(nrows)); + anr_enabled = false; + } +}; + //------------------------------------------------------------------------- // private / implementation methods //------------------------------------------------------------------------- @@ -565,44 +622,6 @@ RuleTreeNode* deleteRtnFromOtn(OptTreeNode* otn, SnortConfig* sc) return deleteRtnFromOtn(otn, get_ips_policy()->policy_id, sc); } -static uint32_t rtn_hash_func(HashFnc*, const unsigned char* k, int) -{ - uint32_t a,b,c; - const RuleTreeNodeKey* rtnk = (const RuleTreeNodeKey*)k; - RuleTreeNode* rtn = rtnk->rtn; - - a = rtn->action; - b = rtn->flags; - c = (uint32_t)(uintptr_t)rtn->listhead; - - mix(a,b,c); - - a += (uint32_t)(uintptr_t)rtn->src_portobject; - b += (uint32_t)(uintptr_t)rtn->dst_portobject; - c += (uint32_t)(uintptr_t)rtnk->policyId; - - finalize(a,b,c); - - return c; -} - -static bool rtn_compare_func(const void* k1, const void* k2, size_t) -{ - const RuleTreeNodeKey* rtnk1 = (const RuleTreeNodeKey*)k1; - const RuleTreeNodeKey* rtnk2 = (const RuleTreeNodeKey*)k2; - - if (!rtnk1 || !rtnk2) - return false; - - if (rtnk1->policyId != rtnk2->policyId) - return false; - - if (same_headers(rtnk1->rtn, rtnk2->rtn)) - return true; - - return false; -} - int addRtnToOtn(SnortConfig* sc, OptTreeNode* otn, RuleTreeNode* rtn, PolicyId policyId) { if (otn->proto_node_num <= policyId) @@ -627,7 +646,6 @@ int addRtnToOtn(SnortConfig* sc, OptTreeNode* otn, RuleTreeNode* rtn, PolicyId p } RuleTreeNode* curr = otn->proto_nodes[policyId]; - if ( curr ) { deleteRtnFromOtn(otn, policyId, sc, (curr->otnRefCount == 1)); @@ -637,11 +655,7 @@ int addRtnToOtn(SnortConfig* sc, OptTreeNode* otn, RuleTreeNode* rtn, PolicyId p rtn->otnRefCount++; if (!sc->rtn_hash_table) - { - sc->rtn_hash_table = new XHash( - 10000, sizeof(RuleTreeNodeKey), 0, 0, false, nullptr, nullptr, true); - sc->rtn_hash_table->set_key_opcodes(rtn_hash_func, rtn_compare_func); - } + sc->rtn_hash_table = new RuleTreeCache(10000, sizeof(RuleTreeNodeKey)); RuleTreeNodeKey key; memset(&key, 0, sizeof(key)); diff --git a/src/ports/port_object.cc b/src/ports/port_object.cc index 5e6114e3d..50d424cab 100644 --- a/src/ports/port_object.cc +++ b/src/ports/port_object.cc @@ -287,7 +287,7 @@ int PortObjectNormalize(PortObject* po) /* PortObjects should be normalized, prior to testing */ -int PortObjectEqual(PortObject* a, PortObject* b) +bool PortObjectEqual(PortObject* a, PortObject* b) { PortObjectItem* pa; PortObjectItem* pb; @@ -295,7 +295,7 @@ int PortObjectEqual(PortObject* a, PortObject* b) SF_LNODE* posb; if ( a->item_list->count != b->item_list->count ) - return 0; + return false; pa = (PortObjectItem*)sflist_first(a->item_list,&posa); pb = (PortObjectItem*)sflist_first(b->item_list,&posb); @@ -303,16 +303,16 @@ int PortObjectEqual(PortObject* a, PortObject* b) while ( pa && pb ) { if ( !PortObjectItemsEqual(pa, pb) ) - return 0; + return false; pa = (PortObjectItem*)sflist_next(&posa); pb = (PortObjectItem*)sflist_next(&posb); } if ( pa || pb ) /* both are not done - cannot match */ - return 0; + return false; - return 1; /* match */ + return true; /* match */ } /* diff --git a/src/ports/port_object.h b/src/ports/port_object.h index f4e68174f..3847ab4a4 100644 --- a/src/ports/port_object.h +++ b/src/ports/port_object.h @@ -61,7 +61,7 @@ PortObject* PortObjectDupPorts(PortObject*); int PortObjectNormalize(PortObject*); void PortObjectToggle(PortObject*); -int PortObjectEqual(PortObject* poa, PortObject* pob); +bool PortObjectEqual(PortObject* poa, PortObject* pob); int PortObjectPortCount(PortObject*); int PortObjectHasPort(PortObject*, int port); diff --git a/src/ports/port_object2.cc b/src/ports/port_object2.cc index 090dd7430..110ec33dc 100644 --- a/src/ports/port_object2.cc +++ b/src/ports/port_object2.cc @@ -23,7 +23,9 @@ #include "port_object2.h" -#include "hash/hashfcn.h" +#include "hash/ghash.h" +#include "hash/hash_defs.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "parser/parser.h" #include "utils/util.h" @@ -55,21 +57,37 @@ using namespace snort; #define SWAP_BYTES(a) #endif -static unsigned po_rule_hash_func(HashFnc* p, const unsigned char* k, int n) +class PortObject2HashKeyOps : public HashKeyOperations { - unsigned char* key; - int ikey = *(const int*)k; +public: + PortObject2HashKeyOps(int rows) + : HashKeyOperations(rows) + { } - /* Since the input is really an int, put the bytes into a normalized - * order so that the hash function returns consistent results across - * on BE & LE hardware. */ - SWAP_BYTES(ikey); + unsigned do_hash(const unsigned char* k, int len) override + { + unsigned char* key; + int ikey = *(const int*)k; - /* Set a pointer to the key to pass to the hashing function */ - key = (unsigned char*)&ikey; + /* Since the input is really an int, put the bytes into a normalized + * order so that the hash function returns consistent results across + * on BE & LE hardware. */ + SWAP_BYTES(ikey); - return hashfcn_hash(p, key, n); -} + /* Set a pointer to the key to pass to the hashing function */ + key = (unsigned char*)&ikey; + + return HashKeyOperations::do_hash(key, len); + } + + bool key_compare(const void* k1, const void* k2, size_t len) override + { + if ( memcmp(k1, k2, len ) ) + return false; + else + return true; + } +}; static int* RuleHashToSortedArray(GHash* rh) { @@ -92,14 +110,6 @@ static int* RuleHashToSortedArray(GHash* rh) return ra; } -static bool port_object_key_compare_func(const void* k1, const void* k2, size_t len) -{ - if ( memcmp(k1, k2, len ) ) - return false; - else - return true; -} - //------------------------------------------------------------------------- // PortObject2 - public //------------------------------------------------------------------------- @@ -109,9 +119,7 @@ PortObject2* PortObject2New(int nrules) PortObject2* po = (PortObject2*)snort_calloc(sizeof(PortObject2)); po->item_list = sflist_new(); po->rule_hash = new GHash(nrules, sizeof(int), 0, snort_free); - - /* Use hash function defined above for hashing the key as an int. */ - po->rule_hash->set_key_opcodes(po_rule_hash_func, port_object_key_compare_func); + po->rule_hash->set_hashkey_ops(new PortObject2HashKeyOps(nrules)); return po; } @@ -189,7 +197,7 @@ PortObject2* PortObject2Dup(PortObject& po) int* prule = (int*)snort_calloc(sizeof(int)); *prule = *prid; - if ( ponew->rule_hash->insert(prule, prule) != GHASH_OK ) + if ( ponew->rule_hash->insert(prule, prule) != HASH_OK ) snort_free(prule); } } @@ -226,7 +234,7 @@ PortObject2* PortObject2AppendPortObject(PortObject2* poa, PortObject* pob) int* prid2 = (int*)snort_calloc(sizeof(int)); *prid2 = *prid; - if ( poa->rule_hash->insert(prid2, prid2) != GHASH_OK ) + if ( poa->rule_hash->insert(prid2, prid2) != HASH_OK ) snort_free(prid2); } return poa; @@ -247,7 +255,7 @@ PortObject2* PortObject2AppendPortObject2(PortObject2* poa, PortObject2* pob) int* prid2 = (int*)snort_calloc(sizeof(int)); *prid2 = *prid; - if ( poa->rule_hash->insert(prid2, prid2) != GHASH_OK ) + if ( poa->rule_hash->insert(prid2, prid2) != HASH_OK ) snort_free(prid2); } return poa; diff --git a/src/ports/port_object2.h b/src/ports/port_object2.h index e95cccf22..67bafcb3c 100644 --- a/src/ports/port_object2.h +++ b/src/ports/port_object2.h @@ -23,13 +23,16 @@ #define PORT_OBJECT2_H #include "framework/bits.h" -#include "hash/ghash.h" #include "utils/sflsq.h" //------------------------------------------------------------------------- // PortObject2 is similar to PortObject //------------------------------------------------------------------------- +namespace snort +{ +class GHash; +} struct PortObject; diff --git a/src/ports/port_table.cc b/src/ports/port_table.cc index 71e30bf0e..26bd6a773 100644 --- a/src/ports/port_table.cc +++ b/src/ports/port_table.cc @@ -25,7 +25,9 @@ #include -#include "hash/hashfcn.h" +#include "hash/ghash.h" +#include "hash/hash_defs.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "main/snort_debug.h" #include "utils/util.h" @@ -77,24 +79,6 @@ static void plx_free(void* p) snort_free(p); } -static unsigned plx_hash(HashFnc* p, const unsigned char* d, int) -{ - unsigned hash = p->seed; - const plx_t* plx = *(plx_t* const*)d; - - for ( int i = 0; i < plx->n; i++ ) - { - unsigned char* pc_ptr = (unsigned char*)&plx->p[i]; - - for ( unsigned k = 0; k < sizeof(void*); k++ ) - { - hash *= p->scale; - hash += pc_ptr[k]; - } - } - return hash ^ p->hardener; -} - /* for sorting an array of pointers */ static inline int p_keycmp(const void* a, const void* b) { @@ -107,84 +91,93 @@ static inline int p_keycmp(const void* a, const void* b) return 0; /* they are equal */ } -/* - Hash Key Comparisons for treating plx_t types as Keys - - return values memcmp style - - this only needs to produce 0 => exact match, otherwise not. - -1, and +1 are not strictly needed, they could both return - a non zero value for the purposes of hashing and searching. -*/ -static bool plx_keycmp(const void* a, const void* b, size_t) +class PlxHashKeyOps : public HashKeyOperations { - const plx_t* pla = *(plx_t* const*)a; - const plx_t* plb = *(plx_t* const*)b; +public: + PlxHashKeyOps(int rows) + : HashKeyOperations(rows) + { } - if ( pla->n < plb->n ) - return false; + unsigned do_hash(const unsigned char* k, int) override + { + unsigned hash = seed; + const plx_t* plx = *(plx_t* const*)k; + + for ( int i = 0; i < plx->n; i++ ) + { + unsigned char* pc_ptr = (unsigned char*)&plx->p[i]; - if ( pla->n > plb->n ) - return false; + for ( unsigned k = 0; k < sizeof(void*); k++ ) + { + hash *= scale; + hash += pc_ptr[k]; + } + } + return hash ^ hardener; + } - for ( int i = 0; i < pla->n; i++ ) + bool key_compare(const void* k1, const void* k2, size_t) override { - if ( p_keycmp(&pla->p[i], &plb->p[i]) ) + const plx_t* pla = *(plx_t* const*)k1; + const plx_t* plb = *(plx_t* const*)k2; + + if ( pla->n < plb->n ) return false; - } - return true; /* they are equal */ -} + if ( pla->n > plb->n ) + return false; + + for ( int i = 0; i < pla->n; i++ ) + { + if ( p_keycmp(&pla->p[i], &plb->p[i]) ) + return false; + } + + return true; /* they are equal */ } +}; //------------------------------------------------------------------------- // PortTable - private - other //------------------------------------------------------------------------- -/* - Hash Key Comparisons for treating PortObjects as Keys - - return values memcmp style -*/ -static bool PortObject_keycmp(const void* a, const void* b, size_t) +class PortObjectHashKeyOps : public HashKeyOperations { - return PortObjectEqual(*(PortObject* const*)a, *(PortObject* const*)b); -} - -/* - Hash routine for hashing PortObjects as Keys +public: + PortObjectHashKeyOps(int rows) + : HashKeyOperations(rows) + { } - p - HashFnc * - d - PortObject * - n = 4 bytes (sizeof*) - not used + unsigned do_hash(const unsigned char* k, int) override + { + unsigned hash = seed; + const PortObject* po = *(PortObject* const*)k; + SF_LNODE* pos; - Don't use this for type=ANY port objects -*/ -static unsigned PortObject_hash(HashFnc* p, const unsigned char* d, int) -{ - unsigned hash = p->seed; - const PortObject* po = *(PortObject* const*)d; - SF_LNODE* pos; + for (PortObjectItem* poi = (PortObjectItem*)sflist_first(po->item_list, &pos); + poi != nullptr; + poi = (PortObjectItem*)sflist_next(&pos) ) + { + if ( poi->any() ) + continue; - /* hash up each item */ - for (PortObjectItem* poi = (PortObjectItem*)sflist_first(po->item_list, &pos); - poi != nullptr; - poi = (PortObjectItem*)sflist_next(&pos) ) - { - if ( poi->any() ) - continue; + hash *= scale; + hash += poi->lport & 0xff; + hash *= scale; + hash += (poi->lport >> 8) & 0xff; - hash *= p->scale; - hash += poi->lport & 0xff; - hash *= p->scale; - hash += (poi->lport >> 8) & 0xff; + hash *= scale; + hash += poi->hport & 0xff; + hash *= scale; + hash += (poi->hport >> 8) & 0xff; + } + return hash ^ hardener; + } - hash *= p->scale; - hash += poi->hport & 0xff; - hash *= p->scale; - hash += (poi->hport >> 8) & 0xff; + bool key_compare(const void* k1, const void* k2, size_t) override + { + return PortObjectEqual(*(PortObject* const*)k1, *(PortObject* const*)k2); } - return hash ^ p->hardener; -} +}; /* * Merge multiple PortObjects into a final PortObject2, @@ -248,7 +241,7 @@ static PortObject2* _merge_N_pol( // Add the Merged PortObject2 to the PortObject2 hash table keyed by ports. int stat = mhash->insert(&ponew, ponew); // This is possible since PLX hash on a different key - if ( stat == GHASH_INTABLE ) + if ( stat == HASH_INTABLE ) { PortObject2* pox = (PortObject2*)mhash->find(&ponew); assert( pox ); @@ -264,7 +257,7 @@ static PortObject2* _merge_N_pol( // Add the plx node to the PLX hash table stat = mhashx->insert(&plx_tmp, ponew); - if ( stat == GHASH_INTABLE ) + if ( stat == HASH_INTABLE ) FatalError("Could not add merged plx to PLX HASH table-INTABLE\n"); return ponew; @@ -460,13 +453,12 @@ static void PortTableCompileMergePortObjects(PortTable* p) // Create a Merged Port Object Table - hash by ports, no user keys, don't free data GHash* mhash = new GHash(PO_HASH_TBL_ROWS, sizeof(PortObject*), 0, nullptr); - - mhash->set_key_opcodes(PortObject_hash, PortObject_keycmp); + mhash->set_hashkey_ops(new PortObjectHashKeyOps(PO_HASH_TBL_ROWS)); p->pt_mpo_hash = mhash; // Create a Merged Port Object Table - hash by pointers, no user keys, don't free data GHash* mhashx = new GHash(PO_HASH_TBL_ROWS, sizeof(plx_t*), 0, nullptr); - mhashx->set_key_opcodes(plx_hash, plx_keycmp); + mhashx->set_hashkey_ops(new PlxHashKeyOps(PO_HASH_TBL_ROWS)); p->pt_mpxo_hash = mhashx; SF_LIST* plx_list = sflist_new(); diff --git a/src/ports/port_table.h b/src/ports/port_table.h index f12fd322a..6048c9c22 100644 --- a/src/ports/port_table.h +++ b/src/ports/port_table.h @@ -22,12 +22,16 @@ #ifndef PORT_TABLE_H #define PORT_TABLE_H -#include "hash/ghash.h" #include "ports/port_item.h" #include "ports/port_object.h" #include "ports/port_object2.h" #include "utils/sflsq.h" +namespace snort +{ +class GHash; +} + //------------------------------------------------------------------------- // PortTable - provides support to analyze the Port List objects defined by // the user as either PortVar entries or simply as inline rule port list diff --git a/src/ports/port_var_table.cc b/src/ports/port_var_table.cc index ea5257ecb..89ed38436 100644 --- a/src/ports/port_var_table.cc +++ b/src/ports/port_var_table.cc @@ -23,6 +23,9 @@ #include "port_var_table.h" +#include "hash/ghash.h" +#include "hash/hash_defs.h" + using namespace snort; //------------------------------------------------------------------------- @@ -72,10 +75,10 @@ int PortVarTableFree(PortVarTable* pvt) int PortVarTableAdd(PortVarTable* h, PortObject* po) { int stat = h->insert(po->name, po); - if ( stat == GHASH_INTABLE ) + if ( stat == HASH_INTABLE ) return 1; - if ( stat == GHASH_OK ) + if ( stat == HASH_OK ) return 0; return -1; diff --git a/src/ports/port_var_table.h b/src/ports/port_var_table.h index 8aedbc6b2..7babca35d 100644 --- a/src/ports/port_var_table.h +++ b/src/ports/port_var_table.h @@ -22,10 +22,14 @@ #ifndef PORT_VAR_TABLE_H #define PORT_VAR_TABLE_H -#include "hash/ghash.h" #include "ports/port_object.h" #include "ports/port_table.h" +namespace snort +{ +class GHash; +} + //------------------------------------------------------------------------- // PortVarTable // port lists may be defined as 'name port-list' diff --git a/src/service_inspectors/cip/ips_cip_attribute.cc b/src/service_inspectors/cip/ips_cip_attribute.cc index ada9362ad..c9ff7af14 100644 --- a/src/service_inspectors/cip/ips_cip_attribute.cc +++ b/src/service_inspectors/cip/ips_cip_attribute.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_class.cc b/src/service_inspectors/cip/ips_cip_class.cc index f7681414a..c5a1d1f16 100644 --- a/src/service_inspectors/cip/ips_cip_class.cc +++ b/src/service_inspectors/cip/ips_cip_class.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_connpathclass.cc b/src/service_inspectors/cip/ips_cip_connpathclass.cc index 22e9a10f0..8f93c5159 100644 --- a/src/service_inspectors/cip/ips_cip_connpathclass.cc +++ b/src/service_inspectors/cip/ips_cip_connpathclass.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_enipcommand.cc b/src/service_inspectors/cip/ips_cip_enipcommand.cc index c02260fd3..5dd433a73 100644 --- a/src/service_inspectors/cip/ips_cip_enipcommand.cc +++ b/src/service_inspectors/cip/ips_cip_enipcommand.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_enipreq.cc b/src/service_inspectors/cip/ips_cip_enipreq.cc index a1866fa6c..f6dc0913b 100644 --- a/src/service_inspectors/cip/ips_cip_enipreq.cc +++ b/src/service_inspectors/cip/ips_cip_enipreq.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_eniprsp.cc b/src/service_inspectors/cip/ips_cip_eniprsp.cc index 64d249df2..d4659b0f3 100644 --- a/src/service_inspectors/cip/ips_cip_eniprsp.cc +++ b/src/service_inspectors/cip/ips_cip_eniprsp.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_instance.cc b/src/service_inspectors/cip/ips_cip_instance.cc index de93fb005..36dc97fe1 100644 --- a/src/service_inspectors/cip/ips_cip_instance.cc +++ b/src/service_inspectors/cip/ips_cip_instance.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_req.cc b/src/service_inspectors/cip/ips_cip_req.cc index bc78481bc..7fbea0cd3 100644 --- a/src/service_inspectors/cip/ips_cip_req.cc +++ b/src/service_inspectors/cip/ips_cip_req.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_rsp.cc b/src/service_inspectors/cip/ips_cip_rsp.cc index 8bed0e85c..a2c24b00e 100644 --- a/src/service_inspectors/cip/ips_cip_rsp.cc +++ b/src/service_inspectors/cip/ips_cip_rsp.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_service.cc b/src/service_inspectors/cip/ips_cip_service.cc index cff11df7a..55c7c17b7 100644 --- a/src/service_inspectors/cip/ips_cip_service.cc +++ b/src/service_inspectors/cip/ips_cip_service.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_status.cc b/src/service_inspectors/cip/ips_cip_status.cc index a9036affa..78517c215 100644 --- a/src/service_inspectors/cip/ips_cip_status.cc +++ b/src/service_inspectors/cip/ips_cip_status.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/dce_rpc/ips_dce_iface.cc b/src/service_inspectors/dce_rpc/ips_dce_iface.cc index 701d1f2ab..553fe4212 100644 --- a/src/service_inspectors/dce_rpc/ips_dce_iface.cc +++ b/src/service_inspectors/dce_rpc/ips_dce_iface.cc @@ -29,7 +29,7 @@ #include "framework/module.h" #include "framework/ips_option.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "target_based/snort_protocols.h" #include "utils/util.h" diff --git a/src/service_inspectors/dce_rpc/ips_dce_opnum.cc b/src/service_inspectors/dce_rpc/ips_dce_opnum.cc index 11e675960..8e74d1d19 100644 --- a/src/service_inspectors/dce_rpc/ips_dce_opnum.cc +++ b/src/service_inspectors/dce_rpc/ips_dce_opnum.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "utils/util.h" diff --git a/src/service_inspectors/dce_rpc/ips_dce_stub_data.cc b/src/service_inspectors/dce_rpc/ips_dce_stub_data.cc index 2f29d382e..522b98ccc 100644 --- a/src/service_inspectors/dce_rpc/ips_dce_stub_data.cc +++ b/src/service_inspectors/dce_rpc/ips_dce_stub_data.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "dce_common.h" diff --git a/src/service_inspectors/dnp3/ips_dnp3_data.cc b/src/service_inspectors/dnp3/ips_dnp3_data.cc index adebb878d..7d3316782 100644 --- a/src/service_inspectors/dnp3/ips_dnp3_data.cc +++ b/src/service_inspectors/dnp3/ips_dnp3_data.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/dnp3/ips_dnp3_func.cc b/src/service_inspectors/dnp3/ips_dnp3_func.cc index 480c97fc5..63a368308 100644 --- a/src/service_inspectors/dnp3/ips_dnp3_func.cc +++ b/src/service_inspectors/dnp3/ips_dnp3_func.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/dnp3/ips_dnp3_ind.cc b/src/service_inspectors/dnp3/ips_dnp3_ind.cc index 311d0988f..1549eefe9 100644 --- a/src/service_inspectors/dnp3/ips_dnp3_ind.cc +++ b/src/service_inspectors/dnp3/ips_dnp3_ind.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/dnp3/ips_dnp3_obj.cc b/src/service_inspectors/dnp3/ips_dnp3_obj.cc index 26b158ea8..eee61eaf2 100644 --- a/src/service_inspectors/dnp3/ips_dnp3_obj.cc +++ b/src/service_inspectors/dnp3/ips_dnp3_obj.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/ftp_telnet/pp_ftp.cc b/src/service_inspectors/ftp_telnet/pp_ftp.cc index fbd3747ae..1a35c2bf2 100644 --- a/src/service_inspectors/ftp_telnet/pp_ftp.cc +++ b/src/service_inspectors/ftp_telnet/pp_ftp.cc @@ -42,7 +42,7 @@ #include "detection/detection_engine.h" #include "detection/detection_util.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "file_api/file_service.h" #include "protocols/packet.h" #include "stream/stream.h" diff --git a/src/service_inspectors/gtp/ips_gtp_info.cc b/src/service_inspectors/gtp/ips_gtp_info.cc index f1d26e648..6ba0373aa 100644 --- a/src/service_inspectors/gtp/ips_gtp_info.cc +++ b/src/service_inspectors/gtp/ips_gtp_info.cc @@ -24,7 +24,7 @@ // gtp_info rule option implementation -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" diff --git a/src/service_inspectors/gtp/ips_gtp_type.cc b/src/service_inspectors/gtp/ips_gtp_type.cc index 5620fc18d..2c2dd6864 100644 --- a/src/service_inspectors/gtp/ips_gtp_type.cc +++ b/src/service_inspectors/gtp/ips_gtp_type.cc @@ -26,7 +26,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/gtp/ips_gtp_version.cc b/src/service_inspectors/gtp/ips_gtp_version.cc index aaa5831da..eca531c30 100644 --- a/src/service_inspectors/gtp/ips_gtp_version.cc +++ b/src/service_inspectors/gtp/ips_gtp_version.cc @@ -26,7 +26,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/http_inspect/http_buffer_info.cc b/src/service_inspectors/http_inspect/http_buffer_info.cc index e72f672d1..0876a08ae 100644 --- a/src/service_inspectors/http_inspect/http_buffer_info.cc +++ b/src/service_inspectors/http_inspect/http_buffer_info.cc @@ -17,7 +17,7 @@ //-------------------------------------------------------------------------- // http_buffer_info.cc author Brandon Stultz -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "http_buffer_info.h" using namespace snort; diff --git a/src/service_inspectors/http_inspect/http_transaction.cc b/src/service_inspectors/http_inspect/http_transaction.cc index 1ad11463e..f05d2556d 100644 --- a/src/service_inspectors/http_inspect/http_transaction.cc +++ b/src/service_inspectors/http_inspect/http_transaction.cc @@ -32,7 +32,7 @@ #include "http_msg_status.h" #include "http_msg_trailer.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" using namespace HttpCommon; using namespace HttpEnums; diff --git a/src/service_inspectors/http_inspect/http_uri.cc b/src/service_inspectors/http_inspect/http_uri.cc index 2b13b6f7b..126eafc06 100644 --- a/src/service_inspectors/http_inspect/http_uri.cc +++ b/src/service_inspectors/http_inspect/http_uri.cc @@ -25,7 +25,7 @@ #include "http_common.h" #include "http_enum.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" using namespace HttpCommon; using namespace HttpEnums; diff --git a/src/service_inspectors/http_inspect/ips_http.cc b/src/service_inspectors/http_inspect/ips_http.cc index 1859f37e3..67b602206 100644 --- a/src/service_inspectors/http_inspect/ips_http.cc +++ b/src/service_inspectors/http_inspect/ips_http.cc @@ -24,7 +24,7 @@ #include "ips_http.h" #include "framework/cursor.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "parser/parse_utils.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/modbus/ips_modbus_data.cc b/src/service_inspectors/modbus/ips_modbus_data.cc index c7fe71ba0..2cf071522 100644 --- a/src/service_inspectors/modbus/ips_modbus_data.cc +++ b/src/service_inspectors/modbus/ips_modbus_data.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/modbus/ips_modbus_func.cc b/src/service_inspectors/modbus/ips_modbus_func.cc index bfdde4114..e564c99a8 100644 --- a/src/service_inspectors/modbus/ips_modbus_func.cc +++ b/src/service_inspectors/modbus/ips_modbus_func.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/modbus/ips_modbus_unit.cc b/src/service_inspectors/modbus/ips_modbus_unit.cc index 5ebf0416a..076637083 100644 --- a/src/service_inspectors/modbus/ips_modbus_unit.cc +++ b/src/service_inspectors/modbus/ips_modbus_unit.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/s7commplus/ips_s7comm_content.cc b/src/service_inspectors/s7commplus/ips_s7comm_content.cc index 740cb90c4..c17b7c54d 100644 --- a/src/service_inspectors/s7commplus/ips_s7comm_content.cc +++ b/src/service_inspectors/s7commplus/ips_s7comm_content.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/s7commplus/ips_s7comm_func.cc b/src/service_inspectors/s7commplus/ips_s7comm_func.cc index 1a5da45c5..aaba91329 100644 --- a/src/service_inspectors/s7commplus/ips_s7comm_func.cc +++ b/src/service_inspectors/s7commplus/ips_s7comm_func.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/s7commplus/ips_s7comm_opcode.cc b/src/service_inspectors/s7commplus/ips_s7comm_opcode.cc index d089ce5db..48478d549 100644 --- a/src/service_inspectors/s7commplus/ips_s7comm_opcode.cc +++ b/src/service_inspectors/s7commplus/ips_s7comm_opcode.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/sip/ips_sip_method.cc b/src/service_inspectors/sip/ips_sip_method.cc index 5382e05f5..f7bad4e01 100644 --- a/src/service_inspectors/sip/ips_sip_method.cc +++ b/src/service_inspectors/sip/ips_sip_method.cc @@ -30,7 +30,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/sip/ips_sip_stat_code.cc b/src/service_inspectors/sip/ips_sip_stat_code.cc index 04044b145..45ae386b9 100644 --- a/src/service_inspectors/sip/ips_sip_stat_code.cc +++ b/src/service_inspectors/sip/ips_sip_stat_code.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/sip/sip_utils.cc b/src/service_inspectors/sip/sip_utils.cc index b2f85e5ed..cf058ee4e 100644 --- a/src/service_inspectors/sip/sip_utils.cc +++ b/src/service_inspectors/sip/sip_utils.cc @@ -27,7 +27,7 @@ #include -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" /* * Trim spaces non-destructively on both sides of string : '', \t, \n, \r diff --git a/src/service_inspectors/ssl/ips_ssl_state.cc b/src/service_inspectors/ssl/ips_ssl_state.cc index 98a1ba0c9..a5db4ac5e 100644 --- a/src/service_inspectors/ssl/ips_ssl_state.cc +++ b/src/service_inspectors/ssl/ips_ssl_state.cc @@ -23,7 +23,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "protocols/ssl.h" diff --git a/src/service_inspectors/ssl/ips_ssl_version.cc b/src/service_inspectors/ssl/ips_ssl_version.cc index 69bea2ad7..2f846c9ea 100644 --- a/src/service_inspectors/ssl/ips_ssl_version.cc +++ b/src/service_inspectors/ssl/ips_ssl_version.cc @@ -23,7 +23,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "protocols/ssl.h" diff --git a/src/stream/tcp/ips_stream_reassemble.cc b/src/stream/tcp/ips_stream_reassemble.cc index 476817e34..cafac897d 100644 --- a/src/stream/tcp/ips_stream_reassemble.cc +++ b/src/stream/tcp/ips_stream_reassemble.cc @@ -24,7 +24,7 @@ #include "detection/detection_engine.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "main/snort_config.h" #include "profiler/profiler.h" diff --git a/src/stream/tcp/ips_stream_size.cc b/src/stream/tcp/ips_stream_size.cc index 0195b4325..70f684ecd 100644 --- a/src/stream/tcp/ips_stream_size.cc +++ b/src/stream/tcp/ips_stream_size.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hashfcn.h" +#include "hash/hash_key_operations.h" #include "profiler/profiler_defs.h" #include "tcp_session.h" diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index 050445200..d83948583 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -7,6 +7,7 @@ set( UTIL_INCLUDES event_gen.h infractions.h kmap.h + memcap_allocator.h primed_allocator.h safec.h segment_mem.h diff --git a/src/utils/memcap_allocator.h b/src/utils/memcap_allocator.h new file mode 100644 index 000000000..4018f866d --- /dev/null +++ b/src/utils/memcap_allocator.h @@ -0,0 +1,107 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// memcap_allocator.h author davis mcpherson davmcphe@cisco.com + +// MemCapAllocator manages allocation and freeing of fixed size blocks of memory with +// an option of limiting total size of allocations to a specified capacity + +#ifndef MEMCAP_ALLOCATOR_H +#define MEMCAP_ALLOCATOR_H + +#include "framework/counts.h" +#include "utils/util.h" + +class MemCapAllocator +{ +public: + MemCapAllocator(unsigned long capacity, unsigned long allocation_size) + : mem_capacity(capacity), allocation_size(allocation_size) + { } + + void* allocate() + { + void* data = nullptr; + if ( is_space_available() ) + { + data = snort_calloc(allocation_size); + mem_allocated += allocation_size; + ++allocation_requests; + } + else + ++no_memory_available; + + return data; + } + + void free(void* mem) + { + snort_free(mem); + mem_allocated -= allocation_size; + ++free_requests; + } + + bool is_space_available() const + { + if ( !mem_capacity ) + return true; + else + return mem_allocated + allocation_size <= mem_capacity; + } + + bool is_over_capacity() const + { + if ( !mem_capacity ) + return false; + else + return mem_allocated > mem_capacity; + } + + void set_mem_capacity(unsigned long capacity) + { mem_capacity = capacity; } + + unsigned long get_mem_capacity() const + { return mem_capacity; } + + unsigned long get_allocation_size() const + { return allocation_size; } + + unsigned long get_mem_allocated() const + { return mem_allocated; } + + PegCount get_allocation_requests() const + { return allocation_requests; } + + PegCount get_free_requests() const + { return free_requests; } + + PegCount get_no_memory_available() const + { return no_memory_available; } + +private: + unsigned long mem_capacity; + unsigned long allocation_size; + unsigned long mem_allocated = 0; + + PegCount allocation_requests = 0; + PegCount free_requests = 0; + PegCount no_memory_available = 0; +}; + +#endif + diff --git a/src/utils/sfmemcap.cc b/src/utils/sfmemcap.cc index 31ef82179..904d5cb19 100644 --- a/src/utils/sfmemcap.cc +++ b/src/utils/sfmemcap.cc @@ -64,9 +64,7 @@ void* sfmemcap_alloc(MEMCAP* mc, unsigned long nbytes) { /* Check if we've maxed out our memory - if we are tracking memory */ if ( (mc->memused + nbytes) > mc->memcap ) - { return nullptr; - } } //data = (long*)snort_alloc( nbytes ); diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index a9bf39e56..181baf831 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -3,4 +3,6 @@ add_cpputest( boyer_moore_test ../boyer_moore.cc ) +add_cpputest( memcap_allocator_test ) + add_catch_test( bitop_test ) diff --git a/src/utils/test/memcap_allocator_test.cc b/src/utils/test/memcap_allocator_test.cc new file mode 100644 index 000000000..beea28088 --- /dev/null +++ b/src/utils/test/memcap_allocator_test.cc @@ -0,0 +1,218 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// memcap_allocator_test.cc author davis mcpherson +// unit tests for MemCapAllocator class + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "../memcap_allocator.h" + +#include +#include + +const unsigned long ALLOCATION_SIZE = 250; +const unsigned long NO_MEM_CAP_LIMIT = 0; +const unsigned NO_MEMCAP_LIMIT_ALLOCATIONS = 10000; + +const unsigned long MEM_CAP_5000 = 5000; +const unsigned MAX_ALLOCATIONS_5000 = MEM_CAP_5000 / ALLOCATION_SIZE; + +const unsigned long MEM_CAP_10000 = 10000; +const unsigned MAX_ALLOCATIONS_10000 = MEM_CAP_10000 / ALLOCATION_SIZE; + +const unsigned long MEM_CAP_20000 = 20000; +const unsigned MAX_ALLOCATIONS_20000 = MEM_CAP_20000 / ALLOCATION_SIZE; + +TEST_GROUP(memcap_allocator) +{ }; + +TEST(memcap_allocator, no_memcap_limit_allocator_test) +{ + MemCapAllocator* mca = new MemCapAllocator(NO_MEM_CAP_LIMIT, ALLOCATION_SIZE); + CHECK(mca); + CHECK(mca->get_allocation_size() == ALLOCATION_SIZE); + CHECK(mca->get_mem_capacity() == NO_MEM_CAP_LIMIT); + CHECK(mca->is_space_available()); + CHECK(!mca->is_over_capacity()); + CHECK(mca->get_mem_allocated() == 0); + CHECK(mca->get_allocation_requests() == 0); + CHECK(mca->get_free_requests() == 0); + CHECK(mca->get_no_memory_available() == 0); + delete mca; +} + +TEST(memcap_allocator, no_memcap_limit_allocations_test) +{ + MemCapAllocator* mca = new MemCapAllocator(NO_MEM_CAP_LIMIT, ALLOCATION_SIZE); + CHECK(mca); + + unsigned bytes_allocated; + uint8_t** mem_blocks = (uint8_t**)snort_alloc(sizeof(uint8_t*) * NO_MEMCAP_LIMIT_ALLOCATIONS); + for ( unsigned i = 0; i < NO_MEMCAP_LIMIT_ALLOCATIONS; i++) + { + mem_blocks[i] = (uint8_t*)mca->allocate(); + bytes_allocated = (i + 1) * ALLOCATION_SIZE; + CHECK(mem_blocks[i]); + CHECK(mca->get_mem_allocated() == bytes_allocated); + CHECK(mca->is_space_available()); + CHECK(!mca->is_over_capacity()); + } + + CHECK(mca->get_allocation_requests() == NO_MEMCAP_LIMIT_ALLOCATIONS); + CHECK(mca->get_free_requests() == 0); + CHECK(mca->get_no_memory_available() == 0); + + for ( unsigned i = 0; i < NO_MEMCAP_LIMIT_ALLOCATIONS; i++) + { + mca->free(mem_blocks[i]); + bytes_allocated = (NO_MEMCAP_LIMIT_ALLOCATIONS - (i + 1)) * ALLOCATION_SIZE; + CHECK(mca->get_mem_allocated() == bytes_allocated); + } + + CHECK(mca->is_space_available()); + CHECK(!mca->is_over_capacity()); + CHECK(mca->get_mem_allocated() == 0); + CHECK(mca->get_free_requests() == NO_MEMCAP_LIMIT_ALLOCATIONS); + CHECK(mca->get_no_memory_available() == 0); + + snort_free(mem_blocks); + delete mca; +} + +TEST(memcap_allocator, create_memcap_allocator_test) +{ + MemCapAllocator* mca = new MemCapAllocator(MEM_CAP_10000, ALLOCATION_SIZE); + CHECK(mca); + CHECK(mca->get_allocation_size() == ALLOCATION_SIZE); + CHECK(mca->get_mem_capacity() == MEM_CAP_10000); + mca->set_mem_capacity(MEM_CAP_20000); + CHECK(mca->get_mem_capacity() == MEM_CAP_20000); + CHECK(mca->get_mem_allocated() == 0); + CHECK(mca->get_allocation_requests() == 0); + CHECK(mca->get_free_requests() == 0); + CHECK(mca->get_no_memory_available() == 0); + delete mca; +} + +TEST(memcap_allocator, allocate_and_free_memory_test) +{ + MemCapAllocator* mca = new MemCapAllocator(MEM_CAP_10000, ALLOCATION_SIZE); + CHECK(mca); + CHECK(mca->is_space_available()); + CHECK(!mca->is_over_capacity()); + + uint8_t* mem = (uint8_t*)mca->allocate(); + CHECK(mem); + CHECK(mca->get_mem_allocated() == ALLOCATION_SIZE); + CHECK(mca->is_space_available()); + CHECK(!mca->is_over_capacity()); + CHECK(mca->get_allocation_requests() == 1); + CHECK(mca->get_free_requests() == 0); + CHECK(mca->get_no_memory_available() == 0); + + mca->free(mem); + CHECK(mca->get_mem_allocated() == 0); + CHECK(mca->is_space_available()); + CHECK(!mca->is_over_capacity()); + CHECK(mca->get_allocation_requests() == 1); + CHECK(mca->get_free_requests() == 1); + CHECK(mca->get_no_memory_available() == 0); + + delete mca; +} + +TEST(memcap_allocator, max_allocations_test) +{ + MemCapAllocator* mca = new MemCapAllocator(MEM_CAP_10000, ALLOCATION_SIZE); + CHECK(mca); + + unsigned bytes_allocated; + uint8_t* mem_blocks[MAX_ALLOCATIONS_10000]; + for ( unsigned i = 0; i < MAX_ALLOCATIONS_10000; i++) + { + mem_blocks[i] = (uint8_t*)mca->allocate(); + bytes_allocated = (i + 1) * ALLOCATION_SIZE; + CHECK(mem_blocks[i]); + CHECK(mca->get_mem_allocated() == bytes_allocated); + if ( i < MAX_ALLOCATIONS_10000 - 1 ) + CHECK(mca->is_space_available()); + else + CHECK(!mca->is_space_available()); + + CHECK(!mca->is_over_capacity()); + } + CHECK(mca->get_allocation_requests() == MAX_ALLOCATIONS_10000); + CHECK(mca->get_free_requests() == 0); + CHECK(mca->get_no_memory_available() == 0); + + uint8_t* mem = (uint8_t*)mca->allocate(); + CHECK(!mem); + CHECK(mca->get_no_memory_available() == 1); + mem = (uint8_t*)mca->allocate(); + CHECK(!mem); + mem = (uint8_t*)mca->allocate(); + CHECK(!mem); + CHECK(mca->get_no_memory_available() == 3); + mca->free(mem_blocks[0]); + mem_blocks[0] = (uint8_t*)mca->allocate(); + CHECK(mem_blocks[0]); + CHECK(mca->get_free_requests() == 1); + CHECK(mca->get_no_memory_available() == 3); + + mca->set_mem_capacity(MEM_CAP_20000); + mem = (uint8_t*)mca->allocate(); + CHECK(mem); + mca->set_mem_capacity(MEM_CAP_5000); + CHECK(mca->is_over_capacity()); + mca->free(mem); + CHECK(mca->get_free_requests() == 2); + + for ( unsigned i = 0; i < MAX_ALLOCATIONS_5000; i++) + { + CHECK(!mca->is_space_available()); + mca->free(mem_blocks[i]); + bytes_allocated = (MAX_ALLOCATIONS_10000 - (i + 1)) * ALLOCATION_SIZE; + CHECK(mca->get_mem_allocated() == bytes_allocated); + } + + CHECK(!mca->is_space_available()); + CHECK(!mca->is_over_capacity()); + + for ( unsigned i = MAX_ALLOCATIONS_5000; i < MAX_ALLOCATIONS_10000; i++) + { + mca->free(mem_blocks[i]); + bytes_allocated = (MAX_ALLOCATIONS_10000 - (i + 1)) * ALLOCATION_SIZE; + CHECK(mca->get_mem_allocated() == bytes_allocated); + CHECK(mca->is_space_available()); + CHECK(!mca->is_over_capacity()); + } + + CHECK(mca->get_mem_allocated() == 0); + CHECK(mca->get_free_requests() == MAX_ALLOCATIONS_10000 + 2); + CHECK(mca->get_no_memory_available() == 3); + + delete mca; +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +}