From: Russ Combs (rucombs) Date: Tue, 3 Mar 2020 15:59:38 +0000 (+0000) Subject: Merge pull request #2054 in SNORT/snort3 from ~DAVMCPHE/snort3:revert_hash_foo to... X-Git-Tag: 3.0.0-269~20 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6b385a10a4c012df7c8dd682b72958ae8e9d9adb;p=thirdparty%2Fsnort3.git Merge pull request #2054 in SNORT/snort3 from ~DAVMCPHE/snort3:revert_hash_foo to master Squashed commit of the following: commit 2b1a1979ac8b05de9e1adbf78cd80a334408f626 Author: davis mcpherson Date: Tue Mar 3 07:38:43 2020 -0500 Revert "Merge pull request #2009 in SNORT/snort3 from ~DAVMCPHE/snort3:lru_cache_for_hash to master" This reverts commit bb26ceaaed7ca78c25ff5b8aa6f0b338fb9ecc1a. commit ed107a307c5b1c05b69a99d6f167133af90e2dba Author: davis mcpherson Date: Tue Mar 3 07:32:19 2020 -0500 Revert "Merge pull request #2037 in SNORT/snort3 from ~DAVMCPHE/snort3:nuking_reload_errors to master" This reverts commit 9ee76016f1abe825d5ebcaa472a2651e89f88171. --- diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc index 884ebe2ab..c8718aace 100644 --- a/src/detection/detection_options.cc +++ b/src/detection/detection_options.cc @@ -37,8 +37,7 @@ #include "filters/detection_filter.h" #include "framework/cursor.h" -#include "hash/hash_defs.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "hash/xhash.h" #include "ips_options/extract.h" #include "ips_options/ips_flowbits.h" @@ -80,82 +79,93 @@ struct detection_option_key_t static inline bool operator==(const struct timeval& a, const struct timeval& b) { return a.tv_sec == b.tv_sec && a.tv_usec == b.tv_usec; } -class DetectionOptionHashKeyOps : public HashKeyOperations +static uint32_t detection_option_hash_func(HashFnc*, const unsigned char* k, int) { -public: - DetectionOptionHashKeyOps(int rows) - : HashKeyOperations(rows) - { } + const detection_option_key_t* key = (const detection_option_key_t*)k; - unsigned do_hash(const unsigned char* k, int) override + if ( key->option_type != RULE_OPTION_TYPE_LEAF_NODE ) { - const detection_option_key_t* key = (const detection_option_key_t*)k; - - if ( key->option_type != RULE_OPTION_TYPE_LEAF_NODE ) - { - IpsOption* opt = (IpsOption*)key->option_data; - return opt->hash(); - } - return 0; + IpsOption* opt = (IpsOption*)key->option_data; + return opt->hash(); } + return 0; +} - bool key_compare(const void* k1, const void* k2, size_t) override - { - const detection_option_key_t* key1 = (const detection_option_key_t*)k1; - const detection_option_key_t* key2 = (const detection_option_key_t*)k2; +static bool detection_option_key_compare_func(const void* k1, const void* k2, size_t) +{ + const detection_option_key_t* key1 = (const detection_option_key_t*)k1; + const detection_option_key_t* key2 = (const detection_option_key_t*)k2; - assert(key1 && key2); + assert(key1 && key2); - if ( key1->option_type != key2->option_type ) - return false; + if ( key1->option_type != key2->option_type ) + return false; - if ( key1->option_type != RULE_OPTION_TYPE_LEAF_NODE ) - { - IpsOption* opt1 = (IpsOption*)key1->option_data; - IpsOption* opt2 = (IpsOption*)key2->option_data; + if ( key1->option_type != RULE_OPTION_TYPE_LEAF_NODE ) + { + IpsOption* opt1 = (IpsOption*)key1->option_data; + IpsOption* opt2 = (IpsOption*)key2->option_data; - if ( *opt1 == *opt2 ) - return true; - } - return false; + if ( *opt1 == *opt2 ) + return true; } -}; + return false; +} -class DetectionOptionHash : public XHash +static int detection_hash_free_func(void* option_key, void*) { -public: + detection_option_key_t* key = (detection_option_key_t*)option_key; - DetectionOptionHash(int rows, int key_len) - : XHash(rows, key_len) + if ( key->option_type != RULE_OPTION_TYPE_LEAF_NODE ) { - initialize(new DetectionOptionHashKeyOps(nrows)); + IpsOption* opt = (IpsOption*)key->option_data; + IpsManager::delete_option(opt); } + return 0; +} - ~DetectionOptionHash() override - { - delete_hash_table(); - } +static XHash* DetectionHashTableNew() +{ + XHash* doht = new XHash(HASH_RULE_OPTIONS, sizeof(detection_option_key_t), + 0, 0, false, nullptr, detection_hash_free_func, true); - void free_user_data(HashNode* hnode) override - { - detection_option_key_t* key = (detection_option_key_t*)hnode->key; + doht->set_key_opcodes(detection_option_hash_func, detection_option_key_compare_func); - if ( key->option_type != RULE_OPTION_TYPE_LEAF_NODE ) - { - IpsOption* opt = (IpsOption*)key->option_data; - IpsManager::delete_option(opt); - } - } -}; + return doht; +} + +void DetectionHashTableFree(XHash* doht) +{ + delete doht; +} + +void* add_detection_option(SnortConfig* sc, option_type_t type, void* option_data) +{ + if ( !sc->detection_option_hash_table ) + sc->detection_option_hash_table = DetectionHashTableNew(); + + detection_option_key_t key; + key.option_type = type; + key.option_data = option_data; + + if ( void* p = sc->detection_option_hash_table->get_user_data(&key) ) + return p; + + sc->detection_option_hash_table->insert(&key, option_data); + return nullptr; +} static uint32_t detection_option_tree_hash(detection_option_tree_node_t* node) { - assert(node); + uint32_t a,b,c; + int i; + + if (!node) + return 0; - uint32_t a, b, c; a = b = c = 0; - for ( int i = 0; i < node->num_children; i++) + for (i=0; inum_children; i++) { #if (defined(__ia64) || defined(__amd64) || defined(_LP64)) { @@ -188,6 +198,19 @@ static uint32_t detection_option_tree_hash(detection_option_tree_node_t* node) return c; } +static uint32_t detection_option_tree_hash_func(HashFnc*, const unsigned char* k, int) +{ + const detection_option_key_t* key = (const detection_option_key_t*)k; + detection_option_tree_node_t* node; + + if (!key || !key->option_data) + return 0; + + node = (detection_option_tree_node_t*)key->option_data; + + return detection_option_tree_hash(node); +} + static bool detection_option_tree_compare( const detection_option_tree_node_t* r, const detection_option_tree_node_t* l) { @@ -209,94 +232,40 @@ static bool detection_option_tree_compare( return true; } -void free_detection_option_tree(detection_option_tree_node_t* node) -{ - for (int i = 0; i < node->num_children; i++) - free_detection_option_tree(node->children[i]); - - snort_free(node->children); - snort_free(node->state); - snort_free(node); -} - -class DetectionOptionTreeHashKeyOps : public HashKeyOperations +static bool detection_option_tree_compare_func(const void* k1, const void* k2, size_t) { -public: - DetectionOptionTreeHashKeyOps(int rows) - : HashKeyOperations(rows) - { } - - unsigned do_hash(const unsigned char* k, int) override - { - assert(k); - const detection_option_key_t* key = (const detection_option_key_t*)k; - if ( !key->option_data ) - return 0; - - detection_option_tree_node_t* node = (detection_option_tree_node_t*)key->option_data; - - return detection_option_tree_hash(node); - } + const detection_option_key_t* key_r = (const detection_option_key_t*)k1; + const detection_option_key_t* key_l = (const detection_option_key_t*)k2; - bool key_compare(const void* k1, const void* k2, size_t) override - { - assert(k1 && k2); - - const detection_option_key_t* key_r = (const detection_option_key_t*)k1; - const detection_option_key_t* key_l = (const detection_option_key_t*)k2; - - const detection_option_tree_node_t* r = (const detection_option_tree_node_t*)key_r->option_data; - const detection_option_tree_node_t* l = (const detection_option_tree_node_t*)key_l->option_data; - - return detection_option_tree_compare(r, l); - } -}; - -class DetectionOptionTreeHash : public XHash -{ -public: - DetectionOptionTreeHash(int rows, int key_len) - : XHash(rows, key_len) - { - initialize(new DetectionOptionTreeHashKeyOps(nrows)); - } - - ~DetectionOptionTreeHash() override - { - delete_hash_table(); - } + if ( !key_r or !key_l ) + return false; - void free_user_data(HashNode* hnode) override - { - free_detection_option_tree((detection_option_tree_node_t*)hnode->data); - } + const detection_option_tree_node_t* r = (const detection_option_tree_node_t*)key_r->option_data; + const detection_option_tree_node_t* l = (const detection_option_tree_node_t*)key_l->option_data; -}; + return detection_option_tree_compare(r, l); +} -static DetectionOptionHash* DetectionHashTableNew() +static int detection_option_tree_free_func(void*, void* data) { - return new DetectionOptionHash(HASH_RULE_OPTIONS, sizeof(detection_option_key_t)); + detection_option_tree_node_t* node = (detection_option_tree_node_t*)data; + free_detection_option_tree(node); + return 0; } -static DetectionOptionTreeHash* DetectionTreeHashTableNew() +void DetectionTreeHashTableFree(XHash* dtht) { - return new DetectionOptionTreeHash(HASH_RULE_TREE, sizeof(detection_option_key_t)); + delete dtht; } -void* add_detection_option(SnortConfig* sc, option_type_t type, void* option_data) +static XHash* DetectionTreeHashTableNew() { - if ( !sc->detection_option_hash_table ) - sc->detection_option_hash_table = DetectionHashTableNew(); + XHash* dtht = new XHash(HASH_RULE_TREE, sizeof(detection_option_key_t), + 0, 0, false, nullptr, detection_option_tree_free_func, true); - detection_option_key_t key; - key.option_type = type; - key.option_data = option_data; - - if ( void* p = sc->detection_option_hash_table->get_user_data(&key) ) - return p; + dtht->set_key_opcodes(detection_option_tree_hash_func, detection_option_tree_compare_func); - sc->detection_option_hash_table->insert(&key, option_data); - return nullptr; + return dtht; } void print_option_tree(detection_option_tree_node_t* node, int level) @@ -558,7 +527,7 @@ int detection_option_node_evaluate( // Back up byte_extract vars so they don't get overwritten between rules trace_log(detection, TRACE_RULE_VARS, "Rule options variables: \n"); - for ( unsigned i = 0; i < NUM_IPS_OPTIONS_VARS; ++i ) + for ( int i = 0; i < NUM_IPS_OPTIONS_VARS; ++i ) { GetVarValueByIndex(&(tmp_byte_extract_vars[i]), (int8_t)i); trace_logf_wo_name(detection, TRACE_RULE_VARS, "var[%d]=%d ", i, @@ -583,7 +552,7 @@ int detection_option_node_evaluate( detection_option_tree_node_t* child_node = node->children[i]; dot_node_state_t* child_state = child_node->state + get_instance_id(); - for ( unsigned j = 0; j < NUM_IPS_OPTIONS_VARS; ++j ) + for ( int j = 0; j < NUM_IPS_OPTIONS_VARS; ++j ) SetVarValueByIndex(tmp_byte_extract_vars[j], (int8_t)j); if ( loop_count > 0 ) @@ -789,7 +758,7 @@ static void detection_option_node_update_otn_stats(detection_option_tree_node_t* if ( node->num_children ) { - for ( int i = 0; i < node->num_children; ++i ) + for ( int i=0; i < node->num_children; ++i ) detection_option_node_update_otn_stats(node->children[i], &local_stats, checks, timeouts, suspends); } @@ -860,3 +829,16 @@ detection_option_tree_node_t* new_node(option_type_t type, void* data) return p; } + +void free_detection_option_tree(detection_option_tree_node_t* node) +{ + int i; + for (i=0; inum_children; i++) + { + free_detection_option_tree(node->children[i]); + } + snort_free(node->children); + snort_free(node->state); + snort_free(node); +} + diff --git a/src/detection/detection_options.h b/src/detection/detection_options.h index f5523cfaa..6f6fdf4a9 100644 --- a/src/detection/detection_options.h +++ b/src/detection/detection_options.h @@ -35,16 +35,15 @@ #include "detection/rule_option_types.h" #include "time/clock_defs.h" -#include "main/snort_debug.h" +#include "main/snort_debug.h" extern Trace TRACE_NAME(detection); namespace snort { -class HashNode; -class XHash; struct Packet; struct SnortConfig; +class XHash; } struct RuleLatencyState; @@ -124,6 +123,9 @@ void* add_detection_option_tree(struct snort::SnortConfig*, detection_option_tre int detection_option_node_evaluate( detection_option_tree_node_t*, detection_option_eval_data_t&, const class Cursor&); +void DetectionHashTableFree(snort::XHash*); +void DetectionTreeHashTableFree(snort::XHash*); + void print_option_tree(detection_option_tree_node_t*, int level); void detection_option_tree_update_otn_stats(snort::XHash*); diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc index 0d933ff0f..9f9fde9f0 100644 --- a/src/detection/fp_create.cc +++ b/src/detection/fp_create.cc @@ -37,7 +37,6 @@ #include "framework/mpse.h" #include "framework/mpse_batch.h" #include "hash/ghash.h" -#include "hash/xhash.h" #include "log/messages.h" #include "main/snort.h" #include "main/snort_config.h" @@ -1661,8 +1660,8 @@ void fpDeleteFastPacketDetection(SnortConfig* sc) return; /* Cleanup the detection option tree */ - delete sc->detection_option_hash_table; - delete sc->detection_option_tree_hash_table; + DetectionHashTableFree(sc->detection_option_hash_table); + DetectionTreeHashTableFree(sc->detection_option_tree_hash_table); fpFreeRuleMaps(sc); ServicePortGroupMapFree(sc->spgmmTable); diff --git a/src/detection/signature.cc b/src/detection/signature.cc index 5d6f8fb4a..bed991f8a 100644 --- a/src/detection/signature.cc +++ b/src/detection/signature.cc @@ -26,7 +26,6 @@ #include "signature.h" -#include "hash/hash_defs.h" #include "hash/ghash.h" #include "log/messages.h" #include "main/snort_config.h" @@ -233,10 +232,10 @@ void OtnLookupAdd(GHash* otn_map, OptTreeNode* otn) key.sid = otn->sigInfo.sid; int status = otn_map->insert(&key, otn); - if ( status == HASH_OK ) + if ( status == GHASH_OK ) return; - assert(status == HASH_INTABLE); + assert(status == GHASH_INTABLE); ParseError("duplicate rule with same gid (%u) and sid (%u)", key.gid, key.sid); } diff --git a/src/detection/tag.cc b/src/detection/tag.cc index 200d31b57..afa2c3b0d 100644 --- a/src/detection/tag.cc +++ b/src/detection/tag.cc @@ -26,7 +26,6 @@ #include "tag.h" #include "events/event.h" -#include "hash/hash_defs.h" #include "hash/xhash.h" #include "log/messages.h" #include "main/snort_config.h" @@ -103,6 +102,11 @@ struct TagNode }; /* G L O B A L S **************************************************/ +static THREAD_LOCAL XHash* host_tag_cache_ptr = nullptr; + +// FIXIT-M utilize Flow instead of separate cache +static THREAD_LOCAL XHash* ssn_tag_cache_ptr = nullptr; + static THREAD_LOCAL uint32_t last_prune_time = 0; static THREAD_LOCAL uint32_t tag_alloc_faults = 0; static THREAD_LOCAL uint32_t tag_memory_usage = 0; @@ -115,10 +119,11 @@ static THREAD_LOCAL unsigned s_sessions = 0; // (consecutive) sessions to be captured. static const unsigned s_max_sessions = 1; - /* P R O T O T Y P E S ********************************************/ static TagNode* TagAlloc(XHash*); static void TagFree(XHash*, TagNode*); +static int TagFreeSessionNodeFunc(void* key, void* data); +static int TagFreeHostNodeFunc(void* key, void* data); static int PruneTagCache(uint32_t, int); static int PruneTime(XHash* tree, uint32_t thetime); static void TagSession(Packet*, TagData*, uint32_t, uint16_t, void*); @@ -126,56 +131,6 @@ static void TagHost(Packet*, TagData*, uint32_t, uint16_t, void*); static void AddTagNode(Packet*, TagData*, int, uint32_t, uint16_t, void*); static inline void SwapTag(TagNode*); -class TagSessionCache : public XHash -{ -public: - TagSessionCache(int rows, int key_len) - : XHash(rows, key_len) - { - initialize(); - anr_enabled = false; - recycle_nodes = false; - } - - ~TagSessionCache() override - { - delete_hash_table(); - } - - void free_user_data(HashNode* hnode) override - { - TagFree(this, (TagNode*)hnode->data); - } -}; - -class TagHostCache : public XHash -{ -public: - TagHostCache(int rows, int key_len) - : XHash(rows, key_len) - { - initialize(); - anr_enabled = false; - recycle_nodes = false; - } - - ~TagHostCache() override - { - delete_hash_table(); - } - - void free_user_data(HashNode* hnode) override - { - TagFree(this, (TagNode*)hnode->data); - } -}; - -static THREAD_LOCAL TagHostCache* host_tag_cache = nullptr; - -// FIXIT-M utilize Flow instead of separate cache -static THREAD_LOCAL TagSessionCache* ssn_tag_cache = nullptr; - - /**Calculated memory needed per node insertion into respective cache. Its includes * memory needed for allocating TagNode, HashNode and key size. * @@ -186,9 +141,9 @@ static THREAD_LOCAL TagSessionCache* ssn_tag_cache = nullptr; */ static inline unsigned int memory_per_node(XHash* hash) { - if ( hash == ssn_tag_cache ) + if ( hash == ssn_tag_cache_ptr ) return sizeof(tTagFlowKey) + sizeof(HashNode) + sizeof(TagNode); - else if ( hash == host_tag_cache ) + else if ( hash == host_tag_cache_ptr ) return sizeof(SfIp) + sizeof(HashNode) + sizeof(TagNode); return 0; @@ -263,6 +218,28 @@ static void TagFree( tag_memory_usage -= memory_per_node(hash); } +/**Callback from session tag cache to free user data. + * @param key - pointer to key to session tag + * @param data - pointer to user data, to be freed. + * @returns 0 + */ +static int TagFreeSessionNodeFunc(void*, void* data) +{ + TagFree(ssn_tag_cache_ptr, (TagNode*)data); + return 0; +} + +/**Callback from host tag cache to free user data. + * @param key - pointer to key to session tag + * @param data - pointer to user data, to be freed. + * @returns 0 + */ +static int TagFreeHostNodeFunc(void*, void* data) +{ + TagFree(host_tag_cache_ptr, (TagNode*)data); + return 0; +} + /** * swap the sips and dips, dp's and sp's * @@ -286,14 +263,17 @@ void InitTag() { unsigned int hashTableSize = TAG_MEMCAP/sizeof(TagNode); - ssn_tag_cache = new TagSessionCache(hashTableSize, sizeof(tTagFlowKey)); - host_tag_cache = new TagHostCache(hashTableSize, sizeof(SfIp)); + ssn_tag_cache_ptr = new XHash(hashTableSize, sizeof(tTagFlowKey), 0, 0, + false, nullptr, TagFreeSessionNodeFunc, false); + + host_tag_cache_ptr = new XHash(hashTableSize, sizeof(SfIp), 0, 0, false, + nullptr, TagFreeHostNodeFunc, false); } void CleanupTag() { - delete ssn_tag_cache; - delete host_tag_cache; + delete ssn_tag_cache_ptr; + delete host_tag_cache_ptr; } static void TagSession(Packet* p, TagData* tag, uint32_t time, uint16_t event_id, void* log_list) @@ -343,11 +323,11 @@ static void AddTagNode(Packet* p, TagData* tag, int mode, uint32_t now, } if (mode == TAG_SESSION) { - tag_cache_ptr = ssn_tag_cache; + tag_cache_ptr = ssn_tag_cache_ptr; } else { - tag_cache_ptr = host_tag_cache; + tag_cache_ptr = host_tag_cache_ptr; } idx = TagAlloc(tag_cache_ptr); @@ -436,7 +416,7 @@ int CheckTagList(Packet* p, Event& event, void** log_list) char create_event = 1; /* check for active tags */ - if (!host_tag_cache->get_num_nodes() && !ssn_tag_cache->get_num_nodes()) + if (!host_tag_cache_ptr->get_node_count() && !ssn_tag_cache_ptr->get_node_count()) { return 0; } @@ -452,7 +432,7 @@ int CheckTagList(Packet* p, Event& event, void** log_list) idx.key.dp = p->ptrs.dp; /* check for session tags... */ - returned = (TagNode*)ssn_tag_cache->get_user_data(&idx); + returned = (TagNode*)ssn_tag_cache_ptr->get_user_data(&idx); if (returned == nullptr) { @@ -461,11 +441,11 @@ int CheckTagList(Packet* p, Event& event, void** log_list) idx.key.dp = p->ptrs.sp; idx.key.sp = p->ptrs.dp; - returned = (TagNode*)ssn_tag_cache->get_user_data(&idx); + returned = (TagNode*)ssn_tag_cache_ptr->get_user_data(&idx); if (returned == nullptr) { - returned = (TagNode*)host_tag_cache->get_user_data(&idx); + returned = (TagNode*)host_tag_cache_ptr->get_user_data(&idx); if (returned == nullptr) { @@ -475,22 +455,22 @@ int CheckTagList(Packet* p, Event& event, void** log_list) */ idx.key.sip = *p->ptrs.ip_api.get_src(); - returned = (TagNode*)host_tag_cache->get_user_data(&idx); + returned = (TagNode*)host_tag_cache_ptr->get_user_data(&idx); } if (returned != nullptr) { - taglist = host_tag_cache; + taglist = host_tag_cache_ptr; } } else { - taglist = ssn_tag_cache; + taglist = ssn_tag_cache_ptr; } } else { - taglist = ssn_tag_cache; + taglist = ssn_tag_cache_ptr; } if (returned != nullptr) @@ -553,7 +533,7 @@ int CheckTagList(Packet* p, Event& event, void** log_list) if ( !returned->metric ) { - if (taglist->release_node(&returned->key) != HASH_OK) + if (taglist->release_node(returned) != HASH_OK) { LogMessage("WARNING: failed to remove tagNode from hash.\n"); } @@ -578,26 +558,35 @@ static int PruneTagCache(uint32_t thetime, int mustdie) if (mustdie == 0) { - if (ssn_tag_cache->get_num_nodes() != 0) - pruned = PruneTime(ssn_tag_cache, thetime); + if (ssn_tag_cache_ptr->get_node_count() != 0) + pruned = PruneTime(ssn_tag_cache_ptr, thetime); - if (host_tag_cache->get_num_nodes() != 0) - pruned += PruneTime(host_tag_cache, thetime); + if (host_tag_cache_ptr->get_node_count() != 0) + pruned += PruneTime(host_tag_cache_ptr, thetime); } else { while (pruned < mustdie && - (ssn_tag_cache->get_num_nodes() > 0 || host_tag_cache->get_num_nodes() > 0)) + (ssn_tag_cache_ptr->get_node_count() > 0 || host_tag_cache_ptr->get_node_count() > 0)) { - if ( ssn_tag_cache->delete_lru_node() ) - ++pruned; - else - LogMessage("WARNING: failed to remove tagNode from ssn hash.\n"); + TagNode* lru_node; - if ( host_tag_cache->delete_lru_node() ) - ++pruned; - else - LogMessage("WARNING: failed to remove tagNode from host hash.\n"); + if ((lru_node = (TagNode*)ssn_tag_cache_ptr->get_lru_user_data()) != nullptr) + { + if (ssn_tag_cache_ptr->release_node(lru_node) != HASH_OK) + { + LogMessage("WARNING: failed to remove tagNode from hash.\n"); + } + pruned++; + } + if ((lru_node = (TagNode*)host_tag_cache_ptr->get_lru_user_data()) != nullptr) + { + if (host_tag_cache_ptr->release_node(lru_node) != HASH_OK) + { + LogMessage("WARNING: failed to remove tagNode from hash.\n"); + } + pruned++; + } } } @@ -613,7 +602,7 @@ static int PruneTime(XHash* tree, uint32_t thetime) { if ((lru_node->last_access + TAG_PRUNE_QUANTUM) < thetime) { - if (tree->release_node(&lru_node->key) != HASH_OK) + if (tree->release_node(lru_node) != HASH_OK) { LogMessage("WARNING: failed to remove tagNode from hash.\n"); } diff --git a/src/file_api/file_cache.cc b/src/file_api/file_cache.cc index 357e3874f..1f435900a 100644 --- a/src/file_api/file_cache.cc +++ b/src/file_api/file_cache.cc @@ -23,7 +23,6 @@ #include "file_cache.h" -#include "hash/hash_defs.h" #include "hash/xhash.h" #include "log/messages.h" #include "main/snort_config.h" @@ -38,39 +37,35 @@ using namespace snort; -class ExpectedFileCache : public XHash +static int file_cache_anr_free_func(void*, void* data) { -public: - ExpectedFileCache(unsigned rows, unsigned key_len, unsigned datasize) - : XHash(rows, key_len, datasize, 0) - { } + FileCache::FileNode* node = (FileCache::FileNode*)data; - ~ExpectedFileCache() override - { - delete_hash_table(); - } + if (!node) + return 0; - bool is_node_recovery_ok(HashNode* hnode) override - { - FileCache::FileNode* node = (FileCache::FileNode*)hnode->data; - if ( !node ) - return true; + struct timeval now; + packet_gettimeofday(&now); - struct timeval now; - packet_gettimeofday(&now); - if ( timercmp(&node->cache_expire_time, &now, <) ) - return true; - else - return false; + // only recycle expired nodes + if (timercmp(&node->cache_expire_time, &now, <)) + { + delete node->file; + return 0; } + else + return 1; +} - void free_user_data(HashNode* hnode) override +static int file_cache_free_func(void*, void* data) +{ + FileCache::FileNode* node = (FileCache::FileNode*)data; + if (node) { - FileCache::FileNode* node = (FileCache::FileNode*)hnode->data; - if ( node ) - delete node->file; + delete node->file; } -}; + return 0; +} // Return the time in ms since we started waiting for pending file lookup. static int64_t time_elapsed_ms(struct timeval* now, struct timeval* expire_time, int64_t lookup_timeout) @@ -84,7 +79,8 @@ static int64_t time_elapsed_ms(struct timeval* now, struct timeval* expire_time, FileCache::FileCache(int64_t max_files_cached) { max_files = max_files_cached; - fileHash = new ExpectedFileCache(max_files, sizeof(FileHashKey), sizeof(FileNode)); + fileHash = new XHash(max_files, sizeof(FileHashKey), sizeof(FileNode), + 0, true, file_cache_anr_free_func, file_cache_free_func, true); fileHash->set_max_nodes(max_files); } @@ -159,15 +155,18 @@ FileContext* FileCache::find(const FileHashKey& hashKey, int64_t timeout) { std::lock_guard lock(cache_mutex); - if ( !fileHash->get_num_nodes() ) + if (!fileHash->get_node_count()) + { return nullptr; + } HashNode* hash_node = fileHash->find_node(&hashKey); - if ( !hash_node ) + + if (!hash_node) return nullptr; FileNode* node = (FileNode*)hash_node->data; - if ( !node ) + if (!node) { fileHash->release_node(hash_node); return nullptr; @@ -176,7 +175,7 @@ FileContext* FileCache::find(const FileHashKey& hashKey, int64_t timeout) struct timeval now; packet_gettimeofday(&now); - if ( timercmp(&node->cache_expire_time, &now, <) ) + if (timercmp(&node->cache_expire_time, &now, <)) { fileHash->release_node(hash_node); return nullptr; diff --git a/src/file_api/file_cache.h b/src/file_api/file_cache.h index 4c56cf10d..1ee25d89d 100644 --- a/src/file_api/file_cache.h +++ b/src/file_api/file_cache.h @@ -28,7 +28,10 @@ #include "file_config.h" -class ExpectedFileCache; +namespace snort +{ +class XHash; +} class FileCache { @@ -71,7 +74,7 @@ private: int store_verdict(snort::Flow*, snort::FileInfo*, int64_t timeout); /* The hash table of expected files */ - ExpectedFileCache* fileHash = nullptr; + snort::XHash* fileHash = nullptr; int64_t block_timeout = DEFAULT_FILE_BLOCK_TIMEOUT; int64_t lookup_timeout = DEFAULT_FILE_LOOKUP_TIMEOUT; int64_t max_files = DEFAULT_MAX_FILES_CACHED; diff --git a/src/file_api/file_identifier.cc b/src/file_api/file_identifier.cc index 5796f101b..2e7143ff5 100644 --- a/src/file_api/file_identifier.cc +++ b/src/file_api/file_identifier.cc @@ -32,7 +32,6 @@ #include #include -#include "hash/ghash.h" #include "log/messages.h" #include "utils/util.h" diff --git a/src/file_api/file_identifier.h b/src/file_api/file_identifier.h index 81c621ae3..c698db991 100644 --- a/src/file_api/file_identifier.h +++ b/src/file_api/file_identifier.h @@ -29,12 +29,9 @@ #include #include -#include "file_lib.h" +#include "hash/ghash.h" -namespace snort -{ -class GHash; -} +#include "file_lib.h" #define MAX_BRANCH (UINT8_MAX + 1) diff --git a/src/file_api/file_module.cc b/src/file_api/file_module.cc index 58be9bf45..4819c388d 100644 --- a/src/file_api/file_module.cc +++ b/src/file_api/file_module.cc @@ -290,7 +290,7 @@ bool FileIdModule::set(const char*, Value& v, SnortConfig*) { if (Snort::is_reloading() && !FileService::is_file_capture_enabled()) { - ReloadError("Enabling file_id.enable_capture requires a restart\n"); + ReloadError("Enabling file capture requires a restart\n"); return false; } fp.set_file_capture(true); @@ -388,7 +388,7 @@ bool FileIdModule::set(const char*, Value& v, SnortConfig*) if (file_rule.use.capture_enabled && Snort::is_reloading() && !FileService::is_file_capture_enabled()) { - ReloadError("Enabling file_id.enable_file_capture requires a restart\n"); + ReloadError("Enabling file capture requires a restart\n"); return false; } } diff --git a/src/file_api/file_service.cc b/src/file_api/file_service.cc index 7aaaf7bdd..d8a53a57a 100644 --- a/src/file_api/file_service.cc +++ b/src/file_api/file_service.cc @@ -87,14 +87,14 @@ void FileService::verify_reload(SnortConfig* sc) return; if (max_files_cached != conf->max_files_cached) - ReloadError("Changing file_id.max_files_cached requires a restart\n"); + ReloadError("Changing file_id:max_files_cached requires a restart\n"); if (file_capture_enabled) { if (capture_memcap != conf->capture_memcap) - ReloadError("Changing file_id.capture_memcap requires a restart\n"); + ReloadError("Changing file_id:capture_memcap requires a restart\n"); if (capture_block_size != conf->capture_block_size) - ReloadError("Changing file_id.capture_block_size requires a restart\n"); + ReloadError("Changing file_id:capture_block_size requires a restart\n"); } } diff --git a/src/filters/sfrf.cc b/src/filters/sfrf.cc index 6e47a7ef5..a985f6cae 100644 --- a/src/filters/sfrf.cc +++ b/src/filters/sfrf.cc @@ -29,7 +29,6 @@ #include "main/thread.h" #include "detection/rules.h" #include "hash/ghash.h" -#include "hash/hash_defs.h" #include "hash/xhash.h" #include "sfip/sf_ip.h" #include "sfip/sf_ipvar.h" @@ -147,12 +146,14 @@ static void SFRF_New(unsigned nbytes) /* Calc max ip nodes for this memory */ if ( nbytes < SFRF_BYTES ) + { nbytes = SFRF_BYTES; - + } nrows = nbytes / (SFRF_BYTES); /* Create global hash table for all of the IP Nodes */ - rf_hash = new XHash(nrows, sizeof(tSFRFTrackingNodeKey), sizeof(tSFRFTrackingNode), nbytes); + rf_hash = new XHash(nrows, sizeof(tSFRFTrackingNodeKey), + sizeof(tSFRFTrackingNode), nbytes, true, nullptr, nullptr, true); } void SFRF_Delete() @@ -167,7 +168,7 @@ void SFRF_Delete() void SFRF_Flush() { if ( rf_hash ) - rf_hash->clear_hash(); + rf_hash->clear(); } static void SFRF_ConfigNodeFree(void* item) @@ -781,24 +782,30 @@ static tSFRFTrackingNode* _getSFRFTrackingNode(const SfIp* ip, unsigned tid, tim key.policyId = get_ips_policy()->policy_id; key.padding = 0; - // Check for any Permanent sid objects for this gid or add this one ... - if ( rf_hash->insert(&key, nullptr) == HASH_NOMEM ) + /* + * Check for any Permanent sid objects for this gid or add this one ... + */ + HashNode* hnode = rf_hash->get_node((void*)&key); + if ( !hnode ) { // xhash_get_node fails to insert only if rf_hash is full. rate_filter_stats.xhash_nomem_peg++; return dynNode; } - dynNode = (tSFRFTrackingNode*)rf_hash->get_user_data(); - if ( dynNode->filterState == FS_NEW ) + if ( hnode->data ) { - // first time initialization - dynNode->tstart = curTime; + dynNode = (tSFRFTrackingNode*)hnode->data; + + if ( dynNode->filterState == FS_NEW ) + { + // first time initialization + dynNode->tstart = curTime; #ifdef SFRF_OVER_RATE - dynNode->tlast = curTime; + dynNode->tlast = curTime; #endif - dynNode->filterState = FS_OFF; + dynNode->filterState = FS_OFF; + } } - return dynNode; } diff --git a/src/filters/sfthd.cc b/src/filters/sfthd.cc index 6eaaa5d2d..b83ba0fa5 100644 --- a/src/filters/sfthd.cc +++ b/src/filters/sfthd.cc @@ -36,7 +36,6 @@ #include #include "hash/ghash.h" -#include "hash/hash_defs.h" #include "hash/xhash.h" #include "main/thread.h" #include "sfip/sf_ipvar.h" @@ -61,7 +60,7 @@ XHash* sfthd_new_hash(unsigned nbytes, size_t key, size_t data) nbytes = size; nrows = nbytes / size; - return new XHash(nrows, key, data, nbytes); + return new XHash(nrows, key, data, nbytes, true, nullptr, nullptr, true); } /*! @@ -892,7 +891,7 @@ int sfthd_test_local( if (status == HASH_INTABLE) { /* Already in the table */ - sfthd_ip_node = (THD_IP_NODE*)local_hash->get_user_data(); + sfthd_ip_node = (THD_IP_NODE*)local_hash->get_cnode()->data; /* Increment the event count */ sfthd_ip_node->count++; @@ -990,7 +989,7 @@ static inline int sfthd_test_global( if (status == HASH_INTABLE) { /* Already in the table */ - sfthd_ip_node = (THD_IP_NODE*)global_hash->get_user_data(); + sfthd_ip_node = (THD_IP_NODE*)global_hash->get_cnode()->data; /* Increment the event count */ sfthd_ip_node->count++; diff --git a/src/flow/expect_cache.cc b/src/flow/expect_cache.cc index d63b6c6a4..69873cc46 100644 --- a/src/flow/expect_cache.cc +++ b/src/flow/expect_cache.cc @@ -134,7 +134,7 @@ void ExpectCache::prune() for (unsigned i = 0; i < MAX_PRUNE; ++i ) { - ExpectNode* node = (ExpectNode*)hash_table->lru_first(); + ExpectNode* node = (ExpectNode*)hash_table->first(); if ( !node || now <= node->expires ) break; @@ -147,7 +147,7 @@ void ExpectCache::prune() ExpectNode* ExpectCache::find_node_by_packet(Packet* p, FlowKey &key) { - if (!hash_table->get_num_nodes()) + if (!hash_table->get_count()) return nullptr; const SfIp* srcIP = p->ptrs.ip_api.get_src(); @@ -171,7 +171,7 @@ ExpectNode* ExpectCache::find_node_by_packet(Packet* p, FlowKey &key) */ // FIXIT-P X This should be optimized to only do full matches when full keys // are present, likewise for partial keys. - ExpectNode* node = (ExpectNode*) hash_table->get_user_data(&key); + ExpectNode* node = (ExpectNode*) hash_table->find(&key); if (!node) { // FIXIT-M X This logic could fail if IPs were equal because the original key @@ -192,12 +192,12 @@ ExpectNode* ExpectCache::find_node_by_packet(Packet* p, FlowKey &key) port2 = key.port_h; key.port_h = 0; } - node = (ExpectNode*) hash_table->get_user_data(&key); + node = (ExpectNode*) hash_table->find(&key); if (!node) { key.port_l = port1; key.port_h = port2; - node = (ExpectNode*) hash_table->get_user_data(&key); + node = (ExpectNode*) hash_table->find(&key); if (!node) return nullptr; } @@ -206,7 +206,7 @@ ExpectNode* ExpectCache::find_node_by_packet(Packet* p, FlowKey &key) { if (node->head) node->clear(free_list); - hash_table->release_node(&key); + hash_table->release(&key); return nullptr; } /* Make sure the packet direction is correct */ @@ -257,7 +257,7 @@ bool ExpectCache::process_expected(ExpectNode* node, FlowKey& key, Packet* p, Fl lws->ssn_state.snort_protocol_id = node->snort_protocol_id; if (!node->count) - hash_table->release_node(&key); + hash_table->release(&key); return ignoring; } @@ -270,9 +270,11 @@ ExpectCache::ExpectCache(uint32_t max) { // -size forces use of abs(size) ie w/o bumping up hash_table = new ZHash(-MAX_HASH, sizeof(FlowKey)); + hash_table->set_key_opcodes(FlowKey::hash, FlowKey::is_equal); + nodes = new ExpectNode[max]; for (unsigned i = 0; i < max; ++i) - hash_table->push(nodes + i); + hash_table->push(nodes+i); /* Preallocate a pool of ExpectFlows big enough to handle the worst case requirement (max number of nodes * max flows per node) and add them all @@ -330,44 +332,54 @@ int ExpectCache::add_flow(const Packet *ctrlPkt, PktType type, IpProtocol ip_pro uint16_t vlanId = (ctrlPkt->proto_bits & PROTO_BIT__VLAN) ? layer::get_vlan_layer(ctrlPkt)->vid() : 0; uint32_t mplsId = (ctrlPkt->proto_bits & PROTO_BIT__MPLS) ? ctrlPkt->ptrs.mplsHdr.label : 0; uint16_t addressSpaceId = ctrlPkt->pkth->address_space_id; + FlowKey key; bool reversed_key = key.init(type, ip_proto, cliIP, cliPort, srvIP, srvPort, vlanId, mplsId, addressSpaceId); bool new_node = false; - ExpectNode* node = (ExpectNode*) hash_table->get_user_data(&key); - if ( !node ) + ExpectNode* node = (ExpectNode*) hash_table->get(&key, &new_node); + if (!node) { prune(); - node = (ExpectNode*) hash_table->get(&key); - assert(node); - new_node = true; + node = (ExpectNode*) hash_table->get(&key, &new_node); + /* The flow free list should never be empty if there was a node + to be (re-)used unless we managed to leak some. Check just + in case. Maybe assert instead? */ + if (!node || !free_list) + { + ++overflows; + return -1; + } } - else if ( packet_time() > node->expires ) + + /* If the node is past its expiration date, whack it and reuse it. */ + if (!new_node && packet_time() > node->expires) { - // node is past its expiration date, whack it and reuse it. node->clear(free_list); new_node = true; } ExpectFlow* last = nullptr; - if ( !new_node ) + if (!new_node) { - // reject if the snort_protocol_id doesn't match - if ( node->snort_protocol_id != snort_protocol_id ) + // Requests will be rejected if the snort_protocol_id doesn't + // match what has already been set. + if (node->snort_protocol_id != snort_protocol_id) { - if ( node->snort_protocol_id && snort_protocol_id ) + if (node->snort_protocol_id && snort_protocol_id) return -1; node->snort_protocol_id = snort_protocol_id; } last = node->tail; - if ( last ) + if (last) { FlowData* lfd = last->data; - while ( lfd ) + + while (lfd) { - if ( lfd->get_id() == fd->get_id() ) + if (lfd->get_id() == fd->get_id()) { last = nullptr; break; @@ -391,9 +403,9 @@ int ExpectCache::add_flow(const Packet *ctrlPkt, PktType type, IpProtocol ip_pro } bool new_expect_flow = false; - if ( !last ) + if (!last) { - if ( node->count >= MAX_LIST ) + if (node->count >= MAX_LIST) { // fail when maxed out ++overflows; @@ -402,7 +414,7 @@ int ExpectCache::add_flow(const Packet *ctrlPkt, PktType type, IpProtocol ip_pro last = free_list; free_list = free_list->next; - if ( !node->tail ) + if (!node->tail) node->head = last; else node->tail->next = last; @@ -415,7 +427,7 @@ int ExpectCache::add_flow(const Packet *ctrlPkt, PktType type, IpProtocol ip_pro last->add_flow_data(fd); node->expires = packet_time() + MAX_WAIT; ++expects; - if ( new_expect_flow ) + if (new_expect_flow) { // chain all expected flows created by this packet packet_expect_flows->emplace_back(last); diff --git a/src/flow/flow_cache.cc b/src/flow/flow_cache.cc index 4054422f9..b7ef59dc9 100644 --- a/src/flow/flow_cache.cc +++ b/src/flow/flow_cache.cc @@ -53,8 +53,11 @@ static const unsigned ALL_FLOWS = 3; FlowCache::FlowCache(const FlowCacheConfig& cfg) : config(cfg) { hash_table = new ZHash(config.max_flows, sizeof(FlowKey)); + hash_table->set_key_opcodes(FlowKey::hash, FlowKey::is_equal); + uni_flows = new FlowUniList; uni_ip_flows = new FlowUniList; + flags = 0x0; assert(prune_stats.get_total() == 0); @@ -76,12 +79,12 @@ void FlowCache::push(Flow* flow) unsigned FlowCache::get_count() { - return hash_table ? hash_table->get_num_nodes() : 0; + return hash_table ? hash_table->get_count() : 0; } Flow* FlowCache::find(const FlowKey* key) { - Flow* flow = (Flow*)hash_table->get_user_data(key); + Flow* flow = (Flow*)hash_table->find(key); if ( flow ) { @@ -116,6 +119,7 @@ Flow* FlowCache::allocate(const FlowKey* key) { time_t timestamp = packet_time(); Flow* flow = (Flow*)hash_table->get(key); + if ( !flow ) { if ( flows_allocated < config.max_flows ) @@ -156,7 +160,7 @@ void FlowCache::remove(Flow* flow) // FIXIT-M This check is added for offload case where both Flow::reset // and Flow::retire try remove the flow from hash. Flow::reset should // just mark the flow as pending instead of trying to remove it. - if ( hash_table->release_node(flow->key) ) + if ( hash_table->release(flow->key) ) memory::MemoryCap::update_deallocations(config.proto[to_utype(flow->key->pkt_type)].cap_weight); } @@ -180,7 +184,7 @@ unsigned FlowCache::prune_stale(uint32_t thetime, const Flow* save_me) ActiveSuspendContext act_susp; unsigned pruned = 0; - auto flow = static_cast(hash_table->lru_first()); + auto flow = static_cast(hash_table->first()); while ( flow and pruned <= cleanup_flows ) { @@ -192,7 +196,7 @@ unsigned FlowCache::prune_stale(uint32_t thetime, const Flow* save_me) if ( hash_table->get_count() == 1 ) break; - hash_table->lru_touch(); + hash_table->touch(); } #else // Reached the current flow. This *should* be the newest flow @@ -209,7 +213,7 @@ unsigned FlowCache::prune_stale(uint32_t thetime, const Flow* save_me) release(flow, PruneReason::IDLE); ++pruned; - flow = static_cast(hash_table->lru_first()); + flow = static_cast(hash_table->first()); } return pruned; @@ -258,11 +262,11 @@ unsigned FlowCache::prune_excess(const Flow* save_me) // initially skip offloads but if that doesn't work the hash table is iterated from the // beginning again. prune offloads at that point. - unsigned ignore_offloads = hash_table->get_num_nodes(); + unsigned ignore_offloads = hash_table->get_count(); - while ( hash_table->get_num_nodes() > max_cap and hash_table->get_num_nodes() > blocks ) + while ( hash_table->get_count() > max_cap and hash_table->get_count() > blocks ) { - auto flow = static_cast(hash_table->lru_first()); + auto flow = static_cast(hash_table->first()); assert(flow); // holds true because hash_table->get_count() > 0 if ( (save_me and flow == save_me) or flow->was_blocked() or @@ -275,7 +279,8 @@ unsigned FlowCache::prune_excess(const Flow* save_me) // FIXIT-M we should update last_data_seen upon touch to ensure // the hash_table LRU list remains sorted by time - hash_table->lru_touch(); + if ( !hash_table->touch() ) + break; } else { @@ -287,7 +292,7 @@ unsigned FlowCache::prune_excess(const Flow* save_me) --ignore_offloads; } - if (!pruned and hash_table->get_num_nodes() > max_cap) + if (!pruned and hash_table->get_count() > max_cap) { prune_one(PruneReason::EXCESS, true); ++pruned; @@ -299,11 +304,11 @@ unsigned FlowCache::prune_excess(const Flow* save_me) bool FlowCache::prune_one(PruneReason reason, bool do_cleanup) { // so we don't prune the current flow (assume current == MRU) - if ( hash_table->get_num_nodes() <= 1 ) + if ( hash_table->get_count() <= 1 ) return false; // ZHash returns in LRU order, which is updated per packet via find --> move_to_front call - auto flow = static_cast(hash_table->lru_first()); + auto flow = static_cast(hash_table->first()); assert(flow); flow->ssn_state.session_flags |= SSNFLAG_PRUNED; @@ -317,10 +322,10 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime) ActiveSuspendContext act_susp; unsigned retired = 0; - auto flow = static_cast(hash_table->lru_current()); + auto flow = static_cast(hash_table->current()); if ( !flow ) - flow = static_cast(hash_table->lru_first()); + flow = static_cast(hash_table->first()); while ( flow and (retired < num_flows) ) { @@ -335,7 +340,7 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime) if ( HighAvailabilityManager::in_standby(flow) or flow->is_suspended() ) { - flow = static_cast(hash_table->lru_next()); + flow = static_cast(hash_table->next()); continue; } @@ -344,7 +349,7 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime) ++retired; - flow = static_cast(hash_table->lru_current()); + flow = static_cast(hash_table->current()); } return retired; @@ -352,20 +357,22 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime) unsigned FlowCache::delete_active_flows(unsigned mode, unsigned num_to_delete, unsigned &deleted) { - unsigned flows_to_check = hash_table->get_num_nodes(); + unsigned flows_to_check = hash_table->get_count(); while ( num_to_delete && flows_to_check-- ) { - auto flow = static_cast(hash_table->lru_first()); + auto flow = static_cast(hash_table->first()); assert(flow); if ( (mode == ALLOWED_FLOWS_ONLY and (flow->was_blocked() || flow->is_suspended())) or (mode == OFFLOADED_FLOWS_TOO and flow->was_blocked()) ) { - hash_table->lru_touch(); + if ( !hash_table->touch() ) + break; + continue; } // we have a winner... - hash_table->remove(); + hash_table->remove(flow->key); if ( flow->next ) unlink_uni(flow); @@ -420,7 +427,7 @@ unsigned FlowCache::purge() unsigned retired = 0; - while ( auto flow = static_cast(hash_table->lru_first()) ) + while ( auto flow = static_cast(hash_table->first()) ) { retire(flow); ++retired; diff --git a/src/flow/flow_key.cc b/src/flow/flow_key.cc index b0c10fa87..72575eb1d 100644 --- a/src/flow/flow_key.cc +++ b/src/flow/flow_key.cc @@ -24,7 +24,7 @@ #include "flow/flow_key.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "main/snort_config.h" #include "protocols/icmp4.h" #include "protocols/icmp6.h" @@ -297,10 +297,46 @@ bool FlowKey::init( // hash foo //------------------------------------------------------------------------- +uint32_t FlowKey::hash(HashFnc* hf, const unsigned char* p, int) +{ + uint32_t a, b, c; + a = b = c = hf->hardener; + + const uint32_t* d = (const uint32_t*)p; + + a += d[0]; // IPv6 lo[0] + b += d[1]; // IPv6 lo[1] + c += d[2]; // IPv6 lo[2] + + mix(a, b, c); + + a += d[3]; // IPv6 lo[3] + b += d[4]; // IPv6 hi[0] + c += d[5]; // IPv6 hi[1] + + mix(a, b, c); + + a += d[6]; // IPv6 hi[2] + b += d[7]; // IPv6 hi[3] + c += d[8]; // mpls label + + mix(a, b, c); + + a += d[9]; // port lo & port hi + b += d[10]; // vlan tag, address space id + c += d[11]; // ip_proto, pkt_type, version, and 8 bits of zeroed pad + + finalize(a, b, c); + + return c; +} + bool FlowKey::is_equal(const void* s1, const void* s2, size_t) { - const uint64_t* a = (const uint64_t*)s1; - const uint64_t* b = (const uint64_t*)s2; + const uint64_t* a,* b; + + a = (const uint64_t*)s1; + b = (const uint64_t*)s2; if (*a - *b) return false; /* Compares IPv4 lo/hi Compares IPv6 low[0,1] */ @@ -334,43 +370,3 @@ bool FlowKey::is_equal(const void* s1, const void* s2, size_t) return true; } -unsigned FlowHashKeyOps::do_hash(const unsigned char* k, int) -{ - uint32_t a, b, c; - a = b = c = hardener; - - const uint32_t* d = (const uint32_t*)k; - - a += d[0]; // IPv6 lo[0] - b += d[1]; // IPv6 lo[1] - c += d[2]; // IPv6 lo[2] - - mix(a, b, c); - - a += d[3]; // IPv6 lo[3] - b += d[4]; // IPv6 hi[0] - c += d[5]; // IPv6 hi[1] - - mix(a, b, c); - - a += d[6]; // IPv6 hi[2] - b += d[7]; // IPv6 hi[3] - c += d[8]; // mpls label - - mix(a, b, c); - - a += d[9]; // port lo & port hi - b += d[10]; // vlan tag, address space id - c += d[11]; // ip_proto, pkt_type, version, and 8 bits of zeroed pad - - finalize(a, b, c); - - return c; -} - -bool FlowHashKeyOps::key_compare(const void* k1, const void* k2, size_t len) -{ - return FlowKey::is_equal(k1, k2, len); -} - - diff --git a/src/flow/flow_key.h b/src/flow/flow_key.h index c85129dc0..4d9fb4ad8 100644 --- a/src/flow/flow_key.h +++ b/src/flow/flow_key.h @@ -26,27 +26,14 @@ #include #include "framework/decode_data.h" -#include "hash/hash_key_operations.h" #include "utils/cpp_macros.h" -class HashKeyOperations; +struct HashFnc; namespace snort { struct SfIp; -class FlowHashKeyOps : public HashKeyOperations -{ -public: - FlowHashKeyOps(int rows) - : HashKeyOperations(rows) - { } - - unsigned do_hash(const unsigned char* k, int len) override; - bool key_compare(const void* k1, const void* k2, size_t) override; -}; - - PADDING_GUARD_BEGIN struct SO_PUBLIC FlowKey { @@ -83,7 +70,8 @@ struct SO_PUBLIC FlowKey void init_address_space(uint16_t); // If this data structure changes size, compare must be updated! - static bool is_equal(const void* k1, const void* k2, size_t); + static uint32_t hash(HashFnc*, const unsigned char* d, int); + static bool is_equal(const void* s1, const void* s2, size_t); private: bool init4( diff --git a/src/flow/test/CMakeLists.txt b/src/flow/test/CMakeLists.txt index df99e24f7..2cb88afba 100644 --- a/src/flow/test/CMakeLists.txt +++ b/src/flow/test/CMakeLists.txt @@ -13,11 +13,9 @@ add_cpputest( flow_cache_test ../flow_cache.cc ../flow_control.cc ../flow_key.cc - ../../hash/hash_key_operations.cc - ../../hash/hash_lru_cache.cc - ../../hash/primetable.cc - ../../hash/xhash.cc ../../hash/zhash.cc + ../../hash/hashfcn.cc + ../../hash/primetable.cc ) add_cpputest( session_test ) diff --git a/src/framework/ips_option.cc b/src/framework/ips_option.cc index 7f6fc3ec5..44f0763ff 100644 --- a/src/framework/ips_option.cc +++ b/src/framework/ips_option.cc @@ -25,7 +25,7 @@ #include -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" using namespace snort; diff --git a/src/hash/CMakeLists.txt b/src/hash/CMakeLists.txt index 029a77ef9..8cbdd87e2 100644 --- a/src/hash/CMakeLists.txt +++ b/src/hash/CMakeLists.txt @@ -1,25 +1,23 @@ set (HASH_INCLUDES - ghash.h hashes.h hash_defs.h - hash_key_operations.h + ghash.h + xhash.h + hashfcn.h lru_cache_shared.h - xhash.h ) add_library( hash OBJECT ${HASH_INCLUDES} - ghash.cc hashes.cc - hash_lru_cache.cc - hash_lru_cache.h - hash_key_operations.cc lru_cache_shared.cc - primetable.cc - primetable.h - xhash.cc - zhash.cc + ghash.cc + hashfcn.cc + primetable.cc + primetable.h + xhash.cc + zhash.cc zhash.h ) diff --git a/src/hash/ghash.cc b/src/hash/ghash.cc index a9e20a8d8..80e264bb4 100644 --- a/src/hash/ghash.cc +++ b/src/hash/ghash.cc @@ -62,7 +62,6 @@ #include "utils/util.h" -#include "hash_defs.h" #include "primetable.h" namespace snort @@ -76,7 +75,7 @@ GHash::GHash(int nrows_, unsigned keysize, bool userkey, gHashFree userfree) else nrows = -nrows_; - hashfcn = new HashKeyOperations(nrows); + hashfcn = hashfcn_new(nrows); table = (GHashNode**)snort_calloc(nrows, sizeof(GHashNode*)); for ( int i = 0; i < nrows; i++ ) table[i] = nullptr; @@ -88,6 +87,8 @@ GHash::GHash(int nrows_, unsigned keysize, bool userkey, gHashFree userfree) GHash::~GHash() { + hashfcn_free(hashfcn); + for (int i = 0; i < nrows; i++) for ( GHashNode* node = table[i]; node; ) { @@ -104,13 +105,22 @@ GHash::~GHash() } snort_free(table); - delete hashfcn; } -GHashNode* GHash::find_node(const void* const key, unsigned index) +// set key length, hashkey, and index parameters required to find/add/remove a node +// the parameters set are valid until for the life of the initial method called +void GHash::set_node_parameters(const void* const key) +{ + klen = ( keysize > 0 ) ? keysize : strlen((const char*)key) + 1; + hashkey = hashfcn->hash_fcn(hashfcn, (const unsigned char*)key, klen); + index = hashkey % nrows; +} + +GHashNode* GHash::find_node(const void* const key) { assert(key); + set_node_parameters(key); for ( GHashNode* hnode = table[index]; hnode; hnode = hnode->next ) { if ( keysize == 0 ) @@ -120,7 +130,7 @@ GHashNode* GHash::find_node(const void* const key, unsigned index) } else { - if ( hashfcn->key_compare(hnode->key, key, keysize) ) + if ( hashfcn->keycmp_fcn(hnode->key, key, keysize) ) return hnode; } } @@ -132,11 +142,10 @@ int GHash::insert(const void* const key, void* const data) { assert(key && data); - unsigned index = get_index(key); - if ( GHashNode* hnode = find_node(key, index) ) + if ( GHashNode* hnode = find_node(key) ) { cnode = hnode; - return HASH_INTABLE; + return GHASH_INTABLE; } GHashNode* hnode = (GHashNode*)snort_calloc(sizeof(GHashNode)); @@ -146,7 +155,6 @@ int GHash::insert(const void* const key, void* const data) } else { - unsigned klen = get_key_length(key); hnode->key = snort_alloc(klen); memcpy(const_cast(hnode->key), key, klen); } @@ -169,15 +177,14 @@ int GHash::insert(const void* const key, void* const data) count++; - return HASH_OK; + return GHASH_OK; } void* GHash::find(const void* const key) { assert(key); - unsigned index = get_index(key); - GHashNode* hnode = find_node(key, index); + GHashNode* hnode = find_node(key); if ( hnode ) return hnode->data; @@ -212,18 +219,17 @@ int GHash::free_node(unsigned index, GHashNode* hnode) snort_free(hnode); count--; - return HASH_OK; + return GHASH_OK; } int GHash::remove(const void* const key) { assert(key); - unsigned index = get_index(key); - if ( GHashNode* hnode = find_node(key, index) ) + if ( GHashNode* hnode = find_node(key) ) return free_node(index, hnode); else - return HASH_NOT_FOUND; + return GHASH_NOT_FOUND; } void GHash::next() @@ -266,10 +272,9 @@ GHashNode* GHash::find_next() return n; } -void GHash::set_hashkey_ops(HashKeyOperations* hk) +void GHash::set_key_opcodes(hash_func hash_fcn, keycmp_func keycmp_fcn) { - delete hashfcn; - hashfcn = hk; + hashfcn_set_keyops(hashfcn, hash_fcn, keycmp_fcn); } } diff --git a/src/hash/ghash.h b/src/hash/ghash.h index 566a39a2d..17b874e74 100644 --- a/src/hash/ghash.h +++ b/src/hash/ghash.h @@ -24,18 +24,22 @@ // generic hash table - stores and maps key + data pairs -#include -#include "hash_key_operations.h" #include "main/snort_types.h" +#include "hashfcn.h" + namespace snort { +#define GHASH_NOT_FOUND (-1) +#define GHASH_OK 0 +#define GHASH_INTABLE 1 + struct GHashNode { struct GHashNode* next; struct GHashNode* prev; - const void* key; - void* data; + const void* key; /* Copy of, or Pointer to, the Users key */ + void* data; /* The users data, this is never copied! */ }; typedef void (* gHashFree)(void*); @@ -51,35 +55,31 @@ public: void* find(const void* const key); GHashNode* find_first(); GHashNode* find_next(); - void set_hashkey_ops(HashKeyOperations*); + void set_key_opcodes(hash_func, keycmp_func); unsigned get_count() const { return count; } private: - GHashNode* find_node(const void* const key, unsigned index); + void set_node_parameters(const void* const key); + GHashNode* find_node(const void* const key); int free_node(unsigned index, GHashNode*); void next(); - unsigned get_key_length(const void* const key) - { return ( keysize > 0 ) ? keysize : strlen((const char*)key) + 1; } - - unsigned get_index(const void* const key) - { - unsigned hashkey = hashfcn->do_hash((const unsigned char*)key, get_key_length(key)); - return hashkey % nrows; - } - unsigned keysize; // bytes in key, if < 0 -> keys are strings bool userkey; // user owns the key */ gHashFree userfree; int nrows; // # rows int the hash table use a prime number 211, 9871 - HashKeyOperations* hashfcn; + HashFnc* hashfcn; GHashNode** table; // array of node ptr's unsigned count; // total # nodes in table int crow; // findfirst/next row in table GHashNode* cnode; // findfirst/next node ptr + // node parameters for search/add/remove + unsigned klen = 0; + unsigned hashkey = 0; + int index = 0; }; diff --git a/src/hash/hash_defs.h b/src/hash/hash_defs.h index 673da8ecc..f66f16495 100644 --- a/src/hash/hash_defs.h +++ b/src/hash/hash_defs.h @@ -20,27 +20,30 @@ #ifndef HASH_DEFS_H #define HASH_DEFS_H -#include "hash_key_operations.h" #include "main/snort_types.h" +#include "utils/sfmemcap.h" + +#include "hashfcn.h" namespace snort { -#define HASH_NOMEM (-2) -#define HASH_NOT_FOUND (-1) +#define HASH_NOMEM (-2) +#define HASH_ERR (-1) #define HASH_OK 0 #define HASH_INTABLE 1 #define HASH_PENDING 2 -class HashNode +struct HashNode { -public: - HashNode* gnext; // lru or free node list - HashNode* gprev; - HashNode* next; // hash row node list - HashNode* prev; - void* key; - void* data; - int rindex; + struct HashNode* gnext; // global node list - used for aging nodes + struct HashNode* gprev; + struct HashNode* next; // row node list + struct HashNode* prev; + int rindex; // row index of table this node belongs to. + void* key; // Pointer to the key. + void* data; // Pointer to the users data, this is not copied ! }; + +typedef int (* Hash_FREE_FCN)(void* key, void* data); } #endif diff --git a/src/hash/hash_lru_cache.cc b/src/hash/hash_lru_cache.cc deleted file mode 100644 index b155e59c0..000000000 --- a/src/hash/hash_lru_cache.cc +++ /dev/null @@ -1,87 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// hash_lru_cache.cc author davis mcpherson davmcphe@cisco.com - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "hash_lru_cache.h" - -#include -#include "utils/util.h" - -using namespace snort; - -HashLruCache::HashLruCache() -{ - head = nullptr; - tail = nullptr; -} - -void HashLruCache::insert(HashNode* hnode) -{ - if ( head ) - { - hnode->gprev = nullptr; - hnode->gnext = head; - head->gprev = hnode; - head = hnode; - } - else - { - hnode->gprev = nullptr; - hnode->gnext = nullptr; - head = hnode; - tail = hnode; - } -} - -void HashLruCache::touch(HashNode* hnode) -{ - if ( hnode == cursor ) - cursor = hnode->gprev; - - if ( hnode != head ) - { - remove_node(hnode); - insert(hnode); - } -} - -void HashLruCache::remove_node(HashNode* hnode) -{ - if ( cursor == hnode ) - cursor = hnode->gprev; - - if ( head == hnode ) - { - head = head->gnext; - if ( head ) - head->gprev = nullptr; - } - - if ( hnode->gprev ) - hnode->gprev->gnext = hnode->gnext; - if ( hnode->gnext ) - hnode->gnext->gprev = hnode->gprev; - - if ( tail == hnode ) - tail = hnode->gprev; -} diff --git a/src/hash/hash_lru_cache.h b/src/hash/hash_lru_cache.h deleted file mode 100644 index 6436c3910..000000000 --- a/src/hash/hash_lru_cache.h +++ /dev/null @@ -1,73 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// hash_lru_cache.h author davis mcpherson davmcphe@cisco.com - -#ifndef HASH_LRU_CACHE_H -#define HASH_LRU_CACHE_H - -#include "hash_defs.h" - -class HashLruCache -{ -public: - HashLruCache(); - - void insert(snort::HashNode*); - void touch(snort::HashNode*); - void remove_node(snort::HashNode*); - - snort::HashNode* get_lru_node() - { - cursor = tail; - return cursor; - } - - snort::HashNode* get_next_lru_node() - { - if ( cursor ) - cursor = cursor->gprev; - return cursor; - } - - snort::HashNode* get_current_node() - { return cursor; } - - void* get_mru_user_data() - { return ( head ) ? head->data : nullptr; } - - void* get_lru_user_data() - { return ( tail ) ? tail->data : nullptr; } - - snort::HashNode* remove_lru_node() - { - snort::HashNode* hnode = tail; - if ( hnode ) - remove_node(hnode); - - return hnode; - } - -private: - snort::HashNode* head = nullptr; - snort::HashNode* tail = nullptr; - snort::HashNode* cursor = nullptr; -}; - -#endif - diff --git a/src/hash/hash_key_operations.cc b/src/hash/hashfcn.cc similarity index 65% rename from src/hash/hash_key_operations.cc rename to src/hash/hashfcn.cc index c70ff91d5..12b73a495 100644 --- a/src/hash/hash_key_operations.cc +++ b/src/hash/hashfcn.cc @@ -17,11 +17,22 @@ // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. //-------------------------------------------------------------------------- +/* + hashfcn.c + + Each hash table allocates it's own GHash struct, using the number of + rows in the hash table to modulo the random values. + + Updates: + + 8/31/2006 - man - changed to use sfprimetable.c +*/ + #ifdef HAVE_CONFIG_H #include "config.h" #endif -#include "hash_key_operations.h" +#include "hashfcn.h" #include @@ -32,48 +43,77 @@ using namespace snort; -HashKeyOperations::HashKeyOperations(int rows) +static bool hashfcn_key_compare(const void* k1, const void* k2, size_t len) +{ + if ( memcmp(k1, k2, len ) == 0 ) + return true; + else + return false; +} + +HashFnc* hashfcn_new(int m) { - static bool one = true; + HashFnc* p; + static int one = 1; if ( one ) /* one time init */ { srand( (unsigned)time(nullptr) ); - one = false; + one = 0; } + p = (HashFnc*)snort_calloc(sizeof(*p)); + if ( SnortConfig::static_hash() ) { - seed = 3193; - scale = 719; - hardener = 133824503; + p->seed = 3193; + p->scale = 719; + p->hardener = 133824503; } else { - seed = nearest_prime( (rand() % rows) + 3191); - scale = nearest_prime( (rand() % rows) + 709); - hardener = ((unsigned) rand() * rand()) + 133824503; + p->seed = nearest_prime( (rand() % m) + 3191); + p->scale = nearest_prime( (rand() % m) + 709); + p->hardener = ((unsigned) rand() * rand()) + 133824503; } + + p->hash_fcn = &hashfcn_hash; + p->keycmp_fcn = &hashfcn_key_compare; + + return p; +} + +void hashfcn_free(HashFnc* p) +{ + if ( p ) + snort_free(p); } -unsigned HashKeyOperations::do_hash(const unsigned char* key, int len) +unsigned hashfcn_hash(HashFnc* p, const unsigned char* d, int n) { - unsigned hash = seed; - while ( len ) + unsigned hash = p->seed; + while ( n ) { - hash *= scale; - hash += *key++; - len--; + hash *= p->scale; + hash += *d++; + n--; } - return hash ^ hardener; + return hash ^ p->hardener; } -bool HashKeyOperations::key_compare(const void* key1, const void* key2, size_t len) +/** + * Make hashfcn use a separate set of opcodes for the backend. + * + * @param h hashfcn ptr + * @param hash_fcn user specified hash function + * @param keycmp_fcn user specified key comparison function + */ +void hashfcn_set_keyops(HashFnc* h, hash_func hash_fcn, keycmp_func keycmp_fcn) { - if ( memcmp(key1, key2, len) ) - return false; - else - return true; + assert(h && hash_fcn && keycmp_fcn); + + h->hash_fcn = hash_fcn; + h->keycmp_fcn = keycmp_fcn; } namespace snort diff --git a/src/hash/hash_key_operations.h b/src/hash/hashfcn.h similarity index 83% rename from src/hash/hash_key_operations.h rename to src/hash/hashfcn.h index b48ab11fb..ad5663404 100644 --- a/src/hash/hash_key_operations.h +++ b/src/hash/hashfcn.h @@ -17,8 +17,8 @@ // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. //-------------------------------------------------------------------------- -#ifndef HASH_KEY_OPERATIONS_H -#define HASH_KEY_OPERATIONS_H +#ifndef HASHFCN_H +#define HASHFCN_H #include "main/snort_types.h" @@ -65,22 +65,26 @@ static inline int hash_nearest_power_of_2(int nrows) return nrows; } -class HashKeyOperations -{ -public: - HashKeyOperations(int rows); - virtual ~HashKeyOperations() - { } +} + +struct HashFnc; - virtual unsigned do_hash(const unsigned char* key, int len); - virtual bool key_compare(const void* key1, const void* key2, size_t len); +typedef uint32_t (* hash_func)(HashFnc*, const unsigned char* d, int n); +typedef bool (* keycmp_func)(const void* s1, const void* s2, size_t n); -protected: +struct HashFnc +{ unsigned seed; unsigned scale; unsigned hardener; + hash_func hash_fcn; + keycmp_func keycmp_fcn; }; -} + +HashFnc* hashfcn_new(int nrows); +void hashfcn_free(HashFnc*); +unsigned hashfcn_hash(HashFnc*, const unsigned char* d, int n); +void hashfcn_set_keyops(HashFnc*, hash_func, keycmp_func); #endif diff --git a/src/hash/test/CMakeLists.txt b/src/hash/test/CMakeLists.txt index 20e176d8b..39896fa9b 100644 --- a/src/hash/test/CMakeLists.txt +++ b/src/hash/test/CMakeLists.txt @@ -2,31 +2,17 @@ add_cpputest( lru_cache_shared_test SOURCES ../lru_cache_shared.cc ) -add_cpputest( hash_lru_cache_test - SOURCES ../hash_lru_cache.cc -) - add_cpputest( xhash_test SOURCES - ../hash_key_operations.cc - ../hash_lru_cache.cc - ../primetable.cc ../xhash.cc + ../hashfcn.cc + ../primetable.cc + ../../utils/sfmemcap.cc ) add_cpputest( ghash_test SOURCES ../ghash.cc - ../hash_key_operations.cc - ../hash_lru_cache.cc + ../hashfcn.cc ../primetable.cc ) - -add_cpputest( zhash_test - SOURCES - ../hash_key_operations.cc - ../hash_lru_cache.cc - ../primetable.cc - ../xhash.cc - ../zhash.cc -) diff --git a/src/hash/test/ghash_test.cc b/src/hash/test/ghash_test.cc index fba122d44..c1175a9cc 100644 --- a/src/hash/test/ghash_test.cc +++ b/src/hash/test/ghash_test.cc @@ -25,7 +25,6 @@ #include "hash/ghash.h" -#include "hash/hash_defs.h" #include "main/snort_config.h" #include "utils/util.h" @@ -120,7 +119,7 @@ TEST(ghash, collision_test) snprintf(str, sizeof(str), "KeyWord%d",1); str[sizeof(str) - 1] = '\0'; i = t->insert(str, (void *)(str + (1))); - CHECK(i == HASH_INTABLE); + CHECK(i == GHASH_INTABLE); // find those nodes for (i=num-1; i>=0; i--) @@ -193,7 +192,7 @@ TEST(ghash, userfree_test) CHECK(t->find(str) == nullptr); // try to remove a node that is not in the table - CHECK(t->remove( str) == HASH_NOT_FOUND); + CHECK(t->remove( str) == GHASH_NOT_FOUND); for ( GHashNode* n = t->find_first(); n; n = t->find_next() ) { diff --git a/src/hash/test/hash_lru_cache_test.cc b/src/hash/test/hash_lru_cache_test.cc deleted file mode 100644 index 5a2877476..000000000 --- a/src/hash/test/hash_lru_cache_test.cc +++ /dev/null @@ -1,140 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// hash_lru_cache_test.cc author davis mcpherson -// unit tests for the HashLruCache class - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "../hash_lru_cache.h" - -#include "../../utils/util.h" - -#include -#include - -using namespace snort; - -HashLruCache* lru = nullptr; - -TEST_GROUP(hash_lru_cache) -{ - void setup() override - { - lru = new HashLruCache; - CHECK(lru); - } - - void teardown() override - { - unsigned node_id = 1; - HashNode* node = lru->remove_lru_node(); - while( node ) - { - CHECK(*(unsigned*)node->data == node_id); - unsigned* data = (unsigned *)lru->get_mru_user_data(); - if ( data ) - CHECK(*data == 5); - data = (unsigned *)lru->get_lru_user_data(); - if ( data ) - CHECK(*data == node_id + 1); - ++node_id; - delete (unsigned*)node->data; - snort_free(node); - node = lru->remove_lru_node(); - } - - delete lru; - } -}; - -TEST(hash_lru_cache, create_hash_lru_cache_test) -{ - CHECK(!lru->get_next_lru_node()); - CHECK(!lru->get_current_node()); - CHECK(!lru->get_mru_user_data()); - CHECK(!lru->get_lru_user_data()); - CHECK(!lru->remove_lru_node()); -} - -TEST(hash_lru_cache, hash_lru_cache_insert_test) -{ - HashNode* node; - for (unsigned i = 0; i < 5; i++) - { - node = (HashNode*)snort_calloc(sizeof(HashNode)); - unsigned* data = new unsigned; - *data = i + 1; - node->data = data; - lru->insert(node); - CHECK(*((unsigned *)lru->get_mru_user_data()) == i + 1); - CHECK(*((unsigned *)lru->get_lru_user_data()) == 1); - } - - for (unsigned i = 0; i < 5; i++) - { - node = lru->get_lru_node(); - CHECK(node); - CHECK(*((unsigned*)node->data) == i + 1); - lru->touch(node); - CHECK(*((unsigned *)lru->get_mru_user_data()) == i + 1); - CHECK(*((unsigned *)lru->get_lru_user_data()) == ((i + 1) % 5) + 1); - } - - node = lru->get_lru_node(); - CHECK(node); - CHECK(*((unsigned*)node->data) == 1); -} - -TEST(hash_lru_cache, hash_lru_cache_browse_test) -{ - HashNode* node; - for (unsigned i = 0; i < 5; i++) - { - node = (HashNode*)snort_calloc(sizeof(HashNode)); - unsigned* data = new unsigned; - *data = i + 1; - node->data = data; - lru->insert(node); - } - - for (unsigned i = 0; i < 5; i++) - { - node = lru->get_lru_node(); - CHECK(node); - CHECK(*((unsigned*)node->data) == (i + 1)); - lru->touch(node); - } - - node = lru->get_lru_node(); - unsigned i = 1; - while( node ) - { - CHECK(*(unsigned*)node->data == i); - CHECK(*(unsigned*)lru->get_current_node()->data == i); - node = lru->get_next_lru_node(); - ++i; - } -} - -int main(int argc, char** argv) -{ - return CommandLineTestRunner::RunAllTests(argc, argv); -} diff --git a/src/hash/test/xhash_test.cc b/src/hash/test/xhash_test.cc index 96002f4e8..a52a4d92d 100644 --- a/src/hash/test/xhash_test.cc +++ b/src/hash/test/xhash_test.cc @@ -25,7 +25,6 @@ #include "hash/xhash.h" -#include "hash/hash_defs.h" #include "main/snort_config.h" #include "utils/util.h" @@ -57,8 +56,9 @@ TEST_GROUP(xhash) // Test create a hash table, add nodes, find and delete. TEST(xhash, create_xhash_test) { - XHash* test_table = new XHash(4, sizeof(struct xhash_test_key), 0, 0); - CHECK(test_table); + XHash* test_table = new XHash(4, sizeof(struct xhash_test_key), + 0, 0, false, nullptr, nullptr, false); + CHECK(test_table->get_keysize() == sizeof(struct xhash_test_key)); void* data = test_table->get_mru_user_data(); CHECK(data == nullptr); @@ -84,40 +84,39 @@ TEST(xhash, create_xhash_test) // Create a free node in xhash and verifies if xhash_free_anr_lru() deletes it TEST(xhash, free_anr_lru_delete_free_node_test) { - XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), 1, 1040); - CHECK(test_table); - + XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), + 1, 1040, false, nullptr, nullptr, true); xhash_test_key xtk; xtk.key = 10; int ret = test_table->insert(&xtk, nullptr); CHECK(ret == HASH_OK); - HashNode* xnode = test_table->find_node(&xtk); - CHECK(xnode); + HashNode* xnode = test_table->get_node(&xtk); + CHECK(xnode != nullptr); ret = test_table->release_node(xnode); CHECK(ret == HASH_OK); - ret = test_table->delete_lru_node(); + ret = test_table->delete_anr_or_lru_node(); CHECK(ret == HASH_OK); HashNode* xhnode = test_table->find_node(&xtk); - CHECK(!xhnode); + CHECK(xhnode == nullptr); delete test_table; } -// No free node is available, verifies the LRU node is deleted +// No free node is available, verifies if xhash_free_anr_lru() deletes the last node TEST(xhash, free_anr_lru_delete_tail_node_test) { - XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), 1, 1040); - CHECK(test_table); - + XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), + 1, 1040, false, nullptr, nullptr, true); xhash_test_key xtk; int ret = test_table->insert(&xtk, nullptr); CHECK(ret == HASH_OK); - CHECK(test_table->delete_lru_node()); + ret = test_table->delete_anr_or_lru_node(); + CHECK(ret == HASH_OK); HashNode* xhnode = test_table->find_node(&xtk); CHECK(xhnode == nullptr); @@ -125,6 +124,20 @@ TEST(xhash, free_anr_lru_delete_tail_node_test) delete test_table; } +// No free node is available [recycle is not enabled], verifies if last node is deleted +TEST(xhash, free_anr_lru_usr_free_delete_tail_node_test) +{ + XHash* test_table = new XHash(3, sizeof(struct xhash_test_key), + 1, 1040, false, nullptr, nullptr, false); + xhash_test_key xtk; + int ret = test_table->insert(&xtk, nullptr); + CHECK(ret == HASH_OK); + + ret = test_table->delete_anr_or_lru_node(); + CHECK(ret == HASH_OK); + delete test_table; +} + int main(int argc, char** argv) { return CommandLineTestRunner::RunAllTests(argc, argv); diff --git a/src/hash/test/zhash_test.cc b/src/hash/test/zhash_test.cc deleted file mode 100644 index 93e4817ac..000000000 --- a/src/hash/test/zhash_test.cc +++ /dev/null @@ -1,149 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// zhash_test.cc author davis mcpherson -// unit tests for the HashLruCache class - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include - -#include "../zhash.h" -#include "../hash_key_operations.h" - -#include "flow/flow_key.h" -#include "main/snort_config.h" - -#include -#include - -using namespace snort; - -namespace snort -{ -unsigned FlowHashKeyOps::do_hash(const unsigned char* k, int len) -{ - unsigned hash = seed; - while ( len ) - { - hash *= scale; - hash += *k++; - len--; - } - return hash ^ hardener; -} - -bool FlowHashKeyOps::key_compare(const void* k1, const void* k2, size_t len) -{ - if ( memcmp(k1, k2, len ) == 0 ) - return true; - else - return false; -} -} - -// Stubs whose sole purpose is to make the test code link -static SnortConfig my_config; -THREAD_LOCAL SnortConfig *snort_conf = &my_config; - -SnortConfig::SnortConfig(const SnortConfig* const) -{ snort_conf->run_flags = 0;} // run_flags is used indirectly from HashFnc class by calling SnortConfig::static_hash() - -SnortConfig::~SnortConfig() = default; - -SnortConfig* SnortConfig::get_conf() -{ return snort_conf; } - -const unsigned ZHASH_ROWS = 1000; -const unsigned ZHASH_KEY_SIZE = 100; -const unsigned MAX_ZHASH_NODES = 100; -char key_buf[ZHASH_KEY_SIZE]; - -ZHash* zh = nullptr; - -TEST_GROUP(zhash) -{ - void setup() override - { - zh = new ZHash(ZHASH_ROWS, ZHASH_KEY_SIZE); - CHECK(zh); - - memset(key_buf, '\0', ZHASH_KEY_SIZE); - } - - void teardown() override - { - delete zh; - } -}; - -TEST(zhash, create_zhash_test) -{ - for (unsigned i = 0; i < MAX_ZHASH_NODES; i++ ) - { - unsigned* data; - data = (unsigned*)snort_calloc(sizeof(unsigned)); - *data = 0; - zh->push(data); - } - - std::string key_prefix = "foo"; - for (unsigned i = 0; i < MAX_ZHASH_NODES; i++ ) - { - std::string key; - key = key_prefix + std::to_string(i + 1); - memcpy(key_buf, key.c_str(), key.size()); - unsigned* data = (unsigned*)zh->get(key_buf); - CHECK(*data == 0); - *data = i + 1; - } - - unsigned nodes_walked = 0; - unsigned* data = (unsigned*)zh->lru_first(); - while ( data ) - { - CHECK(*data == ++nodes_walked); - data = (unsigned*)zh->lru_next(); - } - - CHECK(nodes_walked == MAX_ZHASH_NODES); - - data = (unsigned*)zh->lru_first(); - CHECK(*data == 1); - data = (unsigned*)zh->remove(); - CHECK(*data == 1); - snort_free(data); - data = (unsigned*)zh->lru_current(); - CHECK(*data == 2); - data = (unsigned*)zh->lru_first(); - CHECK(*data == 2); - - for (unsigned i = 1; i < MAX_ZHASH_NODES; i++ ) - { - data = (unsigned*)zh->remove(); - CHECK(*data == (i + 1)); - snort_free(data); - } -} - -int main(int argc, char** argv) -{ - return CommandLineTestRunner::RunAllTests(argc, argv); -} diff --git a/src/hash/xhash.cc b/src/hash/xhash.cc index ee7d74152..bcca7c412 100644 --- a/src/hash/xhash.cc +++ b/src/hash/xhash.cc @@ -95,259 +95,209 @@ #include #include "utils/util.h" -#include "hash_defs.h" -#include "hash_key_operations.h" -#include "hash_lru_cache.h" - using namespace snort; namespace snort { -void XHash::initialize(HashKeyOperations* hk_ops) -{ - hashkey_ops = hk_ops; - table = (HashNode**)snort_calloc(sizeof(HashNode*) * nrows); - lru_cache = new HashLruCache(); - mem_allocator = new MemCapAllocator(mem_cap, sizeof(HashNode) + keysize + datasize); -} - -void XHash::initialize() -{ - initialize(new HashKeyOperations(nrows)); -} - -void XHash::set_number_of_rows (int rows) +XHash::XHash(int nrows_, int keysize_, int datasize_, unsigned long maxmem, + bool anr_enabled, Hash_FREE_FCN anr_free, Hash_FREE_FCN usr_free, + bool recycle_nodes) + : keysize(keysize_), datasize(datasize_), recycle_nodes(recycle_nodes), + anr_enabled(anr_enabled), anr_free(anr_free), usr_free(usr_free) { - if ( rows > 0 ) - nrows = hash_nearest_power_of_2 (rows); + // adjust rows to be power of 2 + if ( nrows_ > 0 ) + nrows = hash_nearest_power_of_2(nrows_); else - nrows = -rows; -} + nrows = -nrows_; // if negative use as is -XHash::XHash(int rows, int keysize) - : keysize(keysize) + table = (HashNode**)snort_calloc(sizeof(HashNode*) * nrows); + hashfcn = hashfcn_new(nrows); + sfmemcap_init(&mc, maxmem); -{ - set_number_of_rows(rows); -} + for ( unsigned i = 0; i < nrows; i++ ) + table[i] = nullptr; -XHash::XHash(int rows, int keysize, int datasize, unsigned long memcap) - : keysize(keysize), datasize(datasize), mem_cap(memcap) -{ - set_number_of_rows(rows); - initialize(); + mem_allocated_per_entry = sizeof(HashNode) + keysize + datasize + sizeof(long); } XHash::~XHash() { - if ( table ) + if ( hashfcn ) + hashfcn_free(hashfcn); + + for (unsigned i = 0; i < nrows; i++) { - for (unsigned i = 0; i < nrows; i++) - for (HashNode* node = table[i]; node;) - { - HashNode* xnode; - xnode = node; - node = node->next; - mem_allocator->free(xnode); - } + for ( HashNode* node = table[i]; node; ) + { + HashNode* onode = node; + node = node->next; - snort_free(table); - } + if ( usr_free ) + usr_free(onode->key, onode->data); + sfmemcap_free(&mc, onode); + } + } + snort_free(table); purge_free_list(); - delete hashkey_ops; - delete lru_cache; - delete mem_allocator; } -void XHash::delete_hash_table() +void XHash::clear() { - assert( table ); - for (unsigned i = 0; i < nrows; i++) - for (HashNode* node = table[i]; node;) + { + HashNode* n = table[i]; + while ( n ) { - HashNode* xnode; - xnode = node; - node = node->next; - free_user_data(xnode); - mem_allocator->free(xnode); + HashNode* tmp; + tmp = n; + n = n->next; + release_node(tmp); } + } - snort_free(table); - table = nullptr; + max_nodes = 0; + crow = 0; + cnode = nullptr; + count = 0; + ghead = nullptr; + gtail = nullptr; + anr_count = 0; + anr_tries = 0; + find_success = 0; + find_fail = 0; } -void XHash::initialize_node(HashNode *hnode, const void *key, void *data, int index) +void XHash::save_free_node(HashNode* hnode) { - hnode->key = (char*) (hnode) + sizeof(HashNode); - memcpy(hnode->key, key, keysize); - if ( datasize ) + if ( fhead ) { - hnode->data = (char*) (hnode) + sizeof(HashNode) + keysize; - if ( data ) - memcpy (hnode->data, data, datasize); + hnode->gprev = nullptr; + hnode->gnext = fhead; + fhead->gprev = hnode; + fhead = hnode; } else - hnode->data = data; - - hnode->rindex = index; - link_node(hnode); - lru_cache->insert(hnode); + { + hnode->gprev = nullptr; + hnode->gnext = nullptr; + fhead = hnode; + ftail = hnode; + } } -HashNode* XHash::allocate_node(const void* key, void* data, int index) +HashNode* XHash::get_free_node() { - // use a free one if available... - HashNode* hnode = get_free_node(); - - // if no free nodes, try to allocate a new one... - if ( !hnode && ((max_nodes == 0) || (num_nodes < max_nodes)) ) - hnode = (HashNode*)mem_allocator->allocate(); - - // if still no node then try to reuse one... - if ( !hnode && anr_enabled ) - hnode = release_lru_node(); + HashNode* node = fhead; - if ( hnode ) + if ( fhead ) { - initialize_node(hnode, key, data, index); - ++num_nodes; + fhead = fhead->gnext; + if ( fhead ) + fhead->gprev = nullptr; + + if ( ftail == node ) + ftail = nullptr; } - return hnode; + return node; } -int XHash::insert(const void* key, void* data) +void XHash::purge_free_list() { - assert(key); - - int index = 0; - HashNode* hnode = find_node_row(key, index); - if ( hnode ) + HashNode* cur = fhead; + while ( cur ) { - cursor = hnode; - return HASH_INTABLE; + HashNode* next = cur->gnext; + sfmemcap_free(&mc, (void*)cur); + cur = next; } - hnode = allocate_node(key, data, index); - cursor = hnode; - return ( hnode ) ? HASH_OK : HASH_NOMEM; + fhead = nullptr; + ftail = nullptr; } -HashNode* XHash::find_node(const void* key) +void XHash::glink_node(HashNode* hnode) { - assert(key); - - int rindex = 0; - return find_node_row(key, rindex); -} - -HashNode* XHash::find_first_node() -{ - for ( crow = 0; crow < nrows; crow++ ) + if ( ghead ) { - cursor = table[crow]; - if ( cursor ) - { - HashNode* n = cursor; - update_cursor(); - return n; - } + hnode->gprev = nullptr; + hnode->gnext = ghead; + ghead->gprev = hnode; + ghead = hnode; + } + else + { + hnode->gprev = nullptr; + hnode->gnext = nullptr; + ghead = hnode; + gtail = hnode; } - - return nullptr; } -HashNode* XHash::find_next_node() +void XHash::gunlink_node(HashNode* hnode) { - HashNode* n = cursor; - if ( !n ) - return nullptr; + if ( gnode == hnode ) + gnode = hnode->gnext; - update_cursor(); - return n; -} + if ( ghead == hnode ) + { + ghead = ghead->gnext; + if ( ghead ) + ghead->gprev = nullptr; + } -void* XHash::get_user_data() -{ - if ( cursor ) - return cursor->data; - else - return nullptr; + if ( hnode->gprev ) + hnode->gprev->gnext = hnode->gnext; + if ( hnode->gnext ) + hnode->gnext->gprev = hnode->gprev; + + if ( gtail == hnode ) + gtail = hnode->gprev; } -void XHash::update_cursor() +void XHash::gmove_to_front(HashNode* hnode) { - if ( !cursor ) - return; - - cursor = cursor->next; - if ( cursor ) - return; - - for ( crow++; crow < nrows; crow++ ) + if ( hnode != ghead ) { - cursor = table[crow]; - if ( cursor ) - return; + gunlink_node(hnode); + glink_node(hnode); } } -void* XHash::get_user_data(const void* key) +HashNode* XHash::gfind_next() { - assert(key); - - int rindex = 0; - HashNode* hnode = find_node_row(key, rindex); - return ( hnode ) ? hnode->data : nullptr; + HashNode* n = gnode; + if ( n ) + gnode = n->gnext; + return n; } -void XHash::release() +HashNode* XHash::gfind_first() { - HashNode* node = lru_cache->get_current_node(); - assert(node); - release_node(node); + if ( ghead ) + gnode = ghead->gnext; + else + gnode = nullptr; + return ghead; } -int XHash::release_node(HashNode* hnode) +void* XHash::get_mru_user_data() { - assert(hnode); - - free_user_data(hnode); - unlink_node(hnode); - lru_cache->remove_node(hnode); - num_nodes--; - - if ( recycle_nodes ) - { - save_free_node(hnode); - ++stats.release_recycles; - } + if ( ghead ) + return ghead->data; else - { - mem_allocator->free(hnode); - ++stats.release_deletes; - } - - return HASH_OK; + return nullptr; } -int XHash::release_node(const void* key) +void* XHash::get_lru_user_data() { - assert(key); - - unsigned hashkey = hashkey_ops->do_hash((const unsigned char*)key, keysize); - - unsigned index = hashkey & (nrows - 1); - for (HashNode* hnode = table[index]; hnode; hnode = hnode->next) - { - if ( hashkey_ops->key_compare(hnode->key, key, keysize) ) - return release_node(hnode); - } - - return HASH_NOT_FOUND; + if ( gtail ) + return gtail->data; + else + return nullptr; } void XHash::link_node(HashNode* hnode) @@ -383,172 +333,327 @@ void XHash::unlink_node(HashNode* hnode) } } -void XHash::move_to_front(HashNode* node) +void XHash::move_to_front(HashNode* n) { - if ( table[node->rindex] != node ) + if ( table[n->rindex] != n ) { - unlink_node(node); - link_node(node); + unlink_node(n); + link_node(n); } - lru_cache->touch(node); + if (n == gnode) + gnode = n->gnext; + gmove_to_front(n); } -HashNode* XHash::find_node_row(const void* key, int& rindex) +/* + * Allocate a new hash node, uses Auto Node Recovery if needed and enabled. + * + * The oldest node is the one with the longest time since it was last touched, + * and does not have any direct indication of how long the node has been around. + * We don't monitor the actual time since last being touched, instead we use a + * splayed global list of node pointers. As nodes are accessed they are splayed + * to the front of the list. The oldest node is just the tail node. + * + */ +HashNode* XHash::allocate_node() { - unsigned hashkey = hashkey_ops->do_hash((const unsigned char*)key, keysize); + // use previously allocated node if there is a free one... + HashNode* hnode = get_free_node(); + if ( !hnode ) + { + if ( (max_nodes == 0) || (count < max_nodes) ) + hnode = (HashNode*)sfmemcap_alloc(&mc, + sizeof(HashNode) + keysize + datasize); - /* Modulus is slow. Switched to a table size that is a power of 2. */ - rindex = hashkey & (nrows - 1); - for (HashNode* hnode = table[rindex]; hnode; hnode = hnode->next ) + if ( !hnode && anr_enabled && gtail ) + { + /* Find the oldest node the users willing to let go. */ + for (hnode = gtail; hnode; hnode = hnode->gprev ) + { + if ( anr_free ) + { + anr_tries++; + if ( anr_free(hnode->key, hnode->data) ) + continue; // don't recycle this one... + } + + gunlink_node(hnode); + unlink_node(hnode); + count--; + anr_count++; + break; + } + } + } + + return hnode; +} + +HashNode* XHash::find_node_row(const void* key, int* rindex) +{ + unsigned hashkey = hashfcn->hash_fcn(hashfcn, (const unsigned char*)key, keysize); + + // Modulus is slow. masking since table size is a power of 2. + int index = hashkey & (nrows - 1); + *rindex = index; + + for (HashNode* hnode = table[index]; hnode; hnode = hnode->next ) { - if ( hashkey_ops->key_compare(hnode->key, key, keysize) ) + if ( hashfcn->keycmp_fcn(hnode->key, key, keysize) ) { - move_to_front(hnode); + if ( splay > 0 ) + move_to_front(hnode); + + find_success++; return hnode; } } + find_fail++; return nullptr; } -void XHash::save_free_node(HashNode* hnode) +int XHash::insert(const void* key, void* data) { - if ( fhead ) + assert(key); + + int index = 0; + + /* Enforce uniqueness: Check for the key in the table */ + HashNode* hnode = find_node_row(key, &index); + if ( hnode ) { - hnode->gprev = nullptr; - hnode->gnext = fhead; - fhead->gprev = hnode; - fhead = hnode; + cnode = hnode; + return HASH_INTABLE; } - else + + hnode = allocate_node(); + if ( !hnode ) + return HASH_NOMEM; + + hnode->key = (char*)hnode + sizeof(HashNode); + memcpy(hnode->key, key, keysize); + hnode->rindex = index; + + if ( datasize ) { - hnode->gprev = nullptr; - hnode->gnext = nullptr; - fhead = hnode; + hnode->data = (char*)hnode + sizeof(HashNode) + keysize; + if ( data ) + memcpy(hnode->data, data, datasize); } + else + hnode->data = data; + + link_node (hnode); + glink_node(hnode); + count++; + + return HASH_OK; } -HashNode* XHash::get_free_node() +HashNode* XHash::get_node(const void* key) { - HashNode* node = fhead; - if ( fhead ) + assert(key); + + int index = 0; + + // Enforce uniqueness: Check for the key in the table + HashNode* hnode = find_node_row( key, &index); + if ( hnode ) { - fhead = fhead->gnext; - if ( fhead ) - fhead->gprev = nullptr; + cnode = hnode; + return hnode; } - return node; + hnode = allocate_node(); + if ( !hnode ) + return nullptr; + + hnode->key = (char*)hnode + sizeof(HashNode); + memcpy(hnode->key, key, keysize); + hnode->rindex = index; + + if ( datasize ) + hnode->data = (char*)hnode + sizeof(HashNode) + keysize; + else + hnode->data = nullptr; + + link_node(hnode); + glink_node(hnode); + count++; + + return hnode; } -bool XHash::delete_free_node() +HashNode* XHash::get_node_with_prune(const void* key, bool* prune_performed) { - HashNode* hnode = get_free_node(); - if ( hnode ) - { - mem_allocator->free(hnode); - return true; - } - return false; + assert(key); + + size_t mem_after_alloc = mc.memused + mem_allocated_per_entry; + bool over_capacity = (mc.memcap < mem_after_alloc); + + if ( over_capacity ) + *prune_performed = (delete_anr_or_lru_node() == HASH_OK); + + HashNode* hnode = nullptr; + if ( *prune_performed or !over_capacity ) + hnode = get_node(key); + + return hnode; } -void XHash::purge_free_list() +HashNode* XHash::find_node(const void* key) { - HashNode* cur = fhead; - while ( cur ) - { - HashNode* next = cur->gnext; - mem_allocator->free(cur); - cur = next; - } + assert(key); - fhead = nullptr; + int rindex = 0; + return find_node_row(key, &rindex); } -void XHash::clear_hash() +void* XHash::get_user_data(void* key) { - for (unsigned i = 0; i < nrows; i++) - for (HashNode* node = table[i]; node;) - { - HashNode* xnode = node; - node = node->next; - release_node(xnode); - } + assert(key); - max_nodes = 0; - num_nodes = 0; - crow = 0; - cursor = nullptr; + int rindex = 0; + HashNode* hnode = find_node_row(key, &rindex); + if ( hnode ) + return hnode->data; + + return nullptr; } -void* XHash::get_mru_user_data() +int XHash::release_node(HashNode* hnode) { - return lru_cache->get_mru_user_data(); + assert(hnode); + + unlink_node(hnode); + gunlink_node(hnode); + count--; + + if ( usr_free ) + usr_free(hnode->key, hnode->data); + + if ( recycle_nodes ) + save_free_node(hnode); + else + sfmemcap_free(&mc, hnode); + + return HASH_OK; } -void* XHash::get_lru_user_data() +int XHash::release_node(void* key) { - return lru_cache->get_lru_user_data(); + assert(key); + + unsigned hashkey = hashfcn->hash_fcn(hashfcn, (unsigned char*)key, keysize); + + unsigned index = hashkey & (nrows - 1); + for ( HashNode* hnode = table[index]; hnode; hnode = hnode->next ) + { + if ( hashfcn->keycmp_fcn(hnode->key, key, keysize) ) + return release_node(hnode); + } + + return HASH_ERR; } -HashNode* XHash::release_lru_node() +int XHash::delete_free_node() { - HashNode* hnode = lru_cache->get_lru_node(); - while ( hnode ) + HashNode* fn = get_free_node(); + if (fn) { - if ( is_node_recovery_ok(hnode) ) + sfmemcap_free(&mc, fn); + return HASH_OK; + } + return HASH_ERR; +} + +int XHash::delete_anr_or_lru_node() +{ + if ( fhead ) + { + if (delete_free_node() == HASH_OK) + return HASH_OK; + } + + if ( gtail ) + { + if ( release_node(gtail) == HASH_OK ) { - lru_cache->remove_node(hnode); - free_user_data(hnode); - unlink_node(hnode); - --num_nodes; - ++stats.memcap_prunes; - break; + if ( fhead ) + { + if ( delete_free_node() == HASH_OK ) + return HASH_OK; + } + else if ( !recycle_nodes ) + return HASH_OK; } - else - hnode = lru_cache->get_next_lru_node (); } - return hnode; + return HASH_ERR; } -bool XHash::delete_lru_node() +int XHash::free_over_allocations(unsigned work_limit, unsigned* num_freed) { - if ( HashNode* hnode = lru_cache->remove_lru_node() ) + + while (mc.memcap < mc.memused and work_limit--) { - unlink_node(hnode); - free_user_data(hnode); - mem_allocator->free(hnode); - --num_nodes; - return true; + if (delete_anr_or_lru_node() != HASH_OK) + return HASH_ERR; + + ++*num_freed; } - return false; + return (mc.memcap >= mc.memused) ? HASH_OK : HASH_PENDING; } -bool XHash::delete_a_node() +void XHash::update_cnode() { - if ( delete_free_node() ) - return true; + if ( !cnode ) + return; - if ( delete_lru_node() ) - return true; + cnode = cnode->next; + if ( cnode ) + return; - return false; + for ( crow++; crow < nrows; crow++ ) + { + cnode = table[crow]; + if ( cnode ) + return; + } } -int XHash::tune_memory_resources(unsigned work_limit, unsigned& num_freed) +HashNode* XHash::find_first_node() { - while ( work_limit-- and mem_allocator->is_over_capacity() ) + for ( crow = 0; crow < nrows; crow++ ) { - if ( !delete_a_node() ) - break; - - ++stats.memcap_deletes; - ++num_freed; + cnode = table[crow]; + if ( cnode ) + { + HashNode* n = cnode; + update_cnode(); + return n; + } } - return ( mem_allocator->is_over_capacity() ) ? HASH_PENDING : HASH_OK; + return nullptr; +} + +HashNode* XHash::find_next_node() +{ + HashNode* n = cnode; + if ( !n ) + return nullptr; + + update_cnode(); + + return n; +} + +void XHash::set_key_opcodes(hash_func hash_fcn, keycmp_func keycmp_fcn) +{ + hashfcn_set_keyops(hashfcn, hash_fcn, keycmp_fcn); } } // namespace snort diff --git a/src/hash/xhash.h b/src/hash/xhash.h index 2d0e2f36c..d9ea5f7df 100644 --- a/src/hash/xhash.h +++ b/src/hash/xhash.h @@ -25,113 +25,121 @@ // generic hash table - stores and maps key + data pairs // (supports memcap and automatic memory recovery when out of memory) -#include "framework/counts.h" +#include "utils/sfmemcap.h" #include "main/snort_types.h" -#include "utils/memcap_allocator.h" -class HashLruCache; +#include "hash_defs.h" +#include "hashfcn.h" namespace snort { -class HashKeyOperations; -class HashNode; - -struct XHashStats -{ - PegCount nodes_created = 0; - PegCount memcap_prunes = 0; - PegCount memcap_deletes = 0; - PegCount release_recycles = 0; - PegCount release_deletes = 0; -}; - class SO_PUBLIC XHash { public: - XHash(int rows, int keysize); - XHash(int rows, int keysize, int datasize, unsigned long memcap); - virtual ~XHash(); + XHash(int nrows_, int keysize_, int datasize_, unsigned long memcap, + bool anr_enabled, Hash_FREE_FCN, Hash_FREE_FCN, bool recycle_nodes); + ~XHash(); + + int free_over_allocations(unsigned work_limit, unsigned* num_freed); + void clear(); int insert(const void* key, void* data); + HashNode* get_node(const void* key); + HashNode* get_node_with_prune(const void* key, bool* prune_performed); + int release_node(void* key); + int release_node(HashNode* node); + int delete_anr_or_lru_node(); HashNode* find_node(const void* key); HashNode* find_first_node(); HashNode* find_next_node(); - void* get_user_data(); - void* get_user_data(const void* key); - void release(); - int release_node(const void* key); - int release_node(HashNode* node); + void* get_user_data(void* key); void* get_mru_user_data(); void* get_lru_user_data(); - bool delete_lru_node(); - void clear_hash(); + void set_key_opcodes(hash_func, keycmp_func); - // set max hash nodes, 0 == no limit + // Set the maximum nodes used in this hash table. + // Specifying 0 is unlimited (or otherwise limited by memcap). void set_max_nodes(int max) { max_nodes = max; } - unsigned get_num_nodes() - { return num_nodes; } + unsigned get_node_count() + { return count; } - void set_memcap(unsigned long memcap) - { mem_allocator->set_mem_capacity(memcap); } + unsigned get_anr_count() + { return anr_count; } - unsigned long get_memcap() - { return mem_allocator->get_mem_capacity(); } - - unsigned long get_mem_used() - { return mem_allocator->get_mem_allocated(); } + unsigned get_total_finds() + { return find_success + find_fail; } - const XHashStats& get_stats() const - { return stats; } + unsigned get_find_fails() + { return find_fail; } - virtual int tune_memory_resources(unsigned work_limit, unsigned& num_freed); + unsigned get_find_successes() + { return find_success; } -protected: - void initialize(HashKeyOperations*); - void initialize(); + void set_memcap(unsigned long new_memcap) + { mc.memcap = new_memcap; } - void initialize_node (HashNode*, const void* key, void* data, int index); - HashNode* allocate_node(const void* key, void* data, int index); - HashNode* find_node_row(const void* key, int& rindex); - void link_node(HashNode*); - void unlink_node(HashNode*); - bool delete_a_node(); - void save_free_node(HashNode*); - HashNode* get_free_node(); - void delete_hash_table(); + unsigned long get_memcap() + { return mc.memcap; } - virtual bool is_node_recovery_ok(HashNode*) - { return true; } + unsigned long get_mem_used() + { return mc.memused; } - virtual void free_user_data(HashNode*) - { } + const HashNode* get_cnode () const + { return cnode; } - MemCapAllocator* mem_allocator = nullptr; - HashLruCache* lru_cache = nullptr; - unsigned nrows = 0; - unsigned keysize = 0; - unsigned num_nodes = 0; - bool recycle_nodes = true; - bool anr_enabled = true; + int get_keysize () const + { return keysize; } private: - HashNode** table = nullptr; - HashKeyOperations* hashkey_ops = nullptr; - HashNode* cursor = nullptr; - HashNode* fhead = nullptr; - unsigned datasize = 0; - unsigned long mem_cap = 0; - unsigned max_nodes = 0; - unsigned crow = 0; - XHashStats stats; - - void set_number_of_rows(int nrows); - void move_to_front(HashNode*); - bool delete_free_node(); - HashNode* release_lru_node(); - void update_cursor(); void purge_free_list(); + void save_free_node(HashNode* hnode); + HashNode* get_free_node(); + void glink_node(HashNode* hnode); + void gunlink_node(HashNode* hnode); + void gmove_to_front(HashNode* hnode); + HashNode* gfind_first(); + HashNode* gfind_next(); + void link_node(HashNode* hnode); + void unlink_node(HashNode* hnode); + void move_to_front(HashNode* n); + HashNode* allocate_node(); + HashNode* find_node_row(const void* key, int* rindex); + void update_cnode(); + int delete_free_node(); + + HashFnc* hashfcn = nullptr; // hash function + int keysize = 0; // bytes in key, if <= 0 -> keys are strings - FIXIT-H does negative keysize work? + int datasize = 0; // bytes in key, if == 0 -> user data + unsigned mem_allocated_per_entry = 0; + HashNode** table = nullptr; // array of node ptr's */ + unsigned nrows = 0; // # rows int the hash table use a prime number 211, 9871 + unsigned count = 0; // total # nodes in table + unsigned crow = 0; // findfirst/next row in table + HashNode* cnode = nullptr; // find_[first|next] node ptr + int splay = 1; // whether to splay nodes with same hash bucket + unsigned max_nodes = 0; // maximum # of nodes within a hash + MEMCAP mc; + unsigned find_fail = 0; + unsigned find_success = 0; + + HashNode* ghead = nullptr; // global - root of all nodes allocated in table + HashNode* gtail = nullptr; + HashNode* gnode = nullptr; // gfirst/gnext node ptr */ + HashNode* fhead = nullptr; // list of free nodes, which are recycled + HashNode* ftail = nullptr; + bool recycle_nodes = false; // recycle nodes... + + // Automatic Node Recover (ANR): When number of nodes in hash is equal + // to max_nodes, remove the least recently used nodes and use it for + // the new node. anr_tries indicates # of ANR tries.*/ + unsigned anr_tries = 0; + unsigned anr_count = 0; // # ANR ops performed + bool anr_enabled = false; // false = anr disable, true = anr enabled + + Hash_FREE_FCN anr_free = nullptr; + Hash_FREE_FCN usr_free = nullptr; }; } // namespace snort diff --git a/src/hash/zhash.cc b/src/hash/zhash.cc index d806d7a3b..f2b87d3ac 100644 --- a/src/hash/zhash.cc +++ b/src/hash/zhash.cc @@ -28,65 +28,233 @@ #include #include -#include "flow/flow_key.h" - #include "hash_defs.h" -#include "hash_key_operations.h" -#include "hash_lru_cache.h" using namespace snort; //------------------------------------------------------------------------- -// public stuff +// private stuff //------------------------------------------------------------------------- +static inline HashNode* s_node_alloc(int keysize) +{ + auto node = static_cast( + ::operator new(sizeof(HashNode) + keysize)); + + *node = {}; + return node; +} + +static inline void s_node_free(HashNode* node) +{ ::operator delete(node); } -ZHash::ZHash(int rows, int key_len) - : XHash(rows, key_len) +void ZHash::delete_free_list() { - initialize(new FlowHashKeyOps(nrows)); - anr_enabled = false; + if ( !fhead ) + return; + + HashNode* cur = fhead; + + while ( cur ) + { + fhead = cur->gnext; + s_node_free(cur); + cur = fhead; + } } -void* ZHash::get(const void* key) +void ZHash::save_free_node(HashNode* node) { - assert(key); + if ( fhead ) + { + node->gprev = nullptr; + node->gnext = fhead; + fhead->gprev = node; + fhead = node; + } + else + { + node->gprev = nullptr; + node->gnext = nullptr; + fhead = node; + } +} - int index; - HashNode* node = find_node_row(key, index); - if ( node ) - return node->data; +HashNode* ZHash::get_free_node() +{ + HashNode* node = fhead; - node = get_free_node(); - if ( !node ) - return nullptr; + if ( fhead ) + { + fhead = fhead->gnext; - memcpy(node->key, key, keysize); - node->rindex = index; - link_node(node); - lru_cache->insert(node); - num_nodes++; - return node->data; + if ( fhead ) + fhead->gprev = nullptr; + } + + return node; } -void* ZHash::remove() +void ZHash::glink_node(HashNode* node) { - HashNode* node = lru_cache->get_current_node(); - assert(node); - void* pv = node->data; + if ( ghead ) + { + node->gprev = nullptr; + node->gnext = ghead; + ghead->gprev = node; + ghead = node; + } + else + { + node->gprev = nullptr; + node->gnext = nullptr; + ghead = node; + gtail = node; + } +} - unlink_node(node); - lru_cache->remove_node(node); - num_nodes--; - mem_allocator->free(node); - return pv; +void ZHash::gunlink_node(HashNode* node) +{ + if ( cursor == node ) + cursor = node->gprev; + + if ( ghead == node ) + { + ghead = ghead->gnext; + if ( ghead ) + ghead->gprev = nullptr; + } + + if ( node->gprev ) + node->gprev->gnext = node->gnext; + + if ( node->gnext ) + node->gnext->gprev = node->gprev; + + if ( gtail == node ) + gtail = node->gprev; +} + +void ZHash::link_node(HashNode* node) +{ + if ( table[node->rindex] ) // UNINITUSE + { + node->prev = nullptr; + node->next = table[node->rindex]; + table[node->rindex]->prev = node; + table[node->rindex] = node; + } + else + { + node->prev = nullptr; + node->next = nullptr; + table[node->rindex] = node; // UNINITUSE + } +} + +void ZHash::unlink_node(HashNode* node) +{ + if ( node->prev ) + { + node->prev->next = node->next; + if ( node->next ) + node->next->prev = node->prev; + } + else if ( table[node->rindex] ) + { + table[node->rindex] = table[node->rindex]->next; + + if ( table[node->rindex] ) + table[node->rindex]->prev = nullptr; + } +} + +void ZHash::move_to_front(HashNode* node) +{ + // move to front of row list + if ( table[node->rindex] != node ) + { + unlink_node(node); + link_node(node); + } + + // move to front of global list + if ( node != ghead ) + { + gunlink_node(node); + glink_node(node); + } +} + +HashNode* ZHash::find_node_row(const void* key, int& row) +{ + unsigned hashkey = hashfcn->hash_fcn( + hashfcn, (const unsigned char*)key, keysize); + + // Modulus is slow; use a table size that is a power of 2. + int index = hashkey & (nrows - 1); + row = index; + + for ( HashNode* node = table[index]; node; node = node->next ) // UNINITUSE + { + if ( hashfcn->keycmp_fcn(node->key, key, keysize) ) + { + move_to_front(node); + find_success++; + return node; + } + } + + find_fail++; + return nullptr; +} + +//------------------------------------------------------------------------- +// public stuff +//------------------------------------------------------------------------- + +ZHash::ZHash(int rows, int keysz) + : keysize(keysz) +{ + // adjust rows to be power of 2 + if ( rows > 0 ) + nrows = hash_nearest_power_of_2(rows); + else + nrows = -rows; // if negative use as is + + table = new HashNode*[nrows](); + hashfcn = hashfcn_new(nrows); + + fhead = cursor = nullptr; + ghead = gtail = nullptr; + count = find_success = find_fail = 0; +} + +ZHash::~ZHash() +{ + hashfcn_free(hashfcn); + + for ( unsigned i = 0; i < nrows; ++i ) + { + for ( HashNode* node = table[i]; node; ) + { + HashNode* onode = node; + node = node->next; + s_node_free(onode); + } + } + + delete[] table; + delete_free_list(); } void* ZHash::push(void* p) { - auto node = (HashNode*)mem_allocator->allocate(); + auto node = s_node_alloc(keysize); + node->key = (char*)node + sizeof(HashNode); node->data = p; + save_free_node(node); return node->key; } @@ -94,36 +262,137 @@ void* ZHash::push(void* p) void* ZHash::pop() { HashNode* node = get_free_node(); + if ( !node ) return nullptr; void* pv = node->data; - mem_allocator->free(node); + s_node_free(node); return pv; } -void* ZHash::lru_first() +void* ZHash::get(const void* key, bool *new_node) { - HashNode* node = lru_cache->get_lru_node(); - return node ? node->data : nullptr; + int row; + HashNode* node = find_node_row(key, row); + + if ( node ) + return node->data; + + node = get_free_node(); + + if ( !node ) + return nullptr; + + memcpy(node->key, key, keysize); + + node->rindex = row; + link_node (node); + glink_node(node); + + count++; + + if (new_node) + *new_node = true; + + return node->data; +} + +void* ZHash::find(const void* key) +{ + int row; + HashNode* node = find_node_row(key, row); + + if ( node ) + return node->data; + + return nullptr; } -void* ZHash::lru_next() +void* ZHash::first() { - HashNode* node = lru_cache->get_next_lru_node(); - return node ? node->data : nullptr; + cursor = gtail; + return cursor ? cursor->data : nullptr; } -void* ZHash::lru_current() +void* ZHash::next() { - HashNode* node = lru_cache->get_current_node(); - return node ? node->data : nullptr; + if ( !cursor ) + return nullptr; + + cursor = cursor->gprev; + return cursor ? cursor->data : nullptr; +} + +void* ZHash::current() +{ + return cursor ? cursor->data : nullptr; +} + +bool ZHash::touch() +{ + HashNode* node = cursor; + + if ( !node ) + return false; + + cursor = cursor->gprev; + + if ( node != ghead ) + { + gunlink_node(node); + glink_node(node); + return true; + } + return false; +} + +bool ZHash::move_to_free_list(HashNode* node) +{ + if ( !node ) + return false; + + unlink_node(node); + gunlink_node(node); + count--; + save_free_node(node); + + return true; +} + +bool ZHash::release() +{ + HashNode* node = cursor; + cursor = nullptr; + return move_to_free_list(node); +} + +bool ZHash::release(const void* key) +{ + int row; + HashNode* node = find_node_row(key, row); + return move_to_free_list(node); +} + +void* ZHash::remove(const void* key) +{ + void* pv = nullptr; + int row; + HashNode* node = find_node_row(key, row); + if ( node ) + { + unlink_node(node); + gunlink_node(node); + count--; + pv = node->data; + s_node_free(node); + } + + return pv; } -void ZHash::lru_touch() +void ZHash::set_key_opcodes(hash_func hash_fcn, keycmp_func keycmp_fcn) { - HashNode* node = lru_cache->get_current_node(); - assert(node); - lru_cache->touch(node); + hashfcn_set_keyops(hashfcn, hash_fcn, keycmp_fcn); } diff --git a/src/hash/zhash.h b/src/hash/zhash.h index f7c2370bc..27584532b 100644 --- a/src/hash/zhash.h +++ b/src/hash/zhash.h @@ -22,12 +22,18 @@ #include -#include "hash/xhash.h" +#include "hashfcn.h" -class ZHash : public snort::XHash +namespace snort +{ +struct HashNode; +} + +class ZHash { public: ZHash(int nrows, int keysize); + ~ZHash(); ZHash(const ZHash&) = delete; ZHash& operator=(const ZHash&) = delete; @@ -35,13 +41,51 @@ public: void* push(void* p); void* pop(); - void* get(const void* key); - void* remove(); + void* first(); + void* next(); + void* current(); + bool touch(); + + void* find(const void* key); + void* get(const void* key, bool *new_node = nullptr); + bool release(const void* key); + bool release(); + void* remove(const void* key); + void set_key_opcodes(hash_func, keycmp_func); + + inline unsigned get_count() + { return count; } + +private: + snort::HashNode* get_free_node(); + snort::HashNode* find_node_row(const void*, int&); + + void glink_node(snort::HashNode*); + void gunlink_node(snort::HashNode*); + + void link_node(snort::HashNode*); + void unlink_node(snort::HashNode*); + + void delete_free_list(); + void save_free_node(snort::HashNode*); + + bool move_to_free_list(snort::HashNode*); + void move_to_front(snort::HashNode*); + +private: + HashFnc* hashfcn; + int keysize; + unsigned nrows; + unsigned count; + + unsigned find_fail; + unsigned find_success; - void* lru_first(); - void* lru_next(); - void* lru_current(); - void lru_touch(); + snort::HashNode** table; + snort::HashNode* ghead; + snort::HashNode* gtail; + snort::HashNode* fhead; + snort::HashNode* cursor; }; #endif diff --git a/src/ips_options/ips_ack.cc b/src/ips_options/ips_ack.cc index 14e382481..3aa0c676e 100644 --- a/src/ips_options/ips_ack.cc +++ b/src/ips_options/ips_ack.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler_defs.h" #include "protocols/packet.h" #include "protocols/tcp.h" diff --git a/src/ips_options/ips_asn1.cc b/src/ips_options/ips_asn1.cc index cb067e213..c10ef9f96 100644 --- a/src/ips_options/ips_asn1.cc +++ b/src/ips_options/ips_asn1.cc @@ -56,7 +56,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_base64.cc b/src/ips_options/ips_base64.cc index df31e3ef5..aebd49263 100644 --- a/src/ips_options/ips_base64.cc +++ b/src/ips_options/ips_base64.cc @@ -25,7 +25,7 @@ #include "detection/detection_engine.h" #include "detection/treenodes.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" diff --git a/src/ips_options/ips_ber_data.cc b/src/ips_options/ips_ber_data.cc index 06b521bee..2a24f7808 100644 --- a/src/ips_options/ips_ber_data.cc +++ b/src/ips_options/ips_ber_data.cc @@ -24,7 +24,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "utils/util_ber.h" diff --git a/src/ips_options/ips_ber_skip.cc b/src/ips_options/ips_ber_skip.cc index 4d4765cde..9cdae7563 100644 --- a/src/ips_options/ips_ber_skip.cc +++ b/src/ips_options/ips_ber_skip.cc @@ -24,7 +24,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "utils/util_ber.h" diff --git a/src/ips_options/ips_bufferlen.cc b/src/ips_options/ips_bufferlen.cc index 0aa305372..a347f7ed3 100644 --- a/src/ips_options/ips_bufferlen.cc +++ b/src/ips_options/ips_bufferlen.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" using namespace snort; diff --git a/src/ips_options/ips_byte_extract.cc b/src/ips_options/ips_byte_extract.cc index 9213600c8..c55bbb3c4 100644 --- a/src/ips_options/ips_byte_extract.cc +++ b/src/ips_options/ips_byte_extract.cc @@ -27,7 +27,7 @@ #include "framework/endianness.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/ips_options/ips_byte_jump.cc b/src/ips_options/ips_byte_jump.cc index ea672a92c..65d58a1f2 100644 --- a/src/ips_options/ips_byte_jump.cc +++ b/src/ips_options/ips_byte_jump.cc @@ -81,7 +81,7 @@ #include "framework/endianness.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_byte_math.cc b/src/ips_options/ips_byte_math.cc index 00ce2dd1e..434495141 100644 --- a/src/ips_options/ips_byte_math.cc +++ b/src/ips_options/ips_byte_math.cc @@ -27,7 +27,7 @@ #include "framework/endianness.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_byte_test.cc b/src/ips_options/ips_byte_test.cc index 2ad0b406e..5ba9bd3db 100644 --- a/src/ips_options/ips_byte_test.cc +++ b/src/ips_options/ips_byte_test.cc @@ -97,7 +97,7 @@ #include "framework/endianness.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_content.cc b/src/ips_options/ips_content.cc index 0830bbdec..3a39ecfc3 100644 --- a/src/ips_options/ips_content.cc +++ b/src/ips_options/ips_content.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "helpers/literal_search.h" #include "log/messages.h" #include "parser/parse_utils.h" diff --git a/src/ips_options/ips_cvs.cc b/src/ips_options/ips_cvs.cc index b17c8f893..d09b2adce 100644 --- a/src/ips_options/ips_cvs.cc +++ b/src/ips_options/ips_cvs.cc @@ -41,7 +41,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_dsize.cc b/src/ips_options/ips_dsize.cc index 14621adf8..52f0f7e7f 100644 --- a/src/ips_options/ips_dsize.cc +++ b/src/ips_options/ips_dsize.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_file_type.cc b/src/ips_options/ips_file_type.cc index 7797fcd5f..61f762aac 100644 --- a/src/ips_options/ips_file_type.cc +++ b/src/ips_options/ips_file_type.cc @@ -29,7 +29,7 @@ #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" using namespace snort; diff --git a/src/ips_options/ips_flags.cc b/src/ips_options/ips_flags.cc index b0453a830..bb83a5cbd 100644 --- a/src/ips_options/ips_flags.cc +++ b/src/ips_options/ips_flags.cc @@ -26,7 +26,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_flow.cc b/src/ips_options/ips_flow.cc index 789c44bbd..ac71613c2 100644 --- a/src/ips_options/ips_flow.cc +++ b/src/ips_options/ips_flow.cc @@ -27,7 +27,7 @@ #include "detection/treenodes.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_flowbits.cc b/src/ips_options/ips_flowbits.cc index 5b6336faf..4205b3264 100644 --- a/src/ips_options/ips_flowbits.cc +++ b/src/ips_options/ips_flowbits.cc @@ -46,8 +46,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "hash/ghash.h" -#include "hash/hash_defs.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "parser/mstring.h" #include "protocols/packet.h" @@ -619,7 +618,7 @@ static FLOWBITS_OBJECT* getFlowBitItem(char* flowbitName, FLOWBITS_OP* flowbits, } int hstatus = flowbit_state->flowbits_hash->insert(flowbitName, flowbits_item); - if (hstatus != HASH_OK) + if (hstatus != GHASH_OK) ParseError("Could not add flowbits key (%s) to hash.",flowbitName); } flowbits_item->toggle = flowbit_state->flowbits_toggle; @@ -812,7 +811,7 @@ static FLOWBITS_GRP* getFlowBitGroup(char* groupName, FlowBitState* flowbit_stat // new group defined, add (bitop set later once we know size) flowbits_grp = (FLOWBITS_GRP*)snort_calloc(sizeof(*flowbits_grp)); int hstatus = flowbit_state->flowbits_grp_hash->insert(groupName, flowbits_grp); - if (hstatus != HASH_OK) + if (hstatus != GHASH_OK) ParseAbort("Could not add flowbits group (%s) to hash.\n",groupName); flowbit_state->flowbits_grp_count++; diff --git a/src/ips_options/ips_fragbits.cc b/src/ips_options/ips_fragbits.cc index 48a147d66..10ae82cee 100644 --- a/src/ips_options/ips_fragbits.cc +++ b/src/ips_options/ips_fragbits.cc @@ -52,7 +52,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_fragoffset.cc b/src/ips_options/ips_fragoffset.cc index e9b6f3e76..614237daa 100644 --- a/src/ips_options/ips_fragoffset.cc +++ b/src/ips_options/ips_fragoffset.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_hash.cc b/src/ips_options/ips_hash.cc index b3b6f741a..db237d187 100644 --- a/src/ips_options/ips_hash.cc +++ b/src/ips_options/ips_hash.cc @@ -27,7 +27,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "hash/hashes.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "parser/parse_utils.h" #include "profiler/profiler.h" diff --git a/src/ips_options/ips_icmp_id.cc b/src/ips_options/ips_icmp_id.cc index a17ad7246..e7296e4ad 100644 --- a/src/ips_options/ips_icmp_id.cc +++ b/src/ips_options/ips_icmp_id.cc @@ -46,7 +46,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/icmp4.h" #include "protocols/icmp6.h" diff --git a/src/ips_options/ips_icmp_seq.cc b/src/ips_options/ips_icmp_seq.cc index d16d372f8..4f56193e9 100644 --- a/src/ips_options/ips_icmp_seq.cc +++ b/src/ips_options/ips_icmp_seq.cc @@ -46,7 +46,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/icmp4.h" #include "protocols/icmp6.h" diff --git a/src/ips_options/ips_icode.cc b/src/ips_options/ips_icode.cc index ac2f3b471..e1b00daf0 100644 --- a/src/ips_options/ips_icode.cc +++ b/src/ips_options/ips_icode.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/icmp4.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_id.cc b/src/ips_options/ips_id.cc index 09fcb78e5..32a72e292 100644 --- a/src/ips_options/ips_id.cc +++ b/src/ips_options/ips_id.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_ip_proto.cc b/src/ips_options/ips_ip_proto.cc index c263d595e..d37f8eced 100644 --- a/src/ips_options/ips_ip_proto.cc +++ b/src/ips_options/ips_ip_proto.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_ipopts.cc b/src/ips_options/ips_ipopts.cc index 48c0480d3..4c80b430d 100644 --- a/src/ips_options/ips_ipopts.cc +++ b/src/ips_options/ips_ipopts.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/ipv4_options.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_isdataat.cc b/src/ips_options/ips_isdataat.cc index f39124891..0de1051a5 100644 --- a/src/ips_options/ips_isdataat.cc +++ b/src/ips_options/ips_isdataat.cc @@ -42,7 +42,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "parser/mstring.h" #include "profiler/profiler.h" diff --git a/src/ips_options/ips_itype.cc b/src/ips_options/ips_itype.cc index eb65fa551..327b3cece 100644 --- a/src/ips_options/ips_itype.cc +++ b/src/ips_options/ips_itype.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/icmp4.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_luajit.cc b/src/ips_options/ips_luajit.cc index 07f66e870..5018984db 100644 --- a/src/ips_options/ips_luajit.cc +++ b/src/ips_options/ips_luajit.cc @@ -24,7 +24,7 @@ #include "framework/cursor.h" #include "framework/decode_data.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "helpers/chunk.h" #include "lua/lua.h" #include "log/messages.h" diff --git a/src/ips_options/ips_pcre.cc b/src/ips_options/ips_pcre.cc index 56f498996..5da5a8e3d 100644 --- a/src/ips_options/ips_pcre.cc +++ b/src/ips_options/ips_pcre.cc @@ -31,7 +31,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/parameter.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "helpers/scratch_allocator.h" #include "log/messages.h" #include "main/snort_config.h" diff --git a/src/ips_options/ips_regex.cc b/src/ips_options/ips_regex.cc index 6159e5e23..74be74f70 100644 --- a/src/ips_options/ips_regex.cc +++ b/src/ips_options/ips_regex.cc @@ -32,7 +32,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "helpers/hyper_scratch_allocator.h" #include "log/messages.h" #include "main/snort_config.h" diff --git a/src/ips_options/ips_replace.cc b/src/ips_options/ips_replace.cc index 50071f2ee..8c10b64d9 100644 --- a/src/ips_options/ips_replace.cc +++ b/src/ips_options/ips_replace.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "main/snort_config.h" #include "main/thread_config.h" diff --git a/src/ips_options/ips_rpc.cc b/src/ips_options/ips_rpc.cc index 7c3f27eb9..7fe436a2e 100644 --- a/src/ips_options/ips_rpc.cc +++ b/src/ips_options/ips_rpc.cc @@ -26,7 +26,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_sd_pattern.cc b/src/ips_options/ips_sd_pattern.cc index 3a09e2ef9..83f68b705 100644 --- a/src/ips_options/ips_sd_pattern.cc +++ b/src/ips_options/ips_sd_pattern.cc @@ -31,7 +31,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "helpers/hyper_scratch_allocator.h" #include "log/messages.h" #include "log/obfuscator.h" diff --git a/src/ips_options/ips_seq.cc b/src/ips_options/ips_seq.cc index c70e8dbfa..d3d145895 100644 --- a/src/ips_options/ips_seq.cc +++ b/src/ips_options/ips_seq.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "protocols/tcp.h" diff --git a/src/ips_options/ips_session.cc b/src/ips_options/ips_session.cc index 598edbe46..c86362ab4 100644 --- a/src/ips_options/ips_session.cc +++ b/src/ips_options/ips_session.cc @@ -49,7 +49,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "main/snort_config.h" #include "profiler/profiler.h" diff --git a/src/ips_options/ips_so.cc b/src/ips_options/ips_so.cc index f94702210..147b142fe 100644 --- a/src/ips_options/ips_so.cc +++ b/src/ips_options/ips_so.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/so_rule.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "main/snort_config.h" #include "managers/so_manager.h" diff --git a/src/ips_options/ips_tos.cc b/src/ips_options/ips_tos.cc index 075eaa813..ddf32df9c 100644 --- a/src/ips_options/ips_tos.cc +++ b/src/ips_options/ips_tos.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_ttl.cc b/src/ips_options/ips_ttl.cc index c190aaa6e..93d7acb0e 100644 --- a/src/ips_options/ips_ttl.cc +++ b/src/ips_options/ips_ttl.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/ips_options/ips_window.cc b/src/ips_options/ips_window.cc index 8b1711377..e0e140848 100644 --- a/src/ips_options/ips_window.cc +++ b/src/ips_options/ips_window.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "protocols/tcp.h" diff --git a/src/log/messages.cc b/src/log/messages.cc index c4c0ad476..6c5948f05 100644 --- a/src/log/messages.cc +++ b/src/log/messages.cc @@ -27,6 +27,8 @@ #include #include +#include +#include #include "main/snort_config.h" #include "parser/parser.h" @@ -41,8 +43,6 @@ static unsigned parse_errors = 0; static unsigned parse_warnings = 0; static unsigned reload_errors = 0; -static std::string reload_errors_description; - void reset_parse_errors() { parse_errors = 0; @@ -63,22 +63,11 @@ unsigned get_parse_warnings() return tmp; } -void reset_reload_errors() -{ - reload_errors = 0; - reload_errors_description.clear(); -} - unsigned get_reload_errors() { return reload_errors; } -std::string& get_reload_errors_description() -{ - return reload_errors_description; -} - static void log_message(FILE* file, const char* type, const char* msg) { const char* file_name; @@ -136,20 +125,12 @@ void ReloadError(const char* format, ...) va_list ap; va_start(ap, format); - vsnprintf(buf, sizeof(buf), format, ap); + vsnprintf(buf, STD_BUF, format, ap); va_end(ap); - buf[sizeof(buf)-1] = '\0'; + buf[STD_BUF] = '\0'; log_message(stderr, "ERROR", buf); - if ( reload_errors_description.empty() ) - reload_errors_description = buf; - else - { - reload_errors_description += ","; - reload_errors_description += buf; - } - reload_errors++; } diff --git a/src/log/messages.h b/src/log/messages.h index 316895d6d..f830a4cbf 100644 --- a/src/log/messages.h +++ b/src/log/messages.h @@ -23,7 +23,6 @@ #include #include -#include #include #include "main/snort_types.h" @@ -50,9 +49,7 @@ enum WarningGroup void reset_parse_errors(); unsigned get_parse_errors(); unsigned get_parse_warnings(); -void reset_reload_errors(); unsigned get_reload_errors(); -std::string& get_reload_errors_description(); namespace snort { diff --git a/src/main.cc b/src/main.cc index f95e191b5..e8b1a3095 100644 --- a/src/main.cc +++ b/src/main.cc @@ -351,12 +351,7 @@ int main_reload_config(lua_State* L) if ( !sc ) { if (get_reload_errors()) - { - std::string response_message = "== reload failed - restart required - "; - response_message += get_reload_errors_description() + "\n"; - current_request->respond(response_message.c_str()); - reset_reload_errors(); - } + current_request->respond("== reload failed - restart required\n"); else current_request->respond("== reload failed - bad config\n"); diff --git a/src/main/analyzer.cc b/src/main/analyzer.cc index c825b2bfd..922f54409 100644 --- a/src/main/analyzer.cc +++ b/src/main/analyzer.cc @@ -616,7 +616,6 @@ void Analyzer::reinit(SnortConfig* sc) { InspectorManager::thread_reinit(sc); ActionManager::thread_reinit(sc); - IpsManager::thread_reinit(sc); } void Analyzer::term() diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc index 3fb2de0cf..a516f4cca 100644 --- a/src/main/snort_config.cc +++ b/src/main/snort_config.cc @@ -498,53 +498,160 @@ void SnortConfig::merge(SnortConfig* cmd_line) state = new std::vector[num_slots]; } -bool SnortConfig::verify() +// FIXIT-L this is a work around till snort supports adding/removing +// stream cache during reload +bool SnortConfig::verify_stream_inspectors() { - bool config_ok = false; + const std::vector inspector_names + { "stream_file", "stream_icmp", "stream_ip", "stream_tcp", "stream_udp", "stream_user" }; + static std::map orig_inspectors; + + // If wasn't initialized before try to initialize from current config + if (orig_inspectors.empty()) + { + const Inspector* const ptr = InspectorManager::get_inspector("stream", true); + if (ptr != nullptr) + { + for (auto name: inspector_names) + { + const bool in_orig = InspectorManager::inspector_exists_in_any_policy(name, get_conf()); + orig_inspectors[name] = in_orig; + } + } + } + // If now available - compare + if (!orig_inspectors.empty()) + { + const Inspector* const ptr = InspectorManager::get_inspector("stream", true, this); + if (ptr != nullptr) + { + for (auto name: inspector_names) + { + const bool in_new = InspectorManager::inspector_exists_in_any_policy(name, this); + if (orig_inspectors[name] != in_new) + { + ReloadError("Snort Reload: Adding/removing %s requires a restart.\n", name); + return false; + } + } + } + } + + return true; +} + +bool SnortConfig::verify() +{ if (get_conf()->asn1_mem != asn1_mem) - ReloadError("Changing detection.asn1_mem requires a restart.\n"); + { + ReloadError("Snort Reload: Changing the asn1 memory configuration " + "requires a restart.\n"); + return false; + } - else if ( get_conf()->bpf_filter != bpf_filter ) - ReloadError("Changing packets.bfp_filter requires a restart.\n"); + if ( bpf_filter != get_conf()->bpf_filter ) + { + ReloadError("Snort Reload: Changing the bpf filter configuration " + "requires a restart.\n"); + return false; + } - else if ( get_conf()->respond_attempts != respond_attempts ) - ReloadError("Changing active.attempts requires a restart.\n"); + if ( respond_attempts != get_conf()->respond_attempts || + respond_device != get_conf()->respond_device ) + { + ReloadError("Snort Reload: Changing config response requires a restart.\n"); + return false; + } - else if ( get_conf()->respond_device != respond_device ) - ReloadError("Changing active.device requires a restart.\n"); + if (get_conf()->chroot_dir != chroot_dir) + { + ReloadError("Snort Reload: Changing the chroot directory " + "configuration requires a restart.\n"); + return false; + } - else if (get_conf()->chroot_dir != chroot_dir) - ReloadError("Changing process.chroot requires a restart.\n"); + if ((get_conf()->run_flags & RUN_FLAG__DAEMON) != + (run_flags & RUN_FLAG__DAEMON)) + { + ReloadError("Snort Reload: Changing to or from daemon mode " + "requires a restart.\n"); + return false; + } - else if ((get_conf()->run_flags & RUN_FLAG__DAEMON) != (run_flags & RUN_FLAG__DAEMON)) - ReloadError("Changing process.daemon requires a restart.\n"); + /* Orig log dir because a chroot might have changed it */ + if (get_conf()->orig_log_dir != orig_log_dir) + { + ReloadError("Snort Reload: Changing the log directory " + "configuration requires a restart.\n"); + return false; + } - else if (get_conf()->orig_log_dir != orig_log_dir) - ReloadError("Changing output.logdir requires a restart.\n"); + if (get_conf()->max_attribute_hosts != max_attribute_hosts) + { + ReloadError("Snort Reload: Changing max_attribute_hosts " + "configuration requires a restart.\n"); + return false; + } + if (get_conf()->max_attribute_services_per_host != max_attribute_services_per_host) + { + ReloadError("Snort Reload: Changing max_attribute_services_per_host " + "configuration requires a restart.\n"); + return false; + } - else if (get_conf()->group_id != group_id) - ReloadError("Changing process.setgid requires a restart.\n"); + if ((get_conf()->run_flags & RUN_FLAG__NO_PROMISCUOUS) != + (run_flags & RUN_FLAG__NO_PROMISCUOUS)) + { + ReloadError("Snort Reload: Changing to or from promiscuous mode " + "requires a restart.\n"); + return false; + } - else if (get_conf()->user_id != user_id) - ReloadError("Changing process.setuid requires a restart.\n"); + if (get_conf()->group_id != group_id) + { + ReloadError("Snort Reload: Changing the group id configuration requires a restart.\n"); + return false; + } - else if (get_conf()->daq_config->get_mru_size() != daq_config->get_mru_size()) - ReloadError("Changing daq.snaplen requires a restart.\n"); + if (get_conf()->user_id != user_id) + { + ReloadError("Snort Reload: Changing the user id configuration requires a restart.\n"); + return false; + } - else if (get_conf()->threshold_config->memcap != threshold_config->memcap) - ReloadError("Changing alerts.event_filter_memcap requires a restart.\n"); + if (get_conf()->daq_config->get_mru_size() != daq_config->get_mru_size()) + { + ReloadError("Snort Reload: Changing the packet snaplen " + "configuration requires a restart.\n"); + return false; + } - else if (get_conf()->rate_filter_config->memcap != rate_filter_config->memcap) - ReloadError("Changing alerts.rate_filter_memcap requires a restart.\n"); + if (get_conf()->threshold_config->memcap != + threshold_config->memcap) + { + ReloadError("Snort Reload: Changing the threshold memcap " + "configuration requires a restart.\n"); + return false; + } - else if (get_conf()->detection_filter_config->memcap != detection_filter_config->memcap) - ReloadError("Changing alerts.detection_filter_memcap requires a restart.\n"); + if (get_conf()->rate_filter_config->memcap != + rate_filter_config->memcap) + { + ReloadError("Snort Reload: Changing the rate filter memcap " + "configuration requires a restart.\n"); + return false; + } - else - config_ok = true; + if (get_conf()->detection_filter_config->memcap != + detection_filter_config->memcap) + { + ReloadError("Snort Reload: Changing the detection filter memcap " + "configuration requires a restart.\n"); + return false; + } - return config_ok; + return verify_stream_inspectors(); } void SnortConfig::set_alert_before_pass(bool enabled) diff --git a/src/main/snort_config.h b/src/main/snort_config.h index ce428f451..af2b992f1 100644 --- a/src/main/snort_config.h +++ b/src/main/snort_config.h @@ -41,7 +41,7 @@ enum RunFlag { RUN_FLAG__READ = 0x00000001, RUN_FLAG__DAEMON = 0x00000002, - // unused = 0x00000004, + RUN_FLAG__NO_PROMISCUOUS = 0x00000004, // unused = 0x00000008, RUN_FLAG__INLINE = 0x00000010, @@ -149,8 +149,8 @@ struct VarNode; namespace snort { -class GHash; class ProtocolReference; +class GHash; class XHash; struct ProfilerConfig; struct SnortConfig; @@ -181,6 +181,7 @@ struct SnortConfig { private: void init(const SnortConfig* const, ProtocolReference*); + bool verify_stream_inspectors(); public: SnortConfig(const SnortConfig* const other_conf = nullptr); diff --git a/src/managers/ips_manager.cc b/src/managers/ips_manager.cc index 9cfab779b..089432347 100644 --- a/src/managers/ips_manager.cc +++ b/src/managers/ips_manager.cc @@ -41,11 +41,10 @@ struct Option { const IpsApi* api; bool init; - bool thread_init; unsigned count; Option(const IpsApi* p) - { api = p; init = false; thread_init = false; count = 0; } + { api = p; init = false; count = 0; } }; typedef map OptionMap; @@ -360,11 +359,8 @@ void IpsManager::reset_options() void IpsManager::setup_options() { for ( auto& p : s_options ) - if ( p.second->init && !p.second->thread_init && p.second->api->tinit ) - { + if ( p.second->init && p.second->api->tinit ) p.second->api->tinit(SnortConfig::get_conf()); - p.second->thread_init = true; - } } void IpsManager::clear_options() @@ -383,9 +379,6 @@ bool IpsManager::verify(SnortConfig* sc) return true; } -void IpsManager::thread_reinit(snort::SnortConfig*) -{ IpsManager::setup_options(); } - #ifdef PIGLET static const IpsApi* find_api(const char* name) diff --git a/src/managers/ips_manager.h b/src/managers/ips_manager.h index 1ee3e1236..54a665c52 100644 --- a/src/managers/ips_manager.h +++ b/src/managers/ips_manager.h @@ -74,7 +74,6 @@ public: static void setup_options(); static void clear_options(); static bool verify(snort::SnortConfig*); - static void thread_reinit(snort::SnortConfig*); #ifdef PIGLET static IpsOptionWrapper* instantiate(const char*, snort::Module*, struct OptTreeNode*); diff --git a/src/mime/file_mime_process.cc b/src/mime/file_mime_process.cc index 4cfcc4603..7c962b619 100644 --- a/src/mime/file_mime_process.cc +++ b/src/mime/file_mime_process.cc @@ -28,7 +28,7 @@ #include "detection/detection_engine.h" #include "file_api/file_flows.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "search_engines/search_tool.h" #include "utils/util_cstring.h" diff --git a/src/network_inspectors/appid/ips_appid_option.cc b/src/network_inspectors/appid/ips_appid_option.cc index 77a53b064..ad8f530e3 100644 --- a/src/network_inspectors/appid/ips_appid_option.cc +++ b/src/network_inspectors/appid/ips_appid_option.cc @@ -27,7 +27,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "utils/util.h" diff --git a/src/network_inspectors/perf_monitor/flow_ip_tracker.cc b/src/network_inspectors/perf_monitor/flow_ip_tracker.cc index 9a4d0c5ac..85c5eb02c 100644 --- a/src/network_inspectors/perf_monitor/flow_ip_tracker.cc +++ b/src/network_inspectors/perf_monitor/flow_ip_tracker.cc @@ -24,7 +24,6 @@ #include "flow_ip_tracker.h" -#include "hash/hash_defs.h" #include "log/messages.h" #include "protocols/packet.h" @@ -47,8 +46,9 @@ FlowStateValue* FlowIPTracker::find_stats(const SfIp* src_addr, const SfIp* dst_ { FlowStateKey key; FlowStateValue* value = nullptr; + bool prune_required = false; - if ( src_addr->less_than(*dst_addr) ) + if (src_addr->less_than(*dst_addr)) { key.ipA = *src_addr; key.ipB = *dst_addr; @@ -62,12 +62,21 @@ FlowStateValue* FlowIPTracker::find_stats(const SfIp* src_addr, const SfIp* dst_ } value = (FlowStateValue*)ip_map->get_user_data(&key); - if ( !value ) + if (!value) { - if ( ip_map->insert(&key, nullptr) != HASH_OK ) + HashNode* node = ip_map->get_node_with_prune(&key, &prune_required); + + if (!node) return nullptr; - value = (FlowStateValue*)ip_map->get_user_data(); - memset(value, 0, sizeof(FlowStateValue)); + + if (prune_required) + { + ++pmstats.total_frees; + ++pmstats.alloc_prunes; + } + + memset(node->data, 0, sizeof(FlowStateValue)); + value = (FlowStateValue*)node->data; } return value; @@ -79,8 +88,8 @@ bool FlowIPTracker::initialize(size_t new_memcap) if ( !ip_map ) { - ip_map = new XHash(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), - sizeof(FlowStateValue), new_memcap); + ip_map = new XHash(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), sizeof(FlowStateValue), + new_memcap, true, nullptr, nullptr, true); } else { @@ -131,25 +140,23 @@ FlowIPTracker::FlowIPTracker(PerfConfig* perf) : PerfTracker(perf, TRACKER_NAME) formatter->finalize_fields(); memcap = perf->flowip_memcap; - ip_map = new XHash(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), sizeof(FlowStateValue), memcap); + ip_map = new XHash(DEFAULT_XHASH_NROWS, sizeof(FlowStateKey), sizeof(FlowStateValue), + memcap, true, nullptr, nullptr, true); } FlowIPTracker::~FlowIPTracker() { - const XHashStats& stats = ip_map->get_stats(); - pmstats.flow_tracker_creates = stats.nodes_created; - pmstats.flow_tracker_total_deletes = stats.memcap_deletes; - pmstats.flow_tracker_prunes = stats.memcap_prunes; - delete ip_map; } void FlowIPTracker::reset() -{ ip_map->clear_hash(); } +{ + ip_map->clear(); +} void FlowIPTracker::update(Packet* p) { - if ( p->has_ip() && !p->is_rebuilt() ) + if (p->has_ip() && !p->is_rebuilt()) { FlowType type = SFS_TYPE_OTHER; int swapped; @@ -164,12 +171,12 @@ void FlowIPTracker::update(Packet* p) type = SFS_TYPE_UDP; FlowStateValue* value = find_stats(src_addr, dst_addr, &swapped); - if ( !value ) + if (!value) return; TrafficStats* stats = &value->traffic_stats[type]; - if ( !swapped ) + if (!swapped) { stats->packets_a_to_b++; stats->bytes_a_to_b += len; @@ -207,10 +214,11 @@ int FlowIPTracker::update_state(const SfIp* src_addr, const SfIp* dst_addr, Flow int swapped; FlowStateValue* value = find_stats(src_addr, dst_addr, &swapped); - if ( !value ) + if (!value) return 1; value->state_changes[state]++; + return 0; } diff --git a/src/network_inspectors/perf_monitor/perf_monitor.cc b/src/network_inspectors/perf_monitor/perf_monitor.cc index 8c038418f..9df381e3b 100644 --- a/src/network_inspectors/perf_monitor/perf_monitor.cc +++ b/src/network_inspectors/perf_monitor/perf_monitor.cc @@ -29,7 +29,6 @@ #endif #include "framework/data_bus.h" -#include "hash/hash_defs.h" #include "hash/xhash.h" #include "log/messages.h" #include "managers/inspector_manager.h" @@ -271,8 +270,9 @@ bool PerfMonReloadTuner::tune_resources(unsigned work_limit) if (flow_ip_tracker) { unsigned num_freed = 0; - int result = flow_ip_tracker->get_ip_map()->tune_memory_resources(work_limit, num_freed); - pmstats.flow_tracker_reload_deletes += num_freed; + int result = flow_ip_tracker->get_ip_map()->free_over_allocations(work_limit, &num_freed); + pmstats.total_frees += num_freed; + pmstats.reload_frees += num_freed; return (result == HASH_OK); } else diff --git a/src/network_inspectors/perf_monitor/perf_pegs.h b/src/network_inspectors/perf_monitor/perf_pegs.h index 8d503970d..26628b198 100644 --- a/src/network_inspectors/perf_monitor/perf_pegs.h +++ b/src/network_inspectors/perf_monitor/perf_pegs.h @@ -27,21 +27,20 @@ static const PegInfo perf_module_pegs[] = { + { CountType::SUM, "packets", "total packets processed by performance monitor" }, - { CountType::SUM, "flow_tracker_creates", "total number of flow trackers created" }, - { CountType::SUM, "flow_tracker_total_deletes", "flow trackers deleted to stay below memcap limit" }, - { CountType::SUM, "flow_tracker_reload_deletes", "flow trackers deleted due to memcap change on config reload" }, - { CountType::SUM, "flow_tracker_prunes", "flow trackers pruned for reuse by new flows" }, + { CountType::SUM, "total_frees", "total flows pruned or freed by performance monitor" }, + { CountType::SUM, "reload_frees", "flows freed on reload with changed memcap" }, + { CountType::SUM, "alloc_prunes", "flows pruned on allocation of IP flows" }, { CountType::END, nullptr, nullptr }, }; struct PerfPegStats { PegCount total_packets; - PegCount flow_tracker_creates; - PegCount flow_tracker_total_deletes; - PegCount flow_tracker_reload_deletes; - PegCount flow_tracker_prunes; + PegCount total_frees; + PegCount reload_frees; + PegCount alloc_prunes; }; #endif diff --git a/src/network_inspectors/port_scan/ps_detect.cc b/src/network_inspectors/port_scan/ps_detect.cc index 693bb7a9f..4ee4800b3 100644 --- a/src/network_inspectors/port_scan/ps_detect.cc +++ b/src/network_inspectors/port_scan/ps_detect.cc @@ -33,7 +33,6 @@ #include "ps_detect.h" -#include "hash/hash_defs.h" #include "hash/xhash.h" #include "log/messages.h" #include "protocols/icmp4.h" @@ -58,32 +57,7 @@ struct PS_HASH_KEY }; PADDING_GUARD_END -class PortScanCache : public XHash -{ -public: - PortScanCache(unsigned rows, unsigned key_len, unsigned datasize, unsigned memcap) - : XHash(rows, key_len, datasize, memcap) - { } - - bool is_node_recovery_ok(HashNode* hnode) override - { - PS_TRACKER* tracker = (PS_TRACKER*)hnode->data; - - if ( !tracker->priority_node ) - return true; - - /* - ** Cycle through the protos to see if it's past the time. - ** We only get here if we ARE a priority node. - */ - if ( tracker->proto.window >= packet_time() ) - return false; - - return true; - } -}; - -static THREAD_LOCAL PortScanCache* portscan_hash = nullptr; +static THREAD_LOCAL XHash* portscan_hash = nullptr; extern THREAD_LOCAL PsPegStats spstats; PS_PKT::PS_PKT(Packet* p) @@ -110,6 +84,32 @@ PortscanConfig::~PortscanConfig() ipset_free(watch_ip); } +/* +** This function is passed into the hash algorithm, so that +** we only reuse nodes that aren't priority nodes. We have to make +** sure that we only track so many priority nodes, otherwise we could +** have all priority nodes and not be able to allocate more. +*/ +static int ps_tracker_free(void* key, void* data) +{ + if (!key || !data) + return 0; + + PS_TRACKER* tracker = (PS_TRACKER*)data; + + if (!tracker->priority_node) + return 0; + + /* + ** Cycle through the protos to see if it's past the time. + ** We only get here if we ARE a priority node. + */ + if (tracker->proto.window >= packet_time()) + return 1; + + return 0; +} + void ps_cleanup() { if ( portscan_hash ) @@ -132,8 +132,8 @@ bool ps_init_hash(unsigned long memcap) } int rows = memcap / ps_node_size(); - portscan_hash = new PortScanCache(rows, sizeof(PS_HASH_KEY), sizeof(PS_TRACKER), - memcap); + portscan_hash = new XHash(rows, sizeof(PS_HASH_KEY), sizeof(PS_TRACKER), + memcap, true, ps_tracker_free, nullptr, true); return false; } @@ -144,7 +144,7 @@ bool ps_prune_hash(unsigned work_limit) return true; unsigned num_pruned = 0; - int result = portscan_hash->tune_memory_resources(work_limit, num_pruned); + int result = portscan_hash->free_over_allocations(work_limit, &num_pruned); spstats.reload_prunes += num_pruned; return result != HASH_PENDING; } @@ -152,7 +152,7 @@ bool ps_prune_hash(unsigned work_limit) void ps_reset() { if ( portscan_hash ) - portscan_hash->clear_hash(); + portscan_hash->clear(); } // Check scanner and scanned ips to see if we can filter them out. @@ -297,12 +297,12 @@ static PS_TRACKER* ps_tracker_get(PS_HASH_KEY* key) if ( ht ) return ht; - auto prev_count = portscan_hash->get_num_nodes(); + auto prev_count = portscan_hash->get_node_count(); if ( portscan_hash->insert((void*)key, nullptr) != HASH_OK ) return nullptr; ++spstats.trackers; - if ( prev_count == portscan_hash->get_num_nodes() ) + if ( prev_count == portscan_hash->get_node_count() ) ++spstats.alloc_prunes; ht = (PS_TRACKER*)portscan_hash->get_mru_user_data(); diff --git a/src/parser/parser.cc b/src/parser/parser.cc index 05a7b299c..8dd5ef27e 100644 --- a/src/parser/parser.cc +++ b/src/parser/parser.cc @@ -34,8 +34,7 @@ #include "filters/detection_filter.h" #include "filters/rate_filter.h" #include "filters/sfthreshold.h" -#include "hash/ghash.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "hash/xhash.h" #include "helpers/directory.h" #include "log/messages.h" @@ -65,62 +64,6 @@ static struct rule_index_map_t* ruleIndexMap = nullptr; static std::string s_aux_rules; -class RuleTreeHashKeyOps : public HashKeyOperations -{ -public: - RuleTreeHashKeyOps(int rows) - : HashKeyOperations(rows) - { } - - unsigned do_hash(const unsigned char* k, int) override - { - uint32_t a,b,c; - const RuleTreeNodeKey* rtnk = (const RuleTreeNodeKey*)k; - RuleTreeNode* rtn = rtnk->rtn; - - a = rtn->action; - b = rtn->flags; - c = (uint32_t)(uintptr_t)rtn->listhead; - - mix(a,b,c); - - a += (uint32_t)(uintptr_t)rtn->src_portobject; - b += (uint32_t)(uintptr_t)rtn->dst_portobject; - c += (uint32_t)(uintptr_t)rtnk->policyId; - - finalize(a,b,c); - - return c; - } - - bool key_compare(const void* k1, const void* k2, size_t) override - { - assert(k1 && k2); - - const RuleTreeNodeKey* rtnk1 = (const RuleTreeNodeKey*)k1; - const RuleTreeNodeKey* rtnk2 = (const RuleTreeNodeKey*)k2; - - if (rtnk1->policyId != rtnk2->policyId) - return false; - - if (same_headers(rtnk1->rtn, rtnk2->rtn)) - return true; - - return false; - } -}; - -class RuleTreeCache : public XHash -{ -public: - RuleTreeCache(int rows, int key_len) - : XHash(rows, key_len) - { - initialize(new RuleTreeHashKeyOps(nrows)); - anr_enabled = false; - } -}; - //------------------------------------------------------------------------- // private / implementation methods //------------------------------------------------------------------------- @@ -622,6 +565,44 @@ RuleTreeNode* deleteRtnFromOtn(OptTreeNode* otn, SnortConfig* sc) return deleteRtnFromOtn(otn, get_ips_policy()->policy_id, sc); } +static uint32_t rtn_hash_func(HashFnc*, const unsigned char* k, int) +{ + uint32_t a,b,c; + const RuleTreeNodeKey* rtnk = (const RuleTreeNodeKey*)k; + RuleTreeNode* rtn = rtnk->rtn; + + a = rtn->action; + b = rtn->flags; + c = (uint32_t)(uintptr_t)rtn->listhead; + + mix(a,b,c); + + a += (uint32_t)(uintptr_t)rtn->src_portobject; + b += (uint32_t)(uintptr_t)rtn->dst_portobject; + c += (uint32_t)(uintptr_t)rtnk->policyId; + + finalize(a,b,c); + + return c; +} + +static bool rtn_compare_func(const void* k1, const void* k2, size_t) +{ + const RuleTreeNodeKey* rtnk1 = (const RuleTreeNodeKey*)k1; + const RuleTreeNodeKey* rtnk2 = (const RuleTreeNodeKey*)k2; + + if (!rtnk1 || !rtnk2) + return false; + + if (rtnk1->policyId != rtnk2->policyId) + return false; + + if (same_headers(rtnk1->rtn, rtnk2->rtn)) + return true; + + return false; +} + int addRtnToOtn(SnortConfig* sc, OptTreeNode* otn, RuleTreeNode* rtn, PolicyId policyId) { if (otn->proto_node_num <= policyId) @@ -646,6 +627,7 @@ int addRtnToOtn(SnortConfig* sc, OptTreeNode* otn, RuleTreeNode* rtn, PolicyId p } RuleTreeNode* curr = otn->proto_nodes[policyId]; + if ( curr ) { deleteRtnFromOtn(otn, policyId, sc, (curr->otnRefCount == 1)); @@ -655,7 +637,11 @@ int addRtnToOtn(SnortConfig* sc, OptTreeNode* otn, RuleTreeNode* rtn, PolicyId p rtn->otnRefCount++; if (!sc->rtn_hash_table) - sc->rtn_hash_table = new RuleTreeCache(10000, sizeof(RuleTreeNodeKey)); + { + sc->rtn_hash_table = new XHash( + 10000, sizeof(RuleTreeNodeKey), 0, 0, false, nullptr, nullptr, true); + sc->rtn_hash_table->set_key_opcodes(rtn_hash_func, rtn_compare_func); + } RuleTreeNodeKey key; memset(&key, 0, sizeof(key)); diff --git a/src/ports/port_object.cc b/src/ports/port_object.cc index 50d424cab..5e6114e3d 100644 --- a/src/ports/port_object.cc +++ b/src/ports/port_object.cc @@ -287,7 +287,7 @@ int PortObjectNormalize(PortObject* po) /* PortObjects should be normalized, prior to testing */ -bool PortObjectEqual(PortObject* a, PortObject* b) +int PortObjectEqual(PortObject* a, PortObject* b) { PortObjectItem* pa; PortObjectItem* pb; @@ -295,7 +295,7 @@ bool PortObjectEqual(PortObject* a, PortObject* b) SF_LNODE* posb; if ( a->item_list->count != b->item_list->count ) - return false; + return 0; pa = (PortObjectItem*)sflist_first(a->item_list,&posa); pb = (PortObjectItem*)sflist_first(b->item_list,&posb); @@ -303,16 +303,16 @@ bool PortObjectEqual(PortObject* a, PortObject* b) while ( pa && pb ) { if ( !PortObjectItemsEqual(pa, pb) ) - return false; + return 0; pa = (PortObjectItem*)sflist_next(&posa); pb = (PortObjectItem*)sflist_next(&posb); } if ( pa || pb ) /* both are not done - cannot match */ - return false; + return 0; - return true; /* match */ + return 1; /* match */ } /* diff --git a/src/ports/port_object.h b/src/ports/port_object.h index 3847ab4a4..f4e68174f 100644 --- a/src/ports/port_object.h +++ b/src/ports/port_object.h @@ -61,7 +61,7 @@ PortObject* PortObjectDupPorts(PortObject*); int PortObjectNormalize(PortObject*); void PortObjectToggle(PortObject*); -bool PortObjectEqual(PortObject* poa, PortObject* pob); +int PortObjectEqual(PortObject* poa, PortObject* pob); int PortObjectPortCount(PortObject*); int PortObjectHasPort(PortObject*, int port); diff --git a/src/ports/port_object2.cc b/src/ports/port_object2.cc index fd801521f..090dd7430 100644 --- a/src/ports/port_object2.cc +++ b/src/ports/port_object2.cc @@ -23,9 +23,7 @@ #include "port_object2.h" -#include "hash/ghash.h" -#include "hash/hash_defs.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "parser/parser.h" #include "utils/util.h" @@ -57,29 +55,21 @@ using namespace snort; #define SWAP_BYTES(a) #endif -class PortObject2HashKeyOps : public HashKeyOperations +static unsigned po_rule_hash_func(HashFnc* p, const unsigned char* k, int n) { -public: - PortObject2HashKeyOps(int rows) - : HashKeyOperations(rows) - { } + unsigned char* key; + int ikey = *(const int*)k; - unsigned do_hash(const unsigned char* k, int len) override - { - unsigned char* key; - int ikey = *(const int*)k; - - /* Since the input is really an int, put the bytes into a normalized - * order so that the hash function returns consistent results across - * on BE & LE hardware. */ - SWAP_BYTES(ikey); + /* Since the input is really an int, put the bytes into a normalized + * order so that the hash function returns consistent results across + * on BE & LE hardware. */ + SWAP_BYTES(ikey); - /* Set a pointer to the key to pass to the hashing function */ - key = (unsigned char*)&ikey; + /* Set a pointer to the key to pass to the hashing function */ + key = (unsigned char*)&ikey; - return HashKeyOperations::do_hash(key, len); - } -}; + return hashfcn_hash(p, key, n); +} static int* RuleHashToSortedArray(GHash* rh) { @@ -102,6 +92,14 @@ static int* RuleHashToSortedArray(GHash* rh) return ra; } +static bool port_object_key_compare_func(const void* k1, const void* k2, size_t len) +{ + if ( memcmp(k1, k2, len ) ) + return false; + else + return true; +} + //------------------------------------------------------------------------- // PortObject2 - public //------------------------------------------------------------------------- @@ -111,7 +109,9 @@ PortObject2* PortObject2New(int nrules) PortObject2* po = (PortObject2*)snort_calloc(sizeof(PortObject2)); po->item_list = sflist_new(); po->rule_hash = new GHash(nrules, sizeof(int), 0, snort_free); - po->rule_hash->set_hashkey_ops(new PortObject2HashKeyOps(nrules)); + + /* Use hash function defined above for hashing the key as an int. */ + po->rule_hash->set_key_opcodes(po_rule_hash_func, port_object_key_compare_func); return po; } @@ -189,7 +189,7 @@ PortObject2* PortObject2Dup(PortObject& po) int* prule = (int*)snort_calloc(sizeof(int)); *prule = *prid; - if ( ponew->rule_hash->insert(prule, prule) != HASH_OK ) + if ( ponew->rule_hash->insert(prule, prule) != GHASH_OK ) snort_free(prule); } } @@ -226,7 +226,7 @@ PortObject2* PortObject2AppendPortObject(PortObject2* poa, PortObject* pob) int* prid2 = (int*)snort_calloc(sizeof(int)); *prid2 = *prid; - if ( poa->rule_hash->insert(prid2, prid2) != HASH_OK ) + if ( poa->rule_hash->insert(prid2, prid2) != GHASH_OK ) snort_free(prid2); } return poa; @@ -247,7 +247,7 @@ PortObject2* PortObject2AppendPortObject2(PortObject2* poa, PortObject2* pob) int* prid2 = (int*)snort_calloc(sizeof(int)); *prid2 = *prid; - if ( poa->rule_hash->insert(prid2, prid2) != HASH_OK ) + if ( poa->rule_hash->insert(prid2, prid2) != GHASH_OK ) snort_free(prid2); } return poa; diff --git a/src/ports/port_object2.h b/src/ports/port_object2.h index 67bafcb3c..e95cccf22 100644 --- a/src/ports/port_object2.h +++ b/src/ports/port_object2.h @@ -23,16 +23,13 @@ #define PORT_OBJECT2_H #include "framework/bits.h" +#include "hash/ghash.h" #include "utils/sflsq.h" //------------------------------------------------------------------------- // PortObject2 is similar to PortObject //------------------------------------------------------------------------- -namespace snort -{ -class GHash; -} struct PortObject; diff --git a/src/ports/port_table.cc b/src/ports/port_table.cc index 26bd6a773..71e30bf0e 100644 --- a/src/ports/port_table.cc +++ b/src/ports/port_table.cc @@ -25,9 +25,7 @@ #include -#include "hash/ghash.h" -#include "hash/hash_defs.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "main/snort_debug.h" #include "utils/util.h" @@ -79,6 +77,24 @@ static void plx_free(void* p) snort_free(p); } +static unsigned plx_hash(HashFnc* p, const unsigned char* d, int) +{ + unsigned hash = p->seed; + const plx_t* plx = *(plx_t* const*)d; + + for ( int i = 0; i < plx->n; i++ ) + { + unsigned char* pc_ptr = (unsigned char*)&plx->p[i]; + + for ( unsigned k = 0; k < sizeof(void*); k++ ) + { + hash *= p->scale; + hash += pc_ptr[k]; + } + } + return hash ^ p->hardener; +} + /* for sorting an array of pointers */ static inline int p_keycmp(const void* a, const void* b) { @@ -91,93 +107,84 @@ static inline int p_keycmp(const void* a, const void* b) return 0; /* they are equal */ } -class PlxHashKeyOps : public HashKeyOperations -{ -public: - PlxHashKeyOps(int rows) - : HashKeyOperations(rows) - { } +/* + Hash Key Comparisons for treating plx_t types as Keys - unsigned do_hash(const unsigned char* k, int) override - { - unsigned hash = seed; - const plx_t* plx = *(plx_t* const*)k; + return values memcmp style - for ( int i = 0; i < plx->n; i++ ) - { - unsigned char* pc_ptr = (unsigned char*)&plx->p[i]; - - for ( unsigned k = 0; k < sizeof(void*); k++ ) - { - hash *= scale; - hash += pc_ptr[k]; - } - } - return hash ^ hardener; - } + this only needs to produce 0 => exact match, otherwise not. + -1, and +1 are not strictly needed, they could both return + a non zero value for the purposes of hashing and searching. +*/ +static bool plx_keycmp(const void* a, const void* b, size_t) +{ + const plx_t* pla = *(plx_t* const*)a; + const plx_t* plb = *(plx_t* const*)b; - bool key_compare(const void* k1, const void* k2, size_t) override - { - const plx_t* pla = *(plx_t* const*)k1; - const plx_t* plb = *(plx_t* const*)k2; + if ( pla->n < plb->n ) + return false; - if ( pla->n < plb->n ) - return false; + if ( pla->n > plb->n ) + return false; - if ( pla->n > plb->n ) + for ( int i = 0; i < pla->n; i++ ) + { + if ( p_keycmp(&pla->p[i], &plb->p[i]) ) return false; + } - for ( int i = 0; i < pla->n; i++ ) - { - if ( p_keycmp(&pla->p[i], &plb->p[i]) ) - return false; - } - - return true; /* they are equal */ } -}; + return true; /* they are equal */ +} //------------------------------------------------------------------------- // PortTable - private - other //------------------------------------------------------------------------- -class PortObjectHashKeyOps : public HashKeyOperations -{ -public: - PortObjectHashKeyOps(int rows) - : HashKeyOperations(rows) - { } +/* + Hash Key Comparisons for treating PortObjects as Keys - unsigned do_hash(const unsigned char* k, int) override - { - unsigned hash = seed; - const PortObject* po = *(PortObject* const*)k; - SF_LNODE* pos; + return values memcmp style +*/ +static bool PortObject_keycmp(const void* a, const void* b, size_t) +{ + return PortObjectEqual(*(PortObject* const*)a, *(PortObject* const*)b); +} - for (PortObjectItem* poi = (PortObjectItem*)sflist_first(po->item_list, &pos); - poi != nullptr; - poi = (PortObjectItem*)sflist_next(&pos) ) - { - if ( poi->any() ) - continue; +/* + Hash routine for hashing PortObjects as Keys - hash *= scale; - hash += poi->lport & 0xff; - hash *= scale; - hash += (poi->lport >> 8) & 0xff; + p - HashFnc * + d - PortObject * + n = 4 bytes (sizeof*) - not used - hash *= scale; - hash += poi->hport & 0xff; - hash *= scale; - hash += (poi->hport >> 8) & 0xff; - } - return hash ^ hardener; - } + Don't use this for type=ANY port objects +*/ +static unsigned PortObject_hash(HashFnc* p, const unsigned char* d, int) +{ + unsigned hash = p->seed; + const PortObject* po = *(PortObject* const*)d; + SF_LNODE* pos; - bool key_compare(const void* k1, const void* k2, size_t) override + /* hash up each item */ + for (PortObjectItem* poi = (PortObjectItem*)sflist_first(po->item_list, &pos); + poi != nullptr; + poi = (PortObjectItem*)sflist_next(&pos) ) { - return PortObjectEqual(*(PortObject* const*)k1, *(PortObject* const*)k2); + if ( poi->any() ) + continue; + + hash *= p->scale; + hash += poi->lport & 0xff; + hash *= p->scale; + hash += (poi->lport >> 8) & 0xff; + + hash *= p->scale; + hash += poi->hport & 0xff; + hash *= p->scale; + hash += (poi->hport >> 8) & 0xff; } -}; + return hash ^ p->hardener; +} /* * Merge multiple PortObjects into a final PortObject2, @@ -241,7 +248,7 @@ static PortObject2* _merge_N_pol( // Add the Merged PortObject2 to the PortObject2 hash table keyed by ports. int stat = mhash->insert(&ponew, ponew); // This is possible since PLX hash on a different key - if ( stat == HASH_INTABLE ) + if ( stat == GHASH_INTABLE ) { PortObject2* pox = (PortObject2*)mhash->find(&ponew); assert( pox ); @@ -257,7 +264,7 @@ static PortObject2* _merge_N_pol( // Add the plx node to the PLX hash table stat = mhashx->insert(&plx_tmp, ponew); - if ( stat == HASH_INTABLE ) + if ( stat == GHASH_INTABLE ) FatalError("Could not add merged plx to PLX HASH table-INTABLE\n"); return ponew; @@ -453,12 +460,13 @@ static void PortTableCompileMergePortObjects(PortTable* p) // Create a Merged Port Object Table - hash by ports, no user keys, don't free data GHash* mhash = new GHash(PO_HASH_TBL_ROWS, sizeof(PortObject*), 0, nullptr); - mhash->set_hashkey_ops(new PortObjectHashKeyOps(PO_HASH_TBL_ROWS)); + + mhash->set_key_opcodes(PortObject_hash, PortObject_keycmp); p->pt_mpo_hash = mhash; // Create a Merged Port Object Table - hash by pointers, no user keys, don't free data GHash* mhashx = new GHash(PO_HASH_TBL_ROWS, sizeof(plx_t*), 0, nullptr); - mhashx->set_hashkey_ops(new PlxHashKeyOps(PO_HASH_TBL_ROWS)); + mhashx->set_key_opcodes(plx_hash, plx_keycmp); p->pt_mpxo_hash = mhashx; SF_LIST* plx_list = sflist_new(); diff --git a/src/ports/port_table.h b/src/ports/port_table.h index 6048c9c22..f12fd322a 100644 --- a/src/ports/port_table.h +++ b/src/ports/port_table.h @@ -22,16 +22,12 @@ #ifndef PORT_TABLE_H #define PORT_TABLE_H +#include "hash/ghash.h" #include "ports/port_item.h" #include "ports/port_object.h" #include "ports/port_object2.h" #include "utils/sflsq.h" -namespace snort -{ -class GHash; -} - //------------------------------------------------------------------------- // PortTable - provides support to analyze the Port List objects defined by // the user as either PortVar entries or simply as inline rule port list diff --git a/src/ports/port_var_table.cc b/src/ports/port_var_table.cc index 89ed38436..ea5257ecb 100644 --- a/src/ports/port_var_table.cc +++ b/src/ports/port_var_table.cc @@ -23,9 +23,6 @@ #include "port_var_table.h" -#include "hash/ghash.h" -#include "hash/hash_defs.h" - using namespace snort; //------------------------------------------------------------------------- @@ -75,10 +72,10 @@ int PortVarTableFree(PortVarTable* pvt) int PortVarTableAdd(PortVarTable* h, PortObject* po) { int stat = h->insert(po->name, po); - if ( stat == HASH_INTABLE ) + if ( stat == GHASH_INTABLE ) return 1; - if ( stat == HASH_OK ) + if ( stat == GHASH_OK ) return 0; return -1; diff --git a/src/ports/port_var_table.h b/src/ports/port_var_table.h index 7babca35d..8aedbc6b2 100644 --- a/src/ports/port_var_table.h +++ b/src/ports/port_var_table.h @@ -22,14 +22,10 @@ #ifndef PORT_VAR_TABLE_H #define PORT_VAR_TABLE_H +#include "hash/ghash.h" #include "ports/port_object.h" #include "ports/port_table.h" -namespace snort -{ -class GHash; -} - //------------------------------------------------------------------------- // PortVarTable // port lists may be defined as 'name port-list' diff --git a/src/service_inspectors/cip/ips_cip_attribute.cc b/src/service_inspectors/cip/ips_cip_attribute.cc index c9ff7af14..ada9362ad 100644 --- a/src/service_inspectors/cip/ips_cip_attribute.cc +++ b/src/service_inspectors/cip/ips_cip_attribute.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_class.cc b/src/service_inspectors/cip/ips_cip_class.cc index c5a1d1f16..f7681414a 100644 --- a/src/service_inspectors/cip/ips_cip_class.cc +++ b/src/service_inspectors/cip/ips_cip_class.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_connpathclass.cc b/src/service_inspectors/cip/ips_cip_connpathclass.cc index 8f93c5159..22e9a10f0 100644 --- a/src/service_inspectors/cip/ips_cip_connpathclass.cc +++ b/src/service_inspectors/cip/ips_cip_connpathclass.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_enipcommand.cc b/src/service_inspectors/cip/ips_cip_enipcommand.cc index 5dd433a73..c02260fd3 100644 --- a/src/service_inspectors/cip/ips_cip_enipcommand.cc +++ b/src/service_inspectors/cip/ips_cip_enipcommand.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_enipreq.cc b/src/service_inspectors/cip/ips_cip_enipreq.cc index f6dc0913b..a1866fa6c 100644 --- a/src/service_inspectors/cip/ips_cip_enipreq.cc +++ b/src/service_inspectors/cip/ips_cip_enipreq.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_eniprsp.cc b/src/service_inspectors/cip/ips_cip_eniprsp.cc index d4659b0f3..64d249df2 100644 --- a/src/service_inspectors/cip/ips_cip_eniprsp.cc +++ b/src/service_inspectors/cip/ips_cip_eniprsp.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_instance.cc b/src/service_inspectors/cip/ips_cip_instance.cc index 36dc97fe1..de93fb005 100644 --- a/src/service_inspectors/cip/ips_cip_instance.cc +++ b/src/service_inspectors/cip/ips_cip_instance.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_req.cc b/src/service_inspectors/cip/ips_cip_req.cc index 7fbea0cd3..bc78481bc 100644 --- a/src/service_inspectors/cip/ips_cip_req.cc +++ b/src/service_inspectors/cip/ips_cip_req.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_rsp.cc b/src/service_inspectors/cip/ips_cip_rsp.cc index a2c24b00e..8bed0e85c 100644 --- a/src/service_inspectors/cip/ips_cip_rsp.cc +++ b/src/service_inspectors/cip/ips_cip_rsp.cc @@ -27,7 +27,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_service.cc b/src/service_inspectors/cip/ips_cip_service.cc index 55c7c17b7..cff11df7a 100644 --- a/src/service_inspectors/cip/ips_cip_service.cc +++ b/src/service_inspectors/cip/ips_cip_service.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/cip/ips_cip_status.cc b/src/service_inspectors/cip/ips_cip_status.cc index 78517c215..a9036affa 100644 --- a/src/service_inspectors/cip/ips_cip_status.cc +++ b/src/service_inspectors/cip/ips_cip_status.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/dce_rpc/ips_dce_iface.cc b/src/service_inspectors/dce_rpc/ips_dce_iface.cc index 553fe4212..701d1f2ab 100644 --- a/src/service_inspectors/dce_rpc/ips_dce_iface.cc +++ b/src/service_inspectors/dce_rpc/ips_dce_iface.cc @@ -29,7 +29,7 @@ #include "framework/module.h" #include "framework/ips_option.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "target_based/snort_protocols.h" #include "utils/util.h" diff --git a/src/service_inspectors/dce_rpc/ips_dce_opnum.cc b/src/service_inspectors/dce_rpc/ips_dce_opnum.cc index 8e74d1d19..11e675960 100644 --- a/src/service_inspectors/dce_rpc/ips_dce_opnum.cc +++ b/src/service_inspectors/dce_rpc/ips_dce_opnum.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "utils/util.h" diff --git a/src/service_inspectors/dce_rpc/ips_dce_stub_data.cc b/src/service_inspectors/dce_rpc/ips_dce_stub_data.cc index 522b98ccc..2f29d382e 100644 --- a/src/service_inspectors/dce_rpc/ips_dce_stub_data.cc +++ b/src/service_inspectors/dce_rpc/ips_dce_stub_data.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "dce_common.h" diff --git a/src/service_inspectors/dnp3/ips_dnp3_data.cc b/src/service_inspectors/dnp3/ips_dnp3_data.cc index 7d3316782..adebb878d 100644 --- a/src/service_inspectors/dnp3/ips_dnp3_data.cc +++ b/src/service_inspectors/dnp3/ips_dnp3_data.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/dnp3/ips_dnp3_func.cc b/src/service_inspectors/dnp3/ips_dnp3_func.cc index 63a368308..480c97fc5 100644 --- a/src/service_inspectors/dnp3/ips_dnp3_func.cc +++ b/src/service_inspectors/dnp3/ips_dnp3_func.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/dnp3/ips_dnp3_ind.cc b/src/service_inspectors/dnp3/ips_dnp3_ind.cc index 1549eefe9..311d0988f 100644 --- a/src/service_inspectors/dnp3/ips_dnp3_ind.cc +++ b/src/service_inspectors/dnp3/ips_dnp3_ind.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/dnp3/ips_dnp3_obj.cc b/src/service_inspectors/dnp3/ips_dnp3_obj.cc index eee61eaf2..26b158ea8 100644 --- a/src/service_inspectors/dnp3/ips_dnp3_obj.cc +++ b/src/service_inspectors/dnp3/ips_dnp3_obj.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/ftp_telnet/pp_ftp.cc b/src/service_inspectors/ftp_telnet/pp_ftp.cc index 1a35c2bf2..fbd3747ae 100644 --- a/src/service_inspectors/ftp_telnet/pp_ftp.cc +++ b/src/service_inspectors/ftp_telnet/pp_ftp.cc @@ -42,7 +42,7 @@ #include "detection/detection_engine.h" #include "detection/detection_util.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "file_api/file_service.h" #include "protocols/packet.h" #include "stream/stream.h" diff --git a/src/service_inspectors/gtp/ips_gtp_info.cc b/src/service_inspectors/gtp/ips_gtp_info.cc index 6ba0373aa..f1d26e648 100644 --- a/src/service_inspectors/gtp/ips_gtp_info.cc +++ b/src/service_inspectors/gtp/ips_gtp_info.cc @@ -24,7 +24,7 @@ // gtp_info rule option implementation -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" diff --git a/src/service_inspectors/gtp/ips_gtp_type.cc b/src/service_inspectors/gtp/ips_gtp_type.cc index 2c2dd6864..5620fc18d 100644 --- a/src/service_inspectors/gtp/ips_gtp_type.cc +++ b/src/service_inspectors/gtp/ips_gtp_type.cc @@ -26,7 +26,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/gtp/ips_gtp_version.cc b/src/service_inspectors/gtp/ips_gtp_version.cc index eca531c30..aaa5831da 100644 --- a/src/service_inspectors/gtp/ips_gtp_version.cc +++ b/src/service_inspectors/gtp/ips_gtp_version.cc @@ -26,7 +26,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/http2_inspect/http2_dummy_packet.h b/src/service_inspectors/http2_inspect/http2_dummy_packet.h index 043ff0eaa..1ab7e1676 100644 --- a/src/service_inspectors/http2_inspect/http2_dummy_packet.h +++ b/src/service_inspectors/http2_inspect/http2_dummy_packet.h @@ -20,7 +20,7 @@ /* * The purpose of this Packet subclass is to enable H2I to take direction from http_inspect on * whether or not to send a frame to detection. When http_inspect is processing normal HTTP/1.1 - * traffic it is dealing with a real packet that has a context, the field on which disable_all() + * traffic it is dealing with a 'real' packet that has a context, the field on which disable_all() * is called to disable detection on that packet. With HTTP/2 traffic, http_inspect is processing a * dummy packet that H2I created, which does not contain a context object. Rather than create an * entire new context object when we really only need a bool, http_inspect checks if the flow is diff --git a/src/service_inspectors/http_inspect/http_buffer_info.cc b/src/service_inspectors/http_inspect/http_buffer_info.cc index 0876a08ae..e72f672d1 100644 --- a/src/service_inspectors/http_inspect/http_buffer_info.cc +++ b/src/service_inspectors/http_inspect/http_buffer_info.cc @@ -17,7 +17,7 @@ //-------------------------------------------------------------------------- // http_buffer_info.cc author Brandon Stultz -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "http_buffer_info.h" using namespace snort; diff --git a/src/service_inspectors/http_inspect/http_transaction.cc b/src/service_inspectors/http_inspect/http_transaction.cc index f05d2556d..1ad11463e 100644 --- a/src/service_inspectors/http_inspect/http_transaction.cc +++ b/src/service_inspectors/http_inspect/http_transaction.cc @@ -32,7 +32,7 @@ #include "http_msg_status.h" #include "http_msg_trailer.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" using namespace HttpCommon; using namespace HttpEnums; diff --git a/src/service_inspectors/http_inspect/http_uri.cc b/src/service_inspectors/http_inspect/http_uri.cc index 126eafc06..2b13b6f7b 100644 --- a/src/service_inspectors/http_inspect/http_uri.cc +++ b/src/service_inspectors/http_inspect/http_uri.cc @@ -25,7 +25,7 @@ #include "http_common.h" #include "http_enum.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" using namespace HttpCommon; using namespace HttpEnums; diff --git a/src/service_inspectors/http_inspect/ips_http.cc b/src/service_inspectors/http_inspect/ips_http.cc index 67b602206..1859f37e3 100644 --- a/src/service_inspectors/http_inspect/ips_http.cc +++ b/src/service_inspectors/http_inspect/ips_http.cc @@ -24,7 +24,7 @@ #include "ips_http.h" #include "framework/cursor.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "parser/parse_utils.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/modbus/ips_modbus_data.cc b/src/service_inspectors/modbus/ips_modbus_data.cc index 2cf071522..c7fe71ba0 100644 --- a/src/service_inspectors/modbus/ips_modbus_data.cc +++ b/src/service_inspectors/modbus/ips_modbus_data.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/modbus/ips_modbus_func.cc b/src/service_inspectors/modbus/ips_modbus_func.cc index e564c99a8..bfdde4114 100644 --- a/src/service_inspectors/modbus/ips_modbus_func.cc +++ b/src/service_inspectors/modbus/ips_modbus_func.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/modbus/ips_modbus_unit.cc b/src/service_inspectors/modbus/ips_modbus_unit.cc index 076637083..5ebf0416a 100644 --- a/src/service_inspectors/modbus/ips_modbus_unit.cc +++ b/src/service_inspectors/modbus/ips_modbus_unit.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/s7commplus/ips_s7comm_content.cc b/src/service_inspectors/s7commplus/ips_s7comm_content.cc index c17b7c54d..740cb90c4 100644 --- a/src/service_inspectors/s7commplus/ips_s7comm_content.cc +++ b/src/service_inspectors/s7commplus/ips_s7comm_content.cc @@ -26,7 +26,7 @@ #include "framework/cursor.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/s7commplus/ips_s7comm_func.cc b/src/service_inspectors/s7commplus/ips_s7comm_func.cc index aaba91329..1a5da45c5 100644 --- a/src/service_inspectors/s7commplus/ips_s7comm_func.cc +++ b/src/service_inspectors/s7commplus/ips_s7comm_func.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/s7commplus/ips_s7comm_opcode.cc b/src/service_inspectors/s7commplus/ips_s7comm_opcode.cc index 48478d549..d089ce5db 100644 --- a/src/service_inspectors/s7commplus/ips_s7comm_opcode.cc +++ b/src/service_inspectors/s7commplus/ips_s7comm_opcode.cc @@ -25,7 +25,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "protocols/packet.h" #include "profiler/profiler.h" diff --git a/src/service_inspectors/sip/ips_sip_method.cc b/src/service_inspectors/sip/ips_sip_method.cc index f7bad4e01..5382e05f5 100644 --- a/src/service_inspectors/sip/ips_sip_method.cc +++ b/src/service_inspectors/sip/ips_sip_method.cc @@ -30,7 +30,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/sip/ips_sip_stat_code.cc b/src/service_inspectors/sip/ips_sip_stat_code.cc index 45ae386b9..04044b145 100644 --- a/src/service_inspectors/sip/ips_sip_stat_code.cc +++ b/src/service_inspectors/sip/ips_sip_stat_code.cc @@ -28,7 +28,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "log/messages.h" #include "profiler/profiler.h" #include "protocols/packet.h" diff --git a/src/service_inspectors/sip/sip_utils.cc b/src/service_inspectors/sip/sip_utils.cc index cf058ee4e..b2f85e5ed 100644 --- a/src/service_inspectors/sip/sip_utils.cc +++ b/src/service_inspectors/sip/sip_utils.cc @@ -27,7 +27,7 @@ #include -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" /* * Trim spaces non-destructively on both sides of string : '', \t, \n, \r diff --git a/src/service_inspectors/ssl/ips_ssl_state.cc b/src/service_inspectors/ssl/ips_ssl_state.cc index a5db4ac5e..98a1ba0c9 100644 --- a/src/service_inspectors/ssl/ips_ssl_state.cc +++ b/src/service_inspectors/ssl/ips_ssl_state.cc @@ -23,7 +23,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "protocols/ssl.h" diff --git a/src/service_inspectors/ssl/ips_ssl_version.cc b/src/service_inspectors/ssl/ips_ssl_version.cc index 2f846c9ea..69bea2ad7 100644 --- a/src/service_inspectors/ssl/ips_ssl_version.cc +++ b/src/service_inspectors/ssl/ips_ssl_version.cc @@ -23,7 +23,7 @@ #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler.h" #include "protocols/packet.h" #include "protocols/ssl.h" diff --git a/src/stream/base/stream_base.cc b/src/stream/base/stream_base.cc index 5e2a9e971..eae32bd43 100644 --- a/src/stream/base/stream_base.cc +++ b/src/stream/base/stream_base.cc @@ -194,11 +194,7 @@ void StreamBase::tinit() if ( config.flow_cache_cfg.max_flows > 0 ) flow_con->init_exp(config.flow_cache_cfg.max_flows); -#ifdef REG_TEST FlushBucket::set(config.footprint); -#else - FlushBucket::set(); -#endif } void StreamBase::tterm() diff --git a/src/stream/base/stream_module.cc b/src/stream/base/stream_module.cc index 56e5cf5d0..c8ba5a7c9 100644 --- a/src/stream/base/stream_module.cc +++ b/src/stream/base/stream_module.cc @@ -64,10 +64,8 @@ FLOW_TYPE_PARAMS(file_params, "180", "32"); static const Parameter s_params[] = { -#ifdef REG_TEST { "footprint", Parameter::PT_INT, "0:max32", "0", "use zero for production, non-zero for testing at given size (for TCP and user)" }, -#endif { "ip_frags_only", Parameter::PT_BOOL, nullptr, "false", "don't process non-frag flows" }, @@ -132,15 +130,12 @@ bool StreamModule::set(const char* fqn, Value& v, SnortConfig* c) { PktType type = PktType::NONE; -#ifdef REG_TEST if ( v.is("footprint") ) { config.footprint = v.get_uint32(); return true; } -#endif - - if ( v.is("ip_frags_only") ) + else if ( v.is("ip_frags_only") ) { if ( v.get_bool() ) c->set_run_flags(RUN_FLAG__IP_FRAGS_ONLY); @@ -181,6 +176,8 @@ bool StreamModule::set(const char* fqn, Value& v, SnortConfig* c) return true; } +// FIXIT-L the detection of stream.xxx_cache changes below is a temporary workaround +// remove this check when stream.xxx_cache params become reloadable bool StreamModule::end(const char*, int, SnortConfig* sc) { if ( reload_resource_manager.initialize(config) ) @@ -201,23 +198,22 @@ void StreamModule::reset_stats() // Stream handler to adjust allocated resources as needed on a config reload bool StreamReloadResourceManager::initialize(const StreamModuleConfig& config_) { - // saving a copy of the config only works here because there is only - // one stream inspector per packet thread... - if ( !Snort::is_reloading() ) + // FIXIT-L - saving config here to check footprint change is a bit of a hack, + if ( Snort::is_reloading() ) { + if ( config.footprint != config_.footprint ) + { + // FIXIT-M - reinit FlushBucket... + ReloadError("Changing of stream.footprint requires a restart\n"); + return false; + } + config = config_; - return false; + return true; } -#ifdef REG_TEST - if ( config.footprint != config_.footprint ) - { - ReloadError("Changing of stream.footprint requires a restart\n"); - return false; - } -#endif config = config_; - return true; + return false; } bool StreamReloadResourceManager::tinit() diff --git a/src/stream/base/stream_module.h b/src/stream/base/stream_module.h index 8bded5983..36af6c365 100644 --- a/src/stream/base/stream_module.h +++ b/src/stream/base/stream_module.h @@ -75,10 +75,7 @@ extern THREAD_LOCAL BaseStats stream_base_stats; struct StreamModuleConfig { FlowCacheConfig flow_cache_cfg; -#ifdef REG_TEST unsigned footprint = 0; -#endif - }; class StreamReloadResourceManager : public snort::ReloadResourceTuner diff --git a/src/stream/flush_bucket.cc b/src/stream/flush_bucket.cc index a8ab9f627..3100ecd6a 100644 --- a/src/stream/flush_bucket.cc +++ b/src/stream/flush_bucket.cc @@ -51,9 +51,6 @@ void FlushBucket::set(unsigned sz) assert(s_flush_bucket); } -void FlushBucket::set() -{ set(0); } - void FlushBucket::clear() { delete s_flush_bucket; diff --git a/src/stream/flush_bucket.h b/src/stream/flush_bucket.h index 3ad3664b1..063870884 100644 --- a/src/stream/flush_bucket.h +++ b/src/stream/flush_bucket.h @@ -33,7 +33,6 @@ public: static uint16_t get_size(); static void set(unsigned sz); - static void set(); static void clear(); protected: diff --git a/src/stream/tcp/ips_stream_reassemble.cc b/src/stream/tcp/ips_stream_reassemble.cc index cafac897d..476817e34 100644 --- a/src/stream/tcp/ips_stream_reassemble.cc +++ b/src/stream/tcp/ips_stream_reassemble.cc @@ -24,7 +24,7 @@ #include "detection/detection_engine.h" #include "framework/ips_option.h" #include "framework/module.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "main/snort_config.h" #include "profiler/profiler.h" diff --git a/src/stream/tcp/ips_stream_size.cc b/src/stream/tcp/ips_stream_size.cc index 70f684ecd..0195b4325 100644 --- a/src/stream/tcp/ips_stream_size.cc +++ b/src/stream/tcp/ips_stream_size.cc @@ -24,7 +24,7 @@ #include "framework/ips_option.h" #include "framework/module.h" #include "framework/range.h" -#include "hash/hash_key_operations.h" +#include "hash/hashfcn.h" #include "profiler/profiler_defs.h" #include "tcp_session.h" diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index c06eff91e..050445200 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -7,11 +7,11 @@ set( UTIL_INCLUDES event_gen.h infractions.h kmap.h - memcap_allocator.h primed_allocator.h safec.h segment_mem.h sflsq.h + sfmemcap.h stats.h util.h util_ber.h @@ -31,6 +31,7 @@ add_library ( utils OBJECT kmap.cc segment_mem.cc sflsq.cc + sfmemcap.cc snort_bounds.h stats.cc util.cc diff --git a/src/utils/memcap_allocator.h b/src/utils/memcap_allocator.h deleted file mode 100644 index 4018f866d..000000000 --- a/src/utils/memcap_allocator.h +++ /dev/null @@ -1,107 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// memcap_allocator.h author davis mcpherson davmcphe@cisco.com - -// MemCapAllocator manages allocation and freeing of fixed size blocks of memory with -// an option of limiting total size of allocations to a specified capacity - -#ifndef MEMCAP_ALLOCATOR_H -#define MEMCAP_ALLOCATOR_H - -#include "framework/counts.h" -#include "utils/util.h" - -class MemCapAllocator -{ -public: - MemCapAllocator(unsigned long capacity, unsigned long allocation_size) - : mem_capacity(capacity), allocation_size(allocation_size) - { } - - void* allocate() - { - void* data = nullptr; - if ( is_space_available() ) - { - data = snort_calloc(allocation_size); - mem_allocated += allocation_size; - ++allocation_requests; - } - else - ++no_memory_available; - - return data; - } - - void free(void* mem) - { - snort_free(mem); - mem_allocated -= allocation_size; - ++free_requests; - } - - bool is_space_available() const - { - if ( !mem_capacity ) - return true; - else - return mem_allocated + allocation_size <= mem_capacity; - } - - bool is_over_capacity() const - { - if ( !mem_capacity ) - return false; - else - return mem_allocated > mem_capacity; - } - - void set_mem_capacity(unsigned long capacity) - { mem_capacity = capacity; } - - unsigned long get_mem_capacity() const - { return mem_capacity; } - - unsigned long get_allocation_size() const - { return allocation_size; } - - unsigned long get_mem_allocated() const - { return mem_allocated; } - - PegCount get_allocation_requests() const - { return allocation_requests; } - - PegCount get_free_requests() const - { return free_requests; } - - PegCount get_no_memory_available() const - { return no_memory_available; } - -private: - unsigned long mem_capacity; - unsigned long allocation_size; - unsigned long mem_allocated = 0; - - PegCount allocation_requests = 0; - PegCount free_requests = 0; - PegCount no_memory_available = 0; -}; - -#endif - diff --git a/src/utils/sfmemcap.cc b/src/utils/sfmemcap.cc new file mode 100644 index 000000000..31ef82179 --- /dev/null +++ b/src/utils/sfmemcap.cc @@ -0,0 +1,106 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved. +// Copyright (C) 2003-2013 Sourcefire, Inc. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +/* + sfmemcap.c + + These functions wrap the alloc & free functions. They enforce a memory cap using + the MEMCAP structure. The MEMCAP structure tracks memory usage. Each allocation + has 4 bytes added to it so we can store the allocation size. This allows us to + free a block and accurately track how much memory was recovered. + + Marc Norton +*/ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "sfmemcap.h" + +#include "util.h" +#include "util_cstring.h" + +/* +* Set max # bytes & init other variables. +*/ +void sfmemcap_init(MEMCAP* mc, unsigned long nbytes) +{ + mc->memcap = nbytes; + mc->memused= 0; + mc->nblocks= 0; +} + +/* +* Allocate some memory +*/ +void* sfmemcap_alloc(MEMCAP* mc, unsigned long nbytes) +{ + long* data; + + //printf("sfmemcap_alloc: %d bytes requested, memcap=%d, + // used=%d\n",nbytes,mc->memcap,mc->memused); + + nbytes += sizeof(long); + + /* Check if we are limiting memory use */ + if ( mc->memcap > 0 ) + { + /* Check if we've maxed out our memory - if we are tracking memory */ + if ( (mc->memused + nbytes) > mc->memcap ) + { + return nullptr; + } + } + + //data = (long*)snort_alloc( nbytes ); + data = (long*)snort_calloc(nbytes); + *data++ = (long)nbytes; + + mc->memused += nbytes; + mc->nblocks++; + + return data; +} + +/* +* Free some memory +*/ +void sfmemcap_free(MEMCAP* mc, void* p) +{ + long* q; + + q = (long*)p; + q--; + mc->memused -= (unsigned)(*q); + mc->nblocks--; + + snort_free(q); +} + +/* +* For debugging. +*/ +void sfmemcap_showmem(MEMCAP* mc) +{ + fprintf(stderr, "memcap: memcap = %lu bytes,",mc->memcap); + fprintf(stderr, " memused= %lu bytes,",mc->memused); + fprintf(stderr, " nblocks= %d blocks\n",mc->nblocks); +} + diff --git a/src/utils/sfmemcap.h b/src/utils/sfmemcap.h new file mode 100644 index 000000000..d2b80434f --- /dev/null +++ b/src/utils/sfmemcap.h @@ -0,0 +1,39 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved. +// Copyright (C) 2003-2013 Sourcefire, Inc. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +#ifndef SFMEMCAP_H +#define SFMEMCAP_H + +// _alloc and _free wrappers that enforce a memory cap + +struct MEMCAP +{ + unsigned long memused; + unsigned long memcap; + int nblocks; +}; + +// FIXIT-L could be refactored as a class but should be deleted +void sfmemcap_init(MEMCAP* mc, unsigned long nbytes); +void* sfmemcap_alloc(MEMCAP* mc, unsigned long nbytes); +void sfmemcap_showmem(MEMCAP* mc); +void sfmemcap_free(MEMCAP* mc, void* memory); + +#endif + diff --git a/src/utils/test/CMakeLists.txt b/src/utils/test/CMakeLists.txt index 181baf831..a9bf39e56 100644 --- a/src/utils/test/CMakeLists.txt +++ b/src/utils/test/CMakeLists.txt @@ -3,6 +3,4 @@ add_cpputest( boyer_moore_test ../boyer_moore.cc ) -add_cpputest( memcap_allocator_test ) - add_catch_test( bitop_test ) diff --git a/src/utils/test/memcap_allocator_test.cc b/src/utils/test/memcap_allocator_test.cc deleted file mode 100644 index 3de048e30..000000000 --- a/src/utils/test/memcap_allocator_test.cc +++ /dev/null @@ -1,221 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2020-2020 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// memcap_allocator_test.cc author davis mcpherson -// unit tests for MemCapAllocator class - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "../memcap_allocator.h" - -#include -#include - -const unsigned long ALLOCATION_SIZE = 250; -const unsigned long NO_MEM_CAP_LIMIT = 0; -const unsigned NO_MEMCAP_LIMIT_ALLOCATIONS = 10000; - -const unsigned long MEM_CAP_5000 = 5000; -const unsigned MAX_ALLOCATIONS_5000 = MEM_CAP_5000 / ALLOCATION_SIZE; - -const unsigned long MEM_CAP_10000 = 10000; -const unsigned MAX_ALLOCATIONS_10000 = MEM_CAP_10000 / ALLOCATION_SIZE; - -const unsigned long MEM_CAP_20000 = 20000; -const unsigned MAX_ALLOCATIONS_20000 = MEM_CAP_20000 / ALLOCATION_SIZE; - -TEST_GROUP(memcap_allocator) -{ }; - -TEST(memcap_allocator, no_memcap_limit_allocator_test) -{ - MemCapAllocator* mca = new MemCapAllocator(NO_MEM_CAP_LIMIT, ALLOCATION_SIZE); - CHECK(mca); - CHECK(mca->get_allocation_size() == ALLOCATION_SIZE); - CHECK(mca->get_mem_capacity() == NO_MEM_CAP_LIMIT); - CHECK(mca->is_space_available()); - CHECK(!mca->is_over_capacity()); - CHECK(mca->get_mem_allocated() == 0); - CHECK(mca->get_allocation_requests() == 0); - CHECK(mca->get_free_requests() == 0); - CHECK(mca->get_no_memory_available() == 0); - delete mca; -} - -TEST(memcap_allocator, no_memcap_limit_allocations_test) -{ - MemCapAllocator* mca = new MemCapAllocator(NO_MEM_CAP_LIMIT, ALLOCATION_SIZE); - CHECK(mca); - - unsigned bytes_allocated; - uint8_t** mem_blocks = (uint8_t**)snort_alloc(sizeof(uint8_t*) * NO_MEMCAP_LIMIT_ALLOCATIONS); - for ( unsigned i = 0; i < NO_MEMCAP_LIMIT_ALLOCATIONS; i++) - { - mem_blocks[i] = (uint8_t*)mca->allocate(); - bytes_allocated = (i + 1) * ALLOCATION_SIZE; - CHECK(mem_blocks[i]); - CHECK(mca->get_mem_allocated() == bytes_allocated); - CHECK(mca->is_space_available()); - CHECK(!mca->is_over_capacity()); - } - - CHECK(mca->get_allocation_requests() == NO_MEMCAP_LIMIT_ALLOCATIONS); - CHECK(mca->get_free_requests() == 0); - CHECK(mca->get_no_memory_available() == 0); - - for ( unsigned i = 0; i < NO_MEMCAP_LIMIT_ALLOCATIONS; i++) - { - mca->free(mem_blocks[i]); - bytes_allocated = (NO_MEMCAP_LIMIT_ALLOCATIONS - (i + 1)) * ALLOCATION_SIZE; - CHECK(mca->get_mem_allocated() == bytes_allocated); - } - - CHECK(mca->is_space_available()); - CHECK(!mca->is_over_capacity()); - CHECK(mca->get_mem_allocated() == 0); - CHECK(mca->get_free_requests() == NO_MEMCAP_LIMIT_ALLOCATIONS); - CHECK(mca->get_no_memory_available() == 0); - - snort_free(mem_blocks); - delete mca; -} - -TEST(memcap_allocator, create_memcap_allocator_test) -{ - MemCapAllocator* mca = new MemCapAllocator(MEM_CAP_10000, ALLOCATION_SIZE); - CHECK(mca); - CHECK(mca->get_allocation_size() == ALLOCATION_SIZE); - CHECK(mca->get_mem_capacity() == MEM_CAP_10000); - mca->set_mem_capacity(MEM_CAP_20000); - CHECK(mca->get_mem_capacity() == MEM_CAP_20000); - CHECK(mca->get_mem_allocated() == 0); - CHECK(mca->get_allocation_requests() == 0); - CHECK(mca->get_free_requests() == 0); - CHECK(mca->get_no_memory_available() == 0); - delete mca; -} - -TEST(memcap_allocator, allocate_and_free_memory_test) -{ - MemCapAllocator* mca = new MemCapAllocator(MEM_CAP_10000, ALLOCATION_SIZE); - CHECK(mca); - CHECK(mca->is_space_available()); - CHECK(!mca->is_over_capacity()); - - uint8_t* mem = (uint8_t*)mca->allocate(); - CHECK(mem); - CHECK(mca->get_mem_allocated() == ALLOCATION_SIZE); - CHECK(mca->is_space_available()); - CHECK(!mca->is_over_capacity()); - CHECK(mca->get_allocation_requests() == 1); - CHECK(mca->get_free_requests() == 0); - CHECK(mca->get_no_memory_available() == 0); - - mca->free(mem); - CHECK(mca->get_mem_allocated() == 0); - CHECK(mca->is_space_available()); - CHECK(!mca->is_over_capacity()); - CHECK(mca->get_allocation_requests() == 1); - CHECK(mca->get_free_requests() == 1); - CHECK(mca->get_no_memory_available() == 0); - - delete mca; -} - -TEST(memcap_allocator, max_allocations_test) -{ - MemCapAllocator* mca = new MemCapAllocator(MEM_CAP_10000, ALLOCATION_SIZE); - CHECK(mca); - - unsigned bytes_allocated; - uint8_t* mem_blocks[MAX_ALLOCATIONS_10000]; - for ( unsigned i = 0; i < MAX_ALLOCATIONS_10000; i++) - { - mem_blocks[i] = (uint8_t*)mca->allocate(); - bytes_allocated = (i + 1) * ALLOCATION_SIZE; - CHECK(mem_blocks[i]); - CHECK(mca->get_mem_allocated() == bytes_allocated); - if ( i < MAX_ALLOCATIONS_10000 - 1 ) - { - CHECK(mca->is_space_available()); - } - else - { - CHECK(!mca->is_space_available()); - } - CHECK(!mca->is_over_capacity()); - } - CHECK(mca->get_allocation_requests() == MAX_ALLOCATIONS_10000); - CHECK(mca->get_free_requests() == 0); - CHECK(mca->get_no_memory_available() == 0); - - uint8_t* mem = (uint8_t*)mca->allocate(); - CHECK(!mem); - CHECK(mca->get_no_memory_available() == 1); - mem = (uint8_t*)mca->allocate(); - CHECK(!mem); - mem = (uint8_t*)mca->allocate(); - CHECK(!mem); - CHECK(mca->get_no_memory_available() == 3); - mca->free(mem_blocks[0]); - mem_blocks[0] = (uint8_t*)mca->allocate(); - CHECK(mem_blocks[0]); - CHECK(mca->get_free_requests() == 1); - CHECK(mca->get_no_memory_available() == 3); - - mca->set_mem_capacity(MEM_CAP_20000); - mem = (uint8_t*)mca->allocate(); - CHECK(mem); - mca->set_mem_capacity(MEM_CAP_5000); - CHECK(mca->is_over_capacity()); - mca->free(mem); - CHECK(mca->get_free_requests() == 2); - - for ( unsigned i = 0; i < MAX_ALLOCATIONS_5000; i++) - { - CHECK(!mca->is_space_available()); - mca->free(mem_blocks[i]); - bytes_allocated = (MAX_ALLOCATIONS_10000 - (i + 1)) * ALLOCATION_SIZE; - CHECK(mca->get_mem_allocated() == bytes_allocated); - } - - CHECK(!mca->is_space_available()); - CHECK(!mca->is_over_capacity()); - - for ( unsigned i = MAX_ALLOCATIONS_5000; i < MAX_ALLOCATIONS_10000; i++) - { - mca->free(mem_blocks[i]); - bytes_allocated = (MAX_ALLOCATIONS_10000 - (i + 1)) * ALLOCATION_SIZE; - CHECK(mca->get_mem_allocated() == bytes_allocated); - CHECK(mca->is_space_available()); - CHECK(!mca->is_over_capacity()); - } - - CHECK(mca->get_mem_allocated() == 0); - CHECK(mca->get_free_requests() == MAX_ALLOCATIONS_10000 + 2); - CHECK(mca->get_no_memory_available() == 3); - - delete mca; -} - -int main(int argc, char** argv) -{ - return CommandLineTestRunner::RunAllTests(argc, argv); -}