From: Davis McPherson (davmcphe) Date: Fri, 9 Aug 2019 13:56:13 +0000 (-0400) Subject: Merge pull request #1697 in SNORT/snort3 from ~PSHINDE2/snort3:port_sfxhash to master X-Git-Tag: 3.0.0-259~11 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=21812e434532278183efffdf218bd79e22387753;p=thirdparty%2Fsnort3.git Merge pull request #1697 in SNORT/snort3 from ~PSHINDE2/snort3:port_sfxhash to master Squashed commit of the following: commit 6dd6e2dd11ee71ff13fa93664fa9b3baecf1460c Author: Pratik Shinde Date: Thu Aug 1 16:00:57 2019 -0400 xhash: Ported sfxhash_change_memcap() from snort2 to snort3 --- diff --git a/src/hash/test/CMakeLists.txt b/src/hash/test/CMakeLists.txt index 3ee116302..39896fa9b 100644 --- a/src/hash/test/CMakeLists.txt +++ b/src/hash/test/CMakeLists.txt @@ -2,6 +2,14 @@ add_cpputest( lru_cache_shared_test SOURCES ../lru_cache_shared.cc ) +add_cpputest( xhash_test + SOURCES + ../xhash.cc + ../hashfcn.cc + ../primetable.cc + ../../utils/sfmemcap.cc +) + add_cpputest( ghash_test SOURCES ../ghash.cc diff --git a/src/hash/test/xhash_test.cc b/src/hash/test/xhash_test.cc new file mode 100644 index 000000000..709573c98 --- /dev/null +++ b/src/hash/test/xhash_test.cc @@ -0,0 +1,233 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// xhash_test.cc author Pratik Shinde +// unit tests for xhash utility functions + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "hash/xhash.h" + +#include "main/snort_config.h" +#include "utils/util.h" + +#include +#include + +using namespace snort; + +// Stubs whose sole purpose is to make the test code link +static SnortConfig my_config; +THREAD_LOCAL SnortConfig *snort_conf = &my_config; + +SnortConfig::SnortConfig(const SnortConfig* const) +{ snort_conf->run_flags = 0;} // run_flags is used indirectly from HashFnc class by calling SnortConfig::static_hash() + +SnortConfig::~SnortConfig() = default; + +SnortConfig* SnortConfig::get_conf() +{ return snort_conf; } + +struct xhash_test_key +{ + int key; +}; + +TEST_GROUP(xhash) +{ }; + +// Test create a hash table, add nodes, find and delete. +TEST(xhash, create_xhash_test) +{ + XHash* test_table = xhash_new(4, sizeof(struct xhash_test_key), + 0, 0, 0, nullptr, nullptr, 0); + CHECK(test_table->keysize == sizeof(struct xhash_test_key)); + xhash_delete(test_table); +} + +// Test verifies if free_anr_lru_function() throws error on invalid table +TEST(xhash, free_anr_lru_invalid_test) +{ + int ret = xhash_free_anr_lru(nullptr); + CHECK(ret == XHASH_ERR); +} + +// Create a free node in xhash and verifies if xhash_free_anr_lru() deletes it +TEST(xhash, free_anr_lru_delete_free_node_test) +{ + XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key), + 1, 1040, 0, nullptr, nullptr, 1); + xhash_test_key xtk; + xtk.key = 10; + int ret = xhash_add(test_table, &xtk, nullptr); + CHECK(ret == XHASH_OK); + + XHashNode *xnode = xhash_get_node(test_table, &xtk); + CHECK(xnode != nullptr); + + ret = xhash_free_node(test_table, xnode); + CHECK(ret == XHASH_OK); + + ret = xhash_free_anr_lru(test_table); + CHECK(ret == XHASH_OK); + + XHashNode* xhnode = xhash_find_node(test_table, &xtk); + CHECK(xhnode == nullptr); + xhash_delete(test_table); +} + +// No free node is available, verifies if xhash_free_anr_lru() deletes the last node +TEST(xhash, free_anr_lru_delete_tail_node_test) +{ + XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key), + 1, 1040, 0, nullptr, nullptr, 1); + xhash_test_key xtk; + int ret = xhash_add(test_table, &xtk, nullptr); + CHECK(ret == XHASH_OK); + + XHashNode* orig_gtail = test_table->gtail; + ret = xhash_free_anr_lru(test_table); + CHECK(ret == XHASH_OK); + CHECK(orig_gtail != test_table->gtail); + + xhash_delete(test_table); +} + +// No free node is available [recycle is not enabled], verifies if last node is deleted +TEST(xhash, free_anr_lru_usr_free_delete_tail_node_test) +{ + XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key), + 1, 1040, 0, nullptr, nullptr, 0); + xhash_test_key xtk; + int ret = xhash_add(test_table, &xtk, nullptr); + CHECK(ret == XHASH_OK); + + XHashNode* orig_gtail = test_table->gtail; + ret = xhash_free_anr_lru(test_table); + CHECK(ret == XHASH_OK); + CHECK(orig_gtail != test_table->gtail); + xhash_delete(test_table); +} + +// if new memcap is same as old memcap, do nothing +TEST(xhash, change_memcap_same_memcap_test) +{ + XHash* test_table = xhash_new(5, sizeof(struct xhash_test_key), + 0, 80, 0, nullptr, nullptr, 1); + unsigned max_work = 0; + int ret = xhash_change_memcap(test_table, 80, &max_work); + CHECK(ret == XHASH_OK); + CHECK(test_table->mc.memcap == 80); + xhash_delete(test_table); +} + +// if new memcap is more than old memcap, only change the memcap +TEST(xhash, change_memcap_more_memcap_test) +{ + XHash* test_table = xhash_new(5, sizeof(struct xhash_test_key), + 0, 80, 0, nullptr, nullptr, 1); + + unsigned max_work = 0; + int ret = xhash_change_memcap(test_table, 100, &max_work); + CHECK(ret == XHASH_OK); + CHECK(test_table->mc.memcap == 100); + xhash_delete(test_table); +} + +// IF new memcap is is less than overhead bytes, throw an error +TEST(xhash, change_memcap_less_than_overhead_memcap_test) +{ + XHash* test_table = xhash_new(5, sizeof(struct xhash_test_key), + 0, 80, 0, nullptr, nullptr, 1); + + unsigned max_work = 0; + int ret = xhash_change_memcap(test_table, test_table->overhead_bytes-1, &max_work); + CHECK(ret == XHASH_ERR); + CHECK(test_table->mc.memcap == 80); + xhash_delete(test_table); +} + +//if new memcap is less than used memcap, do the pruning +TEST(xhash, xhash_change_memcap_less_than_used_test) +{ + XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key), + 1, 1040, 0, nullptr, nullptr, 1); + xhash_test_key xtk[2]; + int ret = xhash_add(test_table, &xtk[0], nullptr); + CHECK(ret == XHASH_OK); + + xtk[1].key = 100; + ret = xhash_add(test_table, &xtk[1], nullptr); + CHECK(ret == XHASH_OK); + + unsigned max_work = 0; + unsigned new_memcap = test_table->mc.memused-1; + ret = xhash_change_memcap(test_table, new_memcap, &max_work); + CHECK(ret == XHASH_OK); + CHECK(test_table->mc.memcap == new_memcap); + xhash_delete(test_table); +} + +// new memcap is less than old memcap and cannot prune +TEST(xhash, xhash_change_memcap_nofree_nodes_test) +{ + XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key), + 1, 1040, 0, nullptr, nullptr, 0); + xhash_test_key xtk; + + int ret = xhash_add(test_table, &xtk, nullptr); + CHECK(ret == XHASH_OK); + unsigned new_memcap = test_table->mc.memused-1; + + + unsigned max_work = 0; + test_table->gtail = nullptr; + ret = xhash_change_memcap(test_table, new_memcap, &max_work); + CHECK(ret == XHASH_NOMEM); + xhash_delete(test_table); +} + +// new memcap is less than old memcap and max_work is than needed +TEST(xhash, xhash_change_memcap_less_max_work_test) +{ + XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key), + 142, 1040, 0, nullptr, nullptr, 0); + xhash_test_key xtk; + + int ret = xhash_add(test_table, &xtk, nullptr); + CHECK(ret == XHASH_OK); + unsigned new_memcap = test_table->mc.memused-1; + + xhash_test_key xtk1; + xtk1.key = 100; + ret = xhash_add(test_table, &xtk1, nullptr); + CHECK(ret == XHASH_OK); + + unsigned max_work = 1; + ret = xhash_change_memcap(test_table, new_memcap, &max_work); + CHECK(ret == XHASH_PENDING); + CHECK(max_work == 0); + xhash_delete(test_table); +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +} diff --git a/src/hash/xhash.cc b/src/hash/xhash.cc index e3e74b438..7a521d661 100644 --- a/src/hash/xhash.cc +++ b/src/hash/xhash.cc @@ -260,6 +260,56 @@ static void xhash_delete_free_list(XHash* t) t->ftail = nullptr; } +/*! + * Try to change the memcap + * Behavior is undefined when t->usrfree is set + * + * t SFXHASH table pointer + * new_memcap the new desired memcap + * max_work the maximum amount of work for the function to do (0 = do all available work) + * + * returns + * XHASH_OK when memcap is successfully decreased + * XHASH_PENDING when more work needs to be done + * XHASH_NOMEM when there isn't enough memory in the hash table + * XHASH_ERR when an error has occurred + */ +int xhash_change_memcap(XHash *t, unsigned long new_memcap, unsigned *max_work) +{ + if (t == nullptr or new_memcap < t->overhead_bytes) + return XHASH_ERR; + + if (new_memcap == t->mc.memcap) + return XHASH_OK; + + if (new_memcap > t->mc.memcap) + { + t->mc.memcap = new_memcap; + return XHASH_OK; + } + + unsigned work = 0; + while (new_memcap < t->mc.memused + and (work < *max_work or *max_work == 0) + and (xhash_free_anr_lru(t) == XHASH_OK)) + work++; + + if (*max_work != 0 and (work == *max_work and new_memcap < t->mc.memused)) + { + *max_work -= work; + return XHASH_PENDING; + } + + //we ran out of nodes to free and there still isn't enough memory + //or (we have undefined behavior: t->usrfree is set and xhash_free_anr_lru is returning XHASH_ERR) + if (new_memcap < t->mc.memused) + return XHASH_NOMEM; + + t->mc.memcap = new_memcap; + return XHASH_OK; +} + + /*! * Delete the hash Table * @@ -965,6 +1015,56 @@ static void xhash_next(XHash* t) } } +static inline int xhash_delete_free_node(XHash *t) +{ + XHashNode* fn = xhash_get_free_node(t); + if (fn) + { + s_free(t, fn); + return XHASH_OK; + } + return XHASH_ERR; +} + +/*! + * Unlink and free an ANR node or the oldest node, if ANR is empty + * behavior is undefined if t->usrfree is set + * + * t XHash table pointer + * + * returns + * XHASH_ERR if error occures + * XHASH_OK if node is freed + */ +int xhash_free_anr_lru(XHash *t) +{ + if (t == nullptr) + return XHASH_ERR; + + if (t->fhead) + { + if (xhash_delete_free_node(t) == XHASH_OK) + return XHASH_OK; + } + + if (t->gtail) + { + if (xhash_free_node(t, t->gtail) == XHASH_OK) + { + if (t->fhead) + { + if (xhash_delete_free_node(t) == XHASH_OK) + return XHASH_OK; + } + else if (!t->recycle_nodes) + { + return XHASH_OK; + } + } + } + return XHASH_ERR; +} + /*! * Find and return the first hash table node * diff --git a/src/hash/xhash.h b/src/hash/xhash.h index fb767f7d4..0283b9f44 100644 --- a/src/hash/xhash.h +++ b/src/hash/xhash.h @@ -36,6 +36,7 @@ namespace snort #define XHASH_ERR (-1) #define XHASH_OK 0 #define XHASH_INTABLE 1 +#define XHASH_PENDING 2 struct XHashNode { @@ -99,7 +100,7 @@ SO_PUBLIC XHash* xhash_new(int nrows, int keysize, int datasize, unsigned long m int recycle_flag); SO_PUBLIC void xhash_set_max_nodes(XHash* h, int max_nodes); - +SO_PUBLIC int xhash_change_memcap(XHash *t, unsigned long new_memcap, unsigned *max_work); SO_PUBLIC void xhash_delete(XHash* h); SO_PUBLIC int xhash_make_empty(XHash*); @@ -135,6 +136,7 @@ inline unsigned xhash_overhead_bytes(XHash* t) inline unsigned xhash_overhead_blocks(XHash* t) { return t->overhead_blocks; } +SO_PUBLIC int xhash_free_anr_lru(XHash* t); SO_PUBLIC void* xhash_mru(XHash* t); SO_PUBLIC void* xhash_lru(XHash* t); SO_PUBLIC void* xhash_find(XHash* h, void* key);