]> git.ipfire.org Git - thirdparty/snort3.git/commitdiff
Merge pull request #1697 in SNORT/snort3 from ~PSHINDE2/snort3:port_sfxhash to master
authorDavis McPherson (davmcphe) <davmcphe@cisco.com>
Fri, 9 Aug 2019 13:56:13 +0000 (09:56 -0400)
committerDavis McPherson (davmcphe) <davmcphe@cisco.com>
Fri, 9 Aug 2019 13:56:13 +0000 (09:56 -0400)
Squashed commit of the following:

commit 6dd6e2dd11ee71ff13fa93664fa9b3baecf1460c
Author: Pratik Shinde <pshinde2@cisco.com>
Date:   Thu Aug 1 16:00:57 2019 -0400

    xhash: Ported sfxhash_change_memcap() from snort2 to snort3

src/hash/test/CMakeLists.txt
src/hash/test/xhash_test.cc [new file with mode: 0644]
src/hash/xhash.cc
src/hash/xhash.h

index 3ee11630277a52d364a096bb48e576ac71eecdc4..39896fa9b8e64d5d6e1cab53fad59f6ed20ba8f4 100644 (file)
@@ -2,6 +2,14 @@ add_cpputest( lru_cache_shared_test
     SOURCES ../lru_cache_shared.cc
 )
 
+add_cpputest( xhash_test
+    SOURCES
+        ../xhash.cc
+        ../hashfcn.cc
+        ../primetable.cc
+        ../../utils/sfmemcap.cc
+)
+
 add_cpputest( ghash_test
     SOURCES
         ../ghash.cc
diff --git a/src/hash/test/xhash_test.cc b/src/hash/test/xhash_test.cc
new file mode 100644 (file)
index 0000000..709573c
--- /dev/null
@@ -0,0 +1,233 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// xhash_test.cc author Pratik Shinde <pshinde2@cisco.com>
+// unit tests for xhash utility functions
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "hash/xhash.h"
+
+#include "main/snort_config.h"
+#include "utils/util.h"
+
+#include <CppUTest/CommandLineTestRunner.h>
+#include <CppUTest/TestHarness.h>
+
+using namespace snort;
+
+// Stubs whose sole purpose is to make the test code link
+static SnortConfig my_config;
+THREAD_LOCAL SnortConfig *snort_conf = &my_config;
+
+SnortConfig::SnortConfig(const SnortConfig* const)
+{ snort_conf->run_flags = 0;} // run_flags is used indirectly from HashFnc class by calling SnortConfig::static_hash()
+
+SnortConfig::~SnortConfig() = default;
+
+SnortConfig* SnortConfig::get_conf()
+{ return snort_conf; }
+
+struct xhash_test_key
+{
+    int key;
+};
+
+TEST_GROUP(xhash)
+{ };
+
+//  Test create a hash table, add nodes, find and delete.
+TEST(xhash, create_xhash_test)
+{
+    XHash* test_table = xhash_new(4, sizeof(struct xhash_test_key),
+                                  0, 0, 0, nullptr, nullptr, 0);
+    CHECK(test_table->keysize == sizeof(struct xhash_test_key));
+    xhash_delete(test_table);
+}
+
+// Test verifies if free_anr_lru_function() throws error on invalid table
+TEST(xhash, free_anr_lru_invalid_test)
+{
+    int ret = xhash_free_anr_lru(nullptr);
+    CHECK(ret == XHASH_ERR); 
+}
+
+// Create a free node in xhash and verifies if xhash_free_anr_lru() deletes it 
+TEST(xhash, free_anr_lru_delete_free_node_test)
+{
+    XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key),
+                                  1, 1040, 0, nullptr, nullptr, 1);
+    xhash_test_key xtk;
+    xtk.key = 10;
+    int ret = xhash_add(test_table, &xtk, nullptr);
+    CHECK(ret == XHASH_OK);
+
+    XHashNode *xnode = xhash_get_node(test_table, &xtk);
+    CHECK(xnode != nullptr);
+
+    ret = xhash_free_node(test_table, xnode);
+    CHECK(ret == XHASH_OK);
+
+    ret = xhash_free_anr_lru(test_table);
+    CHECK(ret == XHASH_OK); 
+
+    XHashNode* xhnode = xhash_find_node(test_table, &xtk);
+    CHECK(xhnode == nullptr);
+    xhash_delete(test_table);
+}
+
+// No free node is available, verifies if xhash_free_anr_lru() deletes the last node
+TEST(xhash, free_anr_lru_delete_tail_node_test)
+{
+    XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key),
+                                  1, 1040, 0, nullptr, nullptr, 1);
+    xhash_test_key xtk;
+    int ret = xhash_add(test_table, &xtk, nullptr);
+    CHECK(ret == XHASH_OK);
+
+    XHashNode* orig_gtail = test_table->gtail;
+    ret = xhash_free_anr_lru(test_table);
+    CHECK(ret == XHASH_OK);
+    CHECK(orig_gtail != test_table->gtail);
+
+    xhash_delete(test_table);
+}
+
+// No free node is available [recycle is not enabled], verifies if last node is deleted
+TEST(xhash, free_anr_lru_usr_free_delete_tail_node_test)
+{
+    XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key),
+                                  1, 1040, 0, nullptr, nullptr, 0);
+    xhash_test_key xtk;
+    int ret = xhash_add(test_table, &xtk, nullptr);
+    CHECK(ret == XHASH_OK);
+
+    XHashNode* orig_gtail = test_table->gtail;
+    ret = xhash_free_anr_lru(test_table);
+    CHECK(ret == XHASH_OK);
+    CHECK(orig_gtail != test_table->gtail);
+    xhash_delete(test_table);
+}
+
+// if new memcap is same as old memcap, do nothing
+TEST(xhash, change_memcap_same_memcap_test)
+{
+    XHash* test_table = xhash_new(5, sizeof(struct xhash_test_key),
+                                  0, 80, 0, nullptr, nullptr, 1);
+    unsigned max_work = 0;
+    int ret = xhash_change_memcap(test_table, 80, &max_work);
+    CHECK(ret == XHASH_OK);
+    CHECK(test_table->mc.memcap == 80);
+    xhash_delete(test_table);
+}
+
+// if new memcap is more than old memcap, only change the memcap
+TEST(xhash, change_memcap_more_memcap_test)
+{
+    XHash* test_table = xhash_new(5, sizeof(struct xhash_test_key),
+                                  0, 80, 0, nullptr, nullptr, 1);
+
+    unsigned max_work = 0;
+    int ret = xhash_change_memcap(test_table, 100, &max_work);
+    CHECK(ret == XHASH_OK);
+    CHECK(test_table->mc.memcap == 100);
+    xhash_delete(test_table);
+}
+
+// IF new memcap is is less than overhead bytes, throw an error
+TEST(xhash, change_memcap_less_than_overhead_memcap_test)
+{
+    XHash* test_table = xhash_new(5, sizeof(struct xhash_test_key),
+                                  0, 80, 0, nullptr, nullptr, 1);
+
+    unsigned max_work = 0;
+    int ret = xhash_change_memcap(test_table, test_table->overhead_bytes-1, &max_work);
+    CHECK(ret == XHASH_ERR);
+    CHECK(test_table->mc.memcap == 80);
+    xhash_delete(test_table);
+}
+
+//if new memcap is less than used memcap, do the pruning
+TEST(xhash, xhash_change_memcap_less_than_used_test)
+{
+    XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key),
+                                  1, 1040, 0, nullptr, nullptr, 1);
+    xhash_test_key xtk[2];
+    int ret = xhash_add(test_table, &xtk[0], nullptr);
+    CHECK(ret == XHASH_OK);
+
+    xtk[1].key = 100;
+    ret = xhash_add(test_table, &xtk[1], nullptr);
+    CHECK(ret == XHASH_OK);
+
+    unsigned max_work = 0;
+    unsigned new_memcap = test_table->mc.memused-1;
+    ret = xhash_change_memcap(test_table, new_memcap, &max_work);  
+    CHECK(ret == XHASH_OK);
+    CHECK(test_table->mc.memcap == new_memcap);
+    xhash_delete(test_table);
+}
+
+// new memcap is less than old memcap and cannot prune
+TEST(xhash, xhash_change_memcap_nofree_nodes_test)
+{
+    XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key),
+                                  1, 1040, 0, nullptr, nullptr, 0);
+    xhash_test_key xtk;
+
+    int ret = xhash_add(test_table, &xtk, nullptr);
+    CHECK(ret == XHASH_OK);
+    unsigned new_memcap = test_table->mc.memused-1;
+
+
+    unsigned max_work = 0;
+    test_table->gtail = nullptr;
+    ret = xhash_change_memcap(test_table, new_memcap, &max_work);
+    CHECK(ret == XHASH_NOMEM);
+    xhash_delete(test_table);
+}
+
+// new memcap is less than old memcap and max_work is than needed
+TEST(xhash, xhash_change_memcap_less_max_work_test)
+{
+    XHash* test_table = xhash_new(3, sizeof(struct xhash_test_key),
+                                  142, 1040, 0, nullptr, nullptr, 0);
+    xhash_test_key xtk;
+
+    int ret = xhash_add(test_table, &xtk, nullptr);
+    CHECK(ret == XHASH_OK);
+    unsigned new_memcap = test_table->mc.memused-1;
+
+    xhash_test_key xtk1;
+    xtk1.key = 100;
+    ret = xhash_add(test_table, &xtk1, nullptr);
+    CHECK(ret == XHASH_OK);
+
+    unsigned max_work = 1;
+    ret = xhash_change_memcap(test_table, new_memcap, &max_work);
+    CHECK(ret == XHASH_PENDING);
+    CHECK(max_work == 0);
+    xhash_delete(test_table);
+}
+
+int main(int argc, char** argv)
+{
+    return CommandLineTestRunner::RunAllTests(argc, argv);
+}
index e3e74b438493df815cb7bfb4920602864b5fc2ca..7a521d661c16b8fccf13b5bb217aa6c0587814b1 100644 (file)
@@ -260,6 +260,56 @@ static void xhash_delete_free_list(XHash* t)
     t->ftail = nullptr;
 }
 
+/*!
+ * Try to change the memcap
+ *  Behavior is undefined when t->usrfree is set
+ *
+ * t             SFXHASH table pointer
+ * new_memcap    the new desired memcap
+ * max_work      the maximum amount of work for the function to do (0 = do all available work)
+ *
+ * returns
+ * XHASH_OK           when memcap is successfully decreased
+ * XHASH_PENDING      when more work needs to be done
+ * XHASH_NOMEM        when there isn't enough memory in the hash table
+ * XHASH_ERR          when an error has occurred
+ */
+int xhash_change_memcap(XHash *t, unsigned long new_memcap, unsigned *max_work)
+{
+    if (t == nullptr or new_memcap < t->overhead_bytes)
+        return XHASH_ERR;
+    
+    if (new_memcap == t->mc.memcap)
+        return XHASH_OK;
+
+    if (new_memcap > t->mc.memcap)
+    {
+        t->mc.memcap = new_memcap;
+        return XHASH_OK;
+    }
+
+    unsigned work = 0;
+    while (new_memcap < t->mc.memused
+            and (work < *max_work or *max_work == 0)
+            and (xhash_free_anr_lru(t) == XHASH_OK))
+        work++;
+
+    if (*max_work != 0 and (work == *max_work and new_memcap < t->mc.memused))
+    {
+        *max_work -= work;
+        return XHASH_PENDING;
+    }
+
+    //we ran out of nodes to free and there still isn't enough memory
+    //or (we have undefined behavior: t->usrfree is set and xhash_free_anr_lru is returning XHASH_ERR)
+    if (new_memcap < t->mc.memused)
+        return XHASH_NOMEM;
+    t->mc.memcap = new_memcap;
+    return XHASH_OK; 
+}
+
+
 /*!
  *  Delete the hash Table
  *
@@ -965,6 +1015,56 @@ static void xhash_next(XHash* t)
     }
 }
 
+static inline int xhash_delete_free_node(XHash *t)
+{
+    XHashNode* fn = xhash_get_free_node(t);
+    if (fn)
+    {
+        s_free(t, fn);
+        return XHASH_OK;
+    }
+    return XHASH_ERR;
+}
+
+/*!
+ * Unlink and free an ANR node or the oldest node, if ANR is empty
+ * behavior is undefined if t->usrfree is set
+ *
+ * t XHash table pointer
+ *
+ * returns
+ * XHASH_ERR if error occures
+ * XHASH_OK  if node is freed
+ */
+int xhash_free_anr_lru(XHash *t)
+{
+    if (t == nullptr)
+        return XHASH_ERR;
+    
+    if (t->fhead)
+    {
+        if (xhash_delete_free_node(t) == XHASH_OK)
+            return XHASH_OK;
+    }
+
+    if (t->gtail)
+    {
+        if (xhash_free_node(t, t->gtail) == XHASH_OK)
+        {
+            if (t->fhead)
+            {
+                if (xhash_delete_free_node(t) == XHASH_OK)
+                    return XHASH_OK;
+            }
+            else if (!t->recycle_nodes)
+            {
+                return XHASH_OK;
+            }
+        }
+    }
+    return XHASH_ERR;
+}
+
 /*!
  * Find and return the first hash table node
  *
index fb767f7d4cda3073e3ae81beafaa770aee6a49d1..0283b9f44b03b4292a0e2a5c24f080566660b613 100644 (file)
@@ -36,6 +36,7 @@ namespace snort
 #define XHASH_ERR      (-1)
 #define XHASH_OK        0
 #define XHASH_INTABLE   1
+#define XHASH_PENDING   2
 
 struct XHashNode
 {
@@ -99,7 +100,7 @@ SO_PUBLIC XHash* xhash_new(int nrows, int keysize, int datasize, unsigned long m
     int recycle_flag);
 
 SO_PUBLIC void xhash_set_max_nodes(XHash* h, int max_nodes);
-
+SO_PUBLIC int xhash_change_memcap(XHash *t, unsigned long new_memcap, unsigned *max_work);
 SO_PUBLIC void xhash_delete(XHash* h);
 SO_PUBLIC int xhash_make_empty(XHash*);
 
@@ -135,6 +136,7 @@ inline unsigned xhash_overhead_bytes(XHash* t)
 inline unsigned xhash_overhead_blocks(XHash* t)
 { return t->overhead_blocks; }
 
+SO_PUBLIC int xhash_free_anr_lru(XHash* t);
 SO_PUBLIC void* xhash_mru(XHash* t);
 SO_PUBLIC void* xhash_lru(XHash* t);
 SO_PUBLIC void* xhash_find(XHash* h, void* key);