util/hash: use randomized hash algorithm

For datasets and http ranges

Ticket: 7209

Prevents abusive hash collisions from known djb2 algorithm

(cherry picked from commit 26da953f6dad3793d29f27ce7ab6628a2db8f471)

diff --git a/src/app-layer-htp-range.c b/src/app-layer-htp-range.c
index f0d75a9750c05a2005d866d3865884efcfa500f7..5af62a1404b58d6ebe930389e60586747deb36db 100644 (file)
--- a/src/app-layer-htp-range.c
+++ b/src/app-layer-htp-range.c
@@ -26,7 +26,7 @@
 #include "util-misc.h"        //ParseSizeStringU64
 #include "util-thash.h"       //HashTable
 #include "util-memcmp.h"      //SCBufferCmp
-#include "util-hash-string.h" //StringHashDjb2
+#include "util-hash-lookup3.h" //hashlittle_safe
 #include "util-validate.h"    //DEBUG_VALIDATE_BUG_ON
 #include "util-byte.h"        //StringParseUint32
 
@@ -102,10 +102,10 @@ static bool ContainerUrlRangeCompare(void *a, void *b)
     return false;
 }
 
-static uint32_t ContainerUrlRangeHash(void *s)
+static uint32_t ContainerUrlRangeHash(uint32_t hash_seed, void *s)
 {
     HttpRangeContainerFile *cur = s;
-    uint32_t h = StringHashDjb2(cur->key, cur->len);
+    uint32_t h = hashlittle_safe(cur->key, cur->len, hash_seed);
     return h;
 }
 

diff --git a/src/datasets-ipv4.c b/src/datasets-ipv4.c
index f1192a0db5d1ff02c7efa0234730784cc707e4a0..67f8778fd2d613fd89e4652fbed151092e8786c5 100644 (file)
--- a/src/datasets-ipv4.c
+++ b/src/datasets-ipv4.c
@@ -25,6 +25,7 @@
 #include "conf.h"
 #include "datasets.h"
 #include "datasets-ipv4.h"
+#include "util-hash-lookup3.h"
 #include "util-thash.h"
 #include "util-print.h"
 
@@ -45,15 +46,10 @@ bool IPv4Compare(void *a, void *b)
     return (memcmp(as->ipv4, bs->ipv4, sizeof(as->ipv4)) == 0);
 }
 
-uint32_t IPv4Hash(void *s)
+uint32_t IPv4Hash(uint32_t hash_seed, void *s)
 {
     const IPv4Type *str = s;
-    uint32_t hash = 5381;
-
-    for (int i = 0; i < (int)sizeof(str->ipv4); i++) {
-        hash = ((hash << 5) + hash) + str->ipv4[i]; /* hash * 33 + c */
-    }
-    return hash;
+    return hashword((uint32_t *)str->ipv4, 1, hash_seed);
 }
 
 // data stays in hash

diff --git a/src/datasets-ipv4.h b/src/datasets-ipv4.h
index 277acc6b88e6824b099c4d85eda82c2d118a6c73..799d22acbb35fef5e395cef4f05bf238be928cc9 100644 (file)
--- a/src/datasets-ipv4.h
+++ b/src/datasets-ipv4.h
@@ -33,7 +33,7 @@ typedef struct IPv4Type {
 
 int IPv4Set(void *dst, void *src);
 bool IPv4Compare(void *a, void *b);
-uint32_t IPv4Hash(void *s);
+uint32_t IPv4Hash(uint32_t hash_seed, void *s);
 void IPv4Free(void *s);
 
 #endif /* __DATASETS_IPV4_H__ */

diff --git a/src/datasets-ipv6.c b/src/datasets-ipv6.c
index f907320f007d0b3645205191fa81a817da743114..ac96374da7c79a9f7e9daed71188bf30891624d8 100644 (file)
--- a/src/datasets-ipv6.c
+++ b/src/datasets-ipv6.c
@@ -25,6 +25,7 @@
 #include "conf.h"
 #include "datasets.h"
 #include "datasets-ipv6.h"
+#include "util-hash-lookup3.h"
 #include "util-thash.h"
 #include "util-print.h"
 
@@ -45,15 +46,10 @@ bool IPv6Compare(void *a, void *b)
     return (memcmp(as->ipv6, bs->ipv6, sizeof(as->ipv6)) == 0);
 }
 
-uint32_t IPv6Hash(void *s)
+uint32_t IPv6Hash(uint32_t hash_seed, void *s)
 {
     const IPv6Type *str = s;
-    uint32_t hash = 5381;
-
-    for (int i = 0; i < (int)sizeof(str->ipv6); i++) {
-        hash = ((hash << 5) + hash) + str->ipv6[i]; /* hash * 33 + c */
-    }
-    return hash;
+    return hashword((uint32_t *)str->ipv6, 4, hash_seed);
 }
 
 // data stays in hash

diff --git a/src/datasets-ipv6.h b/src/datasets-ipv6.h
index a4cabcaf78fe3f16d383954401f5d47a680cd517..51f273664c06c20a9a13d5eec030e1ea9eb31f4f 100644 (file)
--- a/src/datasets-ipv6.h
+++ b/src/datasets-ipv6.h
@@ -33,7 +33,7 @@ typedef struct IPv6Type {
 
 int IPv6Set(void *dst, void *src);
 bool IPv6Compare(void *a, void *b);
-uint32_t IPv6Hash(void *s);
+uint32_t IPv6Hash(uint32_t hash_seed, void *s);
 void IPv6Free(void *s);
 
 #endif /* __DATASETS_IPV4_H__ */

diff --git a/src/datasets-md5.c b/src/datasets-md5.c
index 3b1d8f3fc02ceb0f0fdf3e300bbbea9627f69069..517cdff5be69e00f026cf20b3fb12f65c611c2ba 100644 (file)
--- a/src/datasets-md5.c
+++ b/src/datasets-md5.c
@@ -25,6 +25,8 @@
 #include "conf.h"
 #include "datasets.h"
 #include "datasets-md5.h"
+#include "util-hash-lookup3.h"
+
 #include "util-thash.h"
 #include "util-print.h"
 #include "util-base64.h"    // decode base64
@@ -46,15 +48,10 @@ bool Md5StrCompare(void *a, void *b)
     return (memcmp(as->md5, bs->md5, sizeof(as->md5)) == 0);
 }
 
-uint32_t Md5StrHash(void *s)
+uint32_t Md5StrHash(uint32_t hash_seed, void *s)
 {
     const Md5Type *str = s;
-    uint32_t hash = 5381;
-
-    for (int i = 0; i < (int)sizeof(str->md5); i++) {
-        hash = ((hash << 5) + hash) + str->md5[i]; /* hash * 33 + c */
-    }
-    return hash;
+    return hashword((uint32_t *)str->md5, sizeof(str->md5) / 4, hash_seed);
 }
 
 // data stays in hash

diff --git a/src/datasets-md5.h b/src/datasets-md5.h
index ad4d0e28a3896064e24fc4a9ef35b38869ea44d0..2740ba540ac25d9055ced5c1eb99c2cc523dcc10 100644 (file)
--- a/src/datasets-md5.h
+++ b/src/datasets-md5.h
@@ -33,7 +33,7 @@ typedef struct Md5Type {
 
 int Md5StrSet(void *dst, void *src);
 bool Md5StrCompare(void *a, void *b);
-uint32_t Md5StrHash(void *s);
+uint32_t Md5StrHash(uint32_t hash_seed, void *s);
 void Md5StrFree(void *s);
 
 #endif /* __DATASETS_MD5_H__ */

diff --git a/src/datasets-sha256.c b/src/datasets-sha256.c
index 346397d6d6fa967e593e31e8029b2e1655e9e67f..0929915353ccb039abd5c4ed25e64bb26a0817ba 100644 (file)
--- a/src/datasets-sha256.c
+++ b/src/datasets-sha256.c
@@ -25,9 +25,8 @@
 #include "conf.h"
 #include "datasets.h"
 #include "datasets-sha256.h"
+#include "util-hash-lookup3.h"
 #include "util-thash.h"
-#include "util-print.h"
-#include "util-base64.h"    // decode base64
 
 int Sha256StrSet(void *dst, void *src)
 {
@@ -46,15 +45,10 @@ bool Sha256StrCompare(void *a, void *b)
     return (memcmp(as->sha256, bs->sha256, sizeof(as->sha256)) == 0);
 }
 
-uint32_t Sha256StrHash(void *s)
+uint32_t Sha256StrHash(uint32_t hash_seed, void *s)
 {
     Sha256Type *str = s;
-    uint32_t hash = 5381;
-
-    for (int i = 0; i < (int)sizeof(str->sha256); i++) {
-        hash = ((hash << 5) + hash) + str->sha256[i]; /* hash * 33 + c */
-    }
-    return hash;
+    return hashword((uint32_t *)str->sha256, sizeof(str->sha256) / 4, hash_seed);
 }
 
 // data stays in hash

diff --git a/src/datasets-sha256.h b/src/datasets-sha256.h
index 8793cc17a56ea3a510bda599adb01d1fa38f67d1..a9c81ac75abaffd08f35c610d42021d6b5a61389 100644 (file)
--- a/src/datasets-sha256.h
+++ b/src/datasets-sha256.h
@@ -33,7 +33,7 @@ typedef struct Sha256Type {
 
 int Sha256StrSet(void *dst, void *src);
 bool Sha256StrCompare(void *a, void *b);
-uint32_t Sha256StrHash(void *s);
+uint32_t Sha256StrHash(uint32_t hash_seed, void *s);
 void Sha256StrFree(void *s);
 
 #endif /* __DATASETS_SHA256_H__ */

diff --git a/src/datasets-string.c b/src/datasets-string.c
index 4a572898ceb3b0d9e1492c2c86b6fcacc530e5fd..0a8f499ae33353b60064e7c7cf4bd9954a5098c3 100644 (file)
--- a/src/datasets-string.c
+++ b/src/datasets-string.c
@@ -28,6 +28,7 @@
 #include "util-thash.h"
 #include "util-print.h"
 #include "util-base64.h"    // decode base64
+#include "util-hash-lookup3.h"
 #include "rust.h"
 
 #if 0
@@ -85,17 +86,10 @@ bool StringCompare(void *a, void *b)
     return (memcmp(as->ptr, bs->ptr, as->len) == 0);
 }
 
-uint32_t StringHash(void *s)
+uint32_t StringHash(uint32_t hash_seed, void *s)
 {
-    uint32_t hash = 5381;
     StringType *str = s;
-
-    for (uint32_t i = 0; i < str->len; i++) {
-        int c = str->ptr[i];
-        hash = ((hash << 5) + hash) + c; /* hash * 33 + c */
-    }
-
-    return hash;
+    return hashlittle_safe(str->ptr, str->len, hash_seed);
 }
 
 // base data stays in hash

diff --git a/src/datasets-string.h b/src/datasets-string.h
index b1513e358459a57aa615c96afb2e68aa8a6ce67e..2cd624833b3d76500cc8aaeb2aa7f2525577e745 100644 (file)
--- a/src/datasets-string.h
+++ b/src/datasets-string.h
@@ -34,7 +34,7 @@ typedef struct StringType {
 
 int StringSet(void *dst, void *src);
 bool StringCompare(void *a, void *b);
-uint32_t StringHash(void *s);
+uint32_t StringHash(uint32_t hash_seed, void *s);
 void StringFree(void *s);
 int StringAsBase64(const void *s, char *out, size_t out_size);
 

diff --git a/src/util-thash.c b/src/util-thash.c
index d9500e72622251e69dc855d7948fa4b9672dc599..9ee376f97da80a54d157da6fda50b3f2579b45ba 100644 (file)
--- a/src/util-thash.c
+++ b/src/util-thash.c
@@ -293,7 +293,7 @@ static int THashInitConfig(THashTableContext *ctx, const char *cnf_prefix)
 }
 
 THashTableContext *THashInit(const char *cnf_prefix, size_t data_size,
-        int (*DataSet)(void *, void *), void (*DataFree)(void *), uint32_t (*DataHash)(void *),
+        int (*DataSet)(void *, void *), void (*DataFree)(void *), uint32_t (*DataHash)(uint32_t, void *),
         bool (*DataCompare)(void *, void *), bool reset_memcap, uint64_t memcap, uint32_t hashsize)
 {
     THashTableContext *ctx = SCCalloc(1, sizeof(*ctx));
@@ -459,7 +459,7 @@ static uint32_t THashGetKey(const THashConfig *cnf, void *data)
 {
     uint32_t key;
 
-    key = cnf->DataHash(data);
+    key = cnf->DataHash(cnf->hash_rand, data);
     key %= cnf->hash_size;
 
     return key;

diff --git a/src/util-thash.h b/src/util-thash.h
index 9618d5c06442f2d24c349cf9aeb8775aa96d43c6..3e6883503f85366e3124eaa2377c5cd6633ea96a 100644 (file)
--- a/src/util-thash.h
+++ b/src/util-thash.h
@@ -130,7 +130,7 @@ typedef struct THashDataConfig_ {
     uint32_t data_size;
     int (*DataSet)(void *dst, void *src);
     void (*DataFree)(void *);
-    uint32_t (*DataHash)(void *);
+    uint32_t (*DataHash)(uint32_t, void *);
     bool (*DataCompare)(void *, void *);
 } THashConfig;
 
@@ -169,7 +169,7 @@ typedef struct THashTableContext_ {
 
 THashTableContext *THashInit(const char *cnf_prefix, size_t data_size,
         int (*DataSet)(void *dst, void *src), void (*DataFree)(void *),
-        uint32_t (*DataHash)(void *), bool (*DataCompare)(void *, void *), bool reset_memcap,
+        uint32_t (*DataHash)(uint32_t, void *), bool (*DataCompare)(void *, void *), bool reset_memcap,
         uint64_t memcap, uint32_t hashsize);
 
 void THashShutdown(THashTableContext *ctx);