thash: add expiration logic

Add a callback and helper function to handle data expiration.

Update datasets to explicitly not use expiration.

diff --git a/src/app-layer-htp-range.c b/src/app-layer-htp-range.c
index 394966ac94e01ed389b233e9d65014f56268490b..f7f110ca4fdd37a9044c4f2e71ba9c6ca5365890 100644 (file)
--- a/src/app-layer-htp-range.c
+++ b/src/app-layer-htp-range.c
@@ -127,8 +127,9 @@ static void ContainerUrlRangeFree(void *s)
     }
 }
 
-static inline bool ContainerValueRangeTimeout(HttpRangeContainerFile *cu, const SCTime_t ts)
+static inline bool ContainerValueRangeTimeout(void *data, const SCTime_t ts)
 {
+    HttpRangeContainerFile *cu = data;
     // we only timeout if we have no flow referencing us
     if (SCTIME_CMP_GT(ts, cu->expire) || cu->error) {
         if (SC_ATOMIC_GET(cu->hdata->use_cnt) == 0) {
@@ -171,10 +172,10 @@ void HttpRangeContainersInit(void)
         }
     }
 
-    ContainerUrlRangeList.ht =
-            THashInit("app-layer.protocols.http.byterange", sizeof(HttpRangeContainerFile),
-                    ContainerUrlRangeSet, ContainerUrlRangeFree, ContainerUrlRangeHash,
-                    ContainerUrlRangeCompare, false, memcap, CONTAINER_URLRANGE_HASH_SIZE);
+    ContainerUrlRangeList.ht = THashInit("app-layer.protocols.http.byterange",
+            sizeof(HttpRangeContainerFile), ContainerUrlRangeSet, ContainerUrlRangeFree,
+            ContainerUrlRangeHash, ContainerUrlRangeCompare, ContainerValueRangeTimeout, false,
+            memcap, CONTAINER_URLRANGE_HASH_SIZE);
     ContainerUrlRangeList.timeout = timeout;
 
     SCLogDebug("containers started");

diff --git a/src/datasets.c b/src/datasets.c
index 31fa6b398ca20fff9a853cab9b204eba041ea41c..02e656f35e25156244dbbd07a95af19516a93c1d 100644 (file)
--- a/src/datasets.c
+++ b/src/datasets.c
@@ -701,7 +701,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
     switch (type) {
         case DATASET_TYPE_MD5:
             set->hash = THashInit(cnf_name, sizeof(Md5Type), Md5StrSet, Md5StrFree, Md5StrHash,
-                    Md5StrCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    Md5StrCompare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -710,7 +710,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_STRING:
             set->hash = THashInit(cnf_name, sizeof(StringType), StringSet, StringFree, StringHash,
-                    StringCompare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    StringCompare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -719,7 +719,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_SHA256:
             set->hash = THashInit(cnf_name, sizeof(Sha256Type), Sha256StrSet, Sha256StrFree,
-                    Sha256StrHash, Sha256StrCompare, load != NULL ? 1 : 0,
+                    Sha256StrHash, Sha256StrCompare, NULL, load != NULL ? 1 : 0,
                     memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
@@ -729,7 +729,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_IPV4:
             set->hash = THashInit(cnf_name, sizeof(IPv4Type), IPv4Set, IPv4Free, IPv4Hash,
-                    IPv4Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    IPv4Compare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;
@@ -738,7 +738,7 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
             break;
         case DATASET_TYPE_IPV6:
             set->hash = THashInit(cnf_name, sizeof(IPv6Type), IPv6Set, IPv6Free, IPv6Hash,
-                    IPv6Compare, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
+                    IPv6Compare, NULL, load != NULL ? 1 : 0, memcap > 0 ? memcap : default_memcap,
                     hashsize > 0 ? hashsize : default_hashsize);
             if (set->hash == NULL)
                 goto out_err;

diff --git a/src/util-thash.c b/src/util-thash.c
index a17679605a2273241e683d5bb3433da93a2d9914..cdaf68a326a332167abc6dd14bbe45e7342606b1 100644 (file)
--- a/src/util-thash.c
+++ b/src/util-thash.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2016 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -294,7 +294,8 @@ static int THashInitConfig(THashTableContext *ctx, const char *cnf_prefix)
 
 THashTableContext *THashInit(const char *cnf_prefix, size_t data_size,
         int (*DataSet)(void *, void *), void (*DataFree)(void *), uint32_t (*DataHash)(void *),
-        bool (*DataCompare)(void *, void *), bool reset_memcap, uint64_t memcap, uint32_t hashsize)
+        bool (*DataCompare)(void *, void *), bool (*DataExpired)(void *, SCTime_t),
+        bool reset_memcap, uint64_t memcap, uint32_t hashsize)
 {
     THashTableContext *ctx = SCCalloc(1, sizeof(*ctx));
     BUG_ON(!ctx);
@@ -304,6 +305,7 @@ THashTableContext *THashInit(const char *cnf_prefix, size_t data_size,
     ctx->config.DataFree = DataFree;
     ctx->config.DataHash = DataHash;
     ctx->config.DataCompare = DataCompare;
+    ctx->config.DataExpired = DataExpired;
 
     /* set defaults */
     ctx->config.hash_rand = (uint32_t)RandomGet();
@@ -407,6 +409,59 @@ int THashWalk(THashTableContext *ctx, THashFormatFunc FormatterFunc, THashOutput
     return 0;
 }
 
+/** \brief expire data from the hash
+ *  Walk the hash table and remove data that is exprired according to the
+ *  DataExpired callback.
+ *  \retval cnt number of items successfully expired/removed
+ */
+uint32_t THashExpire(THashTableContext *ctx, const SCTime_t ts)
+{
+    if (ctx->config.DataExpired == NULL)
+        return 0;
+
+    SCLogDebug("timeout: starting");
+    uint32_t cnt = 0;
+
+    for (uint32_t i = 0; i < ctx->config.hash_size; i++) {
+        THashHashRow *hb = &ctx->array[i];
+        if (HRLOCK_TRYLOCK(hb) != 0)
+            continue;
+        /* hash bucket is now locked */
+        THashData *h = hb->head;
+        while (h) {
+            THashData *next = h->next;
+            THashDataLock(h);
+            DEBUG_VALIDATE_BUG_ON(SC_ATOMIC_GET(h->use_cnt) > (uint32_t)INT_MAX);
+            /* only consider items with no references to it */
+            if (SC_ATOMIC_GET(h->use_cnt) == 0 && ctx->config.DataExpired(h->data, ts)) {
+                /* remove from the hash */
+                if (h->prev != NULL)
+                    h->prev->next = h->next;
+                if (h->next != NULL)
+                    h->next->prev = h->prev;
+                if (hb->head == h)
+                    hb->head = h->next;
+                if (hb->tail == h)
+                    hb->tail = h->prev;
+                h->next = NULL;
+                h->prev = NULL;
+                SCLogDebug("timeout: removing data %p", h);
+                ctx->config.DataFree(h->data);
+                THashDataUnlock(h);
+                THashDataMoveToSpare(ctx, h);
+                cnt++;
+            } else {
+                THashDataUnlock(h);
+            }
+            h = next;
+        }
+        HRLOCK_UNLOCK(hb);
+    }
+
+    SCLogDebug("timeout: ending: %u entries expired", cnt);
+    return cnt;
+}
+
 /** \brief Cleanup the thash engine
  *
  * Cleanup the thash engine from tag and threshold.

diff --git a/src/util-thash.h b/src/util-thash.h
index f45b6e84316717bd55a310135a13be491b528b9c..569f1ff9c7951741c61a0e66ee5e3c7e72503f65 100644 (file)
--- a/src/util-thash.h
+++ b/src/util-thash.h
@@ -132,6 +132,7 @@ typedef struct THashDataConfig_ {
     void (*DataFree)(void *);
     uint32_t (*DataHash)(void *);
     bool (*DataCompare)(void *, void *);
+    bool (*DataExpired)(void *, SCTime_t ts);
 } THashConfig;
 
 #define THASH_DATA_SIZE(ctx) (sizeof(THashData) + (ctx)->config.data_size)
@@ -169,8 +170,9 @@ typedef struct THashTableContext_ {
 
 THashTableContext *THashInit(const char *cnf_prefix, size_t data_size,
         int (*DataSet)(void *dst, void *src), void (*DataFree)(void *),
-        uint32_t (*DataHash)(void *), bool (*DataCompare)(void *, void *), bool reset_memcap,
-        uint64_t memcap, uint32_t hashsize);
+        uint32_t (*DataHash)(void *), bool (*DataCompare)(void *, void *),
+        bool (*DataExpired)(void *, SCTime_t), bool reset_memcap, uint64_t memcap,
+        uint32_t hashsize);
 
 void THashShutdown(THashTableContext *ctx);
 
@@ -197,5 +199,6 @@ int THashWalk(THashTableContext *, THashFormatFunc, THashOutputFunc, void *);
 int THashRemoveFromHash (THashTableContext *ctx, void *data);
 void THashConsolidateMemcap(THashTableContext *ctx);
 void THashDataMoveToSpare(THashTableContext *ctx, THashData *h);
+uint32_t THashExpire(THashTableContext *ctx, const SCTime_t ts);
 
 #endif /* SURICATA_THASH_H */