detect: add configurable limits for datasets

Ticket: 7615

Avoids signatures setting extreme hash sizes, which would lead to very
high memory use.

Default to allowing:
- 65536 per dataset
- 16777216 total

To override these built-in defaults:

```yaml
datasets:
  # Limits for per rule dataset instances to avoid rules using too many
  # resources.
  limits:
    # Max value for per dataset `hashsize` setting
    #single-hashsize: 65536
    # Max combined hashsize values for all datasets.
    #total-hashsizes: 16777216
```

(cherry picked from commit a7713db709b8a0be5fc5e5809ab58e9b14a16e85)

diff --git a/src/datasets.c b/src/datasets.c
index 9b098c29814bee097325cbe7b312cc2ddbed4e51..99d66b67de329c0285c29518887b588d90354a37 100644 (file)
--- a/src/datasets.c
+++ b/src/datasets.c
@@ -39,11 +39,16 @@
 #include "util-misc.h"
 #include "util-path.h"
 #include "util-debug.h"
+#include "util-validate.h"
 
 SCMutex sets_lock = SCMUTEX_INITIALIZER;
 static Dataset *sets = NULL;
 static uint32_t set_ids = 0;
 
+uint32_t dataset_max_one_hashsize = 65536;
+uint32_t dataset_max_total_hashsize = 16777216;
+uint32_t dataset_used_hashsize = 0;
+
 static int DatasetAddwRep(Dataset *set, const uint8_t *data, const uint32_t data_len,
         DataRepType *rep);
 
@@ -629,6 +634,34 @@ Dataset *DatasetFind(const char *name, enum DatasetTypes type)
     return set;
 }
 
+static bool DatasetCheckHashsize(const char *name, uint32_t hash_size)
+{
+    if (dataset_max_one_hashsize > 0 && hash_size > dataset_max_one_hashsize) {
+        SCLogError("hashsize %u in dataset '%s' exceeds configured 'single-hashsize' limit (%u)",
+                hash_size, name, dataset_max_one_hashsize);
+        return false;
+    }
+    // we cannot underflow as we know from conf loading that
+    // dataset_max_total_hashsize >= dataset_max_one_hashsize if dataset_max_total_hashsize > 0
+    if (dataset_max_total_hashsize > 0 &&
+            dataset_max_total_hashsize - hash_size < dataset_used_hashsize) {
+        SCLogError("hashsize %u in dataset '%s' exceeds configured 'total-hashsizes' limit (%u, in "
+                   "use %u)",
+                hash_size, name, dataset_max_total_hashsize, dataset_used_hashsize);
+        return false;
+    }
+
+    return true;
+}
+
+static void DatasetUpdateHashsize(const char *name, uint32_t hash_size)
+{
+    if (dataset_max_total_hashsize > 0) {
+        dataset_used_hashsize += hash_size;
+        SCLogDebug("set %s adding with hash_size %u", name, hash_size);
+    }
+}
+
 Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save, const char *load,
         uint64_t memcap, uint32_t hashsize)
 {
@@ -682,6 +715,10 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
         hashsize = default_hashsize;
     }
 
+    if (!DatasetCheckHashsize(name, hashsize)) {
+        goto out_err;
+    }
+
     set = DatasetAlloc(name);
     if (set == NULL) {
         goto out_err;
@@ -760,6 +797,10 @@ Dataset *DatasetGet(const char *name, enum DatasetTypes type, const char *save,
     set->next = sets;
     sets = set;
 
+    /* hash size accounting */
+    DEBUG_VALIDATE_BUG_ON(set->hash->config.hash_size != hashsize);
+    DatasetUpdateHashsize(set->name, set->hash->config.hash_size);
+
     SCMutexUnlock(&sets_lock);
     return set;
 out_err:
@@ -801,6 +842,9 @@ void DatasetReload(void)
             continue;
         }
         set->hidden = true;
+        if (dataset_max_total_hashsize > 0) {
+            dataset_used_hashsize -= set->hash->config.hash_size;
+        }
         SCLogDebug("Set %s at %p hidden successfully", set->name, set);
         set = set->next;
     }
@@ -868,6 +912,27 @@ int DatasetsInit(void)
     uint32_t default_hashsize = 0;
     GetDefaultMemcap(&default_memcap, &default_hashsize);
     if (datasets != NULL) {
+        const char *str = NULL;
+        if (ConfGet("datasets.limits.total-hashsizes", &str) == 1) {
+            if (ParseSizeStringU32(str, &dataset_max_total_hashsize) < 0) {
+                FatalError("failed to parse datasets.limits.total-hashsizes value: %s", str);
+            }
+        }
+        if (ConfGet("datasets.limits.single-hashsize", &str) == 1) {
+            if (ParseSizeStringU32(str, &dataset_max_one_hashsize) < 0) {
+                FatalError("failed to parse datasets.limits.single-hashsize value: %s", str);
+            }
+        }
+        if (dataset_max_total_hashsize > 0 &&
+                dataset_max_total_hashsize < dataset_max_one_hashsize) {
+            FatalError("total-hashsizes (%u) cannot be smaller than single-hashsize (%u)",
+                    dataset_max_total_hashsize, dataset_max_one_hashsize);
+        }
+        if (dataset_max_total_hashsize > 0 && dataset_max_one_hashsize == 0) {
+            // the total limit also applies for single limit
+            dataset_max_one_hashsize = dataset_max_total_hashsize;
+        }
+
         int list_pos = 0;
         ConfNode *iter = NULL;
         TAILQ_FOREACH(iter, &datasets->head, next) {

diff --git a/src/tests/fuzz/confyaml.c b/src/tests/fuzz/confyaml.c
index 194552859909014c1b0853b8d7176ec9deaa558b..05995ea56f57af23895d630abef931104ac14ebe 100644 (file)
--- a/src/tests/fuzz/confyaml.c
+++ b/src/tests/fuzz/confyaml.c
@@ -112,4 +112,8 @@ app-layer:\n\
       enabled: yes\n\
 detect:\n\
   inspection-recursion-limit: 0\n\
+datasets:\n\
+  maximums:\n\
+    single_hashsize: 65536\n\
+    total_hashsizes: 16777216\n\
 ";

diff --git a/src/util-thash.c b/src/util-thash.c
index 3ee006a2c59098188a1827b7972a98a0c9858635..548637916b18774af18a6355076a921ec14f9665 100644 (file)
--- a/src/util-thash.c
+++ b/src/util-thash.c
@@ -311,16 +311,11 @@ THashTableContext *THashInit(const char *cnf_prefix, size_t data_size,
     ctx->config.hash_size = hashsize > 0 ? hashsize : THASH_DEFAULT_HASHSIZE;
     /* Reset memcap in case of loading from file to the highest possible value
      unless defined by the rule keyword */
-#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
-    // limit memcap size to default when fuzzing
-    ctx->config.memcap = THASH_DEFAULT_MEMCAP;
-#else
     if (memcap > 0) {
         ctx->config.memcap = memcap;
     } else {
         ctx->config.memcap = reset_memcap ? UINT64_MAX : THASH_DEFAULT_MEMCAP;
     }
-#endif
     ctx->config.prealloc = THASH_DEFAULT_PREALLOC;
 
     SC_ATOMIC_INIT(ctx->counter);

diff --git a/suricata.yaml.in b/suricata.yaml.in
index d74b4a27d35308a994d5ee58911271c48133f734..97236eb398535e001dd4e24a303acc5ec1c0df58 100644 (file)
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -1188,6 +1188,14 @@ datasets:
     #memcap: 100mb
     #hashsize: 2048
 
+  # Limits for per rule dataset instances to avoid rules using too many
+  # resources.
+  limits:
+    # Max value for per dataset `hashsize` setting
+    #single-hashsize: 65536
+    # Max combined hashsize values for all datasets.
+    #total-hashsizes: 16777216
+
   rules:
     # Set to true to allow absolute filenames and filenames that use
     # ".." components to reference parent directories in rules that specify