detect/iponly: Reduce the size of the SigNumArray bitsets

Instead of tracking ip only rules by the internal signum, track them by
a separate counter that starts at zero.  This results in dense
SigNumArrays instead of sparse ones and a much smaller max_idx.

Issue: 4578
(cherry picked from commit dfbc3da0eb2931ad5ff19f3bdfa5a19839eea101)

diff --git a/src/detect-engine-iponly.c b/src/detect-engine-iponly.c
index d06d28e06f4086d99b2aad34cf4aa69c59f60fa3..91cf07d6c0ad5f788d0a855418b730a7398af960 100644 (file)
--- a/src/detect-engine-iponly.c
+++ b/src/detect-engine-iponly.c
@@ -892,14 +892,24 @@ error:
  */
 void IPOnlyInit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
 {
-    io_ctx->tree_ipv4src = SCRadixCreateRadixTree(SigNumArrayFree,
-                                                  SigNumArrayPrint);
-    io_ctx->tree_ipv4dst = SCRadixCreateRadixTree(SigNumArrayFree,
-                                                  SigNumArrayPrint);
-    io_ctx->tree_ipv6src = SCRadixCreateRadixTree(SigNumArrayFree,
-                                                  SigNumArrayPrint);
-    io_ctx->tree_ipv6dst = SCRadixCreateRadixTree(SigNumArrayFree,
-                                                  SigNumArrayPrint);
+    io_ctx->tree_ipv4src = SCRadixCreateRadixTree(SigNumArrayFree, SigNumArrayPrint);
+    io_ctx->tree_ipv4dst = SCRadixCreateRadixTree(SigNumArrayFree, SigNumArrayPrint);
+    io_ctx->tree_ipv6src = SCRadixCreateRadixTree(SigNumArrayFree, SigNumArrayPrint);
+    io_ctx->tree_ipv6dst = SCRadixCreateRadixTree(SigNumArrayFree, SigNumArrayPrint);
+
+    io_ctx->sig_mapping = SCCalloc(1, de_ctx->sig_array_len * sizeof(uint32_t));
+    if (io_ctx->sig_mapping == NULL) {
+        FatalError(SC_ERR_MEM_ALLOC, "Unable to allocate iponly signature tracking area");
+    }
+    io_ctx->sig_mapping_size = 0;
+}
+
+SigIntId IPOnlyTrackSigNum(DetectEngineIPOnlyCtx *io_ctx, SigIntId signum)
+{
+    SigIntId loc = io_ctx->sig_mapping_size;
+    io_ctx->sig_mapping[loc] = signum;
+    io_ctx->sig_mapping_size++;
+    return loc;
 }
 
 /**
@@ -959,6 +969,10 @@ void IPOnlyDeinit(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx)
     if (io_ctx->tree_ipv6dst != NULL)
         SCRadixReleaseRadixTree(io_ctx->tree_ipv6dst);
     io_ctx->tree_ipv6dst = NULL;
+
+    if (io_ctx->sig_mapping != NULL)
+        SCFree(io_ctx->sig_mapping);
+    io_ctx->sig_mapping = NULL;
 }
 
 /**
@@ -1043,20 +1057,18 @@ void IPOnlyMatchPacket(ThreadVars *tv,
     for (u = 0; u < src->size; u++) {
         SCLogDebug("And %"PRIu8" & %"PRIu8, src->array[u], dst->array[u]);
 
-        /* The final results will be at io_tctx */
-        io_tctx->sig_match_array[u] = dst->array[u] & src->array[u];
+        uint8_t bitarray = dst->array[u] & src->array[u];
 
         /* We have to move the logic of the signature checking
          * to the main detect loop, in order to apply the
          * priority of actions (pass, drop, reject, alert) */
-        if (io_tctx->sig_match_array[u] != 0) {
+        if (bitarray) {
             /* We have a match :) Let's see from which signum's */
-            uint8_t bitarray = io_tctx->sig_match_array[u];
             uint8_t i = 0;
 
             for (; i < 8; i++, bitarray = bitarray >> 1) {
                 if (bitarray & 0x01) {
-                    Signature *s = de_ctx->sig_array[u * 8 + i];
+                    Signature *s = de_ctx->sig_array[io_ctx->sig_mapping[u * 8 + i]];
 
                     if ((s->proto.flags & DETECT_PROTO_IPV4) && !PKT_IS_IPV4(p)) {
                         SCLogDebug("ip version didn't match");
@@ -1564,10 +1576,13 @@ void IPOnlyAddSignature(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx,
     if (!(s->flags & SIG_FLAG_IPONLY))
         return;
 
+    SigIntId mapped_signum = IPOnlyTrackSigNum(io_ctx, s->num);
+    SCLogDebug("Adding IPs from rule: %" PRIu32 " (%s) as %" PRIu32 " mapped to %" PRIu32 "\n",
+            s->id, s->msg, s->num, mapped_signum);
     /* Set the internal signum to the list before merging */
-    IPOnlyCIDRListSetSigNum(s->CidrSrc, s->num);
+    IPOnlyCIDRListSetSigNum(s->CidrSrc, mapped_signum);
 
-    IPOnlyCIDRListSetSigNum(s->CidrDst, s->num);
+    IPOnlyCIDRListSetSigNum(s->CidrDst, mapped_signum);
 
     /**
      * ipv4 and ipv6 are mixed, but later we will separate them into
@@ -1576,8 +1591,8 @@ void IPOnlyAddSignature(DetectEngineCtx *de_ctx, DetectEngineIPOnlyCtx *io_ctx,
     io_ctx->ip_src = IPOnlyCIDRItemInsert(io_ctx->ip_src, s->CidrSrc);
     io_ctx->ip_dst = IPOnlyCIDRItemInsert(io_ctx->ip_dst, s->CidrDst);
 
-    if (s->num > io_ctx->max_idx)
-        io_ctx->max_idx = s->num;
+    if (mapped_signum > io_ctx->max_idx)
+        io_ctx->max_idx = mapped_signum;
 
     /** no longer ref to this, it's in the table now */
     s->CidrSrc = NULL;

diff --git a/src/detect-engine-iponly.h b/src/detect-engine-iponly.h
index 7f4f70bcf080044be7c1ca5708cacb62eb67954e..117cb3c36700480835af12a2f3a3163e928656c6 100644 (file)
--- a/src/detect-engine-iponly.h
+++ b/src/detect-engine-iponly.h
@@ -35,6 +35,7 @@ void IPOnlyDeinit(DetectEngineCtx *, DetectEngineIPOnlyCtx *);
 void IPOnlyPrepare(DetectEngineCtx *);
 void DetectEngineIPOnlyThreadInit(DetectEngineCtx *, DetectEngineIPOnlyThreadCtx *);
 void DetectEngineIPOnlyThreadDeinit(DetectEngineIPOnlyThreadCtx *);
+SigIntId IPOnlyTrackSigNum(DetectEngineIPOnlyCtx *, SigIntId);
 void IPOnlyAddSignature(DetectEngineCtx *, DetectEngineIPOnlyCtx *, Signature *);
 void IPOnlyRegisterTests(void);
 

diff --git a/src/detect.h b/src/detect.h
index df6f14493cc05da8f3e4dc42cbe9eb9dac8785d0..0320dc7a1d7eec24ec2e1cd6ab8129e1c52babf9 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -689,6 +689,11 @@ typedef struct DetectEngineIPOnlyCtx_ {
     /* Used to build the radix trees */
     IPOnlyCIDRItem *ip_src, *ip_dst;
     uint32_t max_idx;
+
+    /* Used to map large signums to smaller values to compact the bitsets
+     * stored in the radix trees */
+    uint32_t *sig_mapping;
+    uint32_t sig_mapping_size;
 } DetectEngineIPOnlyCtx;
 
 typedef struct DetectEngineLookupFlow_ {