Dynamically resize pattern id array as needed

Rather than creating the array of size maxpatid, dynamically resize as needed.
This also handles the case where duplicate pid are added to the array.

Also fix error in bitarray allocation (local version) to always use bitarray_size.

diff --git a/src/util-mpm-ac-bs.c b/src/util-mpm-ac-bs.c
index 6d58d6f61e4b5dc0209c4a76ab3f520ffd3c6558..c706c0982192209e1ab6b1343fc55dd508a489e2 100644 (file)
--- a/src/util-mpm-ac-bs.c
+++ b/src/util-mpm-ac-bs.c
@@ -1502,8 +1502,7 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
                             pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid;
-
+                            MpmAddPid(pmq, lower_pid);
                             MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
@@ -1513,8 +1512,8 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
 
+                            MpmAddPid(pmq, pids[k]);
                             MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;
@@ -1590,8 +1589,8 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
                             pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid;
 
+                            MpmAddPid(pmq, lower_pid);
                             MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
@@ -1601,8 +1600,8 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
 
+                            MpmAddPid(pmq, pids[k]);
                             MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;

diff --git a/src/util-mpm-ac-gfbs.c b/src/util-mpm-ac-gfbs.c
index 5670f66828cd3e362784147fc129bd9a8d3c043e..523aa0ff95acc40cb9fc266b120bf4d570ef4cec 100644 (file)
--- a/src/util-mpm-ac-gfbs.c
+++ b/src/util-mpm-ac-gfbs.c
@@ -1415,8 +1415,8 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
                             pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = (lower_pid);
 
+                            MpmAddPid(pmq, lower_pid);
                             MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
@@ -1426,8 +1426,8 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
 
+                            MpmAddPid(pmq, pids[k]);
                             MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;
@@ -1547,8 +1547,8 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
                             pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = (lower_pid);
 
+                            MpmAddPid(pmq, lower_pid);
                             MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
@@ -1558,8 +1558,8 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
 
+                            MpmAddPid(pmq, pids[k]);
                             MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;

diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c
index 9aed573efd0ca1dc96940bfe49eb02a875c149b5..7df1caa2a9bcdc49fb180317270845720a04f65b 100644 (file)
--- a/src/util-mpm-ac.c
+++ b/src/util-mpm-ac.c
@@ -1333,8 +1333,7 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
                             bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid;
-
+                            MpmAddPid(pmq, lower_pid);
                             MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
@@ -1344,7 +1343,7 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                         } else {
                             pmq->pattern_id_bitarray[(pids[k]) / 8] |= (1 << ((pids[k]) % 8));
                             bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
+                            MpmAddPid(pmq, pids[k]);
                             MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;

diff --git a/src/util-mpm.c b/src/util-mpm.c
index cb11215f7b142f05e380488fb6c9eb0c96358752..5a90e36139b9d78481a32330368d989e99a333f4 100644 (file)
--- a/src/util-mpm.c
+++ b/src/util-mpm.c
@@ -430,14 +430,13 @@ int PmqSetup(PatternMatcherQueue *pmq, uint32_t patmaxid)
     memset(pmq, 0, sizeof(PatternMatcherQueue));
 
     if (patmaxid > 0) {
-        pmq->pattern_id_array_size = patmaxid * sizeof(uint32_t);
+        pmq->pattern_id_array_size = 32; /* Intial size, TODO Make this configure option */
+        pmq->pattern_id_array_cnt = 0;
 
-        pmq->pattern_id_array = SCMalloc(pmq->pattern_id_array_size);
+        pmq->pattern_id_array = SCCalloc(pmq->pattern_id_array_size, sizeof(uint32_t));
         if (pmq->pattern_id_array == NULL) {
             SCReturnInt(-1);
         }
-        memset(pmq->pattern_id_array, 0, pmq->pattern_id_array_size);
-        pmq->pattern_id_array_cnt = 0;
 
         /* lookup bitarray */
         pmq->pattern_id_bitarray_size = (patmaxid / 8) + 1;
@@ -451,7 +450,7 @@ int PmqSetup(PatternMatcherQueue *pmq, uint32_t patmaxid)
         SCLogDebug("pmq->pattern_id_array %p, pmq->pattern_id_bitarray %p",
                 pmq->pattern_id_array, pmq->pattern_id_bitarray);
 
-        pmq->rule_id_array_size = 128;
+        pmq->rule_id_array_size = 128; /* Initial size, TODO: Make configure option. */
         pmq->rule_id_array_cnt = 0;
 
         uint32_t bytes = pmq->rule_id_array_size * sizeof(uint32_t);
@@ -493,6 +492,31 @@ MpmAddSidsResize(PatternMatcherQueue *pmq, uint32_t new_size)
     pmq->rule_id_array_size = new_size;
 }
 
+/** \brief Increase the size of the Pattern rule ID array.
+ *
+ *  \param pmq storage for match results
+ *  \param new_size number of Signature IDs needing to be stored.
+ *
+ */
+void
+MpmAddPidResize(PatternMatcherQueue *pmq, uint32_t new_size)
+{
+    /* Need to make the array bigger. Double the size needed to
+     * also handle the case that sids_size might still be
+     * larger than the old size.
+     */
+    new_size = new_size * 2;
+    uint32_t *new_array = (uint32_t*)SCRealloc(pmq->pattern_id_array,
+                                               new_size * sizeof(uint32_t));
+    if (unlikely(new_array == NULL)) {
+        SCLogError(SC_ERR_MEM_ALLOC, "Failed to realloc PatternMatchQueue"
+                 " pattern ID array. Some new Pattern ID matches were lost.");
+      return;
+    }
+    pmq->pattern_id_array = new_array;
+    pmq->pattern_id_array_size = new_size;
+}
+
 /** \brief Verify and store a match
  *
  *   used at search runtime
@@ -522,10 +546,7 @@ MpmVerifyMatch(MpmThreadCtx *thread_ctx, PatternMatcherQueue *pmq, uint32_t pati
             /* flag this pattern id as being added now */
             pmq->pattern_id_bitarray[(patid / 8)] |= (1<<(patid % 8));
             /* append the pattern_id to the array with matches */
-            pmq->pattern_id_array[pmq->pattern_id_array_cnt] = patid;
-            pmq->pattern_id_array_cnt++;
-            SCLogDebug("pattern_id_array_cnt %u", pmq->pattern_id_array_cnt);
-
+            MpmAddPid(pmq, patid);
             MpmAddSids(pmq, sids, sids_size);
         }
     }
@@ -570,15 +591,9 @@ void PmqReset(PatternMatcherQueue *pmq)
         return;
 
     memset(pmq->pattern_id_bitarray, 0, pmq->pattern_id_bitarray_size);
-    //memset(pmq->pattern_id_array, 0, pmq->pattern_id_array_size);
-    pmq->pattern_id_array_cnt = 0;
-/*
-    uint32_t u;
-    for (u = 0; u < pmq->pattern_id_array_cnt; u++) {
-        pmq->pattern_id_bitarray[(pmq->pattern_id_array[u] / 8)] &= ~(1<<(pmq->pattern_id_array[u] % 8));
-    }
+
     pmq->pattern_id_array_cnt = 0;
-*/
+
     pmq->rule_id_array_cnt = 0;
     /* TODO: Realloc the rule id array smaller at some size? */
 }

diff --git a/src/util-mpm.h b/src/util-mpm.h
index 7c0ccc94a14654a481d17ea343b1fcaac50ce13d..d1ebeef8d9183f6bbda52cc1a221f5f6f2f21d30 100644 (file)
--- a/src/util-mpm.h
+++ b/src/util-mpm.h
@@ -266,7 +266,7 @@ int MpmAddPatternCI(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen,
                     uint16_t offset, uint16_t depth,
                     uint32_t pid, uint32_t sid, uint8_t flags);
 
-/* Resize ID array. Only called from MpmAddSids(). */
+/* Resize Signature ID array. Only called from MpmAddSids(). */
 void MpmAddSidsResize(PatternMatcherQueue *pmq, uint32_t new_size);
 
 /** \brief Add array of Signature IDs to rule ID array.
@@ -284,7 +284,7 @@ MpmAddSids(PatternMatcherQueue *pmq, uint32_t *sids, uint32_t sids_size)
 {
     uint32_t new_size = pmq->rule_id_array_cnt + sids_size;
     if (new_size > pmq->rule_id_array_size) {
-      MpmAddSidsResize(pmq, new_size);
+        MpmAddSidsResize(pmq, new_size);
     }
     SCLogDebug("Adding %u sids", sids_size);
     // Add SIDs for this pattern to the end of the array
@@ -293,4 +293,17 @@ MpmAddSids(PatternMatcherQueue *pmq, uint32_t *sids, uint32_t sids_size)
     pmq->rule_id_array_cnt += sids_size;
 }
 
+/* Resize Pattern ID array. Only called from MpmAddPid(). */
+void MpmAddPidResize(PatternMatcherQueue *pmq, uint32_t new_size);
+
+static inline void
+MpmAddPid(PatternMatcherQueue *pmq, uint32_t patid)
+{
+    uint32_t new_size = pmq->pattern_id_array_cnt + 1;
+    if (new_size > pmq->pattern_id_array_size)
+        MpmAddPidResize(pmq, new_size);
+    pmq->pattern_id_array[pmq->pattern_id_array_cnt] = patid;
+    pmq->pattern_id_array_cnt = new_size;
+    SCLogDebug("pattern_id_array_cnt %u", pmq->pattern_id_array_cnt);
+}
 #endif /* __UTIL_MPM_H__ */