Dynamically resize pmq->rule_id_array

Rather than statically allocate 64K entries in every rule_id_array,
increase the size only when needed. Created a new function MpmAddSids()
to check the size before adding the new sids. If the array is not large
enough, it calls MpmAddSidsResize() that calls realloc and does error
checking. If the realloc fails, it prints an error and drops the new sids
on the floor, which seems better than exiting Suricata.

The size is increased to (current_size + new_count) * 2. This handles the
case where new_count > current_size, which would not be handled by simply
using current_size * 2. It should also be faster than simply reallocing to
current_size + new_count, which would then require another realloc for each
new addition.

diff --git a/src/util-mpm-ac-bs.c b/src/util-mpm-ac-bs.c
index 9ea9eed895c312a778e86770b5ef444689852f6e..6d58d6f61e4b5dc0209c4a76ab3f520ffd3c6558 100644 (file)
--- a/src/util-mpm-ac-bs.c
+++ b/src/util-mpm-ac-bs.c
@@ -1490,23 +1490,21 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                 uint32_t k;
                 for (k = 0; k < no_of_entries; k++) {
                     if (pids[k] & 0xFFFF0000) {
-                        if (SCMemcmp(pid_pat_list[pids[k] & 0x0000FFFF].cs,
-                                     buf + i - pid_pat_list[pids[k] & 0x0000FFFF].patlen + 1,
-                                     pid_pat_list[pids[k] & 0x0000FFFF].patlen) != 0) {
+                        uint32_t lower_pid = pids[k] & 0x0000FFFF;
+                        if (SCMemcmp(pid_pat_list[lower_pid].cs,
+                                     buf + i - pid_pat_list[lower_pid].patlen + 1,
+                                     pid_pat_list[lower_pid].patlen) != 0) {
                             /* inside loop */
                             continue;
                         }
-                        if (bitarray[(pids[k] & 0x0000FFFF) / 8] & (1 << ((pids[k] & 0x0000FFFF) % 8))) {
+                        if (bitarray[(lower_pid) / 8] & (1 << ((lower_pid) % 8))) {
                             ;
                         } else {
-                            bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k] & 0x0000FFFF;
-
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid;
+
+                            MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
                     } else {
@@ -1517,10 +1515,7 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
 
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;
                     }
@@ -1583,23 +1578,21 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                 uint32_t k;
                 for (k = 0; k < no_of_entries; k++) {
                     if (pids[k] & 0xFFFF0000) {
-                        if (SCMemcmp(pid_pat_list[pids[k] & 0x0000FFFF].cs,
-                                     buf + i - pid_pat_list[pids[k] & 0x0000FFFF].patlen + 1,
-                                     pid_pat_list[pids[k] & 0x0000FFFF].patlen) != 0) {
+                        uint32_t lower_pid = pids[k] & 0x0000FFFF;
+                        if (SCMemcmp(pid_pat_list[lower_pid].cs,
+                                     buf + i - pid_pat_list[lower_pid].patlen + 1,
+                                     pid_pat_list[lower_pid].patlen) != 0) {
                             /* inside loop */
                             continue;
                         }
-                        if (bitarray[(pids[k] & 0x0000FFFF) / 8] & (1 << ((pids[k] & 0x0000FFFF) % 8))) {
+                        if (bitarray[(lower_pid) / 8] & (1 << ((lower_pid) % 8))) {
                             ;
                         } else {
-                            bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k] & 0x0000FFFF;
-
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid;
+
+                            MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
                     } else {
@@ -1610,10 +1603,7 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
 
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;
                     }

diff --git a/src/util-mpm-ac-gfbs.c b/src/util-mpm-ac-gfbs.c
index 8ccb8e3f43099ef8bdd9ee274beca6f8f40864ef..5670f66828cd3e362784147fc129bd9a8d3c043e 100644 (file)
--- a/src/util-mpm-ac-gfbs.c
+++ b/src/util-mpm-ac-gfbs.c
@@ -1402,24 +1402,22 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                 uint32_t k = 0;
                 for (k = 0; k < no_of_pid_entries; k++) {
                     if (pids[k] & 0xFFFF0000) {
-                        if (SCMemcmp(pid_pat_list[pids[k] & 0x0000FFFF].cs,
-                                     buf + i - pid_pat_list[pids[k] & 0x0000FFFF].patlen + 1,
-                                     pid_pat_list[pids[k] & 0x0000FFFF].patlen) != 0) {
+                        uint32_t lower_pid = pids[k] & 0x0000FFFF;
+                        if (SCMemcmp(pid_pat_list[lower_pid].cs,
+                                     buf + i - pid_pat_list[lower_pid].patlen + 1,
+                                     pid_pat_list[lower_pid].patlen) != 0) {
                             /* inside loop */
                             continue;
                         }
 
-                        if (bitarray[(pids[k] & 0x0000FFFF) / 8] & (1 << ((pids[k] & 0x0000FFFF) % 8))) {
+                        if (bitarray[(lower_pid) / 8] & (1 << ((lower_pid) % 8))) {
                             ;
                         } else {
-                            bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = (pids[k] & 0x0000FFFF);
+                            bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = (lower_pid);
 
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
                     } else {
@@ -1430,10 +1428,7 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
 
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;
                     }
@@ -1539,24 +1534,22 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                 uint32_t k = 0;
                 for (k = 0; k < no_of_pid_entries; k++) {
                     if (pids[k] & 0xFFFF0000) {
-                        if (SCMemcmp(pid_pat_list[pids[k] & 0x0000FFFF].cs,
-                                     buf + i - pid_pat_list[pids[k] & 0x0000FFFF].patlen + 1,
-                                     pid_pat_list[pids[k] & 0x0000FFFF].patlen) != 0) {
+                        uint32_t lower_pid = pids[k] & 0x0000FFFF;
+                        if (SCMemcmp(pid_pat_list[lower_pid].cs,
+                                     buf + i - pid_pat_list[lower_pid].patlen + 1,
+                                     pid_pat_list[lower_pid].patlen) != 0) {
                             /* inside loop */
                             continue;
                         }
 
-                        if (bitarray[(pids[k] & 0x0000FFFF) / 8] & (1 << ((pids[k] & 0x0000FFFF) % 8))) {
+                        if (bitarray[(lower_pid) / 8] & (1 << ((lower_pid) % 8))) {
                             ;
                         } else {
-                            bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = (pids[k] & 0x0000FFFF);
+                            bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = (lower_pid);
 
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
                     } else {
@@ -1567,10 +1560,7 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                             pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
 
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;
                     }

diff --git a/src/util-mpm-ac-tile.c b/src/util-mpm-ac-tile.c
index e34911de961ff808343355713f9c7f486fed50fb..21750f1a6ec02173c3e83ac77d394e620f194758 100644 (file)
--- a/src/util-mpm-ac-tile.c
+++ b/src/util-mpm-ac-tile.c
@@ -1496,12 +1496,8 @@ int CheckMatch(SCACTileSearchCtx *ctx, PatternMatcherQueue *pmq,
             pmq_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
             *new_pattern++ = lower_pid;
 
-            // Add SIDs for this pattern
-            // TODO - Keep local pointer to update.
-            uint32_t x;
-            for (x = 0; x < pid_pat_list[lower_pid].sids_size; x++) {
-                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[lower_pid].sids[x];
-            }
+            MpmAddSids(pmq, pid_pat_list[lower_pid].sids,
+                       pid_pat_list[lower_pid].sids_size);
         }
         matches++;
     }
@@ -1562,24 +1558,23 @@ uint32_t SCACTileSearchLarge(SCACTileSearchCtx *ctx, MpmThreadCtx *mpm_thread_ct
             uint32_t k;
             for (k = 0; k < no_of_entries; k++) {
                 if (pids[k] & 0xFFFF0000) {
-                    if (SCMemcmp(pid_pat_list[pids[k] & 0x0000FFFF].cs,
-                                 buf + i - pid_pat_list[pids[k] & 0x0000FFFF].patlen + 1,
-                                 pid_pat_list[pids[k] & 0x0000FFFF].patlen) != 0) {
+                    uint32_t lower_pid = pids[k] & 0x0000FFFF;
+                    if (SCMemcmp(pid_pat_list[lower_pid].cs,
+                                 buf + i - pid_pat_list[lower_pid].patlen + 1,
+                                 pid_pat_list[lower_pid].patlen) != 0) {
                         /* inside loop */
                         continue;
                     }
-                    if (bitarray[(pids[k] & 0x0000FFFF) / 8] & (1 << ((pids[k] & 0x0000FFFF) % 8))) {
+                    if (bitarray[(lower_pid) / 8] & (1 << ((lower_pid) % 8))) {
                         ;
                     } else {
-                        bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                        pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |=
-                          (1 << ((pids[k] & 0x0000FFFF) % 8));
-                        pmq->pattern_id_array[pmq->pattern_id_array_cnt++] =
-                          pids[k] & 0x0000FFFF;
-                        uint32_t x;
-                        for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                            pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                        }
+                        bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                        pmq->pattern_id_bitarray[(lower_pid) / 8] |=
+                          (1 << ((lower_pid) % 8));
+                        pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid;
+
+                        MpmAddSids(pmq, pid_pat_list[lower_pid].sids,
+                                   pid_pat_list[lower_pid].sids_size);
                     }
                     matches++;
                 } else {
@@ -1589,10 +1584,9 @@ uint32_t SCACTileSearchLarge(SCACTileSearchCtx *ctx, MpmThreadCtx *mpm_thread_ct
                         bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                         pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                         pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
-                        uint32_t x;
-                        for (x = 0; x < pid_pat_list[pids[k]].sids_size; x++) {
-                            pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k]].sids[x];
-                        }
+
+                        MpmAddSids(pmq, pid_pat_list[pids[k]].sids,
+                                   pid_pat_list[pids[k]].sids_size);
                     }
                     matches++;
                 }

diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c
index f6c462ea25765d62b595aaf56bfb3d3516fc4515..9aed573efd0ca1dc96940bfe49eb02a875c149b5 100644 (file)
--- a/src/util-mpm-ac.c
+++ b/src/util-mpm-ac.c
@@ -1321,23 +1321,21 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                 uint32_t k;
                 for (k = 0; k < no_of_entries; k++) {
                     if (pids[k] & 0xFFFF0000) {
-                        if (SCMemcmp(pid_pat_list[pids[k] & 0x0000FFFF].cs,
-                                     buf + i - pid_pat_list[pids[k] & 0x0000FFFF].patlen + 1,
-                                     pid_pat_list[pids[k] & 0x0000FFFF].patlen) != 0) {
+                        uint32_t lower_pid = pids[k] & 0x0000FFFF;
+                        if (SCMemcmp(pid_pat_list[lower_pid].cs,
+                                     buf + i - pid_pat_list[lower_pid].patlen + 1,
+                                     pid_pat_list[lower_pid].patlen) != 0) {
                             /* inside loop */
                             continue;
                         }
-                        if (bitarray[(pids[k] & 0x0000FFFF) / 8] & (1 << ((pids[k] & 0x0000FFFF) % 8))) {
+                        if (bitarray[(lower_pid) / 8] & (1 << ((lower_pid) % 8))) {
                             ;
                         } else {
-                            pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k] & 0x0000FFFF;
-
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid;
+
+                            MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
                     } else {
@@ -1347,10 +1345,7 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                             pmq->pattern_id_bitarray[(pids[k]) / 8] |= (1 << ((pids[k]) % 8));
                             bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k]].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k]].sids[x];
-                            }
+                            MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;
                     }
@@ -1371,23 +1366,21 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                 uint32_t k;
                 for (k = 0; k < no_of_entries; k++) {
                     if (pids[k] & 0xFFFF0000) {
-                        if (SCMemcmp(pid_pat_list[pids[k] & 0x0000FFFF].cs,
-                                     buf + i - pid_pat_list[pids[k] & 0x0000FFFF].patlen + 1,
-                                     pid_pat_list[pids[k] & 0x0000FFFF].patlen) != 0) {
+                        uint32_t lower_pid = pids[k] & 0x0000FFFF;
+                        if (SCMemcmp(pid_pat_list[lower_pid].cs,
+                                     buf + i - pid_pat_list[lower_pid].patlen + 1,
+                                     pid_pat_list[lower_pid].patlen) != 0) {
                             /* inside loop */
                             continue;
                         }
-                        if (bitarray[(pids[k] & 0x0000FFFF) / 8] & (1 << ((pids[k] & 0x0000FFFF) % 8))) {
+                        if (bitarray[(lower_pid) / 8] & (1 << ((lower_pid) % 8))) {
                             ;
                         } else {
-                            pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
-                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k] & 0x0000FFFF;
-
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k] & 0x0000FFFF].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k] & 0x0000FFFF].sids[x];
-                            }
+                            pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
+                            pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid;
+
+                            MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size);
                         }
                         matches++;
                     } else {
@@ -1398,10 +1391,7 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx,
                             bitarray[pids[k] / 8] |= (1 << (pids[k] % 8));
                             pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k];
 
-                            uint32_t x;
-                            for (x = 0; x < pid_pat_list[pids[k]].sids_size; x++) {
-                                pmq->rule_id_array[pmq->rule_id_array_cnt++] = pid_pat_list[pids[k]].sids[x];
-                            }
+                            MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size);
                         }
                         matches++;
                     }
@@ -2067,16 +2057,17 @@ uint32_t SCACCudaPacketResultsProcessing(Packet *p, MpmCtx *mpm_ctx,
          * the only change */
         for (k = 0; k < no_of_entries; k++) {
             if (pids[k] & 0xFFFF0000) {
-                if (SCMemcmp(pid_pat_list[pids[k] & 0x0000FFFF].cs,
-                             buf + offset - pid_pat_list[pids[k] & 0x0000FFFF].patlen + 1,
-                             pid_pat_list[pids[k] & 0x0000FFFF].patlen) != 0) {
+                uint32_t lower_pid = pids[k] & 0x0000FFFF;
+                if (SCMemcmp(pid_pat_list[lower_pid].cs,
+                             buf + offset - pid_pat_list[lower_pid].patlen + 1,
+                             pid_pat_list[lower_pid].patlen) != 0) {
                     /* inside loop */
                     continue;
                 }
-                if (pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] & (1 << ((pids[k] & 0x0000FFFF) % 8))) {
+                if (pmq->pattern_id_bitarray[(lower_pid) / 8] & (1 << ((lower_pid) % 8))) {
                     ;
                 } else {
-                    pmq->pattern_id_bitarray[(pids[k] & 0x0000FFFF) / 8] |= (1 << ((pids[k] & 0x0000FFFF) % 8));
+                    pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8));
                 }
                 matches++;
             } else {

diff --git a/src/util-mpm.c b/src/util-mpm.c
index cc466177f4a144222d34c9f62831481bf02f97ba..cb11215f7b142f05e380488fb6c9eb0c96358752 100644 (file)
--- a/src/util-mpm.c
+++ b/src/util-mpm.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2014 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -451,29 +451,58 @@ int PmqSetup(PatternMatcherQueue *pmq, uint32_t patmaxid)
         SCLogDebug("pmq->pattern_id_array %p, pmq->pattern_id_bitarray %p",
                 pmq->pattern_id_array, pmq->pattern_id_bitarray);
 
-        pmq->rule_id_array_size = 65536 * sizeof(uint32_t);
+        pmq->rule_id_array_size = 128;
+        pmq->rule_id_array_cnt = 0;
 
-        pmq->rule_id_array = SCMalloc(pmq->rule_id_array_size);
+        uint32_t bytes = pmq->rule_id_array_size * sizeof(uint32_t);
+        pmq->rule_id_array = SCMalloc(bytes);
         if (pmq->rule_id_array == NULL) {
+            pmq->rule_id_array_size = 0;
             SCReturnInt(-1);
         }
-        memset(pmq->rule_id_array, 0, pmq->rule_id_array_size);
-        pmq->rule_id_array_cnt = 0;
-
+        // Don't need to zero memory since it is always written first.
     }
 
     SCReturnInt(0);
 }
 
+/** \brief Add array of Signature IDs to rule ID array.
+ *
+ *   Checks size of the array first
+ *
+ *  \param pmq storage for match results
+ *  \param new_size number of Signature IDs needing to be stored.
+ *
+ */
+void
+MpmAddSidsResize(PatternMatcherQueue *pmq, uint32_t new_size)
+{
+    /* Need to make the array bigger. Double the size needed to
+     * also handle the case that sids_size might still be
+     * larger than the old size.
+     */
+    new_size = new_size * 2;
+    uint32_t *new_array = (uint32_t*)SCRealloc(pmq->rule_id_array,
+                                               new_size * sizeof(uint32_t));
+    if (unlikely(new_array == NULL)) {
+      SCLogError(SC_ERR_MEM_ALLOC, "Failed to realloc PatternMatchQueue"
+                 " rule ID array. Some signature ID matches lost");
+      return;
+    }
+    pmq->rule_id_array = new_array;
+    pmq->rule_id_array_size = new_size;
+}
+
 /** \brief Verify and store a match
  *
  *   used at search runtime
  *
  *  \param thread_ctx mpm thread ctx
  *  \param pmq storage for match results
- *  \param list end match to check against (entire list will be checked)
- *  \param offset match offset in the buffer
- *  \param patlen length of the pattern we're checking
+ *  \param patid pattern ID being checked
+ *  \param bitarray Array of bits for patterns IDs found in current search
+ *  \param sids pointer to array of Signature IDs
+ *  \param sids_size number of Signature IDs in sids array.
  *
  *  \retval 0 no match after all
  *  \retval 1 (new) match
@@ -497,13 +526,7 @@ MpmVerifyMatch(MpmThreadCtx *thread_ctx, PatternMatcherQueue *pmq, uint32_t pati
             pmq->pattern_id_array_cnt++;
             SCLogDebug("pattern_id_array_cnt %u", pmq->pattern_id_array_cnt);
 
-            SCLogDebug("Adding %u sids", sids_size);
-            // Add SIDs for this pattern
-            uint32_t x;
-            for (x = 0; x < sids_size; x++) {
-                pmq->rule_id_array[pmq->rule_id_array_cnt++] = sids[x];
-            }
-
+            MpmAddSids(pmq, sids, sids_size);
         }
     }
 
@@ -557,6 +580,7 @@ void PmqReset(PatternMatcherQueue *pmq)
     pmq->pattern_id_array_cnt = 0;
 */
     pmq->rule_id_array_cnt = 0;
+    /* TODO: Realloc the rule id array smaller at some size? */
 }
 
 /** \brief Cleanup a Pmq
@@ -583,6 +607,7 @@ void PmqCleanup(PatternMatcherQueue *pmq)
     }
 
     pmq->pattern_id_array_cnt = 0;
+    pmq->pattern_id_array_size = 0;
 }
 
 /** \brief Cleanup and free a Pmq

diff --git a/src/util-mpm.h b/src/util-mpm.h
index cf5a7335c9b4fcb0119c875747c476d560146baa..7c0ccc94a14654a481d17ea343b1fcaac50ce13d 100644 (file)
--- a/src/util-mpm.h
+++ b/src/util-mpm.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2014 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -108,9 +108,12 @@ typedef struct PatternMatcherQueue_ {
     uint32_t pattern_id_bitarray_size; /**< size in bytes */
 
     /* used for storing rule id's */
-    uint32_t rule_id_array_size;
+    /* Array of rule IDs found. */
     uint32_t *rule_id_array;
+    /* Number of rule IDs in the array. */
     uint32_t rule_id_array_cnt;
+    /* The number of slots allocated for storing rule IDs */
+    uint32_t rule_id_array_size;
 
 } PatternMatcherQueue;
 
@@ -263,4 +266,31 @@ int MpmAddPatternCI(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen,
                     uint16_t offset, uint16_t depth,
                     uint32_t pid, uint32_t sid, uint8_t flags);
 
+/* Resize ID array. Only called from MpmAddSids(). */
+void MpmAddSidsResize(PatternMatcherQueue *pmq, uint32_t new_size);
+
+/** \brief Add array of Signature IDs to rule ID array.
+ *
+ *   Checks size of the array first. Calls MpmAddSidsResize to increase
+ *   The size of the array, since that is the slow path.
+ *
+ *  \param pmq storage for match results
+ *  \param sids pointer to array of Signature IDs
+ *  \param sids_size number of Signature IDs in sids array.
+ *
+ */
+static inline void
+MpmAddSids(PatternMatcherQueue *pmq, uint32_t *sids, uint32_t sids_size)
+{
+    uint32_t new_size = pmq->rule_id_array_cnt + sids_size;
+    if (new_size > pmq->rule_id_array_size) {
+      MpmAddSidsResize(pmq, new_size);
+    }
+    SCLogDebug("Adding %u sids", sids_size);
+    // Add SIDs for this pattern to the end of the array
+    memcpy(pmq->rule_id_array + pmq->rule_id_array_cnt,
+           sids, sids_size * sizeof(uint32_t));
+    pmq->rule_id_array_cnt += sids_size;
+}
+
 #endif /* __UTIL_MPM_H__ */