From: Ken Steele Date: Wed, 8 Oct 2014 18:36:29 +0000 (-0400) Subject: Dynamically resize pattern id array as needed X-Git-Tag: suricata-2.1beta3~37 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=ab8b1158b07f4d58171d560bc3f34e4cc379e738;p=thirdparty%2Fsuricata.git Dynamically resize pattern id array as needed Rather than creating the array of size maxpatid, dynamically resize as needed. This also handles the case where duplicate pid are added to the array. Also fix error in bitarray allocation (local version) to always use bitarray_size. --- diff --git a/src/util-mpm-ac-bs.c b/src/util-mpm-ac-bs.c index 6d58d6f61e..c706c09821 100644 --- a/src/util-mpm-ac-bs.c +++ b/src/util-mpm-ac-bs.c @@ -1502,8 +1502,7 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid; - + MpmAddPid(pmq, lower_pid); MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size); } matches++; @@ -1513,8 +1512,8 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k]; + MpmAddPid(pmq, pids[k]); MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size); } matches++; @@ -1590,8 +1589,8 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid; + MpmAddPid(pmq, lower_pid); MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size); } matches++; @@ -1601,8 +1600,8 @@ uint32_t SCACBSSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k]; + MpmAddPid(pmq, pids[k]); MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size); } matches++; diff --git a/src/util-mpm-ac-gfbs.c b/src/util-mpm-ac-gfbs.c index 5670f66828..523aa0ff95 100644 --- a/src/util-mpm-ac-gfbs.c +++ b/src/util-mpm-ac-gfbs.c @@ -1415,8 +1415,8 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = (lower_pid); + MpmAddPid(pmq, lower_pid); MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size); } matches++; @@ -1426,8 +1426,8 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k]; + MpmAddPid(pmq, pids[k]); MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size); } matches++; @@ -1547,8 +1547,8 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = (lower_pid); + MpmAddPid(pmq, lower_pid); MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size); } matches++; @@ -1558,8 +1558,8 @@ uint32_t SCACGfbsSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); pmq->pattern_id_bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k]; + MpmAddPid(pmq, pids[k]); MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size); } matches++; diff --git a/src/util-mpm-ac.c b/src/util-mpm-ac.c index 9aed573efd..7df1caa2a9 100644 --- a/src/util-mpm-ac.c +++ b/src/util-mpm-ac.c @@ -1333,8 +1333,7 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { pmq->pattern_id_bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); bitarray[(lower_pid) / 8] |= (1 << ((lower_pid) % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = lower_pid; - + MpmAddPid(pmq, lower_pid); MpmAddSids(pmq, pid_pat_list[lower_pid].sids, pid_pat_list[lower_pid].sids_size); } matches++; @@ -1344,7 +1343,7 @@ uint32_t SCACSearch(MpmCtx *mpm_ctx, MpmThreadCtx *mpm_thread_ctx, } else { pmq->pattern_id_bitarray[(pids[k]) / 8] |= (1 << ((pids[k]) % 8)); bitarray[pids[k] / 8] |= (1 << (pids[k] % 8)); - pmq->pattern_id_array[pmq->pattern_id_array_cnt++] = pids[k]; + MpmAddPid(pmq, pids[k]); MpmAddSids(pmq, pid_pat_list[pids[k]].sids, pid_pat_list[pids[k]].sids_size); } matches++; diff --git a/src/util-mpm.c b/src/util-mpm.c index cb11215f7b..5a90e36139 100644 --- a/src/util-mpm.c +++ b/src/util-mpm.c @@ -430,14 +430,13 @@ int PmqSetup(PatternMatcherQueue *pmq, uint32_t patmaxid) memset(pmq, 0, sizeof(PatternMatcherQueue)); if (patmaxid > 0) { - pmq->pattern_id_array_size = patmaxid * sizeof(uint32_t); + pmq->pattern_id_array_size = 32; /* Intial size, TODO Make this configure option */ + pmq->pattern_id_array_cnt = 0; - pmq->pattern_id_array = SCMalloc(pmq->pattern_id_array_size); + pmq->pattern_id_array = SCCalloc(pmq->pattern_id_array_size, sizeof(uint32_t)); if (pmq->pattern_id_array == NULL) { SCReturnInt(-1); } - memset(pmq->pattern_id_array, 0, pmq->pattern_id_array_size); - pmq->pattern_id_array_cnt = 0; /* lookup bitarray */ pmq->pattern_id_bitarray_size = (patmaxid / 8) + 1; @@ -451,7 +450,7 @@ int PmqSetup(PatternMatcherQueue *pmq, uint32_t patmaxid) SCLogDebug("pmq->pattern_id_array %p, pmq->pattern_id_bitarray %p", pmq->pattern_id_array, pmq->pattern_id_bitarray); - pmq->rule_id_array_size = 128; + pmq->rule_id_array_size = 128; /* Initial size, TODO: Make configure option. */ pmq->rule_id_array_cnt = 0; uint32_t bytes = pmq->rule_id_array_size * sizeof(uint32_t); @@ -493,6 +492,31 @@ MpmAddSidsResize(PatternMatcherQueue *pmq, uint32_t new_size) pmq->rule_id_array_size = new_size; } +/** \brief Increase the size of the Pattern rule ID array. + * + * \param pmq storage for match results + * \param new_size number of Signature IDs needing to be stored. + * + */ +void +MpmAddPidResize(PatternMatcherQueue *pmq, uint32_t new_size) +{ + /* Need to make the array bigger. Double the size needed to + * also handle the case that sids_size might still be + * larger than the old size. + */ + new_size = new_size * 2; + uint32_t *new_array = (uint32_t*)SCRealloc(pmq->pattern_id_array, + new_size * sizeof(uint32_t)); + if (unlikely(new_array == NULL)) { + SCLogError(SC_ERR_MEM_ALLOC, "Failed to realloc PatternMatchQueue" + " pattern ID array. Some new Pattern ID matches were lost."); + return; + } + pmq->pattern_id_array = new_array; + pmq->pattern_id_array_size = new_size; +} + /** \brief Verify and store a match * * used at search runtime @@ -522,10 +546,7 @@ MpmVerifyMatch(MpmThreadCtx *thread_ctx, PatternMatcherQueue *pmq, uint32_t pati /* flag this pattern id as being added now */ pmq->pattern_id_bitarray[(patid / 8)] |= (1<<(patid % 8)); /* append the pattern_id to the array with matches */ - pmq->pattern_id_array[pmq->pattern_id_array_cnt] = patid; - pmq->pattern_id_array_cnt++; - SCLogDebug("pattern_id_array_cnt %u", pmq->pattern_id_array_cnt); - + MpmAddPid(pmq, patid); MpmAddSids(pmq, sids, sids_size); } } @@ -570,15 +591,9 @@ void PmqReset(PatternMatcherQueue *pmq) return; memset(pmq->pattern_id_bitarray, 0, pmq->pattern_id_bitarray_size); - //memset(pmq->pattern_id_array, 0, pmq->pattern_id_array_size); - pmq->pattern_id_array_cnt = 0; -/* - uint32_t u; - for (u = 0; u < pmq->pattern_id_array_cnt; u++) { - pmq->pattern_id_bitarray[(pmq->pattern_id_array[u] / 8)] &= ~(1<<(pmq->pattern_id_array[u] % 8)); - } + pmq->pattern_id_array_cnt = 0; -*/ + pmq->rule_id_array_cnt = 0; /* TODO: Realloc the rule id array smaller at some size? */ } diff --git a/src/util-mpm.h b/src/util-mpm.h index 7c0ccc94a1..d1ebeef8d9 100644 --- a/src/util-mpm.h +++ b/src/util-mpm.h @@ -266,7 +266,7 @@ int MpmAddPatternCI(struct MpmCtx_ *mpm_ctx, uint8_t *pat, uint16_t patlen, uint16_t offset, uint16_t depth, uint32_t pid, uint32_t sid, uint8_t flags); -/* Resize ID array. Only called from MpmAddSids(). */ +/* Resize Signature ID array. Only called from MpmAddSids(). */ void MpmAddSidsResize(PatternMatcherQueue *pmq, uint32_t new_size); /** \brief Add array of Signature IDs to rule ID array. @@ -284,7 +284,7 @@ MpmAddSids(PatternMatcherQueue *pmq, uint32_t *sids, uint32_t sids_size) { uint32_t new_size = pmq->rule_id_array_cnt + sids_size; if (new_size > pmq->rule_id_array_size) { - MpmAddSidsResize(pmq, new_size); + MpmAddSidsResize(pmq, new_size); } SCLogDebug("Adding %u sids", sids_size); // Add SIDs for this pattern to the end of the array @@ -293,4 +293,17 @@ MpmAddSids(PatternMatcherQueue *pmq, uint32_t *sids, uint32_t sids_size) pmq->rule_id_array_cnt += sids_size; } +/* Resize Pattern ID array. Only called from MpmAddPid(). */ +void MpmAddPidResize(PatternMatcherQueue *pmq, uint32_t new_size); + +static inline void +MpmAddPid(PatternMatcherQueue *pmq, uint32_t patid) +{ + uint32_t new_size = pmq->pattern_id_array_cnt + 1; + if (new_size > pmq->pattern_id_array_size) + MpmAddPidResize(pmq, new_size); + pmq->pattern_id_array[pmq->pattern_id_array_cnt] = patid; + pmq->pattern_id_array_cnt = new_size; + SCLogDebug("pattern_id_array_cnt %u", pmq->pattern_id_array_cnt); +} #endif /* __UTIL_MPM_H__ */