From: Victor Julien Date: Tue, 29 Sep 2015 11:20:20 +0000 (+0200) Subject: detect: remove obsolete grouping code X-Git-Tag: suricata-3.1RC1~374 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4223ce9aba0b8303e7b11092ce7e490ac026610a;p=thirdparty%2Fsuricata.git detect: remove obsolete grouping code --- diff --git a/src/detect.c b/src/detect.c index 62aa6a8bc8..d118fa73c2 100644 --- a/src/detect.c +++ b/src/detect.c @@ -3130,204 +3130,11 @@ static int DetectEngineLookupFlowAddSig(DetectEngineCtx *de_ctx, Signature *s) return 0; } -static DetectAddress *GetHeadPtr(DetectAddressHead *head, int family) -{ - DetectAddress *grhead; - - if (head == NULL) - return NULL; - - if (family == AF_INET) { - grhead = head->ipv4_head; - } else if (family == AF_INET6) { - grhead = head->ipv6_head; - } else { - grhead = head->any_head; - } - - return grhead; -} - //#define SMALL_MPM(c) 0 #define SMALL_MPM(c) ((c) == 1) // || (c) == 2) // || (c) == 3) -int CreateGroupedAddrListCmpCnt(DetectAddress *a, DetectAddress *b) -{ - if (a->cnt > b->cnt) - return 1; - return 0; -} - -int CreateGroupedAddrListCmpMpmMinlen(DetectAddress *a, DetectAddress *b) -{ - if (a->sh == NULL || b->sh == NULL) - return 0; - - if (SMALL_MPM(a->sh->mpm_content_minlen)) - return 1; - - if (a->sh->mpm_content_minlen < b->sh->mpm_content_minlen) - return 1; - return 0; -} - -/* set unique_groups to 0 for no grouping. - * - * srchead is a ordered "inserted" list w/o internal overlap - * - */ -int CreateGroupedAddrList(DetectEngineCtx *de_ctx, DetectAddress *srchead, - int family, DetectAddressHead *newhead, - uint32_t unique_groups, - int (*CompareFunc)(DetectAddress *, DetectAddress *), - uint32_t max_idx) -{ - DetectAddress *tmplist = NULL, *tmplist2 = NULL, *joingr = NULL; - char insert = 0; - DetectAddress *gr, *next_gr; - uint32_t groups = 0; - - /* insert the addresses into the tmplist, where it will - * be sorted descending on 'cnt'. */ - for (gr = srchead; gr != NULL; gr = gr->next) { - BUG_ON(gr->ip.family == 0 && !(gr->flags & ADDRESS_FLAG_ANY)); - - if (SMALL_MPM(gr->sh->mpm_content_minlen) && unique_groups > 0) - unique_groups++; - - groups++; - - /* alloc a copy */ - DetectAddress *newtmp = DetectAddressCopy(gr); - if (newtmp == NULL) { - goto error; - } - SigGroupHeadCopySigs(de_ctx, gr->sh,&newtmp->sh); - - DetectPort *port = gr->port; - for ( ; port != NULL; port = port->next) { - DetectPortInsertCopy(de_ctx,&newtmp->port, port); - newtmp->flags |= ADDRESS_HAVEPORT; - } - - /* insert it */ - DetectAddress *tmpgr = tmplist, *prevtmpgr = NULL; - if (tmplist == NULL) { - /* empty list, set head */ - tmplist = newtmp; - } else { - /* look for the place to insert */ - for ( ; tmpgr != NULL&&!insert; tmpgr = tmpgr->next) { - if (CompareFunc(gr, tmpgr)) { - if (tmpgr == tmplist) { - newtmp->next = tmplist; - tmplist = newtmp; - } else { - newtmp->next = prevtmpgr->next; - prevtmpgr->next = newtmp; - } - insert = 1; - } - prevtmpgr = tmpgr; - } - if (insert == 0) { - newtmp->next = NULL; - prevtmpgr->next = newtmp; - } - insert = 0; - } - } - - uint32_t i = unique_groups; - if (i == 0) i = groups; - - for (gr = tmplist; gr != NULL; ) { - BUG_ON(gr->ip.family == 0 && !(gr->flags & ADDRESS_FLAG_ANY)); - - if (i == 0) { - if (joingr == NULL) { - joingr = DetectAddressCopy(gr); - if (joingr == NULL) { - goto error; - } - - SigGroupHeadCopySigs(de_ctx,gr->sh,&joingr->sh); - - DetectPort *port = gr->port; - for ( ; port != NULL; port = port->next) { - DetectPortInsertCopy(de_ctx,&joingr->port, port); - joingr->flags |= ADDRESS_HAVEPORT; - } - } else { - DetectAddressJoin(de_ctx, joingr, gr); - } - } else { - DetectAddress *newtmp = DetectAddressCopy(gr); - if (newtmp == NULL) { - goto error; - } - - SigGroupHeadCopySigs(de_ctx,gr->sh,&newtmp->sh); - - DetectPort *port = gr->port; - for ( ; port != NULL; port = port->next) { - DetectPortInsertCopy(de_ctx,&newtmp->port, port); - newtmp->flags |= ADDRESS_HAVEPORT; - } - - if (tmplist2 == NULL) { - tmplist2 = newtmp; - } else { - newtmp->next = tmplist2; - tmplist2 = newtmp; - } - } - if (i)i--; - - next_gr = gr->next; - DetectAddressFree(gr); - gr = next_gr; - } - - /* we now have a tmplist2 containing the 'unique' groups and - * possibly a joingr that covers the rest. Now build the newhead - * that we will pass back to the caller. - * - * Start with inserting the unique groups */ - for (gr = tmplist2; gr != NULL; ) { - BUG_ON(gr->ip.family == 0 && !(gr->flags & ADDRESS_FLAG_ANY)); - - DetectAddress *newtmp = DetectAddressCopy(gr); - if (newtmp == NULL) { - goto error; - } - SigGroupHeadCopySigs(de_ctx, gr->sh,&newtmp->sh); - - DetectPort *port = gr->port; - for ( ; port != NULL; port = port->next) { - DetectPortInsertCopy(de_ctx, &newtmp->port, port); - newtmp->flags |= ADDRESS_HAVEPORT; - } - - DetectAddressInsert(de_ctx, newhead, newtmp); - - next_gr = gr->next; - DetectAddressFree(gr); - gr = next_gr; - } - - /* if present, insert the joingr that covers the rest */ - if (joingr != NULL) { - DetectAddressInsert(de_ctx, newhead, joingr); - } - - return 0; -error: - return -1; -} - int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b) { if (a->cnt > b->cnt) @@ -3548,399 +3355,6 @@ int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) return 0; } -/** - * \brief Build the destination address portion of the match tree - */ -int BuildDestinationAddressHeads(DetectEngineCtx *de_ctx, - DetectAddressHead *head, - int family, int flow, int ipproto) -{ - Signature *tmp_s = NULL; - DetectAddress *gr = NULL, *sgr = NULL, *lookup_gr = NULL; - uint32_t max_idx = 0; - - DetectAddress *grhead = NULL, *grdsthead = NULL, *grsighead = NULL; - - /* based on the family, select the list we are using in the head */ - grhead = GetHeadPtr(head, family); - - /* loop through the global source address list */ - for (gr = grhead; gr != NULL; gr = gr->next) { - //printf(" * Source group (BuildDestinationAddressHeads): "); DetectAddressPrint(gr); printf(" (%p)\n", gr); - - /* initialize the destination group head */ - gr->dst_gh = DetectAddressHeadInit(); - if (gr->dst_gh == NULL) { - goto error; - } - - /* use a tmp list for speeding up insertions */ - DetectAddress *tmp_gr_list = NULL; - - /* loop through all signatures in this source address group - * and build the temporary destination address list for it */ - uint32_t sig; - for (sig = 0; sig < de_ctx->sig_array_len; sig++) { - if (!(gr->sh->init->sig_array[(sig/8)] & (1<<(sig%8)))) - continue; - - tmp_s = de_ctx->sig_array[sig]; - if (tmp_s == NULL) - continue; - - //printf(" * (tmp) Signature %u (num %u)\n", tmp_s->id, tmp_s->num); - - max_idx = sig; - - /* build the temp list */ - grsighead = GetHeadPtr(&tmp_s->dst, family); - for (sgr = grsighead; sgr != NULL; sgr = sgr->next) { - //printf(" * (tmp) dst group: "); DetectAddressPrint(sgr); printf(" (%p)\n", sgr); - - if ((lookup_gr = DetectAddressLookupInList(tmp_gr_list, sgr)) == NULL) { - DetectAddress *grtmp = DetectAddressCopy(sgr); - if (grtmp == NULL) { - goto error; - } - SigGroupHeadAppendSig(de_ctx,&grtmp->sh,tmp_s); - - DetectAddressAdd(&tmp_gr_list,grtmp); - } else { - /* our group will only have one sig, this one. So add that. */ - SigGroupHeadAppendSig(de_ctx, &lookup_gr->sh, tmp_s); - lookup_gr->cnt++; - } - } - - } - - /* Create the destination address list, keeping in - * mind the limits we use. */ - int groups = (flow ? de_ctx->max_uniq_toserver_dst_groups : de_ctx->max_uniq_toclient_dst_groups); - - CreateGroupedAddrList(de_ctx, tmp_gr_list, family, gr->dst_gh, groups, - CreateGroupedAddrListCmpMpmMinlen, max_idx); - - /* see if the sig group head of each address group is the - * same as an earlier one. If it is, free our head and use - * a pointer to the earlier one. This saves _a lot_ of memory. - */ - grdsthead = GetHeadPtr(gr->dst_gh, family); - for (sgr = grdsthead; sgr != NULL; sgr = sgr->next) { - //printf(" * Destination group: "); DetectAddressPrint(sgr); printf("\n"); - - /* Because a pattern matcher context uses quite some - * memory, we first check if we can reuse it from - * another group head. */ - SigGroupHead *sgh = SigGroupHeadHashLookup(de_ctx, sgr->sh); - if (sgh == NULL) { - /* put the contents in our sig group head */ - SigGroupHeadSetSigCnt(sgr->sh, max_idx); - SigGroupHeadSetProtoAndDirection(sgr->sh, ipproto, flow); - SigGroupHeadBuildMatchArray(de_ctx, sgr->sh, max_idx); - SigGroupHeadHashAdd(de_ctx, sgr->sh); - SigGroupHeadStore(de_ctx, sgr->sh); - de_ctx->gh_unique++; - } else { - SCLogDebug("calling SigGroupHeadFree sgr %p, sgr->sh %p", sgr, sgr->sh); - SigGroupHeadFree(sgr->sh); - sgr->sh = sgh; - - de_ctx->gh_reuse++; - sgr->flags |= ADDRESS_SIGGROUPHEAD_COPY; - sgr->sh->flags |= SIG_GROUP_HEAD_REFERENCED; - } - } - - /* free the temp list */ - DetectAddressCleanupList(tmp_gr_list); - /* clear now unneeded sig group head */ - SCLogDebug("calling SigGroupHeadFree gr %p, gr->sh %p", gr, gr->sh); - SigGroupHeadFree(gr->sh); - gr->sh = NULL; - } - - return 0; -error: - return -1; -} - -//static -int BuildDestinationAddressHeadsWithBothPorts(DetectEngineCtx *de_ctx, - DetectAddressHead *head, - int family, int flow, int ipproto) -{ - Signature *tmp_s = NULL; - DetectAddress *src_gr = NULL, *dst_gr = NULL, *sig_gr = NULL, *lookup_gr = NULL; - DetectAddress *src_gr_head = NULL, *dst_gr_head = NULL, *sig_gr_head = NULL; - uint32_t max_idx = 0; - - /* loop through the global source address list */ - src_gr_head = GetHeadPtr(head,family); - for (src_gr = src_gr_head; src_gr != NULL; src_gr = src_gr->next) { - //printf(" * Source group: "); DetectAddressPrint(src_gr); printf("\n"); - - /* initialize the destination group head */ - src_gr->dst_gh = DetectAddressHeadInit(); - if (src_gr->dst_gh == NULL) { - goto error; - } - - /* use a tmp list for speeding up insertions */ - DetectAddress *tmp_gr_list = NULL; - - /* loop through all signatures in this source address group - * and build the temporary destination address list for it */ - uint32_t sig; - for (sig = 0; sig < de_ctx->sig_array_len; sig++) { - if (!(src_gr->sh->init->sig_array[(sig/8)] & (1<<(sig%8)))) - continue; - - tmp_s = de_ctx->sig_array[sig]; - if (tmp_s == NULL) - continue; - - //printf(" * Source group: "); DetectAddressPrint(src_gr); printf("\n"); - - max_idx = sig; - - /* build the temp list */ - sig_gr_head = GetHeadPtr(&tmp_s->dst,family); - for (sig_gr = sig_gr_head; sig_gr != NULL; sig_gr = sig_gr->next) { - //printf(" * Sig dst addr: "); DetectAddressPrint(sig_gr); printf("\n"); - - if ((lookup_gr = DetectAddressLookupInList(tmp_gr_list, sig_gr)) == NULL) { - DetectAddress *grtmp = DetectAddressCopy(sig_gr); - if (grtmp == NULL) { - goto error; - } - SigGroupHeadAppendSig(de_ctx, &grtmp->sh, tmp_s); - - DetectAddressAdd(&tmp_gr_list,grtmp); - } else { - /* our group will only have one sig, this one. So add that. */ - SigGroupHeadAppendSig(de_ctx, &lookup_gr->sh, tmp_s); - lookup_gr->cnt++; - } - - SCLogDebug("calling SigGroupHeadFree sig_gr %p, sig_gr->sh %p", sig_gr, sig_gr->sh); - SigGroupHeadFree(sig_gr->sh); - sig_gr->sh = NULL; - } - } - - /* Create the destination address list, keeping in - * mind the limits we use. */ - int groups = (flow ? de_ctx->max_uniq_toserver_dst_groups : de_ctx->max_uniq_toclient_dst_groups); - - CreateGroupedAddrList(de_ctx, tmp_gr_list, family, src_gr->dst_gh, groups, - CreateGroupedAddrListCmpMpmMinlen, max_idx); - - /* add the ports to the dst address groups and the sigs - * to the ports */ - dst_gr_head = GetHeadPtr(src_gr->dst_gh,family); - for (dst_gr = dst_gr_head; dst_gr != NULL; dst_gr = dst_gr->next) { - //printf(" * Destination group: "); DetectAddressPrint(dst_gr); printf("\n"); - - dst_gr->flags |= ADDRESS_HAVEPORT; - - if (dst_gr->sh == NULL) - continue; - - /* we will reuse address sig group heads at this points, - * because if the sigs are the same, the ports will be - * the same. Saves memory and a lot of init time. */ - SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, dst_gr->sh); - if (lookup_sgh == NULL) { - DetectPortSpHashReset(de_ctx); - - uint32_t sig2; - for (sig2 = 0; sig2 < max_idx+1; sig2++) { - if (!(dst_gr->sh->init->sig_array[(sig2/8)] & (1<<(sig2%8)))) - continue; - - Signature *s = de_ctx->sig_array[sig2]; - if (s == NULL) - continue; - - //printf(" + Destination group (grouped): "); DetectAddressPrint(dst_gr); printf("\n"); - - DetectPort *sdp = s->sp; - for ( ; sdp != NULL; sdp = sdp->next) { - DetectPort *lookup_port = DetectPortSpHashLookup(de_ctx, sdp); - if (lookup_port == NULL) { - DetectPort *port = DetectPortCopySingle(de_ctx,sdp); - if (port == NULL) - goto error; - - SigGroupHeadAppendSig(de_ctx, &port->sh, s); - DetectPortSpHashAdd(de_ctx, port); - port->cnt = 1; - } else { - SigGroupHeadAppendSig(de_ctx, &lookup_port->sh, s); - lookup_port->cnt++; - } - } - } - - int spgroups = (flow ? de_ctx->max_uniq_toserver_sp_groups : de_ctx->max_uniq_toclient_sp_groups); - - CreateGroupedPortList(de_ctx, de_ctx->sport_hash_table, &dst_gr->port, spgroups, - CreateGroupedPortListCmpMpmMinlen, max_idx); - - SCLogDebug("adding sgh %p to the hash", dst_gr->sh); - SigGroupHeadHashAdd(de_ctx, dst_gr->sh); - - dst_gr->sh->init->port = dst_gr->port; - /* mark this head for deletion once we no longer need - * the hash. We're only using the port ptr, so no problem - * when we remove this after initialization is done */ - dst_gr->sh->flags |= SIG_GROUP_HEAD_FREE; - - /* for each destination port we setup the siggrouphead here */ - DetectPort *sp = dst_gr->port; - for ( ; sp != NULL; sp = sp->next) { - //printf(" * Src Port(range): "); DetectPortPrint(sp); printf("\n"); - - if (sp->sh == NULL) - continue; - - /* we will reuse address sig group heads at this points, - * because if the sigs are the same, the ports will be - * the same. Saves memory and a lot of init time. */ - SigGroupHead *lookup_sp_sgh = SigGroupHeadSPortHashLookup(de_ctx, sp->sh); - if (lookup_sp_sgh == NULL) { - DetectPortDpHashReset(de_ctx); - uint32_t sig2; - for (sig2 = 0; sig2 < max_idx+1; sig2++) { - if (!(sp->sh->init->sig_array[(sig2/8)] & (1<<(sig2%8)))) - continue; - - Signature *s = de_ctx->sig_array[sig2]; - if (s == NULL) - continue; - - DetectPort *sdp = s->dp; - for ( ; sdp != NULL; sdp = sdp->next) { - DetectPort *lookup_port = DetectPortDpHashLookup(de_ctx,sdp); - if (lookup_port == NULL) { - DetectPort *port = DetectPortCopySingle(de_ctx,sdp); - if (port == NULL) - goto error; - - SigGroupHeadAppendSig(de_ctx, &port->sh, s); - DetectPortDpHashAdd(de_ctx,port); - port->cnt = 1; - } else { - SigGroupHeadAppendSig(de_ctx, &lookup_port->sh, s); - lookup_port->cnt++; - } - } - } - - int dpgroups = (flow ? de_ctx->max_uniq_toserver_dp_groups : de_ctx->max_uniq_toclient_dp_groups); - - CreateGroupedPortList(de_ctx, de_ctx->dport_hash_table, - &sp->dst_ph, dpgroups, - CreateGroupedPortListCmpMpmMinlen, max_idx); - - SigGroupHeadSPortHashAdd(de_ctx, sp->sh); - - sp->sh->init->port = sp->dst_ph; - /* mark this head for deletion once we no longer need - * the hash. We're only using the port ptr, so no problem - * when we remove this after initialization is done */ - sp->sh->flags |= SIG_GROUP_HEAD_FREE; - - /* for each destination port we setup the siggrouphead here */ - DetectPort *dp = sp->dst_ph; - for ( ; dp != NULL; dp = dp->next) { - if (dp->sh == NULL) - continue; - - /* Because a pattern matcher context uses quite some - * memory, we first check if we can reuse it from - * another group head. */ - SigGroupHead *lookup_dp_sgh = SigGroupHeadDPortHashLookup(de_ctx, dp->sh); - if (lookup_dp_sgh == NULL) { - SCLogDebug("dp %p dp->sh %p is the original (sp %p, dst_gr %p, src_gr %p)", dp, dp->sh, sp, dst_gr, src_gr); - - SigGroupHeadSetSigCnt(dp->sh, max_idx); - SigGroupHeadSetProtoAndDirection(dp->sh, ipproto, flow); - SigGroupHeadBuildMatchArray(de_ctx,dp->sh, max_idx); - SigGroupHeadDPortHashAdd(de_ctx, dp->sh); - SigGroupHeadStore(de_ctx, dp->sh); - de_ctx->gh_unique++; - } else { - SCLogDebug("dp %p dp->sh %p is a copy", dp, dp->sh); - - SigGroupHeadFree(dp->sh); - dp->sh = lookup_dp_sgh; - dp->flags |= PORT_SIGGROUPHEAD_COPY; - dp->sh->flags |= SIG_GROUP_HEAD_REFERENCED; - - de_ctx->gh_reuse++; - } - } - /* sig group head found in hash, free it and use the hashed one */ - } else { - SigGroupHeadFree(sp->sh); - sp->sh = lookup_sp_sgh; - sp->flags |= PORT_SIGGROUPHEAD_COPY; - sp->sh->flags |= SIG_GROUP_HEAD_REFERENCED; - - SCLogDebug("replacing sp->dst_ph %p with lookup_sp_sgh->init->port %p", sp->dst_ph, lookup_sp_sgh->init->port); - DetectPortCleanupList(sp->dst_ph); - sp->dst_ph = lookup_sp_sgh->init->port; - sp->flags |= PORT_GROUP_PORTS_COPY; - - de_ctx->gh_reuse++; - } - } - } else { - SigGroupHeadFree(dst_gr->sh); - dst_gr->sh = lookup_sgh; - dst_gr->flags |= ADDRESS_SIGGROUPHEAD_COPY; - dst_gr->sh->flags |= SIG_GROUP_HEAD_REFERENCED; - - SCLogDebug("replacing dst_gr->port %p with lookup_sgh->init->port %p", dst_gr->port, lookup_sgh->init->port); - DetectPortCleanupList(dst_gr->port); - dst_gr->port = lookup_sgh->init->port; - dst_gr->flags |= ADDRESS_PORTS_COPY; - - de_ctx->gh_reuse++; - } - } - /* free the temp list */ - DetectAddressCleanupList(tmp_gr_list); - /* clear now unneeded sig group head */ - SigGroupHeadFree(src_gr->sh); - src_gr->sh = NULL; - - /* free dst addr sgh's */ - dst_gr_head = GetHeadPtr(src_gr->dst_gh,family); - for (dst_gr = dst_gr_head; dst_gr != NULL; dst_gr = dst_gr->next) { - if (!(dst_gr->flags & ADDRESS_SIGGROUPHEAD_COPY)) { - if (!(dst_gr->sh->flags & SIG_GROUP_HEAD_REFERENCED)) { - SCLogDebug("removing sgh %p from hash", dst_gr->sh); - - int r = SigGroupHeadHashRemove(de_ctx,dst_gr->sh); - BUG_ON(r == -1); - if (r == 0) { - SCLogDebug("removed sgh %p from hash", dst_gr->sh); - SigGroupHeadFree(dst_gr->sh); - dst_gr->sh = NULL; - } - } - } - } - } - - return 0; -error: - return -1; -} - static void DetectEngineBuildDecoderEventSgh(DetectEngineCtx *de_ctx) { if (de_ctx->decoder_event_sgh == NULL) @@ -4424,7 +3838,6 @@ int SigGroupBuild(DetectEngineCtx *de_ctx) } -// SigAddressPrepareStage5(de_ctx); // DetectAddressPrintMemory(); // DetectSigGroupPrintMemory(); // DetectPortPrintMemory();