From: Victor Julien Date: Sun, 8 Oct 2017 10:17:33 +0000 (+0200) Subject: detect: move grouping/building code into own file X-Git-Tag: suricata-4.1.0-beta1~474 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=90569d5fd6a9e100abbca85632f1c4b5089dd670;p=thirdparty%2Fsuricata.git detect: move grouping/building code into own file Clean up main detect.c file by moving things related to rule grouping out. --- diff --git a/src/Makefile.am b/src/Makefile.am index 76726125ed..abd73aef48 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -106,12 +106,13 @@ detect-tls-cert-subject.c detect-tls-cert-subject.h \ detect-tls-cert-serial.c detect-tls-cert-serial.h \ detect-tls-cert-fingerprint.c detect-tls-cert-fingerprint.h \ detect-dsize.c detect-dsize.h \ +detect-engine.c detect-engine.h \ detect-engine-address.c detect-engine-address.h \ detect-engine-address-ipv4.c detect-engine-address-ipv4.h \ detect-engine-address-ipv6.c detect-engine-address-ipv6.h \ detect-engine-alert.c detect-engine-alert.h \ detect-engine-analyzer.c detect-engine-analyzer.h \ -detect-engine.c detect-engine.h \ +detect-engine-build.c detect-engine-build.h \ detect-engine-content-inspection.c detect-engine-content-inspection.h \ detect-engine-dcepayload.c detect-engine-dcepayload.h \ detect-engine-dns.c detect-engine-dns.h \ diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c new file mode 100644 index 0000000000..76ece89ccc --- /dev/null +++ b/src/detect-engine-build.c @@ -0,0 +1,2054 @@ +/* Copyright (C) 2007-2017 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata-common.h" +#include "detect.h" +#include "detect-engine.h" +#include "detect-parse.h" + +#include "detect-engine-address.h" +#include "detect-engine-iponly.h" +#include "detect-engine-mpm.h" +#include "detect-engine-siggroup.h" +#include "detect-engine-port.h" +#include "detect-engine-prefilter.h" +#include "detect-engine-proto.h" + +#include "detect-dsize.h" +#include "detect-flags.h" +#include "detect-flow.h" +#include "detect-flowbits.h" + +#include "util-profiling.h" + +void SigCleanSignatures(DetectEngineCtx *de_ctx) +{ + Signature *s = NULL, *ns; + + if (de_ctx == NULL) + return; + + for (s = de_ctx->sig_list; s != NULL;) { + ns = s->next; + SigFree(s); + s = ns; + } + + de_ctx->sig_list = NULL; + + DetectEngineResetMaxSigId(de_ctx); + de_ctx->sig_list = NULL; +} + +/** \brief Find a specific signature by sid and gid + * \param de_ctx detection engine ctx + * \param sid the signature id + * \param gid the signature group id + * + * \retval s sig found + * \retval NULL sig not found + */ +Signature *SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid) +{ + Signature *s = NULL; + + if (de_ctx == NULL) + return NULL; + + for (s = de_ctx->sig_list; s != NULL; s = s->next) { + if (s->id == sid && s->gid == gid) + return s; + } + + return NULL; +} + +/** + * \brief Check if a signature contains the filestore keyword. + * + * \param s signature + * + * \retval 0 no + * \retval 1 yes + */ +int SignatureIsFilestoring(const Signature *s) +{ + if (s == NULL) + return 0; + + if (s->flags & SIG_FLAG_FILESTORE) + return 1; + + return 0; +} + +/** + * \brief Check if a signature contains the filemagic keyword. + * + * \param s signature + * + * \retval 0 no + * \retval 1 yes + */ +int SignatureIsFilemagicInspecting(const Signature *s) +{ + if (s == NULL) + return 0; + + if (s->file_flags & FILE_SIG_NEED_MAGIC) + return 1; + + return 0; +} + +/** + * \brief Check if a signature contains the filemd5 keyword. + * + * \param s signature + * + * \retval 0 no + * \retval 1 yes + */ +int SignatureIsFileMd5Inspecting(const Signature *s) +{ + if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5)) + return 1; + + return 0; +} + +/** + * \brief Check if a signature contains the filesha1 keyword. + * + * \param s signature + * + * \retval 0 no + * \retval 1 yes + */ +int SignatureIsFileSha1Inspecting(const Signature *s) +{ + if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1)) + return 1; + + return 0; +} + +/** + * \brief Check if a signature contains the filesha256 keyword. + * + * \param s signature + * + * \retval 0 no + * \retval 1 yes + */ +int SignatureIsFileSha256Inspecting(const Signature *s) +{ + if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA256)) + return 1; + + return 0; +} + +/** + * \brief Check if a signature contains the filesize keyword. + * + * \param s signature + * + * \retval 0 no + * \retval 1 yes + */ +int SignatureIsFilesizeInspecting(const Signature *s) +{ + if (s == NULL) + return 0; + + if (s->file_flags & FILE_SIG_NEED_SIZE) + return 1; + + return 0; +} + +/** \brief Test is a initialized signature is IP only + * \param de_ctx detection engine ctx + * \param s the signature + * \retval 1 sig is ip only + * \retval 0 sig is not ip only + */ +int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s) +{ + if (s->alproto != ALPROTO_UNKNOWN) + return 0; + + if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) + return 0; + + /* for now assume that all registered buffer types are incompatible */ + const int nlists = DetectBufferTypeMaxId(); + for (int i = 0; i < nlists; i++) { + if (s->init_data->smlists[i] == NULL) + continue; + if (!(DetectBufferTypeGetNameById(i))) + continue; + + SCReturnInt(0); + } + + /* TMATCH list can be ignored, it contains TAGs and + * tags are compatible to IP-only. */ + + IPOnlyCIDRItem *cidr_item; + cidr_item = s->CidrSrc; + while (cidr_item != NULL) { + if (cidr_item->negated) + return 0; + + cidr_item = cidr_item->next; + } + cidr_item = s->CidrDst; + while (cidr_item != NULL) { + if (cidr_item->negated) + return 0; + + cidr_item = cidr_item->next; + } + + SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + if (sm == NULL) + goto iponly; + + for ( ; sm != NULL; sm = sm->next) { + if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) + return 0; + /* we have enabled flowbits to be compatible with ip only sigs, as long + * as the sig only has a "set" flowbits */ + if (sm->type == DETECT_FLOWBITS && + (((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) { + return 0; + } + } + +iponly: + if (!(de_ctx->flags & DE_QUIET)) { + SCLogDebug("IP-ONLY (%" PRIu32 "): source %s, dest %s", s->id, + s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET", + s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET"); + } + return 1; +} + +/** \internal + * \brief Test is a initialized signature is inspecting protocol detection only + * \param de_ctx detection engine ctx + * \param s the signature + * \retval 1 sig is dp only + * \retval 0 sig is not dp only + */ +static int SignatureIsPDOnly(const Signature *s) +{ + if (s->alproto != ALPROTO_UNKNOWN) + return 0; + + if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) + return 0; + + /* for now assume that all registered buffer types are incompatible */ + const int nlists = DetectBufferTypeMaxId(); + for (int i = 0; i < nlists; i++) { + if (s->init_data->smlists[i] == NULL) + continue; + if (!(DetectBufferTypeGetNameById(i))) + continue; + + SCReturnInt(0); + } + + /* TMATCH list can be ignored, it contains TAGs and + * tags are compatible to DP-only. */ + + /* match list matches may be compatible to DP only. We follow the same + * logic as IP-only so we can use that flag */ + + SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + if (sm == NULL) + return 0; + + int pd = 0; + for ( ; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_AL_APP_LAYER_PROTOCOL) { + pd = 1; + } else { + /* flowbits are supported for dp only sigs, as long + * as the sig only has a "set" flowbits */ + if (sm->type == DETECT_FLOWBITS) { + if ((((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) { + SCLogDebug("%u: not PD-only: flowbit settings other than 'set'", s->id); + return 0; + } + } else if (sm->type == DETECT_FLOW) { + if (((DetectFlowData *)sm->ctx)->flags & ~(DETECT_FLOW_FLAG_TOSERVER|DETECT_FLOW_FLAG_TOCLIENT)) { + SCLogDebug("%u: not PD-only: flow settings other than toserver/toclient", s->id); + return 0; + } + } else if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) { + SCLogDebug("%u: not PD-only: %s not PD/IP-only compat", s->id, sigmatch_table[sm->type].name); + return 0; + } + } + } + + if (pd) { + SCLogDebug("PD-ONLY (%" PRIu32 ")", s->id); + } + return pd; +} + +/** + * \internal + * \brief Check if the initialized signature is inspecting the packet payload + * \param de_ctx detection engine ctx + * \param s the signature + * \retval 1 sig is inspecting the payload + * \retval 0 sig is not inspecting the payload + */ +static int SignatureIsInspectingPayload(DetectEngineCtx *de_ctx, const Signature *s) +{ + + if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) { + return 1; + } + return 0; +} + +/** + * \internal + * \brief check if a signature is decoder event matching only + * \param de_ctx detection engine + * \param s the signature to test + * \retval 0 not a DEOnly sig + * \retval 1 DEOnly sig + */ +static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s) +{ + if (s->alproto != ALPROTO_UNKNOWN) { + SCReturnInt(0); + } + + if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) + { + SCReturnInt(0); + } + + /* for now assume that all registered buffer types are incompatible */ + const int nlists = DetectBufferTypeMaxId(); + for (int i = 0; i < nlists; i++) { + if (s->init_data->smlists[i] == NULL) + continue; + if (!(DetectBufferTypeGetNameById(i))) + continue; + + SCReturnInt(0); + } + + /* check for conflicting keywords */ + SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + for ( ;sm != NULL; sm = sm->next) { + if ( !(sigmatch_table[sm->type].flags & SIGMATCH_DEONLY_COMPAT)) + SCReturnInt(0); + } + + /* need at least one decode event keyword to be considered decode event. */ + sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; + for ( ;sm != NULL; sm = sm->next) { + if (sm->type == DETECT_DECODE_EVENT) + goto deonly; + if (sm->type == DETECT_ENGINE_EVENT) + goto deonly; + if (sm->type == DETECT_STREAM_EVENT) + goto deonly; + } + + SCReturnInt(0); + +deonly: + if (!(de_ctx->flags & DE_QUIET)) { + SCLogDebug("DE-ONLY (%" PRIu32 "): source %s, dest %s", s->id, + s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET", + s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET"); + } + + SCReturnInt(1); +} + +#define MASK_TCP_INITDEINIT_FLAGS (TH_SYN|TH_RST|TH_FIN) +#define MASK_TCP_UNUSUAL_FLAGS (TH_URG|TH_ECN|TH_CWR) + +/* Create mask for this packet + it's flow if it has one + * + * Sets SIG_MASK_REQUIRE_PAYLOAD, SIG_MASK_REQUIRE_FLOW, + * SIG_MASK_REQUIRE_HTTP_STATE, SIG_MASK_REQUIRE_DCE_STATE + */ +void +PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, + bool has_state, int app_decoder_events) +{ + if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) { + SCLogDebug("packet has payload"); + (*mask) |= SIG_MASK_REQUIRE_PAYLOAD; + } else if (p->flags & PKT_DETECT_HAS_STREAMDATA) { + SCLogDebug("stream data available"); + (*mask) |= SIG_MASK_REQUIRE_PAYLOAD; + } else { + SCLogDebug("packet has no payload"); + (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD; + } + + if (p->events.cnt > 0 || app_decoder_events != 0 || p->app_layer_events != NULL) { + SCLogDebug("packet/flow has events set"); + (*mask) |= SIG_MASK_REQUIRE_ENGINE_EVENT; + } + + if (PKT_IS_TCP(p)) { + if ((p->tcph->th_flags & MASK_TCP_INITDEINIT_FLAGS) != 0) { + (*mask) |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT; + } + if ((p->tcph->th_flags & MASK_TCP_UNUSUAL_FLAGS) != 0) { + (*mask) |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL; + } + } + + if (p->flags & PKT_HAS_FLOW) { + SCLogDebug("packet has flow"); + (*mask) |= SIG_MASK_REQUIRE_FLOW; + + if (has_state) { + switch(alproto) { + case ALPROTO_HTTP: + SCLogDebug("packet/flow has http state"); + (*mask) |= SIG_MASK_REQUIRE_HTTP_STATE; + break; + case ALPROTO_SMB: + case ALPROTO_SMB2: + case ALPROTO_DCERPC: + SCLogDebug("packet/flow has dce state"); + (*mask) |= SIG_MASK_REQUIRE_DCE_STATE; + break; + case ALPROTO_SSH: + SCLogDebug("packet/flow has ssh state"); + (*mask) |= SIG_MASK_REQUIRE_SSH_STATE; + break; + case ALPROTO_TLS: + SCLogDebug("packet/flow has tls state"); + (*mask) |= SIG_MASK_REQUIRE_TLS_STATE; + break; + case ALPROTO_DNS: + SCLogDebug("packet/flow has dns state"); + (*mask) |= SIG_MASK_REQUIRE_DNS_STATE; + break; + case ALPROTO_FTP: + SCLogDebug("packet/flow has ftp state"); + (*mask) |= SIG_MASK_REQUIRE_FTP_STATE; + break; + case ALPROTO_SMTP: + SCLogDebug("packet/flow has smtp state"); + (*mask) |= SIG_MASK_REQUIRE_SMTP_STATE; + break; + case ALPROTO_ENIP: + SCLogDebug("packet/flow has enip state"); + (*mask) |= SIG_MASK_REQUIRE_ENIP_STATE; + break; + case ALPROTO_DNP3: + SCLogDebug("packet/flow has dnp3 state"); + (*mask) |= SIG_MASK_REQUIRE_DNP3_STATE; + break; + case ALPROTO_TEMPLATE: + SCLogDebug("packet/flow has template state"); + (*mask) |= SIG_MASK_REQUIRE_TEMPLATE_STATE; + break; + default: + SCLogDebug("packet/flow has other state"); + break; + } + } else { + SCLogDebug("no alstate"); + } + } +} + +static int SignatureCreateMask(Signature *s) +{ + SCEnter(); + + if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) { + s->mask |= SIG_MASK_REQUIRE_PAYLOAD; + SCLogDebug("sig requires payload"); + } + + SigMatch *sm; + for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) { + switch(sm->type) { + case DETECT_FLOWBITS: + { + /* figure out what flowbit action */ + DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; + if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { + /* not a mask flag, but still set it here */ + s->flags |= SIG_FLAG_REQUIRE_FLOWVAR; + + SCLogDebug("SIG_FLAG_REQUIRE_FLOWVAR set as sig has " + "flowbit isset option."); + } + + /* flow is required for any flowbit manipulation */ + s->mask |= SIG_MASK_REQUIRE_FLOW; + SCLogDebug("sig requires flow to be able to manipulate " + "flowbit(s)"); + break; + } + case DETECT_FLOWINT: + /* flow is required for any flowint manipulation */ + s->mask |= SIG_MASK_REQUIRE_FLOW; + SCLogDebug("sig requires flow to be able to manipulate " + "flowint(s)"); + break; + case DETECT_FLAGS: + { + DetectFlagsData *fl = (DetectFlagsData *)sm->ctx; + + if (fl->flags & TH_SYN) { + s->mask |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT; + SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT"); + } + if (fl->flags & TH_RST) { + s->mask |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT; + SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT"); + } + if (fl->flags & TH_FIN) { + s->mask |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT; + SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT"); + } + if (fl->flags & TH_URG) { + s->mask |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL; + SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL"); + } + if (fl->flags & TH_ECN) { + s->mask |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL; + SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL"); + } + if (fl->flags & TH_CWR) { + s->mask |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL; + SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL"); + } + break; + } + case DETECT_DSIZE: + { + DetectDsizeData *ds = (DetectDsizeData *)sm->ctx; + switch (ds->mode) { + case DETECTDSIZE_LT: + /* LT will include 0, so no payload. + * if GT is used in the same rule the + * flag will be set anyway. */ + break; + case DETECTDSIZE_RA: + case DETECTDSIZE_GT: + s->mask |= SIG_MASK_REQUIRE_PAYLOAD; + SCLogDebug("sig requires payload"); + break; + case DETECTDSIZE_EQ: + if (ds->dsize > 0) { + s->mask |= SIG_MASK_REQUIRE_PAYLOAD; + SCLogDebug("sig requires payload"); + } else if (ds->dsize == 0) { + s->mask |= SIG_MASK_REQUIRE_NO_PAYLOAD; + SCLogDebug("sig requires no payload"); + } + break; + } + break; + } + case DETECT_AL_APP_LAYER_EVENT: + s->mask |= SIG_MASK_REQUIRE_ENGINE_EVENT; + break; + case DETECT_ENGINE_EVENT: + s->mask |= SIG_MASK_REQUIRE_ENGINE_EVENT; + break; + } + } + + if (s->alproto == ALPROTO_SSH) { + s->mask |= SIG_MASK_REQUIRE_SSH_STATE; + SCLogDebug("sig requires ssh state"); + } + if (s->alproto == ALPROTO_TLS) { + s->mask |= SIG_MASK_REQUIRE_TLS_STATE; + SCLogDebug("sig requires tls state"); + } + if (s->alproto == ALPROTO_DNS) { + s->mask |= SIG_MASK_REQUIRE_DNS_STATE; + SCLogDebug("sig requires dns state"); + } + if (s->alproto == ALPROTO_DNP3) { + s->mask |= SIG_MASK_REQUIRE_DNP3_STATE; + SCLogDebug("sig requires dnp3 state"); + } + if (s->alproto == ALPROTO_FTP) { + s->mask |= SIG_MASK_REQUIRE_FTP_STATE; + SCLogDebug("sig requires ftp state"); + } + if (s->alproto == ALPROTO_SMTP) { + s->mask |= SIG_MASK_REQUIRE_SMTP_STATE; + SCLogDebug("sig requires smtp state"); + } + if (s->alproto == ALPROTO_ENIP) { + s->mask |= SIG_MASK_REQUIRE_ENIP_STATE; + SCLogDebug("sig requires enip state"); + } + if (s->alproto == ALPROTO_TEMPLATE) { + s->mask |= SIG_MASK_REQUIRE_TEMPLATE_STATE; + SCLogDebug("sig requires template state"); + } + + if ((s->mask & SIG_MASK_REQUIRE_DCE_STATE) || + (s->mask & SIG_MASK_REQUIRE_HTTP_STATE) || + (s->mask & SIG_MASK_REQUIRE_SSH_STATE) || + (s->mask & SIG_MASK_REQUIRE_DNS_STATE) || + (s->mask & SIG_MASK_REQUIRE_DNP3_STATE) || + (s->mask & SIG_MASK_REQUIRE_FTP_STATE) || + (s->mask & SIG_MASK_REQUIRE_SMTP_STATE) || + (s->mask & SIG_MASK_REQUIRE_ENIP_STATE) || + (s->mask & SIG_MASK_REQUIRE_TEMPLATE_STATE) || + (s->mask & SIG_MASK_REQUIRE_TLS_STATE)) + { + s->mask |= SIG_MASK_REQUIRE_FLOW; + SCLogDebug("sig requires flow"); + } + + if (s->init_data->init_flags & SIG_FLAG_INIT_FLOW) { + s->mask |= SIG_MASK_REQUIRE_FLOW; + SCLogDebug("sig requires flow"); + } + + if (s->flags & SIG_FLAG_APPLAYER) { + s->mask |= SIG_MASK_REQUIRE_FLOW; + SCLogDebug("sig requires flow"); + } + + SCLogDebug("mask %02X", s->mask); + SCReturnInt(0); +} + +static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx) +{ + DetectMpmInitializeBuiltinMpms(de_ctx); + DetectMpmInitializeAppMpms(de_ctx); + + return; +} + +/** \brief Pure-PCRE or bytetest rule */ +static int RuleInspectsPayloadHasNoMpm(const Signature *s) +{ + if (s->init_data->mpm_sm == NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) + return 1; + return 0; +} + +static int RuleGetMpmPatternSize(const Signature *s) +{ + if (s->init_data->mpm_sm == NULL) + return -1; + int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm); + if (mpm_list < 0) + return -1; + const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx; + if (cd == NULL) + return -1; + return (int)cd->content_len; +} + +static int RuleMpmIsNegated(const Signature *s) +{ + if (s->init_data->mpm_sm == NULL) + return 0; + int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm); + if (mpm_list < 0) + return 0; + const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx; + if (cd == NULL) + return 0; + return (cd->flags & DETECT_CONTENT_NEGATED); +} + +#ifdef HAVE_LIBJANSSON +static json_t *RulesGroupPrintSghStats(const SigGroupHead *sgh, + const int add_rules, const int add_mpm_stats) +{ + uint32_t mpm_cnt = 0; + uint32_t nonmpm_cnt = 0; + uint32_t negmpm_cnt = 0; + uint32_t any5_cnt = 0; + uint32_t payload_no_mpm_cnt = 0; + uint32_t syn_cnt = 0; + + uint32_t mpms_total = 0; + uint32_t mpms_min = 0; + uint32_t mpms_max = 0; + + struct { + uint32_t total; + uint32_t cnt; + uint32_t min; + uint32_t max; + } mpm_stats[DETECT_SM_LIST_MAX]; + memset(mpm_stats, 0x00, sizeof(mpm_stats)); + + uint32_t alstats[ALPROTO_MAX] = {0}; + uint32_t mpm_sizes[DETECT_SM_LIST_MAX][256]; + memset(mpm_sizes, 0, sizeof(mpm_sizes)); + uint32_t alproto_mpm_bufs[ALPROTO_MAX][DETECT_SM_LIST_MAX]; + memset(alproto_mpm_bufs, 0, sizeof(alproto_mpm_bufs)); + + json_t *js = json_object(); + if (unlikely(js == NULL)) + return NULL; + + json_object_set_new(js, "id", json_integer(sgh->id)); + + json_t *js_array = json_array(); + + const Signature *s; + uint32_t x; + for (x = 0; x < sgh->sig_cnt; x++) { + s = sgh->match_array[x]; + if (s == NULL) + continue; + + int any = 0; + if (s->proto.flags & DETECT_PROTO_ANY) { + any++; + } + if (s->flags & SIG_FLAG_DST_ANY) { + any++; + } + if (s->flags & SIG_FLAG_SRC_ANY) { + any++; + } + if (s->flags & SIG_FLAG_DP_ANY) { + any++; + } + if (s->flags & SIG_FLAG_SP_ANY) { + any++; + } + if (any == 5) { + any5_cnt++; + } + + if (s->init_data->mpm_sm == NULL) { + nonmpm_cnt++; + + if (s->sm_arrays[DETECT_SM_LIST_MATCH] != NULL) { + SCLogDebug("SGH %p Non-MPM inspecting only packets. Rule %u", sgh, s->id); + } + + DetectPort *sp = s->sp; + DetectPort *dp = s->dp; + + if (s->flags & SIG_FLAG_TOSERVER && (dp->port == 0 && dp->port2 == 65535)) { + SCLogDebug("SGH %p Non-MPM toserver and to 'any'. Rule %u", sgh, s->id); + } + if (s->flags & SIG_FLAG_TOCLIENT && (sp->port == 0 && sp->port2 == 65535)) { + SCLogDebug("SGH %p Non-MPM toclient and to 'any'. Rule %u", sgh, s->id); + } + + if (DetectFlagsSignatureNeedsSynPackets(s)) { + syn_cnt++; + } + + } else { + int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm); + BUG_ON(mpm_list < 0); + const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx; + uint32_t size = cd->content_len < 256 ? cd->content_len : 255; + + mpm_sizes[mpm_list][size]++; + if (s->alproto != ALPROTO_UNKNOWN) { + alproto_mpm_bufs[s->alproto][mpm_list]++; + } + + if (mpm_list == DETECT_SM_LIST_PMATCH) { + if (size == 1) { + DetectPort *sp = s->sp; + DetectPort *dp = s->dp; + if (s->flags & SIG_FLAG_TOSERVER) { + if (dp->port == 0 && dp->port2 == 65535) { + SCLogDebug("SGH %p toserver 1byte fast_pattern to ANY. Rule %u", sgh, s->id); + } else { + SCLogDebug("SGH %p toserver 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, dp->port, dp->port2, s->id); + } + } + if (s->flags & SIG_FLAG_TOCLIENT) { + if (sp->port == 0 && sp->port2 == 65535) { + SCLogDebug("SGH %p toclient 1byte fast_pattern to ANY. Rule %u", sgh, s->id); + } else { + SCLogDebug("SGH %p toclient 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, sp->port, sp->port2, s->id); + } + } + } + } + + uint32_t w = PatternStrength(cd->content, cd->content_len); + mpms_total += w; + if (mpms_min == 0) + mpms_min = w; + if (w < mpms_min) + mpms_min = w; + if (w > mpms_max) + mpms_max = w; + + mpm_stats[mpm_list].total += w; + mpm_stats[mpm_list].cnt++; + if (mpm_stats[mpm_list].min == 0 || w < mpm_stats[mpm_list].min) + mpm_stats[mpm_list].min = w; + if (w > mpm_stats[mpm_list].max) + mpm_stats[mpm_list].max = w; + + mpm_cnt++; + + if (w < 10) { + SCLogDebug("SGH %p Weak MPM Pattern on %s. Rule %u", sgh, DetectListToString(mpm_list), s->id); + } + if (w < 10 && any == 5) { + SCLogDebug("SGH %p Weak MPM Pattern on %s, rule is 5xAny. Rule %u", sgh, DetectListToString(mpm_list), s->id); + } + + if (cd->flags & DETECT_CONTENT_NEGATED) { + SCLogDebug("SGH %p MPM Pattern on %s, is negated. Rule %u", sgh, DetectListToString(mpm_list), s->id); + negmpm_cnt++; + } + } + + if (RuleInspectsPayloadHasNoMpm(s)) { + SCLogDebug("SGH %p No MPM. Payload inspecting. Rule %u", sgh, s->id); + payload_no_mpm_cnt++; + } + + if (s->alproto != ALPROTO_UNKNOWN) { + alstats[s->alproto]++; + } + + if (add_rules) { + json_t *js_sig = json_object(); + if (unlikely(js == NULL)) + continue; + json_object_set_new(js_sig, "sig_id", json_integer(s->id)); + json_array_append_new(js_array, js_sig); + } + } + + json_object_set_new(js, "rules", js_array); + + json_t *stats = json_object(); + json_object_set_new(stats, "total", json_integer(sgh->sig_cnt)); + + json_t *types = json_object(); + json_object_set_new(types, "mpm", json_integer(mpm_cnt)); + json_object_set_new(types, "non_mpm", json_integer(nonmpm_cnt)); + json_object_set_new(types, "negated_mpm", json_integer(negmpm_cnt)); + json_object_set_new(types, "payload_but_no_mpm", json_integer(payload_no_mpm_cnt)); + json_object_set_new(types, "syn", json_integer(syn_cnt)); + json_object_set_new(types, "any5", json_integer(any5_cnt)); + json_object_set_new(stats, "types", types); + + int i; + for (i = 0; i < ALPROTO_MAX; i++) { + if (alstats[i] > 0) { + json_t *app = json_object(); + json_object_set_new(app, "total", json_integer(alstats[i])); + + for (x = 0; x < DETECT_SM_LIST_MAX; x++) { + if (alproto_mpm_bufs[i][x] == 0) + continue; + json_object_set_new(app, DetectListToHumanString(x), json_integer(alproto_mpm_bufs[i][x])); + } + + json_object_set_new(stats, AppProtoToString(i), app); + } + } + + if (add_mpm_stats) { + json_t *mpm_js = json_object(); + + for (i = 0; i < DETECT_SM_LIST_MAX; i++) { + if (mpm_stats[i].cnt > 0) { + + json_t *mpm_sizes_array = json_array(); + for (x = 0; x < 256; x++) { + if (mpm_sizes[i][x] == 0) + continue; + + json_t *e = json_object(); + json_object_set_new(e, "size", json_integer(x)); + json_object_set_new(e, "count", json_integer(mpm_sizes[i][x])); + json_array_append_new(mpm_sizes_array, e); + } + + json_t *buf = json_object(); + json_object_set_new(buf, "total", json_integer(mpm_stats[i].cnt)); + json_object_set_new(buf, "avg_strength", json_integer(mpm_stats[i].total / mpm_stats[i].cnt)); + json_object_set_new(buf, "min_strength", json_integer(mpm_stats[i].min)); + json_object_set_new(buf, "max_strength", json_integer(mpm_stats[i].max)); + + json_object_set_new(buf, "sizes", mpm_sizes_array); + + json_object_set_new(mpm_js, DetectListToHumanString(i), buf); + } + } + + json_object_set_new(stats, "mpm", mpm_js); + } + json_object_set_new(js, "stats", stats); + + json_object_set_new(js, "whitelist", json_integer(sgh->init->whitelist)); + + return js; +} +#endif /* HAVE_LIBJANSSON */ + +static void RulesDumpGrouping(const DetectEngineCtx *de_ctx, + const int add_rules, const int add_mpm_stats) +{ +#ifdef HAVE_LIBJANSSON + json_t *js = json_object(); + if (unlikely(js == NULL)) + return; + + int p; + for (p = 0; p < 256; p++) { + if (p == IPPROTO_TCP || p == IPPROTO_UDP) { + const char *name = (p == IPPROTO_TCP) ? "tcp" : "udp"; + + json_t *tcp = json_object(); + + json_t *ts_array = json_array(); + DetectPort *list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp : + de_ctx->flow_gh[1].udp; + while (list != NULL) { + json_t *port = json_object(); + json_object_set_new(port, "port", json_integer(list->port)); + json_object_set_new(port, "port2", json_integer(list->port2)); + + json_t *tcp_ts = RulesGroupPrintSghStats(list->sh, + add_rules, add_mpm_stats); + json_object_set_new(port, "rulegroup", tcp_ts); + json_array_append_new(ts_array, port); + + list = list->next; + } + json_object_set_new(tcp, "toserver", ts_array); + + json_t *tc_array = json_array(); + list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[0].tcp : + de_ctx->flow_gh[0].udp; + while (list != NULL) { + json_t *port = json_object(); + json_object_set_new(port, "port", json_integer(list->port)); + json_object_set_new(port, "port2", json_integer(list->port2)); + + json_t *tcp_tc = RulesGroupPrintSghStats(list->sh, + add_rules, add_mpm_stats); + json_object_set_new(port, "rulegroup", tcp_tc); + json_array_append_new(tc_array, port); + + list = list->next; + } + json_object_set_new(tcp, "toclient", tc_array); + + json_object_set_new(js, name, tcp); + } + + } + + const char *filename = "rule_group.json"; + const char *log_dir = ConfigGetLogDirectory(); + char log_path[PATH_MAX] = ""; + + snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename); + + FILE *fp = fopen(log_path, "w"); + if (fp == NULL) { + return; + } + + char *js_s = json_dumps(js, + JSON_PRESERVE_ORDER|JSON_ESCAPE_SLASH); + if (unlikely(js_s == NULL)) { + fclose(fp); + return; + } + + json_object_clear(js); + json_decref(js); + + fprintf(fp, "%s\n", js_s); + free(js_s); + fclose(fp); +#endif + return; +} + +static int RulesGroupByProto(DetectEngineCtx *de_ctx) +{ + Signature *s = de_ctx->sig_list; + + uint32_t max_idx = 0; + SigGroupHead *sgh_ts[256] = {NULL}; + SigGroupHead *sgh_tc[256] = {NULL}; + + for ( ; s != NULL; s = s->next) { + if (s->flags & SIG_FLAG_IPONLY) + continue; + + int p; + for (p = 0; p < 256; p++) { + if (p == IPPROTO_TCP || p == IPPROTO_UDP) { + continue; + } + if (!(s->proto.proto[p / 8] & (1<<(p % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) { + continue; + } + + if (s->flags & SIG_FLAG_TOCLIENT) { + SigGroupHeadAppendSig(de_ctx, &sgh_tc[p], s); + max_idx = s->num; + } + if (s->flags & SIG_FLAG_TOSERVER) { + SigGroupHeadAppendSig(de_ctx, &sgh_ts[p], s); + max_idx = s->num; + } + } + } + SCLogDebug("max_idx %u", max_idx); + + /* lets look at deduplicating this list */ + SigGroupHeadHashFree(de_ctx); + SigGroupHeadHashInit(de_ctx); + + uint32_t cnt = 0; + uint32_t own = 0; + uint32_t ref = 0; + int p; + for (p = 0; p < 256; p++) { + if (p == IPPROTO_TCP || p == IPPROTO_UDP) + continue; + if (sgh_ts[p] == NULL) + continue; + + cnt++; + + SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_ts[p]); + if (lookup_sgh == NULL) { + SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]); + + SigGroupHeadSetSigCnt(sgh_ts[p], max_idx); + SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], max_idx); + + SigGroupHeadHashAdd(de_ctx, sgh_ts[p]); + SigGroupHeadStore(de_ctx, sgh_ts[p]); + + de_ctx->gh_unique++; + own++; + } else { + SCLogDebug("proto group %d sgh %p is a copy", p, sgh_ts[p]); + + SigGroupHeadFree(sgh_ts[p]); + sgh_ts[p] = lookup_sgh; + + de_ctx->gh_reuse++; + ref++; + } + } + SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies", + "toserver", cnt, own, ref); + + cnt = 0; + own = 0; + ref = 0; + for (p = 0; p < 256; p++) { + if (p == IPPROTO_TCP || p == IPPROTO_UDP) + continue; + if (sgh_tc[p] == NULL) + continue; + + cnt++; + + SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_tc[p]); + if (lookup_sgh == NULL) { + SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]); + + SigGroupHeadSetSigCnt(sgh_tc[p], max_idx); + SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], max_idx); + + SigGroupHeadHashAdd(de_ctx, sgh_tc[p]); + SigGroupHeadStore(de_ctx, sgh_tc[p]); + + de_ctx->gh_unique++; + own++; + + } else { + SCLogDebug("proto group %d sgh %p is a copy", p, sgh_tc[p]); + + SigGroupHeadFree(sgh_tc[p]); + sgh_tc[p] = lookup_sgh; + + de_ctx->gh_reuse++; + ref++; + } + } + SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies", + "toclient", cnt, own, ref); + + for (p = 0; p < 256; p++) { + if (p == IPPROTO_TCP || p == IPPROTO_UDP) + continue; + + de_ctx->flow_gh[0].sgh[p] = sgh_tc[p]; + de_ctx->flow_gh[1].sgh[p] = sgh_ts[p]; + } + + return 0; +} + +static int PortIsWhitelisted(const DetectEngineCtx *de_ctx, + const DetectPort *a, int ipproto) +{ + DetectPort *w = de_ctx->tcp_whitelist; + if (ipproto == IPPROTO_UDP) + w = de_ctx->udp_whitelist; + + while (w) { + if (a->port >= w->port && a->port2 <= w->port) { + SCLogDebug("port group %u:%u whitelisted -> %d", a->port, a->port2, w->port); + return 1; + } + w = w->next; + } + + return 0; +} + +static int RuleSetWhitelist(Signature *s) +{ + DetectPort *p = NULL; + if (s->flags & SIG_FLAG_TOSERVER) + p = s->dp; + else if (s->flags & SIG_FLAG_TOCLIENT) + p = s->sp; + else + return 0; + + /* for sigs that don't use 'any' as port, see if we want to + * whitelist poor sigs */ + int wl = 0; + if (!(p->port == 0 && p->port2 == 65535)) { + /* pure pcre, bytetest, etc rules */ + if (RuleInspectsPayloadHasNoMpm(s)) { + SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Whitelisting SGH's.", s->id); + wl = 99; + + } else if (RuleMpmIsNegated(s)) { + SCLogDebug("Rule %u MPM is negated. Whitelisting SGH's.", s->id); + wl = 77; + + /* one byte pattern in packet/stream payloads */ + } else if (s->init_data->mpm_sm != NULL && + SigMatchListSMBelongsTo(s, s->init_data->mpm_sm) == DETECT_SM_LIST_PMATCH && + RuleGetMpmPatternSize(s) == 1) + { + SCLogDebug("Rule %u No MPM. Payload inspecting. Whitelisting SGH's.", s->id); + wl = 55; + + } else if (DetectFlagsSignatureNeedsSynPackets(s) && + DetectFlagsSignatureNeedsSynOnlyPackets(s)) + { + SCLogDebug("Rule %u Needs SYN, so inspected often. Whitelisting SGH's.", s->id); + wl = 33; + } + } + + s->init_data->whitelist = wl; + return wl; +} + +int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx); +int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b); + +static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint32_t direction) { + /* step 1: create a hash of 'DetectPort' objects based on all the + * rules. Each object will have a SGH with the sigs added + * that belong to the SGH. */ + DetectPortHashInit(de_ctx); + + uint32_t max_idx = 0; + const Signature *s = de_ctx->sig_list; + DetectPort *list = NULL; + while (s) { + /* IP Only rules are handled separately */ + if (s->flags & SIG_FLAG_IPONLY) + goto next; + if (!(s->proto.proto[ipproto / 8] & (1<<(ipproto % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) + goto next; + if (direction == SIG_FLAG_TOSERVER) { + if (!(s->flags & SIG_FLAG_TOSERVER)) + goto next; + } else if (direction == SIG_FLAG_TOCLIENT) { + if (!(s->flags & SIG_FLAG_TOCLIENT)) + goto next; + } + + DetectPort *p = NULL; + if (direction == SIG_FLAG_TOSERVER) + p = s->dp; + else if (direction == SIG_FLAG_TOCLIENT) + p = s->sp; + else + BUG_ON(1); + + /* see if we want to exclude directionless sigs that really care only for + * to_server syn scans/floods */ + if ((direction == SIG_FLAG_TOCLIENT) && + DetectFlagsSignatureNeedsSynPackets(s) && + DetectFlagsSignatureNeedsSynOnlyPackets(s) && + ((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) && + (!(s->dp->port == 0 && s->dp->port2 == 65535))) + { + SCLogWarning(SC_WARN_POOR_RULE, "rule %u: SYN-only to port(s) %u:%u " + "w/o direction specified, disabling for toclient direction", + s->id, s->dp->port, s->dp->port2); + goto next; + } + + int wl = s->init_data->whitelist; + while (p) { + int pwl = PortIsWhitelisted(de_ctx, p, ipproto) ? 111 : 0; + pwl = MAX(wl,pwl); + + DetectPort *lookup = DetectPortHashLookup(de_ctx, p); + if (lookup) { + SigGroupHeadAppendSig(de_ctx, &lookup->sh, s); + lookup->sh->init->whitelist = MAX(lookup->sh->init->whitelist, pwl); + } else { + DetectPort *tmp2 = DetectPortCopySingle(de_ctx, p); + BUG_ON(tmp2 == NULL); + SigGroupHeadAppendSig(de_ctx, &tmp2->sh, s); + tmp2->sh->init->whitelist = pwl; + DetectPortHashAdd(de_ctx, tmp2); + } + + p = p->next; + } + max_idx = s->num; + next: + s = s->next; + } + + /* step 2: create a list of DetectPort objects */ + HashListTableBucket *htb = NULL; + for (htb = HashListTableGetListHead(de_ctx->dport_hash_table); + htb != NULL; + htb = HashListTableGetListNext(htb)) + { + DetectPort *p = HashListTableGetListData(htb); + DetectPort *tmp = DetectPortCopySingle(de_ctx, p); + BUG_ON(tmp == NULL); + int r = DetectPortInsert(de_ctx, &list , tmp); + BUG_ON(r == -1); + } + DetectPortHashFree(de_ctx); + de_ctx->dport_hash_table = NULL; + + SCLogDebug("rules analyzed"); + + /* step 3: group the list and shrink it if necessary */ + DetectPort *newlist = NULL; + uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups : + de_ctx->max_uniq_toserver_groups; + CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt, max_idx); + list = newlist; + + /* step 4: deduplicate the SGH's */ + SigGroupHeadHashFree(de_ctx); + SigGroupHeadHashInit(de_ctx); + + uint32_t cnt = 0; + uint32_t own = 0; + uint32_t ref = 0; + DetectPort *iter; + for (iter = list ; iter != NULL; iter = iter->next) { + BUG_ON (iter->sh == NULL); + cnt++; + + SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, iter->sh); + if (lookup_sgh == NULL) { + SCLogDebug("port group %p sgh %p is the original", iter, iter->sh); + + SigGroupHeadSetSigCnt(iter->sh, max_idx); + SigGroupHeadBuildMatchArray(de_ctx, iter->sh, max_idx); + SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction); + SigGroupHeadHashAdd(de_ctx, iter->sh); + SigGroupHeadStore(de_ctx, iter->sh); + iter->flags |= PORT_SIGGROUPHEAD_COPY; + de_ctx->gh_unique++; + own++; + } else { + SCLogDebug("port group %p sgh %p is a copy", iter, iter->sh); + + SigGroupHeadFree(iter->sh); + iter->sh = lookup_sgh; + iter->flags |= PORT_SIGGROUPHEAD_COPY; + + de_ctx->gh_reuse++; + ref++; + } + } +#if 0 + for (iter = list ; iter != NULL; iter = iter->next) { + SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s/%d)", + iter->port, iter->port2, iter->sh, + iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own", + iter->sh->init->whitelist ? "true" : "false", + iter->sh->init->whitelist); + } +#endif + SCLogPerf("%s %s: %u port groups, %u unique SGH's, %u copies", + ipproto == 6 ? "TCP" : "UDP", + direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient", + cnt, own, ref); + return list; +} + +/** + * \brief Preprocess signature, classify ip-only, etc, build sig array + * + * \param de_ctx Pointer to the Detection Engine Context + * + * \retval 0 on success + * \retval -1 on failure + */ +int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) +{ + Signature *tmp_s = NULL; + uint32_t cnt_iponly = 0; + uint32_t cnt_payload = 0; + uint32_t cnt_applayer = 0; + uint32_t cnt_deonly = 0; + const int nlists = DetectBufferTypeMaxId(); + + if (!(de_ctx->flags & DE_QUIET)) { + SCLogDebug("building signature grouping structure, stage 1: " + "preprocessing rules..."); + } + + de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx); + de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *)); + de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size); + if (de_ctx->sig_array == NULL) + goto error; + memset(de_ctx->sig_array,0,de_ctx->sig_array_size); + + SCLogDebug("signature lookup array: %" PRIu32 " sigs, %" PRIu32 " bytes", + de_ctx->sig_array_len, de_ctx->sig_array_size); + + /* now for every rule add the source group */ + for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) { + de_ctx->sig_array[tmp_s->num] = tmp_s; + + SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", tmp_s->id, tmp_s->num, tmp_s, de_ctx->sig_array[tmp_s->num]); + + /* see if the sig is dp only */ + if (SignatureIsPDOnly(tmp_s) == 1) { + tmp_s->flags |= SIG_FLAG_PDONLY; + SCLogDebug("Signature %"PRIu32" is considered \"PD only\"", tmp_s->id); + + /* see if the sig is ip only */ + } else if (SignatureIsIPOnly(de_ctx, tmp_s) == 1) { + tmp_s->flags |= SIG_FLAG_IPONLY; + cnt_iponly++; + + SCLogDebug("Signature %"PRIu32" is considered \"IP only\"", tmp_s->id); + + /* see if any sig is inspecting the packet payload */ + } else if (SignatureIsInspectingPayload(de_ctx, tmp_s) == 1) { + cnt_payload++; + + SCLogDebug("Signature %"PRIu32" is considered \"Payload inspecting\"", tmp_s->id); + } else if (SignatureIsDEOnly(de_ctx, tmp_s) == 1) { + tmp_s->init_data->init_flags |= SIG_FLAG_INIT_DEONLY; + SCLogDebug("Signature %"PRIu32" is considered \"Decoder Event only\"", tmp_s->id); + cnt_deonly++; + } + + if (tmp_s->flags & SIG_FLAG_APPLAYER) { + SCLogDebug("Signature %"PRIu32" is considered \"Applayer inspecting\"", tmp_s->id); + cnt_applayer++; + } + +#ifdef DEBUG + if (SCLogDebugEnabled()) { + uint16_t colen = 0; + char copresent = 0; + SigMatch *sm; + DetectContentData *co; + for (sm = tmp_s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; sm = sm->next) { + if (sm->type != DETECT_CONTENT) + continue; + + copresent = 1; + co = (DetectContentData *)sm->ctx; + if (co->content_len > colen) + colen = co->content_len; + } + + if (copresent && colen == 1) { + SCLogDebug("signature %8u content maxlen 1", tmp_s->id); + int proto; + for (proto = 0; proto < 256; proto++) { + if (tmp_s->proto.proto[(proto/8)] & (1<<(proto%8))) + SCLogDebug("=> proto %" PRId32 "", proto); + } + } + } +#endif /* DEBUG */ + + if (RuleMpmIsNegated(tmp_s)) { + tmp_s->flags |= SIG_FLAG_MPM_NEG; + } + + SignatureCreateMask(tmp_s); + DetectContentPropagateLimits(tmp_s); + SigParseApplyDsizeToContent(tmp_s); + + RuleSetWhitelist(tmp_s); + + /* if keyword engines are enabled in the config, handle them here */ + if (de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO && + !(tmp_s->flags & SIG_FLAG_PREFILTER)) + { + int i; + int prefilter_list = DETECT_TBLSIZE; + + /* get the keyword supporting prefilter with the lowest type */ + for (i = 0; i < nlists; i++) { + SigMatch *sm = tmp_s->init_data->smlists[i]; + while (sm != NULL) { + if (sigmatch_table[sm->type].SupportsPrefilter != NULL) { + if (sigmatch_table[sm->type].SupportsPrefilter(tmp_s) == TRUE) { + prefilter_list = MIN(prefilter_list, sm->type); + } + } + sm = sm->next; + } + } + + /* apply that keyword as prefilter */ + if (prefilter_list != DETECT_TBLSIZE) { + for (i = 0; i < nlists; i++) { + SigMatch *sm = tmp_s->init_data->smlists[i]; + while (sm != NULL) { + if (sm->type == prefilter_list) { + tmp_s->init_data->prefilter_sm = sm; + tmp_s->flags |= SIG_FLAG_PREFILTER; + SCLogConfig("sid %u: prefilter is on \"%s\"", tmp_s->id, sigmatch_table[sm->type].name); + break; + } + sm = sm->next; + } + } + } + } + + /* run buffer type callbacks if any */ + int x; + for (x = 0; x < nlists; x++) { + if (tmp_s->init_data->smlists[x]) + DetectBufferRunSetupCallback(x, tmp_s); + } + + de_ctx->sig_cnt++; + } + + if (!(de_ctx->flags & DE_QUIET)) { + SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only " + "rules, %" PRIu32 " are inspecting packet payload, %"PRIu32 + " inspect application layer, %"PRIu32" are decoder event only", + de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer, + cnt_deonly); + + SCLogConfig("building signature grouping structure, stage 1: " + "preprocessing rules... complete"); + } + return 0; + +error: + return -1; +} + +static int PortGroupWhitelist(const DetectPort *a) +{ + return a->sh->init->whitelist; +} + +int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b) +{ + if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) { + SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + return 1; + } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) { + SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + return 0; + } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) { + SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + return 1; + } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) { + if (a->sh->sig_cnt > b->sh->sig_cnt) { + SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + return 1; + } + } + + SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", + a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), + b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); + return 0; +} + +/** \internal + * \brief Create a list of DetectPort objects sorted based on CompareFunc's + * logic. + * + * List can limit the number of groups. In this case an extra "join" group + * is created that contains the sigs belonging to that. It's *appended* to + * the list, meaning that if the list is walked linearly it's found last. + * The joingr is meant to be a catch all. + * + */ +int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx) +{ + DetectPort *tmplist = NULL, *joingr = NULL; + char insert = 0; + uint32_t groups = 0; + DetectPort *list; + + /* insert the addresses into the tmplist, where it will + * be sorted descending on 'cnt' and on wehther a group + * is whitelisted. */ + + DetectPort *oldhead = port_list; + while (oldhead) { + /* take the top of the list */ + list = oldhead; + oldhead = oldhead->next; + list->next = NULL; + + groups++; + + SigGroupHeadSetSigCnt(list->sh, max_idx); + + /* insert it */ + DetectPort *tmpgr = tmplist, *prevtmpgr = NULL; + if (tmplist == NULL) { + /* empty list, set head */ + tmplist = list; + } else { + /* look for the place to insert */ + for ( ; tmpgr != NULL && !insert; tmpgr = tmpgr->next) { + if (CompareFunc(list, tmpgr) == 1) { + if (tmpgr == tmplist) { + list->next = tmplist; + tmplist = list; + SCLogDebug("new list top: %u:%u", tmplist->port, tmplist->port2); + } else { + list->next = prevtmpgr->next; + prevtmpgr->next = list; + } + insert = 1; + break; + } + prevtmpgr = tmpgr; + } + if (insert == 0) { + list->next = NULL; + prevtmpgr->next = list; + } + insert = 0; + } + } + + uint32_t left = unique_groups; + if (left == 0) + left = groups; + + /* create another list: take the port groups from above + * and add them to the 2nd list until we have met our + * count. The rest is added to the 'join' group. */ + DetectPort *tmplist2 = NULL, *tmplist2_tail = NULL; + DetectPort *gr, *next_gr; + for (gr = tmplist; gr != NULL; ) { + next_gr = gr->next; + + SCLogDebug("temp list gr %p %u:%u", gr, gr->port, gr->port2); + DetectPortPrint(gr); + + /* if we've set up all the unique groups, add the rest to the + * catch-all joingr */ + if (left == 0) { + if (joingr == NULL) { + DetectPortParse(de_ctx, &joingr, "0:65535"); + if (joingr == NULL) { + goto error; + } + SCLogDebug("joingr => %u-%u", joingr->port, joingr->port2); + joingr->next = NULL; + } + SigGroupHeadCopySigs(de_ctx,gr->sh,&joingr->sh); + + /* when a group's sigs are added to the joingr, we can free it */ + gr->next = NULL; + DetectPortFree(gr); + gr = NULL; + + /* append */ + } else { + gr->next = NULL; + + if (tmplist2 == NULL) { + tmplist2 = gr; + tmplist2_tail = gr; + } else { + tmplist2_tail->next = gr; + tmplist2_tail = gr; + } + } + + if (left > 0) + left--; + + gr = next_gr; + } + + /* if present, append the joingr that covers the rest */ + if (joingr != NULL) { + SCLogDebug("appending joingr %p %u:%u", joingr, joingr->port, joingr->port2); + + if (tmplist2 == NULL) { + tmplist2 = joingr; + //tmplist2_tail = joingr; + } else { + tmplist2_tail->next = joingr; + //tmplist2_tail = joingr; + } + } else { + SCLogDebug("no joingr"); + } + + /* pass back our new list to the caller */ + *newhead = tmplist2; + DetectPortPrintList(*newhead); + + return 0; +error: + return -1; +} + +/** + * \internal + * \brief add a decoder event signature to the detection engine ctx + */ +static void DetectEngineAddDecoderEventSig(DetectEngineCtx *de_ctx, Signature *s) +{ + SCLogDebug("adding signature %"PRIu32" to the decoder event sgh", s->id); + SigGroupHeadAppendSig(de_ctx, &de_ctx->decoder_event_sgh, s); +} + +/** + * \brief Fill the global src group head, with the sigs included + * + * \param de_ctx Pointer to the Detection Engine Context whose Signatures have + * to be processed + * + * \retval 0 On success + * \retval -1 On failure + */ +int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) +{ + Signature *tmp_s = NULL; + uint32_t sigs = 0; + + if (!(de_ctx->flags & DE_QUIET)) { + SCLogDebug("building signature grouping structure, stage 2: " + "building source address lists..."); + } + + IPOnlyInit(de_ctx, &de_ctx->io_ctx); + + de_ctx->flow_gh[1].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOSERVER); + de_ctx->flow_gh[0].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOCLIENT); + de_ctx->flow_gh[1].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOSERVER); + de_ctx->flow_gh[0].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOCLIENT); + + /* Setup the other IP Protocols (so not TCP/UDP) */ + RulesGroupByProto(de_ctx); + + /* now for every rule add the source group to our temp lists */ + for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) { + SCLogDebug("tmp_s->id %"PRIu32, tmp_s->id); + if (tmp_s->flags & SIG_FLAG_IPONLY) { + IPOnlyAddSignature(de_ctx, &de_ctx->io_ctx, tmp_s); + } + + if (tmp_s->init_data->init_flags & SIG_FLAG_INIT_DEONLY) { + DetectEngineAddDecoderEventSig(de_ctx, tmp_s); + } + + sigs++; + } + + IPOnlyPrepare(de_ctx); + IPOnlyPrint(de_ctx, &de_ctx->io_ctx); + + return 0; +} + +static void DetectEngineBuildDecoderEventSgh(DetectEngineCtx *de_ctx) +{ + if (de_ctx->decoder_event_sgh == NULL) + return; + + uint32_t max_idx = DetectEngineGetMaxSigId(de_ctx); + SigGroupHeadSetSigCnt(de_ctx->decoder_event_sgh, max_idx); + SigGroupHeadBuildMatchArray(de_ctx, de_ctx->decoder_event_sgh, max_idx); +} + +int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) +{ + /* prepare the decoder event sgh */ + DetectEngineBuildDecoderEventSgh(de_ctx); + return 0; +} + +int SigAddressCleanupStage1(DetectEngineCtx *de_ctx) +{ + BUG_ON(de_ctx == NULL); + + if (!(de_ctx->flags & DE_QUIET)) { + SCLogDebug("cleaning up signature grouping structure..."); + } + if (de_ctx->decoder_event_sgh) + SigGroupHeadFree(de_ctx->decoder_event_sgh); + de_ctx->decoder_event_sgh = NULL; + + int f; + for (f = 0; f < FLOW_STATES; f++) { + int p; + for (p = 0; p < 256; p++) { + de_ctx->flow_gh[f].sgh[p] = NULL; + } + + /* free lookup lists */ + DetectPortCleanupList(de_ctx->flow_gh[f].tcp); + de_ctx->flow_gh[f].tcp = NULL; + DetectPortCleanupList(de_ctx->flow_gh[f].udp); + de_ctx->flow_gh[f].udp = NULL; + } + + uint32_t idx; + for (idx = 0; idx < de_ctx->sgh_array_cnt; idx++) { + SigGroupHead *sgh = de_ctx->sgh_array[idx]; + if (sgh == NULL) + continue; + + SCLogDebug("sgh %p", sgh); + SigGroupHeadFree(sgh); + } + SCFree(de_ctx->sgh_array); + de_ctx->sgh_array = NULL; + de_ctx->sgh_array_cnt = 0; + de_ctx->sgh_array_size = 0; + + IPOnlyDeinit(de_ctx, &de_ctx->io_ctx); + + if (!(de_ctx->flags & DE_QUIET)) { + SCLogInfo("cleaning up signature grouping structure... complete"); + } + return 0; +} + +#if 0 +static void DbgPrintSigs(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + if (sgh == NULL) { + printf("\n"); + return; + } + + uint32_t sig; + for (sig = 0; sig < sgh->sig_cnt; sig++) { + printf("%" PRIu32 " ", sgh->match_array[sig]->id); + } + printf("\n"); +} + +static void DbgPrintSigs2(DetectEngineCtx *de_ctx, SigGroupHead *sgh) +{ + if (sgh == NULL || sgh->init == NULL) { + printf("\n"); + return; + } + + uint32_t sig; + for (sig = 0; sig < DetectEngineGetMaxSigId(de_ctx); sig++) { + if (sgh->init->sig_array[(sig/8)] & (1<<(sig%8))) { + printf("%" PRIu32 " ", de_ctx->sig_array[sig]->id); + } + } + printf("\n"); +} +#endif + +/** \brief finalize preparing sgh's */ +int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) +{ + SCEnter(); + + //SCLogInfo("sgh's %"PRIu32, de_ctx->sgh_array_cnt); + + uint32_t cnt = 0; + uint32_t idx = 0; + for (idx = 0; idx < de_ctx->sgh_array_cnt; idx++) { + SigGroupHead *sgh = de_ctx->sgh_array[idx]; + if (sgh == NULL) + continue; + + SCLogDebug("sgh %p", sgh); + + SigGroupHeadSetFilemagicFlag(de_ctx, sgh); + SigGroupHeadSetFileHashFlag(de_ctx, sgh); + SigGroupHeadSetFilesizeFlag(de_ctx, sgh); + SigGroupHeadSetFilestoreCount(de_ctx, sgh); + SCLogDebug("filestore count %u", sgh->filestore_cnt); + + PrefilterSetupRuleGroup(de_ctx, sgh); + + SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh); + + sgh->id = idx; + cnt++; + } + SCLogPerf("Unique rule groups: %u", cnt); + + MpmStoreReportStats(de_ctx); + + if (de_ctx->decoder_event_sgh != NULL) { + /* no need to set filestore count here as that would make a + * signature not decode event only. */ + } + + /* cleanup the hashes now since we won't need them + * after the initialization phase. */ + SigGroupHeadHashFree(de_ctx); + + int dump_grouping = 0; + (void)ConfGetBool("detect.profiling.grouping.dump-to-disk", &dump_grouping); + + if (dump_grouping) { + int add_rules = 0; + (void)ConfGetBool("detect.profiling.grouping.include-rules", &add_rules); + int add_mpm_stats = 0; + (void)ConfGetBool("detect.profiling.grouping.include-mpm-stats", &add_rules); + + RulesDumpGrouping(de_ctx, add_rules, add_mpm_stats); + } + +#ifdef PROFILING + SCProfilingSghInitCounters(de_ctx); +#endif + SCReturnInt(0); +} + +/** \internal + * \brief perform final per signature setup tasks + * + * - Create SigMatchData arrays from the init only SigMatch lists + * - Setup per signature inspect engines + * - remove signature init data. + */ +static int SigMatchPrepare(DetectEngineCtx *de_ctx) +{ + SCEnter(); + + const int nlists = DetectBufferTypeMaxId(); + Signature *s = de_ctx->sig_list; + for (; s != NULL; s = s->next) { + /* set up inspect engines */ + DetectEngineAppInspectionEngine2Signature(s); + + /* built-ins */ + int type; + for (type = 0; type < DETECT_SM_LIST_MAX; type++) { + SigMatch *sm = s->init_data->smlists[type]; + s->sm_arrays[type] = SigMatchList2DataArray(sm); + } + + /* free lists. Ctx' are xferred to sm_arrays so won't get freed */ + int i; + for (i = 0; i < nlists; i++) { + SigMatch *sm = s->init_data->smlists[i]; + while (sm != NULL) { + SigMatch *nsm = sm->next; + SigMatchFree(sm); + sm = nsm; + } + } + SCFree(s->init_data->smlists); + SCFree(s->init_data->smlists_tail); + SCFree(s->init_data); + s->init_data = NULL; + } + + + SCReturnInt(0); +} + +/** + * \brief Convert the signature list into the runtime match structure. + * + * \param de_ctx Pointer to the Detection Engine Context whose Signatures have + * to be processed + * + * \retval 0 On Success. + * \retval -1 On failure. + */ +int SigGroupBuild(DetectEngineCtx *de_ctx) +{ + Signature *s = de_ctx->sig_list; + + /* Assign the unique order id of signatures after sorting, + * so the IP Only engine process them in order too. Also + * reset the old signums and assign new signums. We would + * have experienced Sig reordering by now, hence the new + * signums. */ + de_ctx->signum = 0; + while (s != NULL) { + s->num = de_ctx->signum++; + + s = s->next; + } + + if (DetectSetFastPatternAndItsId(de_ctx) < 0) + return -1; + + SigInitStandardMpmFactoryContexts(de_ctx); + + if (SigAddressPrepareStage1(de_ctx) != 0) { + SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); + exit(EXIT_FAILURE); + } + + if (SigAddressPrepareStage2(de_ctx) != 0) { + SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); + exit(EXIT_FAILURE); + } + + if (SigAddressPrepareStage3(de_ctx) != 0) { + SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); + exit(EXIT_FAILURE); + } + if (SigAddressPrepareStage4(de_ctx) != 0) { + SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); + exit(EXIT_FAILURE); + } + +#ifdef __SC_CUDA_SUPPORT__ + if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE) { + if (PatternMatchDefaultMatcher() == MPM_AC_CUDA) { + /* setting it to default. You've gotta remove it once you fix the state table thing */ + SCACConstructBoth16and32StateTables(); + + MpmCudaConf *conf = CudaHandlerGetCudaProfile("mpm"); + CUcontext cuda_context = CudaHandlerModuleGetContext(MPM_AC_CUDA_MODULE_NAME, conf->device_id); + if (cuda_context == 0) { + SCLogError(SC_ERR_FATAL, "cuda context is NULL."); + exit(EXIT_FAILURE); + } + int r = SCCudaCtxPushCurrent(cuda_context); + if (r < 0) { + SCLogError(SC_ERR_FATAL, "context push failed."); + exit(EXIT_FAILURE); + } + } + + if (PatternMatchDefaultMatcher() == MPM_AC_CUDA) { + int r = SCCudaCtxPopCurrent(NULL); + if (r < 0) { + SCLogError(SC_ERR_FATAL, "cuda context pop failure."); + exit(EXIT_FAILURE); + } + } + + /* too late to call this either ways. Should be called post ac goto. + * \todo Support this. */ + DetermineCudaStateTableSize(de_ctx); + } +#endif + + int r = DetectMpmPrepareBuiltinMpms(de_ctx); + r |= DetectMpmPrepareAppMpms(de_ctx); + if (r != 0) { + SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); + exit(EXIT_FAILURE); + } + + if (SigMatchPrepare(de_ctx) != 0) { + SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); + exit(EXIT_FAILURE); + } + +#ifdef PROFILING + SCProfilingRuleInitCounters(de_ctx); +#endif + SCFree(de_ctx->app_mpms); + de_ctx->app_mpms = NULL; + + if (!DetectEngineMultiTenantEnabled()) { + VarNameStoreActivateStaging(); + } + return 0; +} + +int SigGroupCleanup (DetectEngineCtx *de_ctx) +{ + SigAddressCleanupStage1(de_ctx); + + return 0; +} diff --git a/src/detect-engine-build.h b/src/detect-engine-build.h new file mode 100644 index 0000000000..7eb41d760b --- /dev/null +++ b/src/detect-engine-build.h @@ -0,0 +1,42 @@ +/* Copyright (C) 2007-2017 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef __DETECT_ENGINE_BUILD_H__ +#define __DETECT_ENGINE_BUILD_H__ + +void PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, + bool has_state, int app_decoder_events); + +int SignatureIsFilestoring(const Signature *); +int SignatureIsFilemagicInspecting(const Signature *); +int SignatureIsFileMd5Inspecting(const Signature *); +int SignatureIsFileSha1Inspecting(const Signature *s); +int SignatureIsFileSha256Inspecting(const Signature *s); +int SignatureIsFilesizeInspecting(const Signature *); + +int SigAddressPrepareStage1(DetectEngineCtx *de_ctx); +int SigAddressPrepareStage2(DetectEngineCtx *de_ctx); +int SigAddressPrepareStage3(DetectEngineCtx *de_ctx); +int SigAddressPrepareStage4(DetectEngineCtx *de_ctx); +int SigAddressCleanupStage1(DetectEngineCtx *de_ctx); + +void SigCleanSignatures(DetectEngineCtx *); + +int SigGroupBuild(DetectEngineCtx *); +int SigGroupCleanup (DetectEngineCtx *de_ctx); + +#endif /* __DETECT_ENGINE_BUILD_H__ */ diff --git a/src/detect.c b/src/detect.c index 2e4669f405..69e0187d67 100644 --- a/src/detect.c +++ b/src/detect.c @@ -241,8 +241,6 @@ extern int engine_analysis; static int fp_engine_analysis_set = 0; static int rule_engine_analysis_set = 0; -static void PacketCreateMask(Packet *, SignatureMask *, AppProto, bool, int); - /** * \brief Create the path if default-rule-path was specified * \param sig_file The name of the file @@ -1595,622 +1593,6 @@ error: return TM_ECODE_FAILED; } -void SigCleanSignatures(DetectEngineCtx *de_ctx) -{ - Signature *s = NULL, *ns; - - if (de_ctx == NULL) - return; - - for (s = de_ctx->sig_list; s != NULL;) { - ns = s->next; - SigFree(s); - s = ns; - } - - de_ctx->sig_list = NULL; - - DetectEngineResetMaxSigId(de_ctx); - de_ctx->sig_list = NULL; -} - -/** \brief Find a specific signature by sid and gid - * \param de_ctx detection engine ctx - * \param sid the signature id - * \param gid the signature group id - * - * \retval s sig found - * \retval NULL sig not found - */ -Signature *SigFindSignatureBySidGid(DetectEngineCtx *de_ctx, uint32_t sid, uint32_t gid) -{ - Signature *s = NULL; - - if (de_ctx == NULL) - return NULL; - - for (s = de_ctx->sig_list; s != NULL; s = s->next) { - if (s->id == sid && s->gid == gid) - return s; - } - - return NULL; -} - -/** - * \brief Check if a signature contains the filestore keyword. - * - * \param s signature - * - * \retval 0 no - * \retval 1 yes - */ -int SignatureIsFilestoring(const Signature *s) -{ - if (s == NULL) - return 0; - - if (s->flags & SIG_FLAG_FILESTORE) - return 1; - - return 0; -} - -/** - * \brief Check if a signature contains the filemagic keyword. - * - * \param s signature - * - * \retval 0 no - * \retval 1 yes - */ -int SignatureIsFilemagicInspecting(const Signature *s) -{ - if (s == NULL) - return 0; - - if (s->file_flags & FILE_SIG_NEED_MAGIC) - return 1; - - return 0; -} - -/** - * \brief Check if a signature contains the filemd5 keyword. - * - * \param s signature - * - * \retval 0 no - * \retval 1 yes - */ -int SignatureIsFileMd5Inspecting(const Signature *s) -{ - if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_MD5)) - return 1; - - return 0; -} - -/** - * \brief Check if a signature contains the filesha1 keyword. - * - * \param s signature - * - * \retval 0 no - * \retval 1 yes - */ -int SignatureIsFileSha1Inspecting(const Signature *s) -{ - if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA1)) - return 1; - - return 0; -} - -/** - * \brief Check if a signature contains the filesha256 keyword. - * - * \param s signature - * - * \retval 0 no - * \retval 1 yes - */ -int SignatureIsFileSha256Inspecting(const Signature *s) -{ - if ((s != NULL) && (s->file_flags & FILE_SIG_NEED_SHA256)) - return 1; - - return 0; -} - -/** - * \brief Check if a signature contains the filesize keyword. - * - * \param s signature - * - * \retval 0 no - * \retval 1 yes - */ -int SignatureIsFilesizeInspecting(const Signature *s) -{ - if (s == NULL) - return 0; - - if (s->file_flags & FILE_SIG_NEED_SIZE) - return 1; - - return 0; -} - -/** \brief Test is a initialized signature is IP only - * \param de_ctx detection engine ctx - * \param s the signature - * \retval 1 sig is ip only - * \retval 0 sig is not ip only - */ -int SignatureIsIPOnly(DetectEngineCtx *de_ctx, const Signature *s) -{ - if (s->alproto != ALPROTO_UNKNOWN) - return 0; - - if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) - return 0; - - /* for now assume that all registered buffer types are incompatible */ - const int nlists = DetectBufferTypeMaxId(); - for (int i = 0; i < nlists; i++) { - if (s->init_data->smlists[i] == NULL) - continue; - if (!(DetectBufferTypeGetNameById(i))) - continue; - - SCReturnInt(0); - } - - /* TMATCH list can be ignored, it contains TAGs and - * tags are compatible to IP-only. */ - - IPOnlyCIDRItem *cidr_item; - cidr_item = s->CidrSrc; - while (cidr_item != NULL) { - if (cidr_item->negated) - return 0; - - cidr_item = cidr_item->next; - } - cidr_item = s->CidrDst; - while (cidr_item != NULL) { - if (cidr_item->negated) - return 0; - - cidr_item = cidr_item->next; - } - - SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; - if (sm == NULL) - goto iponly; - - for ( ; sm != NULL; sm = sm->next) { - if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) - return 0; - /* we have enabled flowbits to be compatible with ip only sigs, as long - * as the sig only has a "set" flowbits */ - if (sm->type == DETECT_FLOWBITS && - (((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) { - return 0; - } - } - -iponly: - if (!(de_ctx->flags & DE_QUIET)) { - SCLogDebug("IP-ONLY (%" PRIu32 "): source %s, dest %s", s->id, - s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET", - s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET"); - } - return 1; -} - -/** \internal - * \brief Test is a initialized signature is inspecting protocol detection only - * \param de_ctx detection engine ctx - * \param s the signature - * \retval 1 sig is dp only - * \retval 0 sig is not dp only - */ -static int SignatureIsPDOnly(const Signature *s) -{ - if (s->alproto != ALPROTO_UNKNOWN) - return 0; - - if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) - return 0; - - /* for now assume that all registered buffer types are incompatible */ - const int nlists = DetectBufferTypeMaxId(); - for (int i = 0; i < nlists; i++) { - if (s->init_data->smlists[i] == NULL) - continue; - if (!(DetectBufferTypeGetNameById(i))) - continue; - - SCReturnInt(0); - } - - /* TMATCH list can be ignored, it contains TAGs and - * tags are compatible to DP-only. */ - - /* match list matches may be compatible to DP only. We follow the same - * logic as IP-only so we can use that flag */ - - SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; - if (sm == NULL) - return 0; - - int pd = 0; - for ( ; sm != NULL; sm = sm->next) { - if (sm->type == DETECT_AL_APP_LAYER_PROTOCOL) { - pd = 1; - } else { - /* flowbits are supported for dp only sigs, as long - * as the sig only has a "set" flowbits */ - if (sm->type == DETECT_FLOWBITS) { - if ((((DetectFlowbitsData *)sm->ctx)->cmd != DETECT_FLOWBITS_CMD_SET) ) { - SCLogDebug("%u: not PD-only: flowbit settings other than 'set'", s->id); - return 0; - } - } else if (sm->type == DETECT_FLOW) { - if (((DetectFlowData *)sm->ctx)->flags & ~(DETECT_FLOW_FLAG_TOSERVER|DETECT_FLOW_FLAG_TOCLIENT)) { - SCLogDebug("%u: not PD-only: flow settings other than toserver/toclient", s->id); - return 0; - } - } else if ( !(sigmatch_table[sm->type].flags & SIGMATCH_IPONLY_COMPAT)) { - SCLogDebug("%u: not PD-only: %s not PD/IP-only compat", s->id, sigmatch_table[sm->type].name); - return 0; - } - } - } - - if (pd) { - SCLogDebug("PD-ONLY (%" PRIu32 ")", s->id); - } - return pd; -} - -/** - * \internal - * \brief Check if the initialized signature is inspecting the packet payload - * \param de_ctx detection engine ctx - * \param s the signature - * \retval 1 sig is inspecting the payload - * \retval 0 sig is not inspecting the payload - */ -static int SignatureIsInspectingPayload(DetectEngineCtx *de_ctx, const Signature *s) -{ - - if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) { - return 1; - } - return 0; -} - -/** - * \internal - * \brief check if a signature is decoder event matching only - * \param de_ctx detection engine - * \param s the signature to test - * \retval 0 not a DEOnly sig - * \retval 1 DEOnly sig - */ -static int SignatureIsDEOnly(DetectEngineCtx *de_ctx, const Signature *s) -{ - if (s->alproto != ALPROTO_UNKNOWN) { - SCReturnInt(0); - } - - if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) - { - SCReturnInt(0); - } - - /* for now assume that all registered buffer types are incompatible */ - const int nlists = DetectBufferTypeMaxId(); - for (int i = 0; i < nlists; i++) { - if (s->init_data->smlists[i] == NULL) - continue; - if (!(DetectBufferTypeGetNameById(i))) - continue; - - SCReturnInt(0); - } - - /* check for conflicting keywords */ - SigMatch *sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; - for ( ;sm != NULL; sm = sm->next) { - if ( !(sigmatch_table[sm->type].flags & SIGMATCH_DEONLY_COMPAT)) - SCReturnInt(0); - } - - /* need at least one decode event keyword to be considered decode event. */ - sm = s->init_data->smlists[DETECT_SM_LIST_MATCH]; - for ( ;sm != NULL; sm = sm->next) { - if (sm->type == DETECT_DECODE_EVENT) - goto deonly; - if (sm->type == DETECT_ENGINE_EVENT) - goto deonly; - if (sm->type == DETECT_STREAM_EVENT) - goto deonly; - } - - SCReturnInt(0); - -deonly: - if (!(de_ctx->flags & DE_QUIET)) { - SCLogDebug("DE-ONLY (%" PRIu32 "): source %s, dest %s", s->id, - s->flags & SIG_FLAG_SRC_ANY ? "ANY" : "SET", - s->flags & SIG_FLAG_DST_ANY ? "ANY" : "SET"); - } - - SCReturnInt(1); -} - -#define MASK_TCP_INITDEINIT_FLAGS (TH_SYN|TH_RST|TH_FIN) -#define MASK_TCP_UNUSUAL_FLAGS (TH_URG|TH_ECN|TH_CWR) - -/* Create mask for this packet + it's flow if it has one - * - * Sets SIG_MASK_REQUIRE_PAYLOAD, SIG_MASK_REQUIRE_FLOW, - * SIG_MASK_REQUIRE_HTTP_STATE, SIG_MASK_REQUIRE_DCE_STATE - */ -static void -PacketCreateMask(Packet *p, SignatureMask *mask, AppProto alproto, - bool has_state, int app_decoder_events) -{ - if (!(p->flags & PKT_NOPAYLOAD_INSPECTION) && p->payload_len > 0) { - SCLogDebug("packet has payload"); - (*mask) |= SIG_MASK_REQUIRE_PAYLOAD; - } else if (p->flags & PKT_DETECT_HAS_STREAMDATA) { - SCLogDebug("stream data available"); - (*mask) |= SIG_MASK_REQUIRE_PAYLOAD; - } else { - SCLogDebug("packet has no payload"); - (*mask) |= SIG_MASK_REQUIRE_NO_PAYLOAD; - } - - if (p->events.cnt > 0 || app_decoder_events != 0 || p->app_layer_events != NULL) { - SCLogDebug("packet/flow has events set"); - (*mask) |= SIG_MASK_REQUIRE_ENGINE_EVENT; - } - - if (PKT_IS_TCP(p)) { - if ((p->tcph->th_flags & MASK_TCP_INITDEINIT_FLAGS) != 0) { - (*mask) |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT; - } - if ((p->tcph->th_flags & MASK_TCP_UNUSUAL_FLAGS) != 0) { - (*mask) |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL; - } - } - - if (p->flags & PKT_HAS_FLOW) { - SCLogDebug("packet has flow"); - (*mask) |= SIG_MASK_REQUIRE_FLOW; - - if (has_state) { - switch(alproto) { - case ALPROTO_HTTP: - SCLogDebug("packet/flow has http state"); - (*mask) |= SIG_MASK_REQUIRE_HTTP_STATE; - break; - case ALPROTO_SMB: - case ALPROTO_SMB2: - case ALPROTO_DCERPC: - SCLogDebug("packet/flow has dce state"); - (*mask) |= SIG_MASK_REQUIRE_DCE_STATE; - break; - case ALPROTO_SSH: - SCLogDebug("packet/flow has ssh state"); - (*mask) |= SIG_MASK_REQUIRE_SSH_STATE; - break; - case ALPROTO_TLS: - SCLogDebug("packet/flow has tls state"); - (*mask) |= SIG_MASK_REQUIRE_TLS_STATE; - break; - case ALPROTO_DNS: - SCLogDebug("packet/flow has dns state"); - (*mask) |= SIG_MASK_REQUIRE_DNS_STATE; - break; - case ALPROTO_FTP: - SCLogDebug("packet/flow has ftp state"); - (*mask) |= SIG_MASK_REQUIRE_FTP_STATE; - break; - case ALPROTO_SMTP: - SCLogDebug("packet/flow has smtp state"); - (*mask) |= SIG_MASK_REQUIRE_SMTP_STATE; - break; - case ALPROTO_ENIP: - SCLogDebug("packet/flow has enip state"); - (*mask) |= SIG_MASK_REQUIRE_ENIP_STATE; - break; - case ALPROTO_DNP3: - SCLogDebug("packet/flow has dnp3 state"); - (*mask) |= SIG_MASK_REQUIRE_DNP3_STATE; - break; - case ALPROTO_TEMPLATE: - SCLogDebug("packet/flow has template state"); - (*mask) |= SIG_MASK_REQUIRE_TEMPLATE_STATE; - break; - default: - SCLogDebug("packet/flow has other state"); - break; - } - } else { - SCLogDebug("no alstate"); - } - } -} - -static int SignatureCreateMask(Signature *s) -{ - SCEnter(); - - if (s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) { - s->mask |= SIG_MASK_REQUIRE_PAYLOAD; - SCLogDebug("sig requires payload"); - } - - SigMatch *sm; - for (sm = s->init_data->smlists[DETECT_SM_LIST_MATCH] ; sm != NULL; sm = sm->next) { - switch(sm->type) { - case DETECT_FLOWBITS: - { - /* figure out what flowbit action */ - DetectFlowbitsData *fb = (DetectFlowbitsData *)sm->ctx; - if (fb->cmd == DETECT_FLOWBITS_CMD_ISSET) { - /* not a mask flag, but still set it here */ - s->flags |= SIG_FLAG_REQUIRE_FLOWVAR; - - SCLogDebug("SIG_FLAG_REQUIRE_FLOWVAR set as sig has " - "flowbit isset option."); - } - - /* flow is required for any flowbit manipulation */ - s->mask |= SIG_MASK_REQUIRE_FLOW; - SCLogDebug("sig requires flow to be able to manipulate " - "flowbit(s)"); - break; - } - case DETECT_FLOWINT: - /* flow is required for any flowint manipulation */ - s->mask |= SIG_MASK_REQUIRE_FLOW; - SCLogDebug("sig requires flow to be able to manipulate " - "flowint(s)"); - break; - case DETECT_FLAGS: - { - DetectFlagsData *fl = (DetectFlagsData *)sm->ctx; - - if (fl->flags & TH_SYN) { - s->mask |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT; - SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT"); - } - if (fl->flags & TH_RST) { - s->mask |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT; - SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT"); - } - if (fl->flags & TH_FIN) { - s->mask |= SIG_MASK_REQUIRE_FLAGS_INITDEINIT; - SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_INITDEINIT"); - } - if (fl->flags & TH_URG) { - s->mask |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL; - SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL"); - } - if (fl->flags & TH_ECN) { - s->mask |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL; - SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL"); - } - if (fl->flags & TH_CWR) { - s->mask |= SIG_MASK_REQUIRE_FLAGS_UNUSUAL; - SCLogDebug("sig requires SIG_MASK_REQUIRE_FLAGS_UNUSUAL"); - } - break; - } - case DETECT_DSIZE: - { - DetectDsizeData *ds = (DetectDsizeData *)sm->ctx; - switch (ds->mode) { - case DETECTDSIZE_LT: - /* LT will include 0, so no payload. - * if GT is used in the same rule the - * flag will be set anyway. */ - break; - case DETECTDSIZE_RA: - case DETECTDSIZE_GT: - s->mask |= SIG_MASK_REQUIRE_PAYLOAD; - SCLogDebug("sig requires payload"); - break; - case DETECTDSIZE_EQ: - if (ds->dsize > 0) { - s->mask |= SIG_MASK_REQUIRE_PAYLOAD; - SCLogDebug("sig requires payload"); - } else if (ds->dsize == 0) { - s->mask |= SIG_MASK_REQUIRE_NO_PAYLOAD; - SCLogDebug("sig requires no payload"); - } - break; - } - break; - } - case DETECT_AL_APP_LAYER_EVENT: - s->mask |= SIG_MASK_REQUIRE_ENGINE_EVENT; - break; - case DETECT_ENGINE_EVENT: - s->mask |= SIG_MASK_REQUIRE_ENGINE_EVENT; - break; - } - } - - if (s->alproto == ALPROTO_SSH) { - s->mask |= SIG_MASK_REQUIRE_SSH_STATE; - SCLogDebug("sig requires ssh state"); - } - if (s->alproto == ALPROTO_TLS) { - s->mask |= SIG_MASK_REQUIRE_TLS_STATE; - SCLogDebug("sig requires tls state"); - } - if (s->alproto == ALPROTO_DNS) { - s->mask |= SIG_MASK_REQUIRE_DNS_STATE; - SCLogDebug("sig requires dns state"); - } - if (s->alproto == ALPROTO_DNP3) { - s->mask |= SIG_MASK_REQUIRE_DNP3_STATE; - SCLogDebug("sig requires dnp3 state"); - } - if (s->alproto == ALPROTO_FTP) { - s->mask |= SIG_MASK_REQUIRE_FTP_STATE; - SCLogDebug("sig requires ftp state"); - } - if (s->alproto == ALPROTO_SMTP) { - s->mask |= SIG_MASK_REQUIRE_SMTP_STATE; - SCLogDebug("sig requires smtp state"); - } - if (s->alproto == ALPROTO_ENIP) { - s->mask |= SIG_MASK_REQUIRE_ENIP_STATE; - SCLogDebug("sig requires enip state"); - } - if (s->alproto == ALPROTO_TEMPLATE) { - s->mask |= SIG_MASK_REQUIRE_TEMPLATE_STATE; - SCLogDebug("sig requires template state"); - } - - if ((s->mask & SIG_MASK_REQUIRE_DCE_STATE) || - (s->mask & SIG_MASK_REQUIRE_HTTP_STATE) || - (s->mask & SIG_MASK_REQUIRE_SSH_STATE) || - (s->mask & SIG_MASK_REQUIRE_DNS_STATE) || - (s->mask & SIG_MASK_REQUIRE_DNP3_STATE) || - (s->mask & SIG_MASK_REQUIRE_FTP_STATE) || - (s->mask & SIG_MASK_REQUIRE_SMTP_STATE) || - (s->mask & SIG_MASK_REQUIRE_ENIP_STATE) || - (s->mask & SIG_MASK_REQUIRE_TEMPLATE_STATE) || - (s->mask & SIG_MASK_REQUIRE_TLS_STATE)) - { - s->mask |= SIG_MASK_REQUIRE_FLOW; - SCLogDebug("sig requires flow"); - } - - if (s->init_data->init_flags & SIG_FLAG_INIT_FLOW) { - s->mask |= SIG_MASK_REQUIRE_FLOW; - SCLogDebug("sig requires flow"); - } - - if (s->flags & SIG_FLAG_APPLAYER) { - s->mask |= SIG_MASK_REQUIRE_FLOW; - SCLogDebug("sig requires flow"); - } - - SCLogDebug("mask %02X", s->mask); - SCReturnInt(0); -} - /** \brief disable file features we don't need * Called if we have no detection engine. */ @@ -2220,1368 +1602,6 @@ void DisableDetectFlowFileFlags(Flow *f) DetectPostInspectFileFlagsUpdate(f, NULL /* no sgh */, STREAM_TOCLIENT); } -static void SigInitStandardMpmFactoryContexts(DetectEngineCtx *de_ctx) -{ - DetectMpmInitializeBuiltinMpms(de_ctx); - DetectMpmInitializeAppMpms(de_ctx); - - return; -} - -/** \brief Pure-PCRE or bytetest rule */ -static int RuleInspectsPayloadHasNoMpm(const Signature *s) -{ - if (s->init_data->mpm_sm == NULL && s->init_data->smlists[DETECT_SM_LIST_PMATCH] != NULL) - return 1; - return 0; -} - -static int RuleGetMpmPatternSize(const Signature *s) -{ - if (s->init_data->mpm_sm == NULL) - return -1; - int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm); - if (mpm_list < 0) - return -1; - const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx; - if (cd == NULL) - return -1; - return (int)cd->content_len; -} - -static int RuleMpmIsNegated(const Signature *s) -{ - if (s->init_data->mpm_sm == NULL) - return 0; - int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm); - if (mpm_list < 0) - return 0; - const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx; - if (cd == NULL) - return 0; - return (cd->flags & DETECT_CONTENT_NEGATED); -} - -#ifdef HAVE_LIBJANSSON -static json_t *RulesGroupPrintSghStats(const SigGroupHead *sgh, - const int add_rules, const int add_mpm_stats) -{ - uint32_t mpm_cnt = 0; - uint32_t nonmpm_cnt = 0; - uint32_t negmpm_cnt = 0; - uint32_t any5_cnt = 0; - uint32_t payload_no_mpm_cnt = 0; - uint32_t syn_cnt = 0; - - uint32_t mpms_total = 0; - uint32_t mpms_min = 0; - uint32_t mpms_max = 0; - - struct { - uint32_t total; - uint32_t cnt; - uint32_t min; - uint32_t max; - } mpm_stats[DETECT_SM_LIST_MAX]; - memset(mpm_stats, 0x00, sizeof(mpm_stats)); - - uint32_t alstats[ALPROTO_MAX] = {0}; - uint32_t mpm_sizes[DETECT_SM_LIST_MAX][256]; - memset(mpm_sizes, 0, sizeof(mpm_sizes)); - uint32_t alproto_mpm_bufs[ALPROTO_MAX][DETECT_SM_LIST_MAX]; - memset(alproto_mpm_bufs, 0, sizeof(alproto_mpm_bufs)); - - json_t *js = json_object(); - if (unlikely(js == NULL)) - return NULL; - - json_object_set_new(js, "id", json_integer(sgh->id)); - - json_t *js_array = json_array(); - - const Signature *s; - uint32_t x; - for (x = 0; x < sgh->sig_cnt; x++) { - s = sgh->match_array[x]; - if (s == NULL) - continue; - - int any = 0; - if (s->proto.flags & DETECT_PROTO_ANY) { - any++; - } - if (s->flags & SIG_FLAG_DST_ANY) { - any++; - } - if (s->flags & SIG_FLAG_SRC_ANY) { - any++; - } - if (s->flags & SIG_FLAG_DP_ANY) { - any++; - } - if (s->flags & SIG_FLAG_SP_ANY) { - any++; - } - if (any == 5) { - any5_cnt++; - } - - if (s->init_data->mpm_sm == NULL) { - nonmpm_cnt++; - - if (s->sm_arrays[DETECT_SM_LIST_MATCH] != NULL) { - SCLogDebug("SGH %p Non-MPM inspecting only packets. Rule %u", sgh, s->id); - } - - DetectPort *sp = s->sp; - DetectPort *dp = s->dp; - - if (s->flags & SIG_FLAG_TOSERVER && (dp->port == 0 && dp->port2 == 65535)) { - SCLogDebug("SGH %p Non-MPM toserver and to 'any'. Rule %u", sgh, s->id); - } - if (s->flags & SIG_FLAG_TOCLIENT && (sp->port == 0 && sp->port2 == 65535)) { - SCLogDebug("SGH %p Non-MPM toclient and to 'any'. Rule %u", sgh, s->id); - } - - if (DetectFlagsSignatureNeedsSynPackets(s)) { - syn_cnt++; - } - - } else { - int mpm_list = SigMatchListSMBelongsTo(s, s->init_data->mpm_sm); - BUG_ON(mpm_list < 0); - const DetectContentData *cd = (const DetectContentData *)s->init_data->mpm_sm->ctx; - uint32_t size = cd->content_len < 256 ? cd->content_len : 255; - - mpm_sizes[mpm_list][size]++; - if (s->alproto != ALPROTO_UNKNOWN) { - alproto_mpm_bufs[s->alproto][mpm_list]++; - } - - if (mpm_list == DETECT_SM_LIST_PMATCH) { - if (size == 1) { - DetectPort *sp = s->sp; - DetectPort *dp = s->dp; - if (s->flags & SIG_FLAG_TOSERVER) { - if (dp->port == 0 && dp->port2 == 65535) { - SCLogDebug("SGH %p toserver 1byte fast_pattern to ANY. Rule %u", sgh, s->id); - } else { - SCLogDebug("SGH %p toserver 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, dp->port, dp->port2, s->id); - } - } - if (s->flags & SIG_FLAG_TOCLIENT) { - if (sp->port == 0 && sp->port2 == 65535) { - SCLogDebug("SGH %p toclient 1byte fast_pattern to ANY. Rule %u", sgh, s->id); - } else { - SCLogDebug("SGH %p toclient 1byte fast_pattern to port(s) %u-%u. Rule %u", sgh, sp->port, sp->port2, s->id); - } - } - } - } - - uint32_t w = PatternStrength(cd->content, cd->content_len); - mpms_total += w; - if (mpms_min == 0) - mpms_min = w; - if (w < mpms_min) - mpms_min = w; - if (w > mpms_max) - mpms_max = w; - - mpm_stats[mpm_list].total += w; - mpm_stats[mpm_list].cnt++; - if (mpm_stats[mpm_list].min == 0 || w < mpm_stats[mpm_list].min) - mpm_stats[mpm_list].min = w; - if (w > mpm_stats[mpm_list].max) - mpm_stats[mpm_list].max = w; - - mpm_cnt++; - - if (w < 10) { - SCLogDebug("SGH %p Weak MPM Pattern on %s. Rule %u", sgh, DetectListToString(mpm_list), s->id); - } - if (w < 10 && any == 5) { - SCLogDebug("SGH %p Weak MPM Pattern on %s, rule is 5xAny. Rule %u", sgh, DetectListToString(mpm_list), s->id); - } - - if (cd->flags & DETECT_CONTENT_NEGATED) { - SCLogDebug("SGH %p MPM Pattern on %s, is negated. Rule %u", sgh, DetectListToString(mpm_list), s->id); - negmpm_cnt++; - } - } - - if (RuleInspectsPayloadHasNoMpm(s)) { - SCLogDebug("SGH %p No MPM. Payload inspecting. Rule %u", sgh, s->id); - payload_no_mpm_cnt++; - } - - if (s->alproto != ALPROTO_UNKNOWN) { - alstats[s->alproto]++; - } - - if (add_rules) { - json_t *js_sig = json_object(); - if (unlikely(js == NULL)) - continue; - json_object_set_new(js_sig, "sig_id", json_integer(s->id)); - json_array_append_new(js_array, js_sig); - } - } - - json_object_set_new(js, "rules", js_array); - - json_t *stats = json_object(); - json_object_set_new(stats, "total", json_integer(sgh->sig_cnt)); - - json_t *types = json_object(); - json_object_set_new(types, "mpm", json_integer(mpm_cnt)); - json_object_set_new(types, "non_mpm", json_integer(nonmpm_cnt)); - json_object_set_new(types, "negated_mpm", json_integer(negmpm_cnt)); - json_object_set_new(types, "payload_but_no_mpm", json_integer(payload_no_mpm_cnt)); - json_object_set_new(types, "syn", json_integer(syn_cnt)); - json_object_set_new(types, "any5", json_integer(any5_cnt)); - json_object_set_new(stats, "types", types); - - int i; - for (i = 0; i < ALPROTO_MAX; i++) { - if (alstats[i] > 0) { - json_t *app = json_object(); - json_object_set_new(app, "total", json_integer(alstats[i])); - - for (x = 0; x < DETECT_SM_LIST_MAX; x++) { - if (alproto_mpm_bufs[i][x] == 0) - continue; - json_object_set_new(app, DetectListToHumanString(x), json_integer(alproto_mpm_bufs[i][x])); - } - - json_object_set_new(stats, AppProtoToString(i), app); - } - } - - if (add_mpm_stats) { - json_t *mpm_js = json_object(); - - for (i = 0; i < DETECT_SM_LIST_MAX; i++) { - if (mpm_stats[i].cnt > 0) { - - json_t *mpm_sizes_array = json_array(); - for (x = 0; x < 256; x++) { - if (mpm_sizes[i][x] == 0) - continue; - - json_t *e = json_object(); - json_object_set_new(e, "size", json_integer(x)); - json_object_set_new(e, "count", json_integer(mpm_sizes[i][x])); - json_array_append_new(mpm_sizes_array, e); - } - - json_t *buf = json_object(); - json_object_set_new(buf, "total", json_integer(mpm_stats[i].cnt)); - json_object_set_new(buf, "avg_strength", json_integer(mpm_stats[i].total / mpm_stats[i].cnt)); - json_object_set_new(buf, "min_strength", json_integer(mpm_stats[i].min)); - json_object_set_new(buf, "max_strength", json_integer(mpm_stats[i].max)); - - json_object_set_new(buf, "sizes", mpm_sizes_array); - - json_object_set_new(mpm_js, DetectListToHumanString(i), buf); - } - } - - json_object_set_new(stats, "mpm", mpm_js); - } - json_object_set_new(js, "stats", stats); - - json_object_set_new(js, "whitelist", json_integer(sgh->init->whitelist)); - - return js; -} -#endif /* HAVE_LIBJANSSON */ - -static void RulesDumpGrouping(const DetectEngineCtx *de_ctx, - const int add_rules, const int add_mpm_stats) -{ -#ifdef HAVE_LIBJANSSON - json_t *js = json_object(); - if (unlikely(js == NULL)) - return; - - int p; - for (p = 0; p < 256; p++) { - if (p == IPPROTO_TCP || p == IPPROTO_UDP) { - const char *name = (p == IPPROTO_TCP) ? "tcp" : "udp"; - - json_t *tcp = json_object(); - - json_t *ts_array = json_array(); - DetectPort *list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[1].tcp : - de_ctx->flow_gh[1].udp; - while (list != NULL) { - json_t *port = json_object(); - json_object_set_new(port, "port", json_integer(list->port)); - json_object_set_new(port, "port2", json_integer(list->port2)); - - json_t *tcp_ts = RulesGroupPrintSghStats(list->sh, - add_rules, add_mpm_stats); - json_object_set_new(port, "rulegroup", tcp_ts); - json_array_append_new(ts_array, port); - - list = list->next; - } - json_object_set_new(tcp, "toserver", ts_array); - - json_t *tc_array = json_array(); - list = (p == IPPROTO_TCP) ? de_ctx->flow_gh[0].tcp : - de_ctx->flow_gh[0].udp; - while (list != NULL) { - json_t *port = json_object(); - json_object_set_new(port, "port", json_integer(list->port)); - json_object_set_new(port, "port2", json_integer(list->port2)); - - json_t *tcp_tc = RulesGroupPrintSghStats(list->sh, - add_rules, add_mpm_stats); - json_object_set_new(port, "rulegroup", tcp_tc); - json_array_append_new(tc_array, port); - - list = list->next; - } - json_object_set_new(tcp, "toclient", tc_array); - - json_object_set_new(js, name, tcp); - } - - } - - const char *filename = "rule_group.json"; - const char *log_dir = ConfigGetLogDirectory(); - char log_path[PATH_MAX] = ""; - - snprintf(log_path, sizeof(log_path), "%s/%s", log_dir, filename); - - FILE *fp = fopen(log_path, "w"); - if (fp == NULL) { - return; - } - - char *js_s = json_dumps(js, - JSON_PRESERVE_ORDER|JSON_ESCAPE_SLASH); - if (unlikely(js_s == NULL)) { - fclose(fp); - return; - } - - json_object_clear(js); - json_decref(js); - - fprintf(fp, "%s\n", js_s); - free(js_s); - fclose(fp); -#endif - return; -} - -static int RulesGroupByProto(DetectEngineCtx *de_ctx) -{ - Signature *s = de_ctx->sig_list; - - uint32_t max_idx = 0; - SigGroupHead *sgh_ts[256] = {NULL}; - SigGroupHead *sgh_tc[256] = {NULL}; - - for ( ; s != NULL; s = s->next) { - if (s->flags & SIG_FLAG_IPONLY) - continue; - - int p; - for (p = 0; p < 256; p++) { - if (p == IPPROTO_TCP || p == IPPROTO_UDP) { - continue; - } - if (!(s->proto.proto[p / 8] & (1<<(p % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) { - continue; - } - - if (s->flags & SIG_FLAG_TOCLIENT) { - SigGroupHeadAppendSig(de_ctx, &sgh_tc[p], s); - max_idx = s->num; - } - if (s->flags & SIG_FLAG_TOSERVER) { - SigGroupHeadAppendSig(de_ctx, &sgh_ts[p], s); - max_idx = s->num; - } - } - } - SCLogDebug("max_idx %u", max_idx); - - /* lets look at deduplicating this list */ - SigGroupHeadHashFree(de_ctx); - SigGroupHeadHashInit(de_ctx); - - uint32_t cnt = 0; - uint32_t own = 0; - uint32_t ref = 0; - int p; - for (p = 0; p < 256; p++) { - if (p == IPPROTO_TCP || p == IPPROTO_UDP) - continue; - if (sgh_ts[p] == NULL) - continue; - - cnt++; - - SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_ts[p]); - if (lookup_sgh == NULL) { - SCLogDebug("proto group %d sgh %p is the original", p, sgh_ts[p]); - - SigGroupHeadSetSigCnt(sgh_ts[p], max_idx); - SigGroupHeadBuildMatchArray(de_ctx, sgh_ts[p], max_idx); - - SigGroupHeadHashAdd(de_ctx, sgh_ts[p]); - SigGroupHeadStore(de_ctx, sgh_ts[p]); - - de_ctx->gh_unique++; - own++; - } else { - SCLogDebug("proto group %d sgh %p is a copy", p, sgh_ts[p]); - - SigGroupHeadFree(sgh_ts[p]); - sgh_ts[p] = lookup_sgh; - - de_ctx->gh_reuse++; - ref++; - } - } - SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies", - "toserver", cnt, own, ref); - - cnt = 0; - own = 0; - ref = 0; - for (p = 0; p < 256; p++) { - if (p == IPPROTO_TCP || p == IPPROTO_UDP) - continue; - if (sgh_tc[p] == NULL) - continue; - - cnt++; - - SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, sgh_tc[p]); - if (lookup_sgh == NULL) { - SCLogDebug("proto group %d sgh %p is the original", p, sgh_tc[p]); - - SigGroupHeadSetSigCnt(sgh_tc[p], max_idx); - SigGroupHeadBuildMatchArray(de_ctx, sgh_tc[p], max_idx); - - SigGroupHeadHashAdd(de_ctx, sgh_tc[p]); - SigGroupHeadStore(de_ctx, sgh_tc[p]); - - de_ctx->gh_unique++; - own++; - - } else { - SCLogDebug("proto group %d sgh %p is a copy", p, sgh_tc[p]); - - SigGroupHeadFree(sgh_tc[p]); - sgh_tc[p] = lookup_sgh; - - de_ctx->gh_reuse++; - ref++; - } - } - SCLogPerf("OTHER %s: %u proto groups, %u unique SGH's, %u copies", - "toclient", cnt, own, ref); - - for (p = 0; p < 256; p++) { - if (p == IPPROTO_TCP || p == IPPROTO_UDP) - continue; - - de_ctx->flow_gh[0].sgh[p] = sgh_tc[p]; - de_ctx->flow_gh[1].sgh[p] = sgh_ts[p]; - } - - return 0; -} - -static int PortIsWhitelisted(const DetectEngineCtx *de_ctx, - const DetectPort *a, int ipproto) -{ - DetectPort *w = de_ctx->tcp_whitelist; - if (ipproto == IPPROTO_UDP) - w = de_ctx->udp_whitelist; - - while (w) { - if (a->port >= w->port && a->port2 <= w->port) { - SCLogDebug("port group %u:%u whitelisted -> %d", a->port, a->port2, w->port); - return 1; - } - w = w->next; - } - - return 0; -} - -static int RuleSetWhitelist(Signature *s) -{ - DetectPort *p = NULL; - if (s->flags & SIG_FLAG_TOSERVER) - p = s->dp; - else if (s->flags & SIG_FLAG_TOCLIENT) - p = s->sp; - else - return 0; - - /* for sigs that don't use 'any' as port, see if we want to - * whitelist poor sigs */ - int wl = 0; - if (!(p->port == 0 && p->port2 == 65535)) { - /* pure pcre, bytetest, etc rules */ - if (RuleInspectsPayloadHasNoMpm(s)) { - SCLogDebug("Rule %u MPM has 1 byte fast_pattern. Whitelisting SGH's.", s->id); - wl = 99; - - } else if (RuleMpmIsNegated(s)) { - SCLogDebug("Rule %u MPM is negated. Whitelisting SGH's.", s->id); - wl = 77; - - /* one byte pattern in packet/stream payloads */ - } else if (s->init_data->mpm_sm != NULL && - SigMatchListSMBelongsTo(s, s->init_data->mpm_sm) == DETECT_SM_LIST_PMATCH && - RuleGetMpmPatternSize(s) == 1) - { - SCLogDebug("Rule %u No MPM. Payload inspecting. Whitelisting SGH's.", s->id); - wl = 55; - - } else if (DetectFlagsSignatureNeedsSynPackets(s) && - DetectFlagsSignatureNeedsSynOnlyPackets(s)) - { - SCLogDebug("Rule %u Needs SYN, so inspected often. Whitelisting SGH's.", s->id); - wl = 33; - } - } - - s->init_data->whitelist = wl; - return wl; -} - -int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx); -int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b); - -static DetectPort *RulesGroupByPorts(DetectEngineCtx *de_ctx, int ipproto, uint32_t direction) { - /* step 1: create a hash of 'DetectPort' objects based on all the - * rules. Each object will have a SGH with the sigs added - * that belong to the SGH. */ - DetectPortHashInit(de_ctx); - - uint32_t max_idx = 0; - const Signature *s = de_ctx->sig_list; - DetectPort *list = NULL; - while (s) { - /* IP Only rules are handled separately */ - if (s->flags & SIG_FLAG_IPONLY) - goto next; - if (!(s->proto.proto[ipproto / 8] & (1<<(ipproto % 8)) || (s->proto.flags & DETECT_PROTO_ANY))) - goto next; - if (direction == SIG_FLAG_TOSERVER) { - if (!(s->flags & SIG_FLAG_TOSERVER)) - goto next; - } else if (direction == SIG_FLAG_TOCLIENT) { - if (!(s->flags & SIG_FLAG_TOCLIENT)) - goto next; - } - - DetectPort *p = NULL; - if (direction == SIG_FLAG_TOSERVER) - p = s->dp; - else if (direction == SIG_FLAG_TOCLIENT) - p = s->sp; - else - BUG_ON(1); - - /* see if we want to exclude directionless sigs that really care only for - * to_server syn scans/floods */ - if ((direction == SIG_FLAG_TOCLIENT) && - DetectFlagsSignatureNeedsSynPackets(s) && - DetectFlagsSignatureNeedsSynOnlyPackets(s) && - ((s->flags & (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) == (SIG_FLAG_TOSERVER|SIG_FLAG_TOCLIENT)) && - (!(s->dp->port == 0 && s->dp->port2 == 65535))) - { - SCLogWarning(SC_WARN_POOR_RULE, "rule %u: SYN-only to port(s) %u:%u " - "w/o direction specified, disabling for toclient direction", - s->id, s->dp->port, s->dp->port2); - goto next; - } - - int wl = s->init_data->whitelist; - while (p) { - int pwl = PortIsWhitelisted(de_ctx, p, ipproto) ? 111 : 0; - pwl = MAX(wl,pwl); - - DetectPort *lookup = DetectPortHashLookup(de_ctx, p); - if (lookup) { - SigGroupHeadAppendSig(de_ctx, &lookup->sh, s); - lookup->sh->init->whitelist = MAX(lookup->sh->init->whitelist, pwl); - } else { - DetectPort *tmp2 = DetectPortCopySingle(de_ctx, p); - BUG_ON(tmp2 == NULL); - SigGroupHeadAppendSig(de_ctx, &tmp2->sh, s); - tmp2->sh->init->whitelist = pwl; - DetectPortHashAdd(de_ctx, tmp2); - } - - p = p->next; - } - max_idx = s->num; - next: - s = s->next; - } - - /* step 2: create a list of DetectPort objects */ - HashListTableBucket *htb = NULL; - for (htb = HashListTableGetListHead(de_ctx->dport_hash_table); - htb != NULL; - htb = HashListTableGetListNext(htb)) - { - DetectPort *p = HashListTableGetListData(htb); - DetectPort *tmp = DetectPortCopySingle(de_ctx, p); - BUG_ON(tmp == NULL); - int r = DetectPortInsert(de_ctx, &list , tmp); - BUG_ON(r == -1); - } - DetectPortHashFree(de_ctx); - de_ctx->dport_hash_table = NULL; - - SCLogDebug("rules analyzed"); - - /* step 3: group the list and shrink it if necessary */ - DetectPort *newlist = NULL; - uint16_t groupmax = (direction == SIG_FLAG_TOCLIENT) ? de_ctx->max_uniq_toclient_groups : - de_ctx->max_uniq_toserver_groups; - CreateGroupedPortList(de_ctx, list, &newlist, groupmax, CreateGroupedPortListCmpCnt, max_idx); - list = newlist; - - /* step 4: deduplicate the SGH's */ - SigGroupHeadHashFree(de_ctx); - SigGroupHeadHashInit(de_ctx); - - uint32_t cnt = 0; - uint32_t own = 0; - uint32_t ref = 0; - DetectPort *iter; - for (iter = list ; iter != NULL; iter = iter->next) { - BUG_ON (iter->sh == NULL); - cnt++; - - SigGroupHead *lookup_sgh = SigGroupHeadHashLookup(de_ctx, iter->sh); - if (lookup_sgh == NULL) { - SCLogDebug("port group %p sgh %p is the original", iter, iter->sh); - - SigGroupHeadSetSigCnt(iter->sh, max_idx); - SigGroupHeadBuildMatchArray(de_ctx, iter->sh, max_idx); - SigGroupHeadSetProtoAndDirection(iter->sh, ipproto, direction); - SigGroupHeadHashAdd(de_ctx, iter->sh); - SigGroupHeadStore(de_ctx, iter->sh); - iter->flags |= PORT_SIGGROUPHEAD_COPY; - de_ctx->gh_unique++; - own++; - } else { - SCLogDebug("port group %p sgh %p is a copy", iter, iter->sh); - - SigGroupHeadFree(iter->sh); - iter->sh = lookup_sgh; - iter->flags |= PORT_SIGGROUPHEAD_COPY; - - de_ctx->gh_reuse++; - ref++; - } - } -#if 0 - for (iter = list ; iter != NULL; iter = iter->next) { - SCLogInfo("PORT %u-%u %p (sgh=%s, whitelisted=%s/%d)", - iter->port, iter->port2, iter->sh, - iter->flags & PORT_SIGGROUPHEAD_COPY ? "ref" : "own", - iter->sh->init->whitelist ? "true" : "false", - iter->sh->init->whitelist); - } -#endif - SCLogPerf("%s %s: %u port groups, %u unique SGH's, %u copies", - ipproto == 6 ? "TCP" : "UDP", - direction == SIG_FLAG_TOSERVER ? "toserver" : "toclient", - cnt, own, ref); - return list; -} - -/** - * \brief Preprocess signature, classify ip-only, etc, build sig array - * - * \param de_ctx Pointer to the Detection Engine Context - * - * \retval 0 on success - * \retval -1 on failure - */ -int SigAddressPrepareStage1(DetectEngineCtx *de_ctx) -{ - Signature *tmp_s = NULL; - uint32_t cnt_iponly = 0; - uint32_t cnt_payload = 0; - uint32_t cnt_applayer = 0; - uint32_t cnt_deonly = 0; - const int nlists = DetectBufferTypeMaxId(); - - if (!(de_ctx->flags & DE_QUIET)) { - SCLogDebug("building signature grouping structure, stage 1: " - "preprocessing rules..."); - } - - de_ctx->sig_array_len = DetectEngineGetMaxSigId(de_ctx); - de_ctx->sig_array_size = (de_ctx->sig_array_len * sizeof(Signature *)); - de_ctx->sig_array = (Signature **)SCMalloc(de_ctx->sig_array_size); - if (de_ctx->sig_array == NULL) - goto error; - memset(de_ctx->sig_array,0,de_ctx->sig_array_size); - - SCLogDebug("signature lookup array: %" PRIu32 " sigs, %" PRIu32 " bytes", - de_ctx->sig_array_len, de_ctx->sig_array_size); - - /* now for every rule add the source group */ - for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) { - de_ctx->sig_array[tmp_s->num] = tmp_s; - - SCLogDebug("Signature %" PRIu32 ", internal id %" PRIu32 ", ptrs %p %p ", tmp_s->id, tmp_s->num, tmp_s, de_ctx->sig_array[tmp_s->num]); - - /* see if the sig is dp only */ - if (SignatureIsPDOnly(tmp_s) == 1) { - tmp_s->flags |= SIG_FLAG_PDONLY; - SCLogDebug("Signature %"PRIu32" is considered \"PD only\"", tmp_s->id); - - /* see if the sig is ip only */ - } else if (SignatureIsIPOnly(de_ctx, tmp_s) == 1) { - tmp_s->flags |= SIG_FLAG_IPONLY; - cnt_iponly++; - - SCLogDebug("Signature %"PRIu32" is considered \"IP only\"", tmp_s->id); - - /* see if any sig is inspecting the packet payload */ - } else if (SignatureIsInspectingPayload(de_ctx, tmp_s) == 1) { - cnt_payload++; - - SCLogDebug("Signature %"PRIu32" is considered \"Payload inspecting\"", tmp_s->id); - } else if (SignatureIsDEOnly(de_ctx, tmp_s) == 1) { - tmp_s->init_data->init_flags |= SIG_FLAG_INIT_DEONLY; - SCLogDebug("Signature %"PRIu32" is considered \"Decoder Event only\"", tmp_s->id); - cnt_deonly++; - } - - if (tmp_s->flags & SIG_FLAG_APPLAYER) { - SCLogDebug("Signature %"PRIu32" is considered \"Applayer inspecting\"", tmp_s->id); - cnt_applayer++; - } - -#ifdef DEBUG - if (SCLogDebugEnabled()) { - uint16_t colen = 0; - char copresent = 0; - SigMatch *sm; - DetectContentData *co; - for (sm = tmp_s->init_data->smlists[DETECT_SM_LIST_MATCH]; sm != NULL; sm = sm->next) { - if (sm->type != DETECT_CONTENT) - continue; - - copresent = 1; - co = (DetectContentData *)sm->ctx; - if (co->content_len > colen) - colen = co->content_len; - } - - if (copresent && colen == 1) { - SCLogDebug("signature %8u content maxlen 1", tmp_s->id); - int proto; - for (proto = 0; proto < 256; proto++) { - if (tmp_s->proto.proto[(proto/8)] & (1<<(proto%8))) - SCLogDebug("=> proto %" PRId32 "", proto); - } - } - } -#endif /* DEBUG */ - - if (RuleMpmIsNegated(tmp_s)) { - tmp_s->flags |= SIG_FLAG_MPM_NEG; - } - - SignatureCreateMask(tmp_s); - DetectContentPropagateLimits(tmp_s); - SigParseApplyDsizeToContent(tmp_s); - - RuleSetWhitelist(tmp_s); - - /* if keyword engines are enabled in the config, handle them here */ - if (de_ctx->prefilter_setting == DETECT_PREFILTER_AUTO && - !(tmp_s->flags & SIG_FLAG_PREFILTER)) - { - int i; - int prefilter_list = DETECT_TBLSIZE; - - /* get the keyword supporting prefilter with the lowest type */ - for (i = 0; i < nlists; i++) { - SigMatch *sm = tmp_s->init_data->smlists[i]; - while (sm != NULL) { - if (sigmatch_table[sm->type].SupportsPrefilter != NULL) { - if (sigmatch_table[sm->type].SupportsPrefilter(tmp_s) == TRUE) { - prefilter_list = MIN(prefilter_list, sm->type); - } - } - sm = sm->next; - } - } - - /* apply that keyword as prefilter */ - if (prefilter_list != DETECT_TBLSIZE) { - for (i = 0; i < nlists; i++) { - SigMatch *sm = tmp_s->init_data->smlists[i]; - while (sm != NULL) { - if (sm->type == prefilter_list) { - tmp_s->init_data->prefilter_sm = sm; - tmp_s->flags |= SIG_FLAG_PREFILTER; - SCLogConfig("sid %u: prefilter is on \"%s\"", tmp_s->id, sigmatch_table[sm->type].name); - break; - } - sm = sm->next; - } - } - } - } - de_ctx->sig_cnt++; - } - - if (!(de_ctx->flags & DE_QUIET)) { - SCLogInfo("%" PRIu32 " signatures processed. %" PRIu32 " are IP-only " - "rules, %" PRIu32 " are inspecting packet payload, %"PRIu32 - " inspect application layer, %"PRIu32" are decoder event only", - de_ctx->sig_cnt, cnt_iponly, cnt_payload, cnt_applayer, - cnt_deonly); - - SCLogConfig("building signature grouping structure, stage 1: " - "preprocessing rules... complete"); - } - return 0; - -error: - return -1; -} - -static int PortGroupWhitelist(const DetectPort *a) -{ - return a->sh->init->whitelist; -} - -int CreateGroupedPortListCmpCnt(DetectPort *a, DetectPort *b) -{ - if (PortGroupWhitelist(a) && !PortGroupWhitelist(b)) { - SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); - return 1; - } else if (!PortGroupWhitelist(a) && PortGroupWhitelist(b)) { - SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); - return 0; - } else if (PortGroupWhitelist(a) > PortGroupWhitelist(b)) { - SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); - return 1; - } else if (PortGroupWhitelist(a) == PortGroupWhitelist(b)) { - if (a->sh->sig_cnt > b->sh->sig_cnt) { - SCLogDebug("%u:%u (cnt %u, wl %d) wins against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); - return 1; - } - } - - SCLogDebug("%u:%u (cnt %u, wl %d) loses against %u:%u (cnt %u, wl %d)", - a->port, a->port2, a->sh->sig_cnt, PortGroupWhitelist(a), - b->port, b->port2, b->sh->sig_cnt, PortGroupWhitelist(b)); - return 0; -} - -/** \internal - * \brief Create a list of DetectPort objects sorted based on CompareFunc's - * logic. - * - * List can limit the number of groups. In this case an extra "join" group - * is created that contains the sigs belonging to that. It's *appended* to - * the list, meaning that if the list is walked linearly it's found last. - * The joingr is meant to be a catch all. - * - */ -int CreateGroupedPortList(DetectEngineCtx *de_ctx, DetectPort *port_list, DetectPort **newhead, uint32_t unique_groups, int (*CompareFunc)(DetectPort *, DetectPort *), uint32_t max_idx) -{ - DetectPort *tmplist = NULL, *joingr = NULL; - char insert = 0; - uint32_t groups = 0; - DetectPort *list; - - /* insert the addresses into the tmplist, where it will - * be sorted descending on 'cnt' and on wehther a group - * is whitelisted. */ - - DetectPort *oldhead = port_list; - while (oldhead) { - /* take the top of the list */ - list = oldhead; - oldhead = oldhead->next; - list->next = NULL; - - groups++; - - SigGroupHeadSetSigCnt(list->sh, max_idx); - - /* insert it */ - DetectPort *tmpgr = tmplist, *prevtmpgr = NULL; - if (tmplist == NULL) { - /* empty list, set head */ - tmplist = list; - } else { - /* look for the place to insert */ - for ( ; tmpgr != NULL && !insert; tmpgr = tmpgr->next) { - if (CompareFunc(list, tmpgr) == 1) { - if (tmpgr == tmplist) { - list->next = tmplist; - tmplist = list; - SCLogDebug("new list top: %u:%u", tmplist->port, tmplist->port2); - } else { - list->next = prevtmpgr->next; - prevtmpgr->next = list; - } - insert = 1; - break; - } - prevtmpgr = tmpgr; - } - if (insert == 0) { - list->next = NULL; - prevtmpgr->next = list; - } - insert = 0; - } - } - - uint32_t left = unique_groups; - if (left == 0) - left = groups; - - /* create another list: take the port groups from above - * and add them to the 2nd list until we have met our - * count. The rest is added to the 'join' group. */ - DetectPort *tmplist2 = NULL, *tmplist2_tail = NULL; - DetectPort *gr, *next_gr; - for (gr = tmplist; gr != NULL; ) { - next_gr = gr->next; - - SCLogDebug("temp list gr %p %u:%u", gr, gr->port, gr->port2); - DetectPortPrint(gr); - - /* if we've set up all the unique groups, add the rest to the - * catch-all joingr */ - if (left == 0) { - if (joingr == NULL) { - DetectPortParse(de_ctx, &joingr, "0:65535"); - if (joingr == NULL) { - goto error; - } - SCLogDebug("joingr => %u-%u", joingr->port, joingr->port2); - joingr->next = NULL; - } - SigGroupHeadCopySigs(de_ctx,gr->sh,&joingr->sh); - - /* when a group's sigs are added to the joingr, we can free it */ - gr->next = NULL; - DetectPortFree(gr); - gr = NULL; - - /* append */ - } else { - gr->next = NULL; - - if (tmplist2 == NULL) { - tmplist2 = gr; - tmplist2_tail = gr; - } else { - tmplist2_tail->next = gr; - tmplist2_tail = gr; - } - } - - if (left > 0) - left--; - - gr = next_gr; - } - - /* if present, append the joingr that covers the rest */ - if (joingr != NULL) { - SCLogDebug("appending joingr %p %u:%u", joingr, joingr->port, joingr->port2); - - if (tmplist2 == NULL) { - tmplist2 = joingr; - //tmplist2_tail = joingr; - } else { - tmplist2_tail->next = joingr; - //tmplist2_tail = joingr; - } - } else { - SCLogDebug("no joingr"); - } - - /* pass back our new list to the caller */ - *newhead = tmplist2; - DetectPortPrintList(*newhead); - - return 0; -error: - return -1; -} - -/** - * \internal - * \brief add a decoder event signature to the detection engine ctx - */ -static void DetectEngineAddDecoderEventSig(DetectEngineCtx *de_ctx, Signature *s) -{ - SCLogDebug("adding signature %"PRIu32" to the decoder event sgh", s->id); - SigGroupHeadAppendSig(de_ctx, &de_ctx->decoder_event_sgh, s); -} - -/** - * \brief Fill the global src group head, with the sigs included - * - * \param de_ctx Pointer to the Detection Engine Context whose Signatures have - * to be processed - * - * \retval 0 On success - * \retval -1 On failure - */ -int SigAddressPrepareStage2(DetectEngineCtx *de_ctx) -{ - Signature *tmp_s = NULL; - uint32_t sigs = 0; - - if (!(de_ctx->flags & DE_QUIET)) { - SCLogDebug("building signature grouping structure, stage 2: " - "building source address lists..."); - } - - IPOnlyInit(de_ctx, &de_ctx->io_ctx); - - de_ctx->flow_gh[1].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOSERVER); - de_ctx->flow_gh[0].tcp = RulesGroupByPorts(de_ctx, IPPROTO_TCP, SIG_FLAG_TOCLIENT); - de_ctx->flow_gh[1].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOSERVER); - de_ctx->flow_gh[0].udp = RulesGroupByPorts(de_ctx, IPPROTO_UDP, SIG_FLAG_TOCLIENT); - - /* Setup the other IP Protocols (so not TCP/UDP) */ - RulesGroupByProto(de_ctx); - - /* now for every rule add the source group to our temp lists */ - for (tmp_s = de_ctx->sig_list; tmp_s != NULL; tmp_s = tmp_s->next) { - SCLogDebug("tmp_s->id %"PRIu32, tmp_s->id); - if (tmp_s->flags & SIG_FLAG_IPONLY) { - IPOnlyAddSignature(de_ctx, &de_ctx->io_ctx, tmp_s); - } - - if (tmp_s->init_data->init_flags & SIG_FLAG_INIT_DEONLY) { - DetectEngineAddDecoderEventSig(de_ctx, tmp_s); - } - - sigs++; - } - - IPOnlyPrepare(de_ctx); - IPOnlyPrint(de_ctx, &de_ctx->io_ctx); - - return 0; -} - -static void DetectEngineBuildDecoderEventSgh(DetectEngineCtx *de_ctx) -{ - if (de_ctx->decoder_event_sgh == NULL) - return; - - uint32_t max_idx = DetectEngineGetMaxSigId(de_ctx); - SigGroupHeadSetSigCnt(de_ctx->decoder_event_sgh, max_idx); - SigGroupHeadBuildMatchArray(de_ctx, de_ctx->decoder_event_sgh, max_idx); -} - -int SigAddressPrepareStage3(DetectEngineCtx *de_ctx) -{ - /* prepare the decoder event sgh */ - DetectEngineBuildDecoderEventSgh(de_ctx); - return 0; -} - -int SigAddressCleanupStage1(DetectEngineCtx *de_ctx) -{ - BUG_ON(de_ctx == NULL); - - if (!(de_ctx->flags & DE_QUIET)) { - SCLogDebug("cleaning up signature grouping structure..."); - } - if (de_ctx->decoder_event_sgh) - SigGroupHeadFree(de_ctx->decoder_event_sgh); - de_ctx->decoder_event_sgh = NULL; - - int f; - for (f = 0; f < FLOW_STATES; f++) { - int p; - for (p = 0; p < 256; p++) { - de_ctx->flow_gh[f].sgh[p] = NULL; - } - - /* free lookup lists */ - DetectPortCleanupList(de_ctx->flow_gh[f].tcp); - de_ctx->flow_gh[f].tcp = NULL; - DetectPortCleanupList(de_ctx->flow_gh[f].udp); - de_ctx->flow_gh[f].udp = NULL; - } - - uint32_t idx; - for (idx = 0; idx < de_ctx->sgh_array_cnt; idx++) { - SigGroupHead *sgh = de_ctx->sgh_array[idx]; - if (sgh == NULL) - continue; - - SCLogDebug("sgh %p", sgh); - SigGroupHeadFree(sgh); - } - SCFree(de_ctx->sgh_array); - de_ctx->sgh_array = NULL; - de_ctx->sgh_array_cnt = 0; - de_ctx->sgh_array_size = 0; - - IPOnlyDeinit(de_ctx, &de_ctx->io_ctx); - - if (!(de_ctx->flags & DE_QUIET)) { - SCLogInfo("cleaning up signature grouping structure... complete"); - } - return 0; -} - -/** \brief finalize preparing sgh's */ -int SigAddressPrepareStage4(DetectEngineCtx *de_ctx) -{ - SCEnter(); - - //SCLogInfo("sgh's %"PRIu32, de_ctx->sgh_array_cnt); - - uint32_t cnt = 0; - uint32_t idx = 0; - for (idx = 0; idx < de_ctx->sgh_array_cnt; idx++) { - SigGroupHead *sgh = de_ctx->sgh_array[idx]; - if (sgh == NULL) - continue; - - SCLogDebug("sgh %p", sgh); - - SigGroupHeadSetFilemagicFlag(de_ctx, sgh); - SigGroupHeadSetFileHashFlag(de_ctx, sgh); - SigGroupHeadSetFilesizeFlag(de_ctx, sgh); - SigGroupHeadSetFilestoreCount(de_ctx, sgh); - SCLogDebug("filestore count %u", sgh->filestore_cnt); - - PrefilterSetupRuleGroup(de_ctx, sgh); - - SigGroupHeadBuildNonPrefilterArray(de_ctx, sgh); - - sgh->id = idx; - cnt++; - } - SCLogPerf("Unique rule groups: %u", cnt); - - MpmStoreReportStats(de_ctx); - - if (de_ctx->decoder_event_sgh != NULL) { - /* no need to set filestore count here as that would make a - * signature not decode event only. */ - } - - /* cleanup the hashes now since we won't need them - * after the initialization phase. */ - SigGroupHeadHashFree(de_ctx); - - int dump_grouping = 0; - (void)ConfGetBool("detect.profiling.grouping.dump-to-disk", &dump_grouping); - - if (dump_grouping) { - int add_rules = 0; - (void)ConfGetBool("detect.profiling.grouping.include-rules", &add_rules); - int add_mpm_stats = 0; - (void)ConfGetBool("detect.profiling.grouping.include-mpm-stats", &add_rules); - - RulesDumpGrouping(de_ctx, add_rules, add_mpm_stats); - } - -#ifdef PROFILING - SCProfilingSghInitCounters(de_ctx); -#endif - SCReturnInt(0); -} - -/** \internal - * \brief perform final per signature setup tasks - * - * - Create SigMatchData arrays from the init only SigMatch lists - * - Setup per signature inspect engines - * - remove signature init data. - */ -static int SigMatchPrepare(DetectEngineCtx *de_ctx) -{ - SCEnter(); - - const int nlists = DetectBufferTypeMaxId(); - Signature *s = de_ctx->sig_list; - for (; s != NULL; s = s->next) { - /* set up inspect engines */ - DetectEngineAppInspectionEngine2Signature(s); - - /* built-ins */ - int type; - for (type = 0; type < DETECT_SM_LIST_MAX; type++) { - SigMatch *sm = s->init_data->smlists[type]; - s->sm_arrays[type] = SigMatchList2DataArray(sm); - } - - /* free lists. Ctx' are xferred to sm_arrays so won't get freed */ - int i; - for (i = 0; i < nlists; i++) { - SigMatch *sm = s->init_data->smlists[i]; - while (sm != NULL) { - SigMatch *nsm = sm->next; - SigMatchFree(sm); - sm = nsm; - } - } - SCFree(s->init_data->smlists); - SCFree(s->init_data->smlists_tail); - SCFree(s->init_data); - s->init_data = NULL; - } - - - SCReturnInt(0); -} - -/** - * \brief Convert the signature list into the runtime match structure. - * - * \param de_ctx Pointer to the Detection Engine Context whose Signatures have - * to be processed - * - * \retval 0 On Success. - * \retval -1 On failure. - */ -int SigGroupBuild(DetectEngineCtx *de_ctx) -{ - Signature *s = de_ctx->sig_list; - - /* Assign the unique order id of signatures after sorting, - * so the IP Only engine process them in order too. Also - * reset the old signums and assign new signums. We would - * have experienced Sig reordering by now, hence the new - * signums. */ - de_ctx->signum = 0; - while (s != NULL) { - s->num = de_ctx->signum++; - - s = s->next; - } - - if (DetectSetFastPatternAndItsId(de_ctx) < 0) - return -1; - - SigInitStandardMpmFactoryContexts(de_ctx); - - if (SigAddressPrepareStage1(de_ctx) != 0) { - SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); - exit(EXIT_FAILURE); - } - - if (SigAddressPrepareStage2(de_ctx) != 0) { - SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); - exit(EXIT_FAILURE); - } - - if (SigAddressPrepareStage3(de_ctx) != 0) { - SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); - exit(EXIT_FAILURE); - } - if (SigAddressPrepareStage4(de_ctx) != 0) { - SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); - exit(EXIT_FAILURE); - } - -#ifdef __SC_CUDA_SUPPORT__ - if (de_ctx->sgh_mpm_context == ENGINE_SGH_MPM_FACTORY_CONTEXT_SINGLE) { - if (PatternMatchDefaultMatcher() == MPM_AC_CUDA) { - /* setting it to default. You've gotta remove it once you fix the state table thing */ - SCACConstructBoth16and32StateTables(); - - MpmCudaConf *conf = CudaHandlerGetCudaProfile("mpm"); - CUcontext cuda_context = CudaHandlerModuleGetContext(MPM_AC_CUDA_MODULE_NAME, conf->device_id); - if (cuda_context == 0) { - SCLogError(SC_ERR_FATAL, "cuda context is NULL."); - exit(EXIT_FAILURE); - } - int r = SCCudaCtxPushCurrent(cuda_context); - if (r < 0) { - SCLogError(SC_ERR_FATAL, "context push failed."); - exit(EXIT_FAILURE); - } - } - - if (PatternMatchDefaultMatcher() == MPM_AC_CUDA) { - int r = SCCudaCtxPopCurrent(NULL); - if (r < 0) { - SCLogError(SC_ERR_FATAL, "cuda context pop failure."); - exit(EXIT_FAILURE); - } - } - - /* too late to call this either ways. Should be called post ac goto. - * \todo Support this. */ - DetermineCudaStateTableSize(de_ctx); - } -#endif - - int r = DetectMpmPrepareBuiltinMpms(de_ctx); - r |= DetectMpmPrepareAppMpms(de_ctx); - if (r != 0) { - SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); - exit(EXIT_FAILURE); - } - - if (SigMatchPrepare(de_ctx) != 0) { - SCLogError(SC_ERR_DETECT_PREPARE, "initializing the detection engine failed"); - exit(EXIT_FAILURE); - } - -#ifdef PROFILING - SCProfilingRuleInitCounters(de_ctx); -#endif - SCFree(de_ctx->app_mpms); - de_ctx->app_mpms = NULL; - - if (!DetectEngineMultiTenantEnabled()) { - VarNameStoreActivateStaging(); - } - return 0; -} - -int SigGroupCleanup (DetectEngineCtx *de_ctx) -{ - SigAddressCleanupStage1(de_ctx); - - return 0; -} - static void PrintFeatureList(const SigTableElmt *e, char sep) { const uint8_t flags = e->flags; diff --git a/src/detect.h b/src/detect.h index b123fc1722..fcb871788a 100644 --- a/src/detect.h +++ b/src/detect.h @@ -24,7 +24,7 @@ #ifndef __DETECT_H__ #define __DETECT_H__ -#include +#include "suricata-common.h" #include "flow.h" @@ -1383,11 +1383,6 @@ enum { SigTableElmt sigmatch_table[DETECT_TBLSIZE]; /* detection api */ -int SigAddressPrepareStage1(DetectEngineCtx *de_ctx); -int SigAddressPrepareStage2(DetectEngineCtx *de_ctx); -int SigAddressPrepareStage3(DetectEngineCtx *de_ctx); -int SigAddressPrepareStage4(DetectEngineCtx *de_ctx); -int SigAddressCleanupStage1(DetectEngineCtx *de_ctx); TmEcode Detect(ThreadVars *tv, Packet *p, void *data, PacketQueue *pq, PacketQueue *postpq); SigMatch *SigMatchAlloc(void); @@ -1396,15 +1391,12 @@ void SigMatchSignaturesBuildMatchArray(DetectEngineThreadCtx *, Packet *, SignatureMask, uint16_t); void SigMatchFree(SigMatch *sm); -void SigCleanSignatures(DetectEngineCtx *); void SigTableRegisterTests(void); void SigRegisterTests(void); void DetectSimdRegisterTests(void); void TmModuleDetectRegister (void); -int SigGroupBuild(DetectEngineCtx *); -int SigGroupCleanup (DetectEngineCtx *de_ctx); void SigAddressPrepareBidirectionals (DetectEngineCtx *); void DisableDetectFlowFileFlags(Flow *f); @@ -1420,12 +1412,6 @@ const SigGroupHead *SigMatchSignaturesGetSgh(const DetectEngineCtx *de_ctx, cons Signature *DetectGetTagSignature(void); -int SignatureIsFilestoring(const Signature *); -int SignatureIsFilemagicInspecting(const Signature *); -int SignatureIsFileMd5Inspecting(const Signature *); -int SignatureIsFileSha1Inspecting(const Signature *s); -int SignatureIsFileSha256Inspecting(const Signature *s); -int SignatureIsFilesizeInspecting(const Signature *); int DetectRegisterThreadCtxFuncs(DetectEngineCtx *, const char *name, void *(*InitFunc)(void *), void *data, void (*FreeFunc)(void *), int); void *DetectThreadCtxGetKeywordThreadCtx(DetectEngineThreadCtx *, int); @@ -1435,5 +1421,7 @@ int SigMatchSignaturesRunPostMatch(ThreadVars *tv, const Signature *s); void DetectSignatureApplyActions(Packet *p, const Signature *s, const uint8_t); +#include "detect-engine-build.h" + #endif /* __DETECT_H__ */