From: Victor Julien Date: Tue, 31 Oct 2017 16:48:18 +0000 (+0100) Subject: detect/prefilter: redo profiling X-Git-Tag: suricata-4.1.0-beta1~310 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cf2feeecf430bd7b5f3d2ec1fcf4d7e98981d692;p=thirdparty%2Fsuricata.git detect/prefilter: redo profiling --- diff --git a/src/Makefile.am b/src/Makefile.am index f322120868..ab325a63e2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -442,10 +442,11 @@ util-prefilter.c util-prefilter.h \ util-print.c util-print.h \ util-privs.c util-privs.h \ util-profiling.c util-profiling.h \ -util-profiling-locks.c util-profiling-locks.h \ -util-profiling-rules.c \ util-profiling-keywords.c \ +util-profiling-locks.c util-profiling-locks.h \ +util-profiling-prefilter.c \ util-profiling-rulegroups.c \ +util-profiling-rules.c \ util-proto-name.c util-proto-name.h \ util-radix-tree.c util-radix-tree.h \ util-random.c util-random.h \ diff --git a/src/decode.h b/src/decode.h index 0f0c6020c0..2686f7d934 100644 --- a/src/decode.h +++ b/src/decode.h @@ -374,7 +374,6 @@ typedef struct PktProfiling_ { PktProfilingAppData app[ALPROTO_MAX]; PktProfilingDetectData detect[PROF_DETECT_SIZE]; PktProfilingLoggerData logger[LOGGER_SIZE]; - PktProfilingPrefilterData prefilter; uint64_t proto_detect; } PktProfiling; diff --git a/src/detect-engine-build.c b/src/detect-engine-build.c index f035325a23..7f1a2a63dc 100644 --- a/src/detect-engine-build.c +++ b/src/detect-engine-build.c @@ -1905,6 +1905,7 @@ int SigGroupBuild(DetectEngineCtx *de_ctx) #ifdef PROFILING SCProfilingKeywordInitCounters(de_ctx); + SCProfilingPrefilterInitCounters(de_ctx); de_ctx->profile_match_logging_threshold = UINT_MAX; // disabled intmax_t v = 0; diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c index 11461b33cf..db18af7159 100644 --- a/src/detect-engine-prefilter.c +++ b/src/detect-engine-prefilter.c @@ -55,12 +55,6 @@ #include "util-profiling.h" -typedef struct PrefilterStore_ { - const char *name; - void (*FreeFunc)(void *); - uint32_t id; -} PrefilterStore; - static int PrefilterStoreGetId(const char *name, void (*FreeFunc)(void *)); static const PrefilterStore *PrefilterStoreGetStore(const uint32_t id); @@ -117,10 +111,10 @@ void DetectRunPrefilterTx(DetectEngineThreadCtx *det_ctx, } } - PROFILING_PREFILTER_START(p); + PREFILTER_PROFILING_START; engine->cb.PrefilterTx(det_ctx, engine->pectx, p, p->flow, tx->tx_ptr, tx->tx_id, flow_flags); - PROFILING_PREFILTER_END(p, engine->gid); + PREFILTER_PROFILING_END(det_ctx, engine->gid); if (tx->tx_progress > engine->tx_min_progress) { tx->prefilter_flags |= (1<<(engine->local_id)); @@ -145,16 +139,14 @@ void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, { SCEnter(); - PROFILING_PREFILTER_RESET(p, det_ctx->de_ctx->prefilter_maxid); - if (sgh->pkt_engines) { PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_PKT); /* run packet engines */ PrefilterEngine *engine = sgh->pkt_engines; do { - PROFILING_PREFILTER_START(p); + PREFILTER_PROFILING_START; engine->cb.Prefilter(det_ctx, p, engine->pectx); - PROFILING_PREFILTER_END(p, engine->gid); + PREFILTER_PROFILING_END(det_ctx, engine->gid); if (engine->is_last) break; @@ -171,9 +163,9 @@ void Prefilter(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, PACKET_PROFILING_DETECT_START(p, PROF_DETECT_PF_PAYLOAD); PrefilterEngine *engine = sgh->payload_engines; while (1) { - PROFILING_PREFILTER_START(p); + PREFILTER_PROFILING_START; engine->cb.Prefilter(det_ctx, p, engine->pectx); - PROFILING_PREFILTER_END(p, engine->gid); + PREFILTER_PROFILING_END(det_ctx, engine->gid); if (engine->is_last) break; @@ -479,8 +471,8 @@ static void PrefilterStoreFreeFunc(void *ptr) } static SCMutex g_prefilter_mutex = SCMUTEX_INITIALIZER; -static uint32_t g_prefilter_id = 0; -static HashListTable *g_prefilter_hash_table = NULL; +uint32_t g_prefilter_id = 0; +HashListTable *g_prefilter_hash_table = NULL; static void PrefilterDeinit(void) { diff --git a/src/detect-engine-prefilter.h b/src/detect-engine-prefilter.h index 1a5f03b819..6da302141d 100644 --- a/src/detect-engine-prefilter.h +++ b/src/detect-engine-prefilter.h @@ -26,6 +26,12 @@ #include "detect-engine-state.h" +typedef struct PrefilterStore_ { + const char *name; + void (*FreeFunc)(void *); + uint32_t id; +} PrefilterStore; + void Prefilter(DetectEngineThreadCtx *, const SigGroupHead *, Packet *p, const uint8_t flags); diff --git a/src/detect-engine.c b/src/detect-engine.c index f0d7dc9498..83dec937f8 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -1154,6 +1154,7 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx) if (de_ctx->profile_sgh_ctx != NULL) { SCProfilingSghDestroyCtx(de_ctx); } + SCProfilingPrefilterDestroyCtx(de_ctx); #endif /* Normally the hashes are freed elsewhere, but @@ -1785,6 +1786,7 @@ static TmEcode ThreadCtxDoInit (DetectEngineCtx *de_ctx, DetectEngineThreadCtx * #ifdef PROFILING SCProfilingRuleThreadSetup(de_ctx->profile_ctx, det_ctx); SCProfilingKeywordThreadSetup(de_ctx->profile_keyword_ctx, det_ctx); + SCProfilingPrefilterThreadSetup(de_ctx->profile_prefilter_ctx, det_ctx); SCProfilingSghThreadSetup(de_ctx->profile_sgh_ctx, det_ctx); #endif SC_ATOMIC_INIT(det_ctx->so_far_used_by_detect); @@ -1944,6 +1946,7 @@ static void DetectEngineThreadCtxFree(DetectEngineThreadCtx *det_ctx) #ifdef PROFILING SCProfilingRuleThreadCleanup(det_ctx); SCProfilingKeywordThreadCleanup(det_ctx); + SCProfilingPrefilterThreadCleanup(det_ctx); SCProfilingSghThreadCleanup(det_ctx); #endif diff --git a/src/detect.h b/src/detect.h index a66edf7a51..f4bf189da5 100644 --- a/src/detect.h +++ b/src/detect.h @@ -721,6 +721,7 @@ typedef struct DetectEngineCtx_ { #ifdef PROFILING struct SCProfileDetectCtx_ *profile_ctx; struct SCProfileKeywordDetectCtx_ *profile_keyword_ctx; + struct SCProfilePrefilterDetectCtx_ *profile_prefilter_ctx; struct SCProfileKeywordDetectCtx_ **profile_keyword_ctx_per_list; struct SCProfileSghDetectCtx_ *profile_sgh_ctx; uint32_t profile_match_logging_threshold; @@ -972,6 +973,9 @@ typedef struct DetectEngineThreadCtx_ { struct SCProfileKeywordData_ **keyword_perf_data_per_list; int keyword_perf_list; /**< list we're currently inspecting, DETECT_SM_LIST_* */ struct SCProfileSghData_ *sgh_perf_data; + + struct SCProfilePrefilterData_ *prefilter_perf_data; + int prefilter_perf_size; #endif } DetectEngineThreadCtx; diff --git a/src/suricata.c b/src/suricata.c index 0164f6f919..d78c71a2c8 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2199,6 +2199,7 @@ void PreRunInit(const int runmode) #ifdef PROFILING SCProfilingRulesGlobalInit(); SCProfilingKeywordsGlobalInit(); + SCProfilingPrefilterGlobalInit(); SCProfilingSghsGlobalInit(); SCProfilingInit(); #endif /* PROFILING */ diff --git a/src/util-profiling-prefilter.c b/src/util-profiling-prefilter.c new file mode 100644 index 0000000000..baac3edd0e --- /dev/null +++ b/src/util-profiling-prefilter.c @@ -0,0 +1,311 @@ +/* Copyright (C) 2007-2017 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * \file + * + * \author Endace Technology Limited. + * \author Victor Julien + * + * An API for rule profiling operations. + */ + +#include "suricata-common.h" +#include "decode.h" +#include "detect.h" +#include "detect-engine.h" +#include "detect-engine-prefilter.h" +#include "conf.h" + +#include "tm-threads.h" + +#include "util-unittest.h" +#include "util-byte.h" +#include "util-profiling.h" +#include "util-profiling-locks.h" + +#ifdef PROFILING + +extern uint32_t g_prefilter_id; +extern HashListTable *g_prefilter_hash_table; + +typedef struct SCProfilePrefilterData_ { + uint64_t called; + uint64_t total; + uint64_t max; + const char *name; +} SCProfilePrefilterData; + +typedef struct SCProfilePrefilterDetectCtx_ { + uint32_t id; + uint32_t size; /**< size in elements */ + SCProfilePrefilterData *data; + pthread_mutex_t data_m; +} SCProfilePrefilterDetectCtx; + +static int profiling_prefilter_output_to_file = 0; +int profiling_prefilter_enabled = 0; +__thread int profiling_prefilter_entered = 0; +static char profiling_file_name[PATH_MAX]; +static const char *profiling_file_mode = "a"; + +void SCProfilingPrefilterGlobalInit(void) +{ + ConfNode *conf; + + conf = ConfGetNode("profiling.prefilter"); + if (conf != NULL) { + if (ConfNodeChildValueIsTrue(conf, "enabled")) { + profiling_prefilter_enabled = 1; + const char *filename = ConfNodeLookupChildValue(conf, "filename"); + if (filename != NULL) { + const char *log_dir; + log_dir = ConfigGetLogDirectory(); + + snprintf(profiling_file_name, sizeof(profiling_file_name), "%s/%s", + log_dir, filename); + + const char *v = ConfNodeLookupChildValue(conf, "append"); + if (v == NULL || ConfValIsTrue(v)) { + profiling_file_mode = "a"; + } else { + profiling_file_mode = "w"; + } + + profiling_prefilter_output_to_file = 1; + } + } + } +} + +static void DoDump(SCProfilePrefilterDetectCtx *rules_ctx, FILE *fp, const char *name) +{ + int i; + fprintf(fp, " ----------------------------------------------" + "------------------------------------------------------" + "----------------------------\n"); + fprintf(fp, " Stats for: %s\n", name); + fprintf(fp, " ----------------------------------------------" + "------------------------------------------------------" + "----------------------------\n"); + fprintf(fp, " %-32s %-15s %-15s %-15s %-15s\n", "Prefilter", "Ticks", "Called", "Max Ticks", "Avg"); + fprintf(fp, " -------------------------------- " + "--------------- " + "--------------- " + "--------------- " + "--------------- " + "\n"); + for (i = 0; i < (int)rules_ctx->size; i++) { + SCProfilePrefilterData *d = &rules_ctx->data[i]; + if (d == NULL || d->called== 0) + continue; + + uint64_t ticks = d->total; + double avgticks = 0; + if (ticks && d->called) { + avgticks = (ticks / d->called); + } + + fprintf(fp, + " %-32s %-15"PRIu64" %-15"PRIu64" %-15"PRIu64" %-15.2f\n", + d->name, + ticks, + d->called, + d->max, + avgticks); + } +} + +static void +SCProfilingPrefilterDump(DetectEngineCtx *de_ctx) +{ + FILE *fp; + struct timeval tval; + struct tm *tms; + struct tm local_tm; + + if (profiling_prefilter_enabled == 0 || de_ctx->profile_prefilter_ctx == NULL) + return; + + gettimeofday(&tval, NULL); + tms = SCLocalTime(tval.tv_sec, &local_tm); + + if (profiling_prefilter_output_to_file == 1) { + SCLogDebug("file %s mode %s", profiling_file_name, profiling_file_mode); + + fp = fopen(profiling_file_name, profiling_file_mode); + + if (fp == NULL) { + SCLogError(SC_ERR_FOPEN, "failed to open %s: %s", profiling_file_name, + strerror(errno)); + return; + } + } else { + fp = stdout; + } + + fprintf(fp, " ----------------------------------------------" + "------------------------------------------------------" + "----------------------------\n"); + fprintf(fp, " Date: %" PRId32 "/%" PRId32 "/%04d -- " + "%02d:%02d:%02d\n", tms->tm_mon + 1, tms->tm_mday, tms->tm_year + 1900, + tms->tm_hour,tms->tm_min, tms->tm_sec); + + /* global stats first */ + DoDump(de_ctx->profile_prefilter_ctx, fp, "total"); + + fprintf(fp,"\n"); + if (fp != stdout) + fclose(fp); + + SCLogPerf("Done dumping prefilter profiling data."); +} + +/** + * \brief Update a rule counter. + * + * \param id The ID of this counter. + * \param ticks Number of CPU ticks for this rule. + * \param match Did the rule match? + */ +void +SCProfilingPrefilterUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks) +{ + if (det_ctx != NULL && det_ctx->prefilter_perf_data != NULL && + id < (int)g_prefilter_id) + { + SCProfilePrefilterData *p = &det_ctx->prefilter_perf_data[id]; + + p->called++; + if (ticks > p->max) + p->max = ticks; + p->total += ticks; + } +} + +static SCProfilePrefilterDetectCtx *SCProfilingPrefilterInitCtx(void) +{ + SCProfilePrefilterDetectCtx *ctx = SCMalloc(sizeof(SCProfilePrefilterDetectCtx)); + if (ctx != NULL) { + memset(ctx, 0x00, sizeof(SCProfilePrefilterDetectCtx)); + + if (pthread_mutex_init(&ctx->data_m, NULL) != 0) { + SCLogError(SC_ERR_MUTEX, + "Failed to initialize hash table mutex."); + exit(EXIT_FAILURE); + } + } + + return ctx; +} + +static void DetroyCtx(SCProfilePrefilterDetectCtx *ctx) +{ + if (ctx) { + if (ctx->data != NULL) + SCFree(ctx->data); + pthread_mutex_destroy(&ctx->data_m); + SCFree(ctx); + } +} + +void SCProfilingPrefilterDestroyCtx(DetectEngineCtx *de_ctx) +{ + if (de_ctx != NULL) { + SCProfilingPrefilterDump(de_ctx); + + DetroyCtx(de_ctx->profile_prefilter_ctx); + } +} + +void SCProfilingPrefilterThreadSetup(SCProfilePrefilterDetectCtx *ctx, DetectEngineThreadCtx *det_ctx) +{ + if (ctx == NULL) + return; + + const uint32_t size = g_prefilter_id; + + SCProfilePrefilterData *a = SCMalloc(sizeof(SCProfilePrefilterData) * size); + if (a != NULL) { + memset(a, 0x00, sizeof(SCProfilePrefilterData) * size); + det_ctx->prefilter_perf_data = a; + } +} + +static void SCProfilingPrefilterThreadMerge(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx) +{ + if (de_ctx == NULL || de_ctx->profile_prefilter_ctx == NULL || + de_ctx->profile_prefilter_ctx->data == NULL || det_ctx == NULL || + det_ctx->prefilter_perf_data == NULL) + return; + + for (uint32_t i = 0; i < g_prefilter_id; i++) { + de_ctx->profile_prefilter_ctx->data[i].called += det_ctx->prefilter_perf_data[i].called; + de_ctx->profile_prefilter_ctx->data[i].total += det_ctx->prefilter_perf_data[i].total; + if (det_ctx->prefilter_perf_data[i].max > de_ctx->profile_prefilter_ctx->data[i].max) + de_ctx->profile_prefilter_ctx->data[i].max = det_ctx->prefilter_perf_data[i].max; + } +} + +void SCProfilingPrefilterThreadCleanup(DetectEngineThreadCtx *det_ctx) +{ + if (det_ctx == NULL || det_ctx->de_ctx == NULL || det_ctx->prefilter_perf_data == NULL) + return; + + pthread_mutex_lock(&det_ctx->de_ctx->profile_prefilter_ctx->data_m); + SCProfilingPrefilterThreadMerge(det_ctx->de_ctx, det_ctx); + pthread_mutex_unlock(&det_ctx->de_ctx->profile_prefilter_ctx->data_m); + + SCFree(det_ctx->prefilter_perf_data); + det_ctx->prefilter_perf_data = NULL; +} + +/** + * \brief Register the prefilter profiling counters. + * + * \param de_ctx The active DetectEngineCtx, used to get at the loaded rules. + */ +void +SCProfilingPrefilterInitCounters(DetectEngineCtx *de_ctx) +{ + if (profiling_prefilter_enabled == 0) + return; + + const uint32_t size = g_prefilter_id; + if (g_prefilter_id == 0) + return; + + de_ctx->profile_prefilter_ctx = SCProfilingPrefilterInitCtx(); + BUG_ON(de_ctx->profile_prefilter_ctx == NULL); + de_ctx->profile_prefilter_ctx->size = size; + + de_ctx->profile_prefilter_ctx->data = SCMalloc(sizeof(SCProfilePrefilterData) * size); + BUG_ON(de_ctx->profile_prefilter_ctx->data == NULL); + memset(de_ctx->profile_prefilter_ctx->data, 0x00, sizeof(SCProfilePrefilterData) * size); + + HashListTableBucket *hb = HashListTableGetListHead(g_prefilter_hash_table); + for ( ; hb != NULL; hb = HashListTableGetListNext(hb)) { + PrefilterStore *ctx = HashListTableGetListData(hb); + de_ctx->profile_prefilter_ctx->data[ctx->id].name = ctx->name; + SCLogNotice("prefilter %s set up", de_ctx->profile_prefilter_ctx->data[ctx->id].name); + } + SCLogNotice("size alloc'd %u", (uint32_t)size * (uint32_t)sizeof(SCProfilePrefilterData)); + + SCLogPerf("Registered %"PRIu32" prefilter profiling counters.", size); +} + +#endif /* PROFILING */ diff --git a/src/util-profiling.c b/src/util-profiling.c index b7a74c5f21..e50cb6feb2 100644 --- a/src/util-profiling.c +++ b/src/util-profiling.c @@ -95,9 +95,6 @@ struct ProfileProtoRecords { SCProfilePacketData records4[257]; SCProfilePacketData records6[257]; }; -static SCProfilePacketData prefilter4[256][256]; -static SCProfilePacketData prefilter6[256][256]; - struct ProfileProtoRecords packet_profile_flowworker_data[PROFILE_FLOWWORKER_SIZE]; @@ -178,8 +175,6 @@ SCProfilingInit(void) memset(&packet_profile_log_data4, 0, sizeof(packet_profile_log_data4)); memset(&packet_profile_log_data6, 0, sizeof(packet_profile_log_data6)); memset(&packet_profile_flowworker_data, 0, sizeof(packet_profile_flowworker_data)); - memset(&prefilter4, 0, sizeof(prefilter4)); - memset(&prefilter6, 0, sizeof(prefilter6)); const char *filename = ConfNodeLookupChildValue(conf, "filename"); if (filename != NULL) { @@ -316,96 +311,6 @@ SCProfilingDump(void) SCLogPerf("Done dumping profiling data."); } -static void DumpPrefilterIP(FILE *fp, int ipv, uint64_t total) -{ - char totalstr[256]; - - SCProfilePacketData total_pd; - memset(&total_pd, 0, sizeof(total_pd)); - - int i; - for (i = 0; i < 256; i++) { - const char *name = PrefilterStoreGetName(i); - - for (int p = 0; p < 256; p++) { - SCProfilePacketData *pd = ipv == 4 ? &prefilter4[i][p] : &prefilter6[i][p]; - if (pd->cnt == 0) { - continue; - } - - total_pd.cnt += pd->cnt; - total_pd.tot += pd->tot; - - FormatNumber(pd->tot, totalstr, sizeof(totalstr)); - double percent = (long double)pd->tot / - (long double)total * 100; - - fprintf(fp, "%-30s IPv%d %3d %12"PRIu64" %12"PRIu64" %12"PRIu64" %12"PRIu64" %11s %-6.2f\n", - name, ipv, p, pd->cnt, - pd->min, pd->max, (uint64_t)(pd->tot / pd->cnt), totalstr, percent); - } - } - if (total_pd.cnt) { - FormatNumber(total_pd.tot, totalstr, sizeof(totalstr)); - fprintf(fp, "%-30s IPv%d %12"PRIu64" %12"PRIu64" %11s\n", - "Total", ipv, total_pd.cnt, (uint64_t)(total_pd.tot / total_pd.cnt), totalstr); - } -} - -static void DumpPrefilter(FILE *fp) -{ - uint64_t total = 0; - - int i; - for (i = 0; i < 256; i++) { - for (int p = 0; p < 256; p++) { - SCProfilePacketData *pd = &prefilter4[i][p]; - total += pd->tot; - pd = &prefilter6[i][p]; - total += pd->tot; - } - } - - fprintf(fp, "\n%-30s %-6s %-5s %-12s %-12s %-12s %-12s %-11s %-3s\n", - "Prefilter", "IP ver", "Proto", "cnt", "min", "max", "avg", "tot", "%%"); - fprintf(fp, "%-30s %-6s %-5s %-12s %-12s %-12s %-12s %-11s %-3s\n", - "--------------------", "------", "-----", "----------", - "------------", "------------", "-----------", "---------", "---"); - DumpPrefilterIP(fp, 4, total); - DumpPrefilterIP(fp, 6, total); -} - -static void SCProfilingUpdatePrefilterRecords(Packet *p) -{ - if (p->profile->prefilter.engines != NULL) { - uint32_t x; - for (x = 0; x < p->profile->prefilter.size; x++) { - uint64_t ticks = p->profile->prefilter.engines[x].ticks_spent; - if (ticks == 0) - continue; - - SCProfilePacketData *pd = NULL; - if (PKT_IS_IPV4(p)) { - pd = &prefilter4[x][p->proto]; - } else if (PKT_IS_IPV6(p)) { - pd = &prefilter6[x][p->proto]; - } else { - continue; - } - - if (pd->min == 0 || ticks < pd->min) { - pd->min = ticks; - } - if (pd->max < ticks) { - pd->max = ticks; - } - - pd->tot += ticks; - pd->cnt ++; - } - } -} - static void DumpFlowWorkerIP(FILE *fp, int ipv, uint64_t total) { char totalstr[256]; @@ -832,8 +737,6 @@ void SCProfilingDumpPacketStats(void) } } - DumpPrefilter(fp); - fprintf(fp, "\nGeneral detection engine stats:\n"); total = 0; @@ -1255,7 +1158,6 @@ void SCProfilingAddPacket(Packet *p) SCProfilingUpdatePacketLogRecords(p); } - SCProfilingUpdatePrefilterRecords(p); if (profiling_packets_csv_enabled) SCProfilingPrintPacketProfile(p); diff --git a/src/util-profiling.h b/src/util-profiling.h index c64f93846d..6ecdbdc145 100644 --- a/src/util-profiling.h +++ b/src/util-profiling.h @@ -179,7 +179,6 @@ PktProfiling *SCProfilePacketStart(void); #define PACKET_PROFILING_RESET(p) \ if (profiling_packets_enabled && (p)->profile != NULL) { \ - SCFree((p)->profile->prefilter.engines); \ SCFree((p)->profile); \ (p)->profile = NULL; \ } @@ -275,35 +274,30 @@ PktProfiling *SCProfilePacketStart(void); SCProfilingSghUpdateCounter((det_ctx), (sgh)); \ } -#define PROFILING_PREFILTER_RESET(p, detectsize) \ - if (profiling_packets_enabled && (p)->profile != NULL) { \ - if ((p)->profile->prefilter.size != ((detectsize) + 1)) { \ - if ((p)->profile->prefilter.engines != NULL) \ - SCFree((p)->profile->prefilter.engines); \ - (p)->profile->prefilter.engines = \ - SCCalloc((detectsize)+1, sizeof(PktProfilingPrefilterEngine)); \ - (p)->profile->prefilter.size = (detectsize)+1;\ - } else { \ - memset((p)->profile->prefilter.engines, 0x00, \ - ((detectsize)+1 * sizeof(PktProfilingPrefilterEngine))); \ - } \ - } \ +extern int profiling_prefilter_enabled; +extern __thread int profiling_prefilter_entered; -#define PROFILING_PREFILTER_START(p) \ - uint64_t ticks_start = 0; \ - if (profiling_packets_enabled && (p)->profile != NULL) { \ - ticks_start = UtilCpuGetTicks(); \ - } \ - -#define PROFILING_PREFILTER_END(p, profile_id) \ - if (profiling_packets_enabled && (p)->profile != NULL && \ - ticks_start) \ - { \ - uint64_t ticks_end = UtilCpuGetTicks(); \ - if (ticks_end > ticks_start) \ - (p)->profile->prefilter.engines[(profile_id)].ticks_spent += (ticks_end - ticks_start); \ - ticks_start = 0; \ - } \ +#define PREFILTER_PROFILING_START \ + uint64_t profile_prefilter_start_ = 0; \ + uint64_t profile_prefilter_end_ = 0; \ + if (profiling_prefilter_enabled) { \ + if (profiling_prefilter_entered > 0) { \ + SCLogError(SC_ERR_FATAL, "Re-entered profiling, exiting."); \ + abort(); \ + } \ + profiling_prefilter_entered++; \ + profile_prefilter_start_ = UtilCpuGetTicks(); \ + } + +/* we allow this macro to be called if profiling_prefilter_entered == 0, + * so that we don't have to refactor some of the detection code. */ +#define PREFILTER_PROFILING_END(ctx, profile_id) \ + if (profiling_prefilter_enabled && profiling_prefilter_entered) { \ + profile_prefilter_end_ = UtilCpuGetTicks(); \ + if (profile_prefilter_end_ > profile_prefilter_start_) \ + SCProfilingPrefilterUpdateCounter((ctx),(profile_id),(profile_prefilter_end_ - profile_prefilter_start_)); \ + profiling_prefilter_entered--; \ + } void SCProfilingRulesGlobalInit(void); void SCProfilingRuleDestroyCtx(struct SCProfileDetectCtx_ *); @@ -319,6 +313,14 @@ void SCProfilingKeywordUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uin void SCProfilingKeywordThreadSetup(struct SCProfileKeywordDetectCtx_ *, DetectEngineThreadCtx *); void SCProfilingKeywordThreadCleanup(DetectEngineThreadCtx *); +struct SCProfilePrefilterDetectCtx_; +void SCProfilingPrefilterGlobalInit(void); +void SCProfilingPrefilterDestroyCtx(DetectEngineCtx *); +void SCProfilingPrefilterInitCounters(DetectEngineCtx *); +void SCProfilingPrefilterUpdateCounter(DetectEngineThreadCtx *det_ctx, int id, uint64_t ticks); +void SCProfilingPrefilterThreadSetup(struct SCProfilePrefilterDetectCtx_ *, DetectEngineThreadCtx *); +void SCProfilingPrefilterThreadCleanup(DetectEngineThreadCtx *); + void SCProfilingSghsGlobalInit(void); void SCProfilingSghDestroyCtx(DetectEngineCtx *); void SCProfilingSghInitCounters(DetectEngineCtx *); @@ -368,9 +370,8 @@ void SCProfilingDump(void); #define FLOWWORKER_PROFILING_START(p, id) #define FLOWWORKER_PROFILING_END(p, id) -#define PROFILING_PREFILTER_RESET(p, detectsize) -#define PROFILING_PREFILTER_START(p) -#define PROFILING_PREFILTER_END(p, profile_id) +#define PREFILTER_PROFILING_START +#define PREFILTER_PROFILING_END(ctx, profile_id) #endif /* PROFILING */ diff --git a/suricata.yaml.in b/suricata.yaml.in index 83a4f0d8c5..6294bec45c 100644 --- a/suricata.yaml.in +++ b/suricata.yaml.in @@ -1500,6 +1500,11 @@ profiling: filename: keyword_perf.log append: yes + prefilter: + enabled: yes + filename: prefilter_perf.log + append: yes + # per rulegroup profiling rulegroups: enabled: yes