From: Victor Julien Date: Mon, 10 Apr 2017 18:42:25 +0000 (+0200) Subject: mpm: run engines as few times as possible X-Git-Tag: suricata-4.0.0-beta1~126 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a0fad6bb7fd57080f35ac500cf623cc21bcb2f92;p=thirdparty%2Fsuricata.git mpm: run engines as few times as possible In various scenarios buffers would be checked my MPM more than once. This was because the buffers would be inspected for a certain progress value or higher. For example, for each packet in a file upload, the engine would not just rerun the 'http client body' MPM on the new data, it would also rerun the method, uri, headers, cookie, etc MPMs. This was obviously inefficent, so this patch changes the logic. The patch only runs the MPM engines when the progress is exactly the intended progress. If the progress is beyond the desired value, it is run once. A tracker is added to the app layer API, where the completed MPMs are tracked. Implemented for HTTP, TLS and SSH. --- diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index 3588a9e88b..18662870a7 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -2718,6 +2718,28 @@ static int HTPSetTxDetectState(void *alstate, void *vtx, DetectEngineState *s) return 0; } +static uint64_t HTPGetTxMpmIDs(void *vtx) +{ + htp_tx_t *tx = (htp_tx_t *)vtx; + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + return tx_ud ? tx_ud->mpm_ids : 0; +} + +static int HTPSetTxMpmIDs(void *vtx, uint64_t mpm_ids) +{ + htp_tx_t *tx = (htp_tx_t *)vtx; + HtpTxUserData *tx_ud = htp_tx_get_user_data(tx); + if (tx_ud == NULL) { + tx_ud = HTPMalloc(sizeof(*tx_ud)); + if (unlikely(tx_ud == NULL)) + return -ENOMEM; + memset(tx_ud, 0, sizeof(*tx_ud)); + htp_tx_set_user_data(tx, tx_ud); + } + tx_ud->mpm_ids = mpm_ids; + return 0; +} + static int HTPRegisterPatternsForProtocolDetection(void) { char *methods[] = { "GET", "PUT", "POST", "HEAD", "TRACE", "OPTIONS", @@ -2806,6 +2828,8 @@ void RegisterHTPParsers(void) AppLayerParserRegisterDetectStateFuncs(IPPROTO_TCP, ALPROTO_HTTP, HTPStateHasTxDetectState, HTPGetTxDetectState, HTPSetTxDetectState); + AppLayerParserRegisterMpmIDsFuncs(IPPROTO_TCP, ALPROTO_HTTP, + HTPGetTxMpmIDs, HTPSetTxMpmIDs); AppLayerParserRegisterParser(IPPROTO_TCP, ALPROTO_HTTP, STREAM_TOSERVER, HTPHandleRequestData); diff --git a/src/app-layer-htp.h b/src/app-layer-htp.h index 0e3503c3e8..b8f4f29b70 100644 --- a/src/app-layer-htp.h +++ b/src/app-layer-htp.h @@ -188,6 +188,9 @@ typedef struct HtpBody_ { /** Now the Body Chunks will be stored per transaction, at * the tx user data */ typedef struct HtpTxUserData_ { + /** flags to track which mpm has run */ + uint64_t mpm_ids; + /* Body of the request (if any) */ uint8_t request_body_init; uint8_t response_body_init; @@ -228,7 +231,6 @@ typedef struct HtpTxUserData_ { } HtpTxUserData; typedef struct HtpState_ { - /* Connection parser structure for each connection */ htp_connp_t *connp; /* Connection structure for each connection */ diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index e69997d71f..44a03368cb 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -116,6 +116,9 @@ typedef struct AppLayerParserProtoCtx_ DetectEngineState *(*GetTxDetectState)(void *tx); int (*SetTxDetectState)(void *alstate, void *tx, DetectEngineState *); + uint64_t (*GetTxMpmIDs)(void *tx); + int (*SetTxMpmIDs)(void *tx, uint64_t); + /* each app-layer has its own value */ uint32_t stream_depth; @@ -537,6 +540,18 @@ void AppLayerParserRegisterDetectStateFuncs(uint8_t ipproto, AppProto alproto, SCReturn; } +void AppLayerParserRegisterMpmIDsFuncs(uint8_t ipproto, AppProto alproto, + uint64_t(*GetTxMpmIDs)(void *tx), + int (*SetTxMpmIDs)(void *tx, uint64_t)) +{ + SCEnter(); + + alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].GetTxMpmIDs = GetTxMpmIDs; + alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].SetTxMpmIDs = SetTxMpmIDs; + + SCReturn; +} + /***** Get and transaction functions *****/ void *AppLayerParserGetProtocolParserLocalStorage(uint8_t ipproto, AppProto alproto) @@ -929,6 +944,24 @@ int AppLayerParserSetTxDetectState(uint8_t ipproto, AppProto alproto, SCReturnInt(r); } +uint64_t AppLayerParserGetTxMpmIDs(uint8_t ipproto, AppProto alproto, void *tx) +{ + if (alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].GetTxMpmIDs != NULL) { + return alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].GetTxMpmIDs(tx); + } + + return 0ULL; +} + +int AppLayerParserSetTxMpmIDs(uint8_t ipproto, AppProto alproto, void *tx, uint64_t mpm_ids) +{ + int r = 0; + if (alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].SetTxMpmIDs != NULL) { + r = alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].SetTxMpmIDs(tx, mpm_ids); + } + SCReturnInt(r); +} + /***** General *****/ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *alp_tctx, Flow *f, AppProto alproto, diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index c53cf78129..e887e0008b 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -153,6 +153,9 @@ void AppLayerParserRegisterDetectStateFuncs(uint8_t ipproto, AppProto alproto, void AppLayerParserRegisterGetStreamDepth(uint8_t ipproto, AppProto alproto, uint32_t (*GetStreamDepth)(void)); +void AppLayerParserRegisterMpmIDsFuncs(uint8_t ipproto, AppProto alproto, + uint64_t (*GetTxMpmIDs)(void *tx), + int (*SetTxMpmIDs)(void *tx, uint64_t)); /***** Get and transaction functions *****/ @@ -195,6 +198,9 @@ int AppLayerParserHasTxDetectState(uint8_t ipproto, AppProto alproto, void *alst DetectEngineState *AppLayerParserGetTxDetectState(uint8_t ipproto, AppProto alproto, void *tx); int AppLayerParserSetTxDetectState(uint8_t ipproto, AppProto alproto, void *alstate, void *tx, DetectEngineState *s); +uint64_t AppLayerParserGetTxMpmIDs(uint8_t ipproto, AppProto alproto, void *tx); +int AppLayerParserSetTxMpmIDs(uint8_t ipproto, AppProto alproto, void *tx, uint64_t); + /***** General *****/ int AppLayerParserParse(ThreadVars *tv, AppLayerParserThreadCtx *tctx, Flow *f, AppProto alproto, diff --git a/src/app-layer-ssh.c b/src/app-layer-ssh.c index c4e9c118db..b7a5b2be43 100644 --- a/src/app-layer-ssh.c +++ b/src/app-layer-ssh.c @@ -557,6 +557,19 @@ static int SSHGetTxLogged(void *state, void *tx, uint32_t logger) return 0; } +static uint64_t SSHGetTxMpmIDs(void *vtx) +{ + SshState *ssh_state = (SshState *)vtx; + return ssh_state->mpm_ids; +} + +static int SSHSetTxMpmIDs(void *vtx, uint64_t mpm_ids) +{ + SshState *ssh_state = (SshState *)vtx; + ssh_state->mpm_ids = mpm_ids; + return 0; +} + static int SSHGetAlstateProgressCompletionStatus(uint8_t direction) { return SSH_STATE_FINISHED; @@ -632,6 +645,8 @@ void RegisterSSHParsers(void) AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_SSH, SSHGetAlstateProgress); AppLayerParserRegisterLoggerFuncs(IPPROTO_TCP, ALPROTO_SSH, SSHGetTxLogged, SSHSetTxLogged); + AppLayerParserRegisterMpmIDsFuncs(IPPROTO_TCP, ALPROTO_SSH, + SSHGetTxMpmIDs, SSHSetTxMpmIDs); AppLayerParserRegisterGetStateProgressCompletionStatus(ALPROTO_SSH, SSHGetAlstateProgressCompletionStatus); diff --git a/src/app-layer-ssh.h b/src/app-layer-ssh.h index 7fa368812b..d5b6d3a8a5 100644 --- a/src/app-layer-ssh.h +++ b/src/app-layer-ssh.h @@ -76,6 +76,9 @@ typedef struct SshState_ { /* specifies which loggers are done logging */ uint32_t logged; + /* bit flags of mpms that have already run */ + uint64_t mpm_ids; + DetectEngineState *de_state; } SshState; diff --git a/src/app-layer-ssl.c b/src/app-layer-ssl.c index 47c3e9ef60..0215eed77d 100644 --- a/src/app-layer-ssl.c +++ b/src/app-layer-ssl.c @@ -245,6 +245,19 @@ int SSLGetAlstateProgress(void *tx, uint8_t direction) return TLS_STATE_IN_PROGRESS; } +static uint64_t SSLGetTxMpmIDs(void *vtx) +{ + SSLState *ssl_state = (SSLState *)vtx; + return ssl_state->mpm_ids; +} + +static int SSLSetTxMpmIDs(void *vtx, uint64_t mpm_ids) +{ + SSLState *ssl_state = (SSLState *)vtx; + ssl_state->mpm_ids = mpm_ids; + return 0; +} + static int TLSDecodeHandshakeHello(SSLState *ssl_state, uint8_t *input, uint32_t input_len) { @@ -1832,6 +1845,8 @@ void RegisterSSLParsers(void) AppLayerParserRegisterGetStateProgressFunc(IPPROTO_TCP, ALPROTO_TLS, SSLGetAlstateProgress); AppLayerParserRegisterLoggerFuncs(IPPROTO_TCP, ALPROTO_TLS, SSLGetTxLogged, SSLSetTxLogged); + AppLayerParserRegisterMpmIDsFuncs(IPPROTO_TCP, ALPROTO_TLS, + SSLGetTxMpmIDs, SSLSetTxMpmIDs); AppLayerParserRegisterGetStateProgressCompletionStatus(ALPROTO_TLS, SSLGetAlstateProgressCompletionStatus); diff --git a/src/app-layer-ssl.h b/src/app-layer-ssl.h index 1748803afb..d48835cc13 100644 --- a/src/app-layer-ssl.h +++ b/src/app-layer-ssl.h @@ -191,6 +191,9 @@ typedef struct SSLState_ { /* specifies which loggers are done logging */ uint32_t logged; + /* MPM/prefilter Id's */ + uint64_t mpm_ids; + /* there might be a better place to store this*/ uint16_t hb_record_len; diff --git a/src/detect-engine-prefilter.c b/src/detect-engine-prefilter.c index 5a1580fc05..ed6ce40054 100644 --- a/src/detect-engine-prefilter.c +++ b/src/detect-engine-prefilter.c @@ -118,6 +118,7 @@ static inline void PrefilterTx(DetectEngineThreadCtx *det_ctx, if (tx == NULL) continue; + uint64_t mpm_ids = AppLayerParserGetTxMpmIDs(ipproto, alproto, tx); const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags); SCLogDebug("tx %p progress %d", tx, tx_progress); @@ -127,16 +128,30 @@ static inline void PrefilterTx(DetectEngineThreadCtx *det_ctx, goto next; if (engine->tx_min_progress > tx_progress) goto next; + if (tx_progress > engine->tx_min_progress) { + if (mpm_ids & (1<<(engine->gid))) { + goto next; + } + } PROFILING_PREFILTER_START(p); engine->cb.PrefilterTx(det_ctx, engine->pectx, p, p->flow, tx, idx, flags); PROFILING_PREFILTER_END(p, engine->gid); + + if (tx_progress > engine->tx_min_progress) { + mpm_ids |= (1<<(engine->gid)); + } next: if (engine->is_last) break; engine++; } while (1); + + if (mpm_ids != 0) { + //SCLogNotice("tx %p Mpm IDs: %"PRIx64, tx, mpm_ids); + AppLayerParserSetTxMpmIDs(ipproto, alproto, tx, mpm_ids); + } } }