From: Victor Julien Date: Fri, 27 Feb 2015 14:21:17 +0000 (+0100) Subject: detect-state: split flow and tx state X-Git-Tag: suricata-2.1beta4~140 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=da3e8ad8f6e77583cc7998057d41959a85758868;p=thirdparty%2Fsuricata.git detect-state: split flow and tx state Use separate data structures for storing TX and FLOW (AMATCH) detect state. - move state storing into util funcs - remove de_state_m - simplify reset state logic on reload --- diff --git a/src/detect-engine-state.c b/src/detect-engine-state.c index 261bd5fb78..c19e398f49 100644 --- a/src/detect-engine-state.c +++ b/src/detect-engine-state.c @@ -83,6 +83,28 @@ /******** static internal helpers *********/ +static inline int StateIsValid(uint16_t alproto, void *alstate) +{ + if (alstate != NULL) { + if (alproto == ALPROTO_HTTP) { + HtpState *htp_state = (HtpState *)alstate; + if (htp_state->conn != NULL) { + return 1; + } + } else { + return 1; + } + } + return 0; +} + +static inline int TxIsLast(uint64_t tx_id, uint64_t total_txs) +{ + if (total_txs - tx_id <= 1) + return 1; + return 0; +} + static DeStateStore *DeStateStoreAlloc(void) { DeStateStore *d = SCMalloc(sizeof(DeStateStore)); @@ -92,10 +114,17 @@ static DeStateStore *DeStateStoreAlloc(void) return d; } +static DeStateStoreFlowRules *DeStateStoreFlowRulesAlloc(void) +{ + DeStateStoreFlowRules *d = SCMalloc(sizeof(DeStateStoreFlowRules)); + if (unlikely(d == NULL)) + return NULL; + memset(d, 0, sizeof(DeStateStoreFlowRules)); -static void DeStateSignatureAppend(DetectEngineState *state, Signature *s, - SigMatch *sm, uint32_t inspect_flags, - uint8_t direction) + return d; +} + +static void DeStateSignatureAppend(DetectEngineState *state, Signature *s, uint32_t inspect_flags, uint8_t direction) { int jump = 0; int i = 0; @@ -128,19 +157,56 @@ static void DeStateSignatureAppend(DetectEngineState *state, Signature *s, SigIntId idx = dir_state->cnt++ % DE_STATE_CHUNK_SIZE; store->store[idx].sid = s->num; store->store[idx].flags = inspect_flags; - store->store[idx].nm = sm; return; } -static void DeStateStoreStateVersion(DetectEngineState *de_state, - uint16_t alversion, uint8_t direction) +static void DeStateFlowRuleAppend(DetectEngineStateFlow *state, Signature *s, + SigMatch *sm, uint32_t inspect_flags, + uint8_t direction) { - de_state->dir_state[direction & STREAM_TOSERVER ? 0 : 1].alversion = alversion; + int jump = 0; + int i = 0; + DetectEngineStateDirectionFlow *dir_state = &state->dir_state[direction & STREAM_TOSERVER ? 0 : 1]; + DeStateStoreFlowRules *store = dir_state->head; + + if (store == NULL) { + store = DeStateStoreFlowRulesAlloc(); + if (store != NULL) { + dir_state->head = store; + dir_state->tail = store; + } + } else { + jump = dir_state->cnt / DE_STATE_CHUNK_SIZE; + for (i = 0; i < jump; i++) { + store = store->next; + } + if (store == NULL) { + store = DeStateStoreFlowRulesAlloc(); + if (store != NULL) { + dir_state->tail->next = store; + dir_state->tail = store; + } + } + } + + if (store == NULL) + return; + + SigIntId idx = dir_state->cnt++ % DE_STATE_CHUNK_SIZE; + store->store[idx].sid = s->num; + store->store[idx].flags = inspect_flags; + store->store[idx].nm = sm; return; } +static void DeStateStoreStateVersion(Flow *f, + uint16_t alversion, uint8_t direction) +{ + f->detect_alversion[direction & STREAM_TOSERVER ? 0 : 1] = alversion; +} + static void DeStateStoreFileNoMatchCnt(DetectEngineState *de_state, uint16_t file_no_match, uint8_t direction) { de_state->dir_state[direction & STREAM_TOSERVER ? 0 : 1].filestore_cnt += file_no_match; @@ -166,6 +232,16 @@ DetectEngineState *DetectEngineStateAlloc(void) return d; } +DetectEngineStateFlow *DetectEngineStateFlowAlloc(void) +{ + DetectEngineStateFlow *d = SCMalloc(sizeof(DetectEngineStateFlow)); + if (unlikely(d == NULL)) + return NULL; + memset(d, 0, sizeof(DetectEngineStateFlow)); + + return d; +} + void DetectEngineStateFree(DetectEngineState *state) { DeStateStore *store; @@ -185,7 +261,65 @@ void DetectEngineStateFree(DetectEngineState *state) return; } -/** +void DetectEngineStateFlowFree(DetectEngineStateFlow *state) +{ + DeStateStoreFlowRules *store; + DeStateStoreFlowRules *store_next; + int i = 0; + + for (i = 0; i < 2; i++) { + store = state->dir_state[i].head; + while (store != NULL) { + store_next = store->next; + SCFree(store); + store = store_next; + } + } + SCFree(state); + + return; +} + +static int HasStoredSigs(Flow *f, uint8_t flags) +{ + if (f->de_state != NULL && f->de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1].cnt != 0) { + SCLogDebug("global sigs present"); + return 1; + } + + if (AppLayerParserProtocolSupportsTxs(f->proto, f->alproto)) { + AppProto alproto = f->alproto; + void *alstate = FlowGetAppState(f); + if (!StateIsValid(f->alproto, alstate)) { + return 0; + } + + uint64_t inspect_tx_id = AppLayerParserGetTransactionInspectId(f->alparser, flags); + uint64_t total_txs = AppLayerParserGetTxCnt(f->proto, alproto, alstate); + + for ( ; inspect_tx_id < total_txs; inspect_tx_id++) { + void *inspect_tx = AppLayerParserGetTx(f->proto, alproto, alstate, inspect_tx_id); + if (inspect_tx != NULL) { + DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(f->proto, alproto, inspect_tx); + if (tx_de_state == NULL) { + continue; + } + if (tx_de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1].cnt != 0) { + SCLogDebug("tx %u has sigs present", (uint)inspect_tx_id); + return 1; + } + } + } + } + return 0; +} + +/** \brief Check if we need to inspect this state + * + * State needs to be inspected if: + * 1. state has been updated + * 2. we already have de_state in progress + * * \retval 0 no inspectable state * \retval 1 inspectable state * \retval 2 inspectable state, but no update @@ -194,49 +328,97 @@ int DeStateFlowHasInspectableState(Flow *f, AppProto alproto, uint16_t alversion { int r = 0; - SCMutexLock(&f->de_state_m); - if (f->de_state == NULL || f->de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1].cnt == 0) { - if (AppLayerParserProtocolSupportsTxs(f->proto, alproto)) { - FLOWLOCK_RDLOCK(f); - if (f->alparser != NULL && f->alstate != NULL) { - if (AppLayerParserGetTransactionInspectId(f->alparser, flags) >= - AppLayerParserGetTxCnt(f->proto, alproto, f->alstate)) { - r = 2; - } - } - FLOWLOCK_UNLOCK(f); - } - } else if (!(flags & STREAM_EOF) && - f->de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1].alversion == alversion) { + FLOWLOCK_WRLOCK(f); + + if (!(flags & STREAM_EOF) && f->de_state && + f->detect_alversion[flags & STREAM_TOSERVER ? 0 : 1] == alversion) { + SCLogDebug("unchanged state"); r = 2; - } else { + } else if (HasStoredSigs(f, flags)) { r = 1; + } else { + r = 0; } - SCMutexUnlock(&f->de_state_m); + FLOWLOCK_UNLOCK(f); return r; } -static inline int StateIsValid(uint16_t alproto, void *alstate) +static int StoreState(DetectEngineThreadCtx *det_ctx, + Flow *f, const uint8_t flags, const uint16_t alversion, + Signature *s, SigMatch *sm, const uint32_t inspect_flags, + const uint16_t file_no_match) { - if (alstate != NULL) { - if (alproto == ALPROTO_HTTP) { - HtpState *htp_state = (HtpState *)alstate; - if (htp_state->conn != NULL) { - return 1; + if (f->de_state == NULL) { + f->de_state = DetectEngineStateFlowAlloc(); + if (f->de_state == NULL) { + return 0; + } + } + + DeStateFlowRuleAppend(f->de_state, s, sm, inspect_flags, flags); + DeStateStoreStateVersion(f, alversion, flags); + return 1; +} + +static void StoreStateTxHandleFiles(DetectEngineThreadCtx *det_ctx, Flow *f, + DetectEngineState *destate, const uint8_t flags, + const uint64_t tx_id, const uint16_t file_no_match) +{ + DeStateStoreFileNoMatchCnt(destate, file_no_match, flags); + if (DeStateStoreFilestoreSigsCantMatch(det_ctx->sgh, destate, flags) == 1) { + FileDisableStoringForTransaction(f, flags & (STREAM_TOCLIENT | STREAM_TOSERVER), tx_id); + destate->dir_state[flags & STREAM_TOSERVER ? 0 : 1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_STORE_DISABLED; + } +} + +static void StoreStateTxFileOnly(DetectEngineThreadCtx *det_ctx, + Flow *f, const uint8_t flags, const uint64_t tx_id, void *tx, + const uint16_t file_no_match) +{ + if (AppLayerParserSupportsTxDetectState(f->proto, f->alproto)) { + DetectEngineState *destate = AppLayerParserGetTxDetectState(f->proto, f->alproto, tx); + if (destate == NULL) { + destate = DetectEngineStateAlloc(); + if (destate == NULL) + return; + if (AppLayerParserSetTxDetectState(f->proto, f->alproto, tx, destate) < 0) { + DetectEngineStateFree(destate); + BUG_ON(1); + return; } - } else { - return 1; + SCLogDebug("destate created for %"PRIu64, tx_id); } + StoreStateTxHandleFiles(det_ctx, f, destate, flags, tx_id, file_no_match); } - return 0; } -static inline int TxIsLast(uint64_t tx_id, uint64_t total_txs) +static void StoreStateTx(DetectEngineThreadCtx *det_ctx, + Flow *f, const uint8_t flags, const uint16_t alversion, + const uint64_t tx_id, void *tx, + Signature *s, SigMatch *sm, + const uint32_t inspect_flags, const uint16_t file_no_match) { - if (total_txs - tx_id <= 1) - return 1; - return 0; + if (AppLayerParserSupportsTxDetectState(f->proto, f->alproto)) { + DetectEngineState *destate = AppLayerParserGetTxDetectState(f->proto, f->alproto, tx); + if (destate == NULL) { + destate = DetectEngineStateAlloc(); + if (destate == NULL) + return; + if (AppLayerParserSetTxDetectState(f->proto, f->alproto, tx, destate) < 0) { + DetectEngineStateFree(destate); + BUG_ON(1); + return; + } + SCLogDebug("destate created for %"PRIu64, tx_id); + } + + DeStateSignatureAppend(destate, s, inspect_flags, flags); + DeStateStoreStateVersion(f, alversion, flags); + + StoreStateTxHandleFiles(det_ctx, f, destate, flags, tx_id, file_no_match); + } + SCLogDebug("Stored for TX %"PRIu64, tx_id); } int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, @@ -247,23 +429,21 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, SigMatch *sm = NULL; uint16_t file_no_match = 0; uint32_t inspect_flags = 0; - int store_de_state = 0; uint8_t direction = (flags & STREAM_TOSERVER) ? 0 : 1; int alert_cnt = 0; + FLOWLOCK_WRLOCK(f); + /* TX based matches (inspect engines) */ if (AppLayerParserProtocolSupportsTxs(f->proto, alproto)) { uint64_t tx_id = 0; uint64_t total_txs = 0; - FLOWLOCK_WRLOCK(f); void *alstate = FlowGetAppState(f); if (!StateIsValid(alproto, alstate)) { - FLOWLOCK_UNLOCK(f); goto end; } tx_id = AppLayerParserGetTransactionInspectId(f->alparser, flags); - SCLogDebug("tx_id %"PRIu64, tx_id); total_txs = AppLayerParserGetTxCnt(f->proto, alproto, alstate); SCLogDebug("total_txs %"PRIu64, total_txs); @@ -314,26 +494,31 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, /* if this is the last tx in our list, and it's incomplete: then * we store the state so that ContinueDetection knows about it */ - if (TxIsLast(tx_id, total_txs)) { - if (AppLayerParserGetStateProgress(f->proto, alproto, tx, flags) < - AppLayerParserGetStateProgressCompletionStatus(f->proto, alproto, flags)) { - store_de_state = 1; - if (engine == NULL || inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) + int tx_is_done = (AppLayerParserGetStateProgress(f->proto, alproto, tx, flags) >= + AppLayerParserGetStateProgressCompletionStatus(f->proto, alproto, flags)); + + if ((engine == NULL && total_matches > 0) || (inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH)) { + if (!(TxIsLast(tx_id, total_txs)) || !tx_is_done) { + if (engine == NULL || inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) { inspect_flags |= DE_STATE_FLAG_FULL_INSPECT; + } + + /* store */ + StoreStateTx(det_ctx, f, flags, alversion, tx_id, tx, + s, sm, inspect_flags, file_no_match); + } else { + StoreStateTxFileOnly(det_ctx, f, flags, tx_id, tx, file_no_match); } } } /* for */ - FLOWLOCK_UNLOCK(f); - + /* DCERPC matches */ } else if (s->sm_lists[DETECT_SM_LIST_DMATCH] != NULL && (alproto == ALPROTO_DCERPC || alproto == ALPROTO_SMB || alproto == ALPROTO_SMB2)) { - FLOWLOCK_WRLOCK(f); void *alstate = FlowGetAppState(f); if (alstate == NULL) { - FLOWLOCK_UNLOCK(f); goto end; } @@ -363,18 +548,14 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, alert_cnt = 1; } } - FLOWLOCK_UNLOCK(f); } + /* flow based matches */ KEYWORD_PROFILING_SET_LIST(det_ctx, DETECT_SM_LIST_AMATCH); sm = s->sm_lists[DETECT_SM_LIST_AMATCH]; if (sm != NULL) { - /* RDLOCK would be nicer, but at least tlsstore needs - * write lock currently. */ - FLOWLOCK_WRLOCK(f); void *alstate = FlowGetAppState(f); if (alstate == NULL) { - FLOWLOCK_UNLOCK(f); goto end; } @@ -405,9 +586,7 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, } } } - FLOWLOCK_UNLOCK(f); - store_de_state = 1; if (sm == NULL || inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH) { if (match == 1) { if (!(s->flags & SIG_FLAG_NOALERT)) { @@ -420,34 +599,14 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, } inspect_flags |= DE_STATE_FLAG_FULL_INSPECT; } - } - - if (!store_de_state && file_no_match == 0) - goto end; - SCMutexLock(&f->de_state_m); - if (f->de_state == NULL) { - f->de_state = DetectEngineStateAlloc(); - if (f->de_state == NULL) { - SCMutexUnlock(&f->de_state_m); - goto end; - } + StoreState(det_ctx, f, flags, alversion, + s, sm, inspect_flags, file_no_match); } - if (store_de_state) { - DeStateSignatureAppend(f->de_state, s, sm, inspect_flags, flags); - DeStateStoreStateVersion(f->de_state, alversion, flags); - } - DeStateStoreFileNoMatchCnt(f->de_state, file_no_match, flags); - if (DeStateStoreFilestoreSigsCantMatch(det_ctx->sgh, f->de_state, flags) == 1) { - FLOWLOCK_WRLOCK(f); - FileDisableStoringForTransaction(f, flags & (STREAM_TOCLIENT | STREAM_TOSERVER), - det_ctx->tx_id); - FLOWLOCK_UNLOCK(f); - f->de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1].flags |= DETECT_ENGINE_STATE_FLAG_FILE_STORE_DISABLED; - } - SCMutexUnlock(&f->de_state_m); end: + FLOWLOCK_UNLOCK(f); + det_ctx->tx_id = 0; det_ctx->tx_id_set = 0; return alert_cnt ? 1:0; @@ -455,12 +614,12 @@ int DeStateDetectStartDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, static int DoInspectItem(ThreadVars *tv, DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, - const int alproto_supports_txs, DeStateStoreItem *item, const uint8_t dir_state_flags, Packet *p, Flow *f, AppProto alproto, uint8_t flags, const uint64_t inspect_tx_id, const uint64_t total_txs, - uint16_t *file_no_match) + uint16_t *file_no_match, int inprogress, // is current tx in progress? + const int next_tx_no_progress) // tx after current is still dormant { Signature *s = de_ctx->sig_array[item->sid]; @@ -484,11 +643,7 @@ static int DoInspectItem(ThreadVars *tv, } if (item->flags & DE_STATE_FLAG_FULL_INSPECT) { - if (alproto_supports_txs) { - if (TxIsLast(inspect_tx_id, total_txs)) { - det_ctx->de_state_sig_array[item->sid] = DE_STATE_MATCH_NO_NEW_STATE; - } - } else { + if (TxIsLast(inspect_tx_id, total_txs) || inprogress || next_tx_no_progress) { det_ctx->de_state_sig_array[item->sid] = DE_STATE_MATCH_NO_NEW_STATE; } return 0; @@ -511,11 +666,7 @@ static int DoInspectItem(ThreadVars *tv, item->flags &= ~DE_STATE_FLAG_FILE_TC_INSPECT; item->flags &= ~DE_STATE_FLAG_SIG_CANT_MATCH; } else { - if (alproto_supports_txs) { - if (TxIsLast(inspect_tx_id, total_txs)) { - det_ctx->de_state_sig_array[item->sid] = DE_STATE_MATCH_NO_NEW_STATE; - } - } else { + if (TxIsLast(inspect_tx_id, total_txs) || inprogress || next_tx_no_progress) { det_ctx->de_state_sig_array[item->sid] = DE_STATE_MATCH_NO_NEW_STATE; } return 0; @@ -528,70 +679,105 @@ static int DoInspectItem(ThreadVars *tv, RULE_PROFILING_START(p); - if (alproto_supports_txs) { - FLOWLOCK_WRLOCK(f); - void *alstate = FlowGetAppState(f); - if (!StateIsValid(alproto, alstate)) { - FLOWLOCK_UNLOCK(f); - RULE_PROFILING_END(det_ctx, s, 0, p); - return -1; - } + void *alstate = FlowGetAppState(f); + if (!StateIsValid(alproto, alstate)) { + RULE_PROFILING_END(det_ctx, s, 0, p); + return -1; + } - det_ctx->tx_id = inspect_tx_id; - det_ctx->tx_id_set = 1; - DetectEngineAppInspectionEngine *engine = app_inspection_engine[FlowGetProtoMapping(f->proto)][alproto][(flags & STREAM_TOSERVER) ? 0 : 1]; - void *inspect_tx = AppLayerParserGetTx(f->proto, alproto, alstate, inspect_tx_id); - if (inspect_tx == NULL) { - FLOWLOCK_UNLOCK(f); - RULE_PROFILING_END(det_ctx, s, 0, p); - return -1; - } + det_ctx->tx_id = inspect_tx_id; + det_ctx->tx_id_set = 1; + DetectEngineAppInspectionEngine *engine = app_inspection_engine[FlowGetProtoMapping(f->proto)][alproto][(flags & STREAM_TOSERVER) ? 0 : 1]; + void *inspect_tx = AppLayerParserGetTx(f->proto, alproto, alstate, inspect_tx_id); + if (inspect_tx == NULL) { + RULE_PROFILING_END(det_ctx, s, 0, p); + return -1; + } - while (engine != NULL) { - if (!(item->flags & engine->inspect_flags) && - s->sm_lists[engine->sm_list] != NULL) - { - KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list); - int match = engine->Callback(tv, de_ctx, det_ctx, s, f, - flags, alstate, inspect_tx, inspect_tx_id); - if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) { - inspect_flags |= engine->inspect_flags; - engine = engine->next; - total_matches++; - continue; - } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) { - inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; - inspect_flags |= engine->inspect_flags; - } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE) { - inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; - inspect_flags |= engine->inspect_flags; - (*file_no_match)++; - } - break; + while (engine != NULL) { + if (!(item->flags & engine->inspect_flags) && + s->sm_lists[engine->sm_list] != NULL) + { + KEYWORD_PROFILING_SET_LIST(det_ctx, engine->sm_list); + int match = engine->Callback(tv, de_ctx, det_ctx, s, f, + flags, alstate, inspect_tx, inspect_tx_id); + if (match == DETECT_ENGINE_INSPECT_SIG_MATCH) { + inspect_flags |= engine->inspect_flags; + engine = engine->next; + total_matches++; + continue; + } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH) { + inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; + inspect_flags |= engine->inspect_flags; + } else if (match == DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILESTORE) { + inspect_flags |= DE_STATE_FLAG_SIG_CANT_MATCH; + inspect_flags |= engine->inspect_flags; + (*file_no_match)++; } - engine = engine->next; + break; } - if (total_matches > 0 && (engine == NULL || inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH)) { - if (engine == NULL) - alert = 1; - inspect_flags |= DE_STATE_FLAG_FULL_INSPECT; + engine = engine->next; + } + if (total_matches > 0 && (engine == NULL || inspect_flags & DE_STATE_FLAG_SIG_CANT_MATCH)) { + if (engine == NULL) + alert = 1; + inspect_flags |= DE_STATE_FLAG_FULL_INSPECT; + } + + item->flags |= inspect_flags; + if (TxIsLast(inspect_tx_id, total_txs)) { + det_ctx->de_state_sig_array[item->sid] = DE_STATE_MATCH_NO_NEW_STATE; + } + + if (alert) { + SigMatchSignaturesRunPostMatch(tv, de_ctx, det_ctx, p, s); + + if (!(s->flags & SIG_FLAG_NOALERT)) { + PacketAlertAppend(det_ctx, s, p, inspect_tx_id, + PACKET_ALERT_FLAG_STATE_MATCH|PACKET_ALERT_FLAG_TX); + } else { + PACKET_UPDATE_ACTION(p, s->action); } + } - FLOWLOCK_UNLOCK(f); + DetectFlowvarProcessList(det_ctx, f); + return 1; +} + +/** \internal + * \brief Continue Detection for a single "flow" rule (AMATCH) + */ +static int DoInspectFlowRule(ThreadVars *tv, + DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx, + DeStateStoreFlowRule *item, const uint8_t dir_state_flags, + Packet *p, Flow *f, AppProto alproto, uint8_t flags) +{ + if (item->flags & DE_STATE_FLAG_FULL_INSPECT) { + if (item->flags & DE_STATE_FLAG_FULL_INSPECT) { + det_ctx->de_state_sig_array[item->sid] = DE_STATE_MATCH_NO_NEW_STATE; + return 0; + } + } + + /* check if a sig in state 'cant match' needs to be reconsidered + * as the result of a new file in the existing tx */ + if (item->flags & DE_STATE_FLAG_SIG_CANT_MATCH) { + det_ctx->de_state_sig_array[item->sid] = DE_STATE_MATCH_NO_NEW_STATE; + return 0; } - /* count AMATCH matches */ - total_matches = 0; + uint8_t alert = 0; + uint32_t inspect_flags = 0; + int total_matches = 0; SigMatch *sm = NULL; + Signature *s = de_ctx->sig_array[item->sid]; + + RULE_PROFILING_START(p); KEYWORD_PROFILING_SET_LIST(det_ctx, DETECT_SM_LIST_AMATCH); if (item->nm != NULL) { - /* RDLOCK would be nicer, but at least tlsstore needs - * write lock currently. */ - FLOWLOCK_WRLOCK(f); void *alstate = FlowGetAppState(f); if (alstate == NULL) { - FLOWLOCK_UNLOCK(f); RULE_PROFILING_END(det_ctx, s, 0 /* no match */, p); return -1; } @@ -623,7 +809,6 @@ static int DoInspectItem(ThreadVars *tv, total_matches++; } } - FLOWLOCK_UNLOCK(f); } if (s->sm_lists[DETECT_SM_LIST_AMATCH] != NULL) { @@ -638,20 +823,13 @@ static int DoInspectItem(ThreadVars *tv, item->flags |= inspect_flags; item->nm = sm; - if (TxIsLast(inspect_tx_id, total_txs)) { - det_ctx->de_state_sig_array[item->sid] = DE_STATE_MATCH_NO_NEW_STATE; - } if (alert) { SigMatchSignaturesRunPostMatch(tv, de_ctx, det_ctx, p, s); if (!(s->flags & SIG_FLAG_NOALERT)) { - if (alproto_supports_txs) - PacketAlertAppend(det_ctx, s, p, inspect_tx_id, - PACKET_ALERT_FLAG_STATE_MATCH|PACKET_ALERT_FLAG_TX); - else - PacketAlertAppend(det_ctx, s, p, 0, - PACKET_ALERT_FLAG_STATE_MATCH); + PacketAlertAppend(det_ctx, s, p, 0, + PACKET_ALERT_FLAG_STATE_MATCH); } else { DetectSignatureApplyActions(p, s); } @@ -671,83 +849,111 @@ void DeStateDetectContinueDetection(ThreadVars *tv, DetectEngineCtx *de_ctx, SigIntId state_cnt = 0; uint64_t inspect_tx_id = 0; uint64_t total_txs = 0; - uint8_t alproto_supports_txs = 0; - uint8_t reset_de_state = 0; - SCMutexLock(&f->de_state_m); - DetectEngineStateDirection *dir_state = &f->de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1]; - DeStateStore *store = dir_state->head; + FLOWLOCK_WRLOCK(f); + + SCLogDebug("starting continue detection for packet %"PRIu64, p->pcap_cnt); if (AppLayerParserProtocolSupportsTxs(f->proto, alproto)) { - FLOWLOCK_RDLOCK(f); void *alstate = FlowGetAppState(f); if (!StateIsValid(alproto, alstate)) { FLOWLOCK_UNLOCK(f); - SCMutexUnlock(&f->de_state_m); return; } inspect_tx_id = AppLayerParserGetTransactionInspectId(f->alparser, flags); total_txs = AppLayerParserGetTxCnt(f->proto, alproto, alstate); - void *inspect_tx = AppLayerParserGetTx(f->proto, alproto, alstate, inspect_tx_id); - if (inspect_tx != NULL) { - if (AppLayerParserGetStateProgress(f->proto, alproto, inspect_tx, flags) >= - AppLayerParserGetStateProgressCompletionStatus(f->proto, alproto, flags)) { - reset_de_state = 1; - } - } - FLOWLOCK_UNLOCK(f); - alproto_supports_txs = 1; - } - /* Loop through stored 'items' (stateful rules) and inspect them */ - for (; store != NULL; store = store->next) { - for (store_cnt = 0; - store_cnt < DE_STATE_CHUNK_SIZE && state_cnt < dir_state->cnt; - store_cnt++, state_cnt++) - { - DeStateStoreItem *item = &store->store[store_cnt]; - - int r = DoInspectItem(tv, de_ctx, det_ctx, alproto_supports_txs, - item, dir_state->flags, - p, f, alproto, flags, - inspect_tx_id, total_txs, - &file_no_match); - if (r < 0) { - goto end; + for ( ; inspect_tx_id < total_txs; inspect_tx_id++) { + int inspect_tx_inprogress = 0; + int next_tx_no_progress = 0; + void *inspect_tx = AppLayerParserGetTx(f->proto, alproto, alstate, inspect_tx_id); + if (inspect_tx != NULL) { + int a = AppLayerParserGetStateProgress(f->proto, alproto, inspect_tx, flags); + int b = AppLayerParserGetStateProgressCompletionStatus(f->proto, alproto, flags); + if (a < b) { + inspect_tx_inprogress = 1; + } + SCLogDebug("tx %"PRIu64" (%"PRIu64") => %s", inspect_tx_id, total_txs, + inspect_tx_inprogress ? "in progress" : "done"); + + DetectEngineState *tx_de_state = AppLayerParserGetTxDetectState(f->proto, alproto, inspect_tx); + if (tx_de_state == NULL) { + SCLogDebug("NO STATE tx %"PRIu64" (%"PRIu64")", inspect_tx_id, total_txs); + continue; + } + DetectEngineStateDirection *tx_dir_state = &tx_de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1]; + DeStateStore *tx_store = tx_dir_state->head; + + /* see if we need to consider the next tx in our decision to add + * a sig to the 'no inspect array'. */ + if (!TxIsLast(inspect_tx_id, total_txs)) { + void *next_inspect_tx = AppLayerParserGetTx(f->proto, alproto, alstate, inspect_tx_id+1); + if (next_inspect_tx != NULL) { + int c = AppLayerParserGetStateProgress(f->proto, alproto, next_inspect_tx, flags); + if (c == 0) { + next_tx_no_progress = 1; + } + } + } + + /* Loop through stored 'items' (stateful rules) and inspect them */ + state_cnt = 0; + for (; tx_store != NULL; tx_store = tx_store->next) { + SCLogDebug("tx_store %p", tx_store); + for (store_cnt = 0; + store_cnt < DE_STATE_CHUNK_SIZE && state_cnt < tx_dir_state->cnt; + store_cnt++, state_cnt++) + { + DeStateStoreItem *item = &tx_store->store[store_cnt]; + int r = DoInspectItem(tv, de_ctx, det_ctx, + item, tx_dir_state->flags, + p, f, alproto, flags, + inspect_tx_id, total_txs, + &file_no_match, inspect_tx_inprogress, next_tx_no_progress); + if (r < 0) { + SCLogDebug("failed"); + goto end; + } + } + } + } + if (inspect_tx_inprogress) { + SCLogDebug("break out"); + break; } } } - DeStateStoreStateVersion(f->de_state, alversion, flags); - DeStateStoreFileNoMatchCnt(f->de_state, file_no_match, flags); - - if (!(dir_state->flags & DETECT_ENGINE_STATE_FLAG_FILE_STORE_DISABLED)) { - if (DeStateStoreFilestoreSigsCantMatch(det_ctx->sgh, f->de_state, flags) == 1) { - SCLogDebug("disabling file storage for transaction"); - - FLOWLOCK_WRLOCK(f); - FileDisableStoringForTransaction(f, flags & (STREAM_TOCLIENT|STREAM_TOSERVER), - det_ctx->tx_id); - FLOWLOCK_UNLOCK(f); + /* continue on flow based state rules (AMATCH) */ + if (f->de_state != NULL) { + DetectEngineStateDirectionFlow *dir_state = &f->de_state->dir_state[flags & STREAM_TOSERVER ? 0 : 1]; + DeStateStoreFlowRules *store = dir_state->head; + /* Loop through stored 'items' (stateful rules) and inspect them */ + for (; store != NULL; store = store->next) { + for (store_cnt = 0; + store_cnt < DE_STATE_CHUNK_SIZE && state_cnt < dir_state->cnt; + store_cnt++, state_cnt++) + { + DeStateStoreFlowRule *rule = &store->store[store_cnt]; - dir_state->flags |= DETECT_ENGINE_STATE_FLAG_FILE_STORE_DISABLED; + int r = DoInspectFlowRule(tv, de_ctx, det_ctx, + rule, dir_state->flags, + p, f, alproto, flags); + if (r < 0) { + goto end; + } + } } + DeStateStoreStateVersion(f, alversion, flags); } end: - if (f->de_state != NULL) - dir_state->flags &= ~DETECT_ENGINE_STATE_FLAG_FILE_TC_NEW; - - if (reset_de_state) - DetectEngineStateReset(f->de_state, flags); - - SCMutexUnlock(&f->de_state_m); + FLOWLOCK_UNLOCK(f); det_ctx->tx_id = 0; det_ctx->tx_id_set = 0; return; } - /** \brief update flow's inspection id's * * \note it is possible that f->alstate, f->alparser are NULL */ @@ -762,18 +968,15 @@ void DeStateUpdateInspectTransactionId(Flow *f, uint8_t direction) return; } - -void DetectEngineStateReset(DetectEngineState *state, uint8_t direction) +void DetectEngineStateReset(DetectEngineStateFlow *state, uint8_t direction) { if (state != NULL) { if (direction & STREAM_TOSERVER) { state->dir_state[0].cnt = 0; - state->dir_state[0].filestore_cnt = 0; state->dir_state[0].flags = 0; } if (direction & STREAM_TOCLIENT) { state->dir_state[1].cnt = 0; - state->dir_state[1].filestore_cnt = 0; state->dir_state[1].flags = 0; } } @@ -825,39 +1028,39 @@ static int DeStateTest02(void) uint8_t direction = STREAM_TOSERVER; s.num = 0; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 11; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 22; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 33; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 44; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 55; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 66; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 77; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 88; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 99; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 100; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 111; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 122; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 133; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 144; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 155; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 166; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); if (state->dir_state[direction & STREAM_TOSERVER ? 0 : 1].head == NULL) { goto end; @@ -907,9 +1110,9 @@ static int DeStateTest03(void) uint8_t direction = STREAM_TOSERVER; s.num = 11; - DeStateSignatureAppend(state, &s, NULL, 0, direction); + DeStateSignatureAppend(state, &s, 0, direction); s.num = 22; - DeStateSignatureAppend(state, &s, NULL, DE_STATE_FLAG_URI_INSPECT, direction); + DeStateSignatureAppend(state, &s, DE_STATE_FLAG_URI_INSPECT, direction); if (state->dir_state[direction & STREAM_TOSERVER ? 0 : 1].head == NULL) { goto end; diff --git a/src/detect-engine-state.h b/src/detect-engine-state.h index 4c8bf9b19e..9a5b7ebab1 100644 --- a/src/detect-engine-state.h +++ b/src/detect-engine-state.h @@ -104,8 +104,9 @@ typedef enum { DE_STATE_MATCH_NO_NEW_STATE, } DeStateMatchResult; +/* TX BASED (inspect engines) */ + typedef struct DeStateStoreItem_ { - SigMatch *nm; uint32_t flags; SigIntId sid; } DeStateStoreItem; @@ -120,7 +121,6 @@ typedef struct DetectEngineStateDirection_ { DeStateStore *tail; SigIntId cnt; uint16_t filestore_cnt; - uint8_t alversion; uint8_t flags; } DetectEngineStateDirection; @@ -128,6 +128,30 @@ typedef struct DetectEngineState_ { DetectEngineStateDirection dir_state[2]; } DetectEngineState; +/* FLOW BASED (AMATCH) */ + +typedef struct DeStateStoreFlowRule_ { + SigMatch *nm; + uint32_t flags; + SigIntId sid; +} DeStateStoreFlowRule; + +typedef struct DeStateStoreFlowRules_ { + DeStateStoreFlowRule store[DE_STATE_CHUNK_SIZE]; + struct DeStateStoreFlowRules_ *next; +} DeStateStoreFlowRules; + +typedef struct DetectEngineStateDirectionFlow_ { + DeStateStoreFlowRules *head; + DeStateStoreFlowRules *tail; + SigIntId cnt; + uint8_t flags; +} DetectEngineStateDirectionFlow; + +typedef struct DetectEngineStateFlow_ { + DetectEngineStateDirectionFlow dir_state[2]; +} DetectEngineStateFlow; + /** * \brief Alloc a DetectEngineState object. * @@ -141,6 +165,7 @@ DetectEngineState *DetectEngineStateAlloc(void); * \param state DetectEngineState instance to free. */ void DetectEngineStateFree(DetectEngineState *state); +void DetectEngineStateFlowFree(DetectEngineStateFlow *state); /** * \brief Check if a flow already contains(newly updated as well) de state. @@ -204,7 +229,7 @@ void DeStateUpdateInspectTransactionId(Flow *f, uint8_t direction); * \param state Pointer to the state(LOCKED). * \param direction Direction flags - STREAM_TOSERVER or STREAM_TOCLIENT. */ -void DetectEngineStateReset(DetectEngineState *state, uint8_t direction); +void DetectEngineStateReset(DetectEngineStateFlow *state, uint8_t direction); void DeStateRegisterTests(void); diff --git a/src/detect.c b/src/detect.c index 3bc7d3b46f..98e3959ce9 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1153,7 +1153,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh Signature *s = NULL; Signature *next_s = NULL; uint16_t alversion = 0; - int reset_de_state = 0; int state_alert = 0; int alerts = 0; int app_decoder_events = 0; @@ -1195,11 +1194,13 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh pflow->flags &= ~FLOW_SGH_TOCLIENT; pflow->sgh_toserver = NULL; pflow->sgh_toclient = NULL; - reset_de_state = 1; pflow->de_ctx_id = de_ctx->id; GenericVarFree(pflow->flowvar); pflow->flowvar = NULL; + + DetectEngineStateReset(pflow->de_state, + (STREAM_TOSERVER|STREAM_TOCLIENT)); } /* set the iponly stuff */ @@ -1264,13 +1265,6 @@ int SigMatchSignatures(ThreadVars *th_v, DetectEngineCtx *de_ctx, DetectEngineTh } SCLogDebug("p->flowflags 0x%02x", p->flowflags); - /* reset because of ruleswap */ - if (reset_de_state) { - SCMutexLock(&pflow->de_state_m); - DetectEngineStateReset(pflow->de_state, (STREAM_TOSERVER|STREAM_TOCLIENT)); - SCMutexUnlock(&pflow->de_state_m); - } - if (((p->flowflags & FLOW_PKT_TOSERVER) && !(p->flowflags & FLOW_PKT_TOSERVER_IPONLY_SET)) || ((p->flowflags & FLOW_PKT_TOCLIENT) && !(p->flowflags & FLOW_PKT_TOCLIENT_IPONLY_SET))) { diff --git a/src/flow-util.h b/src/flow-util.h index 152b409836..38257b1289 100644 --- a/src/flow-util.h +++ b/src/flow-util.h @@ -57,13 +57,14 @@ (f)->data_al_so_far[1] = 0; \ (f)->de_ctx_id = 0; \ (f)->thread_id = 0; \ + (f)->detect_alversion[0] = 0; \ + (f)->detect_alversion[1] = 0; \ (f)->alparser = NULL; \ (f)->alstate = NULL; \ (f)->de_state = NULL; \ (f)->sgh_toserver = NULL; \ (f)->sgh_toclient = NULL; \ (f)->flowvar = NULL; \ - SCMutexInit(&(f)->de_state_m, NULL); \ (f)->hnext = NULL; \ (f)->hprev = NULL; \ (f)->lnext = NULL; \ @@ -101,10 +102,10 @@ (f)->data_al_so_far[1] = 0; \ (f)->de_ctx_id = 0; \ (f)->thread_id = 0; \ + (f)->detect_alversion[0] = 0; \ + (f)->detect_alversion[1] = 0; \ if ((f)->de_state != NULL) { \ - SCMutexLock(&(f)->de_state_m); \ DetectEngineStateReset((f)->de_state, (STREAM_TOSERVER | STREAM_TOCLIENT)); \ - SCMutexUnlock(&(f)->de_state_m); \ } \ (f)->sgh_toserver = NULL; \ (f)->sgh_toclient = NULL; \ @@ -123,12 +124,9 @@ \ FLOWLOCK_DESTROY((f)); \ if ((f)->de_state != NULL) { \ - SCMutexLock(&(f)->de_state_m); \ - DetectEngineStateFree((f)->de_state); \ - SCMutexUnlock(&(f)->de_state_m); \ + DetectEngineStateFlowFree((f)->de_state); \ } \ GenericVarFree((f)->flowvar); \ - SCMutexDestroy(&(f)->de_state_m); \ SC_ATOMIC_DESTROY((f)->autofp_tmqh_flow_qid); \ } while(0) diff --git a/src/flow.h b/src/flow.h index 48b4b075c0..b772986db6 100644 --- a/src/flow.h +++ b/src/flow.h @@ -370,6 +370,9 @@ typedef struct Flow_ /** Thread ID for the stream/detect portion of this flow */ FlowThreadId thread_id; + /** detect state 'alversion' inspected for both directions */ + uint8_t detect_alversion[2]; + /** application level storage ptrs. * */ @@ -377,7 +380,7 @@ typedef struct Flow_ void *alstate; /**< application layer state */ /** detection engine state */ - struct DetectEngineState_ *de_state; + struct DetectEngineStateFlow_ *de_state; /** toclient sgh for this flow. Only use when FLOW_SGH_TOCLIENT flow flag * has been set. */ @@ -389,8 +392,6 @@ typedef struct Flow_ /* pointer to the var list */ GenericVar *flowvar; - SCMutex de_state_m; /**< mutex lock for the de_state object */ - /** hash list pointers, protected by fb->s */ struct Flow_ *hnext; /* hash list */ struct Flow_ *hprev;