From: Victor Julien Date: Wed, 31 Jan 2018 14:58:21 +0000 (+0100) Subject: app-layer: add tx iterator API X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=e96d9c11596e6a2b611902f0c7c601de5dccc6f3;p=people%2Fms%2Fsuricata.git app-layer: add tx iterator API Until now, the transaction space is assumed to be terse. Transactions are handled sequentially so the difference between the lowest and highest active tx id's is small. For this reason the logic of walking every id between the 'minimum' and max id made sense. The space might look like: [..........TTTT] Here the looping starts at the first T and loops 4 times. This assumption isn't a great fit though. A protocol like NFS has 2 types of transactions. Long running file transfer transactions and short lived request/reply pairs are causing the id space to be sparse. This leads to a lot of unnecessary looping in various parts of the engine, but most prominently: detection, tx house keeping and tx logging. [.T..T...TTTT.T] Here the looping starts at the first T and loops for every spot, even those where no tx exists anymore. Cases have been observed where the lowest tx id was 2 and the highest was 50k. This lead to a lot of unnecessary looping. This patch add an alternative approach. It allows a protocol to register an iterator function, that simply returns the next transaction until all transactions are returned. To do this it uses a bit of state the caller must keep. The registration is optional. If no iterator is registered the old behaviour will be used. --- diff --git a/rust/gen-c-headers.py b/rust/gen-c-headers.py index 77880a782..a6e803a9b 100755 --- a/rust/gen-c-headers.py +++ b/rust/gen-c-headers.py @@ -88,6 +88,8 @@ type_map = { "core::AppLayerDecoderEvents": "AppLayerDecoderEvents", "AppLayerDecoderEvents": "AppLayerDecoderEvents", "core::AppLayerEventType": "AppLayerEventType", + "applayer::AppLayerGetTxIterTuple": "AppLayerGetTxIterTuple", + "AppLayerGetTxIterTuple": "AppLayerGetTxIterTuple", "AppLayerEventType": "AppLayerEventType", "CLuaState": "lua_State", "Store": "Store", diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs index 4fbe4ad25..6a0a8e0fc 100644 --- a/rust/src/applayer.rs +++ b/rust/src/applayer.rs @@ -15,6 +15,29 @@ * 02110-1301, USA. */ +extern crate libc; +use std; + +#[repr(C)] +pub struct AppLayerGetTxIterTuple { + tx_ptr: *mut libc::c_void, + tx_id: u64, + has_next: bool, +} + +impl AppLayerGetTxIterTuple { + pub fn with_values(tx_ptr: *mut libc::c_void, tx_id: u64, has_next: bool) -> AppLayerGetTxIterTuple { + AppLayerGetTxIterTuple { + tx_ptr: tx_ptr, tx_id: tx_id, has_next: has_next, + } + } + pub fn not_found() -> AppLayerGetTxIterTuple { + AppLayerGetTxIterTuple { + tx_ptr: std::ptr::null_mut(), tx_id: 0, has_next: false, + } + } +} + /// LoggerFlags tracks which loggers have already been executed. #[derive(Debug)] pub struct LoggerFlags { diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs index 4eb2fe7a0..8b2e32ca5 100644 --- a/rust/src/nfs/nfs.rs +++ b/rust/src/nfs/nfs.rs @@ -28,6 +28,7 @@ use nom; use nom::IResult; use log::*; +use applayer; use applayer::LoggerFlags; use core::*; use filetracker::*; @@ -423,6 +424,28 @@ impl NFSState { return None; } + // for use with the C API call StateGetTxIterator + pub fn get_tx_iterator(&mut self, min_tx_id: u64, state: &mut u64) -> + Option<(&NFSTransaction, u64, bool)> + { + let mut index = *state as usize; + let len = self.transactions.len(); + + // find tx that is >= min_tx_id + while index < len { + let tx = &self.transactions[index]; + if tx.id < min_tx_id + 1 { + index += 1; + continue; + } + *state = index as u64 + 1; + SCLogDebug!("returning tx_id {} has_next? {} (len {} index {}), tx {:?}", + tx.id - 1, (len - index) > 1, len, index, tx); + return Some((tx, tx.id - 1, (len - index) > 1)); + } + return None; + } + /// Set an event. The event is set on the most recent transaction. pub fn set_event(&mut self, event: NFSEvent) { let len = self.transactions.len(); @@ -1850,6 +1873,26 @@ pub extern "C" fn rs_nfs3_state_get_tx(state: &mut NFSState, } } +// for use with the C API call StateGetTxIterator +#[no_mangle] +pub extern "C" fn rs_nfs_state_get_tx_iterator( + state: &mut NFSState, + min_tx_id: libc::uint64_t, + istate: &mut libc::uint64_t) + -> applayer::AppLayerGetTxIterTuple +{ + match state.get_tx_iterator(min_tx_id, istate) { + Some((tx, out_tx_id, has_next)) => { + let c_tx = unsafe { transmute(tx) }; + let ires = applayer::AppLayerGetTxIterTuple::with_values(c_tx, out_tx_id, has_next); + return ires; + } + None => { + return applayer::AppLayerGetTxIterTuple::not_found(); + } + } +} + #[no_mangle] pub extern "C" fn rs_nfs3_state_tx_free(state: &mut NFSState, tx_id: libc::uint64_t) diff --git a/src/app-layer-nfs-tcp.c b/src/app-layer-nfs-tcp.c index 676e21cef..e118f4f67 100644 --- a/src/app-layer-nfs-tcp.c +++ b/src/app-layer-nfs-tcp.c @@ -194,6 +194,14 @@ static void *NFSTCPGetTx(void *state, uint64_t tx_id) return rs_nfs3_state_get_tx(state, tx_id); } +static AppLayerGetTxIterTuple RustNFSTCPGetTxIterator( + const uint8_t ipproto, const AppProto alproto, + void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, + AppLayerGetTxIterState *istate) +{ + return rs_nfs_state_get_tx_iterator(alstate, min_tx_id, (uint64_t *)istate); +} + static void NFSTCPSetTxLogged(void *state, void *vtx, LoggerId logged) { rs_nfs3_tx_set_logged(state, vtx, logged); @@ -348,6 +356,8 @@ void RegisterNFSTCPParsers(void) ALPROTO_NFS, NFSTCPGetStateProgress); AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_NFS, NFSTCPGetTx); + AppLayerParserRegisterGetTxIterator(IPPROTO_TCP, ALPROTO_NFS, + RustNFSTCPGetTxIterator); AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_NFS, NFSTCPGetFiles); diff --git a/src/app-layer-nfs-udp.c b/src/app-layer-nfs-udp.c index 71a836c86..6d85ebee7 100644 --- a/src/app-layer-nfs-udp.c +++ b/src/app-layer-nfs-udp.c @@ -187,6 +187,14 @@ static void *NFSGetTx(void *state, uint64_t tx_id) return rs_nfs3_state_get_tx(state, tx_id); } +static AppLayerGetTxIterTuple RustNFSGetTxIterator( + const uint8_t ipproto, const AppProto alproto, + void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, + AppLayerGetTxIterState *istate) +{ + return rs_nfs_state_get_tx_iterator(alstate, min_tx_id, (uint64_t *)istate); +} + static void NFSSetTxLogged(void *state, void *vtx, LoggerId logged) { rs_nfs3_tx_set_logged(state, vtx, logged); @@ -341,6 +349,8 @@ void RegisterNFSUDPParsers(void) ALPROTO_NFS, NFSGetStateProgress); AppLayerParserRegisterGetTx(IPPROTO_UDP, ALPROTO_NFS, NFSGetTx); + AppLayerParserRegisterGetTxIterator(IPPROTO_UDP, ALPROTO_NFS, + RustNFSGetTxIterator); AppLayerParserRegisterGetFilesFunc(IPPROTO_UDP, ALPROTO_NFS, NFSGetFiles); diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index a2ab40a17..23389dc62 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -105,6 +105,7 @@ typedef struct AppLayerParserProtoCtx_ int (*StateGetProgress)(void *alstate, uint8_t direction); uint64_t (*StateGetTxCnt)(void *alstate); void *(*StateGetTx)(void *alstate, uint64_t tx_id); + AppLayerGetTxIteratorFunc StateGetTxIterator; int (*StateGetProgressCompletionStatus)(uint8_t direction); int (*StateGetEventInfo)(const char *event_name, int *event_id, AppLayerEventType *event_type); @@ -518,6 +519,14 @@ void AppLayerParserRegisterGetTx(uint8_t ipproto, AppProto alproto, SCReturn; } +void AppLayerParserRegisterGetTxIterator(uint8_t ipproto, AppProto alproto, + AppLayerGetTxIteratorFunc Func) +{ + SCEnter(); + alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetTxIterator = Func; + SCReturn; +} + void AppLayerParserRegisterGetStateProgressCompletionStatus(AppProto alproto, int (*StateGetProgressCompletionStatus)(uint8_t direction)) { @@ -609,6 +618,49 @@ void AppLayerParserDestroyProtocolParserLocalStorage(uint8_t ipproto, AppProto a SCReturn; } +/** \brief default tx iterator + * + * Used if the app layer parser doesn't register its own iterator. + * Simply walks the tx_id space until it finds a tx. Uses 'state' to + * keep track of where it left off. + * + * \retval txptr or NULL if no more txs in list + */ +static AppLayerGetTxIterTuple AppLayerDefaultGetTxIterator( + const uint8_t ipproto, const AppProto alproto, + void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, + AppLayerGetTxIterState *state) +{ + uint64_t ustate = *(uint64_t *)state; + uint64_t tx_id = MAX(min_tx_id, ustate); + for ( ; tx_id < max_tx_id; tx_id++) { + void *tx_ptr = AppLayerParserGetTx(ipproto, alproto, alstate, tx_id); + if (tx_ptr != NULL) { + ustate = tx_id + 1; + *state = *(AppLayerGetTxIterState *)&ustate; + AppLayerGetTxIterTuple tuple = { + .tx_ptr = tx_ptr, + .tx_id = tx_id, + .has_next = (tx_id + 1 < max_tx_id), + }; + SCLogDebug("tulpe: %p/%"PRIu64"/%s", tuple.tx_ptr, tuple.tx_id, + tuple.has_next ? "true" : "false"); + return tuple; + } + } + + AppLayerGetTxIterTuple no_tuple = { NULL, 0, false }; + return no_tuple; +} + +AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, + const AppProto alproto) +{ + AppLayerGetTxIteratorFunc Func = + alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetTxIterator; + return Func ? Func : AppLayerDefaultGetTxIterator; +} + void AppLayerParserSetTxLogged(uint8_t ipproto, AppProto alproto, void *alstate, void *tx, LoggerId logger) { @@ -677,28 +729,37 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p const uint8_t ipproto = f->proto; const AppProto alproto = f->alproto; + AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto); + AppLayerGetTxIterState state; + memset(&state, 0, sizeof(state)); + SCLogDebug("called: %s, tag_txs_as_inspected %s",direction==0?"toserver":"toclient", tag_txs_as_inspected?"true":"false"); /* mark all txs as inspected if the applayer progress is * at the 'end state'. */ - for (; idx < total_txs; idx++) { - void *tx = AppLayerParserGetTx(ipproto, alproto, alstate, idx); - if (tx == NULL) - continue; + while (1) { + AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, idx, total_txs, &state); + if (ires.tx_ptr == NULL) + break; + + void *tx = ires.tx_ptr; + idx = ires.tx_id; + int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags); - if (state_progress >= state_done_progress) { - if (tag_txs_as_inspected) { - uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags); - if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) { - detect_flags |= APP_LAYER_TX_INSPECTED_FLAG; - AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags); - SCLogDebug("%p/%"PRIu64" in-order tx is done for direction %s. Flag %016"PRIx64, - tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags); - } + if (state_progress < state_done_progress) + break; + + if (tag_txs_as_inspected) { + uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags); + if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) { + detect_flags |= APP_LAYER_TX_INSPECTED_FLAG; + AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags); + SCLogDebug("%p/%"PRIu64" in-order tx is done for direction %s. Flag %016"PRIx64, + tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags); } - continue; - } else + } + if (!ires.has_next) break; } pstate->inspect_id[direction] = idx; @@ -707,29 +768,39 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p /* if necessary we flag all txs that are complete as 'inspected' * also move inspect_id forward. */ if (tag_txs_as_inspected) { - for (; idx < total_txs; idx++) { - bool check_inspect_id = false; - void *tx = AppLayerParserGetTx(ipproto, alproto, alstate, idx); - if (tx == NULL) { - check_inspect_id = true; - } else { - int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags); - if (state_progress >= state_done_progress) { - uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags); - if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) { - detect_flags |= APP_LAYER_TX_INSPECTED_FLAG; - AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags); - SCLogDebug("%p/%"PRIu64" out of order tx is done for direction %s. Flag %016"PRIx64, - tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags); - check_inspect_id = true; - } - } + /* continue at idx */ + while (1) { + AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, idx, total_txs, &state); + if (ires.tx_ptr == NULL) + break; + + void *tx = ires.tx_ptr; + /* if we got a higher id than the minimum we requested, we + * skipped a bunch of 'null-txs'. Lets see if we can up the + * inspect tracker */ + if (ires.tx_id > idx && pstate->inspect_id[direction] == idx) { + pstate->inspect_id[direction] = ires.tx_id; } - if (check_inspect_id) { - SCLogDebug("%p/%"PRIu64" out of order tx. Update inspect_id? %"PRIu64, tx, idx, pstate->inspect_id[direction]); + idx = ires.tx_id; + + const int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags); + if (state_progress < state_done_progress) + break; + + uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags); + if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) { + detect_flags |= APP_LAYER_TX_INSPECTED_FLAG; + AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags); + SCLogDebug("%p/%"PRIu64" out of order tx is done for direction %s. Flag %016"PRIx64, + tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags); + + SCLogDebug("%p/%"PRIu64" out of order tx. Update inspect_id? %"PRIu64, + tx, idx, pstate->inspect_id[direction]); if (pstate->inspect_id[direction]+1 == idx) pstate->inspect_id[direction] = idx; } + if (!ires.has_next) + break; } } @@ -808,31 +879,38 @@ void AppLayerParserTransactionsCleanup(Flow *f) const int tx_end_state_ts = AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOSERVER); const int tx_end_state_tc = AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOCLIENT); - SCLogDebug("checking %"PRIu64" txs from offset %"PRIu64, total_txs, min); - for (uint64_t i = min ; i < total_txs; i++) { - void * const tx = AppLayerParserGetTx(ipproto, alproto, alstate, i); - if (tx == NULL) { - SCLogDebug("%p/%"PRIu64" skipping: no tx", tx, i); - goto wrap_up; - } + AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto); + AppLayerGetTxIterState state; + memset(&state, 0, sizeof(state)); + uint64_t i = min; + uint64_t new_min = min; + + while (1) { + AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, i, total_txs, &state); + if (ires.tx_ptr == NULL) + break; + + void *tx = ires.tx_ptr; + i = ires.tx_id; + SCLogDebug("%p/%"PRIu64" checking", tx, i); const int tx_progress_tc = AppLayerParserGetStateProgress(ipproto, alproto, tx, STREAM_TOCLIENT); if (tx_progress_tc < tx_end_state_tc) { SCLogDebug("%p/%"PRIu64" skipping: tc parser not done", tx, i); - continue; + goto next; } const int tx_progress_ts = AppLayerParserGetStateProgress(ipproto, alproto, tx, STREAM_TOSERVER); if (tx_progress_ts < tx_end_state_ts) { SCLogDebug("%p/%"PRIu64" skipping: ts parser not done", tx, i); - continue; + goto next; } if (f->sgh_toserver != NULL) { uint64_t detect_flags_ts = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, STREAM_TOSERVER); if (!(detect_flags_ts & APP_LAYER_TX_INSPECTED_FLAG)) { SCLogDebug("%p/%"PRIu64" skipping: TS inspect not done: ts:%"PRIx64, tx, i, detect_flags_ts); - continue; + goto next; } } if (f->sgh_toclient != NULL) { @@ -840,7 +918,7 @@ void AppLayerParserTransactionsCleanup(Flow *f) if (!(detect_flags_tc & APP_LAYER_TX_INSPECTED_FLAG)) { SCLogDebug("%p/%"PRIu64" skipping: TC inspect not done: tc:%"PRIx64, tx, i, detect_flags_tc); - continue; + goto next; } } if (logger_expectation != 0) { @@ -848,25 +926,32 @@ void AppLayerParserTransactionsCleanup(Flow *f) if (tx_logged != logger_expectation) { SCLogDebug("%p/%"PRIu64" skipping: logging not done: want:%"PRIx32", have:%"PRIx32, tx, i, logger_expectation, tx_logged); - continue; + goto next; } } /* if we are here, the tx can be freed. */ p->StateTransactionFree(alstate, i); SCLogDebug("%p/%"PRIu64" freed", tx, i); - wrap_up: - /* see if this tx is actually in order. If so, we need to bring all - * trackers up to date. */ - SCLogDebug("%p/%"PRIu64" update f->alparser->min_id? %"PRIu64, tx, i, alparser->min_id); - if (i == alparser->min_id) { - uint64_t next_id = i + 1; - alparser->min_id = next_id; - alparser->inspect_id[0] = MAX(alparser->inspect_id[0], next_id); - alparser->inspect_id[1] = MAX(alparser->inspect_id[1], next_id); - alparser->log_id = MAX(alparser->log_id, next_id); - SCLogDebug("%p/%"PRIu64" updated f->alparser->min_id %"PRIu64, tx, i, alparser->min_id); - } + + /* if this tx was the minimum, up the minimum */ + if (i == new_min) + new_min = i + 1; + +next: + if (!ires.has_next) + break; + } + + /* see if we need to bring all trackers up to date. */ + SCLogDebug("update f->alparser->min_id? %"PRIu64, alparser->min_id); + if (new_min > alparser->min_id) { + const uint64_t next_id = new_min; + alparser->min_id = next_id; + alparser->inspect_id[0] = MAX(alparser->inspect_id[0], next_id); + alparser->inspect_id[1] = MAX(alparser->inspect_id[1], next_id); + alparser->log_id = MAX(alparser->log_id, next_id); + SCLogDebug("updated f->alparser->min_id %"PRIu64, alparser->min_id); } } diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h index ad6fd8b01..251846319 100644 --- a/src/app-layer-parser.h +++ b/src/app-layer-parser.h @@ -92,6 +92,25 @@ typedef int (*AppLayerParserFPtr)(Flow *f, void *protocol_state, uint8_t *buf, uint32_t buf_len, void *local_storage); +typedef struct AppLayerGetTxIterTuple { + void *tx_ptr; + uint64_t tx_id; + bool has_next; +} AppLayerGetTxIterTuple; + +typedef struct AppLayerGetTxIterState { + union { + void *ptr; + uint64_t u64; + } un; +} AppLayerGetTxIterState; + +/** \brief tx iterator prototype */ +typedef AppLayerGetTxIterTuple (*AppLayerGetTxIteratorFunc) + (const uint8_t ipproto, const AppProto alproto, + void *alstate, uint64_t min_tx_id, uint64_t max_tx_id, + AppLayerGetTxIterState *state); + /***** Parser related registration *****/ /** @@ -135,6 +154,8 @@ void AppLayerParserRegisterGetTxCnt(uint8_t ipproto, AppProto alproto, uint64_t (*StateGetTxCnt)(void *alstate)); void AppLayerParserRegisterGetTx(uint8_t ipproto, AppProto alproto, void *(StateGetTx)(void *alstate, uint64_t tx_id)); +void AppLayerParserRegisterGetTxIterator(uint8_t ipproto, AppProto alproto, + AppLayerGetTxIteratorFunc Func); void AppLayerParserRegisterGetStateProgressCompletionStatus(AppProto alproto, int (*StateGetStateProgressCompletionStatus)(uint8_t direction)); void AppLayerParserRegisterGetEventInfo(uint8_t ipproto, AppProto alproto, @@ -156,6 +177,9 @@ void AppLayerParserRegisterDetectFlagsFuncs(uint8_t ipproto, AppProto alproto, /***** Get and transaction functions *****/ +AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto, + const AppProto alproto); + void *AppLayerParserGetProtocolParserLocalStorage(uint8_t ipproto, AppProto alproto); void AppLayerParserDestroyProtocolParserLocalStorage(uint8_t ipproto, AppProto alproto, void *local_data); diff --git a/src/detect-nfs-procedure.c b/src/detect-nfs-procedure.c index 7f7cbd724..05e8c5424 100644 --- a/src/detect-nfs-procedure.c +++ b/src/detect-nfs-procedure.c @@ -34,6 +34,8 @@ #include "detect-pcre.h" #include "detect-nfs-procedure.h" +#include "app-layer-parser.h" + #include "flow.h" #include "flow-util.h" #include "flow-var.h" diff --git a/src/detect-nfs-version.c b/src/detect-nfs-version.c index 5d6f2123f..88e9ae4ad 100644 --- a/src/detect-nfs-version.c +++ b/src/detect-nfs-version.c @@ -34,6 +34,8 @@ #include "detect-pcre.h" #include "detect-nfs-version.h" +#include "app-layer-parser.h" + #include "flow.h" #include "flow-util.h" #include "flow-var.h" diff --git a/src/detect.c b/src/detect.c index b034d8655..978b19c08 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1302,15 +1302,10 @@ static bool DetectRunTxInspectRule(ThreadVars *tv, * \brief get a DetectTransaction object * \retval struct filled with relevant info or all nulls/0s */ -static DetectTransaction GetTx(const uint8_t ipproto, const AppProto alproto, - void *alstate, const uint64_t tx_id, const int tx_end_state, +static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alproto, + void *alstate, const uint64_t tx_id, void *tx_ptr, const int tx_end_state, const uint8_t flow_flags) { - void *tx_ptr = AppLayerParserGetTx(ipproto, alproto, alstate, tx_id); - if (tx_ptr == NULL) { - DetectTransaction no_tx = { NULL, 0, NULL, 0, 0, 0, 0, 0, }; - return no_tx; - } const uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx_ptr, flow_flags); if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) { SCLogDebug("%"PRIu64" tx already fully inspected for %s. Flags %016"PRIx64, @@ -1356,13 +1351,21 @@ static void DetectRunTx(ThreadVars *tv, uint64_t tx_id = AppLayerParserGetTransactionInspectId(f->alparser, flow_flags); const int tx_end_state = AppLayerParserGetStateProgressCompletionStatus(alproto, flow_flags); - for ( ; tx_id < total_txs; tx_id++) { - DetectTransaction tx = GetTx(ipproto, alproto, - alstate, tx_id, tx_end_state, flow_flags); + AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto); + AppLayerGetTxIterState state; + memset(&state, 0, sizeof(state)); + + while (1) { + AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state); + if (ires.tx_ptr == NULL) + break; + + DetectTransaction tx = GetDetectTx(ipproto, alproto, + alstate, ires.tx_id, ires.tx_ptr, tx_end_state, flow_flags); if (tx.tx_ptr == NULL) { SCLogDebug("%p/%"PRIu64" no transaction to inspect", tx.tx_ptr, tx_id); - continue; + goto next; } uint32_t array_idx = 0; @@ -1571,6 +1574,9 @@ static void DetectRunTx(ThreadVars *tv, AppLayerParserSetTxDetectFlags(ipproto, alproto, tx.tx_ptr, flow_flags, new_detect_flags); } +next: + if (!ires.has_next) + break; } } diff --git a/src/output-tx.c b/src/output-tx.c index 4cb7bad83..b061b89d0 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -140,6 +140,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) return TM_ECODE_OK; Flow * const f = p->flow; + const uint8_t ipproto = f->proto; const AppProto alproto = f->alproto; if (AppLayerParserProtocolIsTxAware(p->proto, alproto) == 0) @@ -164,20 +165,23 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) int logged = 0; int gap = 0; - for (; tx_id < total_txs; tx_id++) - { - void *tx = AppLayerParserGetTx(p->proto, alproto, alstate, tx_id); - if (tx == NULL) { - SCLogDebug("tx is NULL not logging"); - continue; - } + AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto); + AppLayerGetTxIterState state; + memset(&state, 0, sizeof(state)); + + while (1) { + AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state); + if (ires.tx_ptr == NULL) + break; + void * const tx = ires.tx_ptr; + tx_id = ires.tx_id; LoggerId tx_logged = AppLayerParserGetTxLogged(f, alstate, tx); const LoggerId tx_logged_old = tx_logged; SCLogDebug("logger: expect %08x, have %08x", logger_expectation, tx_logged); if (tx_logged == logger_expectation) { /* tx already fully logged */ - continue; + goto next_tx; } int tx_progress_ts = AppLayerParserGetStateProgress(p->proto, alproto, @@ -211,17 +215,17 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) int r = logger->LogCondition(tv, p, alstate, tx, tx_id); if (r == FALSE) { SCLogDebug("conditions not met, not logging"); - goto next; + goto next_logger; } } else { if (tx_progress_tc < logger->tc_log_progress) { SCLogDebug("progress not far enough, not logging"); - goto next; + goto next_logger; } if (tx_progress_ts < logger->ts_log_progress) { SCLogDebug("progress not far enough, not logging"); - goto next; + goto next_logger; } } } @@ -235,7 +239,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) tx_logged |= (1<logger_id); } -next: +next_logger: logger = logger->next; store = store->next; #ifdef DEBUG_VALIDATION @@ -263,6 +267,9 @@ next: } else { gap = 1; } +next_tx: + if (!ires.has_next) + break; } /* Update the the last ID that has been logged with all