From: Philippe Antoine Date: Fri, 31 May 2024 08:39:16 +0000 (+0200) Subject: app-layer: track modified/processed txs X-Git-Tag: suricata-8.0.0-beta1~655 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b02557ac7dd6b55187309cee93f849ee2917d0cb;p=thirdparty%2Fsuricata.git app-layer: track modified/processed txs To optimize detection, and logging, to avoid going through all the live transactions when only a few were modified. Two boolean fields are added to the tx data: updated_tc and ts The app-layer parsers are now responsible to set these when needed, and the logging and detection uses them to skip transactions that were not updated. There may some more optimization remaining by when we set both updated_tc and updated_ts in functions returning a mutable transaction, by checking if all the callers are called in one direction only (request or response) Ticket: 7087 --- diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs index ac9800d634..4535e563aa 100644 --- a/rust/src/applayer.rs +++ b/rust/src/applayer.rs @@ -98,6 +98,12 @@ pub struct AppLayerTxData { /// config: log flags pub config: AppLayerTxConfig, + /// The tx has been updated and needs to be processed : detection, logging, cleaning + /// It can then be skipped until new data arrives. + /// There is a boolean for both directions : to server and to client + pub updated_tc: bool, + pub updated_ts: bool, + /// logger flags for tx logging api logged: LoggerFlags, @@ -155,6 +161,8 @@ impl AppLayerTxData { file_flags: 0, file_tx: 0, stream_logged: 0, + updated_tc: true, + updated_ts: true, detect_flags_ts: 0, detect_flags_tc: 0, de_state: std::ptr::null_mut(), @@ -165,9 +173,9 @@ impl AppLayerTxData { /// Create new AppLayerTxData for a transaction in a single /// direction. pub fn for_direction(direction: Direction) -> Self { - let (detect_flags_ts, detect_flags_tc) = match direction { - Direction::ToServer => (0, APP_LAYER_TX_SKIP_INSPECT_FLAG), - Direction::ToClient => (APP_LAYER_TX_SKIP_INSPECT_FLAG, 0), + let (detect_flags_ts, detect_flags_tc, updated_ts, updated_tc) = match direction { + Direction::ToServer => (0, APP_LAYER_TX_SKIP_INSPECT_FLAG, true, false), + Direction::ToClient => (APP_LAYER_TX_SKIP_INSPECT_FLAG, 0, false, true), }; Self { config: AppLayerTxConfig::new(), @@ -178,6 +186,8 @@ impl AppLayerTxData { file_flags: 0, file_tx: 0, stream_logged: 0, + updated_tc, + updated_ts, detect_flags_ts, detect_flags_tc, de_state: std::ptr::null_mut(), diff --git a/rust/src/applayertemplate/template.rs b/rust/src/applayertemplate/template.rs index 1580c4fdf3..eebe7385db 100644 --- a/rust/src/applayertemplate/template.rs +++ b/rust/src/applayertemplate/template.rs @@ -200,6 +200,7 @@ impl TemplateState { start = rem; if let Some(tx) = self.find_request() { + tx.tx_data.updated_tc = true; tx.response = Some(response); SCLogNotice!("Found response for request:"); SCLogNotice!("- Request: {:?}", tx.request); diff --git a/rust/src/dcerpc/dcerpc.rs b/rust/src/dcerpc/dcerpc.rs index 32ebf69901..9253a2bde8 100644 --- a/rust/src/dcerpc/dcerpc.rs +++ b/rust/src/dcerpc/dcerpc.rs @@ -357,6 +357,8 @@ impl DCERPCState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.req_done || !tx_old.resp_done { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.req_done = true; tx_old.resp_done = true; break; @@ -533,6 +535,8 @@ impl DCERPCState { } } } + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/dcerpc/dcerpc_udp.rs b/rust/src/dcerpc/dcerpc_udp.rs index ab7f65cafb..fee460f9b3 100644 --- a/rust/src/dcerpc/dcerpc_udp.rs +++ b/rust/src/dcerpc/dcerpc_udp.rs @@ -88,6 +88,8 @@ impl DCERPCUDPState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.req_done || !tx_old.resp_done { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.req_done = true; tx_old.resp_done = true; break; @@ -164,6 +166,8 @@ impl DCERPCUDPState { } if let Some(tx) = otx { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; let done = (hdr.flags1 & PFCL1_FRAG) == 0 || (hdr.flags1 & PFCL1_LASTFRAG) != 0; match hdr.pkt_type { diff --git a/rust/src/enip/enip.rs b/rust/src/enip/enip.rs index 31c9152e5c..18e7281226 100644 --- a/rust/src/enip/enip.rs +++ b/rust/src/enip/enip.rs @@ -203,6 +203,8 @@ impl EnipState { fn purge_tx_flood(&mut self) { let mut event_set = false; for tx in self.transactions.iter_mut() { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; tx.done = true; if !event_set { tx.tx_data.set_event(EnipEvent::TooManyTransactions as u8); @@ -216,6 +218,8 @@ impl EnipState { if let Some(req) = &tx.request { if tx.response.is_none() { tx.done = true; + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; if response_matches_request(req, pdu) { return Some(tx); } diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs index 89b599a9f8..8ed0157922 100644 --- a/rust/src/http2/http2.rs +++ b/rust/src/http2/http2.rs @@ -752,6 +752,8 @@ impl HTTP2State { let tx = &mut self.transactions[index - 1]; tx.tx_data.update_file_flags(self.state_data.file_flags); tx.update_file_flags(tx.tx_data.file_flags); + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } else { // do not use SETTINGS_MAX_CONCURRENT_STREAMS as it can grow too much @@ -764,6 +766,8 @@ impl HTTP2State { tx_old.set_event(HTTP2Event::TooManyStreams); // use a distinct state, even if we do not log it tx_old.state = HTTP2TransactionState::HTTP2StateTodrop; + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; } return None; } diff --git a/rust/src/ldap/ldap.rs b/rust/src/ldap/ldap.rs index f86c8d8499..4fb8ccc10b 100644 --- a/rust/src/ldap/ldap.rs +++ b/rust/src/ldap/ldap.rs @@ -156,6 +156,8 @@ impl LdapState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.complete { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.complete = true; tx_old .tx_data @@ -308,6 +310,7 @@ impl LdapState { if let Some(tx) = self.find_request(response.message_id) { tx.complete = tx_is_complete(&response.protocol_op, Direction::ToClient); let tx_id = tx.id(); + tx.tx_data.updated_tc = true; tx.responses.push_back(response); let consumed = start.len() - rem.len(); self.set_frame_tc(flow, tx_id, consumed as i64); diff --git a/rust/src/modbus/modbus.rs b/rust/src/modbus/modbus.rs index 0d0c73371e..bbc1915556 100644 --- a/rust/src/modbus/modbus.rs +++ b/rust/src/modbus/modbus.rs @@ -124,6 +124,8 @@ impl ModbusState { for tx in &mut self.transactions { if let Some(req) = &tx.request { if tx.response.is_none() && resp.matches(req) { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -139,6 +141,8 @@ impl ModbusState { for tx in &mut self.transactions { if let Some(resp) = &tx.response { if tx.request.is_none() && req.matches(resp) { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -184,6 +188,8 @@ impl ModbusState { match self.find_response_and_validate(&mut msg) { Some(tx) => { tx.set_events_from_flags(&msg.error_flags); + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; tx.request = Some(msg); } None => { @@ -210,6 +216,8 @@ impl ModbusState { } else { tx.set_events_from_flags(&msg.error_flags); } + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; tx.response = Some(msg); } None => { diff --git a/rust/src/mqtt/mqtt.rs b/rust/src/mqtt/mqtt.rs index 4079daa6de..e2905a4c60 100644 --- a/rust/src/mqtt/mqtt.rs +++ b/rust/src/mqtt/mqtt.rs @@ -174,6 +174,8 @@ impl MQTTState { if !tx.complete { if let Some(mpktid) = tx.pkt_id { if mpktid == pkt_id { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -196,6 +198,8 @@ impl MQTTState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.complete { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.complete = true; MQTTState::set_event(tx_old, MQTTEvent::TooManyTransactions); break; diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs index d4b472e1f6..15effa4f66 100644 --- a/rust/src/nfs/nfs.rs +++ b/rust/src/nfs/nfs.rs @@ -431,6 +431,8 @@ impl NFSState { // set at least one another transaction to the drop state for tx_old in &mut self.transactions { if !tx_old.request_done || !tx_old.response_done { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.request_done = true; tx_old.response_done = true; tx_old.is_file_closed = true; @@ -484,6 +486,8 @@ impl NFSState { pub fn mark_response_tx_done(&mut self, xid: u32, rpc_status: u32, nfs_status: u32, resp_handle: &[u8]) { if let Some(mytx) = self.get_tx_by_xid(xid) { + mytx.tx_data.updated_tc = true; + mytx.tx_data.updated_ts = true; mytx.response_done = true; mytx.rpc_response_status = rpc_status; mytx.nfs_response_status = nfs_status; @@ -736,6 +740,8 @@ impl NFSState { tx.tx_data.update_file_flags(self.state_data.file_flags); d.update_file_flags(tx.tx_data.file_flags); SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid); + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/pgsql/pgsql.rs b/rust/src/pgsql/pgsql.rs index 658c2326ff..ad57590b3a 100644 --- a/rust/src/pgsql/pgsql.rs +++ b/rust/src/pgsql/pgsql.rs @@ -208,6 +208,8 @@ impl PgsqlState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if tx_old.tx_res_state < PgsqlTxProgress::TxDone { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; // we don't check for TxReqDone for the majority of requests are basically completed // when they're parsed, as of now tx_old.tx_req_state = PgsqlTxProgress::TxFlushedOut; @@ -360,6 +362,7 @@ impl PgsqlState { // A simplified finite state machine for PostgreSQL v3 can be found at: // https://samadhiweb.com/blog/2013.04.28.graphviz.postgresv3.html if let Some(tx) = self.find_or_create_tx() { + tx.tx_data.updated_ts = true; tx.request = Some(request); if let Some(state) = new_state { if Self::request_is_complete(state) { @@ -518,6 +521,7 @@ impl PgsqlState { self.state_progress = state; } if let Some(tx) = self.find_or_create_tx() { + tx.tx_data.updated_tc = true; if tx.tx_res_state == PgsqlTxProgress::TxInit { tx.tx_res_state = PgsqlTxProgress::TxReceived; } diff --git a/rust/src/rfb/rfb.rs b/rust/src/rfb/rfb.rs index 810ed1d85d..5c226af3ab 100644 --- a/rust/src/rfb/rfb.rs +++ b/rust/src/rfb/rfb.rs @@ -166,7 +166,13 @@ impl RFBState { fn get_current_tx(&mut self) -> Option<&mut RFBTransaction> { let tx_id = self.tx_id; - self.transactions.iter_mut().find(|tx| tx.tx_id == tx_id) + let r = self.transactions.iter_mut().find(|tx| tx.tx_id == tx_id); + if let Some(tx) = r { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; + return Some(tx); + } + return None; } fn parse_request(&mut self, flow: *const Flow, stream_slice: StreamSlice) -> AppLayerResult { diff --git a/rust/src/smb/dcerpc.rs b/rust/src/smb/dcerpc.rs index 771e2538c2..d528b372a1 100644 --- a/rust/src/smb/dcerpc.rs +++ b/rust/src/smb/dcerpc.rs @@ -168,6 +168,8 @@ impl SMBState { _ => { false }, }; if found { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/smb/files.rs b/rust/src/smb/files.rs index bdc2619a17..e1de7d171a 100644 --- a/rust/src/smb/files.rs +++ b/rust/src/smb/files.rs @@ -126,6 +126,8 @@ impl SMBState { tx.tx_data.update_file_flags(self.state_data.file_flags); d.update_file_flags(tx.tx_data.file_flags); } + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -152,6 +154,8 @@ impl SMBState { tx.tx_data.update_file_flags(self.state_data.file_flags); d.update_file_flags(tx.tx_data.file_flags); } + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/smb/session.rs b/rust/src/smb/session.rs index be7866976d..241973fc2b 100644 --- a/rust/src/smb/session.rs +++ b/rust/src/smb/session.rs @@ -61,6 +61,8 @@ impl SMBState { _ => { false }, }; if hit { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/smb/smb.rs b/rust/src/smb/smb.rs index 9bba6e0345..1b34d94462 100644 --- a/rust/src/smb/smb.rs +++ b/rust/src/smb/smb.rs @@ -833,6 +833,8 @@ impl SMBState { for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) { index += 1; if !tx_old.request_done || !tx_old.response_done { + tx_old.tx_data.updated_tc = true; + tx_old.tx_data.updated_ts = true; tx_old.request_done = true; tx_old.response_done = true; tx_old.set_event(SMBEvent::TooManyTransactions); @@ -951,6 +953,8 @@ impl SMBState { false }; if found { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -975,6 +979,8 @@ impl SMBState { false }; if found { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -1013,6 +1019,8 @@ impl SMBState { _ => { false }, }; if found { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } @@ -1046,6 +1054,8 @@ impl SMBState { _ => { false }, }; if hit { + tx.tx_data.updated_tc = true; + tx.tx_data.updated_ts = true; return Some(tx); } } diff --git a/rust/src/ssh/ssh.rs b/rust/src/ssh/ssh.rs index a6a3871a8e..99c88c9d7c 100644 --- a/rust/src/ssh/ssh.rs +++ b/rust/src/ssh/ssh.rs @@ -415,6 +415,7 @@ pub unsafe extern "C" fn rs_ssh_parse_request( let state = &mut cast_pointer!(state, SSHState); let buf = stream_slice.as_slice(); let hdr = &mut state.transaction.cli_hdr; + state.transaction.tx_data.updated_ts = true; if hdr.flags < SSHConnectionState::SshStateBannerDone { return state.parse_banner(buf, false, pstate, flow, &stream_slice); } else { @@ -431,6 +432,7 @@ pub unsafe extern "C" fn rs_ssh_parse_response( let state = &mut cast_pointer!(state, SSHState); let buf = stream_slice.as_slice(); let hdr = &mut state.transaction.srv_hdr; + state.transaction.tx_data.updated_tc = true; if hdr.flags < SSHConnectionState::SshStateBannerDone { return state.parse_banner(buf, true, pstate, flow, &stream_slice); } else { diff --git a/src/app-layer-dnp3.c b/src/app-layer-dnp3.c index c995667ee0..a1daf3a2e1 100644 --- a/src/app-layer-dnp3.c +++ b/src/app-layer-dnp3.c @@ -882,6 +882,7 @@ static void DNP3HandleUserDataRequest(DNP3State *dnp3, const uint8_t *input, /* Update the saved transport header so subsequent segments * will be matched to this sequence number. */ tx->th = th; + tx->tx_data.updated_ts = true; } else { ah = (DNP3ApplicationHeader *)(input + sizeof(DNP3LinkHeader) + @@ -959,6 +960,7 @@ static void DNP3HandleUserDataResponse(DNP3State *dnp3, const uint8_t *input, /* Replace the transport header in the transaction with this * one in case there are more frames. */ tx->th = th; + tx->tx_data.updated_tc = true; } else { ah = (DNP3ApplicationHeader *)(input + offset); diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c index 15238b9f65..a6a1f632fc 100644 --- a/src/app-layer-ftp.c +++ b/src/app-layer-ftp.c @@ -724,6 +724,7 @@ static AppLayerResult FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserS SCReturnStruct(APP_LAYER_ERROR); } lasttx = tx; + tx->tx_data.updated_tc = true; if (state->command == FTP_COMMAND_UNKNOWN || tx->command_descriptor == NULL) { /* unknown */ tx->command_descriptor = &FtpCommands[FTP_COMMAND_MAX - 1]; @@ -1056,7 +1057,11 @@ static AppLayerResult FTPDataParse(Flow *f, FtpDataState *ftpdata_state, SCTxDataUpdateFileFlags(&ftpdata_state->tx_data, ftpdata_state->state_data.file_flags); if (ftpdata_state->tx_data.file_tx == 0) ftpdata_state->tx_data.file_tx = direction & (STREAM_TOSERVER | STREAM_TOCLIENT); - + if (direction & STREAM_TOSERVER) { + ftpdata_state->tx_data.updated_ts = true; + } else { + ftpdata_state->tx_data.updated_tc = true; + } /* we depend on detection engine for file pruning */ const uint16_t flags = FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction); int ret = 0; diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c index f1d3c56c16..328a10b72c 100644 --- a/src/app-layer-htp.c +++ b/src/app-layer-htp.c @@ -1449,6 +1449,7 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d) if (tx_ud == NULL) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.updated_ts = true; SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags); if (!tx_ud->response_body_init) { @@ -1580,6 +1581,7 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d) if (tx_ud == NULL) { SCReturnInt(HTP_OK); } + tx_ud->tx_data.updated_tc = true; SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags); if (!tx_ud->request_body_init) { tx_ud->request_body_init = 1; @@ -1686,6 +1688,7 @@ static int HTPCallbackRequestHasTrailer(htp_tx_t *tx) { HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated_ts = true; htud->request_has_trailers = 1; } return HTP_OK; @@ -1695,6 +1698,7 @@ static int HTPCallbackResponseHasTrailer(htp_tx_t *tx) { HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated_tc = true; htud->response_has_trailers = 1; } return HTP_OK; @@ -1737,6 +1741,8 @@ static int HTPCallbackRequestStart(htp_tx_t *tx) } tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files htp_tx_set_user_data(tx, tx_ud); + } else { + tx_ud->tx_data.updated_ts = true; } SCReturnInt(HTP_OK); } @@ -1777,6 +1783,8 @@ static int HTPCallbackResponseStart(htp_tx_t *tx) tx_ud->tx_data.file_tx = STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed. htp_tx_set_user_data(tx, tx_ud); + } else { + tx_ud->tx_data.updated_tc = true; } SCReturnInt(HTP_OK); } @@ -1828,6 +1836,7 @@ static int HTPCallbackRequestComplete(htp_tx_t *tx) HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated_ts = true; if (htud->tsflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOSERVER); @@ -1883,6 +1892,7 @@ static int HTPCallbackResponseComplete(htp_tx_t *tx) HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx); if (htud != NULL) { + htud->tx_data.updated_tc = true; if (htud->tcflags & HTP_FILENAME_SET) { SCLogDebug("closing file that was being stored"); (void)HTPFileClose(htud, NULL, 0, 0, STREAM_TOCLIENT); @@ -2001,6 +2011,7 @@ static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data) return HTP_OK; } tx_ud->request_headers_raw = ptmp; + tx_ud->tx_data.updated_ts = true; memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len, tx_data->data, tx_data->len); @@ -2023,6 +2034,7 @@ static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data) if (tx_ud == NULL) { return HTP_OK; } + tx_ud->tx_data.updated_tc = true; ptmp = HTPRealloc(tx_ud->response_headers_raw, tx_ud->response_headers_raw_len, tx_ud->response_headers_raw_len + tx_data->len); diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c index d1cacc572f..045fcb086e 100644 --- a/src/app-layer-parser.c +++ b/src/app-layer-parser.c @@ -935,7 +935,14 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir) (pkt_dir == STREAM_TOSERVER) ? ts_disrupt_flags : tc_disrupt_flags); AppLayerParserFileTxHousekeeping(f, tx, pkt_dir, (bool)pkt_dir_trunc); } - + if (txd) { + // should be reset by parser next time it updates the tx + if (pkt_dir & STREAM_TOSERVER) { + txd->updated_ts = false; + } else { + txd->updated_tc = false; + } + } const int tx_progress_tc = AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); if (tx_progress_tc < tx_end_state_tc) { diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c index ec4799605c..b2bf22a531 100644 --- a/src/app-layer-smtp.c +++ b/src/app-layer-smtp.c @@ -860,6 +860,9 @@ static int SMTPProcessReply( return 0; // to continue processing further } + if (state->curr_tx) { + state->curr_tx->tx_data.updated_tc = true; + } /* the reply code has to contain at least 3 bytes, to hold the 3 digit * reply code */ if (line->len < 3) { @@ -1164,6 +1167,7 @@ static int SMTPProcessRequest( if (frame != NULL && state->curr_tx) { AppLayerFrameSetTxId(frame, state->curr_tx->tx_id); } + tx->tx_data.updated_ts = true; state->toserver_data_count += (line->len + line->delim_len); diff --git a/src/app-layer-ssl.c b/src/app-layer-ssl.c index 05ba2239b2..e387c6cc46 100644 --- a/src/app-layer-ssl.c +++ b/src/app-layer-ssl.c @@ -2673,6 +2673,8 @@ static AppLayerResult SSLDecode(Flow *f, uint8_t direction, void *alstate, AppLayerParserState *pstate, StreamSlice stream_slice) { SSLState *ssl_state = (SSLState *)alstate; + ssl_state->tx_data.updated_tc = true; + ssl_state->tx_data.updated_ts = true; uint32_t counter = 0; ssl_state->f = f; const uint8_t *input = StreamSliceGetData(&stream_slice); diff --git a/src/detect.c b/src/detect.c index 03fa843706..802cf690ce 100644 --- a/src/detect.c +++ b/src/detect.c @@ -1294,6 +1294,12 @@ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alpro DetectTransaction no_tx = NO_TX; return no_tx; } + const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags); + bool updated = (flow_flags & STREAM_TOSERVER) ? txd->updated_ts : txd->updated_tc; + if (!updated && tx_progress < tx_end_state && ((flow_flags & STREAM_EOF) == 0)) { + DetectTransaction no_tx = NO_TX; + return no_tx; + } uint64_t detect_flags = (flow_flags & STREAM_TOSERVER) ? txd->detect_flags_ts : txd->detect_flags_tc; if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) { @@ -1310,7 +1316,6 @@ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alpro return no_tx; } - const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags); const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1; DetectEngineState *tx_de_state = txd->de_state; DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL; diff --git a/src/output-tx.c b/src/output-tx.c index 40b8877067..8e37b1b5f2 100644 --- a/src/output-tx.c +++ b/src/output-tx.c @@ -392,7 +392,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser); uint64_t max_id = tx_id; int logged = 0; - int gap = 0; + bool gap = false; const bool support_files = AppLayerParserSupportsFiles(ipproto, alproto); const uint8_t pkt_dir = STREAM_FLAGS_FOR_PACKET(p); @@ -415,15 +415,6 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) tx_id = ires.tx_id; SCLogDebug("STARTING tx_id %" PRIu64 ", tx %p", tx_id, tx); - const int tx_progress_ts = - AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags); - const int tx_progress_tc = - AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); - const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc); - - SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file, - op_thread_data->filedata); - AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx); if (unlikely(txd == NULL)) { SCLogDebug("NO TXD"); @@ -433,6 +424,15 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) goto next_tx; } + const int tx_progress_ts = + AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags); + const int tx_progress_tc = + AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags); + const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc); + + SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file, + op_thread_data->filedata); + if (file_logging_active) { if (AppLayerParserIsFileTx(txd)) { // need to process each tx that might be a file tx, // even if there are not files (yet) @@ -467,6 +467,11 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) } } SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags); + if (!txd->updated_tc && !txd->updated_ts && !(tx_progress_ts == complete_ts) && + !(tx_progress_tc == complete_tc) && !ts_eof && !tc_eof) { + gap = true; + goto next_tx; + } if (list[ALPROTO_UNKNOWN] != 0) { OutputTxLogList0(tv, op_thread_data, p, f, tx, tx_id); @@ -517,7 +522,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data) max_id = tx_id; SCLogDebug("max_id %" PRIu64, max_id); } else { - gap = 1; + gap = true; } next_tx: if (!ires.has_next)