app-layer: track modified/processed txs

To optimize detection, and logging, to avoid going through
all the live transactions when only a few were modified.

Two boolean fields are added to the tx data: updated_tc and ts
The app-layer parsers are now responsible to set these when
needed, and the logging and detection uses them to skip
transactions that were not updated.

There may some more optimization remaining by when we set
both updated_tc and updated_ts in functions returning
a mutable transaction, by checking if all the callers
are called in one direction only (request or response)

Ticket: 7087
(cherry picked from commit b02557ac7dd6b55187309cee93f849ee2917d0cb)

diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs
index 551a125d979461b2a5b038a48edefddfe04d4cab..023d169585caa18a2987b17013e6957190022663 100644 (file)
--- a/rust/src/applayer.rs
+++ b/rust/src/applayer.rs
@@ -98,6 +98,12 @@ pub struct AppLayerTxData {
     /// config: log flags
     pub config: AppLayerTxConfig,
 
+    /// The tx has been updated and needs to be processed : detection, logging, cleaning
+    /// It can then be skipped until new data arrives.
+    /// There is a boolean for both directions : to server and to client
+    pub updated_tc: bool,
+    pub updated_ts: bool,
+
     /// logger flags for tx logging api
     logged: LoggerFlags,
 
@@ -152,6 +158,8 @@ impl AppLayerTxData {
             files_stored: 0,
             file_flags: 0,
             file_tx: 0,
+            updated_tc: true,
+            updated_ts: true,
             detect_flags_ts: 0,
             detect_flags_tc: 0,
             de_state: std::ptr::null_mut(),
@@ -162,9 +170,9 @@ impl AppLayerTxData {
     /// Create new AppLayerTxData for a transaction in a single
     /// direction.
     pub fn for_direction(direction: Direction) -> Self {
-        let (detect_flags_ts, detect_flags_tc) = match direction {
-            Direction::ToServer => (0, APP_LAYER_TX_SKIP_INSPECT_FLAG),
-            Direction::ToClient => (APP_LAYER_TX_SKIP_INSPECT_FLAG, 0),
+        let (detect_flags_ts, detect_flags_tc, updated_ts, updated_tc) = match direction {
+            Direction::ToServer => (0, APP_LAYER_TX_SKIP_INSPECT_FLAG, true, false),
+            Direction::ToClient => (APP_LAYER_TX_SKIP_INSPECT_FLAG, 0, false, true),
         };
         Self {
             config: AppLayerTxConfig::new(),
@@ -174,6 +182,8 @@ impl AppLayerTxData {
             files_stored: 0,
             file_flags: 0,
             file_tx: 0,
+            updated_tc,
+            updated_ts,
             detect_flags_ts,
             detect_flags_tc,
             de_state: std::ptr::null_mut(),

diff --git a/rust/src/applayertemplate/template.rs b/rust/src/applayertemplate/template.rs
index dbbc7841fad54fe971aca2ed1b269970ff3169f0..0fc6730b4e246a883385cfc84267559ab0b00ecb 100644 (file)
--- a/rust/src/applayertemplate/template.rs
+++ b/rust/src/applayertemplate/template.rs
@@ -200,6 +200,7 @@ impl TemplateState {
                     start = rem;
 
                     if let Some(tx) =  self.find_request() {
+                        tx.tx_data.updated_tc = true;
                         tx.response = Some(response);
                         SCLogNotice!("Found response for request:");
                         SCLogNotice!("- Request: {:?}", tx.request);

diff --git a/rust/src/dcerpc/dcerpc.rs b/rust/src/dcerpc/dcerpc.rs
index 345d4665662463ef227822e0ec902d0f65a4c2a9..b99b23d3f56206bdcb13213bf9ba20108c98096c 100644 (file)
--- a/rust/src/dcerpc/dcerpc.rs
+++ b/rust/src/dcerpc/dcerpc.rs
@@ -361,6 +361,8 @@ impl DCERPCState {
             for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) {
                 index += 1;
                 if !tx_old.req_done || !tx_old.resp_done {
+                    tx_old.tx_data.updated_tc = true;
+                    tx_old.tx_data.updated_ts = true;
                     tx_old.req_done = true;
                     tx_old.resp_done = true;
                     break;
@@ -537,6 +539,8 @@ impl DCERPCState {
                         }
                     }
                 }
+                tx.tx_data.updated_tc = true;
+                tx.tx_data.updated_ts = true;
                 return Some(tx);
             }
         }

diff --git a/rust/src/dcerpc/dcerpc_udp.rs b/rust/src/dcerpc/dcerpc_udp.rs
index 461050c00fe3bbc674a089e3fbc97236a9ab561f..700e38fe36b1253125cae350d1b4441e2847ebb5 100644 (file)
--- a/rust/src/dcerpc/dcerpc_udp.rs
+++ b/rust/src/dcerpc/dcerpc_udp.rs
@@ -88,6 +88,8 @@ impl DCERPCUDPState {
             for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) {
                 index += 1;
                 if !tx_old.req_done || !tx_old.resp_done {
+                    tx_old.tx_data.updated_tc = true;
+                    tx_old.tx_data.updated_ts = true;
                     tx_old.req_done = true;
                     tx_old.resp_done = true;
                     break;
@@ -165,6 +167,8 @@ impl DCERPCUDPState {
         }
 
         if let Some(tx) = otx {
+            tx.tx_data.updated_tc = true;
+            tx.tx_data.updated_ts = true;
             let done = (hdr.flags1 & PFCL1_FRAG) == 0 || (hdr.flags1 & PFCL1_LASTFRAG) != 0;
 
             match hdr.pkt_type {

diff --git a/rust/src/http2/http2.rs b/rust/src/http2/http2.rs
index d679c99e3e828b2e133ec11d7dbe84844ea0a287..eace93ccc36f1a72826c32fff28a0efdf6d590f4 100644 (file)
--- a/rust/src/http2/http2.rs
+++ b/rust/src/http2/http2.rs
@@ -655,6 +655,8 @@ impl HTTP2State {
             let tx = &mut self.transactions[index - 1];
             tx.tx_data.update_file_flags(self.state_data.file_flags);
             tx.update_file_flags(tx.tx_data.file_flags);
+            tx.tx_data.updated_tc = true;
+            tx.tx_data.updated_ts = true;
             return Some(tx);
         } else {
             // do not use SETTINGS_MAX_CONCURRENT_STREAMS as it can grow too much
@@ -667,6 +669,8 @@ impl HTTP2State {
                     tx_old.set_event(HTTP2Event::TooManyStreams);
                     // use a distinct state, even if we do not log it
                     tx_old.state = HTTP2TransactionState::HTTP2StateTodrop;
+                    tx_old.tx_data.updated_tc = true;
+                    tx_old.tx_data.updated_ts = true;
                 }
                 return None;
             }

diff --git a/rust/src/modbus/modbus.rs b/rust/src/modbus/modbus.rs
index b09c26b11894657239420f1cfa5423f81da9c700..80a3f16f0b033769b80b450b109510b5c2ebc60a 100644 (file)
--- a/rust/src/modbus/modbus.rs
+++ b/rust/src/modbus/modbus.rs
@@ -124,6 +124,8 @@ impl ModbusState {
         for tx in &mut self.transactions {
             if let Some(req) = &tx.request {
                 if tx.response.is_none() && resp.matches(req) {
+                    tx.tx_data.updated_tc = true;
+                    tx.tx_data.updated_ts = true;
                     return Some(tx);
                 }
             }
@@ -139,6 +141,8 @@ impl ModbusState {
         for tx in &mut self.transactions {
             if let Some(resp) = &tx.response {
                 if tx.request.is_none() && req.matches(resp) {
+                    tx.tx_data.updated_tc = true;
+                    tx.tx_data.updated_ts = true;
                     return Some(tx);
                 }
             }
@@ -184,6 +188,8 @@ impl ModbusState {
                             match self.find_response_and_validate(&mut msg) {
                                 Some(tx) => {
                                     tx.set_events_from_flags(&msg.error_flags);
+                                    tx.tx_data.updated_tc = true;
+                                    tx.tx_data.updated_ts = true;
                                     tx.request = Some(msg);
                                 }
                                 None => {
@@ -210,6 +216,8 @@ impl ModbusState {
                                 } else {
                                     tx.set_events_from_flags(&msg.error_flags);
                                 }
+                                tx.tx_data.updated_tc = true;
+                                tx.tx_data.updated_ts = true;
                                 tx.response = Some(msg);
                             }
                             None => {

diff --git a/rust/src/mqtt/mqtt.rs b/rust/src/mqtt/mqtt.rs
index b31b6eacea5d7b5bc1ba97d7428f304c67fbe88a..3fd35b30a960ba66d681c12053486e4161992d2f 100644 (file)
--- a/rust/src/mqtt/mqtt.rs
+++ b/rust/src/mqtt/mqtt.rs
@@ -174,6 +174,8 @@ impl MQTTState {
             if !tx.complete {
                 if let Some(mpktid) = tx.pkt_id {
                     if mpktid == pkt_id {
+                        tx.tx_data.updated_tc = true;
+                        tx.tx_data.updated_ts = true;
                         return Some(tx);
                     }
                 }
@@ -196,6 +198,8 @@ impl MQTTState {
             for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) {
                 index += 1;
                 if !tx_old.complete {
+                    tx_old.tx_data.updated_tc = true;
+                    tx_old.tx_data.updated_ts = true;
                     tx_old.complete = true;
                     MQTTState::set_event(tx_old, MQTTEvent::TooManyTransactions);
                     break;

diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs
index b472689a5104056a79c1842742c00a3a2136c53f..98e6f590578782e10de88016a09024d9a851ac7d 100644 (file)
--- a/rust/src/nfs/nfs.rs
+++ b/rust/src/nfs/nfs.rs
@@ -431,6 +431,8 @@ impl NFSState {
             // set at least one another transaction to the drop state
             for tx_old in &mut self.transactions {
                 if !tx_old.request_done || !tx_old.response_done {
+                    tx_old.tx_data.updated_tc = true;
+                    tx_old.tx_data.updated_ts = true;
                     tx_old.request_done = true;
                     tx_old.response_done = true;
                     tx_old.is_file_closed = true;
@@ -500,6 +502,8 @@ impl NFSState {
     pub fn mark_response_tx_done(&mut self, xid: u32, rpc_status: u32, nfs_status: u32, resp_handle: &[u8])
     {
         if let Some(mytx) = self.get_tx_by_xid(xid) {
+            mytx.tx_data.updated_tc = true;
+            mytx.tx_data.updated_ts = true;
             mytx.response_done = true;
             mytx.rpc_response_status = rpc_status;
             mytx.nfs_response_status = nfs_status;
@@ -756,6 +760,8 @@ impl NFSState {
                     tx.tx_data.update_file_flags(self.state_data.file_flags);
                     d.update_file_flags(tx.tx_data.file_flags);
                     SCLogDebug!("Found NFS file TX with ID {} XID {:04X}", tx.id, tx.xid);
+                    tx.tx_data.updated_tc = true;
+                    tx.tx_data.updated_ts = true;
                     return Some(tx);
                 }
             }

diff --git a/rust/src/pgsql/pgsql.rs b/rust/src/pgsql/pgsql.rs
index 4b2b9213f1a89d4ab3978b763437706012bd52fe..9bea2b877d67db45023606c8cfdbe9533d235f0e 100644 (file)
--- a/rust/src/pgsql/pgsql.rs
+++ b/rust/src/pgsql/pgsql.rs
@@ -209,6 +209,8 @@ impl PgsqlState {
             for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) {
                 index += 1;
                 if tx_old.tx_res_state < PgsqlTxProgress::TxDone {
+                    tx_old.tx_data.updated_tc = true;
+                    tx_old.tx_data.updated_ts = true;
                     // we don't check for TxReqDone for the majority of requests are basically completed
                     // when they're parsed, as of now
                     tx_old.tx_req_state = PgsqlTxProgress::TxFlushedOut;
@@ -361,6 +363,7 @@ impl PgsqlState {
                     // A simplified finite state machine for PostgreSQL v3 can be found at:
                     // https://samadhiweb.com/blog/2013.04.28.graphviz.postgresv3.html
                     if let Some(tx) = self.find_or_create_tx() {
+                        tx.tx_data.updated_ts = true;
                         tx.request = Some(request);
                         if let Some(state) = new_state {
                             if Self::request_is_complete(state) {
@@ -519,6 +522,7 @@ impl PgsqlState {
                         self.state_progress = state;
                     }
                     if let Some(tx) = self.find_or_create_tx() {
+                        tx.tx_data.updated_tc = true;
                         if tx.tx_res_state == PgsqlTxProgress::TxInit {
                             tx.tx_res_state = PgsqlTxProgress::TxReceived;
                         }

diff --git a/rust/src/rfb/rfb.rs b/rust/src/rfb/rfb.rs
index 8c3381345012077a431c0d0853bff3733f9dc214..4d16c226192669274495d004e722b4724ac695c4 100644 (file)
--- a/rust/src/rfb/rfb.rs
+++ b/rust/src/rfb/rfb.rs
@@ -165,7 +165,13 @@ impl RFBState {
 
     fn get_current_tx(&mut self) -> Option<&mut RFBTransaction> {
         let tx_id = self.tx_id;
-        self.transactions.iter_mut().find(|tx| tx.tx_id == tx_id)
+        let r = self.transactions.iter_mut().find(|tx| tx.tx_id == tx_id);
+        if let Some(tx) = r {
+            tx.tx_data.updated_tc = true;
+            tx.tx_data.updated_ts = true;
+            return Some(tx);
+        }
+        return None;
     }
 
     fn parse_request(&mut self, flow: *const Flow, stream_slice: StreamSlice) -> AppLayerResult {

diff --git a/rust/src/smb/dcerpc.rs b/rust/src/smb/dcerpc.rs
index de6b8def73b3742f769fa2bd167428b26cf33ce9..6c2a2f9345df1d9c40b45937b2101e8288073bcd 100644 (file)
--- a/rust/src/smb/dcerpc.rs
+++ b/rust/src/smb/dcerpc.rs
@@ -168,6 +168,8 @@ impl SMBState {
                 _ => { false },
             };
             if found {
+                tx.tx_data.updated_tc = true;
+                tx.tx_data.updated_ts = true;
                 return Some(tx);
             }
         }

diff --git a/rust/src/smb/files.rs b/rust/src/smb/files.rs
index b290357428ee7f0f73d7a85c686742c65950541b..ef9e620bb8f3475fb49057922181fdc953202252 100644 (file)
--- a/rust/src/smb/files.rs
+++ b/rust/src/smb/files.rs
@@ -126,6 +126,8 @@ impl SMBState {
                     tx.tx_data.update_file_flags(self.state_data.file_flags);
                     d.update_file_flags(tx.tx_data.file_flags);
                 }
+                tx.tx_data.updated_tc = true;
+                tx.tx_data.updated_ts = true;
                 return Some(tx);
             }
         }
@@ -152,6 +154,8 @@ impl SMBState {
                     tx.tx_data.update_file_flags(self.state_data.file_flags);
                     d.update_file_flags(tx.tx_data.file_flags);
                 }
+                tx.tx_data.updated_tc = true;
+                tx.tx_data.updated_ts = true;
                 return Some(tx);
             }
         }

diff --git a/rust/src/smb/session.rs b/rust/src/smb/session.rs
index be7866976dc9822b9484965a1950fa14fc373325..241973fc2b4d9b47a027c8ec165175669d91f2b2 100644 (file)
--- a/rust/src/smb/session.rs
+++ b/rust/src/smb/session.rs
@@ -61,6 +61,8 @@ impl SMBState {
                 _ => { false },
             };
             if hit {
+                tx.tx_data.updated_tc = true;
+                tx.tx_data.updated_ts = true;
                 return Some(tx);
             }
         }

diff --git a/rust/src/smb/smb.rs b/rust/src/smb/smb.rs
index b936f89211ac14ee8b27c9e4b08e79073792cde1..97d4bf4f22722b0f9fa701533d3360acba924aa2 100644 (file)
--- a/rust/src/smb/smb.rs
+++ b/rust/src/smb/smb.rs
@@ -806,6 +806,8 @@ impl SMBState {
             for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) {
                 index += 1;
                 if !tx_old.request_done || !tx_old.response_done {
+                    tx_old.tx_data.updated_tc = true;
+                    tx_old.tx_data.updated_ts = true;
                     tx_old.request_done = true;
                     tx_old.response_done = true;
                     tx_old.set_event(SMBEvent::TooManyTransactions);
@@ -924,6 +926,8 @@ impl SMBState {
                 false
             };
             if found {
+                tx.tx_data.updated_tc = true;
+                tx.tx_data.updated_ts = true;
                 return Some(tx);
             }
         }
@@ -948,6 +952,8 @@ impl SMBState {
                 false
             };
             if found {
+                tx.tx_data.updated_tc = true;
+                tx.tx_data.updated_ts = true;
                 return Some(tx);
             }
         }
@@ -986,6 +992,8 @@ impl SMBState {
                 _ => { false },
             };
             if found {
+                tx.tx_data.updated_tc = true;
+                tx.tx_data.updated_ts = true;
                 return Some(tx);
             }
         }
@@ -1019,6 +1027,8 @@ impl SMBState {
                 _ => { false },
             };
             if hit {
+                tx.tx_data.updated_tc = true;
+                tx.tx_data.updated_ts = true;
                 return Some(tx);
             }
         }

diff --git a/rust/src/ssh/ssh.rs b/rust/src/ssh/ssh.rs
index a0586894f9fb8efe352dc3e67cd27f3a55098ccb..32526132ec10da4696fa65bbd6101cac94bdd5f1 100644 (file)
--- a/rust/src/ssh/ssh.rs
+++ b/rust/src/ssh/ssh.rs
@@ -359,6 +359,7 @@ pub unsafe extern "C" fn rs_ssh_parse_request(
     let state = &mut cast_pointer!(state, SSHState);
     let buf = stream_slice.as_slice();
     let hdr = &mut state.transaction.cli_hdr;
+    state.transaction.tx_data.updated_ts = true;
     if hdr.flags < SSHConnectionState::SshStateBannerDone {
         return state.parse_banner(buf, false, pstate);
     } else {
@@ -375,6 +376,7 @@ pub unsafe extern "C" fn rs_ssh_parse_response(
     let state = &mut cast_pointer!(state, SSHState);
     let buf = stream_slice.as_slice();
     let hdr = &mut state.transaction.srv_hdr;
+    state.transaction.tx_data.updated_tc = true;
     if hdr.flags < SSHConnectionState::SshStateBannerDone {
         return state.parse_banner(buf, true, pstate);
     } else {

diff --git a/src/app-layer-dnp3.c b/src/app-layer-dnp3.c
index 9501b9f5ea578f877486a6136815027ecb06d7a2..26422cb18021ab4a9f0bae4e601adc5577c117f6 100644 (file)
--- a/src/app-layer-dnp3.c
+++ b/src/app-layer-dnp3.c
@@ -894,6 +894,7 @@ static void DNP3HandleUserDataRequest(DNP3State *dnp3, const uint8_t *input,
         /* Update the saved transport header so subsequent segments
          * will be matched to this sequence number. */
         tx->th = th;
+        tx->tx_data.updated_ts = true;
     }
     else {
         ah = (DNP3ApplicationHeader *)(input + sizeof(DNP3LinkHeader) +
@@ -971,6 +972,7 @@ static void DNP3HandleUserDataResponse(DNP3State *dnp3, const uint8_t *input,
         /* Replace the transport header in the transaction with this
          * one in case there are more frames. */
         tx->th = th;
+        tx->tx_data.updated_tc = true;
     }
     else {
         ah = (DNP3ApplicationHeader *)(input + offset);

diff --git a/src/app-layer-ftp.c b/src/app-layer-ftp.c
index d1804c61e3baef8c27742482a7e35b284aa5f2af..9ee93989d29e7b5b418bb939eb8fac2289cc1961 100644 (file)
--- a/src/app-layer-ftp.c
+++ b/src/app-layer-ftp.c
@@ -718,6 +718,7 @@ static AppLayerResult FTPParseResponse(Flow *f, void *ftp_state, AppLayerParserS
             SCReturnStruct(APP_LAYER_ERROR);
         }
         lasttx = tx;
+        tx->tx_data.updated_tc = true;
         if (state->command == FTP_COMMAND_UNKNOWN || tx->command_descriptor == NULL) {
             /* unknown */
             tx->command_descriptor = &FtpCommands[FTP_COMMAND_MAX - 1];
@@ -1008,7 +1009,11 @@ static AppLayerResult FTPDataParse(Flow *f, FtpDataState *ftpdata_state,
     SCTxDataUpdateFileFlags(&ftpdata_state->tx_data, ftpdata_state->state_data.file_flags);
     if (ftpdata_state->tx_data.file_tx == 0)
         ftpdata_state->tx_data.file_tx = direction & (STREAM_TOSERVER | STREAM_TOCLIENT);
-
+    if (direction & STREAM_TOSERVER) {
+        ftpdata_state->tx_data.updated_ts = true;
+    } else {
+        ftpdata_state->tx_data.updated_tc = true;
+    }
     /* we depend on detection engine for file pruning */
     const uint16_t flags = FileFlowFlagsToFlags(ftpdata_state->tx_data.file_flags, direction);
     int ret = 0;

diff --git a/src/app-layer-htp.c b/src/app-layer-htp.c
index 84cfd2148cf8478eb1bf61fd4d5dfb3844a1069d..16f832fd5d8d6b6a2493163819ecb0c99d66c16a 100644 (file)
--- a/src/app-layer-htp.c
+++ b/src/app-layer-htp.c
@@ -1877,6 +1877,7 @@ static int HTPCallbackRequestBodyData(htp_tx_data_t *d)
     if (tx_ud == NULL) {
         SCReturnInt(HTP_OK);
     }
+    tx_ud->tx_data.updated_ts = true;
     SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags);
 
     if (!tx_ud->response_body_init) {
@@ -2007,6 +2008,7 @@ static int HTPCallbackResponseBodyData(htp_tx_data_t *d)
     if (tx_ud == NULL) {
         SCReturnInt(HTP_OK);
     }
+    tx_ud->tx_data.updated_tc = true;
     SCTxDataUpdateFileFlags(&tx_ud->tx_data, hstate->state_data.file_flags);
     if (!tx_ud->request_body_init) {
         tx_ud->request_body_init = 1;
@@ -2113,6 +2115,7 @@ static int HTPCallbackRequestHasTrailer(htp_tx_t *tx)
 {
     HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
     if (htud != NULL) {
+        htud->tx_data.updated_ts = true;
         htud->request_has_trailers = 1;
     }
     return HTP_OK;
@@ -2122,6 +2125,7 @@ static int HTPCallbackResponseHasTrailer(htp_tx_t *tx)
 {
     HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
     if (htud != NULL) {
+        htud->tx_data.updated_tc = true;
         htud->response_has_trailers = 1;
     }
     return HTP_OK;
@@ -2164,6 +2168,8 @@ static int HTPCallbackRequestStart(htp_tx_t *tx)
         }
         tx_ud->tx_data.file_tx = STREAM_TOSERVER | STREAM_TOCLIENT; // each http tx may xfer files
         htp_tx_set_user_data(tx, tx_ud);
+    } else {
+        tx_ud->tx_data.updated_ts = true;
     }
     SCReturnInt(HTP_OK);
 }
@@ -2204,6 +2210,8 @@ static int HTPCallbackResponseStart(htp_tx_t *tx)
         tx_ud->tx_data.file_tx =
                 STREAM_TOCLIENT; // each http tx may xfer files. Toserver already missed.
         htp_tx_set_user_data(tx, tx_ud);
+    } else {
+        tx_ud->tx_data.updated_tc = true;
     }
     SCReturnInt(HTP_OK);
 }
@@ -2255,6 +2263,7 @@ static int HTPCallbackRequestComplete(htp_tx_t *tx)
 
     HtpTxUserData *htud = (HtpTxUserData *)htp_tx_get_user_data(tx);
     if (htud != NULL) {
+        htud->tx_data.updated_ts = true;
         if (htud->tsflags & HTP_FILENAME_SET) {
             SCLogDebug("closing file that was being stored");
             (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOSERVER);
@@ -2310,6 +2319,7 @@ static int HTPCallbackResponseComplete(htp_tx_t *tx)
 
     HtpTxUserData *htud = (HtpTxUserData *) htp_tx_get_user_data(tx);
     if (htud != NULL) {
+        htud->tx_data.updated_tc = true;
         if (htud->tcflags & HTP_FILENAME_SET) {
             SCLogDebug("closing file that was being stored");
             (void)HTPFileClose(hstate, htud, NULL, 0, 0, STREAM_TOCLIENT);
@@ -2428,6 +2438,7 @@ static int HTPCallbackRequestHeaderData(htp_tx_data_t *tx_data)
         return HTP_OK;
     }
     tx_ud->request_headers_raw = ptmp;
+    tx_ud->tx_data.updated_ts = true;
 
     memcpy(tx_ud->request_headers_raw + tx_ud->request_headers_raw_len,
            tx_data->data, tx_data->len);
@@ -2450,6 +2461,7 @@ static int HTPCallbackResponseHeaderData(htp_tx_data_t *tx_data)
     if (tx_ud == NULL) {
         return HTP_OK;
     }
+    tx_ud->tx_data.updated_tc = true;
     ptmp = HTPRealloc(tx_ud->response_headers_raw,
                      tx_ud->response_headers_raw_len,
                      tx_ud->response_headers_raw_len + tx_data->len);

diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index a051616dd3b94990ec76b787c8e53835c90aa06d..5d33dca714158c9b962126344cdcbea6d2d43717 100644 (file)
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -975,7 +975,14 @@ void AppLayerParserTransactionsCleanup(Flow *f, const uint8_t pkt_dir)
 
             AppLayerParserFileTxHousekeeping(f, tx, pkt_dir, (bool)pkt_dir_trunc);
         }
-
+        if (txd) {
+            // should be reset by parser next time it updates the tx
+            if (pkt_dir & STREAM_TOSERVER) {
+                txd->updated_ts = false;
+            } else {
+                txd->updated_tc = false;
+            }
+        }
         const int tx_progress_tc =
                 AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags);
         if (tx_progress_tc < tx_end_state_tc) {

diff --git a/src/app-layer-smtp.c b/src/app-layer-smtp.c
index 5a08c02ce46ae16d2e370c972ef9359d22ba198a..475d0abe12a36f80ee5c287aaca50b2d1df4cbdc 100644 (file)
--- a/src/app-layer-smtp.c
+++ b/src/app-layer-smtp.c
@@ -912,6 +912,9 @@ static int SMTPProcessReply(SMTPState *state, Flow *f, AppLayerParserState *psta
         return 0; // to continue processing further
     }
 
+    if (state->curr_tx) {
+        state->curr_tx->tx_data.updated_tc = true;
+    }
     /* the reply code has to contain at least 3 bytes, to hold the 3 digit
      * reply code */
     if (line->len < 3) {
@@ -1204,6 +1207,7 @@ static int SMTPProcessRequest(SMTPState *state, Flow *f, AppLayerParserState *ps
         StreamTcpReassemblySetMinInspectDepth(f->protoctx, STREAM_TOSERVER,
                 smtp_config.content_inspect_min_size);
     }
+    tx->tx_data.updated_ts = true;
 
     state->toserver_data_count += (line->len + line->delim_len);
 

diff --git a/src/app-layer-ssl.c b/src/app-layer-ssl.c
index 43d01c7885cab6b652624fb18e7944e5a684f507..574881789aa78b5723c39c73f8363c0bd2aed0a2 100644 (file)
--- a/src/app-layer-ssl.c
+++ b/src/app-layer-ssl.c
@@ -2665,6 +2665,8 @@ static AppLayerResult SSLDecode(Flow *f, uint8_t direction, void *alstate,
         AppLayerParserState *pstate, StreamSlice stream_slice)
 {
     SSLState *ssl_state = (SSLState *)alstate;
+    ssl_state->tx_data.updated_tc = true;
+    ssl_state->tx_data.updated_ts = true;
     uint32_t counter = 0;
     ssl_state->f = f;
     const uint8_t *input = StreamSliceGetData(&stream_slice);

diff --git a/src/detect.c b/src/detect.c
index 82aaafcb94e1d5512f79c08ef41b4a94f45f1cb6..bd3f1240cbd6c5d35fb7f9ba538961058eb9a097 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -1275,6 +1275,12 @@ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alpro
         DetectTransaction no_tx = NO_TX;
         return no_tx;
     }
+    const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags);
+    bool updated = (flow_flags & STREAM_TOSERVER) ? txd->updated_ts : txd->updated_tc;
+    if (!updated && tx_progress < tx_end_state && ((flow_flags & STREAM_EOF) == 0)) {
+        DetectTransaction no_tx = NO_TX;
+        return no_tx;
+    }
     uint64_t detect_flags =
             (flow_flags & STREAM_TOSERVER) ? txd->detect_flags_ts : txd->detect_flags_tc;
     if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) {
@@ -1291,7 +1297,6 @@ static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alpro
         return no_tx;
     }
 
-    const int tx_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx_ptr, flow_flags);
     const int dir_int = (flow_flags & STREAM_TOSERVER) ? 0 : 1;
     DetectEngineState *tx_de_state = txd->de_state;
     DetectEngineStateDirection *tx_dir_state = tx_de_state ? &tx_de_state->dir_state[dir_int] : NULL;

diff --git a/src/output-tx.c b/src/output-tx.c
index 042b424adec9353925fb0dcf73198016e50ddbbb..0936bd054f33694abcab797f8594bdb013507766 100644 (file)
--- a/src/output-tx.c
+++ b/src/output-tx.c
@@ -394,7 +394,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
     uint64_t tx_id = AppLayerParserGetTransactionLogId(f->alparser);
     uint64_t max_id = tx_id;
     int logged = 0;
-    int gap = 0;
+    bool gap = false;
     const bool support_files = AppLayerParserSupportsFiles(ipproto, alproto);
     const uint8_t pkt_dir = STREAM_FLAGS_FOR_PACKET(p);
 
@@ -417,15 +417,6 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
         tx_id = ires.tx_id;
         SCLogDebug("STARTING tx_id %" PRIu64 ", tx %p", tx_id, tx);
 
-        const int tx_progress_ts =
-                AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags);
-        const int tx_progress_tc =
-                AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags);
-        const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc);
-
-        SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file,
-                op_thread_data->filedata);
-
         AppLayerTxData *txd = AppLayerParserGetTxData(ipproto, alproto, tx);
         if (unlikely(txd == NULL)) {
             SCLogDebug("NO TXD");
@@ -435,6 +426,15 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
             goto next_tx;
         }
 
+        const int tx_progress_ts =
+                AppLayerParserGetStateProgress(ipproto, alproto, tx, ts_disrupt_flags);
+        const int tx_progress_tc =
+                AppLayerParserGetStateProgress(ipproto, alproto, tx, tc_disrupt_flags);
+        const bool tx_complete = (tx_progress_ts == complete_ts && tx_progress_tc == complete_tc);
+
+        SCLogDebug("file_thread_data %p filedata_thread_data %p", op_thread_data->file,
+                op_thread_data->filedata);
+
         if (file_logging_active) {
             if (AppLayerParserIsFileTx(txd)) { // need to process each tx that might be a file tx,
                                                // even if there are not files (yet)
@@ -469,6 +469,11 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
             }
         }
         SCLogDebug("logger: expect %08x, have %08x", logger_expectation, txd->logged.flags);
+        if (!txd->updated_tc && !txd->updated_ts && !(tx_progress_ts == complete_ts) &&
+                !(tx_progress_tc == complete_tc) && !ts_eof && !tc_eof) {
+            gap = true;
+            goto next_tx;
+        }
 
         if (list[ALPROTO_UNKNOWN] != 0) {
             OutputTxLogList0(tv, op_thread_data, p, f, tx, tx_id);
@@ -519,7 +524,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
             max_id = tx_id;
             SCLogDebug("max_id %" PRIu64, max_id);
         } else {
-            gap = 1;
+            gap = true;
         }
 next_tx:
         if (!ires.has_next)