app-layer: add tx iterator API

Until now, the transaction space is assumed to be terse. Transactions
are handled sequentially so the difference between the lowest and highest
active tx id's is small. For this reason the logic of walking every id
between the 'minimum' and max id made sense. The space might look like:

    [..........TTTT]

Here the looping starts at the first T and loops 4 times.

This assumption isn't a great fit though. A protocol like NFS has 2 types
of transactions. Long running file transfer transactions and short lived
request/reply pairs are causing the id space to be sparse. This leads to
a lot of unnecessary looping in various parts of the engine, but most
prominently: detection, tx house keeping and tx logging.

    [.T..T...TTTT.T]

Here the looping starts at the first T and loops for every spot, even
those where no tx exists anymore.

Cases have been observed where the lowest tx id was 2 and the highest
was 50k. This lead to a lot of unnecessary looping.

This patch add an alternative approach. It allows a protocol to register
an iterator function, that simply returns the next transaction until
all transactions are returned. To do this it uses a bit of state the
caller must keep.

The registration is optional. If no iterator is registered the old
behaviour will be used.

diff --git a/rust/gen-c-headers.py b/rust/gen-c-headers.py
index 77880a78251ddef481218f4401fd94133612e547..a6e803a9bac336db62b29e508689adf1557037c4 100755 (executable)
--- a/rust/gen-c-headers.py
+++ b/rust/gen-c-headers.py
@@ -88,6 +88,8 @@ type_map = {
     "core::AppLayerDecoderEvents": "AppLayerDecoderEvents",
     "AppLayerDecoderEvents": "AppLayerDecoderEvents",
     "core::AppLayerEventType": "AppLayerEventType",
+    "applayer::AppLayerGetTxIterTuple": "AppLayerGetTxIterTuple",
+    "AppLayerGetTxIterTuple": "AppLayerGetTxIterTuple",
     "AppLayerEventType": "AppLayerEventType",
     "CLuaState": "lua_State",
     "Store": "Store",

diff --git a/rust/src/applayer.rs b/rust/src/applayer.rs
index 4fbe4ad255d0f601d6449379c5920a2484c94928..6a0a8e0fced813b77e22271e6a92414365be853e 100644 (file)
--- a/rust/src/applayer.rs
+++ b/rust/src/applayer.rs
@@ -15,6 +15,29 @@
  * 02110-1301, USA.
  */
 
+extern crate libc;
+use std;
+
+#[repr(C)]
+pub struct AppLayerGetTxIterTuple {
+    tx_ptr: *mut libc::c_void,
+    tx_id: u64,
+    has_next: bool,
+}
+
+impl AppLayerGetTxIterTuple {
+    pub fn with_values(tx_ptr: *mut libc::c_void, tx_id: u64, has_next: bool) -> AppLayerGetTxIterTuple {
+        AppLayerGetTxIterTuple {
+            tx_ptr: tx_ptr, tx_id: tx_id, has_next: has_next,
+        }
+    }
+    pub fn not_found() -> AppLayerGetTxIterTuple {
+        AppLayerGetTxIterTuple {
+            tx_ptr: std::ptr::null_mut(), tx_id: 0, has_next: false,
+        }
+    }
+}
+
 /// LoggerFlags tracks which loggers have already been executed.
 #[derive(Debug)]
 pub struct LoggerFlags {

diff --git a/rust/src/nfs/nfs.rs b/rust/src/nfs/nfs.rs
index 4eb2fe7a02d8b9776db9bfc701c5216f3158f89d..8b2e32ca5f7c1865111696c70f6b5b2e4fbeff6b 100644 (file)
--- a/rust/src/nfs/nfs.rs
+++ b/rust/src/nfs/nfs.rs
@@ -28,6 +28,7 @@ use nom;
 use nom::IResult;
 
 use log::*;
+use applayer;
 use applayer::LoggerFlags;
 use core::*;
 use filetracker::*;
@@ -423,6 +424,28 @@ impl NFSState {
         return None;
     }
 
+    // for use with the C API call StateGetTxIterator
+    pub fn get_tx_iterator(&mut self, min_tx_id: u64, state: &mut u64) ->
+        Option<(&NFSTransaction, u64, bool)>
+    {
+        let mut index = *state as usize;
+        let len = self.transactions.len();
+
+        // find tx that is >= min_tx_id
+        while index < len {
+            let tx = &self.transactions[index];
+            if tx.id < min_tx_id + 1 {
+                index += 1;
+                continue;
+            }
+            *state = index as u64 + 1;
+            SCLogDebug!("returning tx_id {} has_next? {} (len {} index {}), tx {:?}",
+                    tx.id - 1, (len - index) > 1, len, index, tx);
+            return Some((tx, tx.id - 1, (len - index) > 1));
+        }
+        return None;
+    }
+
     /// Set an event. The event is set on the most recent transaction.
     pub fn set_event(&mut self, event: NFSEvent) {
         let len = self.transactions.len();
@@ -1850,6 +1873,26 @@ pub extern "C" fn rs_nfs3_state_get_tx(state: &mut NFSState,
     }
 }
 
+// for use with the C API call StateGetTxIterator
+#[no_mangle]
+pub extern "C" fn rs_nfs_state_get_tx_iterator(
+                                      state: &mut NFSState,
+                                      min_tx_id: libc::uint64_t,
+                                      istate: &mut libc::uint64_t)
+                                      -> applayer::AppLayerGetTxIterTuple
+{
+    match state.get_tx_iterator(min_tx_id, istate) {
+        Some((tx, out_tx_id, has_next)) => {
+            let c_tx = unsafe { transmute(tx) };
+            let ires = applayer::AppLayerGetTxIterTuple::with_values(c_tx, out_tx_id, has_next);
+            return ires;
+        }
+        None => {
+            return applayer::AppLayerGetTxIterTuple::not_found();
+        }
+    }
+}
+
 #[no_mangle]
 pub extern "C" fn rs_nfs3_state_tx_free(state: &mut NFSState,
                                        tx_id: libc::uint64_t)

diff --git a/src/app-layer-nfs-tcp.c b/src/app-layer-nfs-tcp.c
index 676e21cef5a3a62a5ebc0d36607c0830fa994335..e118f4f67249333b4d7240607de13f7ebd3360c9 100644 (file)
--- a/src/app-layer-nfs-tcp.c
+++ b/src/app-layer-nfs-tcp.c
@@ -194,6 +194,14 @@ static void *NFSTCPGetTx(void *state, uint64_t tx_id)
     return rs_nfs3_state_get_tx(state, tx_id);
 }
 
+static AppLayerGetTxIterTuple RustNFSTCPGetTxIterator(
+        const uint8_t ipproto, const AppProto alproto,
+        void *alstate, uint64_t min_tx_id, uint64_t max_tx_id,
+        AppLayerGetTxIterState *istate)
+{
+    return rs_nfs_state_get_tx_iterator(alstate, min_tx_id, (uint64_t *)istate);
+}
+
 static void NFSTCPSetTxLogged(void *state, void *vtx, LoggerId logged)
 {
     rs_nfs3_tx_set_logged(state, vtx, logged);
@@ -348,6 +356,8 @@ void RegisterNFSTCPParsers(void)
             ALPROTO_NFS, NFSTCPGetStateProgress);
         AppLayerParserRegisterGetTx(IPPROTO_TCP, ALPROTO_NFS,
             NFSTCPGetTx);
+        AppLayerParserRegisterGetTxIterator(IPPROTO_TCP, ALPROTO_NFS,
+                RustNFSTCPGetTxIterator);
 
         AppLayerParserRegisterGetFilesFunc(IPPROTO_TCP, ALPROTO_NFS, NFSTCPGetFiles);
 

diff --git a/src/app-layer-nfs-udp.c b/src/app-layer-nfs-udp.c
index 71a836c86964a22858436f2bad3610653ae6a359..6d85ebee7d4f9d55b1578cadb8e47d61cd1c2694 100644 (file)
--- a/src/app-layer-nfs-udp.c
+++ b/src/app-layer-nfs-udp.c
@@ -187,6 +187,14 @@ static void *NFSGetTx(void *state, uint64_t tx_id)
     return rs_nfs3_state_get_tx(state, tx_id);
 }
 
+static AppLayerGetTxIterTuple RustNFSGetTxIterator(
+        const uint8_t ipproto, const AppProto alproto,
+        void *alstate, uint64_t min_tx_id, uint64_t max_tx_id,
+        AppLayerGetTxIterState *istate)
+{
+    return rs_nfs_state_get_tx_iterator(alstate, min_tx_id, (uint64_t *)istate);
+}
+
 static void NFSSetTxLogged(void *state, void *vtx, LoggerId logged)
 {
     rs_nfs3_tx_set_logged(state, vtx, logged);
@@ -341,6 +349,8 @@ void RegisterNFSUDPParsers(void)
             ALPROTO_NFS, NFSGetStateProgress);
         AppLayerParserRegisterGetTx(IPPROTO_UDP, ALPROTO_NFS,
             NFSGetTx);
+        AppLayerParserRegisterGetTxIterator(IPPROTO_UDP, ALPROTO_NFS,
+                RustNFSGetTxIterator);
 
         AppLayerParserRegisterGetFilesFunc(IPPROTO_UDP, ALPROTO_NFS, NFSGetFiles);
 

diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index a2ab40a17cd0fc68946e5999d1883c1303a5462f..23389dc622379ccead3719a60ff4c6bd173dc4c9 100644 (file)
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -105,6 +105,7 @@ typedef struct AppLayerParserProtoCtx_
     int (*StateGetProgress)(void *alstate, uint8_t direction);
     uint64_t (*StateGetTxCnt)(void *alstate);
     void *(*StateGetTx)(void *alstate, uint64_t tx_id);
+    AppLayerGetTxIteratorFunc StateGetTxIterator;
     int (*StateGetProgressCompletionStatus)(uint8_t direction);
     int (*StateGetEventInfo)(const char *event_name,
                              int *event_id, AppLayerEventType *event_type);
@@ -518,6 +519,14 @@ void AppLayerParserRegisterGetTx(uint8_t ipproto, AppProto alproto,
     SCReturn;
 }
 
+void AppLayerParserRegisterGetTxIterator(uint8_t ipproto, AppProto alproto,
+                      AppLayerGetTxIteratorFunc Func)
+{
+    SCEnter();
+    alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetTxIterator = Func;
+    SCReturn;
+}
+
 void AppLayerParserRegisterGetStateProgressCompletionStatus(AppProto alproto,
     int (*StateGetProgressCompletionStatus)(uint8_t direction))
 {
@@ -609,6 +618,49 @@ void AppLayerParserDestroyProtocolParserLocalStorage(uint8_t ipproto, AppProto a
     SCReturn;
 }
 
+/** \brief default tx iterator
+ *
+ *  Used if the app layer parser doesn't register its own iterator.
+ *  Simply walks the tx_id space until it finds a tx. Uses 'state' to
+ *  keep track of where it left off.
+ *
+ *  \retval txptr or NULL if no more txs in list
+ */
+static AppLayerGetTxIterTuple AppLayerDefaultGetTxIterator(
+        const uint8_t ipproto, const AppProto alproto,
+        void *alstate, uint64_t min_tx_id, uint64_t max_tx_id,
+        AppLayerGetTxIterState *state)
+{
+    uint64_t ustate = *(uint64_t *)state;
+    uint64_t tx_id = MAX(min_tx_id, ustate);
+    for ( ; tx_id < max_tx_id; tx_id++) {
+        void *tx_ptr = AppLayerParserGetTx(ipproto, alproto, alstate, tx_id);
+        if (tx_ptr != NULL) {
+            ustate = tx_id + 1;
+            *state = *(AppLayerGetTxIterState *)&ustate;
+            AppLayerGetTxIterTuple tuple = {
+                .tx_ptr = tx_ptr,
+                .tx_id = tx_id,
+                .has_next = (tx_id + 1 < max_tx_id),
+            };
+            SCLogDebug("tulpe: %p/%"PRIu64"/%s", tuple.tx_ptr, tuple.tx_id,
+                    tuple.has_next ? "true" : "false");
+            return tuple;
+        }
+    }
+
+    AppLayerGetTxIterTuple no_tuple = { NULL, 0, false };
+    return no_tuple;
+}
+
+AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto,
+        const AppProto alproto)
+{
+    AppLayerGetTxIteratorFunc Func =
+        alp_ctx.ctxs[FlowGetProtoMapping(ipproto)][alproto].StateGetTxIterator;
+    return Func ? Func : AppLayerDefaultGetTxIterator;
+}
+
 void AppLayerParserSetTxLogged(uint8_t ipproto, AppProto alproto,
                                void *alstate, void *tx, LoggerId logger)
 {
@@ -677,28 +729,37 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p
     const uint8_t ipproto = f->proto;
     const AppProto alproto = f->alproto;
 
+    AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
+    AppLayerGetTxIterState state;
+    memset(&state, 0, sizeof(state));
+
     SCLogDebug("called: %s, tag_txs_as_inspected %s",direction==0?"toserver":"toclient",
             tag_txs_as_inspected?"true":"false");
 
     /* mark all txs as inspected if the applayer progress is
      * at the 'end state'. */
-    for (; idx < total_txs; idx++) {
-        void *tx = AppLayerParserGetTx(ipproto, alproto, alstate, idx);
-        if (tx == NULL)
-            continue;
+    while (1) {
+        AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, idx, total_txs, &state);
+        if (ires.tx_ptr == NULL)
+            break;
+
+        void *tx = ires.tx_ptr;
+        idx = ires.tx_id;
+
         int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags);
-        if (state_progress >= state_done_progress) {
-            if (tag_txs_as_inspected) {
-                uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
-                if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
-                    detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
-                    AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
-                    SCLogDebug("%p/%"PRIu64" in-order tx is done for direction %s. Flag %016"PRIx64,
-                            tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
-                }
+        if (state_progress < state_done_progress)
+            break;
+
+        if (tag_txs_as_inspected) {
+            uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
+            if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
+                detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
+                AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
+                SCLogDebug("%p/%"PRIu64" in-order tx is done for direction %s. Flag %016"PRIx64,
+                        tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
             }
-            continue;
-        } else
+        }
+        if (!ires.has_next)
             break;
     }
     pstate->inspect_id[direction] = idx;
@@ -707,29 +768,39 @@ void AppLayerParserSetTransactionInspectId(const Flow *f, AppLayerParserState *p
     /* if necessary we flag all txs that are complete as 'inspected'
      * also move inspect_id forward. */
     if (tag_txs_as_inspected) {
-        for (; idx < total_txs; idx++) {
-            bool check_inspect_id = false;
-            void *tx = AppLayerParserGetTx(ipproto, alproto, alstate, idx);
-            if (tx == NULL) {
-                check_inspect_id = true;
-            } else {
-                int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags);
-                if (state_progress >= state_done_progress) {
-                    uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
-                    if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
-                        detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
-                        AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
-                        SCLogDebug("%p/%"PRIu64" out of order tx is done for direction %s. Flag %016"PRIx64,
-                                tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
-                        check_inspect_id = true;
-                    }
-                }
+        /* continue at idx */
+        while (1) {
+            AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, idx, total_txs, &state);
+            if (ires.tx_ptr == NULL)
+                break;
+
+            void *tx = ires.tx_ptr;
+            /* if we got a higher id than the minimum we requested, we
+             * skipped a bunch of 'null-txs'. Lets see if we can up the
+             * inspect tracker */
+            if (ires.tx_id > idx && pstate->inspect_id[direction] == idx) {
+                pstate->inspect_id[direction] = ires.tx_id;
             }
-            if (check_inspect_id) {
-                SCLogDebug("%p/%"PRIu64" out of order tx. Update inspect_id? %"PRIu64, tx, idx, pstate->inspect_id[direction]);
+            idx = ires.tx_id;
+
+            const int state_progress = AppLayerParserGetStateProgress(ipproto, alproto, tx, flags);
+            if (state_progress < state_done_progress)
+                break;
+
+            uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, flags);
+            if ((detect_flags & APP_LAYER_TX_INSPECTED_FLAG) == 0) {
+                detect_flags |= APP_LAYER_TX_INSPECTED_FLAG;
+                AppLayerParserSetTxDetectFlags(ipproto, alproto, tx, flags, detect_flags);
+                SCLogDebug("%p/%"PRIu64" out of order tx is done for direction %s. Flag %016"PRIx64,
+                        tx, idx, flags & STREAM_TOSERVER ? "toserver" : "toclient", detect_flags);
+
+                SCLogDebug("%p/%"PRIu64" out of order tx. Update inspect_id? %"PRIu64,
+                        tx, idx, pstate->inspect_id[direction]);
                 if (pstate->inspect_id[direction]+1 == idx)
                     pstate->inspect_id[direction] = idx;
             }
+            if (!ires.has_next)
+                break;
         }
     }
 
@@ -808,31 +879,38 @@ void AppLayerParserTransactionsCleanup(Flow *f)
     const int tx_end_state_ts = AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOSERVER);
     const int tx_end_state_tc = AppLayerParserGetStateProgressCompletionStatus(alproto, STREAM_TOCLIENT);
 
-    SCLogDebug("checking %"PRIu64" txs from offset %"PRIu64, total_txs, min);
-    for (uint64_t i = min ; i < total_txs; i++) {
-        void * const tx = AppLayerParserGetTx(ipproto, alproto, alstate, i);
-        if (tx == NULL) {
-            SCLogDebug("%p/%"PRIu64" skipping: no tx", tx, i);
-            goto wrap_up;
-        }
+    AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
+    AppLayerGetTxIterState state;
+    memset(&state, 0, sizeof(state));
+    uint64_t i = min;
+    uint64_t new_min = min;
+
+    while (1) {
+        AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, i, total_txs, &state);
+        if (ires.tx_ptr == NULL)
+            break;
+
+        void *tx = ires.tx_ptr;
+        i = ires.tx_id;
+
         SCLogDebug("%p/%"PRIu64" checking", tx, i);
 
         const int tx_progress_tc = AppLayerParserGetStateProgress(ipproto, alproto, tx, STREAM_TOCLIENT);
         if (tx_progress_tc < tx_end_state_tc) {
             SCLogDebug("%p/%"PRIu64" skipping: tc parser not done", tx, i);
-            continue;
+            goto next;
         }
         const int tx_progress_ts = AppLayerParserGetStateProgress(ipproto, alproto, tx, STREAM_TOSERVER);
         if (tx_progress_ts < tx_end_state_ts) {
             SCLogDebug("%p/%"PRIu64" skipping: ts parser not done", tx, i);
-            continue;
+            goto next;
         }
         if (f->sgh_toserver != NULL) {
             uint64_t detect_flags_ts = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx, STREAM_TOSERVER);
             if (!(detect_flags_ts & APP_LAYER_TX_INSPECTED_FLAG)) {
                 SCLogDebug("%p/%"PRIu64" skipping: TS inspect not done: ts:%"PRIx64,
                         tx, i, detect_flags_ts);
-                continue;
+                goto next;
             }
         }
         if (f->sgh_toclient != NULL) {
@@ -840,7 +918,7 @@ void AppLayerParserTransactionsCleanup(Flow *f)
             if (!(detect_flags_tc & APP_LAYER_TX_INSPECTED_FLAG)) {
                 SCLogDebug("%p/%"PRIu64" skipping: TC inspect not done: tc:%"PRIx64,
                         tx, i, detect_flags_tc);
-                continue;
+                goto next;
             }
         }
         if (logger_expectation != 0) {
@@ -848,25 +926,32 @@ void AppLayerParserTransactionsCleanup(Flow *f)
             if (tx_logged != logger_expectation) {
                 SCLogDebug("%p/%"PRIu64" skipping: logging not done: want:%"PRIx32", have:%"PRIx32,
                         tx, i, logger_expectation, tx_logged);
-                continue;
+                goto next;
             }
         }
 
         /* if we are here, the tx can be freed. */
         p->StateTransactionFree(alstate, i);
         SCLogDebug("%p/%"PRIu64" freed", tx, i);
-    wrap_up:
-        /* see if this tx is actually in order. If so, we need to bring all
-         * trackers up to date. */
-        SCLogDebug("%p/%"PRIu64" update f->alparser->min_id? %"PRIu64, tx, i, alparser->min_id);
-        if (i == alparser->min_id) {
-            uint64_t next_id = i + 1;
-            alparser->min_id = next_id;
-            alparser->inspect_id[0] = MAX(alparser->inspect_id[0], next_id);
-            alparser->inspect_id[1] = MAX(alparser->inspect_id[1], next_id);
-            alparser->log_id = MAX(alparser->log_id, next_id);
-            SCLogDebug("%p/%"PRIu64" updated f->alparser->min_id %"PRIu64, tx, i, alparser->min_id);
-        }
+
+        /* if this tx was the minimum, up the minimum */
+        if (i == new_min)
+            new_min = i + 1;
+
+next:
+        if (!ires.has_next)
+            break;
+    }
+
+    /* see if we need to bring all trackers up to date. */
+    SCLogDebug("update f->alparser->min_id? %"PRIu64, alparser->min_id);
+    if (new_min > alparser->min_id) {
+        const uint64_t next_id = new_min;
+        alparser->min_id = next_id;
+        alparser->inspect_id[0] = MAX(alparser->inspect_id[0], next_id);
+        alparser->inspect_id[1] = MAX(alparser->inspect_id[1], next_id);
+        alparser->log_id = MAX(alparser->log_id, next_id);
+        SCLogDebug("updated f->alparser->min_id %"PRIu64, alparser->min_id);
     }
 }
 

diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h
index ad6fd8b01809df7a399138bfdd617540abe51a9a..2518463191429bd44a59e3ae7d345703bef41371 100644 (file)
--- a/src/app-layer-parser.h
+++ b/src/app-layer-parser.h
@@ -92,6 +92,25 @@ typedef int (*AppLayerParserFPtr)(Flow *f, void *protocol_state,
         uint8_t *buf, uint32_t buf_len,
         void *local_storage);
 
+typedef struct AppLayerGetTxIterTuple {
+    void *tx_ptr;
+    uint64_t tx_id;
+    bool has_next;
+} AppLayerGetTxIterTuple;
+
+typedef struct AppLayerGetTxIterState {
+    union {
+        void *ptr;
+        uint64_t u64;
+    } un;
+} AppLayerGetTxIterState;
+
+/** \brief tx iterator prototype */
+typedef AppLayerGetTxIterTuple (*AppLayerGetTxIteratorFunc)
+       (const uint8_t ipproto, const AppProto alproto,
+        void *alstate, uint64_t min_tx_id, uint64_t max_tx_id,
+        AppLayerGetTxIterState *state);
+
 /***** Parser related registration *****/
 
 /**
@@ -135,6 +154,8 @@ void AppLayerParserRegisterGetTxCnt(uint8_t ipproto, AppProto alproto,
                          uint64_t (*StateGetTxCnt)(void *alstate));
 void AppLayerParserRegisterGetTx(uint8_t ipproto, AppProto alproto,
                       void *(StateGetTx)(void *alstate, uint64_t tx_id));
+void AppLayerParserRegisterGetTxIterator(uint8_t ipproto, AppProto alproto,
+                      AppLayerGetTxIteratorFunc Func);
 void AppLayerParserRegisterGetStateProgressCompletionStatus(AppProto alproto,
     int (*StateGetStateProgressCompletionStatus)(uint8_t direction));
 void AppLayerParserRegisterGetEventInfo(uint8_t ipproto, AppProto alproto,
@@ -156,6 +177,9 @@ void AppLayerParserRegisterDetectFlagsFuncs(uint8_t ipproto, AppProto alproto,
 
 /***** Get and transaction functions *****/
 
+AppLayerGetTxIteratorFunc AppLayerGetTxIterator(const uint8_t ipproto,
+         const AppProto alproto);
+
 void *AppLayerParserGetProtocolParserLocalStorage(uint8_t ipproto, AppProto alproto);
 void AppLayerParserDestroyProtocolParserLocalStorage(uint8_t ipproto, AppProto alproto,
                                           void *local_data);

diff --git a/src/detect-nfs-procedure.c b/src/detect-nfs-procedure.c
index 7f7cbd72469b1a4deec1b0c50095b6b1a49a75c6..05e8c542421724bf18aea089a32864b677557eee 100644 (file)
--- a/src/detect-nfs-procedure.c
+++ b/src/detect-nfs-procedure.c
@@ -34,6 +34,8 @@
 #include "detect-pcre.h"
 #include "detect-nfs-procedure.h"
 
+#include "app-layer-parser.h"
+
 #include "flow.h"
 #include "flow-util.h"
 #include "flow-var.h"

diff --git a/src/detect-nfs-version.c b/src/detect-nfs-version.c
index 5d6f2123fcff1dc80c6d8fb5d9aeb981002202bd..88e9ae4ad2fc87208fa9f9aface399070b6814a4 100644 (file)
--- a/src/detect-nfs-version.c
+++ b/src/detect-nfs-version.c
@@ -34,6 +34,8 @@
 #include "detect-pcre.h"
 #include "detect-nfs-version.h"
 
+#include "app-layer-parser.h"
+
 #include "flow.h"
 #include "flow-util.h"
 #include "flow-var.h"

diff --git a/src/detect.c b/src/detect.c
index b034d86551352571aab678422252a21e87813753..978b19c08c5be5a1353f27bddd5f7770c9fe0716 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -1302,15 +1302,10 @@ static bool DetectRunTxInspectRule(ThreadVars *tv,
  *  \brief get a DetectTransaction object
  *  \retval struct filled with relevant info or all nulls/0s
  */
-static DetectTransaction GetTx(const uint8_t ipproto, const AppProto alproto,
-        void *alstate, const uint64_t tx_id, const int tx_end_state,
+static DetectTransaction GetDetectTx(const uint8_t ipproto, const AppProto alproto,
+        void *alstate, const uint64_t tx_id, void *tx_ptr, const int tx_end_state,
         const uint8_t flow_flags)
 {
-    void *tx_ptr = AppLayerParserGetTx(ipproto, alproto, alstate, tx_id);
-    if (tx_ptr == NULL) {
-        DetectTransaction no_tx = { NULL, 0, NULL, 0, 0, 0, 0, 0, };
-        return no_tx;
-    }
     const uint64_t detect_flags = AppLayerParserGetTxDetectFlags(ipproto, alproto, tx_ptr, flow_flags);
     if (detect_flags & APP_LAYER_TX_INSPECTED_FLAG) {
         SCLogDebug("%"PRIu64" tx already fully inspected for %s. Flags %016"PRIx64,
@@ -1356,13 +1351,21 @@ static void DetectRunTx(ThreadVars *tv,
     uint64_t tx_id = AppLayerParserGetTransactionInspectId(f->alparser, flow_flags);
     const int tx_end_state = AppLayerParserGetStateProgressCompletionStatus(alproto, flow_flags);
 
-    for ( ; tx_id < total_txs; tx_id++) {
-        DetectTransaction tx = GetTx(ipproto, alproto,
-                alstate, tx_id, tx_end_state, flow_flags);
+    AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
+    AppLayerGetTxIterState state;
+    memset(&state, 0, sizeof(state));
+
+    while (1) {
+        AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state);
+        if (ires.tx_ptr == NULL)
+            break;
+
+        DetectTransaction tx = GetDetectTx(ipproto, alproto,
+                alstate, ires.tx_id, ires.tx_ptr, tx_end_state, flow_flags);
         if (tx.tx_ptr == NULL) {
             SCLogDebug("%p/%"PRIu64" no transaction to inspect",
                     tx.tx_ptr, tx_id);
-            continue;
+            goto next;
         }
 
         uint32_t array_idx = 0;
@@ -1571,6 +1574,9 @@ static void DetectRunTx(ThreadVars *tv,
             AppLayerParserSetTxDetectFlags(ipproto, alproto, tx.tx_ptr,
                     flow_flags, new_detect_flags);
         }
+next:
+        if (!ires.has_next)
+            break;
     }
 }
 

diff --git a/src/output-tx.c b/src/output-tx.c
index 4cb7bad834e450afe56779130d35552c5ebdf1cc..b061b89d0358040f6e1d5fdbbdc8fc5af6799f1c 100644 (file)
--- a/src/output-tx.c
+++ b/src/output-tx.c
@@ -140,6 +140,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
         return TM_ECODE_OK;
 
     Flow * const f = p->flow;
+    const uint8_t ipproto = f->proto;
     const AppProto alproto = f->alproto;
 
     if (AppLayerParserProtocolIsTxAware(p->proto, alproto) == 0)
@@ -164,20 +165,23 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
     int logged = 0;
     int gap = 0;
 
-    for (; tx_id < total_txs; tx_id++)
-    {
-        void *tx = AppLayerParserGetTx(p->proto, alproto, alstate, tx_id);
-        if (tx == NULL) {
-            SCLogDebug("tx is NULL not logging");
-            continue;
-        }
+    AppLayerGetTxIteratorFunc IterFunc = AppLayerGetTxIterator(ipproto, alproto);
+    AppLayerGetTxIterState state;
+    memset(&state, 0, sizeof(state));
+
+    while (1) {
+        AppLayerGetTxIterTuple ires = IterFunc(ipproto, alproto, alstate, tx_id, total_txs, &state);
+        if (ires.tx_ptr == NULL)
+            break;
+        void * const tx = ires.tx_ptr;
+        tx_id = ires.tx_id;
 
         LoggerId tx_logged = AppLayerParserGetTxLogged(f, alstate, tx);
         const LoggerId tx_logged_old = tx_logged;
         SCLogDebug("logger: expect %08x, have %08x", logger_expectation, tx_logged);
         if (tx_logged == logger_expectation) {
             /* tx already fully logged */
-            continue;
+            goto next_tx;
         }
 
         int tx_progress_ts = AppLayerParserGetStateProgress(p->proto, alproto,
@@ -211,17 +215,17 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
                         int r = logger->LogCondition(tv, p, alstate, tx, tx_id);
                         if (r == FALSE) {
                             SCLogDebug("conditions not met, not logging");
-                            goto next;
+                            goto next_logger;
                         }
                     } else {
                         if (tx_progress_tc < logger->tc_log_progress) {
                             SCLogDebug("progress not far enough, not logging");
-                            goto next;
+                            goto next_logger;
                         }
 
                         if (tx_progress_ts < logger->ts_log_progress) {
                             SCLogDebug("progress not far enough, not logging");
-                            goto next;
+                            goto next_logger;
                         }
                     }
                 }
@@ -235,7 +239,7 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
                 tx_logged |= (1<<logger->logger_id);
             }
 
-next:
+next_logger:
             logger = logger->next;
             store = store->next;
 #ifdef DEBUG_VALIDATION
@@ -263,6 +267,9 @@ next:
         } else {
             gap = 1;
         }
+next_tx:
+        if (!ires.has_next)
+            break;
     }
 
     /* Update the the last ID that has been logged with all