app-layer: reduce app cleanup and output-tx calls

Track packets that updated the app-layer, and for those run
the transaction housekeeping and output-tx logging loops.

Do the same of end of flow packets.

This skips needless iterations over the transaction stores.

diff --git a/src/app-layer.c b/src/app-layer.c
index fb0d407d22e21efce4b4ef95076c73925250d391..ed5ebe5ecfb7399c2f045d6830bd9047f62cd430 100644 (file)
--- a/src/app-layer.c
+++ b/src/app-layer.c
@@ -507,6 +507,7 @@ static int TCPProtoDetect(ThreadVars *tv,
         int r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto,
                 flags, data, data_len);
         PACKET_PROFILING_APP_END(app_tctx, f->alproto);
+        p->flags |= PKT_APPLAYER_UPDATE;
         if (r != 1) {
             StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
         }
@@ -580,6 +581,7 @@ static int TCPProtoDetect(ThreadVars *tv,
                             f->alproto, flags,
                             data, data_len);
                     PACKET_PROFILING_APP_END(app_tctx, f->alproto);
+                    p->flags |= PKT_APPLAYER_UPDATE;
                     if (r != 1) {
                         StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
                     }
@@ -684,6 +686,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
         r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto,
                 flags, data, data_len);
         PACKET_PROFILING_APP_END(app_tctx, f->alproto);
+        p->flags |= PKT_APPLAYER_UPDATE;
         /* ignore parser result for gap */
         StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
         if (r < 0) {
@@ -767,6 +770,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
             r = AppLayerParserParse(tv, app_tctx->alp_tctx, f, f->alproto,
                                     flags, data, data_len);
             PACKET_PROFILING_APP_END(app_tctx, f->alproto);
+            p->flags |= PKT_APPLAYER_UPDATE;
             if (r != 1) {
                 StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
                 if (r < 0) {
@@ -891,6 +895,7 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow *
             r = AppLayerParserParse(tv, tctx->alp_tctx, f, f->alproto,
                                     flags, p->payload, p->payload_len);
             PACKET_PROFILING_APP_END(tctx, f->alproto);
+            p->flags |= PKT_APPLAYER_UPDATE;
         }
         PACKET_PROFILING_APP_STORE(tctx, p);
         /* we do only inspection in one direction, so flag both
@@ -907,6 +912,7 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow *
                 flags, p->payload, p->payload_len);
         PACKET_PROFILING_APP_END(tctx, f->alproto);
         PACKET_PROFILING_APP_STORE(tctx, p);
+        p->flags |= PKT_APPLAYER_UPDATE;
     }
     if (r < 0) {
         ExceptionPolicyApply(p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR);

diff --git a/src/decode.h b/src/decode.h
index f59570e14a7a10751ca942ded7a0b2452771471b..e98725d327f909f1d6210055912ff45cdf0197eb 100644 (file)
--- a/src/decode.h
+++ b/src/decode.h
@@ -1102,6 +1102,9 @@ void DecodeUnregisterCounters(void);
 #define PKT_FIRST_ALERTS BIT_U32(29)
 #define PKT_FIRST_TAG    BIT_U32(30)
 
+/** Packet updated the app-layer. */
+#define PKT_APPLAYER_UPDATE BIT_U32(31)
+
 /** \brief return 1 if the packet is a pseudo packet */
 #define PKT_IS_PSEUDOPKT(p) \
     ((p)->flags & (PKT_PSEUDO_STREAM_END|PKT_PSEUDO_DETECTLOG_FLUSH))

diff --git a/src/flow-worker.c b/src/flow-worker.c
index 27328f266ece7e0fd81601e7c187d68e34041fd3..f75182b8535f12e5fca5bbf01ec2ed999a7cad30 100644 (file)
--- a/src/flow-worker.c
+++ b/src/flow-worker.c
@@ -574,9 +574,13 @@ static TmEcode FlowWorker(ThreadVars *tv, Packet *p, void *data)
             FramesPrune(p->flow, p);
         }
 
-        /* run tx cleanup last */
-        AppLayerParserTransactionsCleanup(p->flow, STREAM_FLAGS_FOR_PACKET(p));
-
+        if ((PKT_IS_PSEUDOPKT(p)) || ((p->flags & PKT_APPLAYER_UPDATE) != 0)) {
+            SCLogDebug("pseudo or app update: run cleanup");
+            /* run tx cleanup last */
+            AppLayerParserTransactionsCleanup(p->flow, STREAM_FLAGS_FOR_PACKET(p));
+        } else {
+            SCLogDebug("not pseudo, no app update: skip");
+        }
         Flow *f = p->flow;
         FlowDeReference(&p->flow);
         FLOWLOCK_UNLOCK(f);

diff --git a/src/output-tx.c b/src/output-tx.c
index 9f5512a77ad60398bab4a8e42dedea7e9cb0bb91..3b4e55a6ed07f8393fdaeebe7e1a76b81ef9809b 100644 (file)
--- a/src/output-tx.c
+++ b/src/output-tx.c
@@ -335,6 +335,11 @@ static TmEcode OutputTxLog(ThreadVars *tv, Packet *p, void *thread_data)
     DEBUG_VALIDATE_BUG_ON(thread_data == NULL);
     if (p->flow == NULL)
         return TM_ECODE_OK;
+    if (!((PKT_IS_PSEUDOPKT(p)) || (p->flags & PKT_APPLAYER_UPDATE) != 0)) {
+        SCLogDebug("not pseudo, no app update: skip");
+        return TM_ECODE_OK;
+    }
+    SCLogDebug("pseudo, or app update: run output");
 
     OutputTxLoggerThreadData *op_thread_data = (OutputTxLoggerThreadData *)thread_data;
 

diff --git a/src/stream-tcp-reassemble.c b/src/stream-tcp-reassemble.c
index 20d5d255ac3fa525687796d654a4b4a772bd2085..2a26ae6568ef99071114ffcc1c30111467f85c5d 100644 (file)
--- a/src/stream-tcp-reassemble.c
+++ b/src/stream-tcp-reassemble.c
@@ -734,6 +734,7 @@ int StreamTcpReassembleHandleSegmentHandleData(ThreadVars *tv, TcpReassemblyThre
         StreamTcpSetEvent(p, STREAM_REASSEMBLY_DEPTH_REACHED);
         /* increment stream depth counter */
         StatsIncr(tv, ra_ctx->counter_tcp_stream_depth);
+        p->flags |= PKT_APPLAYER_UPDATE;
     }
     if (size == 0) {
         SCLogDebug("ssn %p: depth reached, not reassembling", ssn);