detect/stats: log out total of discarded alerts

Add a counter to our stats log with the total of alerts that have been
discarded due to packet alert queue overflow.

Also included a fix for
Bug #5354

Task #5179

(cherry picked from commit 04eefa5ab8008c06c8c19e56b06774d07bab91c7)

diff --git a/src/decode.h b/src/decode.h
index 60949a82038ee6fc2a58c374b7c14ee5090fa4fd..428e7eebff293af0a01c1e770f6b7a51766604d8 100644 (file)
--- a/src/decode.h
+++ b/src/decode.h
@@ -293,6 +293,7 @@ extern uint16_t packet_alert_max;
 
 typedef struct PacketAlerts_ {
     uint16_t cnt;
+    uint16_t discarded;
     PacketAlert *alerts;
     /* single pa used when we're dropping,
      * so we can log it out in the drop log. */
@@ -757,74 +758,76 @@ void CaptureStatsSetup(ThreadVars *tv, CaptureStats *s);
 /**
  *  \brief Recycle a packet structure for reuse.
  */
-#define PACKET_REINIT(p) do {             \
-        CLEAR_ADDR(&(p)->src);                  \
-        CLEAR_ADDR(&(p)->dst);                  \
-        (p)->sp = 0;                            \
-        (p)->dp = 0;                            \
-        (p)->proto = 0;                         \
-        (p)->recursion_level = 0;               \
-        PACKET_FREE_EXTDATA((p));               \
-        (p)->flags = (p)->flags & PKT_ALLOC;    \
-        (p)->flowflags = 0;                     \
-        (p)->pkt_src = 0;                       \
-        (p)->vlan_id[0] = 0;                    \
-        (p)->vlan_id[1] = 0;                    \
-        (p)->vlan_idx = 0;                      \
-        (p)->ts.tv_sec = 0;                     \
-        (p)->ts.tv_usec = 0;                    \
-        (p)->datalink = 0;                      \
-        (p)->action = 0;                        \
-        if ((p)->pktvar != NULL) {              \
-            PktVarFree((p)->pktvar);            \
-            (p)->pktvar = NULL;                 \
-        }                                       \
-        (p)->ethh = NULL;                       \
-        if ((p)->ip4h != NULL) {                \
-            CLEAR_IPV4_PACKET((p));             \
-        }                                       \
-        if ((p)->ip6h != NULL) {                \
-            CLEAR_IPV6_PACKET((p));             \
-        }                                       \
-        if ((p)->tcph != NULL) {                \
-            CLEAR_TCP_PACKET((p));              \
-        }                                       \
-        if ((p)->udph != NULL) {                \
-            CLEAR_UDP_PACKET((p));              \
-        }                                       \
-        if ((p)->sctph != NULL) {               \
-            CLEAR_SCTP_PACKET((p));             \
-        }                                       \
-        if ((p)->icmpv4h != NULL) {             \
-            CLEAR_ICMPV4_PACKET((p));           \
-        }                                       \
-        if ((p)->icmpv6h != NULL) {             \
-            CLEAR_ICMPV6_PACKET((p));           \
-        }                                       \
-        (p)->ppph = NULL;                       \
-        (p)->pppoesh = NULL;                    \
-        (p)->pppoedh = NULL;                    \
-        (p)->greh = NULL;                       \
-        (p)->payload = NULL;                    \
-        (p)->payload_len = 0;                   \
-        (p)->BypassPacketsFlow = NULL;          \
-        (p)->pktlen = 0;                        \
-        (p)->alerts.cnt = 0;                    \
-        (p)->alerts.drop.action = 0;            \
-        (p)->pcap_cnt = 0;                      \
-        (p)->tunnel_rtv_cnt = 0;                \
-        (p)->tunnel_tpr_cnt = 0;                \
-        (p)->events.cnt = 0;                    \
-        AppLayerDecoderEventsResetEvents((p)->app_layer_events); \
-        (p)->next = NULL;                       \
-        (p)->prev = NULL;                       \
-        (p)->root = NULL;                       \
-        (p)->livedev = NULL;                    \
-        PACKET_RESET_CHECKSUMS((p));            \
-        PACKET_PROFILING_RESET((p));            \
-        p->tenant_id = 0;                       \
-        p->nb_decoded_layers = 0;               \
-    } while (0)
+#define PACKET_REINIT(p)                                                       \
+  do {                                                                         \
+    CLEAR_ADDR(&(p)->src);                                                     \
+    CLEAR_ADDR(&(p)->dst);                                                     \
+    (p)->sp = 0;                                                               \
+    (p)->dp = 0;                                                               \
+    (p)->proto = 0;                                                            \
+    (p)->recursion_level = 0;                                                  \
+    PACKET_FREE_EXTDATA((p));                                                  \
+    (p)->flags = (p)->flags & PKT_ALLOC;                                       \
+    (p)->flowflags = 0;                                                        \
+    (p)->pkt_src = 0;                                                          \
+    (p)->vlan_id[0] = 0;                                                       \
+    (p)->vlan_id[1] = 0;                                                       \
+    (p)->vlan_idx = 0;                                                         \
+    (p)->ts.tv_sec = 0;                                                        \
+    (p)->ts.tv_usec = 0;                                                       \
+    (p)->datalink = 0;                                                         \
+    (p)->action = 0;                                                           \
+    if ((p)->pktvar != NULL) {                                                 \
+      PktVarFree((p)->pktvar);                                                 \
+      (p)->pktvar = NULL;                                                      \
+    }                                                                          \
+    (p)->ethh = NULL;                                                          \
+    if ((p)->ip4h != NULL) {                                                   \
+      CLEAR_IPV4_PACKET((p));                                                  \
+    }                                                                          \
+    if ((p)->ip6h != NULL) {                                                   \
+      CLEAR_IPV6_PACKET((p));                                                  \
+    }                                                                          \
+    if ((p)->tcph != NULL) {                                                   \
+      CLEAR_TCP_PACKET((p));                                                   \
+    }                                                                          \
+    if ((p)->udph != NULL) {                                                   \
+      CLEAR_UDP_PACKET((p));                                                   \
+    }                                                                          \
+    if ((p)->sctph != NULL) {                                                  \
+      CLEAR_SCTP_PACKET((p));                                                  \
+    }                                                                          \
+    if ((p)->icmpv4h != NULL) {                                                \
+      CLEAR_ICMPV4_PACKET((p));                                                \
+    }                                                                          \
+    if ((p)->icmpv6h != NULL) {                                                \
+      CLEAR_ICMPV6_PACKET((p));                                                \
+    }                                                                          \
+    (p)->ppph = NULL;                                                          \
+    (p)->pppoesh = NULL;                                                       \
+    (p)->pppoedh = NULL;                                                       \
+    (p)->greh = NULL;                                                          \
+    (p)->payload = NULL;                                                       \
+    (p)->payload_len = 0;                                                      \
+    (p)->BypassPacketsFlow = NULL;                                             \
+    (p)->pktlen = 0;                                                           \
+    (p)->alerts.cnt = 0;                                                       \
+    (p)->alerts.discarded = 0;                                                 \
+    (p)->alerts.drop.action = 0;                                               \
+    (p)->pcap_cnt = 0;                                                         \
+    (p)->tunnel_rtv_cnt = 0;                                                   \
+    (p)->tunnel_tpr_cnt = 0;                                                   \
+    (p)->events.cnt = 0;                                                       \
+    AppLayerDecoderEventsResetEvents((p)->app_layer_events);                   \
+    (p)->next = NULL;                                                          \
+    (p)->prev = NULL;                                                          \
+    (p)->root = NULL;                                                          \
+    (p)->livedev = NULL;                                                       \
+    PACKET_RESET_CHECKSUMS((p));                                               \
+    PACKET_PROFILING_RESET((p));                                               \
+    p->tenant_id = 0;                                                          \
+    p->nb_decoded_layers = 0;                                                  \
+  } while (0)
 
 #define PACKET_RECYCLE(p) do { \
         PACKET_RELEASE_REFS((p)); \

diff --git a/src/detect-engine-alert.c b/src/detect-engine-alert.c
index 26c9ac933863d2d6cb92727f0e1d1a02b3de085b..9b1b56117008f7df6ccea322ee6480ad96e784a7 100644 (file)
--- a/src/detect-engine-alert.c
+++ b/src/detect-engine-alert.c
@@ -267,6 +267,7 @@ void AlertQueueAppend(DetectEngineThreadCtx *det_ctx, const Signature *s, Packet
         /* we must grow the alert queue */
         if (pos == AlertQueueExpand(det_ctx)) {
             /* this means we failed to expand the queue */
+            p->alerts.discarded++;
             return;
         }
     }
@@ -368,6 +369,7 @@ void PacketAlertFinalize(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx
         /* Thresholding removes this alert */
         if (res == 0 || res == 2 || (s->flags & SIG_FLAG_NOALERT)) {
             /* we will not copy this to the AlertQueue */
+            p->alerts.discarded++;
         } else if (p->alerts.cnt < packet_alert_max) {
             p->alerts.alerts[p->alerts.cnt] = det_ctx->alert_queue[i];
             SCLogDebug("Appending sid %" PRIu32 " alert to Packet::alerts at pos %u", s->id, i);
@@ -378,6 +380,8 @@ void PacketAlertFinalize(DetectEngineCtx *de_ctx, DetectEngineThreadCtx *det_ctx
                 break;
             }
             p->alerts.cnt++;
+        } else {
+            p->alerts.discarded++;
         }
         i++;
     }

diff --git a/src/detect-engine.c b/src/detect-engine.c
index 0d48d2a440402c06a8a17f3971fa54a1e8607ead..53e8d0c9cdfaa9e71fd52920406ee9c8878c582c 100644 (file)
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -2910,6 +2910,8 @@ TmEcode DetectEngineThreadCtxInit(ThreadVars *tv, void *initdata, void **data)
 
     /** alert counter setup */
     det_ctx->counter_alerts = StatsRegisterCounter("detect.alert", tv);
+    det_ctx->counter_alerts_overflow =
+        StatsRegisterCounter("detect.alert_queue_overflow", tv);
 #ifdef PROFILING
     det_ctx->counter_mpm_list = StatsRegisterAvgCounter("detect.mpm_list", tv);
     det_ctx->counter_nonmpm_list = StatsRegisterAvgCounter("detect.nonmpm_list", tv);

diff --git a/src/detect.c b/src/detect.c
index 841ef778e2fd51198de08e6d589d4b61cf6166d4..d487a941ad3c64e7eed7817a7f0816f94ec444b1 100644 (file)
--- a/src/detect.c
+++ b/src/detect.c
@@ -822,6 +822,7 @@ static DetectRunScratchpad DetectRunSetup(
 
 #ifdef UNITTESTS
     p->alerts.cnt = 0;
+    p->alerts.discarded = 0;
 #endif
     det_ctx->ticker++;
     det_ctx->filestore_cnt = 0;
@@ -931,6 +932,9 @@ static inline void DetectRunPostRules(
     if (p->alerts.cnt > 0) {
         StatsAddUI64(tv, det_ctx->counter_alerts, (uint64_t)p->alerts.cnt);
     }
+    if (p->alerts.discarded > 0) {
+        StatsAddUI64(tv, det_ctx->counter_alerts_overflow, (uint64_t)p->alerts.discarded);
+    }
     PACKET_PROFILING_DETECT_END(p, PROF_DETECT_ALERT);
 }
 

diff --git a/src/detect.h b/src/detect.h
index ea3c0b8ae5457ad0e801e691098cd0a796409b62..ba4283c06c81662ad421fd7915ae285523f09ac0 100644 (file)
--- a/src/detect.h
+++ b/src/detect.h
@@ -1045,6 +1045,8 @@ typedef struct DetectEngineThreadCtx_ {
 
     /** id for alert counter */
     uint16_t counter_alerts;
+    /** id for discarded alerts counter**/
+    uint16_t counter_alerts_overflow;
 #ifdef PROFILING
     uint16_t counter_mpm_list;
     uint16_t counter_nonmpm_list;