decode/flow: add exception policy stats counters

We will register stats counters for all policies, even though for now
Suri only uses one possible configuration policy at a time. The idea is
that this could change in the near future, so we want to have this
ready.

Task #5816

diff --git a/etc/schema.json b/etc/schema.json
index d39c35c704f99c2f207e4ab45497be60649a3816..8aa3a028f38b1655755f7cc6ecffa1124349c3aa 100644 (file)
--- a/etc/schema.json
+++ b/etc/schema.json
@@ -5043,6 +5043,11 @@
                         "memcap": {
                             "type": "integer"
                         },
+                        "memcap_exception_policy": {
+                            "description":
+                                    "How many times flow memcap exception policy was applied, and which one",
+                            "$ref": "#/$defs/exceptionPolicy"
+                        },
                         "memuse": {
                             "type": "integer"
                         },
@@ -5723,6 +5728,35 @@
                     ]
                 }
             }
+        },
+        "exceptionPolicy": {
+            "type": "object",
+            "properties": {
+                "drop_flow": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "drop_packet": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "pass_flow": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "pass_packet": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "bypass": {
+                    "type": "integer",
+                    "minimum": 0
+                },
+                "reject": {
+                    "type": "integer",
+                    "minimum": 0
+                }
+            }
         }
     }
 }

diff --git a/src/counters.c b/src/counters.c
index 790f416ba050274aecce457c91c6af239a762a48..36172d913d15cef4c7ccb2fae225e0f299173b3c 100644 (file)
--- a/src/counters.c
+++ b/src/counters.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2021 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free

diff --git a/src/decode.c b/src/decode.c
index 70ef282b9f1a284fb9a4a8230816555cd81ba4db..64a6d43d56f9fb99935271f721df5ab3dd8ff619 100644 (file)
--- a/src/decode.c
+++ b/src/decode.c
@@ -76,6 +76,32 @@ extern bool stats_stream_events;
 uint8_t decoder_max_layers = PKT_DEFAULT_MAX_DECODED_LAYERS;
 uint16_t packet_alert_max = PACKET_ALERT_MAX;
 
+/* Settings order as in the enum */
+// clang-format off
+ExceptionPolicyStatsSetts flow_memcap_eps_stats = {
+    .valid_settings_ids = {
+    /* EXCEPTION_POLICY_NOT_SET */      false,
+    /* EXCEPTION_POLICY_AUTO */         false,
+    /* EXCEPTION_POLICY_PASS_PACKET */  true,
+    /* EXCEPTION_POLICY_PASS_FLOW */    false,
+    /* EXCEPTION_POLICY_BYPASS_FLOW */  true,
+    /* EXCEPTION_POLICY_DROP_PACKET */  false,
+    /* EXCEPTION_POLICY_DROP_FLOW */    false,
+    /* EXCEPTION_POLICY_REJECT */       true,
+    },
+    .valid_settings_ips = {
+    /* EXCEPTION_POLICY_NOT_SET */      false,
+    /* EXCEPTION_POLICY_AUTO */         false,
+    /* EXCEPTION_POLICY_PASS_PACKET */  true,
+    /* EXCEPTION_POLICY_PASS_FLOW */    false,
+    /* EXCEPTION_POLICY_BYPASS_FLOW */  true,
+    /* EXCEPTION_POLICY_DROP_PACKET */  true,
+    /* EXCEPTION_POLICY_DROP_FLOW */    false,
+    /* EXCEPTION_POLICY_REJECT */       true,
+    },
+};
+// clang-format on
+
 /**
  * \brief Initialize PacketAlerts with dynamic alerts array size
  *
@@ -525,6 +551,14 @@ void DecodeUnregisterCounters(void)
     SCMutexUnlock(&g_counter_table_mutex);
 }
 
+static bool IsFlowMemcapExceptionPolicyStatsValid(enum ExceptionPolicy policy)
+{
+    if (EngineModeIsIPS()) {
+        return flow_memcap_eps_stats.valid_settings_ips[policy];
+    }
+    return flow_memcap_eps_stats.valid_settings_ids[policy];
+}
+
 void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
 {
     /* register counters */
@@ -572,6 +606,9 @@ void DecodeRegisterPerfCounters(DecodeThreadVars *dtv, ThreadVars *tv)
     dtv->counter_erspan = StatsRegisterMaxCounter("decoder.erspan", tv);
     dtv->counter_nsh = StatsRegisterMaxCounter("decoder.nsh", tv);
     dtv->counter_flow_memcap = StatsRegisterCounter("flow.memcap", tv);
+    ExceptionPolicySetStatsCounters(tv, &dtv->counter_flow_memcap_eps, &flow_memcap_eps_stats,
+            FlowGetMemcapExceptionPolicy(), "flow.memcap_exception_policy.",
+            IsFlowMemcapExceptionPolicyStatsValid);
 
     dtv->counter_tcp_active_sessions = StatsRegisterCounter("tcp.active_sessions", tv);
     dtv->counter_flow_total = StatsRegisterCounter("flow.total", tv);

diff --git a/src/decode.h b/src/decode.h
index ea73bb2837f0ad90473ec20ca09913488cbdb0a3..65e7ac1bb9293209e46d7a66e7151fe419b45c87 100644 (file)
--- a/src/decode.h
+++ b/src/decode.h
@@ -32,6 +32,7 @@
 #include "threadvars.h"
 #include "util-debug.h"
 #include "decode-events.h"
+#include "util-exception-policy-types.h"
 #ifdef PROFILING
 #include "flow-worker.h"
 #include "app-layer-protos.h"
@@ -740,6 +741,7 @@ typedef struct DecodeThreadVars_
     uint16_t counter_defrag_max_hit;
 
     uint16_t counter_flow_memcap;
+    ExceptionPolicyCounters counter_flow_memcap_eps;
 
     uint16_t counter_tcp_active_sessions;
     uint16_t counter_flow_total;

diff --git a/src/flow-hash.c b/src/flow-hash.c
index 3b221e2ffffc352f86a9fb374273ebcef53f91da..57e3f99158174f91895d8e6e039a323023e4ba5b 100644 (file)
--- a/src/flow-hash.c
+++ b/src/flow-hash.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2023 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -641,9 +641,24 @@ static inline Flow *FlowSpareSync(ThreadVars *tv, FlowLookupStruct *fls,
     return f;
 }
 
-static inline void NoFlowHandleIPS(Packet *p)
+static void FlowExceptionPolicyStatsIncr(
+        ThreadVars *tv, FlowLookupStruct *fls, enum ExceptionPolicy policy)
+{
+#ifdef UNITTESTS
+    if (tv == NULL) {
+        return;
+    }
+#endif
+    uint16_t id = fls->dtv->counter_flow_memcap_eps.eps_id[policy];
+    if (likely(id > 0)) {
+        StatsIncr(tv, id);
+    }
+}
+
+static inline void NoFlowHandleIPS(ThreadVars *tv, FlowLookupStruct *fls, Packet *p)
 {
     ExceptionPolicyApply(p, flow_config.memcap_policy, PKT_DROP_REASON_FLOW_MEMCAP);
+    FlowExceptionPolicyStatsIncr(tv, fls, flow_config.memcap_policy);
 }
 
 /**
@@ -663,7 +678,7 @@ static Flow *FlowGetNew(ThreadVars *tv, FlowLookupStruct *fls, Packet *p)
     const bool emerg = ((SC_ATOMIC_GET(flow_flags) & FLOW_EMERGENCY) != 0);
 #ifdef DEBUG
     if (g_eps_flow_memcap != UINT64_MAX && g_eps_flow_memcap == p->pcap_cnt) {
-        NoFlowHandleIPS(p);
+        NoFlowHandleIPS(tv, fls, p);
         StatsIncr(tv, fls->dtv->counter_flow_memcap);
         return NULL;
     }
@@ -689,7 +704,14 @@ static Flow *FlowGetNew(ThreadVars *tv, FlowLookupStruct *fls, Packet *p)
 
             f = FlowGetUsedFlow(tv, fls->dtv, p->ts);
             if (f == NULL) {
-                NoFlowHandleIPS(p);
+                NoFlowHandleIPS(tv, fls, p);
+#ifdef UNITTESTS
+                if (tv != NULL && fls->dtv != NULL) {
+#endif
+                    StatsIncr(tv, fls->dtv->counter_flow_memcap);
+#ifdef UNITTESTS
+                }
+#endif
                 return NULL;
             }
 #ifdef UNITTESTS
@@ -714,7 +736,7 @@ static Flow *FlowGetNew(ThreadVars *tv, FlowLookupStruct *fls, Packet *p)
 #ifdef UNITTESTS
             }
 #endif
-            NoFlowHandleIPS(p);
+            NoFlowHandleIPS(tv, fls, p);
             return NULL;
         }
 

diff --git a/src/flow.c b/src/flow.c
index b326a0d0fadbbcb34a747f9e7dfee0abd8d437ba..aff504754c31204eea7d5ed94bab6504677851d5 100644 (file)
--- a/src/flow.c
+++ b/src/flow.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -143,6 +143,11 @@ uint64_t FlowGetMemuse(void)
     return memusecopy;
 }
 
+enum ExceptionPolicy FlowGetMemcapExceptionPolicy(void)
+{
+    return flow_config.memcap_policy;
+}
+
 void FlowCleanupAppLayer(Flow *f)
 {
     if (f == NULL || f->proto == 0)

diff --git a/src/flow.h b/src/flow.h
index 7066965c5196d4e14c07225b621e8b8982c8a214..97424c97fade4ddf07993a9c7ed79e9b4dd04266 100644 (file)
--- a/src/flow.h
+++ b/src/flow.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2013 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -30,6 +30,7 @@ typedef struct FlowStorageId FlowStorageId;
 #include "decode.h"
 #include "util-time.h"
 #include "util-exception-policy.h"
+#include "util-exception-policy-types.h"
 #include "util-var.h"
 #include "util-optimize.h"
 #include "app-layer-protos.h"
@@ -576,6 +577,7 @@ void FlowUpdateState(Flow *f, enum FlowState s);
 int FlowSetMemcap(uint64_t size);
 uint64_t FlowGetMemcap(void);
 uint64_t FlowGetMemuse(void);
+enum ExceptionPolicy FlowGetMemcapExceptionPolicy(void);
 
 FlowStorageId GetFlowBypassInfoID(void);
 void RegisterFlowBypassInfo(void);

diff --git a/src/output-json-stats.c b/src/output-json-stats.c
index ecc7dce4b092351fc36a4bbc96c9c113cec05620..41df64f7ce5073d3a378c197b72f034fddbe8101 100644 (file)
--- a/src/output-json-stats.c
+++ b/src/output-json-stats.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2014-2020 Open Information Security Foundation
+/* Copyright (C) 2014-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free

diff --git a/src/util-exception-policy-types.h b/src/util-exception-policy-types.h
index 04e6d3bca4e467c48176c04c3ef0eb268a302bbe..b5295d19305b36fe11512b2076645e79d178c644 100644 (file)
--- a/src/util-exception-policy-types.h
+++ b/src/util-exception-policy-types.h
@@ -38,7 +38,7 @@ enum ExceptionPolicy {
 /* Max length = possible exception policy scenarios + counter names
  * + exception policy type. E.g.:
  * "tcp.reassembly_exception_policy.drop_packet" + 1 */
-#define EXCEPTION_POLICY_COUNTER_MAX_LEN 44
+#define EXCEPTION_POLICY_COUNTER_MAX_LEN 45
 
 typedef struct ExceptionPolicyCounters_ {
     /* Follows enum order */

diff --git a/src/util-exception-policy.c b/src/util-exception-policy.c
index 879f0b67a702b2a588df324fec8e0a982ff92c22..eebda2f70ef44e7573fe3e32deb9824637bbacdb 100644 (file)
--- a/src/util-exception-policy.c
+++ b/src/util-exception-policy.c
@@ -295,6 +295,22 @@ enum ExceptionPolicy ExceptionPolicyMidstreamParse(bool midstream_enabled)
     return policy;
 }
 
+void ExceptionPolicySetStatsCounters(ThreadVars *tv, ExceptionPolicyCounters *counter,
+        ExceptionPolicyStatsSetts *setting, enum ExceptionPolicy conf_policy,
+        const char *default_str, bool (*isExceptionPolicyValid)(enum ExceptionPolicy))
+{
+    if (conf_policy != EXCEPTION_POLICY_NOT_SET) {
+        /* set-up policy counters */
+        for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1; i < EXCEPTION_POLICY_MAX; i++) {
+            if (isExceptionPolicyValid(i)) {
+                snprintf(setting->eps_name[i], sizeof(setting->eps_name[i]), "%s%s", default_str,
+                        ExceptionPolicyEnumToString(i, true));
+                counter->eps_id[i] = StatsRegisterCounter(setting->eps_name[i], tv);
+            }
+        }
+    }
+}
+
 #ifndef DEBUG
 
 int ExceptionSimulationCommandLineParser(const char *name, const char *arg)

diff --git a/src/util-exception-policy.h b/src/util-exception-policy.h
index ffd199fe527db28d6e4d8e6a6582cfeb67bd9880..6f50c929ed8ad85e09337a3d83cee1fc058c0a94 100644 (file)
--- a/src/util-exception-policy.h
+++ b/src/util-exception-policy.h
@@ -31,6 +31,9 @@ void ExceptionPolicyApply(
         Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason);
 enum ExceptionPolicy ExceptionPolicyParse(const char *option, const bool support_flow);
 enum ExceptionPolicy ExceptionPolicyMidstreamParse(bool midstream_enabled);
+void ExceptionPolicySetStatsCounters(ThreadVars *tv, ExceptionPolicyCounters *counter,
+        ExceptionPolicyStatsSetts *setting, enum ExceptionPolicy conf_policy,
+        const char *default_str, bool (*isExceptionPolicyValid)(enum ExceptionPolicy));
 
 extern enum ExceptionPolicy g_eps_master_switch;
 #ifdef DEBUG