applayer: add stats counters for exception errors

Add stats counters for exception policy are applied for app-layer errors

Part of
Task #5816

diff --git a/etc/schema.json b/etc/schema.json
index 587135d8bce0b507fc1a9cf6c1a5f59b4d45308e..4c852ec9b3d1db188fc05b982f33bba36ebdb165 100644 (file)
--- a/etc/schema.json
+++ b/etc/schema.json
@@ -3753,6 +3753,11 @@
                         "error": {
                             "type": "object",
                             "properties": {
+                                "exception_policy": {
+                                    "description":
+                                            "Consolidated stats on how many times app-layer error exception policy was applied, and which one",
+                                    "$ref": "#/$defs/exceptionPolicy"
+                                },
                                 "bittorrent-dht": {
                                     "description":
                                             "Errors encountered parsing BitTorrent DHT protocol",
@@ -5691,6 +5696,11 @@
                 "internal": {
                     "description": "Number of internal parser errors",
                     "type": "integer"
+                },
+                "exception_policy": {
+                    "description":
+                            "How many times app-layer error exception policy was applied, and which one",
+                    "$ref": "#/$defs/exceptionPolicy"
                 }
             },
             "additionalProperties": false

diff --git a/src/app-layer.c b/src/app-layer.c
index e159b932afc38f92ed1a1682b4df672952505924..2566861025fc3331a5c75090f542b2f0052885e0 100644 (file)
--- a/src/app-layer.c
+++ b/src/app-layer.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2023 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -49,6 +49,7 @@
 #include "app-layer-htp-mem.h"
 #include "util-exception-policy.h"
 
+extern bool g_stats_eps_per_app_proto_errors;
 /**
  * \brief This is for the app layer in general and it contains per thread
  *        context relevant to both the alpd and alp.
@@ -80,6 +81,7 @@ typedef struct AppLayerCounterNames_ {
     char parser_error[MAX_COUNTER_SIZE];
     char internal_error[MAX_COUNTER_SIZE];
     char alloc_error[MAX_COUNTER_SIZE];
+    char eps_name[EXCEPTION_POLICY_MAX][MAX_COUNTER_SIZE];
 } AppLayerCounterNames;
 
 typedef struct AppLayerCounters_ {
@@ -89,12 +91,41 @@ typedef struct AppLayerCounters_ {
     uint16_t parser_error_id;
     uint16_t internal_error_id;
     uint16_t alloc_error_id;
+    ExceptionPolicyCounters eps_error;
 } AppLayerCounters;
 
 /* counter names. Only used at init. */
 AppLayerCounterNames applayer_counter_names[FLOW_PROTO_APPLAYER_MAX][ALPROTO_MAX];
 /* counter id's. Used that runtime. */
 AppLayerCounters applayer_counters[FLOW_PROTO_APPLAYER_MAX][ALPROTO_MAX];
+/* Exception policy global counters ids */
+ExceptionPolicyCounters eps_error_summary;
+
+/* Settings order as in the enum */
+// clang-format off
+ExceptionPolicyStatsSetts app_layer_error_eps_stats = {
+    .valid_settings_ids = {
+       /* EXCEPTION_POLICY_NOT_SET */      false,
+       /* EXCEPTION_POLICY_AUTO */         false,
+       /* EXCEPTION_POLICY_PASS_PACKET */  true,
+       /* EXCEPTION_POLICY_PASS_FLOW */    true,
+       /* EXCEPTION_POLICY_BYPASS_FLOW */  true,
+       /* EXCEPTION_POLICY_DROP_PACKET */  false,
+       /* EXCEPTION_POLICY_DROP_FLOW */    false,
+       /* EXCEPTION_POLICY_REJECT */       true,
+    },
+    .valid_settings_ips = {
+       /* EXCEPTION_POLICY_NOT_SET */      false,
+       /* EXCEPTION_POLICY_AUTO */         false,
+       /* EXCEPTION_POLICY_PASS_PACKET */  true,
+       /* EXCEPTION_POLICY_PASS_FLOW */    true,
+       /* EXCEPTION_POLICY_BYPASS_FLOW */  true,
+       /* EXCEPTION_POLICY_DROP_PACKET */  true,
+       /* EXCEPTION_POLICY_DROP_FLOW */    true,
+       /* EXCEPTION_POLICY_REJECT */       true,
+    },
+};
+// clang-format on
 
 void AppLayerSetupCounters(void);
 void AppLayerDeSetupCounters(void);
@@ -159,6 +190,25 @@ void AppLayerIncInternalErrorCounter(ThreadVars *tv, Flow *f)
     }
 }
 
+static void AppLayerIncrErrorExcPolicyCounter(ThreadVars *tv, Flow *f, enum ExceptionPolicy policy)
+{
+#ifdef UNITTESTS
+    if (tv == NULL) {
+        return;
+    }
+#endif
+    uint16_t id = applayer_counters[f->protomap][f->alproto].eps_error.eps_id[policy];
+    /* for the summary values */
+    uint16_t g_id = eps_error_summary.eps_id[policy];
+
+    if (likely(id > 0)) {
+        StatsIncr(tv, id);
+    }
+    if (likely(g_id > 0)) {
+        StatsIncr(tv, g_id);
+    }
+}
+
 /* in IDS mode protocol detection is done in reverse order:
  * when TCP data is ack'd. We want to flag the correct packet,
  * so in this case we set a flag in the flow so that the first
@@ -640,6 +690,7 @@ static int TCPProtoDetect(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx,
     SCReturnInt(0);
 parser_error:
     ExceptionPolicyApply(p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR);
+    AppLayerIncrErrorExcPolicyCounter(tv, f, g_applayerparser_error_policy);
     SCReturnInt(-1);
 detect_error:
     DisableAppLayer(tv, f, p);
@@ -707,6 +758,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
         StreamTcpUpdateAppLayerProgress(ssn, direction, data_len);
         if (r < 0) {
             ExceptionPolicyApply(p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR);
+            AppLayerIncrErrorExcPolicyCounter(tv, f, g_applayerparser_error_policy);
             SCReturnInt(-1);
         }
         goto end;
@@ -793,6 +845,7 @@ int AppLayerHandleTCPData(ThreadVars *tv, TcpReassemblyThreadCtx *ra_ctx, Packet
                 if (r < 0) {
                     ExceptionPolicyApply(
                             p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR);
+                    AppLayerIncrErrorExcPolicyCounter(tv, f, g_applayerparser_error_policy);
                     SCReturnInt(-1);
                 }
             }
@@ -933,6 +986,7 @@ int AppLayerHandleUdp(ThreadVars *tv, AppLayerThreadCtx *tctx, Packet *p, Flow *
     }
     if (r < 0) {
         ExceptionPolicyApply(p, g_applayerparser_error_policy, PKT_DROP_REASON_APPLAYER_ERROR);
+        AppLayerIncrErrorExcPolicyCounter(tv, f, g_applayerparser_error_policy);
         SCReturnInt(-1);
     }
 
@@ -1062,6 +1116,30 @@ void AppLayerRegisterGlobalCounters(void)
     StatsRegisterGlobalCounter("app_layer.expectations", ExpectationGetCounter);
 }
 
+static bool IsAppLayerErrorExceptionPolicyStatsValid(enum ExceptionPolicy policy)
+{
+    if (EngineModeIsIPS()) {
+        return app_layer_error_eps_stats.valid_settings_ips[policy];
+    }
+    return app_layer_error_eps_stats.valid_settings_ids[policy];
+}
+
+static void AppLayerSetupExceptionPolicyPerProtoCounters(
+        uint8_t ipproto_map, AppProto alproto, const char *alproto_str, const char *ipproto_suffix)
+{
+    if (g_stats_eps_per_app_proto_errors &&
+            g_applayerparser_error_policy != EXCEPTION_POLICY_NOT_SET) {
+        for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1; i < EXCEPTION_POLICY_MAX; i++) {
+            if (IsAppLayerErrorExceptionPolicyStatsValid(i)) {
+                snprintf(applayer_counter_names[ipproto_map][alproto].eps_name[i],
+                        sizeof(applayer_counter_names[ipproto_map][alproto].eps_name[i]),
+                        "app_layer.error.%s%s.exception_policy.%s", alproto_str, ipproto_suffix,
+                        ExceptionPolicyEnumToString(i, true));
+            }
+        }
+    }
+}
+
 #define IPPROTOS_MAX 2
 void AppLayerSetupCounters(void)
 {
@@ -1070,6 +1148,19 @@ void AppLayerSetupCounters(void)
     const char *str = "app_layer.flow.";
     const char *estr = "app_layer.error.";
 
+    /* We don't log stats counters if exception policy is `ignore`/`not set` */
+    if (g_applayerparser_error_policy != EXCEPTION_POLICY_NOT_SET) {
+        /* Register global counters for app layer error exception policy summary */
+        const char *eps_default_str = "app_layer.error.exception_policy.";
+        for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1; i < EXCEPTION_POLICY_MAX; i++) {
+            if (IsAppLayerErrorExceptionPolicyStatsValid(i)) {
+                snprintf(app_layer_error_eps_stats.eps_name[i],
+                        sizeof(app_layer_error_eps_stats.eps_name[i]), "%s%s", eps_default_str,
+                        ExceptionPolicyEnumToString(i, true));
+            }
+        }
+    }
+
     AppLayerProtoDetectSupportedAppProtocols(alprotos);
 
     for (uint8_t p = 0; p < IPPROTOS_MAX; p++) {
@@ -1108,6 +1199,9 @@ void AppLayerSetupCounters(void)
                     snprintf(applayer_counter_names[ipproto_map][alproto].internal_error,
                             sizeof(applayer_counter_names[ipproto_map][alproto].internal_error),
                             "%s%s%s.internal", estr, alproto_str, ipproto_suffix);
+
+                    AppLayerSetupExceptionPolicyPerProtoCounters(
+                            ipproto_map, alproto, alproto_str, ipproto_suffix);
                 } else {
                     snprintf(applayer_counter_names[ipproto_map][alproto].name,
                             sizeof(applayer_counter_names[ipproto_map][alproto].name),
@@ -1130,6 +1224,8 @@ void AppLayerSetupCounters(void)
                     snprintf(applayer_counter_names[ipproto_map][alproto].internal_error,
                             sizeof(applayer_counter_names[ipproto_map][alproto].internal_error),
                             "%s%s.internal", estr, alproto_str);
+                    AppLayerSetupExceptionPolicyPerProtoCounters(
+                            ipproto_map, alproto, alproto_str, "");
                 }
             } else if (alproto == ALPROTO_FAILED) {
                 snprintf(applayer_counter_names[ipproto_map][alproto].name,
@@ -1151,6 +1247,17 @@ void AppLayerRegisterThreadCounters(ThreadVars *tv)
     AppProto alprotos[ALPROTO_MAX];
     AppLayerProtoDetectSupportedAppProtocols(alprotos);
 
+    /* We don't log stats counters if exception policy is `ignore`/`not set` */
+    if (g_applayerparser_error_policy != EXCEPTION_POLICY_NOT_SET) {
+        /* Register global counters for app layer error exception policy summary */
+        for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1; i < EXCEPTION_POLICY_MAX; i++) {
+            if (IsAppLayerErrorExceptionPolicyStatsValid(i)) {
+                eps_error_summary.eps_id[i] =
+                        StatsRegisterCounter(app_layer_error_eps_stats.eps_name[i], tv);
+            }
+        }
+    }
+
     for (uint8_t p = 0; p < IPPROTOS_MAX; p++) {
         const uint8_t ipproto = ipprotos[p];
         const uint8_t ipproto_map = FlowGetProtoMapping(ipproto);
@@ -1173,6 +1280,18 @@ void AppLayerRegisterThreadCounters(ThreadVars *tv)
                         applayer_counter_names[ipproto_map][alproto].parser_error, tv);
                 applayer_counters[ipproto_map][alproto].internal_error_id = StatsRegisterCounter(
                         applayer_counter_names[ipproto_map][alproto].internal_error, tv);
+                /* We don't log stats counters if exception policy is `ignore`/`not set` */
+                if (g_stats_eps_per_app_proto_errors &&
+                        g_applayerparser_error_policy != EXCEPTION_POLICY_NOT_SET) {
+                    for (enum ExceptionPolicy i = EXCEPTION_POLICY_NOT_SET + 1;
+                            i < EXCEPTION_POLICY_MAX; i++) {
+                        if (IsAppLayerErrorExceptionPolicyStatsValid(i)) {
+                            applayer_counters[ipproto_map][alproto]
+                                    .eps_error.eps_id[i] = StatsRegisterCounter(
+                                    applayer_counter_names[ipproto_map][alproto].eps_name[i], tv);
+                        }
+                    }
+                }
             } else if (alproto == ALPROTO_FAILED) {
                 applayer_counters[ipproto_map][alproto].counter_id =
                     StatsRegisterCounter(applayer_counter_names[ipproto_map][alproto].name, tv);

diff --git a/src/suricata.c b/src/suricata.c
index 6fdedf6c55eeb383dac517fd1d561c882975407e..4e064c6c98bc65ceaf19f089ac39d0d3be3063f2 100644 (file)
--- a/src/suricata.c
+++ b/src/suricata.c
@@ -215,6 +215,9 @@ bool g_disable_hashing = false;
 /* snapshot of the system's hugepages before system intitialization. */
 SystemHugepageSnapshot *prerun_snap = NULL;
 
+/** add per-proto app-layer error counters for exception policies stats? disabled by default */
+bool g_stats_eps_per_app_proto_errors = false;
+
 /** Suricata instance */
 SCInstance suricata;
 
@@ -2698,6 +2701,13 @@ int PostConfLoadedSetup(SCInstance *suri)
 
     SetMasterExceptionPolicy();
 
+    ConfNode *eps = ConfGetNode("stats.exception-policy");
+    if (eps != NULL) {
+        if (ConfNodeChildValueIsTrue(eps, "per-app-proto-errors")) {
+            g_stats_eps_per_app_proto_errors = true;
+        }
+    }
+
     AppLayerSetup();
 
     /* Suricata will use this umask if provided. By default it will use the

diff --git a/suricata.yaml.in b/suricata.yaml.in
index 38f5152f5dade67f328e85888b951aba3f4ea572..15796700d0166067fd8fd0b2c592f394263a1de5 100644 (file)
--- a/suricata.yaml.in
+++ b/suricata.yaml.in
@@ -74,6 +74,9 @@ stats:
   #decoder-events-prefix: "decoder.event"
   # Add stream events as stats.
   #stream-events: false
+  exception-policy:
+    #per-app-proto-errors: false  # default: false. True will log errors for
+                                  # each app-proto. Warning: VERY verbose
 
 # Plugins -- Experimental -- specify the filename for each plugin shared object
 plugins: