flow/output: log triggered exception policies

To accompany the Exception Policy stats, also add information about any
Exception Policy triggered and for which target to the flow log event.

Task #6215

diff --git a/doc/userguide/configuration/exception-policies.rst b/doc/userguide/configuration/exception-policies.rst
index a50289d9eebb6eb1d3805df2b45c76fcf50c2ff0..d559604445e06e5f53f4346faf08aaa044493e02 100644 (file)
--- a/doc/userguide/configuration/exception-policies.rst
+++ b/doc/userguide/configuration/exception-policies.rst
@@ -59,6 +59,8 @@ It is possible to disable this default, by setting the exception policies'
 **In IDS mode**, setting ``auto`` mode actually means disabling the
 ``master-switch``, or ignoring the exception policies.
 
+.. _eps_settings:
+
 Specific settings
 ~~~~~~~~~~~~~~~~~
 
@@ -210,10 +212,57 @@ Notes:
    * Not valid means that Suricata will error out and won't start.
    * ``REJECT`` will make Suricata send a Reset-packet unreach error to the sender of the matching packet.
 
+.. _eps_output:
+
+Log Output
+----------
+
+.. _eps_flow_event:
+
+Flow Event
+~~~~~~~~~~
+
+When an Exception Policy is triggered, this will be indicated in the flow log
+event for the associated flow, also indicating which target triggered that, and
+what policy was applied. If no exception policy is triggered, that field won't
+be present in the logs.
+
+Note that this is true even if the policy is applied only to certain packets from
+a flow.
+
+In the log sample below, the flow triggered the ``midstream policy``, leading
+to Suricata applying the behavior that had been configured for such scenario:
+*to pass the flow* (``pass_flow``). It also did trigger the ``app_layer_error``
+exception policy, but that is set up to ``ignore``::
+
+  "flow": {
+    "pkts_toserver": 4,
+    "pkts_toclient": 5,
+    "bytes_toserver": 495,
+    "bytes_toclient": 351,
+    "start": "2016-07-13T22:42:07.199672+0000",
+    "end": "2016-07-13T22:42:07.573174+0000",
+    "age": 0,
+    "state": "new",
+    "reason": "shutdown",
+    "alerted": false,
+    "action": "pass",
+    "exception_policy": [
+       {
+         "target": "stream_midstream",
+         "policy": "pass_flow"
+       },
+       {
+         "target": "app_layer_error",
+         "policy": "ignore"
+       }
+     ]
+  }
+
 .. _eps_stats:
 
 Available Stats
----------------
+~~~~~~~~~~~~~~~
 
 There are stats counters for each supported exception policy scenario:
 

diff --git a/doc/userguide/output/eve/eve-json-format.rst b/doc/userguide/output/eve/eve-json-format.rst
index 952945dffc98671b918cad25590294df9cb3ea78..744591b8a9810611942ce45cd1285274c7cc17d4 100644 (file)
--- a/doc/userguide/output/eve/eve-json-format.rst
+++ b/doc/userguide/output/eve/eve-json-format.rst
@@ -314,7 +314,13 @@ Event type: ``flow``::
         "age": 40,
         "state": "closed",
         "reason": "shutdown",
-        "alerted": true
+        "alerted": true,
+        "exception_policy": [
+          {
+            "target": "stream_midstream",
+            "policy": "ignore"
+          }
+        ]
       },
       "ether": {
         "dest_macs": [
@@ -1669,6 +1675,14 @@ Fields
 * "state": display state of the flow (include "new", "established", "closed", "bypassed")
 * "reason": mechanism that did trigger the end of the flow (include "timeout", "forced" and "shutdown")
 * "alerted": "true" or "false" depending if an alert has been seen on flow
+* "action": "pass" or "drop" depending if flow was PASS'ed or DROP'ed (no present if none)
+* "exception_policy": array consisting of exception policies that have been triggered by
+  the flow:
+
+    * "target": if an exception policy was triggered, what setting exceptions
+      led to this (cf. :ref:`Exception Policy - Specific Settings<eps_settings>`).
+    * "policy": if an exception policy was triggered, what policy was applied
+      (to the flow or to any packet(s) from it).
 
 Example ::
 
@@ -1689,7 +1703,14 @@ Example ::
     "bypass": "capture",
     "state": "bypassed",
     "reason": "timeout",
-    "alerted": false
+    "alerted": false,
+    "action": "pass",
+    "exception_policy": [
+      {
+        "target": "stream_midstream",
+        "policy": "pass_flow"
+      }
+    ]
   }
 
 Event type: RDP

diff --git a/etc/schema.json b/etc/schema.json
index ff05ed0bc738eab9c3be0eae68f7dc5b4990ab26..2f516f840d95f54e6b3d4dcfd9a1c37262798ff1 100644 (file)
--- a/etc/schema.json
+++ b/etc/schema.json
@@ -1932,6 +1932,20 @@
                 "end": {
                     "type": "string"
                 },
+                "exception_policy": {
+                    "description": "The exception policy(ies) triggered by the flow. Not logged if none was triggered",
+                    "type": "array",
+                    "properties": {
+                        "target": {
+                            "description": "What triggered the exception",
+                            "type": "string"
+                        },
+                        "policy": {
+                            "description": "Which exception policy was applied",
+                            "type": "string"
+                        }
+                    }
+                },
                 "pkts_toclient": {
                     "type": "integer"
                 },

diff --git a/src/app-layer-parser.c b/src/app-layer-parser.c
index 8350e0cec0037f9d24693851468297af9b3d8e1b..47012a7db02c79e14d55d7c1e9776a0d4e7002aa 100644 (file)
--- a/src/app-layer-parser.c
+++ b/src/app-layer-parser.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2021 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -153,6 +153,11 @@ static void AppLayerConfig(void)
     g_applayerparser_error_policy = ExceptionPolicyParse("app-layer.error-policy", true);
 }
 
+enum ExceptionPolicy AppLayerErrorGetExceptionPolicy(void)
+{
+    return g_applayerparser_error_policy;
+}
+
 static void AppLayerParserFramesFreeContainer(FramesContainer *frames)
 {
     if (frames != NULL) {

diff --git a/src/app-layer-parser.h b/src/app-layer-parser.h
index d233edf9eb1f28c6ac8d46b35e29c684c433630c..ddde6db91ca567c58d8b59aa58fa80b57f8593ff 100644 (file)
--- a/src/app-layer-parser.h
+++ b/src/app-layer-parser.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2020 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -136,6 +136,8 @@ void AppLayerParserThreadCtxFree(AppLayerParserThreadCtx *tctx);
 int AppLayerParserConfParserEnabled(const char *ipproto,
                                     const char *alproto_name);
 
+enum ExceptionPolicy AppLayerErrorGetExceptionPolicy(void);
+
 /** \brief Prototype for parsing functions */
 typedef AppLayerResult (*AppLayerParserFPtr)(Flow *f, void *protocol_state,
         AppLayerParserState *pstate, StreamSlice stream_slice, void *local_storage);

diff --git a/src/flow.h b/src/flow.h
index 18bbad4c5409fa25152ca0b60780ffc0ef0c9a6a..cf083387a308801fe718b934bdf7574441297e73 100644 (file)
--- a/src/flow.h
+++ b/src/flow.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2024 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -467,6 +467,9 @@ typedef struct Flow_
     uint8_t min_ttl_toclient;
     uint8_t max_ttl_toclient;
 
+    /** which exception policies were applied, if any */
+    uint8_t applied_exception_policy;
+
     /** application level storage ptrs.
      *
      */

diff --git a/src/output-json-flow.c b/src/output-json-flow.c
index d30866636f170737994105e99ea16d738e9af4cd..eb7c554c06bed08b443c365cbc4da2af9dbfc277 100644 (file)
--- a/src/output-json-flow.c
+++ b/src/output-json-flow.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2020 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -50,6 +50,7 @@
 #include "stream-tcp.h"
 #include "stream-tcp-private.h"
 #include "flow-storage.h"
+#include "util-exception-policy.h"
 
 static JsonBuilder *CreateEveHeaderFromFlow(const Flow *f)
 {
@@ -214,6 +215,65 @@ void EveAddFlow(Flow *f, JsonBuilder *js)
     jb_set_string(js, "start", timebuf1);
 }
 
+static void EveExceptionPolicyLog(JsonBuilder *js, uint16_t flag)
+{
+    if (flag & EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP) {
+        jb_start_object(js);
+        jb_set_string(js, "target",
+                ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP), true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_SESSION_MEMCAP) {
+        jb_start_object(js);
+        jb_set_string(js, "target",
+                ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_SESSION_MEMCAP));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_SESSION_MEMCAP), true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP) {
+        jb_start_object(js);
+        jb_set_string(js, "target",
+                ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP),
+                        true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_FLOW_MEMCAP) {
+        jb_start_object(js);
+        jb_set_string(
+                js, "target", ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_FLOW_MEMCAP));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_FLOW_MEMCAP), true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_MIDSTREAM) {
+        jb_start_object(js);
+        jb_set_string(
+                js, "target", ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_MIDSTREAM));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_MIDSTREAM), true));
+        jb_close(js);
+    }
+    if (flag & EXCEPTION_TARGET_FLAG_APPLAYER_ERROR) {
+        jb_start_object(js);
+        jb_set_string(js, "target",
+                ExceptionPolicyTargetFlagToString(EXCEPTION_TARGET_FLAG_APPLAYER_ERROR));
+        jb_set_string(js, "policy",
+                ExceptionPolicyEnumToString(
+                        ExceptionPolicyTargetPolicy(EXCEPTION_TARGET_FLAG_APPLAYER_ERROR), true));
+        jb_close(js);
+    }
+}
+
 /* Eve format logging */
 static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f)
 {
@@ -280,6 +340,11 @@ static void EveFlowLogJSON(OutputJsonThreadCtx *aft, JsonBuilder *jb, Flow *f)
     } else if (f->flags & FLOW_ACTION_PASS) {
         JB_SET_STRING(jb, "action", "pass");
     }
+    if (f->applied_exception_policy != 0) {
+        jb_open_array(jb, "exception_policy");
+        EveExceptionPolicyLog(jb, f->applied_exception_policy);
+        jb_close(jb); /* close array */
+    }
 
     /* Close flow. */
     jb_close(jb);

diff --git a/src/stream-tcp.c b/src/stream-tcp.c
index 47f2b36ecd6ea08ee56cccc453ee92b61ac88443..0d313dc5e2a0cc64ad626dde8c965eca002dd5c9 100644 (file)
--- a/src/stream-tcp.c
+++ b/src/stream-tcp.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2024 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -896,6 +896,21 @@ static void StreamTcpSsnMemcapExceptionPolicyStatsIncr(
     }
 }
 
+enum ExceptionPolicy StreamTcpSsnMemcapGetExceptionPolicy(void)
+{
+    return stream_config.ssn_memcap_policy;
+}
+
+enum ExceptionPolicy StreamTcpReassemblyMemcapGetExceptionPolicy(void)
+{
+    return stream_config.reassembly_memcap_policy;
+}
+
+enum ExceptionPolicy StreamMidstreamGetExceptionPolicy(void)
+{
+    return stream_config.midstream_policy;
+}
+
 /** \internal
  *  \brief The function is used to fetch a TCP session from the
  *         ssn_pool, when a TCP SYN is received.

diff --git a/src/stream-tcp.h b/src/stream-tcp.h
index 6303874a1a96410b2191e08b380d5fe2ac09d92c..3e2768db07e6a45a099f525912fd3c8b3254ad8b 100644 (file)
--- a/src/stream-tcp.h
+++ b/src/stream-tcp.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2024 Open Information Security Foundation
+/* Copyright (C) 2007-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -158,6 +158,10 @@ void StreamTcpDetectLogFlush(ThreadVars *tv, StreamTcpThread *stt, Flow *f, Pack
 const char *StreamTcpStateAsString(const enum TcpState);
 const char *StreamTcpSsnStateAsString(const TcpSession *ssn);
 
+enum ExceptionPolicy StreamTcpSsnMemcapGetExceptionPolicy(void);
+enum ExceptionPolicy StreamTcpReassemblyMemcapGetExceptionPolicy(void);
+enum ExceptionPolicy StreamMidstreamGetExceptionPolicy(void);
+
 /** ------- Inline functions: ------ */
 
 /**

diff --git a/src/util-exception-policy-types.h b/src/util-exception-policy-types.h
index a6139acc8934eef18dab8b3f280b01fe73a00e9e..7df6d0d82c06f71cba015ac4d7a231fb9f6480a3 100644 (file)
--- a/src/util-exception-policy-types.h
+++ b/src/util-exception-policy-types.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2024 Open Information Security Foundation
+/* Copyright (C) 2024-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -40,6 +40,14 @@ enum ExceptionPolicy {
  * "tcp.reassembly_exception_policy.drop_packet" + 1 */
 #define EXCEPTION_POLICY_COUNTER_MAX_LEN 45
 
+/** Flags for possible scenario/ config settings for exception policies */
+#define EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP     BIT_U8(0)
+#define EXCEPTION_TARGET_FLAG_SESSION_MEMCAP    BIT_U8(1)
+#define EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP BIT_U8(2)
+#define EXCEPTION_TARGET_FLAG_FLOW_MEMCAP       BIT_U8(3)
+#define EXCEPTION_TARGET_FLAG_MIDSTREAM         BIT_U8(4)
+#define EXCEPTION_TARGET_FLAG_APPLAYER_ERROR    BIT_U8(5)
+
 typedef struct ExceptionPolicyCounters_ {
     /* Follows enum order */
     uint16_t eps_id[EXCEPTION_POLICY_MAX];

diff --git a/src/util-exception-policy.c b/src/util-exception-policy.c
index 9900720202690c486d92ac757b1f70cbb53de076..d586dc499d3cff57b68f139ac3e9fc6e7b13d66d 100644 (file)
--- a/src/util-exception-policy.c
+++ b/src/util-exception-policy.c
@@ -1,4 +1,4 @@
-/* Copyright (C) 2022-2024 Open Information Security Foundation
+/* Copyright (C) 2022-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -19,14 +19,18 @@
  * \file
  */
 
+#include "util-exception-policy.h"
 #include "suricata-common.h"
 #include "suricata.h"
 #include "packet.h"
-#include "util-exception-policy.h"
 #include "util-misc.h"
 #include "stream-tcp-reassemble.h"
 #include "action-globals.h"
 #include "conf.h"
+#include "flow.h"
+#include "stream-tcp.h"
+#include "defrag-hash.h"
+#include "app-layer-parser.h"
 
 enum ExceptionPolicy g_eps_master_switch = EXCEPTION_POLICY_NOT_SET;
 /** true if exception policy was defined in config */
@@ -66,9 +70,76 @@ static enum ExceptionPolicy GetMasterExceptionPolicy(void)
     return g_eps_master_switch;
 }
 
+static uint8_t ExceptionPolicyFlag(enum PacketDropReason drop_reason)
+{
+    switch (drop_reason) {
+        case PKT_DROP_REASON_DEFRAG_MEMCAP:
+            return EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP;
+        case PKT_DROP_REASON_STREAM_MEMCAP:
+            return EXCEPTION_TARGET_FLAG_SESSION_MEMCAP;
+        case PKT_DROP_REASON_STREAM_REASSEMBLY:
+            return EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP;
+        case PKT_DROP_REASON_FLOW_MEMCAP:
+            return EXCEPTION_TARGET_FLAG_FLOW_MEMCAP;
+        case PKT_DROP_REASON_STREAM_MIDSTREAM:
+            return EXCEPTION_TARGET_FLAG_MIDSTREAM;
+        case PKT_DROP_REASON_APPLAYER_ERROR:
+            return EXCEPTION_TARGET_FLAG_APPLAYER_ERROR;
+        default:
+            return 0;
+    }
+
+    return 0;
+}
+
+const char *ExceptionPolicyTargetFlagToString(uint8_t target_flag)
+{
+    switch (target_flag) {
+        case EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP:
+            return "defrag_memcap";
+        case EXCEPTION_TARGET_FLAG_SESSION_MEMCAP:
+            return "stream_memcap";
+        case EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP:
+            return "stream_reassembly_memcap";
+        case EXCEPTION_TARGET_FLAG_FLOW_MEMCAP:
+            return "flow_memcap";
+        case EXCEPTION_TARGET_FLAG_MIDSTREAM:
+            return "stream_midstream";
+        case EXCEPTION_TARGET_FLAG_APPLAYER_ERROR:
+            return "app_layer_error";
+        default:
+            return "none";
+    }
+    return "none";
+}
+
+enum ExceptionPolicy ExceptionPolicyTargetPolicy(uint8_t target_flag)
+{
+    switch (target_flag) {
+        case EXCEPTION_TARGET_FLAG_DEFRAG_MEMCAP:
+            return DefragGetMemcapExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_SESSION_MEMCAP:
+            return StreamTcpSsnMemcapGetExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_REASSEMBLY_MEMCAP:
+            return StreamTcpReassemblyMemcapGetExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_FLOW_MEMCAP:
+            return FlowGetMemcapExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_MIDSTREAM:
+            return StreamMidstreamGetExceptionPolicy();
+        case EXCEPTION_TARGET_FLAG_APPLAYER_ERROR:
+            return AppLayerErrorGetExceptionPolicy();
+        default:
+            return EXCEPTION_POLICY_NOT_SET;
+    }
+    return EXCEPTION_POLICY_NOT_SET;
+}
+
 void ExceptionPolicyApply(Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason)
 {
     SCLogDebug("start: pcap_cnt %" PRIu64 ", policy %u", p->pcap_cnt, policy);
+    if (p->flow) {
+        p->flow->applied_exception_policy |= ExceptionPolicyFlag(drop_reason);
+    }
     switch (policy) {
         case EXCEPTION_POLICY_AUTO:
             break;

diff --git a/src/util-exception-policy.h b/src/util-exception-policy.h
index 6f50c929ed8ad85e09337a3d83cee1fc058c0a94..9d3dcc4260d6d98ac9caebf6cbbedad79e726713 100644 (file)
--- a/src/util-exception-policy.h
+++ b/src/util-exception-policy.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2022-2024 Open Information Security Foundation
+/* Copyright (C) 2022-2025 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -26,6 +26,8 @@
 #include "util-exception-policy-types.h"
 
 const char *ExceptionPolicyEnumToString(enum ExceptionPolicy policy, bool is_json);
+const char *ExceptionPolicyTargetFlagToString(uint8_t target_flag);
+enum ExceptionPolicy ExceptionPolicyTargetPolicy(uint8_t target_flag);
 void SetMasterExceptionPolicy(void);
 void ExceptionPolicyApply(
         Packet *p, enum ExceptionPolicy policy, enum PacketDropReason drop_reason);