Pull request #4134: detection: collect matched buffers

Merge in SNORT/snort3 from ~ASERBENI/snort3:ips_buf_dump to master

Squashed commit of the following:

commit 210d825c271a41d02e04a850fac384e38a04b397
Author: Andrii Serbeniuk <aserbeni@cisco.com>
Date:   Tue Nov 28 14:53:31 2023 +0200

    detection: collect matched buffers on IpsContext

diff --git a/src/detection/detect.cc b/src/detection/detect.cc
index a655cd4728209ca7b1d4be61ebee2d622e55ce43..8b9da1f9d075547399a9c6e615a813934787f45c 100644 (file)
--- a/src/detection/detect.cc
+++ b/src/detection/detect.cc
@@ -121,6 +121,7 @@ void CallAlertFuncs(Packet* p, const OptTreeNode* otn, ListHead* head)
     Event event;
 
     event.sig_info = const_cast<SigInfo*>(&otn->sigInfo);
+    event.buffs_to_dump = otn->buffer_setters;
     event.ref_time.tv_sec = p->pkth->ts.tv_sec;
     event.ref_time.tv_usec = p->pkth->ts.tv_usec;
     event.update_event_id_and_ref(p->context->conf->get_event_log_id());

diff --git a/src/detection/detection_continuation.h b/src/detection/detection_continuation.h
index 891b6c25fdd374aa03971230ecf6b5ab25002339..c28b88990f6a67d3e45906a7e29c836214f6195a 100644 (file)
--- a/src/detection/detection_continuation.h
+++ b/src/detection/detection_continuation.h
@@ -37,6 +37,7 @@
 #include "detect_trace.h"
 #include "ips_context.h"
 #include "rule_option_types.h"
+#include "treenodes.h"
 
 class Continuation
 {
@@ -257,6 +258,14 @@ bool Continuation::State::eval(snort::Packet& p)
         result = detection_option_node_evaluate(root_node, data, cursor);
     }
 
+    if (data.leaf_reached and !data.otn->sigInfo.file_id)
+    {
+        data.p->context->matched_buffers.emplace_back(cursor.get_name(), cursor.buffer(), cursor.size());
+        debug_logf(detection_trace, TRACE_BUFFER, data.p, "Collecting \"%s\" buffer of size %u on continuation root\n",
+            cursor.get_name(), cursor.size());
+        snort::pc.buf_dumps++;
+    }
+
     clear_trace_cursor_info();
 
     if (result)

diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc
index 9b8ebdabeca49ab65414c33f7deb9820bb894b11..5cb815285833841392945347ab8b09abff1667fe 100644 (file)
--- a/src/detection/detection_options.cc
+++ b/src/detection/detection_options.cc
@@ -685,6 +685,16 @@ int detection_option_node_evaluate(
                     Continuation::recall(state, p);
                 }
 
+                if ( eval_data.leaf_reached and !eval_data.otn->sigInfo.file_id and
+                    node->option_type != RULE_OPTION_TYPE_LEAF_NODE and
+                    ((IpsOption*)node->option_data)->is_buffer_setter() )
+                {
+                    debug_logf(detection_trace, TRACE_BUFFER, p, "Collecting \"%s\" buffer of size %u\n",
+                        cursor.get_name(), cursor.size());
+                    p->context->matched_buffers.emplace_back(cursor.get_name(), cursor.buffer(), cursor.size());
+                    pc.buf_dumps++;
+                }
+
                 // Don't need to reset since it's only checked after we've gone
                 // through the loop at least once and the result will have
                 // been set again already

diff --git a/src/detection/detection_util.h b/src/detection/detection_util.h
index f563e4797dbe11eaa19034428e68874f8f165c58..8cced22bb46bdd056d254b85d40acd8f43674398 100644 (file)
--- a/src/detection/detection_util.h
+++ b/src/detection/detection_util.h
@@ -43,6 +43,18 @@ struct DataBuffer
     unsigned len;
 };
 
+struct MatchedBuffer
+{
+    MatchedBuffer(const char* const n, const uint8_t* const d, unsigned s) :
+        name(n), data(d), size(s)
+    {}
+    MatchedBuffer() = delete;
+
+    const char* const name = nullptr;
+    const uint8_t* const data = nullptr;
+    unsigned size = 0;
+};
+
 // FIXIT-RC event trace should be placed in its own files
 void EventTrace_Init();
 void EventTrace_Term();

diff --git a/src/detection/dev_notes.txt b/src/detection/dev_notes.txt
index fad64db17c6590027737123244d3ae416c34a0f0..c9c0f660e4ad8129d5a02cb9a472a37779504580 100644 (file)
--- a/src/detection/dev_notes.txt
+++ b/src/detection/dev_notes.txt
@@ -269,3 +269,17 @@ But just-created continuations are not evaluated immediately on the same packet,
 the next PDU. Additionally, if an inspector calls for detection on a single data block (like, a full
 attachment in HTTP), continuations can be disabled by providing 'no_flow' flag to file_data buffer or
 any other buffer to indicate that the block is complete and no continuations needed.
+
+
+*Matched Buffers Dumps*
+
+Buffers that have been matched during detection are collected and stored on IPS context
+until logging takes place.
+Amount of collected buffers is represented with "detection.buf_dumps" peg count;
+
+Caveats:
+
+1. Only buffers included in the rules explicitly will be collected.
+
+2. For multiple detection cases, all found instances of a matched buffer will be logged,
+   even ones that may not be related to the rule.

diff --git a/src/detection/ips_context.cc b/src/detection/ips_context.cc
index 6a85b59e209313b82d7e0cc09a44edfcb761d54b..a68cd0af61775b4a2fd971e78e6badcf6a36f6e6 100644 (file)
--- a/src/detection/ips_context.cc
+++ b/src/detection/ips_context.cc
@@ -113,6 +113,7 @@ void IpsContext::clear()
     }
     remove_gadget = false;
     assert(post_callbacks.empty());
+    matched_buffers.clear();
 }
 
 void IpsContext::set_context_data(unsigned id, IpsContextData* cd)

diff --git a/src/detection/ips_context.h b/src/detection/ips_context.h
index 3bc61d6199f1e9eae72c3360a5e2ae2d3fb6240a..0b4f526820ab9111116bb6c7edcc947c007a977e 100644 (file)
--- a/src/detection/ips_context.h
+++ b/src/detection/ips_context.h
@@ -176,6 +176,8 @@ public:
     // Only 5 inspectors currently use the ips context data.
     static constexpr unsigned max_ips_id = 8;
 
+    std::vector<MatchedBuffer> matched_buffers;
+
 private:
     FlowSnapshot flow = {};
     std::vector<IpsContextData*> data;

diff --git a/src/detection/signature.cc b/src/detection/signature.cc
index c50a3243d2b0baba0568569fb212a5b2518ea008..b88a16d2ba1c87c6a1f52f21574ec354ce543252 100644 (file)
--- a/src/detection/signature.cc
+++ b/src/detection/signature.cc
@@ -165,6 +165,7 @@ OptTreeNode::~OptTreeNode()
         snort_free(detection_filter);
 
     delete sigInfo.body;
+    delete[] buffer_setters;
     delete[] state;
 }
 

diff --git a/src/detection/treenodes.h b/src/detection/treenodes.h
index 17f2b3969b8501bb4ce05a40dca7fbb822428ddf..5b733f5520b4044c46dfcf41acffc47fe70f8a37 100644 (file)
--- a/src/detection/treenodes.h
+++ b/src/detection/treenodes.h
@@ -186,6 +186,7 @@ struct OptTreeNode
     OptFpList* opt_func = nullptr;
     OutputSet* outputFuncs = nullptr; /* per sid enabled output functions */
     snort::IpsOption* agent = nullptr;
+    const char** buffer_setters = nullptr;
 
     OptFpList* normal_fp_only = nullptr;
     OptFpList* offload_fp_only = nullptr;

diff --git a/src/events/event.h b/src/events/event.h
index 6d4b0c58243761eed98416214e7313d4a46ee0ce..e569017f28447eec7d1c822a4a2913071372e82e 100644 (file)
--- a/src/events/event.h
+++ b/src/events/event.h
@@ -40,6 +40,7 @@ struct Event
     struct sf_timeval32 ref_time = { 0, 0 };   /* reference time for the event reference */
     const char* alt_msg = nullptr;
     std::string action_string;
+    const char** buffs_to_dump = nullptr;
 
     Event() = default;
     Event(SigInfo& si)

diff --git a/src/loggers/alert_fast.cc b/src/loggers/alert_fast.cc
index 572d280993ea0c9f09acd7c8857543a20f2142de..39b35cab30483c5bc1a48b7eb982888ba0a775d2 100644 (file)
--- a/src/loggers/alert_fast.cc
+++ b/src/loggers/alert_fast.cc
@@ -71,6 +71,12 @@ static const Parameter s_params[] =
     { "packet", Parameter::PT_BOOL, nullptr, "false",
       "output packet dump with alert" },
 
+    { "buffers", Parameter::PT_BOOL, nullptr, "false",
+      "output IPS buffer dump" },
+
+    { "buffers_depth", Parameter::PT_INT, "0:maxSZ", "0",
+      "number of IPS buffer bytes to dump per buffer (0 is unlimited)" },
+
     { "limit", Parameter::PT_INT, "0:maxSZ", "0",
       "set maximum size in MB before rollover (0 is unlimited)" },
 
@@ -93,8 +99,10 @@ public:
 
 public:
     size_t limit = 0;
+    size_t buffers_depth = 0;
     bool file = false;
     bool packet = false;
+    bool buffers = false;
 };
 
 bool FastModule::set(const char*, Value& v, SnortConfig*)
@@ -105,9 +113,15 @@ bool FastModule::set(const char*, Value& v, SnortConfig*)
     else if ( v.is("packet") )
         packet = v.get_bool();
 
+    else if ( v.is("buffers") )
+        buffers = v.get_bool();
+
     else if ( v.is("limit") )
         limit = v.get_size() * 1024 * 1024;
 
+    else if ( v.is("buffers_depth") )
+        buffers_depth = v.get_size();
+
     return true;
 }
 
@@ -116,6 +130,8 @@ bool FastModule::begin(const char*, int, SnortConfig*)
     file = false;
     limit = 0;
     packet = false;
+    buffers = false;
+    buffers_depth = 0;
     return true;
 }
 
@@ -148,6 +164,55 @@ static void ObfuscateLogNetData(TextLog* log, const uint8_t* data, const int len
     LogNetData(log, (const uint8_t*)buf.c_str(), len, p, buf_name, ins_name);
 }
 
+static bool should_dump_buffer(const char* buf_name, const char** buffs_to_dump)
+{
+    if ( !buf_name )
+        return false;
+
+    size_t cmp_idx = 0;
+
+    while ( buffs_to_dump[cmp_idx] )
+        if ( !strcmp(buf_name, buffs_to_dump[cmp_idx++]) )
+            return true;
+
+    return false;
+}
+
+static void log_ips_buffers(Packet* p, const char** buffs_to_dump, unsigned long depth)
+{
+    if ( !buffs_to_dump or !buffs_to_dump[0] )
+        return;
+
+    auto& all_buffs = p->context->matched_buffers;
+    std::vector<MatchedBuffer*> to_dump;
+
+    for ( auto& b : all_buffs )
+        if ( should_dump_buffer(b.name, buffs_to_dump) and to_dump.cend() ==
+            find_if(to_dump.begin(), to_dump.end(), [b](MatchedBuffer*& cmp_b)
+            {
+                bool same_buffers = cmp_b->name == b.name and cmp_b->data == b.data;
+                if ( same_buffers and cmp_b->size < b.size )
+                {
+                    cmp_b->size = b.size;
+                    return true;
+                }
+
+                return same_buffers;
+            }) )
+            to_dump.push_back(&b);
+
+    for ( auto b : to_dump )
+    {
+        const char* buf_name = b->name;
+
+        if ( !buf_name )
+            continue;
+
+        int log_depth = depth && depth < b->size ? depth : b->size;
+        ObfuscateLogNetData(fast_log, b->data, log_depth, p, buf_name, buf_name, "detection");
+    }
+}
+
 using BufferIds = std::vector<unsigned>;
 
 //-------------------------------------------------------------------------
@@ -173,7 +238,9 @@ private:
 private:
     string file;
     unsigned long limit;
+    unsigned long buffers_depth;
     bool packet;
+    bool log_buffers;
 
     static std::vector<unsigned> req_ids;
     static std::vector<unsigned> rsp_ids;
@@ -182,7 +249,8 @@ private:
 std::vector<unsigned> FastLogger::req_ids;
 std::vector<unsigned> FastLogger::rsp_ids;
 
-FastLogger::FastLogger(FastModule* m) : file(m->file ? F_NAME : "stdout"), limit(m->limit), packet(m->packet)
+FastLogger::FastLogger(FastModule* m) : file(m->file ? F_NAME : "stdout"), limit(m->limit),
+    buffers_depth(m->buffers_depth), packet(m->packet), log_buffers(m->buffers)
 { }
 
 //-----------------------------------------------------------------
@@ -320,6 +388,9 @@ void FastLogger::log_data(Packet* p, const Event& event)
 
     if ( buf.len and event.sig_info->gid != 116 )
         LogNetData(fast_log, buf.data, buf.len, p, "alt");
+
+    if ( log_buffers )
+        log_ips_buffers(p, event.buffs_to_dump, buffers_depth);
 }
 
 //-------------------------------------------------------------------------

diff --git a/src/parser/parse_rule.cc b/src/parser/parse_rule.cc
index 7ee3a256b1635b1bcc698759dee82062736a311a..ac55397683750af37fa3b8fc7fe95a76ab49e096 100644 (file)
--- a/src/parser/parse_rule.cc
+++ b/src/parser/parse_rule.cc
@@ -93,6 +93,8 @@ static bool buf_is_set = false;
 static std::string s_type;
 static std::string s_body;
 
+static std::list<const char*> buffer_setters;
+
 static bool action_file_id = false;
 static bool fp_file_id = false;
 static bool only_file_data = true;
@@ -1028,6 +1030,11 @@ void parse_rule_opt_end(SnortConfig* sc, const char* key, OptTreeNode* otn)
             fp_file_id = true;
     }
 
+    // only add unique buffer setters
+    if ( !action_file_id and ips and ips->is_buffer_setter() and
+        std::find(buffer_setters.cbegin(), buffer_setters.cend(), ips->get_name()) == buffer_setters.cend() )
+        buffer_setters.emplace_back(ips->get_name());
+
     if ( type != OPT_TYPE_META )
         otn->num_detection_opts++;
 
@@ -1070,6 +1077,8 @@ OptTreeNode* parse_rule_open(SnortConfig* sc, RuleTreeNode& rtn, bool stub)
     s_body = "(";
     buf_is_set = false;
 
+    buffer_setters.clear();
+
     return otn;
 }
 
@@ -1249,6 +1258,18 @@ void parse_rule_close(SnortConfig* sc, RuleTreeNode& rtn, OptTreeNode* otn)
     if ( otn->sigInfo.message.empty() )
         otn->sigInfo.message = "\"no msg in rule\"";
 
+    if ( !otn->sigInfo.file_id and buffer_setters.size() )
+    {
+        otn->buffer_setters = new const char*[buffer_setters.size() + 1];
+
+        size_t idx = 0;
+        for (auto& setter : buffer_setters)
+        {
+            otn->buffer_setters[idx++] = setter;
+        }
+        otn->buffer_setters[idx] = nullptr;
+    }
+
     OptFpList* fpl = AddOptFuncToList(nullptr, otn);
     fpl->type = RULE_OPTION_TYPE_LEAF_NODE;
 

diff --git a/src/utils/stats.cc b/src/utils/stats.cc
index bd70a05f7501d89fc549833d7340a0cda0091147..69af1e17c1274fb38a6238bf882c5c8bc345ca46 100644 (file)
--- a/src/utils/stats.cc
+++ b/src/utils/stats.cc
@@ -239,6 +239,7 @@ const PegInfo pc_names[] =
     { CountType::MAX, "cont_max_num", "peak number of simultaneous continuations per flow" },
     { CountType::SUM, "cont_match_distance", "total number of bytes jumped over by matched continuations"},
     { CountType::SUM, "cont_mismatch_distance", "total number of bytes jumped over by mismatched continuations"},
+    { CountType::SUM, "buf_dumps", "total number of IPS buffers collected from matched rules" },
     { CountType::END, nullptr, nullptr }
 };
 

diff --git a/src/utils/stats.h b/src/utils/stats.h
index 0f34d9a1fb3d8fc4876edce4f967f8e8ea2236af..3f117c8371660e3e5f1c793de8396e6caa65cca4 100644 (file)
--- a/src/utils/stats.h
+++ b/src/utils/stats.h
@@ -72,6 +72,7 @@ struct PacketCount
     PegCount cont_max_num;
     PegCount cont_match_distance;
     PegCount cont_mismatch_distance;
+    PegCount buf_dumps;
 };
 
 struct ProcessCount