Pull request #4709: extractor: support conn.log orig_bytes, resp_bytes

Merge in SNORT/snort3 from ~MDAGON/snort3:conn_bytes_final to master

Squashed commit of the following:

commit ee59534a98148aaed8a16339ced286afbe3d1e80
Author: maya dagon <mdagon@cisco.com>
Date:   Fri Aug 30 12:54:48 2024 -0400

    extractor: support conn.log orig_bytes, resp_bytes

diff --git a/doc/user/extractor.txt b/doc/user/extractor.txt
index bb866c7e16358cda222cfa61bb4d63bc38b8c280..08f6326a2f6897ea58afec72054a1359ad6b302f 100644 (file)
--- a/doc/user/extractor.txt
+++ b/doc/user/extractor.txt
@@ -138,8 +138,13 @@ Fields supported for connection:
 * `duration` - connection duration in seconds
 * `proto` - transport layer protocol of the connection
 * `service` - connection's application protocol
-* `orig_pkts` - number of packets originator sent
-* `resp_pkts` - number of packets responder sent
+* `orig_pkts` - number of packets client sent
+* `resp_pkts` - number of packets server sent
+* `orig_bytes` - tcp/udp payload bytes client sent
+* `resp_bytes` - tcp/udp payload bytes server sent
+
+For TCP orig_bytes and resp_bytes are calculated using first seen sequence number and next expected sequence number.
+These are reset during TCP flow restart. For this case only bytes seen following the restart will be reported.
 
 ==== Example
 

diff --git a/src/network_inspectors/extractor/extractor_conn.cc b/src/network_inspectors/extractor/extractor_conn.cc
index c9d965ce9319cbf270ced965c0e7483f62cd463d..969d72a942440ad31761c15bbb75f4ddcca1dbf9 100644 (file)
--- a/src/network_inspectors/extractor/extractor_conn.cc
+++ b/src/network_inspectors/extractor/extractor_conn.cc
@@ -28,6 +28,8 @@
 #include "profiler/profiler.h"
 #include "pub_sub/intrinsic_event_ids.h"
 #include "sfip/sf_ip.h"
+#include "stream/tcp/tcp_session.h"
+#include "stream/udp/udp_session.h"
 #include "utils/util.h"
 #include "utils/util_net.h"
 
@@ -52,11 +54,55 @@ static uint64_t get_duration(const DataEvent*, const Flow* f)
     return f->last_data_seen - f->flowstats.start_time.tv_sec;
 }
 
+static uint64_t get_orig_bytes_tcp(const TcpSession* tcpssn)
+{
+    uint32_t client_bytes = tcpssn->client.get_snd_nxt() - tcpssn->client.get_iss();
+    // get_snd_nxt is the next expected sequence number (last seen + 1)
+    if (client_bytes != 0)
+        client_bytes -= 1;
+    return client_bytes; 
+}
+
+static uint64_t get_orig_bytes(const DataEvent*, const Flow* f)
+{
+    if (f->session == nullptr)
+        return 0;
+    if (f->pkt_type == PktType::TCP)
+        return get_orig_bytes_tcp((const TcpSession*)f->session);
+    else if (f->pkt_type == PktType::UDP)
+        return ((UdpSession*)(f->session))->payload_bytes_seen_client;
+
+    return 0;
+}
+
+static uint64_t get_resp_bytes_tcp(const TcpSession* tcpssn)
+{
+    uint32_t server_bytes = tcpssn->server.get_snd_nxt() - tcpssn->server.get_iss();
+    if (server_bytes != 0)
+        server_bytes -= 1;
+    return server_bytes;
+}
+
+static uint64_t get_resp_bytes(const DataEvent*, const Flow* f)
+{
+    if (f->session == nullptr)
+        return 0;
+    
+    if (f->pkt_type == PktType::TCP)
+        return get_resp_bytes_tcp((const TcpSession*)f->session);
+    else if (f->pkt_type == PktType::UDP)
+        return ((UdpSession*)(f->session))->payload_bytes_seen_server;
+
+    return 0;
+}
+
 static const map<string, ExtractorEvent::NumGetFn> sub_num_getters =
 {
     {"orig_pkts", get_orig_pkts},
     {"resp_pkts", get_resp_pkts},
-    {"duration", get_duration}
+    {"duration", get_duration},
+    {"orig_bytes", get_orig_bytes},
+    {"resp_bytes", get_resp_bytes}
 };
 
 static const char* get_service(const DataEvent*, const Flow* f)
@@ -157,4 +203,23 @@ TEST_CASE("Conn Proto", "[extractor]")
     delete flow;
 }
 
+TEST_CASE("Conn payload bytes", "[extractor]")
+{
+    Flow* flow = new Flow;
+    InspectionPolicy ins;
+    set_inspection_policy(&ins);
+    NetworkPolicy net;
+    set_network_policy(&net);
+
+    SECTION("no session")
+    {
+        uint64_t bytes = get_orig_bytes(nullptr, flow);
+        CHECK(bytes == 0);
+        bytes = get_resp_bytes(nullptr, flow);
+        CHECK(bytes == 0);
+    }
+
+    delete flow;
+}
+
 #endif

diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc
index 5401977c6b3ca1726283902857d5fc5739eb552e..b52922a007f91bc37df2aae6a97ba388b7518f0d 100644 (file)
--- a/src/network_inspectors/extractor/extractor_service.cc
+++ b/src/network_inspectors/extractor/extractor_service.cc
@@ -351,7 +351,9 @@ const ServiceBlueprint ConnExtractorService::blueprint =
         "service",
         "orig_pkts",
         "resp_pkts",
-        "duration"
+        "duration",
+        "orig_bytes",
+        "resp_bytes"
     },
 };
 

diff --git a/src/network_inspectors/extractor/extractors.h b/src/network_inspectors/extractor/extractors.h
index 773a63b17498a2c3b65bcee4c059c1e55141f28d..d47de5c7d1b69a8100401e708c62299a0c752f7e 100644 (file)
--- a/src/network_inspectors/extractor/extractors.h
+++ b/src/network_inspectors/extractor/extractors.h
@@ -104,10 +104,10 @@ protected:
     }
 
     static const SfIp& get_ip_src(const DataEvent*, const Flow* flow)
-    { return flow->flags.client_initiated ? flow->client_ip : flow->server_ip; }
+    { return  flow->client_ip; }
 
     static const SfIp& get_ip_dst(const DataEvent*, const Flow* flow)
-    { return flow->flags.client_initiated ? flow->server_ip : flow->client_ip; }
+    { return flow->server_ip; }
 
     static uint64_t get_ip_src_port(const DataEvent*, const Flow* flow)
     { return flow->client_port; }

diff --git a/src/stream/udp/udp_session.cc b/src/stream/udp/udp_session.cc
index 1065071749271214fa666779ea04f2a75f8e670e..5abe5c203225ed66778720ff1f92a67939394fbd 100644 (file)
--- a/src/stream/udp/udp_session.cc
+++ b/src/stream/udp/udp_session.cc
@@ -60,47 +60,6 @@ static void UdpSessionCleanup(Flow* lwssn)
         udpStats.released++;
 }
 
-static int ProcessUdp(Flow* lwssn, Packet* p, StreamUdpConfig*)
-{
-    assert(lwssn->pkt_type == PktType::UDP);
-
-    if ( Stream::blocked_flow(p) )
-        return 0;
-
-    if ( Stream::ignored_flow(lwssn, p) )
-    {
-        udpStats.ignored++;
-        return 0;
-    }
-    udpStats.total_bytes += p->dsize;
-    /* if both seen, mark established */
-    if (p->is_from_server())
-    {
-        lwssn->ssn_state.session_flags |= SSNFLAG_SEEN_RESPONDER;
-        lwssn->set_ttl(p, false);
-    }
-    else
-    {
-        lwssn->ssn_state.session_flags |= SSNFLAG_SEEN_SENDER;
-        lwssn->set_ttl(p, true);
-    }
-
-    if (!(lwssn->ssn_state.session_flags & SSNFLAG_ESTABLISHED))
-    {
-        if ((lwssn->ssn_state.session_flags & SSNFLAG_SEEN_SENDER) &&
-            (lwssn->ssn_state.session_flags & SSNFLAG_SEEN_RESPONDER))
-        {
-            lwssn->ssn_state.session_flags |= SSNFLAG_ESTABLISHED;
-            DataBus::publish(Stream::get_pub_id(), StreamEventIds::UDP_BIDIRECTIONAL, p);
-        }
-    }
-
-    if ( lwssn->clouseau )
-        lwssn->clouseau->eval(p);
-
-    return 0;
-}
-
 //-------------------------------------------------------------------------
 // UdpSession methods
 //-------------------------------------------------------------------------
@@ -168,11 +127,53 @@ void UdpSession::update_direction(
     flow->swap_roles();
 }
 
+int UdpSession::process_udp(Flow* lwssn, Packet* p)
+{
+    assert(lwssn->pkt_type == PktType::UDP);
+
+    if ( Stream::blocked_flow(p) )
+        return 0;
+
+    if ( Stream::ignored_flow(lwssn, p) )
+    {
+        udpStats.ignored++;
+        return 0;
+    }
+    udpStats.total_bytes += p->dsize;
+    /* if both seen, mark established */
+    if (p->is_from_server())
+    {
+        lwssn->ssn_state.session_flags |= SSNFLAG_SEEN_RESPONDER;
+        lwssn->set_ttl(p, false);
+        payload_bytes_seen_server += p->dsize;
+    }
+    else
+    {
+        lwssn->ssn_state.session_flags |= SSNFLAG_SEEN_SENDER;
+        lwssn->set_ttl(p, true);
+        payload_bytes_seen_client += p->dsize;
+    }
+
+    if (!(lwssn->ssn_state.session_flags & SSNFLAG_ESTABLISHED))
+    {
+        if ((lwssn->ssn_state.session_flags & SSNFLAG_SEEN_SENDER) &&
+            (lwssn->ssn_state.session_flags & SSNFLAG_SEEN_RESPONDER))
+        {
+            lwssn->ssn_state.session_flags |= SSNFLAG_ESTABLISHED;
+            DataBus::publish(Stream::get_pub_id(), StreamEventIds::UDP_BIDIRECTIONAL, p);
+        }
+    }
+
+    if ( lwssn->clouseau )
+        lwssn->clouseau->eval(p);
+
+    return 0;
+}
+
 int UdpSession::process(Packet* p)
 {
     Profile profile(udp_perf_stats);    // cppcheck-suppress unreadVariable
 
-    StreamUdpConfig* pc = get_udp_cfg(flow->ssn_server);
     // Check if the session is expired.
     // Should be done before we do something with the packet...
     if ( Stream::expired_flow(flow, p) )
@@ -185,7 +186,7 @@ int UdpSession::process(Packet* p)
         UdpHAManager::process_deletion(*flow);
     }
 
-    ProcessUdp(flow, p, pc);
+    process_udp(flow, p);
     flow->markup_packet_flags(p);
 
     flow->set_expire(p, flow->default_session_timeout);

diff --git a/src/stream/udp/udp_session.h b/src/stream/udp/udp_session.h
index b70875b4ebe89171907a9f5416a07fbeba752891..3884c3acf7532db071e9659debac092fc0ada26c 100644 (file)
--- a/src/stream/udp/udp_session.h
+++ b/src/stream/udp/udp_session.h
@@ -37,6 +37,11 @@ public:
 
 public:
     struct timeval ssn_time = {};
+    uint64_t payload_bytes_seen_client = 0;
+    uint64_t payload_bytes_seen_server = 0;
+
+private:
+    int process_udp(snort::Flow*, snort::Packet*);
 };
 
 #endif