Merge pull request #2038 in SNORT/snort3 from ~MMATIRKO/snort3:improved_flowstats3...

Squashed commit of the following:

commit 8382ed77ac0571ff4601934ae6dec7f1ce26493d
Author: Michael Matirko <mmatirko@cisco.com>
Date:   Thu Feb 6 12:02:12 2020 -0500

    flow: added initiator bytes/packets onto flow

diff --git a/src/flow/flow.cc b/src/flow/flow.cc
index 284bfd6d3c644d54b5799b4379dc2038a51509f7..be3815b6e4bcfd69ef4c2874cc0606025e104556 100644 (file)
--- a/src/flow/flow.cc
+++ b/src/flow/flow.cc
@@ -534,3 +534,12 @@ void Flow::set_service(Packet* pkt, const char* new_service)
     service = new_service;
     DataBus::publish(FLOW_SERVICE_CHANGE_EVENT, pkt);
 }
+
+void Flow::swap_roles()
+{
+    client_initiated = !client_initiated;
+    std::swap(client_ip, server_ip);
+    std::swap(client_port, server_port);
+    std::swap(flowstats.client_pkts, flowstats.server_pkts);
+    std::swap(flowstats.client_bytes, flowstats.server_bytes);
+}

diff --git a/src/flow/flow.h b/src/flow/flow.h
index 7dd30f7f7169e2fdab3b9156f7cf2a05b0a888c8..f5299c2c1380af4997894e79d53c37dbee07eb0b 100644 (file)
--- a/src/flow/flow.h
+++ b/src/flow/flow.h
@@ -142,6 +142,15 @@ private:
     unsigned id;
 };
 
+struct FlowStats
+{
+    uint64_t client_pkts;
+    uint64_t server_pkts;
+    uint64_t client_bytes;
+    uint64_t server_bytes;
+    struct timeval start_time;
+};
+
 struct LwState
 {
     uint32_t session_flags;
@@ -200,6 +209,7 @@ public:
     void set_ttl(Packet*, bool client);
     void set_mpls_layer_per_dir(Packet*);
     Layer get_mpls_layer_per_dir(bool);
+    void swap_roles();
     void set_service(Packet* pkt, const char* new_service);
     bool get_attr(const std::string& key, int32_t& val);
     bool get_attr(const std::string& key, std::string& val);
@@ -395,6 +405,14 @@ public:  // FIXIT-M privatize if possible
     // everything from here down is zeroed
     IpsContextChain context_chain;
     FlowData* flow_data;
+    FlowStats flowstats;
+
+    SfIp client_ip;
+    SfIp server_ip;
+
+    LwState ssn_state;
+    LwState previous_ssn_state;
+
     Inspector* clouseau;  // service identifier
     Inspector* gadget;    // service handler
     Inspector* assistant_gadget;
@@ -403,12 +421,8 @@ public:  // FIXIT-M privatize if possible
 
     uint64_t expire_time;
 
-    SfIp client_ip;
-    SfIp server_ip;
+    DeferredWhitelist deferred_whitelist = WHITELIST_DEFER_OFF;
 
-    LwState ssn_state;
-    LwState previous_ssn_state;
-    FlowState flow_state;
     unsigned inspection_policy_id;
     unsigned ips_policy_id;
     unsigned network_policy_id;
@@ -429,8 +443,9 @@ public:  // FIXIT-M privatize if possible
 
     bool disable_inspect;
     bool trigger_finalize_event;
+    bool client_initiated;
 
-    DeferredWhitelist deferred_whitelist = WHITELIST_DEFER_OFF;
+    FlowState flow_state;
 
 private:
     void clean();

diff --git a/src/flow/flow_control.cc b/src/flow/flow_control.cc
index 2df1b4c1195db8f00d9a3824d88f36ac736578ce..c6be9bd52eac20af28696166d06404eabe779909 100644 (file)
--- a/src/flow/flow_control.cc
+++ b/src/flow/flow_control.cc
@@ -419,6 +419,8 @@ unsigned FlowControl::process(Flow* flow, Packet* p)
             flow->set_state(Flow::FlowState::ALLOW);
 
         ++news;
+        flow->flowstats.start_time = p->pkth->ts;
+        flow->client_initiated = p->is_from_client();
     }
 
     // This requires the packet direction to be set
@@ -465,9 +467,24 @@ unsigned FlowControl::process(Flow* flow, Packet* p)
         break;
     }
 
+    update_stats(flow, p);
     return news;
 }
 
+void FlowControl::update_stats(Flow* flow, Packet* p)
+{
+    if (p->is_from_client())
+    {
+        flow->flowstats.client_pkts++;
+        flow->flowstats.client_bytes += p->pktlen;
+    }
+    else
+    {
+        flow->flowstats.server_pkts++;
+        flow->flowstats.server_bytes += p->pktlen;
+    }
+}
+
 //-------------------------------------------------------------------------
 // expected
 //-------------------------------------------------------------------------

diff --git a/src/flow/flow_control.h b/src/flow/flow_control.h
index ded6be0fc16145cc567f1a8215c0a5e36a50ce74..e446aa8338730f1fe8e3311dfd781e5f453fcf6b 100644 (file)
--- a/src/flow/flow_control.h
+++ b/src/flow/flow_control.h
@@ -99,6 +99,7 @@ private:
     void set_key(snort::FlowKey*, snort::Packet*);
     unsigned process(snort::Flow*, snort::Packet*);
     void preemptive_cleanup();
+    void update_stats(snort::Flow*, snort::Packet*);
 
 private:
     snort::InspectSsnFunc get_proto_session[to_utype(PktType::MAX)] = {};

diff --git a/src/loggers/alert_csv.cc b/src/loggers/alert_csv.cc
index 516eb54d395ca26f16f7b671a30d37abf80367f2..51c25ff90dec06e5c35d5daddf09b51faddca199 100644 (file)
--- a/src/loggers/alert_csv.cc
+++ b/src/loggers/alert_csv.cc
@@ -107,6 +107,18 @@ static void ff_b64_data(const Args& a)
     TextLog_Putc(csv_log, '"');
 }
 
+static void ff_client_bytes(const Args& a)
+{
+    if (a.pkt->flow)
+        TextLog_Print(csv_log, "%" PRIu64, a.pkt->flow->flowstats.client_bytes);
+}
+
+static void ff_client_pkts(const Args& a)
+{
+    if (a.pkt->flow)
+        TextLog_Print(csv_log, "%" PRIu64, a.pkt->flow->flowstats.client_pkts);
+}
+
 static void ff_dir(const Args& a)
 {
     const char* dir;
@@ -191,6 +203,12 @@ static void ff_eth_type(const Args& a)
     TextLog_Print(csv_log, "0x%X", ntohs(eh->ether_type));
 }
 
+static void ff_flowstart_time(const Args& a)
+{
+    if (a.pkt->flow)
+        TextLog_Print(csv_log, "%u", a.pkt->flow->flowstats.start_time.tv_sec);
+}
+
 static void ff_gid(const Args& a)
 {
     TextLog_Print(csv_log, "%u",  a.event.sig_info->gid);
@@ -302,6 +320,18 @@ static void ff_seconds(const Args& a)
     TextLog_Print(csv_log, "%u",  a.pkt->pkth->ts.tv_sec);
 }
 
+static void ff_server_bytes(const Args& a)
+{
+    if (a.pkt->flow)
+        TextLog_Print(csv_log, "%" PRIu64, a.pkt->flow->flowstats.server_bytes);
+}
+
+static void ff_server_pkts(const Args& a)
+{
+    if (a.pkt->flow)
+        TextLog_Print(csv_log, "%" PRIu64, a.pkt->flow->flowstats.server_pkts);
+}
+
 static void ff_service(const Args& a)
 {
     const char* svc = "unknown";
@@ -430,23 +460,25 @@ typedef void (*CsvFunc)(const Args&);
 
 static const CsvFunc csv_func[] =
 {
-    ff_action, ff_class, ff_b64_data, ff_dir, ff_dst_addr, ff_dst_ap,
-    ff_dst_port, ff_eth_dst, ff_eth_len, ff_eth_src, ff_eth_type, ff_gid,
-    ff_icmp_code, ff_icmp_id, ff_icmp_seq, ff_icmp_type, ff_iface, ff_ip_id,
-    ff_ip_len, ff_msg, ff_mpls, ff_pkt_gen, ff_pkt_len, ff_pkt_num, ff_priority,
-    ff_proto, ff_rev, ff_rule, ff_seconds, ff_service, ff_sid, ff_src_addr, ff_src_ap,
-    ff_src_port, ff_target, ff_tcp_ack, ff_tcp_flags, ff_tcp_len, ff_tcp_seq,
-    ff_tcp_win, ff_timestamp, ff_tos, ff_ttl, ff_udp_len, ff_vlan
+    ff_action, ff_class, ff_b64_data, ff_client_bytes, ff_client_pkts, ff_dir,
+    ff_dst_addr, ff_dst_ap, ff_dst_port, ff_eth_dst, ff_eth_len, ff_eth_src,
+    ff_eth_type, ff_flowstart_time, ff_gid, ff_icmp_code, ff_icmp_id, ff_icmp_seq,
+    ff_icmp_type, ff_iface, ff_ip_id, ff_ip_len, ff_msg, ff_mpls, ff_pkt_gen, ff_pkt_len,
+    ff_pkt_num, ff_priority, ff_proto, ff_rev, ff_rule, ff_seconds, ff_server_bytes,
+    ff_server_pkts, ff_service, ff_sid, ff_src_addr, ff_src_ap, ff_src_port, ff_target,
+    ff_tcp_ack, ff_tcp_flags,ff_tcp_len, ff_tcp_seq, ff_tcp_win, ff_timestamp, ff_tos,
+    ff_ttl, ff_udp_len, ff_vlan
 };
 
 #define csv_range \
-    "action | class | b64_data | dir | dst_addr | dst_ap | " \
-    "dst_port | eth_dst | eth_len | eth_src | eth_type | gid | " \
-    "icmp_code | icmp_id | icmp_seq | icmp_type | iface | ip_id | " \
-    "ip_len | msg | mpls | pkt_gen | pkt_len | pkt_num | priority | " \
-    "proto | rev | rule | seconds | service | sid | src_addr | src_ap | " \
-    "src_port | target | tcp_ack | tcp_flags | tcp_len | tcp_seq | " \
-    "tcp_win | timestamp | tos | ttl | udp_len | vlan"
+    "action | class | b64_data | client_bytes | client_pkts | dir | dst_addr | " \
+    "dst_ap | dst_port | eth_dst | eth_len | eth_src | eth_type | flowstart_time | "\
+    "gid | icmp_code | icmp_id | icmp_seq | icmp_type | iface | "\
+    "ip_id | ip_len | msg | mpls | pkt_gen | pkt_len | " \
+    "pkt_num | priority | proto | rev | rule | seconds | server_bytes | " \
+    "server_pkts | service | sid | src_addr | src_ap | src_port | target | " \
+    "tcp_ack | tcp_flags | tcp_len | tcp_seq | tcp_win | timestamp | " \
+    "tos | ttl | udp_len | vlan"
 
 #define csv_deflt \
     "timestamp pkt_num proto pkt_gen pkt_len dir src_ap dst_ap rule action"

diff --git a/src/loggers/alert_json.cc b/src/loggers/alert_json.cc
index 30a5ec665dccbf335daa02bc65039bc0e6874510..812e17b1a8f380a9b4041ee18e8eb272e406546c 100644 (file)
--- a/src/loggers/alert_json.cc
+++ b/src/loggers/alert_json.cc
@@ -128,6 +128,28 @@ static bool ff_b64_data(const Args& a)
     return true;
 }
 
+static bool ff_client_bytes(const Args& a)
+{
+    if (a.pkt->flow)
+    {
+        print_label(a, "client_bytes");
+        TextLog_Print(json_log, "%" PRIu64, a.pkt->flow->flowstats.client_bytes);
+        return true;
+    }
+    return false;
+}
+
+static bool ff_client_pkts(const Args& a)
+{
+    if (a.pkt->flow)
+    {
+        print_label(a, "client_pkts");
+        TextLog_Print(json_log, "%" PRIu64, a.pkt->flow->flowstats.client_pkts);
+        return true;
+    }
+    return false;
+}
+
 static bool ff_dir(const Args& a)
 {
     const char* dir;
@@ -234,6 +256,17 @@ static bool ff_eth_type(const Args& a)
     return true;
 }
 
+static bool ff_flowstart_time(const Args& a)
+{
+    if (a.pkt->flow)
+    {
+        print_label(a, "flowstart_time");
+        TextLog_Print(json_log, "%u", a.pkt->flow->flowstats.start_time.tv_sec);
+        return true;
+    }
+    return false;
+}
+
 static bool ff_gid(const Args& a)
 {
     print_label(a, "gid");
@@ -403,6 +436,28 @@ static bool ff_seconds(const Args& a)
     return true;
 }
 
+static bool ff_server_bytes(const Args& a)
+{
+    if (a.pkt->flow)
+    {
+        print_label(a, "server_bytes");
+        TextLog_Print(json_log, "%" PRIu64, a.pkt->flow->flowstats.server_bytes);
+        return true;
+    }
+    return false;
+}
+
+static bool ff_server_pkts(const Args& a)
+{
+    if (a.pkt->flow)
+    {
+        print_label(a, "server_pkts");
+        TextLog_Print(json_log, "%" PRIu64, a.pkt->flow->flowstats.server_pkts);
+        return true;
+    }
+    return false;
+}
+
 static bool ff_service(const Args& a)
 {
     const char* svc = "unknown";
@@ -594,23 +649,24 @@ typedef bool (*JsonFunc)(const Args&);
 
 static const JsonFunc json_func[] =
 {
-    ff_action, ff_class, ff_b64_data, ff_dir, ff_dst_addr, ff_dst_ap,
-    ff_dst_port, ff_eth_dst, ff_eth_len, ff_eth_src, ff_eth_type, ff_gid,
-    ff_icmp_code, ff_icmp_id, ff_icmp_seq, ff_icmp_type, ff_iface, ff_ip_id,
-    ff_ip_len, ff_msg, ff_mpls, ff_pkt_gen, ff_pkt_len, ff_pkt_num, ff_priority,
-    ff_proto, ff_rev, ff_rule, ff_seconds, ff_service, ff_sid, ff_src_addr, ff_src_ap,
-    ff_src_port, ff_target, ff_tcp_ack, ff_tcp_flags, ff_tcp_len, ff_tcp_seq,
-    ff_tcp_win, ff_timestamp, ff_tos, ff_ttl, ff_udp_len, ff_vlan
+    ff_action, ff_class, ff_b64_data, ff_client_bytes, ff_client_pkts, ff_dir,
+    ff_dst_addr, ff_dst_ap, ff_dst_port, ff_eth_dst, ff_eth_len, ff_eth_src, ff_eth_type,
+    ff_flowstart_time, ff_gid, ff_icmp_code, ff_icmp_id, ff_icmp_seq, ff_icmp_type,
+    ff_iface, ff_ip_id, ff_ip_len, ff_msg, ff_mpls, ff_pkt_gen, ff_pkt_len,
+    ff_pkt_num, ff_priority, ff_proto, ff_rev, ff_rule, ff_seconds, ff_server_bytes,
+    ff_server_pkts, ff_service, ff_sid, ff_src_addr, ff_src_ap, ff_src_port, ff_target,
+    ff_tcp_ack, ff_tcp_flags, ff_tcp_len, ff_tcp_seq, ff_tcp_win, ff_timestamp, ff_tos,
+    ff_ttl, ff_udp_len, ff_vlan
 };
 
 #define json_range \
-    "action | class | b64_data | dir | dst_addr | dst_ap | " \
-    "dst_port | eth_dst | eth_len | eth_src | eth_type | gid | " \
-    "icmp_code | icmp_id | icmp_seq | icmp_type | iface | ip_id | " \
-    "ip_len | msg | mpls | pkt_gen | pkt_len | pkt_num | priority | " \
-    "proto | rev | rule | seconds | service | sid | src_addr | src_ap | " \
-    "src_port | target | tcp_ack | tcp_flags | tcp_len | tcp_seq | " \
-    "tcp_win | timestamp | tos | ttl | udp_len | vlan"
+    "action | class | b64_data | client_bytes | client_pkts | dir | dst_addr | dst_ap | " \
+    "dst_port | eth_dst | eth_len | eth_src | eth_type | flowstart_time | " \
+    "gid | icmp_code | icmp_id | icmp_seq | icmp_type | iface | ip_id | ip_len | " \
+    "msg | mpls | pkt_gen | pkt_len | pkt_num | priority | proto | rev | rule | " \
+    "seconds | server_bytes | server_pkts | service | sid | src_addr | src_ap | src_port | target | " \
+    "tcp_ack | tcp_flags | tcp_len | tcp_seq | tcp_win | timestamp | " \
+    "tos | ttl | udp_len | vlan"
 
 #define json_deflt \
     "timestamp pkt_num proto pkt_gen pkt_len dir src_ap dst_ap rule action"

diff --git a/src/stream/udp/udp_session.cc b/src/stream/udp/udp_session.cc
index b8f8314ff87247ba37f7a9b4ab6663905807bd6d..05a8d33c7ff2c8fcf4c2e7840d9d5a3fff64e776 100644 (file)
--- a/src/stream/udp/udp_session.cc
+++ b/src/stream/udp/udp_session.cc
@@ -144,9 +144,6 @@ void UdpSession::clear()
 void UdpSession::update_direction(
     char dir, const SfIp* ip, uint16_t port)
 {
-    SfIp tmpIp;
-    uint16_t tmpPort;
-
     if (flow->client_ip.equals(*ip) && (flow->client_port == port))
     {
         if ((dir == SSN_DIR_FROM_CLIENT) && (flow->ssn_state.direction == FROM_CLIENT))
@@ -164,13 +161,8 @@ void UdpSession::update_direction(
         }
     }
 
-    // Swap them -- leave flow->ssn_state.direction the same
-    tmpIp = flow->client_ip;
-    tmpPort = flow->client_port;
-    flow->client_ip = flow->server_ip;
-    flow->client_port = flow->server_port;
-    flow->server_ip = tmpIp;
-    flow->server_port = tmpPort;
+    // Swap client/server ip, ports, and stats -- leave flow->ssn_state.direction the same
+    flow->swap_roles();
 }
 
 int UdpSession::process(Packet* p)

diff --git a/src/stream/user/user_session.cc b/src/stream/user/user_session.cc
index 89eadb95f414081adb1e88e478582aa18e852ffd..b7ba6ea495003f99bdce4345c08a9db154ef80ab 100644 (file)
--- a/src/stream/user/user_session.cc
+++ b/src/stream/user/user_session.cc
@@ -333,14 +333,7 @@ void UserSession::start(Packet* p, Flow* f)
         if ( (f->ssn_state.session_flags & SSNFLAG_CLIENT_SWAP) &&
             !(f->ssn_state.session_flags & SSNFLAG_CLIENT_SWAPPED) )
         {
-            SfIp ip = f->client_ip;
-            uint16_t port = f->client_port;
-
-            f->client_ip = f->server_ip;
-            f->server_ip = ip;
-
-            f->client_port = f->server_port;
-            f->server_port = port;
+            f->swap_roles();
 
             if ( !f->two_way_traffic() )
             {