Merge pull request #941 in SNORT/snort3 from pkt_trace to master

Squashed commit of the following:

commit dfec01c9671e3d12df4d89b4fd45c35f5cd3834a
Author: Steven Baigal <sbaigal@cisco.com>
Date:   Thu Jun 29 17:21:10 2017 -0400

    add packet trace feature

diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc
index eb50227378260fc9f8e655de98a77b90c97e5a65..b69f9a7f1003b97a78f5b526b98706eeb1952c44 100644 (file)
--- a/src/detection/fp_detect.cc
+++ b/src/detection/fp_detect.cc
@@ -49,6 +49,7 @@
 #include "latency/packet_latency.h"
 #include "latency/rule_latency.h"
 #include "log/messages.h"
+#include "log/packet_tracer.h"
 #include "main/snort.h"
 #include "main/snort_config.h"
 #include "main/snort_debug.h"
@@ -112,6 +113,10 @@ static inline void fpLogOther(
     if ( EventTrace_IsEnabled() )
         EventTrace_Log(p, otn, action);
 
+    PacketTracer::log("Event: %d:%d:%d, Action %s\n",
+        otn->sigInfo.gid, otn->sigInfo.sid, otn->sigInfo.rev,
+        get_action_string((RuleType)action));
+
     // rule option actions are queued here (eg replace)
     otn_trigger_actions(otn, p);
 

diff --git a/src/log/CMakeLists.txt b/src/log/CMakeLists.txt
index 066cb2ca6487fe91a164d70c0e100165c8067c13..969defee2978eb7c328c88a318d77aed157e3e3a 100644 (file)
--- a/src/log/CMakeLists.txt
+++ b/src/log/CMakeLists.txt
@@ -14,6 +14,8 @@ add_library ( log STATIC
     messages.cc
     obfuscator.cc
     obfuscator.h
+    packet_tracer.cc
+    packet_tracer.h
     text_log.cc
 )
 

diff --git a/src/log/Makefile.am b/src/log/Makefile.am
index 4f7a27a30f54980fa53a1a7a4e253ba32a7fd03e..7fa028aeca543b33a58f4244e7e06bc29282e1f3 100644 (file)
--- a/src/log/Makefile.am
+++ b/src/log/Makefile.am
@@ -16,6 +16,8 @@ log_text.h \
 messages.cc \
 obfuscator.cc \
 obfuscator.h \
+packet_tracer.cc \
+packet_tracer.h \
 text_log.cc
 
 if ENABLE_UNIT_TESTS

diff --git a/src/log/packet_tracer.cc b/src/log/packet_tracer.cc
new file mode 100644 (file)
index 0000000..aa9110e
--- /dev/null
+++ b/src/log/packet_tracer.cc
@@ -0,0 +1,216 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2017-2017 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// pkt_tracer.cc author Steven Baigal <sbaigal@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "packet_tracer.h"
+
+#include <cstdarg>
+
+#include "log.h"
+#include "protocols/eth.h"
+#include "protocols/ip.h"
+#include "protocols/packet.h"
+#include "protocols/tcp.h"
+
+#ifdef UNIT_TEST
+#include "catch/catch.hpp"
+#endif
+
+static THREAD_LOCAL PacketTracer* s_pkt_trace = nullptr;
+
+void PacketTracer::thread_init()
+{
+    if (s_pkt_trace == nullptr)
+    {
+        s_pkt_trace = new PacketTracer();
+    }
+}
+
+void PacketTracer::thread_term()
+{
+    if (s_pkt_trace)
+    {
+        delete s_pkt_trace;
+        s_pkt_trace = nullptr;
+    }
+}
+
+void PacketTracer::dump(char* output_buff, unsigned int len)
+{
+    if (!s_pkt_trace)
+        return;
+    if (output_buff)
+    {
+        memcpy(output_buff, s_pkt_trace->buffer,
+            (len < s_pkt_trace->buff_len + 1 ? len : s_pkt_trace->buff_len + 1));
+    }
+    else
+        printf("%s\n", s_pkt_trace->buffer);
+
+    s_pkt_trace->buff_len = 0;
+    s_pkt_trace->buffer[0] = '\0';
+}
+
+void PacketTracer::log(const char* format, ...)
+{
+    if (!s_pkt_trace)
+        return;
+
+    va_list ap;
+
+    va_start(ap, format);
+    const int buff_space = max_buff_size - s_pkt_trace->buff_len;
+    const int len = vsnprintf(s_pkt_trace->buffer + s_pkt_trace->buff_len,
+            buff_space, format, ap);
+    va_end(ap);
+
+    if (len >= 0 and len < buff_space)
+    {
+        s_pkt_trace->buff_len += len;
+    }
+    else
+    {
+        s_pkt_trace->buff_len = max_buff_size - 1;
+    }
+}
+
+void PacketTracer::add_header_info(Packet* p)
+{
+    if (!s_pkt_trace)
+        return;
+
+    if ( auto eh = layer::get_eth_layer(p) )
+    {
+        // MAC layer
+        log("%02X:%02X:%02X:%02X:%02X:%02X - ", eh->ether_src[0],
+            eh->ether_src[1], eh->ether_src[2], eh->ether_src[3],
+            eh->ether_src[4], eh->ether_src[5]);
+        log("%02X:%02X:%02X:%02X:%02X:%02X ", eh->ether_dst[0],
+            eh->ether_dst[1], eh->ether_dst[2], eh->ether_dst[3],
+            eh->ether_dst[4], eh->ether_dst[5]);
+        // protocol and pkt size
+        log("%04X\n", (uint16_t)eh->ethertype());
+    }
+
+    if (p->ptrs.ip_api.get_src() and p->ptrs.ip_api.get_src())
+    {
+        char sipstr[INET6_ADDRSTRLEN], dipstr[INET6_ADDRSTRLEN];
+
+        p->ptrs.ip_api.get_src()->ntop(sipstr, sizeof(sipstr));
+        p->ptrs.ip_api.get_dst()->ntop(dipstr, sizeof(dipstr));
+
+        log("%s-%u - %s-%u %u\n",
+            sipstr, p->ptrs.sp, dipstr, p->ptrs.dp, (unsigned)p->ptrs.ip_api.proto());
+        log("Packet: %s", p->get_type());
+        if (p->type() == PktType::TCP)
+        {
+            char tcpFlags[10];
+            CreateTCPFlagString(p->ptrs.tcph, tcpFlags);
+            log( " %s, seq %u, ack %u", tcpFlags,
+                p->ptrs.tcph->seq(), p->ptrs.tcph->ack());
+        }
+        log("\n");
+    }
+}
+
+#ifdef UNIT_TEST
+
+char* PacketTracer::get_buff()
+{
+    return this->buffer;
+}
+
+int PacketTracer::get_buff_len()
+{
+    return this->buff_len;
+}
+
+#define MAX_PKT_TRACE_BUFF_SIZE 2048
+
+TEST_CASE("basic log", "[PacketTracer]")
+{
+    char test_str[] = "1234567890";
+    // instantiate a packet tracer
+    PacketTracer::thread_init();
+
+    // basic logging
+    PacketTracer::log("%s", test_str);
+    CHECK((memcmp(s_pkt_trace->get_buff(), test_str, 10) == 0));
+    CHECK((s_pkt_trace->get_buff_len() == 10));
+    // continue log will add message to the buffer
+    PacketTracer::log("%s", "ABCDEFG");
+    CHECK((strcmp(s_pkt_trace->get_buff(), "1234567890ABCDEFG") == 0));
+    CHECK((s_pkt_trace->get_buff_len() == (int)strlen(s_pkt_trace->get_buff())));
+    // log empty string won't change existed buffer
+    int curr_len = s_pkt_trace->get_buff_len();
+    char empty_str[] = "";
+    PacketTracer::log("%s", empty_str);
+    CHECK((s_pkt_trace->get_buff_len() == curr_len));
+
+    PacketTracer::thread_term();
+}
+
+TEST_CASE("corner cases", "[PacketTracer]")
+{
+    char test_str[] = "1234567890", empty_str[] = "";
+    PacketTracer::thread_init();
+
+    // init length check
+    CHECK((s_pkt_trace->get_buff_len() == 0));
+    // logging empty string to start with
+    PacketTracer::log("%s", empty_str);
+    CHECK((s_pkt_trace->get_buff_len() == 0));
+
+    // log messages larger than buffer size
+    for(int i=0; i<1024; i++)
+        PacketTracer::log("%s", test_str);
+    // when buffer limit is  reached, buffer length will stoppped at max_buff_size-1
+    CHECK((s_pkt_trace->get_buff_len() == (MAX_PKT_TRACE_BUFF_SIZE-1)));
+
+    // continue logging will not change anything
+    PacketTracer::log("%s", test_str);
+    CHECK((s_pkt_trace->get_buff_len() == (MAX_PKT_TRACE_BUFF_SIZE-1)));
+
+    PacketTracer::thread_term();
+}
+
+TEST_CASE("dump", "[PacketTracer]")
+{
+    char test_string[MAX_PKT_TRACE_BUFF_SIZE];
+    char test_str[] = "ABCD", results[] = "ABCD3=400";
+
+    PacketTracer::thread_init();
+
+    PacketTracer::log("%s%d=%d", test_str, 3, 400);
+    PacketTracer::dump(test_string, MAX_PKT_TRACE_BUFF_SIZE);
+    CHECK(!strcmp(test_string, results));
+    CHECK((s_pkt_trace->get_buff_len() == 0));
+
+    // dump again
+    PacketTracer::dump(test_string, MAX_PKT_TRACE_BUFF_SIZE);
+    CHECK(!strcmp(test_string, ""));
+    CHECK((s_pkt_trace->get_buff_len() == 0));
+
+    PacketTracer::thread_term();
+}
+#endif

diff --git a/src/log/packet_tracer.h b/src/log/packet_tracer.h
new file mode 100644 (file)
index 0000000..25c4d70
--- /dev/null
+++ b/src/log/packet_tracer.h
@@ -0,0 +1,44 @@
+//--------------------------------------------------------------------------
+// Copyright (C) 2017-2017 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation.  You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+//--------------------------------------------------------------------------
+
+// packet_tracer.h author Steven Baigal <sbaigal@cisco.com>
+
+#ifndef PACKET_TRACER_H
+#define PACKET_TRACER_H
+
+struct Packet;
+
+class PacketTracer
+{
+public:
+    static void thread_init();
+    static void thread_term();
+    static void log(const char* format, ...) __attribute__((format (printf, 1, 2)));
+    static void dump(char* output_buff=nullptr, unsigned int len=0);
+    static void add_header_info(Packet* p);
+#ifdef UNIT_TEST
+    char* get_buff();
+    int get_buff_len();
+#endif
+private:
+    static const int max_buff_size = 2048;
+    char buffer[max_buff_size];
+    unsigned int buff_len = 0;
+};
+
+#endif

diff --git a/src/main/modules.cc b/src/main/modules.cc
index 7fc945697981ad66eafd8087ef11f32ae7ae2190..6e0823cab108b4b64305ef3d911f20577df9a717 100644 (file)
--- a/src/main/modules.cc
+++ b/src/main/modules.cc
@@ -37,6 +37,7 @@
 #include "host_tracker/host_cache_module.h"
 #include "latency/latency_module.h"
 #include "log/messages.h"
+#include "log/packet_tracer.h"
 #include "managers/module_manager.h"
 #include "managers/plugin_manager.h"
 #include "memory/memory_module.h"
@@ -748,6 +749,9 @@ static const Parameter output_params[] =
 #endif
       "output 20 bytes per lines instead of 16 when dumping buffers" },
 
+    { "enable_packet_trace", Parameter::PT_BOOL, nullptr, "false",
+      "enable summary output of state that determined packet verdict" },
+
     { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr }
 };
 
@@ -772,6 +776,9 @@ bool OutputModule::set(const char*, Value& v, SnortConfig* sc)
     else if ( v.is("dump_payload_verbose") )
         v.update_mask(sc->output_flags, OUTPUT_FLAG__VERBOSE_DUMP);
 
+    else if ( v.is("enable_packet_trace") )
+        sc->enable_packet_trace = v.get_bool();
+
     else if ( v.is("log_ipv6_extra_data") )
         // FIXIT-M move to output|logging_flags
         sc->log_ipv6_extra = v.get_bool() ? 0 : 1;

diff --git a/src/main/snort.cc b/src/main/snort.cc
index d9cb48a71f04356b092157fda48d6f2abe67d09b..ebf4c684ec98da21c77ca099b30dc9dcdba215b0 100644 (file)
--- a/src/main/snort.cc
+++ b/src/main/snort.cc
@@ -53,6 +53,7 @@
 #include "latency/rule_latency.h"
 #include "log/log.h"
 #include "log/messages.h"
+#include "log/packet_tracer.h"
 #include "loggers/loggers.h"
 #include "main.h"
 #include "managers/action_manager.h"
@@ -720,6 +721,8 @@ void Snort::thread_init_unprivileged()
     SideChannelManager::thread_init();
     HighAvailabilityManager::thread_init(); // must be before InspectorManager::thread_init();
     InspectorManager::thread_init(snort_conf);
+    if (SnortConfig::packet_trace_enabled())
+        PacketTracer::thread_init();
 
     // in case there are HA messages waiting, process them first
     HighAvailabilityManager::process_receive();
@@ -762,6 +765,7 @@ void Snort::thread_term()
     EventTrace_Term();
     CleanupTag();
     FileService::thread_term();
+    PacketTracer::thread_term();
 
     Active::term();
     delete s_switcher;
@@ -785,6 +789,8 @@ DAQ_Verdict Snort::process_packet(
     PacketManager::decode(p, pkthdr, pkt);
     assert(p->pkth && p->pkt);
 
+    PacketTracer::add_header_info(p);
+
     if (is_frag)
     {
         p->packet_flags |= (PKT_PSEUDO | PKT_REBUILT_FRAG);
@@ -896,6 +902,12 @@ DAQ_Verdict Snort::packet_callback(
     int inject = 0;
     verdict = update_verdict(verdict, inject);
 
+    PacketTracer::log("NAP id %d, IPS id %d, Verdict %s\n",
+        get_network_policy()->policy_id, get_ips_policy()->policy_id,
+        SFDAQ::verdict_to_string(verdict));
+
+    PacketTracer::dump();
+
     // FIXIT-H move this to the appropriate struct
     //perfBase->UpdateWireStats(pkthdr->caplen, Active::packet_was_dropped(), inject);
     HighAvailabilityManager::process_update(s_packet->flow, pkthdr);

diff --git a/src/main/snort_config.h b/src/main/snort_config.h
index d5ef0ededd4c94f462c103d06d34b8d5f615f185..88adb16d6ab9d72b1f073f22d44478e4d05700b5 100644 (file)
--- a/src/main/snort_config.h
+++ b/src/main/snort_config.h
@@ -172,6 +172,7 @@ public:
     uint8_t log_ipv6_extra = 0;
     uint16_t event_trace_max = 0;
     long int tagged_packet_limit = 256;
+    bool enable_packet_trace = false;
 
     std::string log_dir;
 
@@ -616,6 +617,11 @@ public:
         return snort_conf->user_id != -1 || snort_conf->group_id != -1 ||
             !snort_conf->chroot_dir.empty();
     }
+
+    static bool packet_trace_enabled()
+    {
+        return snort_conf->enable_packet_trace;
+    }
 };
 
 #endif

diff --git a/src/network_inspectors/appid/appid_inspector.cc b/src/network_inspectors/appid/appid_inspector.cc
index 562dbf4c4888cf313ef0eabc98bf58ae6ec6a60c..93a09c5bee337a67093362d769efa6b076d69013 100644 (file)
--- a/src/network_inspectors/appid/appid_inspector.cc
+++ b/src/network_inspectors/appid/appid_inspector.cc
@@ -44,6 +44,7 @@
 #include "detector_plugins/detector_sip.h"
 #include "detector_plugins/detector_pattern.h"
 #include "log/messages.h"
+#include "log/packet_tracer.h"
 #include "main/snort_config.h"
 #include "managers/inspector_manager.h"
 #include "protocols/packet.h"
@@ -59,6 +60,27 @@ static void openssl_cleanup()
     CRYPTO_cleanup_all_ex_data();
 }
 
+static void add_appid_to_packet_trace(Flow* flow)
+{
+    AppIdSession* session = appid_api.get_appid_session(flow);
+    if (session)
+    {
+        AppId service_id, client_id, payload_id, misc_id;
+        const char *service_app_name, *client_app_name, *payload_app_name, *misc_name;
+        session->get_application_ids(service_id, client_id, payload_id, misc_id);
+        service_app_name = appid_api.get_application_name(service_id);
+        client_app_name = appid_api.get_application_name(client_id);
+        payload_app_name = appid_api.get_application_name(payload_id);
+        misc_name = appid_api.get_application_name(misc_id);
+
+        PacketTracer::log("AppID: service: %s(%d), client: %s(%d), payload: %s(%d), misc: %s(%d)\n",
+            (service_app_name ? service_app_name : ""), service_id,
+            (client_app_name ? client_app_name : ""), client_id,
+            (payload_app_name ? payload_app_name : ""), payload_id,
+            (misc_name ? misc_name : ""), misc_id);
+    }
+}
+
 AppIdInspector::AppIdInspector(const AppIdModuleConfig* pc)
 {
     assert(pc);
@@ -174,7 +196,11 @@ void AppIdInspector::eval(Packet* p)
 
     appid_stats.packets++;
     if (p->flow)
+    {
         AppIdDiscovery::do_application_discovery(p);
+        if (SnortConfig::packet_trace_enabled())
+            add_appid_to_packet_trace(p->flow);
+    }
     else
         appid_stats.ignored_packets++;
 }
@@ -308,4 +334,3 @@ AppId getOpenAppId(Flow* flow)
     AppIdSession* asd = appid_api.get_appid_session(flow);
     return asd->payload_app_id;
 }
-

diff --git a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
index 746230281ceab4abf8dc9eccd1e97c1fdb824779..ffe619d77ac6406b0a93723a6d9252b97119d560 100644 (file)
--- a/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
+++ b/src/network_inspectors/appid/detector_plugins/http_url_patterns.h
@@ -25,14 +25,15 @@
 #include <list>
 #include <vector>
 
-#include "application_ids.h"
-#include "appid_utils/sf_multi_mpse.h"
-#include "appid_utils/sf_mlmp.h"
 #include "flow/flow.h"
 #include "log/messages.h"
 #include "search_engines/search_tool.h"
 #include "utils/util.h"
 
+#include "application_ids.h"
+#include "appid_utils/sf_multi_mpse.h"
+#include "appid_utils/sf_mlmp.h"
+
 struct Packet;
 struct AppIdServiceSubtype;
 class AppIdHttpSession;

diff --git a/src/packet_io/sfdaq.cc b/src/packet_io/sfdaq.cc
index c725af12463bc07fdb410a71301888a9d1c02502..f5d0f97c006d35d1eb672dad011be59ab4cbe708 100644 (file)
--- a/src/packet_io/sfdaq.cc
+++ b/src/packet_io/sfdaq.cc
@@ -178,6 +178,11 @@ void SFDAQ::term()
     daq_mod = nullptr;
 }
 
+const char* SFDAQ::verdict_to_string(DAQ_Verdict verdict)
+{
+    return daq_verdict_string(verdict);
+}
+
 bool SFDAQ::forwarding_packet(const DAQ_PktHdr_t* h)
 {
     // DAQ mode is inline and the packet will be forwarded?

diff --git a/src/packet_io/sfdaq.h b/src/packet_io/sfdaq.h
index 69c15c7f0e587d4102cac65fe36d53b877140a85..2633e5bd472864d30a4525c174d039d5ae91f3c2 100644 (file)
--- a/src/packet_io/sfdaq.h
+++ b/src/packet_io/sfdaq.h
@@ -85,6 +85,7 @@ public:
     static void unload();
 
     static void print_types(std::ostream&);
+    static const char* verdict_to_string(DAQ_Verdict verdict);
     static void init(const SnortConfig*);
     static void term();