Merge pull request #645 in SNORT/snort3 from late to master

Squashed commit of the following:

commit a797535dfab7af50483a891e95d37a5fd22ca00c
Author: Russ Combs <rucombs@cisco.com>
Date:   Wed Sep 28 21:42:23 2016 -0400

    fix ac_full initialization performance

commit a4edbc38a49e7d54e19dae65a29257db9c99129a
Author: Russ Combs (rucombs) <rucombs@cisco.com>
Date:   Wed Sep 28 06:07:36 2016 -0400

    latency log tweaks

diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc
index dde3863b610314ca06ed999a683fff88155570fb..65c2c851e280fb1600c37634968bcb7c33026293 100644 (file)
--- a/src/detection/detection_options.cc
+++ b/src/detection/detection_options.cc
@@ -806,12 +806,13 @@ void detection_option_tree_update_otn_stats(SFXHASH* doth)
     }
 }
 
-detection_option_tree_root_t* new_root()
+detection_option_tree_root_t* new_root(OptTreeNode* otn)
 {
     detection_option_tree_root_t* p = (detection_option_tree_root_t*)
         snort_calloc(sizeof(detection_option_tree_root_t));
 
     p->latency_state = new RuleLatencyState[ThreadConfig::get_instance_max()]();
+    p->otn = otn;
 
     return p;
 }

diff --git a/src/detection/detection_options.h b/src/detection/detection_options.h
index ca3f1474840f8c93ff27cf89fcf9ca2590d3bcf8..ab591bcd2b78fd0da565d6e92f83a61f94ae68ff 100644 (file)
--- a/src/detection/detection_options.h
+++ b/src/detection/detection_options.h
@@ -100,6 +100,8 @@ struct detection_option_tree_root_t
     int num_children;
     detection_option_tree_node_t** children;
     RuleLatencyState* latency_state;
+
+    struct OptTreeNode* otn;  // first rule in tree
 };
 
 struct detection_option_eval_data_t
@@ -126,7 +128,7 @@ void print_option_tree(detection_option_tree_node_t*, int level);
 #endif
 void detection_option_tree_update_otn_stats(SFXHASH*);
 
-detection_option_tree_root_t* new_root();
+detection_option_tree_root_t* new_root(OptTreeNode*);
 void free_detection_option_root(void** existing_tree);
 
 detection_option_tree_node_t* new_node(option_type_t, void*);

diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc
index 819da873bddcb58e1a504bdc9cf2474c4a5b958f..56500daf44a38e8ec43eec5944ab86aedcae452e 100644 (file)
--- a/src/detection/fp_create.cc
+++ b/src/detection/fp_create.cc
@@ -111,7 +111,7 @@ static int otn_create_tree(OptTreeNode* otn, void** existing_tree)
         return -1;
 
     if (!*existing_tree)
-        *existing_tree = new_root();
+        *existing_tree = new_root(otn);
 
     detection_option_tree_root_t* root = (detection_option_tree_root_t*)*existing_tree;
 
@@ -322,11 +322,9 @@ static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree)
     if (!existing_tree)
         return -1;
 
-    if (!*existing_tree)
-        *existing_tree = new_root();
-
     if (!id)
     {
+        assert(*existing_tree);
         /* NULL input id (PMX *), last call for this pattern state */
         return finalize_detection_option_tree(sc, (detection_option_tree_root_t*)*existing_tree);
     }
@@ -334,6 +332,9 @@ static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree)
     PMX* pmx = (PMX*)id;
     OptTreeNode* otn = (OptTreeNode*)pmx->rule_node.rnRuleData;
 
+    if (!*existing_tree)
+        *existing_tree = new_root(otn);
+
     return otn_create_tree(otn, existing_tree);
 }
 

diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc
index d90f72698a5e92bd2319825643b768616584e149..871fcb25a094b9441adf891b7e9221b41cdadf48 100644 (file)
--- a/src/detection/fp_detect.cc
+++ b/src/detection/fp_detect.cc
@@ -362,7 +362,7 @@ static int detection_option_tree_evaluate(detection_option_tree_root_t* root,
     if ( !root )
         return 0;
 
-    RuleLatency::Context rule_latency_ctx(root);
+    RuleLatency::Context rule_latency_ctx(root, eval_data->p);
 
     if ( RuleLatency::suspended() )
         return 0;

diff --git a/src/latency/packet_latency.cc b/src/latency/packet_latency.cc
index 4c171bab4fe2161ecb4645886e6db32590afa57f..671ce55fbbe9760224e4b676de20f3d9ff778f1e 100644 (file)
--- a/src/latency/packet_latency.cc
+++ b/src/latency/packet_latency.cc
@@ -73,29 +73,29 @@ public:
 using ConfigWrapper = ReferenceWrapper<PacketLatencyConfig>;
 using EventHandler = EventingWrapper<Event>;
 
-static inline std::ostream& operator<<(std::ostream& os, const sfip_t* addr)
-{
-    char str[INET6_ADDRSTRLEN + 1];
-    sfip_ntop(addr, str, sizeof(str));
-    str[INET6_ADDRSTRLEN] = '\0';
-    os << str;
-    return os;
-}
-
 static inline std::ostream& operator<<(std::ostream& os, const Event& e)
 {
     using std::chrono::duration_cast;
     using std::chrono::microseconds;
 
-    os << "latency: packet " << pc.total_from_daq << " timed out";
+    os << "latency: " << pc.total_from_daq << " packet";
+
     if ( e.fastpathed )
-        os << " (fastpathed)";
+        os << " fastpathed: ";
+    else
+        os << " timed out: ";
+
+    os << clock_usecs(duration_cast<microseconds>(e.elapsed).count()) << " usec, ";
+
+    if ( e.packet->is_cooked() )
+        os << e.packet->get_pseudo_type();
+    else
+        os << e.packet->get_type();
 
-    os << ": ";
+    os << "[" << e.packet->dsize << "]";
 
-    os << clock_usecs(duration_cast<microseconds>(e.elapsed).count()) << " usec, [";
-    os << e.packet->ptrs.ip_api.get_src() << " -> " <<
-        e.packet->ptrs.ip_api.get_dst() << "]";
+    os << ", " << e.packet->ptrs.ip_api.get_src() << ":" << e.packet->ptrs.sp;
+    os << " -> " << e.packet->ptrs.ip_api.get_dst() << ":" << e.packet->ptrs.dp;
 
     return os;
 }

diff --git a/src/latency/rule_latency.cc b/src/latency/rule_latency.cc
index a9a1aab4d8b8adb9ddabb8b6b23879397b4fabdd..d8df600f0f3ae29e024afdf92dff9c65b31c45b6 100644 (file)
--- a/src/latency/rule_latency.cc
+++ b/src/latency/rule_latency.cc
@@ -29,9 +29,12 @@
 #include <sstream>
 
 #include "detection/detection_options.h"
+#include "detection/treenodes.h"
 #include "events/event_queue.h"
 #include "log/messages.h"
 #include "main/snort_config.h"
+#include "sfip/sf_ip.h"
+#include "utils/stats.h"
 
 #include "latency_config.h"
 #include "latency_rules.h"
@@ -62,16 +65,18 @@ struct Event
     Type type;
     typename SnortClock::duration elapsed;
     detection_option_tree_root_t* root;
+    Packet* packet;
 };
 
 template<typename Clock>
 class RuleTimer : public LatencyTimer<Clock>
 {
 public:
-    RuleTimer(typename Clock::duration d, detection_option_tree_root_t* root) :
-        LatencyTimer<Clock>(d), root(root) { }
+    RuleTimer(typename Clock::duration d, detection_option_tree_root_t* root, Packet* p) :
+        LatencyTimer<Clock>(d), root(root), packet(p) { }
 
     detection_option_tree_root_t* root;
+    Packet* packet;
 };
 
 using ConfigWrapper = ReferenceWrapper<RuleLatencyConfig>;
@@ -82,26 +87,31 @@ static inline std::ostream& operator<<(std::ostream& os, const Event& e)
     using std::chrono::duration_cast;
     using std::chrono::microseconds;
 
-    os << "latency: ";
+    os << "latency: " << pc.total_from_daq << " rule tree ";
 
-    if ( e.type == Event::EVENT_ENABLED )
-        os << "rule tree enabled: ";
-
-    else
+    switch ( e.type )
     {
-        os << "rule tree timed out";
-        if ( e.type == Event::EVENT_SUSPENDED )
-            os << " (suspended)";
+    case Event::EVENT_ENABLED:
+        os << "enabled: ";
+        break;
 
-        os << ": ";
+    case Event::EVENT_TIMED_OUT:
+        os << "timed out: ";
+        break;
 
-        os << clock_usecs(duration_cast<microseconds>(e.elapsed).count()) << " usec, ";
+    case Event::EVENT_SUSPENDED:
+        os << "suspended: ";
+        break;
     }
 
-    // FIXIT-L clean up rule latency logging; delete or make useful
-    // seeing the address of the dot root is not particularly helpful
-    // except during debugging (ported from legacy ppm)
-    os << "[" << e.root << "]";
+    os << clock_usecs(duration_cast<microseconds>(e.elapsed).count()) << " usec, ";
+    os << e.root->otn->sigInfo.generator << ":" << e.root->otn->sigInfo.id;
+
+    if ( e.root->num_children > 1 )
+        os << " (of " << e.root->num_children << ")";
+
+    os << ", " << e.packet->ptrs.ip_api.get_src() << ":" << e.packet->ptrs.sp;
+    os << " -> " << e.packet->ptrs.ip_api.get_dst() << ":" << e.packet->ptrs.dp;
 
     return os;
 }
@@ -180,7 +190,7 @@ class Impl
 public:
     Impl(const ConfigWrapper&, EventHandler&, EventHandler&);
 
-    bool push(detection_option_tree_root_t*);
+    bool push(detection_option_tree_root_t*, Packet*);
     bool pop();
     bool suspended() const;
 
@@ -199,23 +209,18 @@ inline Impl<Clock, RuleTree>::Impl(const ConfigWrapper& cfg, EventHandler& eh, E
 { }
 
 template<typename Clock, typename RuleTree>
-inline bool Impl<Clock, RuleTree>::push(detection_option_tree_root_t* root)
+inline bool Impl<Clock, RuleTree>::push(detection_option_tree_root_t* root, Packet* p)
 {
-    assert(root);
+    assert(root and p);
 
     // FIXIT-L rule timer is pushed even if rule is not enabled (no visible side-effects)
-    timers.emplace_back(config->max_time, root);
+    timers.emplace_back(config->max_time, root, p);
 
     if ( config->allow_reenable() )
     {
         if ( RuleTree::reenable(*root, config->max_suspend_time, Clock::now()) )
         {
-            Event e {
-                Event::EVENT_ENABLED,
-                typename Clock::duration(0),
-                root
-            };
-
+            Event e { Event::EVENT_ENABLED, config->max_suspend_time, root, p };
             handle(e);
             return true;
         }
@@ -241,10 +246,10 @@ inline bool Impl<Clock, RuleTree>::pop()
             auto suspended = RuleTree::timeout_and_suspend(*timer.root, config->suspend_threshold,
                 Clock::now(), config->suspend);
 
-            Event e {
+            Event e
+            {
                 suspended ? Event::EVENT_SUSPENDED : Event::EVENT_TIMED_OUT,
-                timer.elapsed(),
-                timer.root
+                timer.elapsed(), timer.root, timer.packet
             };
 
             handle(e);
@@ -332,11 +337,11 @@ static inline Impl<>& get_impl()
 // rule latency interface
 // -----------------------------------------------------------------------------
 
-void RuleLatency::push(detection_option_tree_root_t* root)
+void RuleLatency::push(detection_option_tree_root_t* root, Packet* p)
 {
     if ( rule_latency::config->enabled() )
     {
-        if ( rule_latency::get_impl().push(root) )
+        if ( rule_latency::get_impl().push(root, p) )
             ++latency_stats.rule_tree_enables;
 
         ++latency_stats.total_rule_evals;
@@ -465,6 +470,7 @@ TEST_CASE ( "rule latency impl", "[latency]" )
     config.config.action = RuleLatencyConfig::ALERT_AND_LOG;
 
     detection_option_tree_root_t root;
+    Packet pkt(false);
 
     rule_latency::Impl<MockClock, RuleInterfaceSpy> impl(config, event_handler, log_handler);
 
@@ -476,7 +482,7 @@ TEST_CASE ( "rule latency impl", "[latency]" )
 
             SECTION( "push rule" )
             {
-                CHECK_FALSE( impl.push(&root) );
+                CHECK_FALSE( impl.push(&root, &pkt) );
                 CHECK( log_handler.count == 0 );
                 CHECK( event_handler.count == 0 );
                 CHECK( RuleInterfaceSpy::reenable_called );
@@ -486,7 +492,7 @@ TEST_CASE ( "rule latency impl", "[latency]" )
             {
                 RuleInterfaceSpy::reenable_result = true;
 
-                CHECK( impl.push(&root) );
+                CHECK( impl.push(&root, &pkt) );
                 CHECK( log_handler.count == 1 );
                 CHECK( event_handler.count == 1 );
                 CHECK( RuleInterfaceSpy::reenable_called );
@@ -499,7 +505,7 @@ TEST_CASE ( "rule latency impl", "[latency]" )
 
             SECTION( "push rule" )
             {
-                CHECK_FALSE( impl.push(&root) );
+                CHECK_FALSE( impl.push(&root, &pkt) );
                 CHECK( log_handler.count == 0 );
                 CHECK( event_handler.count == 0 );
                 CHECK_FALSE( RuleInterfaceSpy::reenable_called );
@@ -511,7 +517,7 @@ TEST_CASE ( "rule latency impl", "[latency]" )
     {
         RuleInterfaceSpy::is_suspended_result = true;
 
-        impl.push(&root);
+        impl.push(&root, &pkt);
 
         SECTION( "suspending of rules disabled" )
         {
@@ -534,7 +540,7 @@ TEST_CASE ( "rule latency impl", "[latency]" )
     {
         config.config.max_time = 1_ticks;
 
-        impl.push(&root);
+        impl.push(&root, &pkt);
 
         SECTION( "rule timeout" )
         {

diff --git a/src/latency/rule_latency.h b/src/latency/rule_latency.h
index 5ada2c4e20201b96fb5c32838c4b9a916b147e93..46c5a4250a4e151feb647ccf10783e888da8f7f4 100644 (file)
--- a/src/latency/rule_latency.h
+++ b/src/latency/rule_latency.h
@@ -22,11 +22,12 @@
 #define RULE_LATENCY_H
 
 struct detection_option_tree_root_t;
+struct Packet;
 
 class RuleLatency
 {
 public:
-    static void push(detection_option_tree_root_t*);
+    static void push(detection_option_tree_root_t*, Packet*);
     static void pop();
     static bool suspended();
 
@@ -35,8 +36,8 @@ public:
     class Context
     {
     public:
-        Context(detection_option_tree_root_t* root)
-        { RuleLatency::push(root); }
+        Context(detection_option_tree_root_t* root, Packet* p)
+        { RuleLatency::push(root, p); }
 
         ~Context()
         { RuleLatency::pop(); }

diff --git a/src/search_engines/acsmx2.cc b/src/search_engines/acsmx2.cc
index 79a803c7c196a9a9da504ef610bb4413813a20f6..88e4a9d6300f68d42733827499a6e42e7548f9d7 100644 (file)
--- a/src/search_engines/acsmx2.cc
+++ b/src/search_engines/acsmx2.cc
@@ -354,15 +354,6 @@ static void AC_FREE_DFA(void* p, int n, int sizeofstate)
     }
 }
 
-static void queue_add(std::list<int>& q, int s)
-{
-    for ( auto qs : q )
-        if ( qs == s )
-            return;
-
-    q.push_back(s);
-}
-
 /*
 *  Get Next State-NFA
 */
@@ -639,7 +630,7 @@ static void Build_NFA(ACSM_STRUCT2* acsm)
 
         if ( s )
         {
-            queue_add(queue, s);
+            queue.push_back(s);
             FailState[s] = 0;
         }
     }
@@ -655,8 +646,7 @@ static void Build_NFA(ACSM_STRUCT2* acsm)
 
             if ( (acstate_t)s != ACSM_FAIL_STATE2 )
             {
-                queue_add(queue, s);
-
+                queue.push_back(s);
                 int fs = FailState[r];
 
                 /*
@@ -707,7 +697,7 @@ static void Convert_NFA_To_DFA(ACSM_STRUCT2* acsm)
     for (int i=0; i<acsm->acsmAlphabetSize; i++)
     {
         if ( int s = List_GetNextState(acsm,0,i) )
-            queue_add(queue, s);
+            queue.push_back(s);
     }
 
     /* Start building the next layer of transitions */
@@ -720,7 +710,7 @@ static void Convert_NFA_To_DFA(ACSM_STRUCT2* acsm)
 
             if ( (acstate_t)s != ACSM_FAIL_STATE2 && s!= 0)
             {
-                queue_add(queue, s);
+                queue.push_back(s);
             }
             else
             {

diff --git a/src/sfip/sf_ip.h b/src/sfip/sf_ip.h
index cd56fcba4442b3557e9d39e4eaa8558c4efee1a9..367015bd607d939da341100a37d21de638966171 100644 (file)
--- a/src/sfip/sf_ip.h
+++ b/src/sfip/sf_ip.h
@@ -42,6 +42,8 @@
 #include "main/snort_debug.h"
 #include "main/snort_types.h"
 
+#include <sstream>
+
 /* define SFIP_ROBUST to check pointers passed into the sfip libs.
  * Robustification should not be enabled if the client code is trustworthy.
  * Namely, if pointers are checked once in the client, or are pointers to
@@ -700,5 +702,13 @@ inline void sfip_copy(sfip_t& lhs, const sfip_t* const rhs)
 SO_PUBLIC void sfip_raw_ntop(int family, const void* ip_raw, char* buf, int bufsize);
 SO_PUBLIC void sfip_ntop(const sfip_t* ip, char* buf, int bufsize);
 
+inline std::ostream& operator<<(std::ostream& os, const sfip_t* addr)
+{
+    char str[INET6_ADDRSTRLEN];
+    sfip_ntop(addr, str, sizeof(str));
+    os << str;
+    return os;
+}
+
 #endif