From: Russ Combs (rucombs) Date: Thu, 29 Sep 2016 17:42:16 +0000 (-0400) Subject: Merge pull request #645 in SNORT/snort3 from late to master X-Git-Tag: 3.0.0-233~253 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5e84c30f0e886e87a6e8725c1661ddf3aec5020b;p=thirdparty%2Fsnort3.git Merge pull request #645 in SNORT/snort3 from late to master Squashed commit of the following: commit a797535dfab7af50483a891e95d37a5fd22ca00c Author: Russ Combs Date: Wed Sep 28 21:42:23 2016 -0400 fix ac_full initialization performance commit a4edbc38a49e7d54e19dae65a29257db9c99129a Author: Russ Combs (rucombs) Date: Wed Sep 28 06:07:36 2016 -0400 latency log tweaks --- diff --git a/src/detection/detection_options.cc b/src/detection/detection_options.cc index dde3863b6..65c2c851e 100644 --- a/src/detection/detection_options.cc +++ b/src/detection/detection_options.cc @@ -806,12 +806,13 @@ void detection_option_tree_update_otn_stats(SFXHASH* doth) } } -detection_option_tree_root_t* new_root() +detection_option_tree_root_t* new_root(OptTreeNode* otn) { detection_option_tree_root_t* p = (detection_option_tree_root_t*) snort_calloc(sizeof(detection_option_tree_root_t)); p->latency_state = new RuleLatencyState[ThreadConfig::get_instance_max()](); + p->otn = otn; return p; } diff --git a/src/detection/detection_options.h b/src/detection/detection_options.h index ca3f14748..ab591bcd2 100644 --- a/src/detection/detection_options.h +++ b/src/detection/detection_options.h @@ -100,6 +100,8 @@ struct detection_option_tree_root_t int num_children; detection_option_tree_node_t** children; RuleLatencyState* latency_state; + + struct OptTreeNode* otn; // first rule in tree }; struct detection_option_eval_data_t @@ -126,7 +128,7 @@ void print_option_tree(detection_option_tree_node_t*, int level); #endif void detection_option_tree_update_otn_stats(SFXHASH*); -detection_option_tree_root_t* new_root(); +detection_option_tree_root_t* new_root(OptTreeNode*); void free_detection_option_root(void** existing_tree); detection_option_tree_node_t* new_node(option_type_t, void*); diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc index 819da873b..56500daf4 100644 --- a/src/detection/fp_create.cc +++ b/src/detection/fp_create.cc @@ -111,7 +111,7 @@ static int otn_create_tree(OptTreeNode* otn, void** existing_tree) return -1; if (!*existing_tree) - *existing_tree = new_root(); + *existing_tree = new_root(otn); detection_option_tree_root_t* root = (detection_option_tree_root_t*)*existing_tree; @@ -322,11 +322,9 @@ static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree) if (!existing_tree) return -1; - if (!*existing_tree) - *existing_tree = new_root(); - if (!id) { + assert(*existing_tree); /* NULL input id (PMX *), last call for this pattern state */ return finalize_detection_option_tree(sc, (detection_option_tree_root_t*)*existing_tree); } @@ -334,6 +332,9 @@ static int pmx_create_tree(SnortConfig* sc, void* id, void** existing_tree) PMX* pmx = (PMX*)id; OptTreeNode* otn = (OptTreeNode*)pmx->rule_node.rnRuleData; + if (!*existing_tree) + *existing_tree = new_root(otn); + return otn_create_tree(otn, existing_tree); } diff --git a/src/detection/fp_detect.cc b/src/detection/fp_detect.cc index d90f72698..871fcb25a 100644 --- a/src/detection/fp_detect.cc +++ b/src/detection/fp_detect.cc @@ -362,7 +362,7 @@ static int detection_option_tree_evaluate(detection_option_tree_root_t* root, if ( !root ) return 0; - RuleLatency::Context rule_latency_ctx(root); + RuleLatency::Context rule_latency_ctx(root, eval_data->p); if ( RuleLatency::suspended() ) return 0; diff --git a/src/latency/packet_latency.cc b/src/latency/packet_latency.cc index 4c171bab4..671ce55fb 100644 --- a/src/latency/packet_latency.cc +++ b/src/latency/packet_latency.cc @@ -73,29 +73,29 @@ public: using ConfigWrapper = ReferenceWrapper; using EventHandler = EventingWrapper; -static inline std::ostream& operator<<(std::ostream& os, const sfip_t* addr) -{ - char str[INET6_ADDRSTRLEN + 1]; - sfip_ntop(addr, str, sizeof(str)); - str[INET6_ADDRSTRLEN] = '\0'; - os << str; - return os; -} - static inline std::ostream& operator<<(std::ostream& os, const Event& e) { using std::chrono::duration_cast; using std::chrono::microseconds; - os << "latency: packet " << pc.total_from_daq << " timed out"; + os << "latency: " << pc.total_from_daq << " packet"; + if ( e.fastpathed ) - os << " (fastpathed)"; + os << " fastpathed: "; + else + os << " timed out: "; + + os << clock_usecs(duration_cast(e.elapsed).count()) << " usec, "; + + if ( e.packet->is_cooked() ) + os << e.packet->get_pseudo_type(); + else + os << e.packet->get_type(); - os << ": "; + os << "[" << e.packet->dsize << "]"; - os << clock_usecs(duration_cast(e.elapsed).count()) << " usec, ["; - os << e.packet->ptrs.ip_api.get_src() << " -> " << - e.packet->ptrs.ip_api.get_dst() << "]"; + os << ", " << e.packet->ptrs.ip_api.get_src() << ":" << e.packet->ptrs.sp; + os << " -> " << e.packet->ptrs.ip_api.get_dst() << ":" << e.packet->ptrs.dp; return os; } diff --git a/src/latency/rule_latency.cc b/src/latency/rule_latency.cc index a9a1aab4d..d8df600f0 100644 --- a/src/latency/rule_latency.cc +++ b/src/latency/rule_latency.cc @@ -29,9 +29,12 @@ #include #include "detection/detection_options.h" +#include "detection/treenodes.h" #include "events/event_queue.h" #include "log/messages.h" #include "main/snort_config.h" +#include "sfip/sf_ip.h" +#include "utils/stats.h" #include "latency_config.h" #include "latency_rules.h" @@ -62,16 +65,18 @@ struct Event Type type; typename SnortClock::duration elapsed; detection_option_tree_root_t* root; + Packet* packet; }; template class RuleTimer : public LatencyTimer { public: - RuleTimer(typename Clock::duration d, detection_option_tree_root_t* root) : - LatencyTimer(d), root(root) { } + RuleTimer(typename Clock::duration d, detection_option_tree_root_t* root, Packet* p) : + LatencyTimer(d), root(root), packet(p) { } detection_option_tree_root_t* root; + Packet* packet; }; using ConfigWrapper = ReferenceWrapper; @@ -82,26 +87,31 @@ static inline std::ostream& operator<<(std::ostream& os, const Event& e) using std::chrono::duration_cast; using std::chrono::microseconds; - os << "latency: "; + os << "latency: " << pc.total_from_daq << " rule tree "; - if ( e.type == Event::EVENT_ENABLED ) - os << "rule tree enabled: "; - - else + switch ( e.type ) { - os << "rule tree timed out"; - if ( e.type == Event::EVENT_SUSPENDED ) - os << " (suspended)"; + case Event::EVENT_ENABLED: + os << "enabled: "; + break; - os << ": "; + case Event::EVENT_TIMED_OUT: + os << "timed out: "; + break; - os << clock_usecs(duration_cast(e.elapsed).count()) << " usec, "; + case Event::EVENT_SUSPENDED: + os << "suspended: "; + break; } - // FIXIT-L clean up rule latency logging; delete or make useful - // seeing the address of the dot root is not particularly helpful - // except during debugging (ported from legacy ppm) - os << "[" << e.root << "]"; + os << clock_usecs(duration_cast(e.elapsed).count()) << " usec, "; + os << e.root->otn->sigInfo.generator << ":" << e.root->otn->sigInfo.id; + + if ( e.root->num_children > 1 ) + os << " (of " << e.root->num_children << ")"; + + os << ", " << e.packet->ptrs.ip_api.get_src() << ":" << e.packet->ptrs.sp; + os << " -> " << e.packet->ptrs.ip_api.get_dst() << ":" << e.packet->ptrs.dp; return os; } @@ -180,7 +190,7 @@ class Impl public: Impl(const ConfigWrapper&, EventHandler&, EventHandler&); - bool push(detection_option_tree_root_t*); + bool push(detection_option_tree_root_t*, Packet*); bool pop(); bool suspended() const; @@ -199,23 +209,18 @@ inline Impl::Impl(const ConfigWrapper& cfg, EventHandler& eh, E { } template -inline bool Impl::push(detection_option_tree_root_t* root) +inline bool Impl::push(detection_option_tree_root_t* root, Packet* p) { - assert(root); + assert(root and p); // FIXIT-L rule timer is pushed even if rule is not enabled (no visible side-effects) - timers.emplace_back(config->max_time, root); + timers.emplace_back(config->max_time, root, p); if ( config->allow_reenable() ) { if ( RuleTree::reenable(*root, config->max_suspend_time, Clock::now()) ) { - Event e { - Event::EVENT_ENABLED, - typename Clock::duration(0), - root - }; - + Event e { Event::EVENT_ENABLED, config->max_suspend_time, root, p }; handle(e); return true; } @@ -241,10 +246,10 @@ inline bool Impl::pop() auto suspended = RuleTree::timeout_and_suspend(*timer.root, config->suspend_threshold, Clock::now(), config->suspend); - Event e { + Event e + { suspended ? Event::EVENT_SUSPENDED : Event::EVENT_TIMED_OUT, - timer.elapsed(), - timer.root + timer.elapsed(), timer.root, timer.packet }; handle(e); @@ -332,11 +337,11 @@ static inline Impl<>& get_impl() // rule latency interface // ----------------------------------------------------------------------------- -void RuleLatency::push(detection_option_tree_root_t* root) +void RuleLatency::push(detection_option_tree_root_t* root, Packet* p) { if ( rule_latency::config->enabled() ) { - if ( rule_latency::get_impl().push(root) ) + if ( rule_latency::get_impl().push(root, p) ) ++latency_stats.rule_tree_enables; ++latency_stats.total_rule_evals; @@ -465,6 +470,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) config.config.action = RuleLatencyConfig::ALERT_AND_LOG; detection_option_tree_root_t root; + Packet pkt(false); rule_latency::Impl impl(config, event_handler, log_handler); @@ -476,7 +482,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) SECTION( "push rule" ) { - CHECK_FALSE( impl.push(&root) ); + CHECK_FALSE( impl.push(&root, &pkt) ); CHECK( log_handler.count == 0 ); CHECK( event_handler.count == 0 ); CHECK( RuleInterfaceSpy::reenable_called ); @@ -486,7 +492,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) { RuleInterfaceSpy::reenable_result = true; - CHECK( impl.push(&root) ); + CHECK( impl.push(&root, &pkt) ); CHECK( log_handler.count == 1 ); CHECK( event_handler.count == 1 ); CHECK( RuleInterfaceSpy::reenable_called ); @@ -499,7 +505,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) SECTION( "push rule" ) { - CHECK_FALSE( impl.push(&root) ); + CHECK_FALSE( impl.push(&root, &pkt) ); CHECK( log_handler.count == 0 ); CHECK( event_handler.count == 0 ); CHECK_FALSE( RuleInterfaceSpy::reenable_called ); @@ -511,7 +517,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) { RuleInterfaceSpy::is_suspended_result = true; - impl.push(&root); + impl.push(&root, &pkt); SECTION( "suspending of rules disabled" ) { @@ -534,7 +540,7 @@ TEST_CASE ( "rule latency impl", "[latency]" ) { config.config.max_time = 1_ticks; - impl.push(&root); + impl.push(&root, &pkt); SECTION( "rule timeout" ) { diff --git a/src/latency/rule_latency.h b/src/latency/rule_latency.h index 5ada2c4e2..46c5a4250 100644 --- a/src/latency/rule_latency.h +++ b/src/latency/rule_latency.h @@ -22,11 +22,12 @@ #define RULE_LATENCY_H struct detection_option_tree_root_t; +struct Packet; class RuleLatency { public: - static void push(detection_option_tree_root_t*); + static void push(detection_option_tree_root_t*, Packet*); static void pop(); static bool suspended(); @@ -35,8 +36,8 @@ public: class Context { public: - Context(detection_option_tree_root_t* root) - { RuleLatency::push(root); } + Context(detection_option_tree_root_t* root, Packet* p) + { RuleLatency::push(root, p); } ~Context() { RuleLatency::pop(); } diff --git a/src/search_engines/acsmx2.cc b/src/search_engines/acsmx2.cc index 79a803c7c..88e4a9d63 100644 --- a/src/search_engines/acsmx2.cc +++ b/src/search_engines/acsmx2.cc @@ -354,15 +354,6 @@ static void AC_FREE_DFA(void* p, int n, int sizeofstate) } } -static void queue_add(std::list& q, int s) -{ - for ( auto qs : q ) - if ( qs == s ) - return; - - q.push_back(s); -} - /* * Get Next State-NFA */ @@ -639,7 +630,7 @@ static void Build_NFA(ACSM_STRUCT2* acsm) if ( s ) { - queue_add(queue, s); + queue.push_back(s); FailState[s] = 0; } } @@ -655,8 +646,7 @@ static void Build_NFA(ACSM_STRUCT2* acsm) if ( (acstate_t)s != ACSM_FAIL_STATE2 ) { - queue_add(queue, s); - + queue.push_back(s); int fs = FailState[r]; /* @@ -707,7 +697,7 @@ static void Convert_NFA_To_DFA(ACSM_STRUCT2* acsm) for (int i=0; iacsmAlphabetSize; i++) { if ( int s = List_GetNextState(acsm,0,i) ) - queue_add(queue, s); + queue.push_back(s); } /* Start building the next layer of transitions */ @@ -720,7 +710,7 @@ static void Convert_NFA_To_DFA(ACSM_STRUCT2* acsm) if ( (acstate_t)s != ACSM_FAIL_STATE2 && s!= 0) { - queue_add(queue, s); + queue.push_back(s); } else { diff --git a/src/sfip/sf_ip.h b/src/sfip/sf_ip.h index cd56fcba4..367015bd6 100644 --- a/src/sfip/sf_ip.h +++ b/src/sfip/sf_ip.h @@ -42,6 +42,8 @@ #include "main/snort_debug.h" #include "main/snort_types.h" +#include + /* define SFIP_ROBUST to check pointers passed into the sfip libs. * Robustification should not be enabled if the client code is trustworthy. * Namely, if pointers are checked once in the client, or are pointers to @@ -700,5 +702,13 @@ inline void sfip_copy(sfip_t& lhs, const sfip_t* const rhs) SO_PUBLIC void sfip_raw_ntop(int family, const void* ip_raw, char* buf, int bufsize); SO_PUBLIC void sfip_ntop(const sfip_t* ip, char* buf, int bufsize); +inline std::ostream& operator<<(std::ostream& os, const sfip_t* addr) +{ + char str[INET6_ADDRSTRLEN]; + sfip_ntop(addr, str, sizeof(str)); + os << str; + return os; +} + #endif