From: Davis McPherson (davmcphe) Date: Wed, 31 Jul 2019 15:55:02 +0000 (-0400) Subject: Merge pull request #1668 in SNORT/snort3 from ~DAVMCPHE/snort3:single_flowCache to... X-Git-Tag: 3.0.0-259~19 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c7149d0a37bb5157e78a38f9b7bfba4a4d57cfca;p=thirdparty%2Fsnort3.git Merge pull request #1668 in SNORT/snort3 from ~DAVMCPHE/snort3:single_flowCache to master Squashed commit of the following: commit 9ba243badce51f88109251156be8efaf97ff1c3c Author: Devendra Dahiphale Date: Sun Jun 30 03:36:52 2019 -0400 Flow: make a single flow cache for all the protocols flow: refactor flow config object to work with single flow cache concept flow: if no 'get_ssn' handler configured then skip processing of the flow flow: release session object allocated for a flow when the Flow object is reused and the PktType of the new flow is different from the previous use stream: update checks for modified stream config to work with updates to stream config options flow: refactor uni list managment into a separate class and instantiate an instance for ip flows and another for all non-ip flows snort2lua: Combine proto specific cache options for max_session in one max_flows option --- diff --git a/src/flow/CMakeLists.txt b/src/flow/CMakeLists.txt index f84e14131..fb5a7d036 100644 --- a/src/flow/CMakeLists.txt +++ b/src/flow/CMakeLists.txt @@ -17,8 +17,9 @@ add_library (flow OBJECT flow_control.cc flow_control.h flow_key.cc - flow_stash.h flow_stash.cc + flow_stash.h + flow_uni_list.h ha.cc ha_module.cc ha_module.h diff --git a/src/flow/flow.cc b/src/flow/flow.cc index 83c6a1ede..58678296c 100644 --- a/src/flow/flow.cc +++ b/src/flow/flow.cc @@ -120,10 +120,16 @@ void Flow::term() delete bitop; if ( ssn_client ) + { ssn_client->rem_ref(); + ssn_client = nullptr; + } if ( ssn_server ) + { ssn_server->rem_ref(); + ssn_server = nullptr; + } if ( clouseau ) clouseau->rem_ref(); @@ -135,7 +141,10 @@ void Flow::term() delete ha_state; if (stash) + { delete stash; + stash = nullptr; + } } inline void Flow::clean() diff --git a/src/flow/flow_cache.cc b/src/flow/flow_cache.cc index 0ed2f33b0..80581a12e 100644 --- a/src/flow/flow_cache.cc +++ b/src/flow/flow_cache.cc @@ -24,7 +24,6 @@ #include "flow/flow_cache.h" -#include "flow/ha.h" #include "hash/zhash.h" #include "helpers/flag_context.h" #include "ips_options/ips_flowbits.h" @@ -33,7 +32,11 @@ #include "time/packet_time.h" #include "utils/stats.h" +#include "flow.h" #include "flow_key.h" +#include "flow_uni_list.h" +#include "ha.h" +#include "session.h" using namespace snort; @@ -43,28 +46,23 @@ using namespace snort; // FlowCache stuff //------------------------------------------------------------------------- -FlowCache::FlowCache (const FlowConfig& cfg) : config(cfg) +FlowCache::FlowCache(const FlowCacheConfig& cfg) : config(cfg) { - hash_table = new ZHash(config.max_sessions, sizeof(FlowKey)); + hash_table = new ZHash(config.max_flows, sizeof(FlowKey)); hash_table->set_keyops(FlowKey::hash, FlowKey::compare); - uni_head = new Flow; - uni_tail = new Flow; + uni_flows = new FlowUniList; + uni_ip_flows = new FlowUniList; - uni_head->next = uni_tail; - uni_tail->prev = uni_head; - - uni_count = 0; flags = 0x0; assert(prune_stats.get_total() == 0); } -FlowCache::~FlowCache () +FlowCache::~FlowCache() { - delete uni_head; - delete uni_tail; - + delete uni_flows; + delete uni_ip_flows; delete hash_table; } @@ -97,27 +95,19 @@ Flow* FlowCache::find(const FlowKey* key) // always prepend void FlowCache::link_uni(Flow* flow) { - flow->next = uni_head->next; - flow->prev = uni_head; - - uni_head->next->prev = flow; - uni_head->next = flow; - - ++uni_count; + if ( flow->pkt_type == PktType::IP ) + uni_ip_flows->link_uni(flow); + else + uni_flows->link_uni(flow); } // but remove from any point void FlowCache::unlink_uni(Flow* flow) { - if ( !flow->next ) - return; - - --uni_count; - - flow->next->prev = flow->prev; - flow->prev->next = flow->next; - - flow->next = flow->prev = nullptr; + if ( flow->pkt_type == PktType::IP ) + uni_ip_flows->unlink_uni(flow); + else + uni_flows->unlink_uni(flow); } Flow* FlowCache::get(const FlowKey* key) @@ -129,18 +119,24 @@ Flow* FlowCache::get(const FlowKey* key) { if ( !prune_stale(timestamp, nullptr) ) { - if ( !prune_unis() ) + if ( !prune_unis(key->pkt_type) ) prune_excess(nullptr); } flow = (Flow*)hash_table->get(key); - assert(flow); - flow->reset(); + + if ( flow->session && flow->pkt_type != key->pkt_type ) + flow->term(); + else + flow->reset(); link_uni(flow); } - memory::MemoryCap::update_allocations(config.cap_weight); + if ( flow->session && flow->pkt_type != key->pkt_type ) + flow->term(); + + memory::MemoryCap::update_allocations(config.proto[to_utype(key->pkt_type)].cap_weight); flow->last_data_seen = timestamp; return flow; @@ -164,7 +160,7 @@ int FlowCache::remove(Flow* flow) // and Flow::retire try remove the flow from hash. Flow::reset should // just mark the flow as pending instead of trying to remove it. if ( deleted ) - memory::MemoryCap::update_deallocations(config.cap_weight); + memory::MemoryCap::update_deallocations(config.proto[to_utype(flow->key->pkt_type)].cap_weight); return deleted; } @@ -221,26 +217,31 @@ unsigned FlowCache::prune_stale(uint32_t thetime, const Flow* save_me) return pruned; } -unsigned FlowCache::prune_unis() +unsigned FlowCache::prune_unis(PktType pkt_type) { ActiveSuspendContext act_susp; // we may have many or few unis; need to find reasonable ratio // FIXIT-M max_uni should be based on typical ratios seen in perfmon - const unsigned max_uni = (config.max_sessions >> 2) + 1; - - Flow* curr = uni_tail->prev; + const unsigned max_uni = (config.max_flows >> 2) + 1; unsigned pruned = 0; + FlowUniList* uni_list; + + if ( pkt_type == PktType::IP ) + uni_list = uni_ip_flows; + else + uni_list = uni_flows; - while ( (uni_count > max_uni) && curr && (pruned < cleanup_flows) ) + Flow* flow = uni_list->get_oldest_uni(); + while ( (uni_list->get_count() > max_uni) && flow && (pruned < cleanup_flows) ) { - Flow* flow = curr; - curr = curr->prev; + Flow* prune_me = flow; + flow = prune_me->prev; - if ( flow->was_blocked() ) + if ( prune_me->was_blocked() ) continue; - release(flow, PruneReason::UNI); + release(prune_me, PruneReason::UNI); ++pruned; } @@ -251,7 +252,7 @@ unsigned FlowCache::prune_excess(const Flow* save_me) { ActiveSuspendContext act_susp; - auto max_cap = config.max_sessions - cleanup_flows; + auto max_cap = config.max_flows - cleanup_flows; assert(max_cap > 0); unsigned pruned = 0; @@ -327,7 +328,7 @@ unsigned FlowCache::timeout(unsigned num_flows, time_t thetime) while ( flow and (retired < num_flows) ) { - if ( flow->last_data_seen + config.nominal_timeout > thetime ) + if ( flow->last_data_seen + config.proto[to_utype(flow->key->pkt_type)].nominal_timeout > thetime ) break; if ( HighAvailabilityManager::in_standby(flow) or diff --git a/src/flow/flow_cache.h b/src/flow/flow_cache.h index cff941974..b0393ee98 100644 --- a/src/flow/flow_cache.h +++ b/src/flow/flow_cache.h @@ -37,10 +37,12 @@ class Flow; struct FlowKey; } +class FlowUniList; + class FlowCache { public: - FlowCache(const FlowConfig&); + FlowCache(const FlowCacheConfig&); ~FlowCache(); FlowCache(const FlowCache&) = delete; @@ -53,7 +55,6 @@ public: int release(snort::Flow*, PruneReason = PruneReason::NONE, bool do_cleanup = true); - unsigned prune_unis(); unsigned prune_stale(uint32_t thetime, const snort::Flow* save_me); unsigned prune_excess(const snort::Flow* save_me); bool prune_one(PruneReason, bool do_cleanup); @@ -63,7 +64,7 @@ public: unsigned get_count(); unsigned get_max_flows() const - { return config.max_sessions; } + { return config.max_flows; } PegCount get_total_prunes() const { return prune_stats.get_total(); } @@ -83,14 +84,16 @@ private: private: static const unsigned cleanup_flows = 1; - const FlowConfig config; - unsigned uni_count; + const FlowCacheConfig config; uint32_t flags; class ZHash* hash_table; - snort::Flow* uni_head, * uni_tail; + FlowUniList* uni_flows; + FlowUniList* uni_ip_flows; + + //snort::Flow* uni_head_ip, *uni_tail_ip, *uni_head_nonip, *uni_tail_nonip; PruneStats prune_stats; + unsigned prune_unis(PktType); }; - #endif diff --git a/src/flow/flow_config.h b/src/flow/flow_config.h index 95ab46bdb..f3cc7277d 100644 --- a/src/flow/flow_config.h +++ b/src/flow/flow_config.h @@ -21,15 +21,21 @@ #ifndef FLOW_CONFIG_H #define FLOW_CONFIG_H -// configured by the stream module for each cache instance +#include "framework/decode_data.h" -struct FlowConfig +// configured by the stream module +struct FlowTypeConfig { - unsigned max_sessions = 0; - unsigned pruning_timeout = 0; unsigned nominal_timeout = 0; unsigned cap_weight = 0; }; +struct FlowCacheConfig +{ + unsigned max_flows = 0; + unsigned pruning_timeout = 0; + FlowTypeConfig proto[to_utype(PktType::MAX)]; +}; + #endif diff --git a/src/flow/flow_control.cc b/src/flow/flow_control.cc index 625a4e9aa..09c274db2 100644 --- a/src/flow/flow_control.cc +++ b/src/flow/flow_control.cc @@ -41,17 +41,22 @@ using namespace snort; -FlowControl::FlowControl() = default; +FlowControl::FlowControl(const FlowCacheConfig& fc) +{ + cache = new FlowCache(fc); + + mem = (Flow*)snort_calloc(fc.max_flows, sizeof(Flow)); + + for ( unsigned i = 0; i < fc.max_flows; ++i ) + cache->push(mem + i); +} FlowControl::~FlowControl() { DetectionEngine de; - for ( int i = 0; i < to_utype(PktType::MAX); ++i ) - { - delete proto[i].cache; - snort_free(proto[i].mem); - } + delete cache; + snort_free(mem); delete exp_cache; } @@ -59,15 +64,15 @@ FlowControl::~FlowControl() // count foo //------------------------------------------------------------------------- -PegCount FlowControl::get_total_prunes(PktType type) const +PegCount FlowControl::get_total_prunes() const { - auto cache = get_cache(type); + auto cache = get_cache(); return cache ? cache->get_total_prunes() : 0; } -PegCount FlowControl::get_prunes(PktType type, PruneReason reason) const +PegCount FlowControl::get_prunes(PruneReason reason) const { - auto cache = get_cache(type); + auto cache = get_cache(); return cache ? cache->get_prunes(reason) : 0; } @@ -75,10 +80,10 @@ void FlowControl::clear_counts() { for ( int i = 0; i < to_utype(PktType::MAX); ++i ) { - if ( proto[i].cache ) - proto[i].cache->reset_stats(); + if ( cache ) + cache->reset_stats(); - proto[i].num_flows = 0; + num_flows = 0; } } @@ -88,7 +93,7 @@ void FlowControl::clear_counts() Flow* FlowControl::find_flow(const FlowKey* key) { - if ( auto cache = get_cache(key->pkt_type) ) + if ( auto cache = get_cache() ) return cache->find(key); return nullptr; @@ -96,7 +101,7 @@ Flow* FlowControl::find_flow(const FlowKey* key) Flow* FlowControl::new_flow(const FlowKey* key) { - if ( auto cache = get_cache(key->pkt_type) ) + if ( auto cache = get_cache() ) return cache->get(key); return nullptr; @@ -106,7 +111,7 @@ Flow* FlowControl::new_flow(const FlowKey* key) // packet type are obviated for existing / initialized flows void FlowControl::delete_flow(const FlowKey* key) { - FlowCache* cache = get_cache(key->pkt_type); + FlowCache* cache = get_cache(); if ( !cache ) return; @@ -117,20 +122,20 @@ void FlowControl::delete_flow(const FlowKey* key) void FlowControl::delete_flow(Flow* flow, PruneReason reason) { - if ( auto cache = get_cache(flow->pkt_type) ) + if ( auto cache = get_cache() ) cache->release(flow, reason); } -void FlowControl::purge_flows (PktType type) +void FlowControl::purge_flows () { - if ( auto cache = get_cache(type) ) + if ( auto cache = get_cache() ) cache->purge(); } // hole for memory manager/prune handler bool FlowControl::prune_one(PruneReason reason, bool do_cleanup) { - auto cache = get_cache(last_pkt_type); + auto cache = get_cache(); return cache ? cache->prune_one(reason, do_cleanup) : false; } @@ -140,7 +145,7 @@ void FlowControl::timeout_flows(time_t cur_time) return; ActiveSuspendContext act_susp; - FlowCache* fc = get_cache(types[next]); + FlowCache* fc = get_cache(); if ( ++next >= types.size() ) next = 0; @@ -315,21 +320,11 @@ static void init_roles(Packet* p, Flow* flow) // proto //------------------------------------------------------------------------- -void FlowControl::init_proto( - PktType type, const FlowConfig& fc, InspectSsnFunc get_ssn) +void FlowControl::init_proto(PktType type, InspectSsnFunc get_ssn) { - if ( !fc.max_sessions || !get_ssn ) - return; - - auto& con = proto[to_utype(type)]; + assert(get_ssn); - con.cache = new FlowCache(fc); - con.mem = (Flow*)snort_calloc(fc.max_sessions, sizeof(Flow)); - - for ( unsigned i = 0; i < fc.max_sessions; ++i ) - con.cache->push(con.mem + i); - - con.get_ssn = get_ssn; + proto[to_utype(type)].get_ssn = get_ssn; types.emplace_back(type); } @@ -355,14 +350,12 @@ static bool want_flow(PktType type, Packet* p) bool FlowControl::process(PktType type, Packet* p, bool* new_flow) { - auto& con = proto[to_utype(type)]; - - if ( !con.cache ) + if ( !proto[to_utype(type)].get_ssn ) return false; FlowKey key; set_key(&key, p); - Flow* flow = con.cache->find(&key); + Flow* flow = cache->find(&key); if ( !flow ) { flow = HighAvailabilityManager::import(*p, key); @@ -372,7 +365,7 @@ bool FlowControl::process(PktType type, Packet* p, bool* new_flow) if ( !want_flow(type, p) ) return true; - flow = con.cache->get(&key); + flow = cache->get(&key); if ( !flow ) return true; @@ -385,15 +378,15 @@ bool FlowControl::process(PktType type, Packet* p, bool* new_flow) if ( !flow->session ) { flow->init(type); - flow->session = con.get_ssn(flow); + flow->session = proto[to_utype(type)].get_ssn(flow); } - con.num_flows += process(flow, p); + num_flows += process(flow, p); // FIXIT-M refactor to unlink_uni immediately after session // is processed by inspector manager (all flows) if ( flow->next && is_bidirectional(flow) ) - con.cache->unlink_uni(flow); + cache->unlink_uni(flow); return true; } diff --git a/src/flow/flow_control.h b/src/flow/flow_control.h index 4690d801c..2c29477e5 100644 --- a/src/flow/flow_control.h +++ b/src/flow/flow_control.h @@ -48,7 +48,7 @@ enum class PruneReason : uint8_t; class FlowControl { public: - FlowControl(); + FlowControl(const FlowCacheConfig& fc); ~FlowControl(); public: @@ -57,12 +57,12 @@ public: snort::Flow* find_flow(const snort::FlowKey*); snort::Flow* new_flow(const snort::FlowKey*); - void init_proto(PktType, const FlowConfig&, snort::InspectSsnFunc); + void init_proto(PktType, snort::InspectSsnFunc); void init_exp(uint32_t max); void delete_flow(const snort::FlowKey*); void delete_flow(snort::Flow*, PruneReason); - void purge_flows(PktType); + void purge_flows(); bool prune_one(PruneReason, bool do_cleanup); void timeout_flows(time_t cur_time); @@ -82,20 +82,20 @@ public: const snort::SfIp *dstIP, uint16_t dstPort, SnortProtocolId snort_protocol_id, snort::FlowData*); - PegCount get_flows(PktType pt) - { return proto[to_utype(pt)].num_flows; } + PegCount get_flows() + { return num_flows; } - PegCount get_total_prunes(PktType) const; - PegCount get_prunes(PktType, PruneReason) const; + PegCount get_total_prunes() const; + PegCount get_prunes(PruneReason) const; void clear_counts(); private: - FlowCache* get_cache(PktType pt) - { return proto[to_utype(pt)].cache; } + FlowCache* get_cache() + { return cache; } - const FlowCache* get_cache(PktType pt) const - { return proto[to_utype(pt)].cache; } + const FlowCache* get_cache() const + { return cache; } void set_key(snort::FlowKey*, snort::Packet*); @@ -105,12 +105,12 @@ private: private: struct { - FlowCache* cache = nullptr; - snort::Flow* mem = nullptr; snort::InspectSsnFunc get_ssn = nullptr; - PegCount num_flows = 0; } proto[to_utype(PktType::MAX)]; + PegCount num_flows = 0; + FlowCache* cache = nullptr; + snort::Flow* mem = nullptr; class ExpectCache* exp_cache = nullptr; PktType last_pkt_type = PktType::NONE; diff --git a/src/flow/flow_uni_list.h b/src/flow/flow_uni_list.h new file mode 100644 index 000000000..7c138e086 --- /dev/null +++ b/src/flow/flow_uni_list.h @@ -0,0 +1,83 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// flow_uni_list.h author davis mcpherson davmchpe@cisco.com + +#ifndef FLOW_UNI_LIST_H +#define FLOW_UNI_LIST_H + +#include "flow.h" + +class FlowUniList +{ +public: + + FlowUniList() + { + head = new snort::Flow; + tail = new snort::Flow; + + head->next = tail; + tail->prev = head; + } + + ~FlowUniList() + { + delete head; + delete tail; + } + + void link_uni(snort::Flow* flow) + { + + flow->next = head->next; + flow->prev = head; + + head->next->prev = flow; + head->next = flow; + + ++count; + } + + void unlink_uni(snort::Flow* flow) + { + if ( !flow->next ) + return; + + --count; + + flow->next->prev = flow->prev; + flow->prev->next = flow->next; + + flow->next = flow->prev = nullptr; + } + + snort::Flow* get_oldest_uni() + { return tail->prev; } + + unsigned get_count() const + { return count; } + +private: + snort::Flow* head = nullptr; + snort::Flow* tail = nullptr; + unsigned count = 0; + +}; + +#endif diff --git a/src/flow/test/CMakeLists.txt b/src/flow/test/CMakeLists.txt index 819801eb9..d4c128476 100644 --- a/src/flow/test/CMakeLists.txt +++ b/src/flow/test/CMakeLists.txt @@ -3,3 +3,5 @@ add_cpputest( ha_test ) add_cpputest( flow_stash_test SOURCES ../flow_stash.cc ) + +add_cpputest( session_test ) diff --git a/src/flow/test/session_test.cc b/src/flow/test/session_test.cc new file mode 100644 index 000000000..df50b2d7c --- /dev/null +++ b/src/flow/test/session_test.cc @@ -0,0 +1,81 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// session_test.cc authors Devendra Dahiphale +// unit test main + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "flow/session.h" + +#include +#include + +using namespace snort; + +class DummySession : public Session +{ + public: + DummySession(snort::Flow* f) : Session(f) { } + void clear() { } + ~DummySession() { } +}; + +//------------------------------------------------------------------------- +// tests +//------------------------------------------------------------------------- + +TEST_GROUP(session_test) +{ +}; + +TEST(session_test, seesion_class_test) +{ + Session *ssn = new DummySession(nullptr); + CHECK(true == ssn->setup(nullptr)); + + ssn->update_direction(1, nullptr, 1234); + CHECK(0 == ssn->process(nullptr)); + ssn->restart(nullptr); + ssn->flush_client(nullptr); + ssn->flush_server(nullptr); + ssn->flush_talker(nullptr, false); + ssn->flush_listener(nullptr, false); + CHECK(nullptr == ssn->get_splitter(true)); + + ssn->set_extra_data(nullptr, 1); + + CHECK(true == ssn->is_sequenced(1)); + CHECK(true == ssn->are_packets_missing(1)); + + + CHECK(SSN_DIR_NONE == ssn->get_reassembly_direction()); + CHECK(SSN_MISSING_NONE == ssn->missing_in_reassembled(1)); + + CHECK(false == ssn->set_packet_action_to_hold(nullptr)); + + delete ssn; +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +} + diff --git a/src/stream/base/stream_base.cc b/src/stream/base/stream_base.cc index 837cc2c33..70f989e54 100644 --- a/src/stream/base/stream_base.cc +++ b/src/stream/base/stream_base.cc @@ -49,41 +49,17 @@ THREAD_LOCAL FlowControl* flow_con = nullptr; static BaseStats g_stats; THREAD_LOCAL BaseStats stream_base_stats; -#define PROTO_PEGS(proto_str) \ - { CountType::SUM, proto_str "_flows", "total " proto_str " sessions" }, \ - { CountType::SUM, proto_str "_total_prunes", "total " proto_str " sessions pruned" }, \ - { CountType::SUM, proto_str "_idle_prunes", proto_str " sessions pruned due to timeout" }, \ - { CountType::SUM, proto_str "_excess_prunes", proto_str " sessions pruned due to excess" }, \ - { CountType::SUM, proto_str "_uni_prunes", proto_str " uni sessions pruned" }, \ - { CountType::SUM, proto_str "_preemptive_prunes", proto_str " sessions pruned during preemptive pruning" }, \ - { CountType::SUM, proto_str "_memcap_prunes", proto_str " sessions pruned due to memcap" }, \ - { CountType::SUM, proto_str "_ha_prunes", proto_str " sessions pruned by high availability sync" } - -#define SET_PROTO_COUNTS(proto, pkttype) \ - stream_base_stats.proto ## _flows = flow_con->get_flows(PktType::pkttype); \ - stream_base_stats.proto ## _total_prunes = flow_con->get_total_prunes(PktType::pkttype), \ - stream_base_stats.proto ## _timeout_prunes = \ - flow_con->get_prunes(PktType::pkttype, PruneReason::IDLE), \ - stream_base_stats.proto ## _excess_prunes = \ - flow_con->get_prunes(PktType::pkttype, PruneReason::EXCESS), \ - stream_base_stats.proto ## _uni_prunes = \ - flow_con->get_prunes(PktType::pkttype, PruneReason::UNI), \ - stream_base_stats.proto ## _preemptive_prunes = \ - flow_con->get_prunes(PktType::pkttype, PruneReason::PREEMPTIVE), \ - stream_base_stats.proto ## _memcap_prunes = \ - flow_con->get_prunes(PktType::pkttype, PruneReason::MEMCAP), \ - stream_base_stats.proto ## _ha_prunes = \ - flow_con->get_prunes(PktType::pkttype, PruneReason::HA) - // FIXIT-L dependency on stats define in another file const PegInfo base_pegs[] = { - PROTO_PEGS("ip"), - PROTO_PEGS("icmp"), - PROTO_PEGS("tcp"), - PROTO_PEGS("udp"), - PROTO_PEGS("user"), - PROTO_PEGS("file"), + { CountType::SUM, "flows", "total sessions" }, + { CountType::SUM, "total_prunes", "total sessions pruned" }, + { CountType::SUM, "idle_prunes", " sessions pruned due to timeout" }, + { CountType::SUM, "excess_prunes", "sessions pruned due to excess" }, + { CountType::SUM, "uni_prunes", "uni sessions pruned" }, + { CountType::SUM, "preemptive_prunes", "sessions pruned during preemptive pruning" }, + { CountType::SUM, "memcap_prunes", "sessions pruned due to memcap" }, + { CountType::SUM, "ha_prunes", "sessions pruned by high availability sync" }, { CountType::END, nullptr, nullptr } }; @@ -93,12 +69,14 @@ void base_sum() if ( !flow_con ) return; - SET_PROTO_COUNTS(ip, IP); - SET_PROTO_COUNTS(icmp, ICMP); - SET_PROTO_COUNTS(tcp, TCP); - SET_PROTO_COUNTS(udp, UDP); - SET_PROTO_COUNTS(user, PDU); - SET_PROTO_COUNTS(file, FILE); + stream_base_stats.flows = flow_con->get_flows(); + stream_base_stats.prunes = flow_con->get_total_prunes(); + stream_base_stats.timeout_prunes = flow_con->get_prunes(PruneReason::IDLE); + stream_base_stats.excess_prunes = flow_con->get_prunes(PruneReason::EXCESS); + stream_base_stats.uni_prunes = flow_con->get_prunes(PruneReason::UNI); + stream_base_stats.preemptive_prunes = flow_con->get_prunes(PruneReason::PREEMPTIVE); + stream_base_stats.memcap_prunes = flow_con->get_prunes(PruneReason::MEMCAP); + stream_base_stats.ha_prunes = flow_con->get_prunes(PruneReason::HA); sum_stats((PegCount*)&g_stats, (PegCount*)&stream_base_stats, array_size(base_pegs)-1); @@ -159,47 +137,34 @@ StreamBase::StreamBase(const StreamModuleConfig* c) void StreamBase::tinit() { - assert(!flow_con); - flow_con = new FlowControl; + assert(!flow_con && config.flow_cache_cfg.max_flows); + + // this is temp added to suppress the compiler error only + flow_con = new FlowControl(config.flow_cache_cfg); InspectSsnFunc f; StreamHAManager::tinit(); - if ( config.ip_cfg.max_sessions ) - { - if ( (f = InspectorManager::get_session(PROTO_BIT__IP)) ) - flow_con->init_proto(PktType::IP, config.ip_cfg, f); - } - if ( config.icmp_cfg.max_sessions ) - { - if ( (f = InspectorManager::get_session(PROTO_BIT__ICMP)) ) - flow_con->init_proto(PktType::ICMP, config.icmp_cfg, f); - } - if ( config.tcp_cfg.max_sessions ) - { - if ( (f = InspectorManager::get_session(PROTO_BIT__TCP)) ) - flow_con->init_proto(PktType::TCP, config.tcp_cfg, f); - } - if ( config.udp_cfg.max_sessions ) - { - if ( (f = InspectorManager::get_session(PROTO_BIT__UDP)) ) - flow_con->init_proto(PktType::UDP, config.udp_cfg, f); - } - if ( config.user_cfg.max_sessions ) - { - if ( (f = InspectorManager::get_session(PROTO_BIT__PDU)) ) - flow_con->init_proto(PktType::PDU, config.user_cfg, f); - } - if ( config.file_cfg.max_sessions ) - { - if ( (f = InspectorManager::get_session(PROTO_BIT__FILE)) ) - flow_con->init_proto(PktType::FILE, config.file_cfg, f); - } - uint32_t max = config.tcp_cfg.max_sessions + config.udp_cfg.max_sessions - + config.user_cfg.max_sessions; + if ( (f = InspectorManager::get_session(PROTO_BIT__IP)) ) + flow_con->init_proto(PktType::IP, f); + + if ( (f = InspectorManager::get_session(PROTO_BIT__ICMP)) ) + flow_con->init_proto(PktType::ICMP, f); + + if ( (f = InspectorManager::get_session(PROTO_BIT__TCP)) ) + flow_con->init_proto(PktType::TCP, f); + + if ( (f = InspectorManager::get_session(PROTO_BIT__UDP)) ) + flow_con->init_proto(PktType::UDP, f); + + if ( (f = InspectorManager::get_session(PROTO_BIT__PDU)) ) + flow_con->init_proto(PktType::PDU, f); + + if ( (f = InspectorManager::get_session(PROTO_BIT__FILE)) ) + flow_con->init_proto(PktType::FILE, f); - if ( max > 0 ) - flow_con->init_exp(max); + if ( config.flow_cache_cfg.max_flows > 0 ) + flow_con->init_exp(config.flow_cache_cfg.max_flows); FlushBucket::set(config.footprint); } @@ -213,12 +178,8 @@ void StreamBase::tterm() void StreamBase::show(SnortConfig*) { LogMessage("Stream Base config:\n"); - LogMessage(" IP max sessions: %d\n", config.ip_cfg.max_sessions); - LogMessage(" ICMP max sessions: %d\n", config.icmp_cfg.max_sessions); - LogMessage(" TCP max sessions: %d\n", config.tcp_cfg.max_sessions); - LogMessage(" UDP max sessions: %d\n", config.udp_cfg.max_sessions); - LogMessage(" User max sessions: %d\n", config.user_cfg.max_sessions); - LogMessage(" File max sessions: %d\n", config.file_cfg.max_sessions); + LogMessage(" Max flows: %d\n", config.flow_cache_cfg.max_flows); + LogMessage(" Pruning timeout: %d\n", config.flow_cache_cfg.pruning_timeout); } void StreamBase::eval(Packet* p) diff --git a/src/stream/base/stream_module.cc b/src/stream/base/stream_module.cc index afdcd6d65..127adf531 100644 --- a/src/stream/base/stream_module.cc +++ b/src/stream/base/stream_module.cc @@ -38,15 +38,9 @@ using namespace std; //------------------------------------------------------------------------- Trace TRACE_NAME(stream); -#define CACHE_PARAMS(name, max, prune, idle, weight) \ +#define FLOW_TYPE_PARAMS(name, idle, weight) \ static const Parameter name[] = \ { \ - { "max_sessions", Parameter::PT_INT, "2:max32", max, \ - "maximum simultaneous sessions tracked before pruning" }, \ - \ - { "pruning_timeout", Parameter::PT_INT, "1:max32", prune, \ - "minimum inactive time before being eligible for pruning" }, \ - \ { "idle_timeout", Parameter::PT_INT, "1:max32", idle, \ "maximum inactive time before retiring session tracker" }, \ \ @@ -56,31 +50,37 @@ static const Parameter name[] = \ { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } \ } -CACHE_PARAMS(ip_params, "16384", "30", "180", "64"); -CACHE_PARAMS(icmp_params, "65536", "30", "180", "8"); -CACHE_PARAMS(tcp_params, "262144", "30", "3600", "11500"); -CACHE_PARAMS(udp_params, "131072", "30", "180", "128"); -CACHE_PARAMS(user_params, "1024", "30", "180", "256"); -CACHE_PARAMS(file_params, "128", "30", "180", "32"); +FLOW_TYPE_PARAMS(ip_params, "180", "64"); +FLOW_TYPE_PARAMS(icmp_params, "180", "8"); +FLOW_TYPE_PARAMS(tcp_params, "3600", "11500"); +FLOW_TYPE_PARAMS(udp_params, "180", "128"); +FLOW_TYPE_PARAMS(user_params,"180", "256"); +FLOW_TYPE_PARAMS(file_params, "180", "32"); -#define CACHE_TABLE(cache, proto, params) \ - { cache, Parameter::PT_TABLE, params, nullptr, \ +#define FLOW_TYPE_TABLE(flow_type, proto, params) \ + { flow_type, Parameter::PT_TABLE, params, nullptr, \ "configure " proto " cache limits" } static const Parameter s_params[] = { { "footprint", Parameter::PT_INT, "0:max32", "0", - "use zero for production, non-zero for testing at given size (for TCP and user)" }, + "use zero for production, non-zero for testing at given size (for TCP and user)" }, { "ip_frags_only", Parameter::PT_BOOL, nullptr, "false", - "don't process non-frag flows" }, + "don't process non-frag flows" }, + + { "max_flows", Parameter::PT_INT, "2:max32", "476288", + "maximum simultaneous flows tracked before pruning" }, - CACHE_TABLE("ip_cache", "ip", ip_params), - CACHE_TABLE("icmp_cache", "icmp", icmp_params), - CACHE_TABLE("tcp_cache", "tcp", tcp_params), - CACHE_TABLE("udp_cache", "udp", udp_params), - CACHE_TABLE("user_cache", "user", user_params), - CACHE_TABLE("file_cache", "file", file_params), + { "pruning_timeout", Parameter::PT_INT, "1:max32", "30", + "minimum inactive time before being eligible for pruning" }, + + FLOW_TYPE_TABLE("ip_cache", "ip", ip_params), + FLOW_TYPE_TABLE("icmp_cache", "icmp", icmp_params), + FLOW_TYPE_TABLE("tcp_cache", "tcp", tcp_params), + FLOW_TYPE_TABLE("udp_cache", "udp", udp_params), + FLOW_TYPE_TABLE("user_cache", "user", user_params), + FLOW_TYPE_TABLE("file_cache", "file", file_params), { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } }; @@ -129,7 +129,7 @@ bool StreamModule::begin(const char* fqn, int, SnortConfig*) bool StreamModule::set(const char* fqn, Value& v, SnortConfig* c) { - FlowConfig* fc = nullptr; + PktType type = PktType::NONE; if ( v.is("footprint") ) { @@ -142,59 +142,65 @@ bool StreamModule::set(const char* fqn, Value& v, SnortConfig* c) c->set_run_flags(RUN_FLAG__IP_FRAGS_ONLY); return true; } + else if ( v.is("max_flows") ) + { + config.flow_cache_cfg.max_flows = v.get_uint32(); + return true; + } + else if ( v.is("pruning_timeout") ) + { + config.flow_cache_cfg.pruning_timeout = v.get_uint32(); + return true; + } else if ( strstr(fqn, "ip_cache") ) - fc = &config.ip_cfg; - + type = PktType::IP; else if ( strstr(fqn, "icmp_cache") ) - fc = &config.icmp_cfg; - + type = PktType::ICMP; else if ( strstr(fqn, "tcp_cache") ) - fc = &config.tcp_cfg; - + type = PktType::TCP; else if ( strstr(fqn, "udp_cache") ) - fc = &config.udp_cfg; - + type = PktType::UDP; else if ( strstr(fqn, "user_cache") ) - fc = &config.user_cfg; - + type = PktType::PDU; else if ( strstr(fqn, "file_cache") ) - fc = &config.file_cfg; - + type = PktType::FILE; else return Module::set(fqn, v, c); - if ( v.is("max_sessions") ) - fc->max_sessions = v.get_uint32(); - - else if ( v.is("pruning_timeout") ) - fc->pruning_timeout = v.get_uint32(); - - else if ( v.is("idle_timeout") ) - fc->nominal_timeout = v.get_uint32(); - + if ( v.is("idle_timeout") ) + config.flow_cache_cfg.proto[to_utype(type)].nominal_timeout = v.get_uint32(); else if ( v.is("cap_weight") ) - fc->cap_weight = v.get_uint16(); - + config.flow_cache_cfg.proto[to_utype(type)].cap_weight = v.get_uint16(); else return false; return true; } -static int check_cache_change(const char* fqn, const char* name, const FlowConfig& new_cfg, - const FlowConfig& saved_cfg) +static int check_stream_config(const FlowCacheConfig& new_cfg, const FlowCacheConfig& saved_cfg) { int ret = 0; - if ( saved_cfg.max_sessions and strstr(fqn, name) ) + + if ( saved_cfg.max_flows != new_cfg.max_flows + or saved_cfg.pruning_timeout != new_cfg.pruning_timeout ) { - if ( saved_cfg.max_sessions != new_cfg.max_sessions - or saved_cfg.pruning_timeout != new_cfg.pruning_timeout - or saved_cfg.nominal_timeout != new_cfg.nominal_timeout ) - { - ReloadError("Changing of %s requires a restart\n", name); - ret = 1; - } + ReloadError("Change of stream flow cache options requires a restart\n"); + ret = 1; } + + return ret; +} + +static int check_stream_proto_config(const FlowCacheConfig& new_cfg, const FlowCacheConfig& saved_cfg, PktType type) +{ + int ret = 0; + + if ( saved_cfg.proto[to_utype(type)].nominal_timeout != new_cfg.proto[to_utype(type)].nominal_timeout ) + { + ReloadError("Change of stream protocol configuration options requires a restart\n"); + ret = 1; + } + return ret; } @@ -205,16 +211,21 @@ bool StreamModule::end(const char* fqn, int, SnortConfig*) static StreamModuleConfig saved_config = {}; static int issue_found = 0; - issue_found += check_cache_change(fqn, "ip_cache", config.ip_cfg, saved_config.ip_cfg); - issue_found += check_cache_change(fqn, "icmp_cache", config.icmp_cfg, saved_config.icmp_cfg); - issue_found += check_cache_change(fqn, "tcp_cache", config.tcp_cfg, saved_config.tcp_cfg); - issue_found += check_cache_change(fqn, "udp_cache", config.udp_cfg, saved_config.udp_cfg); - issue_found += check_cache_change(fqn, "user_cache", config.ip_cfg, saved_config.user_cfg); - issue_found += check_cache_change(fqn, "file_cache", config.ip_cfg, saved_config.file_cfg); + if ( saved_config.flow_cache_cfg.max_flows ) + { + // FIXIT-H - stream reload story will change this to look for change to max_flows config option + issue_found += check_stream_config(config.flow_cache_cfg, saved_config.flow_cache_cfg); + issue_found += check_stream_proto_config(config.flow_cache_cfg, saved_config.flow_cache_cfg, PktType::IP); + issue_found += check_stream_proto_config(config.flow_cache_cfg, saved_config.flow_cache_cfg, PktType::UDP); + issue_found += check_stream_proto_config(config.flow_cache_cfg, saved_config.flow_cache_cfg, PktType::TCP); + issue_found += check_stream_proto_config(config.flow_cache_cfg, saved_config.flow_cache_cfg, PktType::ICMP); + issue_found += check_stream_proto_config(config.flow_cache_cfg, saved_config.flow_cache_cfg, PktType::PDU); + issue_found += check_stream_proto_config(config.flow_cache_cfg, saved_config.flow_cache_cfg, PktType::FILE); + } if ( !strcmp(fqn, "stream") ) { - if ( saved_config.ip_cfg.max_sessions // saved config is valid + if ( saved_config.flow_cache_cfg.max_flows // saved config is valid and config.footprint != saved_config.footprint ) { ReloadError("Changing of stream.footprint requires a restart\n"); @@ -224,6 +235,7 @@ bool StreamModule::end(const char* fqn, int, SnortConfig*) saved_config = config; issue_found = 0; } + return true; } diff --git a/src/stream/base/stream_module.h b/src/stream/base/stream_module.h index 1710fe477..2305a07bd 100644 --- a/src/stream/base/stream_module.h +++ b/src/stream/base/stream_module.h @@ -39,24 +39,16 @@ extern Trace TRACE_NAME(stream); #define MOD_NAME "stream" #define MOD_HELP "common flow tracking" -#define PROTO_FIELDS(proto) \ - PegCount proto ## _flows; \ - PegCount proto ## _total_prunes; \ - PegCount proto ## _timeout_prunes; \ - PegCount proto ## _excess_prunes; \ - PegCount proto ## _uni_prunes; \ - PegCount proto ## _preemptive_prunes; \ - PegCount proto ## _memcap_prunes; \ - PegCount proto ## _ha_prunes - struct BaseStats { - PROTO_FIELDS(ip); - PROTO_FIELDS(icmp); - PROTO_FIELDS(tcp); - PROTO_FIELDS(udp); - PROTO_FIELDS(user); - PROTO_FIELDS(file); + PegCount flows; + PegCount prunes; + PegCount timeout_prunes; + PegCount excess_prunes; + PegCount uni_prunes; + PegCount preemptive_prunes; + PegCount memcap_prunes; + PegCount ha_prunes; }; extern const PegInfo base_pegs[]; @@ -65,13 +57,7 @@ extern THREAD_LOCAL BaseStats stream_base_stats; struct StreamModuleConfig { - FlowConfig ip_cfg; - FlowConfig icmp_cfg; - FlowConfig tcp_cfg; - FlowConfig udp_cfg; - FlowConfig user_cfg; - FlowConfig file_cfg; - + FlowCacheConfig flow_cache_cfg; unsigned footprint; }; diff --git a/src/stream/stream.cc b/src/stream/stream.cc index 0e3ee4fab..13d9f0d02 100644 --- a/src/stream/stream.cc +++ b/src/stream/stream.cc @@ -350,12 +350,7 @@ void Stream::purge_flows() if ( !flow_con ) return; - flow_con->purge_flows(PktType::IP); - flow_con->purge_flows(PktType::ICMP); - flow_con->purge_flows(PktType::TCP); - flow_con->purge_flows(PktType::UDP); - flow_con->purge_flows(PktType::PDU); - flow_con->purge_flows(PktType::FILE); + flow_con->purge_flows(); } void Stream::timeout_flows(time_t cur_time) diff --git a/src/stream/tcp/tcp_ha.cc b/src/stream/tcp/tcp_ha.cc index 9b37fee63..dd56ff9da 100644 --- a/src/stream/tcp/tcp_ha.cc +++ b/src/stream/tcp/tcp_ha.cc @@ -42,7 +42,6 @@ Flow* TcpHA::create_session(const FlowKey* key) } return flow; - } void TcpHA::deactivate_session(Flow* flow) diff --git a/tools/snort2lua/preprocessor_states/pps_stream5_global.cc b/tools/snort2lua/preprocessor_states/pps_stream5_global.cc index d9f9e9702..2b312a7b8 100644 --- a/tools/snort2lua/preprocessor_states/pps_stream5_global.cc +++ b/tools/snort2lua/preprocessor_states/pps_stream5_global.cc @@ -43,6 +43,9 @@ bool StreamGlobal::convert(std::istringstream& data_stream) table_api.open_table("stream"); + int tcp_max = 262144, udp_max = 131072, ip_max = 16384, icmp_max = 65536; + int pruning_timeout = INT_MAX; + while (util::get_string(data_stream, keyword, ",")) { bool tmpval = true; @@ -87,44 +90,46 @@ bool StreamGlobal::convert(std::istringstream& data_stream) else if (keyword == "max_tcp") { - table_api.open_table("tcp_cache"); if (cv.do_convert_max_session()) { - table_api.add_diff_option_comment("max_tcp", "max_sessions"); - tmpval = parse_int_option("max_sessions", arg_stream, false); + int val; + if (arg_stream >> val) + tcp_max = val; } - table_api.close_table(); } else if (keyword == "tcp_cache_nominal_timeout") { table_api.open_table("tcp_cache"); - table_api.add_diff_option_comment("tcp_cache_nominal_timeout", "pruning_timeout"); - tmpval = parse_int_option("pruning_timeout", arg_stream, false); + table_api.add_diff_option_comment("tcp_cache_nominal_timeout", "idle_timeout"); + tmpval = parse_int_option("idle_timeout", arg_stream, false); table_api.close_table(); } else if (keyword == "tcp_cache_pruning_timeout") { - table_api.open_table("tcp_cache"); - table_api.add_diff_option_comment("tcp_cache_pruning_timeout", "idle_timeout"); - tmpval = parse_int_option("idle_timeout", arg_stream, false); - table_api.close_table(); + int val; + if (arg_stream >> val) + { + if (pruning_timeout > val) + pruning_timeout = val; + } } else if (keyword == "max_udp") { - table_api.open_table("udp_cache"); if (cv.do_convert_max_session()) { - table_api.add_diff_option_comment("max_udp","max_sessions"); - tmpval = parse_int_option("max_sessions", arg_stream, false); + int val; + if (arg_stream >> val) + udp_max = val; } - table_api.close_table(); } else if (keyword == "udp_cache_pruning_timeout") { - table_api.open_table("udp_cache"); - table_api.add_diff_option_comment("udp_cache_pruning_timeout","pruning_timeout"); - tmpval = parse_int_option("pruning_timeout", arg_stream, false); - table_api.close_table(); + int val; + if (arg_stream >> val) + { + if (pruning_timeout > val) + pruning_timeout = val; + } } else if (keyword == "udp_cache_nominal_timeout") { @@ -135,23 +140,21 @@ bool StreamGlobal::convert(std::istringstream& data_stream) } else if (keyword == "max_icmp") { - table_api.open_table("icmp_cache"); if (cv.do_convert_max_session()) { - table_api.add_diff_option_comment("max_icmp","max_sessions"); - tmpval = parse_int_option("max_sessions", arg_stream, false); + int val; + if (arg_stream >> val) + icmp_max = val; } - table_api.close_table(); } else if (keyword == "max_ip") { - table_api.open_table("ip_cache"); if (cv.do_convert_max_session()) { - table_api.add_diff_option_comment("max_ip","max_sessions"); - tmpval = parse_int_option("max_sessions", arg_stream, false); + int val; + if (arg_stream >> val) + ip_max = val; } - table_api.close_table(); } else if (keyword == "show_rebuilt_packets") { @@ -187,7 +190,8 @@ bool StreamGlobal::convert(std::istringstream& data_stream) retval = false; } } - + table_api.add_option("max_flows", tcp_max + udp_max + icmp_max + ip_max); + table_api.add_option("pruning_timeout", INT_MAX == pruning_timeout ? 30 : pruning_timeout); return retval; }