From: Mike Stepanek (mstepane) <mstepane@cisco.com>
Date: Tue, 8 Oct 2019 18:17:05 +0000 (-0400)
Subject: Merge pull request #1735 in SNORT/snort3 from ~DAVMCPHE/snort3:flow_object_allocation... 
X-Git-Tag: 3.0.0-262~3
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0e014a88133252ec5a3b65f4f615d150b4aa101a;p=thirdparty%2Fsnort3.git

Merge pull request #1735 in SNORT/snort3 from ~DAVMCPHE/snort3:flow_object_allocation to master

Squashed commit of the following:

commit 3b8ffbfb453e155f805ec859198ca08b945d0cdf
Author: davis mcpherson <davmcphe.cisco.com>
Date:   Wed Sep 4 10:57:08 2019 -0400

    flow: patch to allocate Flow objects individually on demand. Once allocated the Flow objects are reused until snort exits or reload changes the max_flows setting
---

diff --git a/src/flow/expect_cache.cc b/src/flow/expect_cache.cc
index 1bc043e64..73e12bb69 100644
--- a/src/flow/expect_cache.cc
+++ b/src/flow/expect_cache.cc
@@ -323,10 +323,8 @@ ExpectCache::~ExpectCache()
  * snort_protocol_id and associated data is not stored.
  *
  */
-int ExpectCache::add_flow(const Packet *ctrlPkt,
-    PktType type, IpProtocol ip_proto,
-    const SfIp* cliIP, uint16_t cliPort,
-    const SfIp* srvIP, uint16_t srvPort,
+int ExpectCache::add_flow(const Packet *ctrlPkt, PktType type, IpProtocol ip_proto,
+    const SfIp* cliIP, uint16_t cliPort, const SfIp* srvIP, uint16_t srvPort,
     char direction, FlowData* fd, SnortProtocolId snort_protocol_id)
 {
     /* Just pull the VLAN ID, MPLS ID, and Address Space ID from the
@@ -339,11 +337,8 @@ int ExpectCache::add_flow(const Packet *ctrlPkt,
     bool reversed_key = key.init(type, ip_proto, cliIP, cliPort, srvIP, srvPort,
             vlanId, mplsId, addressSpaceId);
 
-    ExpectNode* node;
-    ExpectFlow* last;
     bool new_node = false;
-
-    node = (ExpectNode*) hash_table->get(&key, &new_node);
+    ExpectNode* node = (ExpectNode*) hash_table->get(&key, &new_node);
     if (!node)
     {
         prune();
@@ -365,6 +360,7 @@ int ExpectCache::add_flow(const Packet *ctrlPkt,
         new_node = true;
     }
 
+    ExpectFlow* last = nullptr;
     if (!new_node)
     {
         // Requests will be rejected if the snort_protocol_id doesn't
diff --git a/src/flow/flow.h b/src/flow/flow.h
index 86a8765ca..218dfbb50 100644
--- a/src/flow/flow.h
+++ b/src/flow/flow.h
@@ -346,7 +346,6 @@ public:  // FIXIT-M privatize if possible
 
     // these fields are const after initialization
     const FlowKey* key;
-    Session* session;
     BitOp* bitop;
     FlowHAState* ha_state;
     FlowStash* stash;
@@ -356,6 +355,7 @@ public:  // FIXIT-M privatize if possible
 
     // these fields are always set; not zeroed
     Flow* prev, * next;
+    Session* session;
     Inspector* ssn_client;
     Inspector* ssn_server;
 
diff --git a/src/flow/flow_cache.cc b/src/flow/flow_cache.cc
index 28f332fa4..785668c49 100644
--- a/src/flow/flow_cache.cc
+++ b/src/flow/flow_cache.cc
@@ -70,6 +70,7 @@ void FlowCache::push(Flow* flow)
 {
     void* key = hash_table->push(flow);
     flow->key = (FlowKey*)key;
+    ++flows_allocated;
 }
 
 unsigned FlowCache::get_count()
@@ -110,14 +111,19 @@ void FlowCache::unlink_uni(Flow* flow)
         uni_flows->unlink_uni(flow);
 }
 
-Flow* FlowCache::get(const FlowKey* key)
+Flow* FlowCache::allocate(const FlowKey* key)
 {
     time_t timestamp = packet_time();
     Flow* flow = (Flow*)hash_table->get(key);
 
     if ( !flow )
     {
-        if ( !prune_stale(timestamp, nullptr) )
+        if ( flows_allocated < config.max_flows )
+        {
+            Flow* new_flow = new Flow;
+            push(new_flow);
+        }
+        else if ( !prune_stale(timestamp, nullptr) )
         {
             if ( !prune_unis(key->pkt_type) )
                 prune_excess(nullptr);
@@ -130,9 +136,9 @@ Flow* FlowCache::get(const FlowKey* key)
             flow->term();
         else
             flow->reset();
-        link_uni(flow);
     }
 
+    link_uni(flow);
     if ( flow->session && flow->pkt_type != key->pkt_type )
         flow->term();
 
@@ -369,7 +375,7 @@ unsigned FlowCache::purge()
     }
 
     while ( Flow* flow = (Flow*)hash_table->pop() )
-        flow->term();
+        delete flow;
 
     return retired;
 }
diff --git a/src/flow/flow_cache.h b/src/flow/flow_cache.h
index b0393ee98..2a37d99e2 100644
--- a/src/flow/flow_cache.h
+++ b/src/flow/flow_cache.h
@@ -48,13 +48,10 @@ public:
     FlowCache(const FlowCache&) = delete;
     FlowCache& operator=(const FlowCache&) = delete;
 
-    void push(snort::Flow*);
-
     snort::Flow* find(const snort::FlowKey*);
-    snort::Flow* get(const snort::FlowKey*);
+    snort::Flow* allocate(const snort::FlowKey*);
 
     int release(snort::Flow*, PruneReason = PruneReason::NONE, bool do_cleanup = true);
-
     unsigned prune_stale(uint32_t thetime, const snort::Flow* save_me);
     unsigned prune_excess(const snort::Flow* save_me);
     bool prune_one(PruneReason, bool do_cleanup);
@@ -78,9 +75,11 @@ public:
     void unlink_uni(snort::Flow*);
 
 private:
+    void push(snort::Flow*);
     void link_uni(snort::Flow*);
     int remove(snort::Flow*);
     int retire(snort::Flow*);
+    unsigned prune_unis(PktType);
 
 private:
     static const unsigned cleanup_flows = 1;
@@ -88,12 +87,11 @@ private:
     uint32_t flags;
 
     class ZHash* hash_table;
+    unsigned flows_allocated = 0;
     FlowUniList* uni_flows;
     FlowUniList* uni_ip_flows;
 
-    //snort::Flow* uni_head_ip, *uni_tail_ip, *uni_head_nonip, *uni_tail_nonip;
     PruneStats prune_stats;
-    unsigned prune_unis(PktType);
 };
 #endif
 
diff --git a/src/flow/flow_control.cc b/src/flow/flow_control.cc
index d814d9d79..9a4b4477f 100644
--- a/src/flow/flow_control.cc
+++ b/src/flow/flow_control.cc
@@ -47,11 +47,6 @@ using namespace snort;
 FlowControl::FlowControl(const FlowCacheConfig& fc)
 {
     cache = new FlowCache(fc);
-
-    mem = (Flow*)snort_calloc(fc.max_flows, sizeof(Flow));
-
-    for ( unsigned i = 0; i < fc.max_flows; ++i )
-        cache->push(mem + i);
 }
 
 FlowControl::~FlowControl()
@@ -69,25 +64,18 @@ FlowControl::~FlowControl()
 
 PegCount FlowControl::get_total_prunes() const
 {
-    auto cache = get_cache();
-    return cache ? cache->get_total_prunes() : 0;
+    return cache->get_total_prunes();
 }
 
 PegCount FlowControl::get_prunes(PruneReason reason) const
 {
-    auto cache = get_cache();
-    return cache ? cache->get_prunes(reason) : 0;
+    return cache->get_prunes(reason);
 }
 
 void FlowControl::clear_counts()
 {
-    for ( int i = 0; i < to_utype(PktType::MAX); ++i )
-    {
-        if ( cache )
-            cache->reset_stats();
-
-        num_flows = 0;
-    }
+    cache->reset_stats();
+    num_flows = 0;
 }
 
 //-------------------------------------------------------------------------
@@ -96,65 +84,41 @@ void FlowControl::clear_counts()
 
 Flow* FlowControl::find_flow(const FlowKey* key)
 {
-    if ( auto cache = get_cache() )
-        return cache->find(key);
-
-    return nullptr;
+    return cache->find(key);
 }
 
 Flow* FlowControl::new_flow(const FlowKey* key)
 {
-    if ( auto cache = get_cache() )
-        return cache->get(key);
-
-    return nullptr;
+    return cache->allocate(key);
 }
 
-// FIXIT-L cache* can be put in flow so that lookups by
-// packet type are obviated for existing / initialized flows
 void FlowControl::delete_flow(const FlowKey* key)
 {
-    FlowCache* cache = get_cache();
-
-    if ( !cache )
-        return;
-
     if ( auto flow = cache->find(key) )
         cache->release(flow, PruneReason::HA);
 }
 
 void FlowControl::delete_flow(Flow* flow, PruneReason reason)
 {
-    if ( auto cache = get_cache() )
-        cache->release(flow, reason);
+    cache->release(flow, reason);
 }
 
 void FlowControl::purge_flows ()
 {
-    if ( auto cache = get_cache() )
-        cache->purge();
+    cache->purge();
 }
 
 // hole for memory manager/prune handler
 bool FlowControl::prune_one(PruneReason reason, bool do_cleanup)
 {
-    auto cache = get_cache();
-    return cache ? cache->prune_one(reason, do_cleanup) : false;
+    return cache->prune_one(reason, do_cleanup);
 }
 
 void FlowControl::timeout_flows(time_t cur_time)
 {
-    if ( types.empty() )
-        return;
-
     ActiveSuspendContext act_susp;
-    FlowCache* fc = get_cache();
-
-    if ( ++next >= types.size() )
-        next = 0;
 
-    if ( fc )
-        fc->timeout(1, cur_time);
+    cache->timeout(1, cur_time);
 }
 
 void FlowControl::preemptive_cleanup()
@@ -341,8 +305,7 @@ void FlowControl::init_proto(PktType type, InspectSsnFunc get_ssn)
 {
     assert(get_ssn);
 
-    proto[to_utype(type)].get_ssn = get_ssn;
-    types.emplace_back(type);
+    get_proto_session[to_utype(type)] = get_ssn;
 }
 
 // FIXIT-P apply more filtering logic here, eg require_3whs
@@ -367,7 +330,7 @@ static bool want_flow(PktType type, Packet* p)
 
 bool FlowControl::process(PktType type, Packet* p, bool* new_flow)
 {
-    if ( !proto[to_utype(type)].get_ssn )
+    if ( !get_proto_session[to_utype(type)] )
         return false;
 
     FlowKey key;
@@ -386,7 +349,7 @@ bool FlowControl::process(PktType type, Packet* p, bool* new_flow)
             if ( !want_flow(type, p) )
                 return true;
 
-            flow = cache->get(&key);
+            flow = cache->allocate(&key);
 
             if ( !flow )
                 return true;
@@ -399,7 +362,7 @@ bool FlowControl::process(PktType type, Packet* p, bool* new_flow)
     if ( !flow->session )
     {
         flow->init(type);
-        flow->session = proto[to_utype(type)].get_ssn(flow);
+        flow->session = get_proto_session[to_utype(type)](flow);
     }
 
     num_flows += process(flow, p);
diff --git a/src/flow/flow_control.h b/src/flow/flow_control.h
index 825733ff9..0bc1fd92d 100644
--- a/src/flow/flow_control.h
+++ b/src/flow/flow_control.h
@@ -95,31 +95,17 @@ public:
     void clear_counts();
 
 private:
-    FlowCache* get_cache()
-    { return cache; }
-
-    const FlowCache* get_cache() const
-    { return cache; }
-
     void set_key(snort::FlowKey*, snort::Packet*);
-
     unsigned process(snort::Flow*, snort::Packet*);
     void preemptive_cleanup();
 
 private:
-    struct
-    {
-        snort::InspectSsnFunc get_ssn = nullptr;
-    } proto[to_utype(PktType::MAX)];
-
+    snort::InspectSsnFunc get_proto_session[to_utype(PktType::MAX)] = {};
     PegCount num_flows = 0;
     FlowCache* cache = nullptr;
     snort::Flow* mem = nullptr;
     class ExpectCache* exp_cache = nullptr;
     PktType last_pkt_type = PktType::NONE;
-
-    std::vector<PktType> types;
-    unsigned next = 0;
 };
 
 #endif
diff --git a/src/flow/flow_uni_list.h b/src/flow/flow_uni_list.h
index fd04f747d..75718fa96 100644
--- a/src/flow/flow_uni_list.h
+++ b/src/flow/flow_uni_list.h
@@ -61,7 +61,6 @@ public:
         flow->prev->next = flow->next;
         flow->next = flow->prev = nullptr;
         --count;
-
     }
 
     snort::Flow* get_oldest_uni()
diff --git a/src/flow/test/flow_control_test.cc b/src/flow/test/flow_control_test.cc
index 5b5050c21..5d51af823 100644
--- a/src/flow/test/flow_control_test.cc
+++ b/src/flow/test/flow_control_test.cc
@@ -70,7 +70,7 @@ DetectionEngine::~DetectionEngine() = default;
 ExpectCache::~ExpectCache() = default;
 unsigned FlowCache::purge() { return 1; }
 Flow* FlowCache::find(const FlowKey*) { return nullptr; }
-Flow* FlowCache::get(const FlowKey*) { return nullptr; }
+Flow* FlowCache::allocate(const FlowKey*) { return nullptr; }
 void FlowCache::push(Flow*) { }
 bool FlowCache::prune_one(PruneReason, bool) { return true; }
 unsigned FlowCache::timeout(unsigned, time_t) { return 1; }