From: Shravan Rangarajuvenkata (shrarang) Date: Wed, 6 Jan 2021 19:05:49 +0000 (+0000) Subject: Merge pull request #2687 in SNORT/snort3 from ~SHRARANG/snort3:appid_test_cleanup... X-Git-Tag: 3.1.0.0~8 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2311c3a47e88158f7e0aa5b1579f0d612f9d20f3;p=thirdparty%2Fsnort3.git Merge pull request #2687 in SNORT/snort3 from ~SHRARANG/snort3:appid_test_cleanup to master Squashed commit of the following: commit 4110a15eb824ce2ef4b4535ce7dae21ed831931b Author: Shravan Rangaraju Date: Sat Jan 2 06:50:48 2021 -0500 appid: store stats in map --- diff --git a/src/network_inspectors/appid/CMakeLists.txt b/src/network_inspectors/appid/CMakeLists.txt index 0d0f37a85..20431ae59 100644 --- a/src/network_inspectors/appid/CMakeLists.txt +++ b/src/network_inspectors/appid/CMakeLists.txt @@ -139,8 +139,6 @@ set ( DP_APPID_SOURCES ) set ( UTIL_APPID_SOURCES - appid_utils/fw_avltree.cc - appid_utils/fw_avltree.h appid_utils/ip_funcs.h appid_utils/sf_mlmp.cc appid_utils/sf_mlmp.h diff --git a/src/network_inspectors/appid/appid_stats.cc b/src/network_inspectors/appid/appid_stats.cc index fe2065b9f..80f49c0b2 100644 --- a/src/network_inspectors/appid/appid_stats.cc +++ b/src/network_inspectors/appid/appid_stats.cc @@ -40,64 +40,49 @@ using namespace snort; #define URLCATBUCKETS 100 #define URLREPBUCKETS 5 -struct AppIdStatRecord -{ - char* app_name = nullptr; - uint64_t initiatorBytes; - uint64_t responderBytes; -}; - static const char appid_stats_filename[] = "appid_stats.log"; static THREAD_LOCAL AppIdStatistics* appid_stats_manager = nullptr; -static void delete_record(void* record) -{ - snort_free(((AppIdStatRecord*)record)->app_name); - snort_free(record); -} - void AppIdStatistics::end_stats_period() { - SF_LIST* bucketList = logBuckets; - logBuckets = currBuckets; - currBuckets = bucketList; + SF_LIST* bucketList = log_buckets; + log_buckets = curr_buckets; + curr_buckets = bucketList; } -StatsBucket* AppIdStatistics::get_stats_bucket(time_t startTime) +StatsBucket* AppIdStatistics::get_stats_bucket(time_t start_time) { StatsBucket* bucket = nullptr; - if ( !currBuckets ) - currBuckets = sflist_new(); + if ( !curr_buckets ) + curr_buckets = sflist_new(); - SF_LNODE* lNode = nullptr; - StatsBucket* lBucket = nullptr; + SF_LNODE* l_node = nullptr; + StatsBucket* l_bucket = nullptr; - for ( lBucket = (StatsBucket*)sflist_first(currBuckets, &lNode); lNode && lBucket; - lBucket = (StatsBucket*)sflist_next(&lNode) ) + for ( l_bucket = (StatsBucket*)sflist_first(curr_buckets, &l_node); l_node && l_bucket; + l_bucket = (StatsBucket*)sflist_next(&l_node) ) { - if (startTime == lBucket->startTime) + if (start_time == l_bucket->start_time) { - bucket = lBucket; + bucket = l_bucket; break; } - else if (startTime < lBucket->startTime) + else if (start_time < l_bucket->start_time) { - bucket = (StatsBucket*)snort_calloc(sizeof(StatsBucket)); - bucket->startTime = startTime; - bucket->appsTree = fwAvlInit(); - sflist_add_before(currBuckets, lNode, bucket); + bucket = new StatsBucket; + bucket->start_time = start_time; + sflist_add_before(curr_buckets, l_node, bucket); break; } } - if ( !lNode ) + if ( !l_node ) { - bucket = (StatsBucket*)snort_calloc(sizeof(StatsBucket)); - bucket->startTime = startTime; - bucket->appsTree = fwAvlInit(); - sflist_add_tail(currBuckets, bucket); + bucket = new StatsBucket; + bucket->start_time = start_time; + sflist_add_tail(curr_buckets, bucket); } return bucket; @@ -105,12 +90,12 @@ StatsBucket* AppIdStatistics::get_stats_bucket(time_t startTime) void AppIdStatistics::open_stats_log_file() { - log = TextLog_Init(appid_stats_filename, 4096, rollSize); + log = TextLog_Init(appid_stats_filename, 4096, roll_size); } void AppIdStatistics::dump_statistics() { - if ( !logBuckets ) + if ( !log_buckets ) return; if ( !log ) @@ -118,25 +103,20 @@ void AppIdStatistics::dump_statistics() struct StatsBucket* bucket = nullptr; - while ((bucket = (struct StatsBucket*)sflist_remove_head(logBuckets)) != nullptr) + while ((bucket = (struct StatsBucket*)sflist_remove_head(log_buckets)) != nullptr) { - if ( bucket->appRecordCnt ) + if ( bucket->app_record_cnt ) { - struct FwAvlNode* node; - - for (node = fwAvlFirst(bucket->appsTree); node != nullptr; node = fwAvlNext(node)) + for (auto it : bucket->apps_tree) { - struct AppIdStatRecord* record; - - record = (struct AppIdStatRecord*)node->data; + struct AppIdStatRecord& record = it.second; // FIXIT-M %lu won't do time_t on 32-bit systems TextLog_Print(log, "%lu,%s," STDu64 "," STDu64 "\n", - packet_time(), record->app_name, record->initiatorBytes, record->responderBytes); + packet_time(), record.app_name.c_str(), record.initiator_bytes, record.responder_bytes); } } - fwAvlDeleteTree(bucket->appsTree, delete_record); - snort_free(bucket); + delete bucket; } } @@ -144,8 +124,8 @@ AppIdStatistics::AppIdStatistics(const AppIdConfig& config) { enabled = true; - rollSize = config.app_stats_rollover_size; - bucketInterval = config.app_stats_period; + roll_size = config.app_stats_rollover_size; + bucket_interval = config.app_stats_period; time_t now = get_time(); start_stats_period(now); @@ -163,17 +143,16 @@ AppIdStatistics::~AppIdStatistics() if ( log ) TextLog_Term(log); - if ( logBuckets ) - snort_free(logBuckets); + if ( log_buckets ) + snort_free(log_buckets); - if ( currBuckets ) + if ( curr_buckets ) { - while (auto bucket = (StatsBucket*)sflist_remove_head(currBuckets)) + while (auto bucket = (StatsBucket*)sflist_remove_head(curr_buckets)) { - fwAvlDeleteTree(bucket->appsTree, delete_record); - snort_free(bucket); + delete bucket; } - snort_free(currBuckets); + snort_free(curr_buckets); } } @@ -194,44 +173,40 @@ void AppIdStatistics::cleanup() static void update_stats(const AppIdSession& asd, AppId app_id, StatsBucket* bucket) { - AppIdStatRecord* record = (AppIdStatRecord*)(fwAvlLookup(app_id, bucket->appsTree)); - if ( !record ) + auto it = bucket->apps_tree.find(app_id); + if ( it == bucket->apps_tree.end() ) { - char tmp_buff[MAX_EVENT_APPNAME_LEN]; bool cooked_client = false; - record = (AppIdStatRecord*)(snort_calloc(sizeof(struct AppIdStatRecord))); - if ( app_id >= 2000000000 ) cooked_client = true; // Skip stats for sessions using old odp context after reload detectors if (!pkt_thread_odp_ctxt or (pkt_thread_odp_ctxt->get_version() != asd.get_odp_ctxt_version())) - { - snort_free(record); return; - } OdpContext& odp_ctxt = asd.get_odp_ctxt(); AppInfoTableEntry* entry = odp_ctxt.get_app_info_mgr().get_app_info_entry(app_id); + const char* app_name; + char tmp_buff[MAX_EVENT_APPNAME_LEN]; if ( entry ) { if (cooked_client) { snprintf(tmp_buff, MAX_EVENT_APPNAME_LEN, "_cl_%s", entry->app_name); - tmp_buff[MAX_EVENT_APPNAME_LEN-1] = 0; - record->app_name = snort_strdup(tmp_buff); + tmp_buff[MAX_EVENT_APPNAME_LEN-1] = '\0'; + app_name = tmp_buff; } else - record->app_name = snort_strdup(entry->app_name); + app_name = entry->app_name; } else if ( app_id == APP_ID_UNKNOWN ) - record->app_name = snort_strdup("__unknown"); + app_name = "__unknown"; else if ( app_id == APP_ID_NONE ) - record->app_name = snort_strdup("__none"); + app_name = "__none"; else { if (cooked_client) @@ -239,26 +214,19 @@ static void update_stats(const AppIdSession& asd, AppId app_id, StatsBucket* buc else snprintf(tmp_buff, MAX_EVENT_APPNAME_LEN, "_err_%d",app_id); - tmp_buff[MAX_EVENT_APPNAME_LEN - 1] = 0; - record->app_name = snort_strdup(tmp_buff); + tmp_buff[MAX_EVENT_APPNAME_LEN - 1] = '\0'; + app_name = tmp_buff; } - if (fwAvlInsert(app_id, record, bucket->appsTree) == 0) - { - bucket->appRecordCnt += 1; - } - else - { - WarningMessage("Error saving statistics record for app id: %d", app_id); - snort_free(record); - record = nullptr; - } + bucket->apps_tree.emplace(app_id, AppIdStatRecord(app_name, asd.stats.initiator_bytes, + asd.stats.responder_bytes)); + bucket->app_record_cnt += 1; } - - if ( record ) + else { - record->initiatorBytes += asd.stats.initiator_bytes; - record->responderBytes += asd.stats.responder_bytes; + auto& record = it->second; + record.initiator_bytes += asd.stats.initiator_bytes; + record.responder_bytes += asd.stats.responder_bytes; } } @@ -266,7 +234,7 @@ void AppIdStatistics::update(const AppIdSession& asd) { time_t now = get_time(); - if ( now >= bucketEnd ) + if ( now >= bucket_end ) { end_stats_period(); dump_statistics(); @@ -274,14 +242,14 @@ void AppIdStatistics::update(const AppIdSession& asd) } time_t bucketTime = asd.stats.first_packet_second - - (asd.stats.first_packet_second % bucketInterval); + (asd.stats.first_packet_second % bucket_interval); StatsBucket* bucket = get_stats_bucket(bucketTime); if ( !bucket ) return; - bucket->totalStats.txByteCnt += asd.stats.initiator_bytes; - bucket->totalStats.rxByteCnt += asd.stats.responder_bytes; + bucket->totalStats.tx_byte_cnt += asd.stats.initiator_bytes; + bucket->totalStats.rx_byte_cnt += asd.stats.responder_bytes; AppId web_app_id, service_id, client_id; asd.get_api().get_first_stream_app_ids(service_id, client_id, web_app_id); @@ -304,7 +272,7 @@ void AppIdStatistics::flush() return; time_t now = get_time(); - if (now >= bucketEnd) + if (now >= bucket_end) { end_stats_period(); dump_statistics(); diff --git a/src/network_inspectors/appid/appid_stats.h b/src/network_inspectors/appid/appid_stats.h index 51faeb89c..881b743c5 100644 --- a/src/network_inspectors/appid/appid_stats.h +++ b/src/network_inspectors/appid/appid_stats.h @@ -24,23 +24,37 @@ #include #include +#include -#include "appid_utils/fw_avltree.h" #include "utils/sflsq.h" +#include "utils/util.h" + +#include "application_ids.h" class AppIdSession; class AppIdConfig; +struct AppIdStatRecord +{ + std::string app_name; + uint64_t initiator_bytes = 0; + uint64_t responder_bytes = 0; + + AppIdStatRecord(const char* app_name, uint64_t initiator_bytes, uint64_t responder_bytes) : + app_name(app_name), initiator_bytes(initiator_bytes), responder_bytes(responder_bytes) + { } +}; + struct StatsBucket { - uint32_t startTime; - FwAvlTree* appsTree; + uint32_t start_time = 0; + std::map apps_tree; struct { - size_t txByteCnt; - size_t rxByteCnt; + size_t tx_byte_cnt = 0; + size_t rx_byte_cnt = 0; } totalStats; - uint32_t appRecordCnt; + uint32_t app_record_cnt = 0; }; class AppIdStatistics @@ -60,13 +74,13 @@ private: time_t get_time() { auto now = time(nullptr); - return now - (now % bucketInterval); + return now - (now % bucket_interval); } - void start_stats_period(time_t startTime) + void start_stats_period(time_t start_time) { - bucketStart = startTime; - bucketEnd = bucketStart + bucketInterval; + bucket_start = start_time; + bucket_end = bucket_start + bucket_interval; } void end_stats_period(); @@ -75,14 +89,13 @@ private: void dump_statistics(); bool enabled = false; - SF_LIST* currBuckets = nullptr; - SF_LIST* logBuckets = nullptr; + SF_LIST* curr_buckets = nullptr; + SF_LIST* log_buckets = nullptr; struct TextLog* log = nullptr; - time_t bucketStart = 0; - time_t bucketInterval = 0; - time_t bucketEnd = 0; - size_t rollSize = 0; + time_t bucket_start = 0; + time_t bucket_interval = 0; + time_t bucket_end = 0; + size_t roll_size = 0; }; #endif - diff --git a/src/network_inspectors/appid/appid_utils/fw_avltree.cc b/src/network_inspectors/appid/appid_utils/fw_avltree.cc deleted file mode 100644 index 62d96165f..000000000 --- a/src/network_inspectors/appid/appid_utils/fw_avltree.cc +++ /dev/null @@ -1,393 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2005-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// fw_avltree.cc author Sourcefire Inc. - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "fw_avltree.h" - -#include "utils/util.h" - -static inline int is_root(FwAvlNode* node) { return (node->parent == nullptr); } -static inline int get_balance(FwAvlNode* node) { return node->balance; } -static inline void set_balance(int balance, FwAvlNode* node) { node->balance = balance; } -static inline int inc_balance(FwAvlNode* node) { return ++node->balance; } -static inline int dec_balance(FwAvlNode* node) { return --node->balance; } -static inline FwAvlNode* get_parent(FwAvlNode* node) { return node->parent; } - -static inline void set_parent(FwAvlNode* parent, FwAvlNode* node) -{ - node->parent = parent; -} - -static inline FwAvlNode* new_node(uint32_t key, void* data) -{ - FwAvlNode* node = (FwAvlNode*)snort_calloc(sizeof(FwAvlNode)); - - node->key = key; - node->data = data; - return node; -} - -static inline FwAvlNode* get_first(FwAvlNode* node) -{ - while (node->left != nullptr) - node = node->left; - return node; -} - -FwAvlNode* fwAvlFirst(const FwAvlTree* tree) -{ - if ((tree != nullptr) && (tree->root != nullptr)) - return get_first(tree->root); - else - return nullptr; -} - -FwAvlNode* fwAvlNext(FwAvlNode* node) -{ - - if (node->right != nullptr) - { - return get_first(node->right); - } - else - { - FwAvlNode* parent = nullptr; - FwAvlNode* tmp = node; - while ( ((parent = get_parent(tmp)) != nullptr) && (parent->right == tmp) ) - tmp = parent; - - return parent; - } -} - -static void rotate_left(FwAvlNode* node, FwAvlTree* tree) -{ - FwAvlNode* p = node; - FwAvlNode* q = node->right; - FwAvlNode* parent = get_parent(node); - - if (!is_root(p)) - { - if (parent->left == p) - parent->left = q; - else - parent->right = q; - } - else - { - tree->root = q; - } - - set_parent(parent, q); - set_parent(q, p); - - p->right = q->left; - if (p->right != nullptr) - { - set_parent(p, p->right); - } - q->left = p; -} - -static void rotate_right(FwAvlNode* node, FwAvlTree* tree) -{ - FwAvlNode* p = node; - FwAvlNode* q = node->left; - FwAvlNode* parent = get_parent(node); - - if (!is_root(p)) - { - if (parent->left == p) - parent->left = q; - else - parent->right = q; - } - else - { - tree->root = q; - } - - set_parent(parent, q); - set_parent(q, p); - - p->left = q->right; - if (p->left != nullptr) - { - set_parent(p, p->left); - } - q->right = p; -} - -static inline FwAvlNode* do_lookup(const uint32_t key, - const FwAvlTree* tree, FwAvlNode** pparent, - FwAvlNode** unbalanced, int* is_left) -{ - FwAvlNode* node = tree->root; - - *pparent = nullptr; - *unbalanced = node; - *is_left = 0; - - while (node != nullptr) - { - if (get_balance(node) != 0) - { - *unbalanced = node; - } - - *pparent = node; - - if (key == node->key) - { - return node; - } - else - { - if ((*is_left = node->key > key) != 0) - node = node->left; - else - node = node->right; - } - } - return nullptr; -} - -void* fwAvlLookup(const uint32_t key, const FwAvlTree* tree) -{ - FwAvlNode* node = nullptr; - FwAvlNode* pparent; - FwAvlNode* unbalanced; - int is_left; - - if (tree == nullptr) - { - return nullptr; - } - - node = do_lookup(key, tree, &pparent, &unbalanced, &is_left); - - if (node != nullptr) - { - return node->data; - } - - return nullptr; -} - -static inline void set_child(FwAvlNode* child, FwAvlNode* node, - int left) -{ - if (left != 0) - node->left = child; - else - node->right = child; -} - -int fwAvlInsert(uint32_t key, void* data, FwAvlTree* tree) -{ - int is_left; - FwAvlNode* parent; - FwAvlNode* right; - FwAvlNode* left; - FwAvlNode* unbalanced; - FwAvlNode* node; - - if (do_lookup(key, tree, &parent, &unbalanced, &is_left) != nullptr) - return 1; - - if (!(node = new_node(key, data))) - return -1; - - tree->count++; - if (parent == nullptr) - { - tree->root = node; - tree->first = node; - tree->last = node; - return 0; - } - - if (is_left != 0) - { - if (parent == tree->first) - tree->first = node; - } - else - { - if (parent == tree->last) - tree->last = node; - } - set_parent(parent, node); - set_child(node, parent, is_left); - - for (;; ) - { - if (parent->left == node) - dec_balance(parent); - else - inc_balance(parent); - - if (parent == unbalanced) - break; - - node = parent; - parent = get_parent(node); - } - - switch (get_balance(unbalanced)) - { - case 1: - case -1: - tree->height++; - break; - case 0: - break; - case 2: - right = unbalanced->right; - - if (get_balance(right) == 1) - { - set_balance(0, unbalanced); - set_balance(0, right); - } - else - { - switch (get_balance(right->left)) - { - case 1: - set_balance(-1, unbalanced); - set_balance(0, right); - break; - case 0: - set_balance(0, unbalanced); - set_balance(0, right); - break; - case -1: - set_balance(0, unbalanced); - set_balance(1, right); - break; - } - set_balance(0, right->left); - rotate_right(right, tree); - } - rotate_left(unbalanced, tree); - break; - case -2: - left = unbalanced->left; - - if (get_balance(left) == -1) - { - set_balance(0, unbalanced); - set_balance(0, left); - } - else - { - switch (get_balance(left->right)) - { - case 1: - set_balance(0, unbalanced); - set_balance(-1, left); - break; - case 0: - set_balance(0, unbalanced); - set_balance(0, left); - break; - case -1: - set_balance(1, unbalanced); - set_balance(0, left); - break; - } - set_balance(0, left->right); - rotate_left(left, tree); - } - rotate_right(unbalanced, tree); - break; - } - return 0; -} - -FwAvlTree* fwAvlInit() -{ - return (FwAvlTree*)snort_calloc(sizeof(FwAvlTree)); -} - -static FwQNode* newFwQNode(FwAvlNode* treeNode) -{ - FwQNode* q_node = (FwQNode*)snort_calloc(sizeof(FwQNode)); - - q_node->treeNode = treeNode; - q_node->next = nullptr; - return(q_node); -} - -FwQNode* fwAvlSerialize(FwAvlTree* tree) -{ - FwQNode* head; - FwQNode* node; - FwQNode* tail; - - if ((tree == nullptr) || (tree->root == nullptr)) - return nullptr; - - head = newFwQNode(tree->root); - node = head; - tail = head; - - while (node) - { - if (node->treeNode->left != nullptr) - { - tail->next = newFwQNode(node->treeNode->left); - tail = tail->next; - } - - if (node->treeNode->right != nullptr) - { - tail->next = newFwQNode(node->treeNode->right); - tail = tail->next; - } - - node = node->next; - } - return head; -} - -void fwAvlDeleteTree(FwAvlTree* tree, void (* dataDelete)(void* data)) -{ - FwQNode* node = fwAvlSerialize(tree); - - while (node != nullptr) - { - if (dataDelete) - dataDelete(node->treeNode->data); - - snort_free(node->treeNode); - - FwQNode* tmp = node; - node = node->next; - snort_free(tmp); - } - snort_free(tree); -} - diff --git a/src/network_inspectors/appid/appid_utils/fw_avltree.h b/src/network_inspectors/appid/appid_utils/fw_avltree.h deleted file mode 100644 index d8bb4d45f..000000000 --- a/src/network_inspectors/appid/appid_utils/fw_avltree.h +++ /dev/null @@ -1,62 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2005-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// fw_avltree.h author Sourcefire Inc. - -#ifndef FW_AVL_TREE_H -#define FW_AVL_TREE_H - -#include -#include - -struct FwAvlNode -{ - uint32_t key; - void* data; - int balance; - FwAvlNode* left; - FwAvlNode* right; - FwAvlNode* parent; -}; - -struct FwAvlTree -{ - unsigned count; - size_t height; - FwAvlNode* root; - FwAvlNode* first; - FwAvlNode* last; -}; - -struct FwQNode -{ - FwAvlNode* treeNode; - FwQNode* next; -}; - -FwAvlTree* fwAvlInit(); -int fwAvlInsert(uint32_t key, void* data, FwAvlTree*); -void* fwAvlLookup(const uint32_t key, const FwAvlTree*); -FwAvlNode* fwAvlFirst(const FwAvlTree*); -FwAvlNode* fwAvlNext(FwAvlNode*); -FwQNode* fwAvlSerialize(FwAvlTree*); -void fwAvlDeleteTree(FwAvlTree*, void (* dataDelete)(void* data)); - -#endif -