From: Sreeja Athirkandathil Narayanan (sathirka) Date: Fri, 28 Apr 2023 17:55:50 +0000 (+0000) Subject: Pull request #3803: appid: AppIdPegCounters thread fixes X-Git-Tag: 3.1.61.0~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=fc23ca37c90949c4125401e09c1c953ec10d74ae;p=thirdparty%2Fsnort3.git Pull request #3803: appid: AppIdPegCounters thread fixes Merge in SNORT/snort3 from ~OSTEPANO/snort3:appid_pegs_data_ref to master Squashed commit of the following: commit 49fedbbdbfb3e6e06a131f51aefec1603a1b3d83 Author: Oleksandr Stepanov Date: Fri Apr 7 12:02:51 2023 -0400 appid: AppIdPegCounters thread data handling refactored to prevent data races --- diff --git a/src/network_inspectors/appid/appid_peg_counts.cc b/src/network_inspectors/appid/appid_peg_counts.cc index 45c3e4b24..937eaf650 100644 --- a/src/network_inspectors/appid/appid_peg_counts.cc +++ b/src/network_inspectors/appid/appid_peg_counts.cc @@ -34,139 +34,165 @@ using namespace snort; -std::unordered_map AppIdPegCounts::appid_detector_pegs_idx; -std::vector AppIdPegCounts::appid_detectors_info; -THREAD_LOCAL std::vector* AppIdPegCounts::appid_peg_counts; +THREAD_LOCAL AppIdPegCounts::AppIdDynamicPeg* AppIdPegCounts::unknown_appids_peg; +THREAD_LOCAL std::unordered_map* AppIdPegCounts::appid_peg_counts; +std::unordered_map> AppIdPegCounts::appid_peg_ids; AppIdPegCounts::AppIdDynamicPeg AppIdPegCounts::appid_dynamic_sum[SF_APPID_MAX + 1]; AppIdPegCounts::AppIdDynamicPeg AppIdPegCounts::zeroed_peg; PegCount AppIdPegCounts::all_zeroed_peg[DetectorPegs::NUM_APPID_DETECTOR_PEGS] = {}; +static std::mutex dynamic_stats_mutex; + void AppIdPegCounts::init_pegs() { assert(!appid_peg_counts); - appid_peg_counts = new std::vector( - appid_detectors_info.size() + 1, zeroed_peg); + appid_peg_counts = new std::unordered_map(); + + for (auto& appid : appid_peg_ids) + { + appid_peg_counts->emplace(appid.first, zeroed_peg); + } + unknown_appids_peg = new AppIdDynamicPeg(zeroed_peg); } void AppIdPegCounts::cleanup_pegs() { delete appid_peg_counts; appid_peg_counts = nullptr; + delete unknown_appids_peg; + unknown_appids_peg = nullptr; } void AppIdPegCounts::cleanup_peg_info() { - appid_detectors_info.clear(); - appid_detector_pegs_idx.clear(); + appid_peg_ids.clear(); } void AppIdPegCounts::cleanup_dynamic_sum() { - for (unsigned app_num = 0; app_num < AppIdPegCounts::appid_detectors_info.size(); app_num++) + const std::lock_guard _lock(dynamic_stats_mutex); + + for (unsigned app_num = 0; app_num < SF_APPID_MAX; app_num++) { memset(appid_dynamic_sum[app_num].stats, 0, sizeof(PegCount) * DetectorPegs::NUM_APPID_DETECTOR_PEGS); - if ( appid_peg_counts ) - memset((*appid_peg_counts)[app_num].stats, 0, sizeof(PegCount) * - DetectorPegs::NUM_APPID_DETECTOR_PEGS); + } + + if (appid_peg_counts) + { + for (auto& peg : (*appid_peg_counts)) + { + memset(&peg.second.stats, 0, sizeof(PegCount) * DetectorPegs::NUM_APPID_DETECTOR_PEGS); + } } // reset unknown_app stats memset(appid_dynamic_sum[SF_APPID_MAX].stats, 0, sizeof(PegCount) * DetectorPegs::NUM_APPID_DETECTOR_PEGS); - if ( appid_peg_counts ) - memset((*appid_peg_counts)[appid_peg_counts->size() - 1].stats, 0, sizeof(PegCount) * - DetectorPegs::NUM_APPID_DETECTOR_PEGS); + if (unknown_appids_peg) + memset(&unknown_appids_peg->stats, 0, sizeof(PegCount) * DetectorPegs::NUM_APPID_DETECTOR_PEGS); + } void AppIdPegCounts::add_app_peg_info(std::string app_name, AppId app_id) { std::replace(app_name.begin(), app_name.end(), ' ', '_'); - appid_detector_pegs_idx[app_id] = appid_detectors_info.size(); - appid_detectors_info.emplace_back(app_name); + appid_peg_ids.emplace(app_id, std::make_pair(app_name, appid_peg_ids.size())); } void AppIdPegCounts::sum_stats() { - if (!appid_peg_counts) + if (!appid_peg_counts or !unknown_appids_peg) return; - static std::mutex r_mutex; + const std::lock_guard _lock(dynamic_stats_mutex); + for (auto& peg : (*appid_peg_counts)) { - std::lock_guard _lock(r_mutex); - const unsigned peg_num = appid_peg_counts->size() ? (appid_peg_counts->size() - 1) : 0; - AppIdDynamicPeg* ptr = (AppIdDynamicPeg*)appid_peg_counts->data(); + auto dyn_indx = appid_peg_ids[peg.first].second; + for (unsigned j = 0; j < DetectorPegs::NUM_APPID_DETECTOR_PEGS; ++j) + appid_dynamic_sum[dyn_indx].stats[j] += peg.second.stats[j]; - for (unsigned i = 0; i < peg_num; ++i) - { - for (unsigned j = 0; j < DetectorPegs::NUM_APPID_DETECTOR_PEGS; ++j) - appid_dynamic_sum[i].stats[j] += ptr[i].stats[j]; + peg.second.zero_out(); + } - ptr[i].zero_out(); - } + // unknown_app stats + for (unsigned j = 0; j < DetectorPegs::NUM_APPID_DETECTOR_PEGS; ++j) + appid_dynamic_sum[SF_APPID_MAX].stats[j] += unknown_appids_peg->stats[j]; - // unknown_app stats - for (unsigned j = 0; j < DetectorPegs::NUM_APPID_DETECTOR_PEGS; ++j) - appid_dynamic_sum[SF_APPID_MAX].stats[j] += ptr[peg_num].stats[j]; - ptr[peg_num].zero_out(); - } - + unknown_appids_peg->zero_out(); } void AppIdPegCounts::inc_service_count(AppId id) { - (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::SERVICE_DETECTS]++; + auto peg = appid_peg_counts->find(id); + if (peg != appid_peg_counts->end()) + peg->second.stats[DetectorPegs::SERVICE_DETECTS]++; + else + unknown_appids_peg->stats[DetectorPegs::SERVICE_DETECTS]++; } void AppIdPegCounts::inc_client_count(AppId id) { - (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::CLIENT_DETECTS]++; + auto peg = appid_peg_counts->find(id); + if (peg != appid_peg_counts->end()) + peg->second.stats[DetectorPegs::CLIENT_DETECTS]++; + else + unknown_appids_peg->stats[DetectorPegs::CLIENT_DETECTS]++; } void AppIdPegCounts::inc_payload_count(AppId id) { - (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::PAYLOAD_DETECTS]++; + auto peg = appid_peg_counts->find(id); + if (peg != appid_peg_counts->end()) + peg->second.stats[DetectorPegs::PAYLOAD_DETECTS]++; + else + unknown_appids_peg->stats[DetectorPegs::PAYLOAD_DETECTS]++; } void AppIdPegCounts::inc_user_count(AppId id) { - (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::USER_DETECTS]++; + auto peg = appid_peg_counts->find(id); + if (peg != appid_peg_counts->end()) + peg->second.stats[DetectorPegs::USER_DETECTS]++; + else + unknown_appids_peg->stats[DetectorPegs::USER_DETECTS]++; } void AppIdPegCounts::inc_misc_count(AppId id) { - (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::MISC_DETECTS]++; + auto peg = appid_peg_counts->find(id); + if (peg != appid_peg_counts->end()) + peg->second.stats[DetectorPegs::MISC_DETECTS]++; + else + unknown_appids_peg->stats[DetectorPegs::MISC_DETECTS]++; } void AppIdPegCounts::inc_referred_count(AppId id) { - (*appid_peg_counts)[get_stats_index(id)].stats[DetectorPegs::REFERRED_DETECTS]++; -} - -uint32_t AppIdPegCounts::get_stats_index(AppId id) -{ - std::unordered_map::iterator stats_idx_it = appid_detector_pegs_idx.find(id); - if ( stats_idx_it != appid_detector_pegs_idx.end() ) - return stats_idx_it->second; + auto peg = appid_peg_counts->find(id); + if (peg != appid_peg_counts->end()) + peg->second.stats[DetectorPegs::REFERRED_DETECTS]++; else - return appid_detectors_info.size(); + unknown_appids_peg->stats[DetectorPegs::REFERRED_DETECTS]++; } void AppIdPegCounts::print() { bool print = false; - unsigned app_num = AppIdPegCounts::appid_detectors_info.size(); + unsigned app_num = AppIdPegCounts::appid_peg_ids.size(); + + std::unordered_map tmp_sorting_map; - for (unsigned i = 0; i < app_num; i++) + for (auto& det_info : AppIdPegCounts::appid_peg_ids) { - AppIdDynamicPeg* pegs = &appid_dynamic_sum[i]; + AppIdDynamicPeg* pegs = &appid_dynamic_sum[det_info.second.second]; if (!pegs->all_zeros()) { - print = true; - break; + print |= true; } + tmp_sorting_map.emplace(det_info.second.second, &det_info.second.first); } AppIdDynamicPeg* unknown_pegs = &appid_dynamic_sum[SF_APPID_MAX]; @@ -181,14 +207,13 @@ void AppIdPegCounts::print() "Application", "Services", "Clients", "Users", "Payloads", "Misc", "Referred"); LogText(buff); - for (unsigned i = 0; i < app_num; i++) + for (uint32_t i = 0; i < app_num; i++) { AppIdDynamicPeg* pegs = &appid_dynamic_sum[i]; if (pegs->all_zeros()) continue; - std::string app_name = AppIdPegCounts::appid_detectors_info[i]; - pegs->print(app_name.c_str(), buff, sizeof(buff)); + pegs->print(tmp_sorting_map[i]->c_str(), buff, sizeof(buff)); LogText(buff); } diff --git a/src/network_inspectors/appid/appid_peg_counts.h b/src/network_inspectors/appid/appid_peg_counts.h index 010ff7175..b74847bd6 100644 --- a/src/network_inspectors/appid/appid_peg_counts.h +++ b/src/network_inspectors/appid/appid_peg_counts.h @@ -112,11 +112,10 @@ public: static void print(); private: - static std::unordered_map appid_detector_pegs_idx; - static std::vector appid_detectors_info; static AppIdDynamicPeg appid_dynamic_sum[SF_APPID_MAX+1]; - static THREAD_LOCAL std::vector* appid_peg_counts; - static uint32_t get_stats_index(AppId id); + static THREAD_LOCAL AppIdDynamicPeg* unknown_appids_peg; + static THREAD_LOCAL std::unordered_map* appid_peg_counts; + static std::unordered_map> appid_peg_ids; static AppIdDynamicPeg zeroed_peg; static PegCount all_zeroed_peg[DetectorPegs::NUM_APPID_DETECTOR_PEGS]; };