From: Shravan Rangarajuvenkata (shrarang) Date: Thu, 18 Feb 2021 19:51:10 +0000 (+0000) Subject: Merge pull request #2749 in SNORT/snort3 from ~SHRARANG/snort3:appid_remove_forecast... X-Git-Tag: 3.1.2.0~38 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d98291bf12bd60f02fc254de039234b665dd0ea9;p=thirdparty%2Fsnort3.git Merge pull request #2749 in SNORT/snort3 from ~SHRARANG/snort3:appid_remove_forecast to master Squashed commit of the following: commit 8b16b5b54d078478ddffa3b4899b68eda7a4641d Author: Shravan Rangaraju Date: Fri Feb 12 17:17:55 2021 -0500 appid: remove app forecast method --- diff --git a/src/network_inspectors/appid/CMakeLists.txt b/src/network_inspectors/appid/CMakeLists.txt index 5e9c8a402..151ffaaec 100644 --- a/src/network_inspectors/appid/CMakeLists.txt +++ b/src/network_inspectors/appid/CMakeLists.txt @@ -139,8 +139,6 @@ set ( UTIL_APPID_SOURCES ) set ( APPID_SOURCES - app_forecast.cc - app_forecast.h appid_api.cc appid_api.h appid_app_descriptor.cc diff --git a/src/network_inspectors/appid/app_forecast.cc b/src/network_inspectors/appid/app_forecast.cc deleted file mode 100644 index e87674f9d..000000000 --- a/src/network_inspectors/appid/app_forecast.cc +++ /dev/null @@ -1,72 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2005-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// app_forecast.cc author Sourcefire Inc. - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "app_forecast.h" -#include "appid_inspector.h" - -#include "log/messages.h" -#include "time/packet_time.h" -#include "appid_session.h" - -using namespace snort; - -void check_session_for_AF_indicator(Packet* p, AppidSessionDirection dir, AppId indicator, const OdpContext& odp_ctxt) -{ - const std::unordered_map& AF_indicators = odp_ctxt.get_af_indicators(); - auto af_indicator_entry = AF_indicators.find(indicator); - - if (af_indicator_entry == AF_indicators.end()) - return; - - AFElement ind_element = af_indicator_entry->second; - AFActKey master_key(p, dir, ind_element.forecast); - - AFActVal new_active_value = AFActVal(ind_element.target, packet_time()); - - odp_thread_local_ctxt->add_af_actives(master_key, new_active_value); -} - -AppId check_session_for_AF_forecast(AppIdSession& asd, Packet* p, AppidSessionDirection dir, AppId forecast) -{ - AFActKey master_key(p, dir, forecast); - - //get out if there is no value - std::map* AF_actives = odp_thread_local_ctxt->get_af_actives(); - assert(AF_actives); - auto check_act_val = AF_actives->find(master_key); - if (check_act_val == AF_actives->end()) - return APP_ID_UNKNOWN; - - //if the value is older than 5 minutes, remove it and get out - time_t age = packet_time() - check_act_val->second.last; - if (age < 0 || age > 300) - { - odp_thread_local_ctxt->erase_af_actives(master_key); - return APP_ID_UNKNOWN; - } - asd.set_payload_id(check_act_val->second.target); - return forecast; -} - diff --git a/src/network_inspectors/appid/app_forecast.h b/src/network_inspectors/appid/app_forecast.h deleted file mode 100644 index 2f383f05e..000000000 --- a/src/network_inspectors/appid/app_forecast.h +++ /dev/null @@ -1,92 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2014-2020 Cisco and/or its affiliates. All rights reserved. -// Copyright (C) 2005-2013 Sourcefire, Inc. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- - -// app_forecast.h author Sourcefire Inc. - -#ifndef APP_FORECAST_H -#define APP_FORECAST_H - -#include - -#include "flow/flow.h" -#include "protocols/packet.h" -#include "utils/cpp_macros.h" - -#include "appid_types.h" -#include "application_ids.h" - -class AppIdSession; -class OdpContext; -namespace snort -{ -struct Packet; -} - -// indicator - the appId that indicates there may be subsequent flows to look for, -// from the same host -// forecast - the appId in the subsequent flow that we are looking for -// target - the appId we want to set in that subsequent flow -// -// for now, indicator and target are WEB APPLICATIONS. The forecast is APP PROTOCOL. -// We can change this later by adding app type info for each, if we find a use case. - -struct AFElement -{ - AFElement(AppId forecast, AppId target) : forecast(forecast), target(target) { } - - AppId forecast; - AppId target; -}; - -PADDING_GUARD_BEGIN -class AFActKey -{ - public: - AFActKey(snort::Packet* p, AppidSessionDirection dir, AppId forecast) : - forecast(forecast) - { - const snort::SfIp* src = dir ? p->ptrs.ip_api.get_dst() : p->ptrs.ip_api.get_src(); - - memcpy(ip, src->get_ip6_ptr(), sizeof(ip)); - } - - bool operator<(const AFActKey &key) const - { - return (forecast < key.forecast || ip[0] < key.ip[0] || - ip[1] < key.ip[1] || ip[2] < key.ip[2] || ip[3] < key.ip[3]); - } - private: - uint32_t ip[4]; - AppId forecast; -}; -PADDING_GUARD_END - -struct AFActVal -{ - AFActVal(AppId target, time_t last) : target(target), last(last) { } - - AppId target; - time_t last; -}; - -void check_session_for_AF_indicator(snort::Packet*, AppidSessionDirection, AppId, const OdpContext&); -AppId check_session_for_AF_forecast(AppIdSession&, snort::Packet*, AppidSessionDirection, AppId); - -#endif - diff --git a/src/network_inspectors/appid/appid_config.cc b/src/network_inspectors/appid/appid_config.cc index 0a3b92828..5fe5daa6d 100644 --- a/src/network_inspectors/appid/appid_config.cc +++ b/src/network_inspectors/appid/appid_config.cc @@ -28,7 +28,6 @@ #include #include -#include "app_forecast.h" #include "app_info_table.h" #include "appid_discovery.h" #include "appid_http_session.h" @@ -108,7 +107,7 @@ bool AppIdContext::init_appid(SnortConfig* sc) odp_ctxt = new OdpContext(config, sc); if (!odp_thread_local_ctxt) - odp_thread_local_ctxt = new OdpThreadContext(true); + odp_thread_local_ctxt = new OdpThreadContext; static bool once = false; if (!once) @@ -159,11 +158,6 @@ OdpContext::OdpContext(const AppIdConfig& config, SnortConfig* sc) version = next_version++; } -OdpContext::~OdpContext() -{ - AF_indicators.clear(); -} - void OdpContext::initialize() { service_pattern_detector->finalize_service_port_patterns(); @@ -225,26 +219,6 @@ AppId OdpContext::get_protocol_service_id(IpProtocol proto) return ip_protocol[(uint16_t)proto]; } -void OdpContext::add_af_indicator(AppId indicator, AppId forecast, AppId target) -{ - if (AF_indicators.find(indicator) != AF_indicators.end()) - { - ErrorMessage("LuaDetectorApi:Attempt to add more than one AFElement per appId %d", - indicator); - return; - } - - AFElement val = AFElement(forecast, target); - if (false == AF_indicators.emplace(indicator, val).second) - ErrorMessage("LuaDetectorApi:Failed to add AFElement for appId %d", indicator); -} - -OdpThreadContext::OdpThreadContext(bool is_control) -{ - if (!is_control) - AF_actives = new std::map; -} - void OdpThreadContext::initialize(AppIdContext& ctxt, bool is_control, bool reload_odp) { if (!is_control and reload_odp) @@ -257,10 +231,4 @@ OdpThreadContext::~OdpThreadContext() { assert(lua_detector_mgr); delete lua_detector_mgr; - - if (AF_actives != nullptr) - { - AF_actives->clear(); - delete AF_actives; - } } diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h index 0a12c658c..20bd67a7a 100644 --- a/src/network_inspectors/appid/appid_config.h +++ b/src/network_inspectors/appid/appid_config.h @@ -27,7 +27,6 @@ #include "target_based/snort_protocols.h" -#include "app_forecast.h" #include "app_info_table.h" #include "client_plugins/client_discovery.h" #include "detector_plugins/dns_patterns.h" @@ -119,7 +118,6 @@ public: uint16_t max_packet_service_fail_ignore_bytes = MIN_MAX_PKT_BEFORE_SERVICE_FAIL_IGNORE_BYTES; OdpContext(const AppIdConfig&, snort::SnortConfig*); - ~OdpContext(); void initialize(); void reload(); @@ -194,16 +192,10 @@ public: return *service_pattern_detector; } - const std::unordered_map& get_af_indicators() const - { - return AF_indicators; - } - void add_port_service_id(IpProtocol, uint16_t, AppId); void add_protocol_service_id(IpProtocol, AppId); AppId get_port_service_id(IpProtocol, uint16_t); AppId get_protocol_service_id(IpProtocol); - void add_af_indicator(AppId, AppId, AppId); private: AppInfoManager app_info_mgr; @@ -217,7 +209,6 @@ private: SslPatternMatchers ssl_matchers; PatternClientDetector* client_pattern_detector; PatternServiceDetector* service_pattern_detector; - std::unordered_map AF_indicators; // list of "indicator apps" std::array tcp_port_only = {}; // port-only TCP services std::array udp_port_only = {}; // port-only UDP services @@ -230,7 +221,6 @@ private: class OdpThreadContext { public: - OdpThreadContext(bool is_control=false); ~OdpThreadContext(); void initialize(AppIdContext& ctxt, bool is_control=false, bool reload_odp=false); @@ -245,26 +235,8 @@ public: return *lua_detector_mgr; } - std::map* get_af_actives() const - { - return AF_actives; - } - - void add_af_actives(AFActKey key, AFActVal value) - { - assert(AF_actives); - AF_actives->emplace(key, value); - } - - void erase_af_actives(AFActKey key) - { - assert(AF_actives); - AF_actives->erase(key); - } - private: LuaDetectorManager* lua_detector_mgr = nullptr; - std::map* AF_actives = nullptr; // list of hosts to watch }; class AppIdContext diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index 0a53fc23d..27be6db79 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -31,7 +31,6 @@ #include "protocols/packet.h" #include "protocols/tcp.h" -#include "app_forecast.h" #include "appid_config.h" #include "appid_debug.h" #include "appid_detector.h" @@ -124,8 +123,8 @@ void AppIdDiscovery::do_application_discovery(Packet* p, AppIdInspector& inspect bool is_discovery_done = do_discovery(p, *asd, protocol, outer_protocol, direction, service_id, client_id, payload_id, misc_id, change_bits, tp_appid_ctxt); - do_post_discovery(p, *asd, direction, is_discovery_done, service_id, client_id, payload_id, - misc_id, change_bits); + do_post_discovery(p, *asd, is_discovery_done, service_id, client_id, payload_id, misc_id, + change_bits); } static inline unsigned get_ipfuncs_flags(const Packet* p, bool dst) @@ -859,8 +858,8 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto } void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd, - AppidSessionDirection direction, bool is_discovery_done, AppId service_id, - AppId client_id, AppId payload_id, AppId misc_id, AppidChangeBits& change_bits) + bool is_discovery_done, AppId service_id, AppId client_id, AppId payload_id, AppId misc_id, + AppidChangeBits& change_bits) { if (service_id > APP_ID_NONE) { @@ -876,23 +875,6 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd, asd.set_session_flags(APPID_SESSION_CONTINUE); } - if (service_id != APP_ID_NONE) - { - if (payload_id != asd.past_indicator and payload_id != APP_ID_NONE) - { - asd.past_indicator = payload_id; - check_session_for_AF_indicator(p, direction, (AppId)payload_id, asd.get_odp_ctxt()); - } - - if (asd.past_forecast != service_id and asd.past_forecast != APP_ID_UNKNOWN and - asd.get_payload_id() == APP_ID_NONE) - { - asd.past_forecast = check_session_for_AF_forecast(asd, p, direction, service_id); - if (asd.past_forecast != APP_ID_UNKNOWN) - payload_id = asd.pick_ss_payload_app_id(service_id); - } - } - if (asd.get_session_flags(APPID_SESSION_OOO_CHECK_TP) and asd.tpsession and (asd.scan_flags & SCAN_HOST_PORT_FLAG) and (service_id or payload_id)) { diff --git a/src/network_inspectors/appid/appid_discovery.h b/src/network_inspectors/appid/appid_discovery.h index 845e92414..ef790892c 100644 --- a/src/network_inspectors/appid/appid_discovery.h +++ b/src/network_inspectors/appid/appid_discovery.h @@ -144,8 +144,8 @@ private: AppId& client_id, AppId& payload_id, AppId& misc_id, AppidChangeBits& change_bits, ThirdPartyAppIdContext* tp_appid_ctxt); static void do_post_discovery(snort::Packet* p, AppIdSession& asd, - AppidSessionDirection direction, bool is_discovery_done, AppId service_id, AppId client_id, - AppId payload_id, AppId misc_id, AppidChangeBits& change_bits); + bool is_discovery_done, AppId service_id, AppId client_id, AppId payload_id, AppId misc_id, + AppidChangeBits& change_bits); static void do_port_based_discovery(snort::Packet* p, AppIdSession& asd, IpProtocol protocol, AppidSessionDirection direction); static bool do_host_port_based_discovery(snort::Packet* p, AppIdSession& asd, diff --git a/src/network_inspectors/appid/appid_inspector.cc b/src/network_inspectors/appid/appid_inspector.cc index df0504b73..767a670ba 100644 --- a/src/network_inspectors/appid/appid_inspector.cc +++ b/src/network_inspectors/appid/appid_inspector.cc @@ -34,7 +34,6 @@ #include "packet_tracer/packet_tracer.h" #include "profiler/profiler.h" -#include "app_forecast.h" #include "appid_data_decrypt_event_handler.h" #include "appid_dcerpc_event_handler.h" #include "appid_debug.h" diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 788875ec8..56f8a5ff9 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -246,7 +246,7 @@ bool ACOdpContextSwap::execute(Analyzer&, void**) pkt_thread_odp_ctxt = ¤t_odp_ctxt; assert(odp_thread_local_ctxt); delete odp_thread_local_ctxt; - odp_thread_local_ctxt = new OdpThreadContext(); + odp_thread_local_ctxt = new OdpThreadContext; odp_thread_local_ctxt->initialize(ctxt, false, true); return true; } @@ -375,7 +375,7 @@ static int reload_detectors(lua_State* L) ctxt.create_odp_ctxt(); assert(odp_thread_local_ctxt); delete odp_thread_local_ctxt; - odp_thread_local_ctxt = new OdpThreadContext(true); + odp_thread_local_ctxt = new OdpThreadContext; OdpContext& odp_ctxt = ctxt.get_odp_ctxt(); odp_ctxt.get_client_disco_mgr().initialize(); diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc index b3fbe4703..d196ad623 100644 --- a/src/network_inspectors/appid/appid_session.cc +++ b/src/network_inspectors/appid/appid_session.cc @@ -38,7 +38,6 @@ #include "target_based/snort_protocols.h" #include "time/packet_time.h" -#include "app_forecast.h" #include "app_info_table.h" #include "appid_config.h" #include "appid_debug.h" diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h index 1f4360092..fd5886da9 100644 --- a/src/network_inspectors/appid/appid_session.h +++ b/src/network_inspectors/appid/appid_session.h @@ -308,9 +308,6 @@ public: AppId referred_id; } encrypted = { APP_ID_NONE, APP_ID_NONE, APP_ID_NONE, APP_ID_NONE, APP_ID_NONE }; - AppId past_indicator = APP_ID_NONE; - AppId past_forecast = APP_ID_NONE; - bool in_expected_cache = false; static unsigned inspector_id; static std::mutex inferred_svcs_lock; diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h index 861ef9e51..801a262b4 100644 --- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h +++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h @@ -224,6 +224,5 @@ int ServiceDiscovery::add_service_port(AppIdDetector*, const ServiceDetectorPort OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } -OdpContext::~OdpContext() { } #endif diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc index 7ed533a84..beb28cde3 100644 --- a/src/network_inspectors/appid/lua_detector_api.cc +++ b/src/network_inspectors/appid/lua_detector_api.cc @@ -34,7 +34,6 @@ #include "profiler/profiler.h" #include "protocols/packet.h" -#include "app_forecast.h" #include "app_info_table.h" #include "appid_debug.h" #include "appid_inspector.h" @@ -1842,23 +1841,6 @@ static int detector_add_length_app_cache(lua_State* L) return 1; } -static int detector_add_af_application(lua_State* L) -{ - auto& ud = *UserData::check(L, DETECTOR, 1); - // Verify detector user data and that we are NOT in packet context - ud->validate_lua_state(false); - if (!init(L)) return 0; - - int index = 1; - - AppId indicator = (AppId)lua_tointeger(L, ++index); - AppId forecast = (AppId)lua_tointeger(L, ++index); - AppId target = (AppId)lua_tointeger(L, ++index); - ud->get_odp_ctxt().add_af_indicator(indicator, forecast, target); - - return 0; -} - static int detector_add_url_application(lua_State* L) { // Verify detector user data and that we are NOT in packet context @@ -2691,9 +2673,6 @@ static const luaL_Reg detector_methods[] = // same appId { "CHPMultiAddAction", detector_add_chp_multi_action }, - //App Forecasting engine - { "AFAddApp", detector_add_af_application }, - { "portOnlyService", detector_port_only_service }, /* Length-based detectors. */ diff --git a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h index 6df7daa5b..16edfde29 100644 --- a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h +++ b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h @@ -212,6 +212,4 @@ void ServiceDiscoveryState::set_service_id_valid(ServiceDetector*) { } OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } -OdpContext::~OdpContext() { } - #endif diff --git a/src/network_inspectors/appid/test/appid_debug_test.cc b/src/network_inspectors/appid/test/appid_debug_test.cc index 5c9592b08..9b14e7495 100644 --- a/src/network_inspectors/appid/test/appid_debug_test.cc +++ b/src/network_inspectors/appid/test/appid_debug_test.cc @@ -56,7 +56,6 @@ public: AppIdConfig::~AppIdConfig() { } OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } -OdpContext::~OdpContext() { } AppIdConfig stub_config; AppIdContext stub_ctxt(stub_config); diff --git a/src/network_inspectors/appid/test/appid_http_session_test.cc b/src/network_inspectors/appid/test/appid_http_session_test.cc index b23f4e528..e3629a63b 100644 --- a/src/network_inspectors/appid/test/appid_http_session_test.cc +++ b/src/network_inspectors/appid/test/appid_http_session_test.cc @@ -169,7 +169,6 @@ void memory::MemoryCap::update_allocations(size_t) { } void memory::MemoryCap::update_deallocations(size_t) { } OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } -OdpContext::~OdpContext() { } AppIdConfig::~AppIdConfig() { } diff --git a/src/network_inspectors/appid/test/appid_mock_session.h b/src/network_inspectors/appid/test/appid_mock_session.h index b647ac93e..2074f5f8e 100644 --- a/src/network_inspectors/appid/test/appid_mock_session.h +++ b/src/network_inspectors/appid/test/appid_mock_session.h @@ -73,7 +73,6 @@ public: AppIdConfig::~AppIdConfig() { } OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } -OdpContext::~OdpContext() { } void FlowHAState::add(uint8_t) { } diff --git a/src/network_inspectors/appid/test/service_state_test.cc b/src/network_inspectors/appid/test/service_state_test.cc index ed16c4519..ddea1ab07 100644 --- a/src/network_inspectors/appid/test/service_state_test.cc +++ b/src/network_inspectors/appid/test/service_state_test.cc @@ -84,7 +84,6 @@ void ClientAppDescriptor::update_stats(AppId, bool) {} void PayloadAppDescriptor::update_stats(AppId, bool) {} AppIdConfig::~AppIdConfig() { } OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } -OdpContext::~OdpContext() { } AppIdConfig stub_config; AppIdContext stub_ctxt(stub_config); OdpContext stub_odp_ctxt(stub_config, nullptr); diff --git a/src/network_inspectors/appid/test/tp_lib_handler_test.cc b/src/network_inspectors/appid/test/tp_lib_handler_test.cc index 46d17b618..bd14e0123 100644 --- a/src/network_inspectors/appid/test/tp_lib_handler_test.cc +++ b/src/network_inspectors/appid/test/tp_lib_handler_test.cc @@ -60,7 +60,6 @@ SipPatternMatchers::~SipPatternMatchers() { } SslPatternMatchers::~SslPatternMatchers() { } AppIdConfig::~AppIdConfig() { } OdpContext::OdpContext(const AppIdConfig&, snort::SnortConfig*) { } -OdpContext::~OdpContext() { } void ServiceDiscovery::initialize() { } void ServiceDiscovery::reload() { } int ServiceDiscovery::add_service_port(AppIdDetector*, const ServiceDetectorPort&)