From: Russ Combs (rucombs) Date: Wed, 20 Jul 2016 11:43:49 +0000 (-0400) Subject: Merge pull request #563 in SNORT/snort3 from appid_ws2_plugins_104 to master X-Git-Tag: 3.0.0-233~329 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=360e3beddc49d925a597b3c1bbfa702e215d0f89;p=thirdparty%2Fsnort3.git Merge pull request #563 in SNORT/snort3 from appid_ws2_plugins_104 to master Squashed commit of the following: commit 441baf1b5a602430abf2e23fa893bf9e2b94c0a6 Author: davis mcpherson Date: Wed Jul 13 15:43:15 2016 -0400 add stats counters and regression tests for mdns, timbuktu, battlefield, bgp & netbios services --- diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc index d1ba1435c..500691216 100644 --- a/src/network_inspectors/appid/app_info_table.cc +++ b/src/network_inspectors/appid/app_info_table.cc @@ -392,7 +392,7 @@ void appInfoTableInit(const char* path, AppIdConfig* pConfig) pAppidActiveConfig->mod_config->mdns_user_reporting = 1; pAppidActiveConfig->mod_config->dns_host_reporting = 1; pAppidActiveConfig->mod_config->max_tp_flow_depth = 5; - pAppidActiveConfig->mod_config->http2_detection_enabled = 0; + pAppidActiveConfig->mod_config->http2_detection_enabled = false; snprintf(filepath, sizeof(filepath), "%s/odp/%s", path, APP_CONFIG_FILE); appIdConfLoad (filepath); @@ -765,12 +765,12 @@ static void appIdConfLoad(const char* path) if (!(strcasecmp(conf_val, "disabled"))) { LogMessage("AppId: disabling internal HTTP/2 detection.\n"); - pAppidActiveConfig->mod_config->http2_detection_enabled = 0; + pAppidActiveConfig->mod_config->http2_detection_enabled = false; } else if (!(strcasecmp(conf_val, "enabled"))) { LogMessage("AppId: enabling internal HTTP/2 detection.\n"); - pAppidActiveConfig->mod_config->http2_detection_enabled = 1; + pAppidActiveConfig->mod_config->http2_detection_enabled = true; } else { diff --git a/src/network_inspectors/appid/appid_config.cc b/src/network_inspectors/appid/appid_config.cc index 90d44c56a..278f370bc 100644 --- a/src/network_inspectors/appid/appid_config.cc +++ b/src/network_inspectors/appid/appid_config.cc @@ -804,9 +804,7 @@ bool AppIdConfig::init_appid( ) ClientAppInit(pAppidActiveConfig); ServiceInit(pAppidActiveConfig); LuaDetectorModuleManager::FinalizeLuaModules(pAppidActiveConfig); - finalize_pattern_modules(); - http_detector_finalize(pAppidActiveConfig); #ifdef REMOVED_WHILE_NOT_IN_USE sipUaFinalize(&pAppidActiveConfig->detectorSipConfig); diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 6adf7dfab..c77c20f8e 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -37,28 +37,35 @@ THREAD_LOCAL ProfileStats appidPerfStats; const PegInfo appid_pegs[] = { { "packets", "count of packets processed by appid" }, + { "battlefield_flows", "count of battle field flows discovered by appid" }, + { "bgp_flows", "count of bgp flows discovered by appid" }, { "bit_clients", "count of bittorrent clients discovered by appid" }, { "bit_flows", "count of bittorrent flows discovered by appid" }, { "bittracker_clients", "count of bittorrent tracker clients discovered by appid" }, - { "dns_udp_flows", "count of dns flows over udp discovered by appid" }, + { "dcerpc_tcp_flows", "count of dce rpc flows over tcp discovered by appid" }, + { "dcerpc_udp_flows", "count of dce rpc flows over udp discovered by appid" }, { "dns_tcp_flows", "count of dns flows over tcp discovered by appid" }, + { "dns_udp_flows", "count of dns flows over udp discovered by appid" }, { "ftp_flows", "count of ftp flows discovered by appid" }, { "ftps_flows", "count of ftps flows discovered by appid" }, - { "imap flows", "count of imap service flows discovered by appid" }, - { "imaps flows", "count of imap TLS service flows discovered by appid" }, - { "irc flows", "count of irc service flows discovered by appid" }, - { "kerberos clients", "count of kerberos clients discovered by appid" }, - { "kerberos flows", "count of kerberos service flows discovered by appid" }, - { "kerberos users", "count of kerberos users discovered by appid" }, - { "lpr flows", "count of lpr service flows discovered by appid" }, - { "mysql flows", "count of mysql service flows discovered by appid" }, - { "pop flows", "count of pop service flows discovered by appid" }, + { "imap_flows", "count of imap service flows discovered by appid" }, + { "imaps_flows", "count of imap TLS service flows discovered by appid" }, + { "irc_flows", "count of irc service flows discovered by appid" }, + { "kerberos_clients", "count of kerberos clients discovered by appid" }, + { "kerberos_flows", "count of kerberos service flows discovered by appid" }, + { "kerberos_users", "count of kerberos users discovered by appid" }, + { "lpr_flows", "count of lpr service flows discovered by appid" }, + { "mdns_flows", "count of mdns service flows discovered by appid" }, + { "mysql_flows", "count of mysql service flows discovered by appid" }, + { "netbios_flows", "count of netbios service flows discovered by appid" }, + { "pop_flows", "count of pop service flows discovered by appid" }, { "smtp_flows", "count of smtp flows discovered by appid" }, { "smtps_flows", "count of smtps flows discovered by appid" }, { "ssh_clients", "count of ssh clients discovered by appid" }, { "ssh_flows", "count of ssh flows discovered by appid" }, { "ssl_flows", "count of ssl flows discovered by appid" }, { "telnet_flows", "count of telnet flows discovered by appid" }, + { "timbuktu_flows", "count of timbuktu flows discovered by appid" }, { nullptr, nullptr } }; diff --git a/src/network_inspectors/appid/appid_module.h b/src/network_inspectors/appid/appid_module.h index d20f5a2ca..093288c96 100644 --- a/src/network_inspectors/appid/appid_module.h +++ b/src/network_inspectors/appid/appid_module.h @@ -38,11 +38,15 @@ extern THREAD_LOCAL ProfileStats appidPerfStats; struct AppIdStats { PegCount packets; + PegCount battlefield_flows; + PegCount bgp_flows; PegCount bit_clients; PegCount bit_flows; PegCount bittracker_clients; - PegCount dns_udp_flows; + PegCount dcerpc_tcp_flows; + PegCount dcerpc_udp_flows; PegCount dns_tcp_flows; + PegCount dns_udp_flows; PegCount ftp_flows; PegCount ftps_flows; PegCount imap_flows; @@ -52,7 +56,9 @@ struct AppIdStats PegCount kerberos_flows; PegCount kerberos_users; PegCount lpr_flows; + PegCount mdns_flows; PegCount mysql_flows; + PegCount netbios_flows; PegCount pop_flows; PegCount smtp_flows; PegCount smtps_flows; @@ -60,6 +66,7 @@ struct AppIdStats PegCount ssh_flows; PegCount ssl_flows; PegCount telnet_flows; + PegCount timbuktu_flows; }; extern THREAD_LOCAL AppIdStats appid_stats; diff --git a/src/network_inspectors/appid/client_plugins/client_app_base.cc b/src/network_inspectors/appid/client_plugins/client_app_base.cc index a68d1b1ae..008532d5e 100644 --- a/src/network_inspectors/appid/client_plugins/client_app_base.cc +++ b/src/network_inspectors/appid/client_plugins/client_app_base.cc @@ -121,22 +121,17 @@ static RNAClientAppModule* static_client_list[] = { &smtp_client_mod, &ssh_client_mod, - -#ifdef REMOVED_WHILE_NOT_IN_USE &msn_client_mod, &aim_client_mod, &ym_client_mod, &sip_udp_client_mod, &sip_tcp_client_mod, -#endif &bit_client_mod, &bit_tracker_client_mod, -#ifdef REMOVED_WHILE_NOT_IN_USE &rtp_client_mod, &timbuktu_client_mod, &tns_client_mod, &vnc_client_mod, -#endif &pattern_udp_client_mod, &pattern_tcp_client_mod, &dns_udp_client_mod, diff --git a/src/network_inspectors/appid/detector_plugins/detector_http.cc b/src/network_inspectors/appid/detector_plugins/detector_http.cc index 3831e6a16..256f43f53 100644 --- a/src/network_inspectors/appid/detector_plugins/detector_http.cc +++ b/src/network_inspectors/appid/detector_plugins/detector_http.cc @@ -630,9 +630,7 @@ static int processCHPList(CHPListElement* chplist, DetectorHttpConfig* pHttpConf return 1; } -static SearchTool* registerHeaderPatterns( - HeaderPattern* patternList, - size_t patternListCount) +static SearchTool* registerHeaderPatterns(HeaderPattern* patternList, size_t patternListCount) { SearchTool* patternMatcher = new SearchTool("ac_full"); @@ -802,7 +800,7 @@ static char* normalize_userid(char* user) // find number of '%' for (i = 0; i < old_size; i++) { - if (*(user+i) == '%') + if (*(user + i) == '%') percent_count++; } if (0 == percent_count) @@ -850,9 +848,7 @@ static char* normalize_userid(char* user) return user; } -static void extractCHP(char* buf, int bs, int start, - int psize, char* adata, - char** outbuf) +static void extractCHP(char* buf, int bs, int start, int psize, char* adata, char** outbuf) { char* begin = buf+start+psize; char* end = nullptr; @@ -1018,8 +1014,7 @@ static IpProtocol ffSetProtocol(char* buf, int buf_size, int start, int psize) return (IpProtocol)temp_protocol; } -static void fflowCreate(char* adata, fflow_info* fflow, - Packet* p, AppId target_appid) +static void fflowCreate(char* adata, fflow_info* fflow, Packet* p, AppId target_appid) { char* saddr_string = nullptr; char* daddr_string = nullptr; @@ -1466,8 +1461,7 @@ static inline int optionallyReplaceWithStrdup(char** optionalStr, const char* st } void identifyUserAgent(const uint8_t* start, int size, AppId* serviceAppId, AppId* ClientAppId, - char** version, - const DetectorHttpConfig* pHttpConfig) + char** version, const DetectorHttpConfig* pHttpConfig) { int skypeDetect; int mobileDetect; @@ -1796,8 +1790,8 @@ done: FreeMatchStructures(mp); } -int geAppidByViaPattern(const uint8_t* data, unsigned size, char** version, const - DetectorHttpConfig* pHttpConfig) +int geAppidByViaPattern(const uint8_t* data, unsigned size, char** version, + const DetectorHttpConfig* pHttpConfig) { unsigned i; const uint8_t* data_ptr; @@ -2287,12 +2281,9 @@ int webdav_found(HeaderMatchedPatterns* hmp) // knowledge" case for HTTP/2 (i.e., the client knows the server supports // HTTP/2 and jumps right in with the preface). -static CLIENT_APP_RETCODE http_client_init(const IniClientAppAPI* const init_api, - SF_LIST* config); +static CLIENT_APP_RETCODE http_client_init(const IniClientAppAPI* const init_api, SF_LIST* config); static CLIENT_APP_RETCODE http_client_validate(const uint8_t* data, uint16_t size, const int dir, - AppIdData* flowp, Packet* pkt, struct Detector* userData, - const AppIdConfig* pConfig); - + AppIdData* flowp, Packet* pkt, struct Detector* userData, const AppIdConfig* pConfig); static int http_service_init(const IniServiceAPI* const init_api); static int http_service_validate(ServiceValidationArgs* args); diff --git a/src/network_inspectors/appid/fw_appid.h b/src/network_inspectors/appid/fw_appid.h index 8395952f2..ee0d42f74 100644 --- a/src/network_inspectors/appid/fw_appid.h +++ b/src/network_inspectors/appid/fw_appid.h @@ -61,22 +61,11 @@ AppId appGetAppFromClientId(uint32_t clientId, AppIdConfig*); AppId appGetAppFromPayloadId(uint32_t payloadId, AppIdConfig*); void appSharedDataDelete(AppIdData*); void AppIdAddUser(AppIdData*, const char* username, AppId, int success); -void AppIdAddDnsQueryInfo( - AppIdData*, - uint16_t id, - const uint8_t* host, uint8_t host_len, uint16_t host_offset, - uint16_t record_type -); - -void AppIdAddDnsResponseInfo( - AppIdData*, - uint16_t id, - const uint8_t* host, uint8_t host_len, uint16_t host_offset, - uint8_t response_type, uint32_t ttl -); - +void AppIdAddDnsQueryInfo(AppIdData*, uint16_t id, const uint8_t* host, uint8_t host_len, + uint16_t host_offset, uint16_t record_type); +void AppIdAddDnsResponseInfo(AppIdData*, uint16_t id, const uint8_t* host, uint8_t host_len, + uint16_t host_offset, uint8_t response_type, uint32_t ttl); void AppIdResetDnsInfo(AppIdData*); - void AppIdAddPayload(AppIdData*, AppId); AppIdData* appSharedDataAlloc(IpProtocol proto, const sfip_t*); AppId getOpenAppId(void* ssnptr); diff --git a/src/network_inspectors/appid/service_plugins/service_base.cc b/src/network_inspectors/appid/service_plugins/service_base.cc index d563e386c..808bca760 100644 --- a/src/network_inspectors/appid/service_plugins/service_base.cc +++ b/src/network_inspectors/appid/service_plugins/service_base.cc @@ -53,6 +53,7 @@ #include "service_telnet.h" #include "service_tftp.h" #include "appid_flow_data.h" +#include "appid_config.h" #include "fw_appid.h" #include "lua_detector_api.h" #include "lua_detector_module.h" @@ -169,20 +170,15 @@ extern RNAServiceValidationModule http_service_mod; static RNAServiceValidationModule* static_service_list[] = { -#ifdef REMOVED_WHILE_NOT_IN_USE &bgp_service_mod, &bootp_service_mod, &dcerpc_service_mod, -#endif &dns_service_mod, -#ifdef REMOVED_WHILE_NOT_IN_USE &flap_service_mod, -#endif &ftp_service_mod, &irc_service_mod, &lpr_service_mod, &mysql_service_mod, -#ifdef REMOVED_WHILE_NOT_IN_USE &netbios_service_mod, &nntp_service_mod, &ntp_service_mod, @@ -194,15 +190,11 @@ static RNAServiceValidationModule* static_service_list[] = &rshell_service_mod, &rsync_service_mod, &rtmp_service_mod, -#endif &smtp_service_mod, -#ifdef REMOVED_WHILE_NOT_IN_USE &snmp_service_mod, -#endif &ssh_service_mod, &ssl_service_mod, &telnet_service_mod, -#ifdef REMOVED_WHILE_NOT_IN_USE &tftp_service_mod, &sip_service_mod, &directconnect_service_mod, @@ -210,7 +202,6 @@ static RNAServiceValidationModule* static_service_list[] = &mdns_service_mod, &timbuktu_service_mod, &tns_service_mod, -#endif &bit_service_mod, &pattern_service_mod, &http_service_mod diff --git a/src/network_inspectors/appid/service_plugins/service_battle_field.cc b/src/network_inspectors/appid/service_plugins/service_battle_field.cc index 487f23a6c..4350efbbf 100644 --- a/src/network_inspectors/appid/service_plugins/service_battle_field.cc +++ b/src/network_inspectors/appid/service_plugins/service_battle_field.cc @@ -20,6 +20,7 @@ // service_battle_field.cc author Sourcefire Inc. #include "service_battle_field.h" +#include "appid_module.h" #include "application_ids.h" #include "main/snort_debug.h" @@ -222,6 +223,7 @@ success: battlefield_service_mod.api->add_service(flowp, pkt, args->dir, &svc_element, APP_ID_BATTLEFIELD, nullptr, nullptr, nullptr); + appid_stats.battlefield_flows++; return SERVICE_SUCCESS; fail: diff --git a/src/network_inspectors/appid/service_plugins/service_bgp.cc b/src/network_inspectors/appid/service_plugins/service_bgp.cc index cefa73b9d..df479ba3a 100644 --- a/src/network_inspectors/appid/service_plugins/service_bgp.cc +++ b/src/network_inspectors/appid/service_plugins/service_bgp.cc @@ -20,6 +20,7 @@ // service_bgp.cc author Sourcefire Inc. #include "service_bgp.h" +#include "appid_module.h" #include "application_ids.h" #include "main/snort_debug.h" @@ -251,6 +252,7 @@ fail: success: bgp_service_mod.api->add_service(flowp, args->pkt, args->dir, &svc_element, APP_ID_BGP, nullptr, nullptr, nullptr); + appid_stats.bgp_flows++; return SERVICE_SUCCESS; } diff --git a/src/network_inspectors/appid/service_plugins/service_dcerpc.cc b/src/network_inspectors/appid/service_plugins/service_dcerpc.cc index b5baaa5ac..2c2b42c34 100644 --- a/src/network_inspectors/appid/service_plugins/service_dcerpc.cc +++ b/src/network_inspectors/appid/service_plugins/service_dcerpc.cc @@ -20,6 +20,7 @@ // service_dcerpc.cc author Sourcefire Inc. #include "service_dcerpc.h" +#include "appid_module.h" #include "application_ids.h" #include "dcerpc.h" @@ -137,6 +138,7 @@ static int dcerpc_tcp_validate(ServiceValidationArgs* args) { dcerpc_service_mod.api->add_service(flowp, args->pkt, args->dir, &tcp_svc_element, APP_ID_DCE_RPC, nullptr, nullptr, nullptr); + appid_stats.dcerpc_tcp_flows++; return SERVICE_SUCCESS; } @@ -188,6 +190,7 @@ static int dcerpc_udp_validate(ServiceValidationArgs* args) { dcerpc_service_mod.api->add_service(flowp, args->pkt, args->dir, &udp_svc_element, APP_ID_DCE_RPC, nullptr, nullptr, nullptr); + appid_stats.dcerpc_udp_flows++; return SERVICE_SUCCESS; } diff --git a/src/network_inspectors/appid/service_plugins/service_mdns.cc b/src/network_inspectors/appid/service_plugins/service_mdns.cc index c8347f4b1..60263d629 100644 --- a/src/network_inspectors/appid/service_plugins/service_mdns.cc +++ b/src/network_inspectors/appid/service_plugins/service_mdns.cc @@ -21,6 +21,7 @@ #include "service_mdns.h" +#include "appid_module.h" #include "search_engines/search_tool.h" #include "client_plugins/client_app_base.h" #include "detector_plugins/http_url_patterns.h" @@ -311,7 +312,7 @@ static int MDNSUserAnalyser(AppIdData* flowp, const Packet* pkt, uint16_t size, user_index++; } - AppIdAddUser(flowp, user_name, APP_ID_MDNS, 1); + mdns_service_mod.api->add_user(flowp, user_name, APP_ID_MDNS, 1); break; } /* Find the length to Jump to the next response */ @@ -364,7 +365,7 @@ static int MDNSUserAnalyser(AppIdData* flowp, const Packet* pkt, uint16_t size, memcpy(user_name, user_name_bkp + user_index, user_name_len - user_index); user_name[ user_name_len - user_index ] = '\0'; - AppIdAddUser(flowp, user_name, APP_ID_MDNS, 1); + mdns_service_mod.api->add_user(flowp, user_name, APP_ID_MDNS, 1); return 1; } else @@ -427,6 +428,7 @@ static int MDNS_validate(ServiceValidationArgs* args) success: mdns_service_mod.api->add_service(flowp, pkt, args->dir, &svc_element, APP_ID_MDNS, nullptr, nullptr, nullptr); + appid_stats.mdns_flows++; return SERVICE_SUCCESS; fail: diff --git a/src/network_inspectors/appid/service_plugins/service_netbios.cc b/src/network_inspectors/appid/service_plugins/service_netbios.cc index c2057dce4..f0c45c53f 100644 --- a/src/network_inspectors/appid/service_plugins/service_netbios.cc +++ b/src/network_inspectors/appid/service_plugins/service_netbios.cc @@ -29,6 +29,7 @@ #include "main/snort_debug.h" #include "utils/util.h" +#include "appid_module.h" #include "app_info_table.h" #include "appid_api.h" #include "appid_flow_data.h" @@ -614,6 +615,7 @@ static int nbns_validate(ServiceValidationArgs* args) success: netbios_service_mod.api->add_service(flowp, args->pkt, dir, &nbns_svc_element, APP_ID_NETBIOS_NS, nullptr, nullptr, nullptr); + appid_stats.netbios_flows++; return SERVICE_SUCCESS; inprocess: @@ -1067,6 +1069,7 @@ static int nbss_validate(ServiceValidationArgs* args) { netbios_service_mod.api->add_misc(flowp, nd->miscAppId); } + appid_stats.netbios_flows++; } return SERVICE_SUCCESS; @@ -1229,6 +1232,7 @@ success: { netbios_service_mod.api->add_misc(flowp, miscAppId); } + appid_stats.netbios_flows++; } } return SERVICE_SUCCESS; diff --git a/src/network_inspectors/appid/service_plugins/service_timbuktu.cc b/src/network_inspectors/appid/service_plugins/service_timbuktu.cc index 630f44c75..1e4164672 100644 --- a/src/network_inspectors/appid/service_plugins/service_timbuktu.cc +++ b/src/network_inspectors/appid/service_plugins/service_timbuktu.cc @@ -19,6 +19,7 @@ // service_timbuktu.cc author Sourcefire Inc. +#include "appid_module.h" #include "appid_flow_data.h" #include "application_ids.h" #include "service_api.h" @@ -189,6 +190,7 @@ inprocess: success: timbuktu_service_mod.api->add_service(flowp, args->pkt, args->dir, &svc_element, APP_ID_TIMBUKTU, nullptr, nullptr, nullptr); + appid_stats.timbuktu_flows++; return SERVICE_SUCCESS; fail: