From: Russ Combs (rucombs) Date: Tue, 25 Oct 2016 20:45:02 +0000 (-0400) Subject: Merge pull request #682 in SNORT/snort3 from appid_http3 to master X-Git-Tag: 3.0.0-233~210 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b423eccb3609ab72edb1756d87f7786573951934;p=thirdparty%2Fsnort3.git Merge pull request #682 in SNORT/snort3 from appid_http3 to master Squashed commit of the following: commit 7a1b322bb866ec5d5eeb0b91e8419a1722d4028e Author: Steve Chew Date: Wed Oct 19 12:21:28 2016 -0400 Added subscribe/publish communication between HTTP inspector and AppId. Make http_inspect and appid always build statically for now to avoid dependency problem. --- diff --git a/configure.ac b/configure.ac index 96a63d540..83defd843 100644 --- a/configure.ac +++ b/configure.ac @@ -1134,6 +1134,7 @@ src/main/Makefile \ src/managers/Makefile \ src/memory/Makefile \ src/mime/Makefile \ +src/pub_sub/Makefile \ src/stream/Makefile \ src/stream/base/Makefile \ src/stream/ip/Makefile \ @@ -1145,6 +1146,7 @@ src/stream/user/Makefile \ src/stream/file/Makefile \ src/network_inspectors/Makefile \ src/network_inspectors/appid/Makefile \ +src/network_inspectors/appid/test/Makefile \ src/network_inspectors/appid/client_plugins/test/Makefile \ src/network_inspectors/appid/service_plugins/test/Makefile \ src/network_inspectors/arp_spoof/Makefile \ diff --git a/extra/src/inspectors/data_log/data_log.cc b/extra/src/inspectors/data_log/data_log.cc index 5daa5435e..3e645ec29 100644 --- a/extra/src/inspectors/data_log/data_log.cc +++ b/extra/src/inspectors/data_log/data_log.cc @@ -128,7 +128,7 @@ void DataLog::show(SnortConfig*) static const Parameter dl_params[] = { - { "key", Parameter::PT_STRING, nullptr, "http_uri", + { "key", Parameter::PT_SELECT, "http_uri | http_raw_uri", "http_raw_uri", "name of data buffer to log" }, { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fc78cf39e..7da85f453 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -43,15 +43,15 @@ include_directories(BEFORE network_inspectors) # Build options for specific libraries # FIXIT is this needed? see also -# STATIC_INSECTOR_LIBS in service_inspectors/CMakeLists.txt +# STATIC_INSPECTOR_LIBS in service_inspectors/CMakeLists.txt if (STATIC_INSPECTORS) set (STATIC_INSPECTOR_LIBRARIES - appid +# appid arp_spoof back_orifice dns ftp_telnet - http_inspect +# http_inspect rpc_decode sip ssh @@ -97,6 +97,8 @@ target_link_libraries( snort ${STATIC_CODEC_LIBRARIES} network_inspectors service_inspectors + appid + http_inspect ${STATIC_INSPECTOR_LIBRARIES} port_scan reputation @@ -123,6 +125,7 @@ target_link_libraries( snort codec_module memory host_tracker + pub_sub parser flow side_channel @@ -178,6 +181,7 @@ add_subdirectory(service_inspectors) add_subdirectory(stream) add_subdirectory(target_based) add_subdirectory(host_tracker) +add_subdirectory(pub_sub) add_subdirectory(time) add_subdirectory(profiler) add_subdirectory(utils) diff --git a/src/Makefile.am b/src/Makefile.am index a2e3ca2af..78a327599 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,6 @@ main.h if STATIC_INSPECTORS lib_list = \ -network_inspectors/appid/libappid.a \ network_inspectors/arp_spoof/libarp_spoof.a \ network_inspectors/packet_capture/libpacket_capture.a \ service_inspectors/back_orifice/libback_orifice.a \ @@ -17,7 +16,6 @@ service_inspectors/dns/libdns.a \ service_inspectors/ftp_telnet/libftp_telnet.a \ service_inspectors/gtp/libgtp_inspect.a \ service_inspectors/modbus/libmodbus.a \ -service_inspectors/http_inspect/libhttp_inspect.a \ service_inspectors/rpc_decode/librpc_decode.a \ service_inspectors/sip/libsip.a \ service_inspectors/ssh/libssh.a \ @@ -58,7 +56,10 @@ network_inspectors/normalize/libnormalize.a \ network_inspectors/perf_monitor/libperf_monitor.a \ network_inspectors/reputation/libreputation.a \ service_inspectors/libservice_inspectors.a \ +network_inspectors/appid/libappid.a \ +service_inspectors/http_inspect/libhttp_inspect.a \ $(lib_list) \ +pub_sub/libpub_sub.a \ service_inspectors/imap/libimap.a \ service_inspectors/pop/libpop.a \ service_inspectors/smtp/libsmtp.a \ @@ -139,6 +140,7 @@ packet_io \ parser \ ports \ protocols \ +pub_sub \ search_engines \ service_inspectors \ sfip \ diff --git a/src/connectors/file_connector/test/file_connector_module_test.cc b/src/connectors/file_connector/test/file_connector_module_test.cc index 5f843ab1d..eb51caeee 100644 --- a/src/connectors/file_connector/test/file_connector_module_test.cc +++ b/src/connectors/file_connector/test/file_connector_module_test.cc @@ -36,7 +36,9 @@ void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*) { } void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*, FILE*) { } +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif char* snort_strdup(const char* s) { return strdup(s); } diff --git a/src/connectors/file_connector/test/file_connector_test.cc b/src/connectors/file_connector/test/file_connector_test.cc index fd8287277..d39f72322 100644 --- a/src/connectors/file_connector/test/file_connector_test.cc +++ b/src/connectors/file_connector/test/file_connector_test.cc @@ -53,7 +53,9 @@ void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*, FILE*) { } const char* get_instance_file(std::string& file, const char* name) { file += name; return nullptr; } +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif FileConnectorModule::FileConnectorModule() : Module("FC", "FC Help", nullptr) diff --git a/src/connectors/tcp_connector/test/tcp_connector_module_test.cc b/src/connectors/tcp_connector/test/tcp_connector_module_test.cc index 8dde7d319..9685e562a 100644 --- a/src/connectors/tcp_connector/test/tcp_connector_module_test.cc +++ b/src/connectors/tcp_connector/test/tcp_connector_module_test.cc @@ -36,7 +36,9 @@ void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*) { } void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*, FILE*) { } +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif char* snort_strdup(const char* s) { return strdup(s); } diff --git a/src/connectors/tcp_connector/test/tcp_connector_test.cc b/src/connectors/tcp_connector/test/tcp_connector_test.cc index d4f02a7d1..0c60258c9 100644 --- a/src/connectors/tcp_connector/test/tcp_connector_test.cc +++ b/src/connectors/tcp_connector/test/tcp_connector_test.cc @@ -75,7 +75,9 @@ void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*, FILE*) { } unsigned get_instance_id() { return s_instance; } +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif void ErrorMessage(const char*, ...) { } void LogMessage(const char*, ...) { } diff --git a/src/flow/test/ha_module_test.cc b/src/flow/test/ha_module_test.cc index 32dd569dd..684f07ec9 100644 --- a/src/flow/test/ha_module_test.cc +++ b/src/flow/test/ha_module_test.cc @@ -42,7 +42,9 @@ void ParseWarning(WarningGroup, const char*, ...) { } char* snort_strdup(const char* str) { return strdup(str); } +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif static bool s_port_1_set = false; static bool s_use_daq = false; diff --git a/src/flow/test/ha_test.cc b/src/flow/test/ha_test.cc index 81ece5f41..71894ec96 100644 --- a/src/flow/test/ha_test.cc +++ b/src/flow/test/ha_test.cc @@ -148,7 +148,9 @@ void Stream::delete_flow(const FlowKey* flowkey) void ErrorMessage(const char*,...) { } void LogMessage(const char*,...) { } +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif void packet_gettimeofday(struct timeval* tv) { *tv = s_packet_time; } diff --git a/src/network_inspectors/appid/CMakeLists.txt b/src/network_inspectors/appid/CMakeLists.txt index 0646818a9..41bc93c01 100644 --- a/src/network_inspectors/appid/CMakeLists.txt +++ b/src/network_inspectors/appid/CMakeLists.txt @@ -158,6 +158,8 @@ set ( APPID_SOURCES host_port_app_cache.cc host_port_app_cache.h http_common.h + appid_http_event_handler.cc + appid_http_event_handler.h ips_appid_option.cc length_app_cache.cc length_app_cache.h @@ -177,7 +179,7 @@ set ( APPID_SOURCES ) -if (STATIC_INSPECTORS) +#if (STATIC_INSPECTORS) add_library(appid STATIC ${APPID_SOURCES} ${CP_APPID_SOURCES} @@ -185,20 +187,21 @@ if (STATIC_INSPECTORS) ${SP_APPID_SOURCES} ${UTIL_APPID_SOURCES} ) -else (STATIC_INSPECTORS) - add_shared_library(appid inspectors - ${APPID_SOURCES} - ${CP_APPID_SOURCES} - ${DP_APPID_SOURCES} - ${SP_APPID_SOURCES} - ${UTIL_APPID_SOURCES} - ) -endif (STATIC_INSPECTORS) +#else (STATIC_INSPECTORS) +# add_shared_library(appid inspectors +# ${APPID_SOURCES} +# ${CP_APPID_SOURCES} +# ${DP_APPID_SOURCES} +# ${SP_APPID_SOURCES} +# ${UTIL_APPID_SOURCES} +# ) +#endif (STATIC_INSPECTORS) target_include_directories ( appid PRIVATE ${APPID_INCLUDE_DIR} ) add_subdirectory(service_plugins/test) add_subdirectory(client_plugins/test) +add_subdirectory(test) #install (FILES ${APPID_INCLUDES} # DESTINATION "${INCLUDE_INSTALL_PATH}/appid" diff --git a/src/network_inspectors/appid/Makefile.am b/src/network_inspectors/appid/Makefile.am index fc2cfbb64..44f94fc85 100644 --- a/src/network_inspectors/appid/Makefile.am +++ b/src/network_inspectors/appid/Makefile.am @@ -154,6 +154,8 @@ fw_appid.h \ host_port_app_cache.cc \ host_port_app_cache.h \ http_common.h \ +appid_http_event_handler.cc \ +appid_http_event_handler.h \ ips_appid_option.cc \ length_app_cache.cc \ length_app_cache.h \ @@ -171,27 +173,28 @@ thirdparty_appid_types.h \ thirdparty_appid_utils.cc \ thirdparty_appid_utils.h -if STATIC_INSPECTORS +#if STATIC_INSPECTORS noinst_LIBRARIES = libappid.a libappid_a_SOURCES = $(file_list) \ $(cp_file_list) \ $(sp_file_list) \ $(dp_file_list) \ $(util_file_list) -else -shlibdir = $(pkglibdir)/inspectors -shlib_LTLIBRARIES = libappid.la -libappid_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO -libappid_la_LDFLAGS = $(AM_LDFLAGS) -export-dynamic -shared -libappid_la_SOURCES = $(file_list) \ -$(cp_file_list) \ -$(sp_file_list) \ -$(dp_file_list) \ -$(util_file_list) -endif +#else +#shlibdir = $(pkglibdir)/inspectors +#shlib_LTLIBRARIES = libappid.la +#libappid_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO +#libappid_la_LDFLAGS = $(AM_LDFLAGS) -export-dynamic -shared +#libappid_la_SOURCES = $(file_list) \ +#$(cp_file_list) \ +#$(sp_file_list) \ +#$(dp_file_list) \ +#$(util_file_list) +#endif if ENABLE_UNIT_TESTS -SUBDIRS=service_plugins/test \ +SUBDIRS=test\ +service_plugins/test \ client_plugins/test endif diff --git a/src/network_inspectors/appid/appid_http_event_handler.cc b/src/network_inspectors/appid/appid_http_event_handler.cc new file mode 100644 index 000000000..0c6ef2313 --- /dev/null +++ b/src/network_inspectors/appid/appid_http_event_handler.cc @@ -0,0 +1,133 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2016-2016 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// appid_http_event_handler.cc author Steve Chew + +// Receive events from the HTTP inspector containing header information +// to be used to detect AppIds. + +#include "appid_http_event_handler.h" +#include "appid_config.h" +#include "appid_session.h" +#include "appid_module.h" +#include "thirdparty_appid_utils.h" +#include "utils/util.h" + +static void replace_header_data(char **data, const uint8_t *header_start, + unsigned header_length) +{ + if(header_length <= 0) + return; + + assert(data); + if(*data) + snort_free(*data); + + *data = snort_strndup((char*)header_start, header_length); +} + +void HttpEventHandler::handle(DataEvent& event, Flow* flow) +{ + AppIdSession* session; + int direction; + const uint8_t* header_start; + unsigned header_length; + HttpEvent* http_event = (HttpEvent*)&event; + + assert(flow); + session = appid_api.get_appid_data(flow); + if (!session) + return; + + direction = event_type == REQUEST_EVENT ? APP_ID_FROM_INITIATOR : APP_ID_FROM_RESPONDER; + + if (!session->hsession) + session->hsession = (decltype(session->hsession))snort_calloc(sizeof(httpSession)); + + if (direction == APP_ID_FROM_INITIATOR) + { + header_start = http_event->get_host(header_length); + if(header_length > 0) + { + replace_header_data(&session->hsession->host, header_start, + header_length); + session->scan_flags |= SCAN_HTTP_HOST_URL_FLAG; + + header_start = http_event->get_uri(header_length); + replace_header_data(&session->hsession->url, header_start, + header_length); + } + + header_start = http_event->get_user_agent(header_length); + if(header_length > 0) + { + replace_header_data(&session->hsession->useragent, header_start, + header_length); + session->scan_flags |= SCAN_HTTP_USER_AGENT_FLAG; + } + + header_start = http_event->get_referer(header_length); + replace_header_data(&session->hsession->referer, header_start, + header_length); + + header_start = http_event->get_x_working_with(header_length); + replace_header_data(&session->hsession->x_working_with, header_start, + header_length); + } + else // Response headers. + { + header_start = http_event->get_content_type(header_length); + replace_header_data(&session->hsession->content_type, header_start, + header_length); + + header_start = http_event->get_server(header_length); + replace_header_data(&session->hsession->server, header_start, + header_length); + + int32_t responseCodeNum = http_event->get_response_code(); + if (responseCodeNum > 0 && responseCodeNum < 700) + { + unsigned int ret; + char tmpstr[32]; + ret = snprintf(tmpstr, sizeof(tmpstr), "%d", responseCodeNum); + if(ret < sizeof(tmpstr)) + { + snort_free(session->hsession->response_code); + session->hsession->response_code = snort_strdup(tmpstr); + } + } + } + + // The Via header can be in both the request and response. + header_start = http_event->get_via(header_length); + if(header_length > 0) + { + replace_header_data(&session->hsession->via, header_start, + header_length); + session->scan_flags |= SCAN_HTTP_VIA_FLAG; + } + + session->processHTTPPacket(direction); + session->set_session_flags(APPID_SESSION_SERVICE_DETECTED | APPID_SESSION_HTTP_SESSION); + if (direction == APP_ID_FROM_INITIATOR) + appid_stats.http_flows++; + flow->set_application_ids(session->pick_service_app_id(), + session->pick_client_app_id(), session->pick_payload_app_id(), + session->pick_misc_app_id()); +} + diff --git a/src/network_inspectors/appid/appid_http_event_handler.h b/src/network_inspectors/appid/appid_http_event_handler.h new file mode 100644 index 000000000..c87509702 --- /dev/null +++ b/src/network_inspectors/appid/appid_http_event_handler.h @@ -0,0 +1,52 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2016-2016 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// appid_http_event_handler.h author Steve Chew + +// Receive events from the HTTP inspector containing header information +// to be used to detect AppIds. + +#ifndef APPID_HTTP_EVENT_HANDLER_H +#define APPID_HTTP_EVENT_HANDLER_H + +#include "framework/data_bus.h" + +#include "pub_sub/http_events.h" + +class HttpEventHandler : public DataHandler +{ +public: + enum HttpEventType + { + REQUEST_EVENT, + RESPONSE_EVENT, + }; + + HttpEventHandler(HttpEventType type) + { + event_type = type; + } + + void handle(DataEvent&, Flow*); + +private: + HttpEventType event_type; +}; + +#endif + diff --git a/src/network_inspectors/appid/appid_inspector.cc b/src/network_inspectors/appid/appid_inspector.cc index e123b4d15..27b914360 100644 --- a/src/network_inspectors/appid/appid_inspector.cc +++ b/src/network_inspectors/appid/appid_inspector.cc @@ -43,6 +43,7 @@ #include "detector_plugins/detector_http.h" #include "detector_plugins/detector_sip.h" #include "detector_plugins/detector_pattern.h" +#include "appid_http_event_handler.h" THREAD_LOCAL LuaDetectorManager* lua_detector_mgr; @@ -77,6 +78,10 @@ bool AppIdInspector::configure(SnortConfig*) active_config = new AppIdConfig( ( AppIdModuleConfig* )config); if(config->debug) show(nullptr); + + get_data_bus().subscribe(HTTP_REQUEST_HEADER_EVENT_KEY, new HttpEventHandler(HttpEventHandler::REQUEST_EVENT)); + get_data_bus().subscribe(HTTP_RESPONSE_HEADER_EVENT_KEY, new HttpEventHandler(HttpEventHandler::RESPONSE_EVENT)); + return active_config->init_appid(); // FIXIT-M some of this stuff may be needed in some fashion... diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 12396b316..a591f8603 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -56,6 +56,7 @@ const PegInfo appid_pegs[] = { "dns_udp_flows", "count of dns flows over udp discovered by appid" }, { "ftp_flows", "count of ftp flows discovered by appid" }, { "ftps_flows", "count of ftps flows discovered by appid" }, + { "http_flows", "count of http flows discovered by appid" }, { "imap_flows", "count of imap service flows discovered by appid" }, { "imaps_flows", "count of imap TLS service flows discovered by appid" }, { "irc_flows", "count of irc service flows discovered by appid" }, diff --git a/src/network_inspectors/appid/appid_module.h b/src/network_inspectors/appid/appid_module.h index 7c3df0265..204e7b490 100644 --- a/src/network_inspectors/appid/appid_module.h +++ b/src/network_inspectors/appid/appid_module.h @@ -50,6 +50,7 @@ struct AppIdStats PegCount dns_udp_flows; PegCount ftp_flows; PegCount ftps_flows; + PegCount http_flows; PegCount imap_flows; PegCount imaps_flows; PegCount irc_flows; diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc index e9416ff32..3f1b46b10 100644 --- a/src/network_inspectors/appid/appid_session.cc +++ b/src/network_inspectors/appid/appid_session.cc @@ -441,9 +441,9 @@ bool AppIdSession::is_packet_ignored(Packet* p) return false; } -#ifdef REMOVED_WHILE_NOT_IN_USE static int ptype_scan_counts[NUMBER_OF_PTYPES]; +#ifdef REMOVED_WHILE_NOT_IN_USE void AppIdSession::ProcessThirdPartyResults(Packet* p, int confidence, AppId* proto_list, ThirdPartyAppIDAttributeData* attribute_data) { @@ -1137,6 +1137,76 @@ bool AppIdSession::do_third_party_discovery(IpProtocol protocol, const sfip_t* i return isTpAppidDiscoveryDone; } + +void AppIdSession::pickHttpXffAddress(Packet*, ThirdPartyAppIDAttributeData* attribute_data) +{ + int i; + static const char* defaultXffPrecedence[] = + { + HTTP_XFF_FIELD_X_FORWARDED_FOR, + HTTP_XFF_FIELD_TRUE_CLIENT_IP + }; + + // XFF precedence configuration cannot change for a session. Do not get it again if we already + // got it. +// FIXIT-M: +#ifdef REMOVED_WHILE_NOT_IN_USE + if (!hsession->xffPrecedence) + hsession->xffPrecedence = _dpd.sessionAPI->get_http_xff_precedence( + p->flow, p->packet_flags, &hsession->numXffFields); +#endif + + if (!hsession->xffPrecedence) + { + hsession->xffPrecedence = defaultXffPrecedence; + hsession->numXffFields = sizeof(defaultXffPrecedence) / + sizeof(defaultXffPrecedence[0]); + } + + if (session_logging_enabled) + { + for (i = 0; i < attribute_data->numXffFields; i++) + LogMessage("AppIdDbg %s %s : %s\n", app_id_debug_session, + attribute_data->xffFieldValue[i].field, attribute_data->xffFieldValue[i].value); + } + + // xffPrecedence array is sorted based on precedence + for (i = 0; (i < hsession->numXffFields) && + hsession->xffPrecedence[i]; i++) + { + int j; + for (j = 0; j < attribute_data->numXffFields; j++) + { + if (hsession->xffAddr) + sfip_free(hsession->xffAddr); + + if (strncasecmp(attribute_data->xffFieldValue[j].field, + hsession->xffPrecedence[i], UINT8_MAX) == 0) + { + char* tmp = strchr(attribute_data->xffFieldValue[j].value, ','); + SFIP_RET status; + + if (!tmp) + { + hsession->xffAddr = sfip_alloc( + attribute_data->xffFieldValue[j].value, &status); + } + // For a comma-separated list of addresses, pick the first address + else + { + attribute_data->xffFieldValue[j].value[tmp - + attribute_data->xffFieldValue[j].value] = '\0'; + hsession->xffAddr = sfip_alloc( + attribute_data->xffFieldValue[j].value, &status); + } + break; + } + } + if (hsession->xffAddr) + break; + } +} + #endif bool AppIdSession::do_service_discovery(IpProtocol protocol, int direction, AppId ClientAppId, @@ -2450,6 +2520,11 @@ void AppIdSession::clear_http_field() if (hsession == nullptr) return; + if (hsession->x_working_with) + { + snort_free(hsession->x_working_with); + hsession->x_working_with = nullptr; + } if (hsession->referer) { snort_free(hsession->referer); @@ -2544,6 +2619,11 @@ void AppIdSession::free_http_session_data() snort_free(hsession->response_code); hsession->response_code = nullptr; } + if (hsession->server) + { + snort_free(hsession->server); + hsession->server = nullptr; + } snort_free(hsession); hsession = nullptr; @@ -2928,8 +3008,6 @@ bool AppIdSession::is_ssl_session_decrypted() return get_session_flags(APPID_SESSION_DECRYPTED); } -#ifdef REMOVED_WHILE_NOT_IN_USE - static const char* httpFieldName[ NUMBER_OF_PTYPES ] = // for use in debug messages { "useragent", @@ -3068,6 +3146,22 @@ int AppIdSession::initial_CHP_sweep(char** chp_buffers, MatchedCHPAction** ppmat return 1; } +bool AppIdSession::is_payload_appid_set() +{ + return ( payload_app_id || tp_payload_app_id ); +} + +void AppIdSession::clearMiscHttpFlags() +{ + if (!get_session_flags(APPID_SESSION_SPDY_SESSION)) + { + clear_session_flags(APPID_SESSION_CHP_INSPECTING); + if (thirdparty_appid_module) + thirdparty_appid_module->session_attr_clear(tpsession, + TP_ATTR_CONTINUE_MONITORING); + } +} + void AppIdSession::processCHP(char** version, Packet* p) { int i, size; @@ -3273,97 +3367,13 @@ void AppIdSession::processCHP(char** version, Packet* p) } } -bool AppIdSession::is_payload_appid_set() -{ - return ( payload_app_id || tp_payload_app_id ); -} - -void AppIdSession::clearMiscHttpFlags() -{ - if (!get_session_flags(APPID_SESSION_SPDY_SESSION)) - { - clear_session_flags(APPID_SESSION_CHP_INSPECTING); - if (thirdparty_appid_module) - thirdparty_appid_module->session_attr_clear(tpsession, - TP_ATTR_CONTINUE_MONITORING); - } -} - -void AppIdSession::pickHttpXffAddress(Packet*, ThirdPartyAppIDAttributeData* attribute_data) -{ - int i; - static const char* defaultXffPrecedence[] = - { - HTTP_XFF_FIELD_X_FORWARDED_FOR, - HTTP_XFF_FIELD_TRUE_CLIENT_IP - }; - - // XFF precedence configuration cannot change for a asd. Do not get it again if we already - // got it. -#ifdef REMOVED_WHILE_NOT_IN_USE - if (!hsession->xffPrecedence) - hsession->xffPrecedence = _dpd.sessionAPI->get_http_xff_precedence( - p->flow, p->packet_flags, &hsession->numXffFields); -#endif - - if (!hsession->xffPrecedence) - { - hsession->xffPrecedence = defaultXffPrecedence; - hsession->numXffFields = sizeof(defaultXffPrecedence) / - sizeof(defaultXffPrecedence[0]); - } - - if (session_logging_enabled) - { - for (i = 0; i < attribute_data->numXffFields; i++) - LogMessage("AppIdDbg %s %s : %s\n", session_logging_id, - attribute_data->xffFieldValue[i].field, attribute_data->xffFieldValue[i].value); - } - - // xffPrecedence array is sorted based on precedence - for (i = 0; (i < hsession->numXffFields) && - hsession->xffPrecedence[i]; i++) - { - int j; - for (j = 0; j < attribute_data->numXffFields; j++) - { - if (hsession->xffAddr) - sfip_free(hsession->xffAddr); - - if (strncasecmp(attribute_data->xffFieldValue[j].field, - hsession->xffPrecedence[i], UINT8_MAX) == 0) - { - char* tmp = strchr(attribute_data->xffFieldValue[j].value, ','); - SFIP_RET status; - - if (!tmp) - { - hsession->xffAddr = sfip_alloc( - attribute_data->xffFieldValue[j].value, &status); - } - // For a comma-separated list of addresses, pick the first address - else - { - attribute_data->xffFieldValue[j].value[tmp - - attribute_data->xffFieldValue[j].value] = '\0'; - hsession->xffAddr = sfip_alloc( - attribute_data->xffFieldValue[j].value, &status); - } - break; - } - } - if (hsession->xffAddr) - break; - } -} - -int AppIdSession::processHTTPPacket(Packet* p, int direction, HttpParsedHeaders* const) +int AppIdSession::processHTTPPacket(int direction) { Profile http_profile_context(httpPerfStats); constexpr auto RESPONSE_CODE_LENGTH = 3; HeaderMatchedPatterns hmp; httpSession* http_session; - int start, end, size; + int size; char* version = nullptr; char* vendorVersion = nullptr; char* vendor = nullptr; @@ -3448,7 +3458,7 @@ int AppIdSession::processHTTPPacket(Packet* p, int direction, HttpParsedHeaders* http_session->chp_finished, http_session->chp_hold_flow); if (!http_session->chp_finished || http_session->chp_hold_flow) - processCHP(&version, p); + processCHP(&version, nullptr); if (!http_session->skip_simple_detect) // false unless a match happened with a call to // processCHP(). @@ -3456,23 +3466,20 @@ int AppIdSession::processHTTPPacket(Packet* p, int direction, HttpParsedHeaders* if (!get_session_flags(APPID_SESSION_APP_REINSPECT)) { // Scan Server Header for Vendor & Version - if ( (thirdparty_appid_module && (scan_flags & SCAN_HTTP_VENDOR_FLAG) && + + // FIXIT-M: Should we be checking the scan_flags even when + // thirdparty_appid_module is off? + if ((thirdparty_appid_module && (scan_flags & SCAN_HTTP_VENDOR_FLAG) && hsession->server) || - (!thirdparty_appid_module && - get_http_header_location(p->data, p->dsize, HTTP_ID_SERVER, - &start, &end, &hmp) == 1) ) + (!thirdparty_appid_module && hsession->server)) { if (serviceAppId == APP_ID_NONE || serviceAppId == APP_ID_HTTP) { - RNAServiceSubtype* subtype = nullptr; + RNAServiceSubtype* local_subtype = nullptr; RNAServiceSubtype** tmpSubtype; - if (thirdparty_appid_module) - get_server_vendor_version((uint8_t*)hsession->server, + get_server_vendor_version((uint8_t*)hsession->server, strlen(hsession->server), &vendorVersion, &vendor, &subtype); - else - get_server_vendor_version(p->data + start, end - start, &vendorVersion, - &vendor, &subtype); if (vendor || vendorVersion) { if (serviceVendor) @@ -3491,13 +3498,13 @@ int AppIdSession::processHTTPPacket(Packet* p, int direction, HttpParsedHeaders* serviceVersion = vendorVersion; scan_flags &= ~SCAN_HTTP_VENDOR_FLAG; } - if (subtype) + if (local_subtype) { for (tmpSubtype = &subtype; *tmpSubtype; tmpSubtype = &(*tmpSubtype)->next) ; - *tmpSubtype = subtype; + *tmpSubtype = local_subtype; } } } @@ -3553,19 +3560,16 @@ int AppIdSession::processHTTPPacket(Packet* p, int direction, HttpParsedHeaders* } /* Scan X-Working-With HTTP header */ + // FIXIT-M: Should we be checking the scan_flags even when + // thirdparty_appid_module is off? if ((thirdparty_appid_module && (scan_flags & SCAN_HTTP_XWORKINGWITH_FLAG) && hsession->x_working_with) || - (!thirdparty_appid_module && get_http_header_location(p->data, p->dsize, - HTTP_ID_X_WORKING_WITH, &start, &end, &hmp) == 1)) + (!thirdparty_appid_module && hsession->x_working_with)) { AppId appId; - if (thirdparty_appid_module) - appId = scan_header_x_working_with((uint8_t*)hsession->x_working_with, + appId = scan_header_x_working_with((uint8_t*)hsession->x_working_with, strlen(hsession->x_working_with), &version); - else - appId = scan_header_x_working_with(p->data + start, end - start, &version); - if (appId) { if (direction == APP_ID_FROM_INITIATOR) @@ -3587,17 +3591,15 @@ int AppIdSession::processHTTPPacket(Packet* p, int direction, HttpParsedHeaders* } // Scan Content-Type Header for multimedia types and scan contents + // FIXIT-M: Should we be checking the scan_flags even when + // thirdparty_appid_module is off? if ((thirdparty_appid_module && (scan_flags & SCAN_HTTP_CONTENT_TYPE_FLAG) && hsession->content_type && !is_payload_appid_set()) || (!thirdparty_appid_module && !is_payload_appid_set() && - get_http_header_location(p->data, p->dsize, HTTP_ID_CONTENT_TYPE, - &start, &end, &hmp) == 1)) + hsession->content_type)) { - if (thirdparty_appid_module) - payload_id = get_appid_by_content_type((uint8_t*)hsession->content_type, + payload_id = get_appid_by_content_type((uint8_t*)hsession->content_type, strlen(hsession->content_type)); - else - payload_id = get_appid_by_content_type(p->data + start, end - start); if (session_logging_enabled && payload_id > APP_ID_NONE && payload_app_id != payload_id) LogMessage("AppIdDbg %s Content-Type is data %d\n", session_logging_id, @@ -3670,6 +3672,7 @@ int AppIdSession::processHTTPPacket(Packet* p, int direction, HttpParsedHeaders* clearMiscHttpFlags(); } // end DON'T skip_simple_detect + snort_free(version); return 0; } -#endif + diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h index 63d18dfeb..8e05b525a 100644 --- a/src/network_inspectors/appid/appid_session.h +++ b/src/network_inspectors/appid/appid_session.h @@ -203,6 +203,7 @@ public: static AppIdSession* create_future_session(const Packet*, const sfip_t*, uint16_t, const sfip_t*, uint16_t, IpProtocol, int16_t, int); static void do_application_discovery(Packet*); + int processHTTPPacket(int); AppIdConfig* config = nullptr; CommonAppIdData common; @@ -371,21 +372,18 @@ private: void set_payload_app_id_data( ApplicationId, char**); void stop_rna_service_inspection(Packet*, int); void set_session_logging_state(const Packet* pkt, int direction); + void clear_app_id_data(); + int initial_CHP_sweep(char**, MatchedCHPAction**); + void clearMiscHttpFlags(); + void processCHP(char**, Packet*); #ifdef REMOVED_WHILE_NOT_IN_USE // FIXIT-M these are not needed until appid for snort3 supports 3rd party detectors (e.g. NAVL) void ProcessThirdPartyResults(Packet*, int, AppId*, ThirdPartyAppIDAttributeData*); void checkTerminateTpModule(uint16_t tpPktCount); bool do_third_party_discovery(IpProtocol, const sfip_t*, Packet*, int&); - - // FIXIT-H when http detection is made functional we need to look at these methods and determine if they are - // needed and what changes are required for snort3 - void clear_app_id_data(); void pickHttpXffAddress(Packet*, ThirdPartyAppIDAttributeData*); - int initial_CHP_sweep(char**, MatchedCHPAction**); - void clearMiscHttpFlags(); - int processHTTPPacket(Packet*, int, HttpParsedHeaders* const); - void processCHP(char**, Packet*); + #endif void create_session_logging_id(int direction, Packet* pkt); diff --git a/src/network_inspectors/appid/client_plugins/test/client_app_smtp_test.cc b/src/network_inspectors/appid/client_plugins/test/client_app_smtp_test.cc index 25bede943..7768d032c 100644 --- a/src/network_inspectors/appid/client_plugins/test/client_app_smtp_test.cc +++ b/src/network_inspectors/appid/client_plugins/test/client_app_smtp_test.cc @@ -27,7 +27,9 @@ #include +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif struct AddAppData { diff --git a/src/network_inspectors/appid/detector_plugins/detector_http.cc b/src/network_inspectors/appid/detector_plugins/detector_http.cc index 464089350..eaaf62610 100644 --- a/src/network_inspectors/appid/detector_plugins/detector_http.cc +++ b/src/network_inspectors/appid/detector_plugins/detector_http.cc @@ -55,12 +55,12 @@ #define HTTP_PUT_SIZE (sizeof(HTTP_PUT)-1) #define HTTP_POST_SIZE (sizeof(HTTP_POST)-1) #define HTTP_HEAD_SIZE (sizeof(HTTP_HEAD)-1) -#define HTTP_TRACE_SIZE (sizeof(HTTP_GET)-1) +#define HTTP_TRACE_SIZE (sizeof(HTTP_TRACE)-1) #define HTTP_DELETE_SIZE (sizeof(HTTP_DELETE)-1) #define HTTP_OPTIONS_SIZE (sizeof(HTTP_OPTIONS)-1) #define HTTP_PROPFIND_SIZE (sizeof(HTTP_PROPFIND)-1) #define HTTP_PROPPATCH_SIZE (sizeof(HTTP_PROPPATCH)-1) -#define HTTP_MKCOL_SIZE (sizeof(HTTP_GET)-1) +#define HTTP_MKCOL_SIZE (sizeof(HTTP_MKCOL)-1) #define HTTP_COPY_SIZE (sizeof(HTTP_COPY)-1) #define HTTP_MOVE_SIZE (sizeof(HTTP_MOVE)-1) #define HTTP_LOCK_SIZE (sizeof(HTTP_LOCK)-1) @@ -1123,6 +1123,9 @@ static IpProtocol ffSetProtocol(char* buf, int buf_size, int start, int psize) return (IpProtocol)temp_protocol; } +#if MUST_FIX + // FIXIT-H: We do not have a packet when we get called from + // the HTTP inspector. Is there an alternative? static void fflowCreate(char* adata, fflow_info* fflow, Packet* p, AppId target_appid) { char* saddr_string = nullptr; @@ -1316,6 +1319,7 @@ static void fflowCreate(char* adata, fflow_info* fflow, Packet* p, AppId target_ fflow->flow_prepared = 1; } +#endif void finalize_fflow(fflow_info* fflow, unsigned app_type_flags, AppId target_appId, Packet* p) { @@ -1353,7 +1357,7 @@ void scan_key_chp(PatternType ptype, char* buf, int buf_size, CHPTallyAndActions } AppId scan_chp(PatternType ptype, char* buf, int buf_size, MatchedCHPAction* mp, char** version, - char** user, char** new_field, int* total_found, httpSession* hsession, Packet* p) + char** user, char** new_field, int* total_found, httpSession* hsession, Packet*) { MatchedCHPAction* second_sweep_for_inserts = nullptr; int do_not_further_modify_field = 0; @@ -1487,7 +1491,11 @@ AppId scan_chp(PatternType ptype, char* buf, int buf_size, MatchedCHPAction* mp, break; if (!hsession->fflow) hsession->fflow = (fflow_info*)snort_calloc(sizeof(fflow_info)); +#if MUST_FIX + // FIXIT-H: We do not have a packet when we get called from + // the HTTP inspector. Is there an alternative? fflowCreate(match->action_data, hsession->fflow, p, hsession->chp_candidate); +#endif break; case INSERT_FIELD: diff --git a/src/network_inspectors/appid/service_plugins/test/service_rsync_test.cc b/src/network_inspectors/appid/service_plugins/test/service_rsync_test.cc index 963c9f6a6..235f69305 100644 --- a/src/network_inspectors/appid/service_plugins/test/service_rsync_test.cc +++ b/src/network_inspectors/appid/service_plugins/test/service_rsync_test.cc @@ -25,7 +25,9 @@ #include #include +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif extern int rsync_validate(ServiceValidationArgs*); diff --git a/src/network_inspectors/appid/test/CMakeLists.txt b/src/network_inspectors/appid/test/CMakeLists.txt new file mode 100644 index 000000000..8edf550c3 --- /dev/null +++ b/src/network_inspectors/appid/test/CMakeLists.txt @@ -0,0 +1,8 @@ + +add_library(appid_test_depends_on_lib ../appid_stats_counter.cc) + +add_cpputest(appid_http_event_test appid_test_depends_on_lib) + +include_directories ( appid PRIVATE ${APPID_INCLUDE_DIR} ) + + diff --git a/src/network_inspectors/appid/test/Makefile.am b/src/network_inspectors/appid/test/Makefile.am new file mode 100644 index 000000000..9d6586dc9 --- /dev/null +++ b/src/network_inspectors/appid/test/Makefile.am @@ -0,0 +1,14 @@ + +AM_DEFAULT_SOURCE_EXT = .cc + +check_PROGRAMS = \ +appid_http_event_test + +TESTS = $(check_PROGRAMS) + +appid_http_event_test_CPPFLAGS = -I$(top_srcdir)/src/network_inspectors/appid @AM_CPPFLAGS@ @CPPUTEST_CPPFLAGS@ + +appid_http_event_test_LDADD = \ +../appid_stats_counter.o \ +@CPPUTEST_LDFLAGS@ + diff --git a/src/network_inspectors/appid/test/appid_http_event_test.cc b/src/network_inspectors/appid/test/appid_http_event_test.cc new file mode 100644 index 000000000..34f219a75 --- /dev/null +++ b/src/network_inspectors/appid/test/appid_http_event_test.cc @@ -0,0 +1,557 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2016-2016 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// appid_http_event_test.cc author Steve Chew +// unit test for the http inspection event handler. + +// Must be included before CppUTest files to avoid compiler error. +#include "network_inspectors/appid/appid_http_event_handler.cc" + +#include +#include +#include + +#include + +#include "framework/data_bus.h" +#include "protocols/protocol_ids.h" +#include "service_inspectors/http_inspect/http_msg_header.h" + +AppIdConfig* pAppidActiveConfig = nullptr; +AppIdApi appid_api; +THREAD_LOCAL ThirdPartyAppIDModule* thirdparty_appid_module = nullptr; + +char* snort_strndup(const char* src, size_t dst_size) +{ + return strndup(src, dst_size); +} + +char* snort_strdup(const char* src) +{ + return strdup(src); +} + +FlowData::FlowData(unsigned, Inspector *) +{ +} + +FlowData::~FlowData() +{ +} + +void Flow::set_application_ids(AppId, AppId, AppId, AppId) { } + +const char *content_type = nullptr; +const char *host = nullptr; +const char *referer = nullptr; +int32_t response_code = 0; +const char *server = nullptr; +const char *x_working_with = nullptr; +const char *url = nullptr; +const char *useragent = nullptr; +const char *via = nullptr; + +void Field::set(int32_t length_, const uint8_t* start_) +{ + start = start_; + length = length_; +} + +Field global_field; + +const Field& HttpMsgSection::get_classic_buffer(unsigned id, uint64_t sub_id, uint64_t) +{ + global_field.set(0, nullptr); + if(id == HttpEnums::HTTP_BUFFER_HEADER) + { + if(sub_id == HttpEnums::HEAD_HOST) + { + if(host) + global_field.set(strlen(host), (const uint8_t*)host); + } + + if(sub_id == HttpEnums::HEAD_USER_AGENT) + { + if(useragent) + global_field.set(strlen(useragent), (const uint8_t*)useragent); + } + + if(sub_id == HttpEnums::HEAD_REFERER) + { + if(referer) + global_field.set(strlen(referer), (const uint8_t*)referer); + } + + if(sub_id == HttpEnums::HEAD_SERVER) + { + if(server) + global_field.set(strlen(server), (const uint8_t*)server); + } + + if(sub_id == HttpEnums::HEAD_X_WORKING_WITH) + { + if(x_working_with) + global_field.set(strlen(x_working_with), (const uint8_t*)x_working_with); + } + + if(sub_id == HttpEnums::HEAD_VIA) + { + if(via) + global_field.set(strlen(via), (const uint8_t*)via); + } + + if(sub_id == HttpEnums::HEAD_CONTENT_TYPE) + { + if(content_type) + global_field.set(strlen(content_type), (const uint8_t*)content_type); + } + } + + if(id == HttpEnums::HTTP_BUFFER_URI) + { + if(sub_id == 0) + { + if(url) + global_field.set(strlen(url), (const uint8_t*)url); + } + } + + return global_field; +} + +class FakeHttpMsgHeader +{ +}; + +unsigned AppIdSession::flow_id = 0; +AppIdSession *fake_session = nullptr; +FakeHttpMsgHeader *fake_msg_header = nullptr; + +AppIdSession::AppIdSession(IpProtocol, const sfip_t*) : FlowData(flow_id, nullptr) +{ +} + +AppIdSession::~AppIdSession() +{ + if(!hsession) + return; + + if(hsession->content_type) free(hsession->content_type); + if(hsession->host) free(hsession->host); + if(hsession->referer) free(hsession->referer); + if(hsession->response_code) free(hsession->response_code); + if(hsession->server) free(hsession->server); + if(hsession->url) free(hsession->url); + if(hsession->useragent) free(hsession->useragent); + if(hsession->via) free(hsession->via); + if(hsession->x_working_with) free(hsession->x_working_with); + snort_free(hsession); +} + +int AppIdSession::processHTTPPacket(int) +{ + return 0; +} + +AppId AppIdSession::pick_service_app_id() +{ + return 0; +} + +AppId AppIdSession::pick_client_app_id() +{ + return 0; +} + +AppId AppIdSession::pick_payload_app_id() +{ + return 0; +} + +AppId AppIdSession::pick_misc_app_id() +{ + return 0; +} + +AppIdSession* AppIdApi::get_appid_data(Flow*) +{ + mock().actualCall("get_appid_data"); + return fake_session; +} + +const uint8_t* HttpEvent::get_content_type(unsigned &length) +{ + global_field.set(0, nullptr); + if(content_type) + global_field.set(strlen(content_type), (const uint8_t*)content_type); + length = global_field.length; + return global_field.start; +} + +const uint8_t* HttpEvent::get_host(unsigned &length) +{ + global_field.set(0, nullptr); + if(host) + global_field.set(strlen(host), (const uint8_t*)host); + length = global_field.length; + return global_field.start; +} + +const uint8_t* HttpEvent::get_referer(unsigned &length) +{ + global_field.set(0, nullptr); + if(referer) + global_field.set(strlen(referer), (const uint8_t*)referer); + length = global_field.length; + return global_field.start; +} + +int32_t HttpEvent::get_response_code() +{ + return response_code; +} + +const uint8_t* HttpEvent::get_server(unsigned &length) +{ + global_field.set(0, nullptr); + if(server) + global_field.set(strlen(server), (const uint8_t*)server); + length = global_field.length; + return global_field.start; +} + +const uint8_t* HttpEvent::get_uri(unsigned &length) +{ + global_field.set(0, nullptr); + if(url) + global_field.set(strlen(url), (const uint8_t*)url); + length = global_field.length; + return global_field.start; +} + +const uint8_t* HttpEvent::get_user_agent(unsigned &length) +{ + global_field.set(0, nullptr); + if(useragent) + global_field.set(strlen(useragent), (const uint8_t*)useragent); + length = global_field.length; + return global_field.start; +} + +const uint8_t* HttpEvent::get_via(unsigned &length) +{ + global_field.set(0, nullptr); + if(via) + global_field.set(strlen(via), (const uint8_t*)via); + length = global_field.length; + return global_field.start; +} + +const uint8_t* HttpEvent::get_x_working_with(unsigned &length) +{ + global_field.set(0, nullptr); + if(x_working_with) + global_field.set(strlen(x_working_with), (const uint8_t*)x_working_with); + length = global_field.length; + return global_field.start; +} + +Flow::Flow() {} +Flow::~Flow() {} + +class FakeFlow : public Flow +{ +}; + +#ifdef DEBUG_MSGS +void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif + +TEST_GROUP(appid_http_event) +{ + void setup() + { + appid_stats.http_flows = 0; + } + + void teardown() + { + fake_msg_header = nullptr; + fake_session = nullptr; + mock().clear(); + } +}; + +TEST(appid_http_event, handle_null_appid_data) +{ + FakeFlow flow; + HttpEvent event(nullptr); + HttpEventHandler event_handler(HttpEventHandler::REQUEST_EVENT); + mock().expectOneCall("get_appid_data"); + event_handler.handle(event, &flow); + mock().checkExpectations(); +} + +TEST(appid_http_event, handle_null_msg_header) +{ + FakeFlow flow; + HttpEvent event(nullptr); + AppIdSession session(IpProtocol::TCP, nullptr); + HttpEventHandler event_handler(HttpEventHandler::REQUEST_EVENT); + fake_session = &session; + + mock().strictOrder(); + mock().expectOneCall("get_appid_data"); + event_handler.handle(event, &flow); + mock().checkExpectations(); +} + +#define CONTENT_TYPE "html/text" +#define HOST "www.google.com" +#define URL "http://www.google.com/path/to/index.html" +#define USERAGENT "Mozilla/5.0 (Macintosh; Intel Mac OS X)" +#define REFERER "http://www.yahoo.com/search" +#define RESPONSE_CODE 301 +#define SERVER "Apache" +#define X_WORKING_WITH "working with string" +#define VIA "via string" + +struct TestData +{ + HttpEventHandler::HttpEventType type = HttpEventHandler::REQUEST_EVENT; + unsigned scan_flags = 0; + PegCount http_flows = 1; // Default to 1 since most tests have 1. + const char *content_type = nullptr; + const char *host = nullptr; + const char *referer = nullptr; + int32_t response_code = 0; + const char *server = nullptr; + const char *x_working_with = nullptr; + const char *url = nullptr; + const char *useragent = nullptr; + const char *via = nullptr; +}; + +void run_event_handler(TestData test_data, TestData *expect_data = nullptr) +{ + FakeFlow flow; + HttpEvent event(nullptr); + AppIdSession session(IpProtocol::TCP, nullptr); + FakeHttpMsgHeader http_msg_header; + HttpEventHandler event_handler(test_data.type); + fake_session = &session; + fake_msg_header = &http_msg_header; + + host = test_data.host; + referer = test_data.referer; + server = test_data.server; + x_working_with = test_data.x_working_with; + url = test_data.url; + useragent = test_data.useragent; + via = test_data.via; + content_type = test_data.content_type; + response_code = test_data.response_code; + + if(expect_data == nullptr) + expect_data = &test_data; + + mock().strictOrder(); + mock().expectOneCall("get_appid_data"); + event_handler.handle(event, &flow); + LONGS_EQUAL(expect_data->scan_flags, session.scan_flags); + LONGS_EQUAL(expect_data->http_flows, appid_stats.http_flows); + STRCMP_EQUAL(expect_data->host, session.hsession->host); + STRCMP_EQUAL(expect_data->url, session.hsession->url); + STRCMP_EQUAL(expect_data->content_type, session.hsession->content_type); + STRCMP_EQUAL(expect_data->referer, session.hsession->referer); + STRCMP_EQUAL(expect_data->server, session.hsession->server); + STRCMP_EQUAL(expect_data->x_working_with, session.hsession->x_working_with); + STRCMP_EQUAL(expect_data->useragent, session.hsession->useragent); + STRCMP_EQUAL(expect_data->via, session.hsession->via); + if(nullptr == session.hsession->response_code) + { + LONGS_EQUAL(0, expect_data->response_code); + } + else + { + LONGS_EQUAL(expect_data->response_code, strtol(session.hsession->response_code, nullptr, 10)); + } + mock().checkExpectations(); +} + +TEST(appid_http_event, handle_msg_header_no_headers_exist) +{ + TestData test_data; + test_data.scan_flags = 0; + + run_event_handler(test_data); +} + + +TEST(appid_http_event, handle_msg_header_only_host) +{ + TestData test_data; + test_data.scan_flags = SCAN_HTTP_HOST_URL_FLAG; + test_data.host = HOST; + + run_event_handler(test_data); +} + +TEST(appid_http_event, handle_msg_header_host_and_url) +{ + TestData test_data; + test_data.scan_flags = SCAN_HTTP_HOST_URL_FLAG; + test_data.host = HOST; + test_data.url = URL; + + run_event_handler(test_data); +} + +TEST(appid_http_event, handle_msg_header_user_agent) +{ + TestData test_data; + test_data.scan_flags = SCAN_HTTP_USER_AGENT_FLAG; + test_data.useragent = USERAGENT; + + run_event_handler(test_data); +} + +TEST(appid_http_event, handle_msg_header_x_working_with) +{ + TestData test_data; + test_data.scan_flags = 0; + test_data.x_working_with = X_WORKING_WITH; + + run_event_handler(test_data); +} + +TEST(appid_http_event, handle_msg_header_referer) +{ + TestData test_data; + test_data.scan_flags = 0; + test_data.referer = REFERER; + + run_event_handler(test_data); +} + +TEST(appid_http_event, handle_msg_header_via) +{ + TestData test_data; + test_data.scan_flags = SCAN_HTTP_VIA_FLAG; + test_data.via = VIA; + + run_event_handler(test_data); +} + +TEST(appid_http_event, handle_msg_header_content_type) +{ + TestData test_data; + test_data.type = HttpEventHandler::RESPONSE_EVENT; + test_data.scan_flags = 0; + test_data.http_flows = 0; // Flows are only counted on request header + test_data.content_type = CONTENT_TYPE; + + run_event_handler(test_data); +} + +TEST(appid_http_event, handle_msg_header_server) +{ + TestData test_data; + test_data.type = HttpEventHandler::RESPONSE_EVENT; + test_data.scan_flags = 0; + test_data.http_flows = 0; // Flows are only counted on request header + test_data.server = SERVER; + + run_event_handler(test_data); +} + +TEST(appid_http_event, handle_msg_header_response_code) +{ + TestData test_data; + test_data.type = HttpEventHandler::RESPONSE_EVENT; + test_data.scan_flags = 0; + test_data.http_flows = 0; // Flows are only counted on request header + test_data.response_code = RESPONSE_CODE; + + run_event_handler(test_data); +} + +TEST(appid_http_event, handle_msg_header_response_code_out_of_range) +{ + TestData test_data; + test_data.type = HttpEventHandler::RESPONSE_EVENT; + test_data.scan_flags = 0; + test_data.http_flows = 0; // Flows are only counted on request header + test_data.response_code = 1000; + + TestData expect_data = test_data; + expect_data.response_code = 0; + + run_event_handler(test_data, &expect_data); +} + +TEST(appid_http_event, handle_msg_header_all_response_headers) +{ + TestData test_data; + test_data.type = HttpEventHandler::RESPONSE_EVENT; + test_data.scan_flags = 1; + test_data.http_flows = 0; // Flows are only counted on request header + test_data.response_code = RESPONSE_CODE; + test_data.content_type = CONTENT_TYPE; + test_data.via = VIA; + + TestData expect_data = test_data; + + // Fill in the request data too to demonstrate that it is not copied + // during response header handling. + test_data.url = URL; + test_data.host = HOST; + test_data.referer = REFERER; + test_data.useragent = USERAGENT; + + run_event_handler(test_data, &expect_data); +} + +TEST(appid_http_event, handle_msg_header_all_request_headers) +{ + TestData test_data; + test_data.type = HttpEventHandler::REQUEST_EVENT; + test_data.scan_flags = SCAN_HTTP_VIA_FLAG | SCAN_HTTP_USER_AGENT_FLAG | + SCAN_HTTP_HOST_URL_FLAG; + test_data.url = URL; + test_data.host = HOST; + test_data.referer = REFERER; + test_data.useragent = USERAGENT; + test_data.via = VIA; + + TestData expect_data = test_data; + + // Fill in the request data too to demonstrate that it is not copied + // during response header handling. + test_data.response_code = RESPONSE_CODE; + test_data.content_type = CONTENT_TYPE; + + run_event_handler(test_data, &expect_data); +} + +int main(int argc, char** argv) +{ + int return_value = CommandLineTestRunner::RunAllTests(argc, argv); + return return_value; +} + diff --git a/src/pub_sub/CMakeLists.txt b/src/pub_sub/CMakeLists.txt new file mode 100644 index 000000000..68862c27f --- /dev/null +++ b/src/pub_sub/CMakeLists.txt @@ -0,0 +1,4 @@ +add_library( pub_sub STATIC + http_events.cc + http_events.h +) diff --git a/src/pub_sub/Makefile.am b/src/pub_sub/Makefile.am new file mode 100644 index 000000000..f52d3b369 --- /dev/null +++ b/src/pub_sub/Makefile.am @@ -0,0 +1,11 @@ + +noinst_LIBRARIES = libpub_sub.a + +libpub_sub_a_SOURCES = \ +http_events.cc \ +http_events.h + +#if ENABLE_UNIT_TESTS +#SUBDIRS = test +#endif + diff --git a/src/pub_sub/http_events.cc b/src/pub_sub/http_events.cc new file mode 100644 index 000000000..1b2b58284 --- /dev/null +++ b/src/pub_sub/http_events.cc @@ -0,0 +1,93 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2016-2016 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// http_events.cc author Steve Chew +// Inspection events published by the Http Inspector. Modules can subscribe +// to receive the events. + +#include "http_events.h" +#include "service_inspectors/http_inspect/http_msg_header.h" + +const uint8_t* HttpEvent::get_header(unsigned id, uint64_t sub_id, unsigned& length) +{ + Field field; + field = http_msg_header->get_classic_buffer(id, sub_id, 0); + if(field.length > 0) + { + length = field.length; + return field.start; + } + else + { + length = 0; + return nullptr; + } +} + +const uint8_t* HttpEvent::get_content_type(unsigned& length) +{ + return get_header(HttpEnums::HTTP_BUFFER_HEADER, + HttpEnums::HEAD_CONTENT_TYPE, length); +} + +const uint8_t* HttpEvent::get_host(unsigned& length) +{ + return get_header(HttpEnums::HTTP_BUFFER_HEADER, HttpEnums::HEAD_HOST, + length); +} + +const uint8_t* HttpEvent::get_referer(unsigned& length) +{ + return get_header(HttpEnums::HTTP_BUFFER_HEADER, HttpEnums::HEAD_REFERER, + length); +} + +int32_t HttpEvent::get_response_code() +{ + return http_msg_header->get_status_code(); +} + +const uint8_t* HttpEvent::get_server(unsigned& length) +{ + return get_header(HttpEnums::HTTP_BUFFER_HEADER, HttpEnums::HEAD_SERVER, + length); +} + +const uint8_t* HttpEvent::get_uri(unsigned& length) +{ + return get_header(HttpEnums::HTTP_BUFFER_URI, 0, length); +} + +const uint8_t* HttpEvent::get_user_agent(unsigned& length) +{ + return get_header(HttpEnums::HTTP_BUFFER_HEADER, HttpEnums::HEAD_USER_AGENT, + length); +} + +const uint8_t* HttpEvent::get_via(unsigned& length) +{ + return get_header(HttpEnums::HTTP_BUFFER_HEADER, HttpEnums::HEAD_VIA, + length); +} + +const uint8_t* HttpEvent::get_x_working_with(unsigned& length) +{ + return get_header(HttpEnums::HTTP_BUFFER_HEADER, + HttpEnums::HEAD_X_WORKING_WITH, length); +} + diff --git a/src/pub_sub/http_events.h b/src/pub_sub/http_events.h new file mode 100644 index 000000000..422308026 --- /dev/null +++ b/src/pub_sub/http_events.h @@ -0,0 +1,59 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2016-2016 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// http_events.h author Steve Chew +// Inspection events published by the Http Inspector. Modules can subscribe +// to receive the events. + +#include "framework/data_bus.h" + +// These are common values between the HTTP inspector and the subscribers. +#define HTTP_REQUEST_HEADER_EVENT_KEY "http_request_header_event" +#define HTTP_RESPONSE_HEADER_EVENT_KEY "http_response_header_event" + +class HttpMsgHeader; + +class SO_PUBLIC HttpEvent : public DataEvent +{ +public: + HttpEvent(HttpMsgHeader* http_msg_header_) : + http_msg_header(http_msg_header_) + { + } + + ~HttpEvent() + { + } + + const uint8_t* get_content_type(unsigned &length); + const uint8_t* get_host(unsigned &length); + const uint8_t* get_referer(unsigned &length); + const uint8_t* get_server(unsigned &length); + const uint8_t* get_uri(unsigned &length); + const uint8_t* get_user_agent(unsigned &length); + const uint8_t* get_via(unsigned &length); + const uint8_t* get_x_working_with(unsigned &length); + int32_t get_response_code(); + +private: + HttpMsgHeader* const http_msg_header; + + const uint8_t* get_header(unsigned, uint64_t, unsigned&); + +}; + diff --git a/src/service_inspectors/CMakeLists.txt b/src/service_inspectors/CMakeLists.txt index ee910b054..d4ca46fa5 100644 --- a/src/service_inspectors/CMakeLists.txt +++ b/src/service_inspectors/CMakeLists.txt @@ -17,7 +17,7 @@ add_subdirectory(ssl) add_subdirectory(wizard) if (STATIC_INSPECTORS) - set (STATIC_INSECTOR_LIBS + set (STATIC_INSPECTOR_LIBS back_orifice dce_rpc dnp3 @@ -26,7 +26,7 @@ if (STATIC_INSPECTORS) gtp_inspect imap modbus - http_inspect +# http_inspect pop rpc_decode sip @@ -43,5 +43,5 @@ add_library( service_inspectors STATIC ) target_link_libraries( service_inspectors - ${STATIC_INSECTOR_LIBS} + http_inspect ${STATIC_INSPECTOR_LIBS} ) diff --git a/src/service_inspectors/http_inspect/CMakeLists.txt b/src/service_inspectors/http_inspect/CMakeLists.txt index 778798672..92659dd98 100644 --- a/src/service_inspectors/http_inspect/CMakeLists.txt +++ b/src/service_inspectors/http_inspect/CMakeLists.txt @@ -62,13 +62,13 @@ set (FILE_LIST http_event_gen.h ) -if (STATIC_INSPECTORS) +#if (STATIC_INSPECTORS) add_library(http_inspect STATIC ${FILE_LIST}) -else(STATIC_INSPECTORS) - add_shared_library(http_inspect inspectors ${FILE_LIST}) +#else(STATIC_INSPECTORS) + #add_shared_library(http_inspect inspectors ${FILE_LIST}) -endif(STATIC_INSPECTORS) +#endif(STATIC_INSPECTORS) add_subdirectory ( test ) diff --git a/src/service_inspectors/http_inspect/Makefile.am b/src/service_inspectors/http_inspect/Makefile.am index fa621be2b..f6cc10d50 100644 --- a/src/service_inspectors/http_inspect/Makefile.am +++ b/src/service_inspectors/http_inspect/Makefile.am @@ -31,18 +31,18 @@ http_infractions.h \ http_event_gen.h \ ips_http.cc ips_http.h -if STATIC_INSPECTORS +#if STATIC_INSPECTORS noinst_LIBRARIES = libhttp_inspect.a libhttp_inspect_a_SOURCES = $(file_list) -else -shlibdir = $(pkglibdir)/inspectors -shlib_LTLIBRARIES = libhttp_inspect.la -libhttp_inspect_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO -libhttp_inspect_la_LDFLAGS = $(AM_LDFLAGS) -export-dynamic -shared -libhttp_inspect_la_SOURCES = $(file_list) - -endif +#else +#shlibdir = $(pkglibdir)/inspectors +#shlib_LTLIBRARIES = libhttp_inspect.la +#libhttp_inspect_la_CXXFLAGS = $(AM_CXXFLAGS) -DBUILDING_SO +#libhttp_inspect_la_LDFLAGS = $(AM_LDFLAGS) -export-dynamic -shared +#libhttp_inspect_la_SOURCES = $(file_list) +# +#endif if BUILD_CPPUTESTS SUBDIRS = test diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h index 20b4fc2f1..3d48efce5 100644 --- a/src/service_inspectors/http_inspect/http_enum.h +++ b/src/service_inspectors/http_inspect/http_enum.h @@ -114,6 +114,7 @@ enum HeaderId { HEAD__NOT_COMPUTE=-14, HEAD__PROBLEMATIC=-12, HEAD__NOT_PRESENT= HEAD_WWW_AUTHENTICATE, HEAD_ALLOW, HEAD_CONTENT_ENCODING, HEAD_CONTENT_LANGUAGE, HEAD_CONTENT_LENGTH, HEAD_CONTENT_LOCATION, HEAD_CONTENT_MD5, HEAD_CONTENT_RANGE, HEAD_CONTENT_TYPE, HEAD_EXPIRES, HEAD_LAST_MODIFIED, HEAD_X_FORWARDED_FOR, HEAD_TRUE_CLIENT_IP, + HEAD_X_WORKING_WITH, HEAD__MAX_VALUE }; // All the infractions we might find while parsing and analyzing a message diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc index 0112179f4..ebccff036 100644 --- a/src/service_inspectors/http_inspect/http_inspect.cc +++ b/src/service_inspectors/http_inspect/http_inspect.cc @@ -186,6 +186,7 @@ const Field& HttpInspect::process(const uint8_t* data, const uint16_t dsize, Flo } #endif + latest_section->publish(); return latest_section->get_detect_buf(); } diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc index abffe42b7..c1cd1cddd 100644 --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -31,6 +31,7 @@ #include "http_normalizers.h" #include "http_msg_request.h" #include "http_msg_header.h" +#include "pub_sub/http_events.h" using namespace HttpEnums; @@ -42,6 +43,19 @@ HttpMsgHeader::HttpMsgHeader(const uint8_t* buffer, const uint16_t buf_size, transaction->set_header(this, source_id); } +void HttpMsgHeader::publish() +{ + HttpEvent http_event(this); + if(source_id == SRC_CLIENT) + { + get_data_bus().publish(HTTP_REQUEST_HEADER_EVENT_KEY, http_event, flow); + } + else if(source_id == SRC_SERVER) + { + get_data_bus().publish(HTTP_RESPONSE_HEADER_EVENT_KEY, http_event, flow); + } +} + void HttpMsgHeader::update_flow() { session_data->section_type[source_id] = SEC__NOT_COMPUTE; diff --git a/src/service_inspectors/http_inspect/http_msg_header.h b/src/service_inspectors/http_inspect/http_msg_header.h index 98d62c28c..c0e993949 100644 --- a/src/service_inspectors/http_inspect/http_msg_header.h +++ b/src/service_inspectors/http_inspect/http_msg_header.h @@ -38,6 +38,14 @@ public: HttpEnums::InspectSection get_inspection_section() const override { return detection_section ? HttpEnums::IS_DETECTION : HttpEnums::IS_NONE; } void update_flow() override; + + void publish() override; + + int32_t get_status_code() + { + return status_code_num; + } + private: // Dummy configurations to support MIME processing MailLogConfig mime_conf; diff --git a/src/service_inspectors/http_inspect/http_msg_section.h b/src/service_inspectors/http_inspect/http_msg_section.h index 02c5e610e..383838a18 100644 --- a/src/service_inspectors/http_inspect/http_msg_section.h +++ b/src/service_inspectors/http_inspect/http_msg_section.h @@ -53,6 +53,9 @@ public: HttpEnums::MethodId get_method_id() const { return method_id; } + // Publish an inspection event for other modules to consume. + virtual void publish() { } + #ifdef REG_TEST // Test tool prints all derived message parts virtual void print_section(FILE* output) = 0; diff --git a/src/service_inspectors/http_inspect/http_tables.cc b/src/service_inspectors/http_inspect/http_tables.cc index 65657c88b..bd51f1ce2 100644 --- a/src/service_inspectors/http_inspect/http_tables.cc +++ b/src/service_inspectors/http_inspect/http_tables.cc @@ -147,6 +147,7 @@ const StrCode HttpMsgHeadShared::header_list[] = { HEAD_LAST_MODIFIED, "last-modified" }, { HEAD_X_FORWARDED_FOR, "x-forwarded-for" }, { HEAD_TRUE_CLIENT_IP, "true-client-ip" }, + { HEAD_X_WORKING_WITH, "x-working-with" }, { 0, nullptr } }; @@ -272,7 +273,8 @@ const HeaderNormalizer* const HttpMsgHeadShared::header_norms[HEAD__MAX_VALUE] = [HEAD_EXPIRES] = &NORMALIZER_BASIC, [HEAD_LAST_MODIFIED] = &NORMALIZER_BASIC, [HEAD_X_FORWARDED_FOR] = &NORMALIZER_CAT, - [HEAD_TRUE_CLIENT_IP] = &NORMALIZER_BASIC + [HEAD_TRUE_CLIENT_IP] = &NORMALIZER_BASIC, + [HEAD_X_WORKING_WITH] = &NORMALIZER_BASIC }; /* *INDENT-ON* */ diff --git a/src/side_channel/test/side_channel_module_test.cc b/src/side_channel/test/side_channel_module_test.cc index 3c0af5a44..600ebdb2d 100644 --- a/src/side_channel/test/side_channel_module_test.cc +++ b/src/side_channel/test/side_channel_module_test.cc @@ -56,7 +56,9 @@ void show_stats(PegCount*, const PegInfo*, IndexVec&, const char*, FILE*) { } void ParseWarning(WarningGroup, const char*, ...) { } +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif char* snort_strdup(const char* s) { return strdup(s); } diff --git a/src/side_channel/test/side_channel_test.cc b/src/side_channel/test/side_channel_test.cc index 1d75be8f5..b7302b9ff 100644 --- a/src/side_channel/test/side_channel_test.cc +++ b/src/side_channel/test/side_channel_test.cc @@ -137,7 +137,9 @@ Connector* ConnectorManager::get_connector(const std::string connector_name) void ParseWarning(WarningGroup, const char*, ...) { } +#ifdef DEBUG_MSGS void Debug::print(const char*, int, uint64_t, const char*, ...) { } +#endif TEST_GROUP(side_channel) {