From: Shravan Rangarajuvenkata (shrarang) Date: Tue, 5 May 2020 17:37:19 +0000 (+0000) Subject: Merge pull request #2174 in SNORT/snort3 from ~AGIURGIU/snort3:appid_http2_ms_detect... X-Git-Tag: 3.0.1-3~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=659b66a7ba84787c7255e62d6f2b281a91023a5c;p=thirdparty%2Fsnort3.git Merge pull request #2174 in SNORT/snort3 from ~AGIURGIU/snort3:appid_http2_ms_detect to master Squashed commit of the following: commit 3b0b9e7dbc443903f9f19def32648bc57bafb4da Author: Shravan Rangaraju Date: Thu Apr 30 01:23:44 2020 -0400 appid: support for multi-stream http2 session commit c55b5718ef6877bbf16f4d1a3fde16f16117a2d3 Author: Shravan Rangaraju Date: Thu Apr 23 02:27:23 2020 -0400 appid: store appids for http traffic in http session commit 8a78d00add4080c77538196d0ae98ffa74db1dfa Author: Kanimozhi Murthi Date: Tue Apr 7 23:43:51 2020 -0400 appid: remove old http2 support --- diff --git a/src/network_inspectors/appid/app_info_table.cc b/src/network_inspectors/appid/app_info_table.cc index d07c95d17..0ed788361 100644 --- a/src/network_inspectors/appid/app_info_table.cc +++ b/src/network_inspectors/appid/app_info_table.cc @@ -612,28 +612,6 @@ void AppInfoManager::load_odp_config(OdpContext& odp_ctxt, const char* path) { set_app_info_flags(atoi(conf_val), APPINFO_FLAG_IGNORE); } - else if (!(strcasecmp(conf_key, "http2_detection"))) - { - // This option will control our own HTTP/2 detection. We can - // still be told externally, though, that it's HTTP/2 (either - // from HTTP Inspect or 3rd Party). This is intended to be - // used to ask AppID to detect unencrypted HTTP/2 on non-std - // ports. - if (!(strcasecmp(conf_val, "disabled"))) - { - odp_ctxt.http2_detection_enabled = false; - } - else if (!(strcasecmp(conf_val, "enabled"))) - { - odp_ctxt.http2_detection_enabled = true; - } - else - { - ParseWarning(WARN_CONF, - "AppId: ignoring invalid option for http2_detection: %s\n", - conf_val); - } - } else ParseWarning(WARN_CONF, "AppId: unsupported configuration: %s\n", conf_key); } diff --git a/src/network_inspectors/appid/appid_api.cc b/src/network_inspectors/appid/appid_api.cc index 64e539195..aada456bc 100644 --- a/src/network_inspectors/appid/appid_api.cc +++ b/src/network_inspectors/appid/appid_api.cc @@ -61,12 +61,12 @@ const char* AppIdApi::get_application_name(const Flow& flow, bool from_client) AppIdSession* asd = get_appid_session(flow); if (asd) { - AppId appid = asd->pick_payload_app_id(); + AppId appid = asd->pick_ss_payload_app_id(); if (appid <= APP_ID_NONE) - appid = asd->pick_misc_app_id(); + appid = asd->pick_ss_misc_app_id(); if (!appid and from_client) { - appid = asd->pick_client_app_id(); + appid = asd->pick_ss_client_app_id(); if (!appid) appid = asd->pick_service_app_id(); } @@ -74,7 +74,7 @@ const char* AppIdApi::get_application_name(const Flow& flow, bool from_client) { appid = asd->pick_service_app_id(); if (!appid) - appid = asd->pick_client_app_id(); + appid = asd->pick_ss_client_app_id(); } if (appid > APP_ID_NONE && appid < SF_APPID_MAX) app_name = asd->ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(appid); @@ -112,9 +112,16 @@ uint32_t AppIdApi::produce_ha_state(const Flow& flow, uint8_t* buf) appHA->appId[1] = asd->service.get_id(); appHA->appId[2] = asd->client_inferred_service_id; appHA->appId[3] = asd->service.get_port_service_id(); - appHA->appId[4] = asd->payload.get_id(); + AppIdHttpSession* hsession = asd->get_http_session(); + if (hsession) + appHA->appId[4] = hsession->payload.get_id(); + else + appHA->appId[4] = asd->payload.get_id(); appHA->appId[5] = asd->get_tp_payload_app_id(); - appHA->appId[6] = asd->client.get_id(); + if (hsession) + appHA->appId[6] = hsession->client.get_id(); + else + appHA->appId[6] = asd->client.get_id(); appHA->appId[7] = asd->misc_app_id; } else @@ -180,9 +187,18 @@ uint32_t AppIdApi::consume_ha_state(Flow& flow, const uint8_t* buf, uint8_t, IpP asd->service.set_id(appHA->appId[1], asd->ctxt.get_odp_ctxt()); asd->client_inferred_service_id = appHA->appId[2]; asd->service.set_port_service_id(appHA->appId[3]); - asd->payload.set_id(appHA->appId[4]); + AppIdHttpSession* hsession = nullptr; + if (appHA->appId[1] == APP_ID_HTTP or appHA->appId[1] == APP_ID_RTMP) + hsession = asd->create_http_session(); + if (hsession) + hsession->payload.set_id(appHA->appId[4]); + else + asd->payload.set_id(appHA->appId[4]); asd->set_tp_payload_app_id(appHA->appId[5]); - asd->client.set_id(appHA->appId[6]); + if (hsession) + hsession->client.set_id(appHA->appId[6]); + else + asd->client.set_id(appHA->appId[6]); asd->misc_app_id = appHA->appId[7]; } return sizeof(*appHA); @@ -245,7 +261,7 @@ bool AppIdApi::ssl_app_group_id_lookup(Flow* flow, const char* server_name, cons else asd->payload.set_id(payload_id); - asd->set_application_ids(service_id, client_id, payload_id, misc_id, change_bits); + asd->set_ss_application_ids(service_id, client_id, payload_id, misc_id, change_bits); asd->publish_appid_event(change_bits, flow); } diff --git a/src/network_inspectors/appid/appid_app_descriptor.cc b/src/network_inspectors/appid/appid_app_descriptor.cc index 17fb65159..34553769a 100644 --- a/src/network_inspectors/appid/appid_app_descriptor.cc +++ b/src/network_inspectors/appid/appid_app_descriptor.cc @@ -66,7 +66,8 @@ void ServiceAppDescriptor::set_id(AppId app_id, OdpContext& odp_ctxt) void ClientAppDescriptor::update_user(AppId app_id, const char* username) { - my_username = username; + if ( my_username != username ) + my_username = username; if ( my_user_id != app_id ) { diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h index 7f39f9a4f..c6a1e4354 100644 --- a/src/network_inspectors/appid/appid_config.h +++ b/src/network_inspectors/appid/appid_config.h @@ -88,7 +88,6 @@ public: bool referred_appId_disabled = false; bool mdns_user_reporting = true; bool chp_userid_disabled = false; - bool http2_detection_enabled = false; bool is_host_port_app_cache_runtime = false; bool check_host_port_app_cache = false; bool check_host_cache_unknown_ssl = false; @@ -100,7 +99,6 @@ public: bool tp_allow_probes = false; uint32_t host_port_app_cache_lookup_interval = 10; uint32_t host_port_app_cache_lookup_range = 100000; - bool http_response_version_enabled = false; bool allow_port_wildcard_host_cache = false; bool recheck_for_portservice_appid = false; uint64_t max_bytes_before_service_fail = MIN_MAX_BYTES_BEFORE_SERVICE_FAIL; diff --git a/src/network_inspectors/appid/appid_detector.cc b/src/network_inspectors/appid/appid_detector.cc index 465a0a4b7..5a815bc11 100644 --- a/src/network_inspectors/appid/appid_detector.cc +++ b/src/network_inspectors/appid/appid_detector.cc @@ -73,14 +73,6 @@ int AppIdDetector::data_add(AppIdSession& asd, void* data, AppIdFreeFCN fcn) return asd.add_flow_data(data, flow_data_index, fcn); } -void AppIdDetector::add_info(AppIdSession& asd, const char* info, AppidChangeBits& change_bits) -{ - AppIdHttpSession* hsession = asd.get_http_session(); - - if ( !hsession->get_field(MISC_URL_FID) ) - hsession->set_field(MISC_URL_FID, new std::string(info), change_bits); -} - void AppIdDetector::add_user(AppIdSession& asd, const char* username, AppId appId, bool success) { asd.client.update_user(appId, username); diff --git a/src/network_inspectors/appid/appid_detector.h b/src/network_inspectors/appid/appid_detector.h index 0578fe3e9..3846e3198 100644 --- a/src/network_inspectors/appid/appid_detector.h +++ b/src/network_inspectors/appid/appid_detector.h @@ -118,7 +118,6 @@ public: virtual void* data_get(AppIdSession&); virtual int data_add(AppIdSession&, void*, AppIdFreeFCN); - virtual void add_info(AppIdSession&, const char*, AppidChangeBits&); virtual void add_user(AppIdSession&, const char*, AppId, bool); virtual void add_payload(AppIdSession&, AppId); virtual void add_app(AppIdSession& asd, AppId service_id, AppId client_id, const char* version, AppidChangeBits& change_bits) diff --git a/src/network_inspectors/appid/appid_discovery.cc b/src/network_inspectors/appid/appid_discovery.cc index db3053bbe..0b85c22ef 100644 --- a/src/network_inspectors/appid/appid_discovery.cc +++ b/src/network_inspectors/appid/appid_discovery.cc @@ -501,26 +501,25 @@ bool AppIdDiscovery::do_pre_discovery(Packet* p, AppIdSession** p_asd, AppIdInsp if (!asd->get_session_flags(APPID_SESSION_PAYLOAD_SEEN) and p->dsize) asd->set_session_flags(APPID_SESSION_PAYLOAD_SEEN); - if (asd->get_session_flags(APPID_SESSION_IGNORE_FLOW)) + if (asd->get_session_flags(APPID_SESSION_FUTURE_FLOW)) { - if (!asd->get_session_flags(APPID_SESSION_IGNORE_FLOW_IDED)) + if (!asd->get_session_flags(APPID_SESSION_FUTURE_FLOW_IDED)) { AppidChangeBits change_bits; - asd->set_application_ids(asd->pick_service_app_id(), asd->pick_client_app_id(), - asd->pick_payload_app_id(), asd->pick_misc_app_id(), change_bits); + + asd->set_ss_application_ids(asd->pick_service_app_id(), asd->pick_ss_client_app_id(), + asd->pick_ss_payload_app_id(), asd->pick_ss_misc_app_id(), change_bits); asd->publish_appid_event(change_bits, p->flow); - asd->set_session_flags(APPID_SESSION_IGNORE_FLOW_IDED); - } - if (appidDebug->is_active() && - !asd->get_session_flags(APPID_SESSION_IGNORE_FLOW_LOGGED)) - { - asd->set_session_flags(APPID_SESSION_IGNORE_FLOW_LOGGED); + asd->set_session_flags(APPID_SESSION_FUTURE_FLOW_IDED); - const char *app_name = - asd->ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(asd->service.get_id()); - LogMessage("AppIdDbg %s Ignoring connection with service %s (%d)\n", - appidDebug->get_debug_session(), app_name ? app_name : "unknown", - asd->service.get_id()); + if (appidDebug->is_active()) + { + const char *app_name = + asd->ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(asd->service.get_id()); + LogMessage("AppIdDbg %s Ignoring connection with service %s (%d)\n", + appidDebug->get_debug_session(), app_name ? app_name : "unknown", + asd->service.get_id()); + } } return false; @@ -804,13 +803,14 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto else { service_id = asd.pick_service_app_id(); - misc_id = asd.pick_misc_app_id(); + misc_id = asd.pick_ss_misc_app_id(); } return true; } // Third party detection - if (tp_appid_ctxt && !asd.is_http2) + // Skip third-party detection for http2 + if (tp_appid_ctxt and ((service_id = asd.pick_service_app_id()) != APP_ID_HTTP2)) { // Skip third-party inspection for sessions using old config if (asd.tpsession and &(asd.tpsession->get_ctxt()) != tp_appid_ctxt) @@ -885,33 +885,34 @@ bool AppIdDiscovery::do_discovery(Packet* p, AppIdSession& asd, IpProtocol proto } } - payload_id = asd.pick_payload_app_id(); - client_id = asd.pick_client_app_id(); - misc_id = asd.pick_misc_app_id(); + payload_id = asd.pick_ss_payload_app_id(); + client_id = asd.pick_ss_client_app_id(); + misc_id = asd.pick_ss_misc_app_id(); + + AppIdHttpSession* hsession = asd.get_http_session(); + bool is_http_tunnel = false; - bool is_http_tunnel = ((asd.payload.get_id() == APP_ID_HTTP_TUNNEL) || - (asd.payload.get_id() == APP_ID_HTTP_SSL_TUNNEL)) ? true:false; + if (hsession) + is_http_tunnel = ((hsession->payload.get_id() == APP_ID_HTTP_TUNNEL) || + (hsession->payload.get_id() == APP_ID_HTTP_SSL_TUNNEL)) ? true : false; if (is_check_host_cache_valid(asd, service_id, client_id, payload_id, misc_id) or (is_http_tunnel)) { - if (is_http_tunnel) + if (is_http_tunnel and (asd.scan_flags & SCAN_HTTP_URI_FLAG)) { - AppIdHttpSession* hsession = asd.get_http_session(); - if (hsession and (asd.scan_flags & SCAN_HTTP_URI_FLAG)) - { - if (hsession->get_tun_dest()) - hsession->free_tun_dest(); - hsession->set_tun_dest(); - asd.scan_flags &= ~SCAN_HTTP_URI_FLAG; - } + if (hsession->get_tun_dest()) + hsession->free_tun_dest(); + hsession->set_tun_dest(); + asd.scan_flags &= ~SCAN_HTTP_URI_FLAG; } + if (do_host_port_based_discovery(p, asd, protocol, direction)) { asd.service.set_port_service_id(APP_ID_NONE); service_id = asd.pick_service_app_id(); - client_id = asd.pick_client_app_id(); - payload_id = asd.pick_payload_app_id(); + client_id = asd.pick_ss_client_app_id(); + payload_id = asd.pick_ss_payload_app_id(); } } @@ -949,7 +950,7 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd, { asd.past_forecast = check_session_for_AF_forecast(asd, p, direction, service_id); if (asd.past_forecast != APP_ID_UNKNOWN) - payload_id = asd.pick_payload_app_id(); + payload_id = asd.pick_ss_payload_app_id(); } } @@ -966,6 +967,6 @@ void AppIdDiscovery::do_post_discovery(Packet* p, AppIdSession& asd, } } - asd.set_application_ids(service_id, client_id, payload_id, misc_id, change_bits); + asd.set_ss_application_ids(service_id, client_id, payload_id, misc_id, change_bits); asd.publish_appid_event(change_bits, p->flow); } diff --git a/src/network_inspectors/appid/appid_http_event_handler.cc b/src/network_inspectors/appid/appid_http_event_handler.cc index 75fc50f21..0ac8b3de4 100644 --- a/src/network_inspectors/appid/appid_http_event_handler.cc +++ b/src/network_inspectors/appid/appid_http_event_handler.cc @@ -55,15 +55,30 @@ void HttpEventHandler::handle(DataEvent& event, Flow* flow) return; if (appidDebug->is_active()) - LogMessage("AppIdDbg %s Processing HTTP metadata from HTTP Inspector\n", - appidDebug->get_debug_session()); + LogMessage("AppIdDbg %s Processing HTTP metadata from HTTP Inspector for stream %u\n", + appidDebug->get_debug_session(), http_event->get_http2_stream_id()); + asd->set_session_flags(APPID_SESSION_HTTP_SESSION); direction = event_type == REQUEST_EVENT ? APP_ID_FROM_INITIATOR : APP_ID_FROM_RESPONDER; - AppIdHttpSession* hsession = asd->get_http_session(0); + AppIdHttpSession* hsession; + if (http_event->get_is_http2()) + { + if (asd->get_prev_http2_raw_packet() != asd->session_packet_count) + { + asd->delete_all_http_sessions(); + asd->set_prev_http2_raw_packet(asd->session_packet_count); + } + + hsession = asd->create_http_session(http_event->get_http2_stream_id()); + } + else + { + hsession = asd->get_http_session(0); - if (!hsession) - hsession = asd->create_http_session(); + if (!hsession) + hsession = asd->create_http_session(); + } if (direction == APP_ID_FROM_INITIATOR) { @@ -101,11 +116,19 @@ void HttpEventHandler::handle(DataEvent& event, Flow* flow) else // Response headers. { header_start = http_event->get_content_type(header_length); - hsession->set_field(RSP_CONTENT_TYPE_FID, header_start, header_length, change_bits); + if (header_length > 0) + { + hsession->set_field(RSP_CONTENT_TYPE_FID, header_start, header_length, change_bits); + asd->scan_flags |= SCAN_HTTP_CONTENT_TYPE_FLAG; + } header_start = http_event->get_location(header_length); hsession->set_field(RSP_LOCATION_FID, header_start, header_length, change_bits); header_start = http_event->get_server(header_length); - hsession->set_field(MISC_SERVER_FID, header_start, header_length, change_bits); + if (header_length > 0) + { + hsession->set_field(MISC_SERVER_FID, header_start, header_length, change_bits); + asd->scan_flags |= SCAN_HTTP_VENDOR_FLAG; + } int32_t responseCodeNum = http_event->get_response_code(); if (responseCodeNum > 0 && responseCodeNum < 700) @@ -141,16 +164,18 @@ void HttpEventHandler::handle(DataEvent& event, Flow* flow) hsession->process_http_packet(direction, change_bits, asd->ctxt.get_odp_ctxt().get_http_matchers()); - if (asd->service.get_id() == APP_ID_HTTP) + if (http_event->get_is_http2()) { - if (http_event->get_is_http2() && http_event->get_http2_stream_id() != 0) - { - asd->service.set_id(APP_ID_HTTP2, asd->ctxt.get_odp_ctxt()); - } - asd->set_application_ids(asd->pick_service_app_id(), asd->pick_client_app_id(), - asd->pick_payload_app_id(), asd->pick_misc_app_id(), change_bits); + asd->service.set_id(APP_ID_HTTP2, asd->ctxt.get_odp_ctxt()); } - asd->publish_appid_event(change_bits, flow); + if (asd->service.get_id() != APP_ID_HTTP2) + asd->set_ss_application_ids(asd->pick_service_app_id(), asd->pick_ss_client_app_id(), + asd->pick_ss_payload_app_id(), asd->pick_ss_misc_app_id(), change_bits); + else + asd->set_application_ids_service(APP_ID_HTTP2, change_bits); + + asd->publish_appid_event(change_bits, flow, http_event->get_is_http2(), + asd->get_hsessions_size() - 1); } diff --git a/src/network_inspectors/appid/appid_http_session.cc b/src/network_inspectors/appid/appid_http_session.cc index 293a964e3..92967a065 100644 --- a/src/network_inspectors/appid/appid_http_session.cc +++ b/src/network_inspectors/appid/appid_http_session.cc @@ -37,8 +37,8 @@ using namespace snort; -AppIdHttpSession::AppIdHttpSession(AppIdSession& asd) - : asd(asd) +AppIdHttpSession::AppIdHttpSession(AppIdSession& asd, uint32_t http2_stream_id) + : asd(asd), http2_stream_id(http2_stream_id) { for ( int i = 0; i < NUM_HTTP_FIELDS; i++) { @@ -360,10 +360,10 @@ void AppIdHttpSession::process_chp_buffers(AppidChangeBits& change_bits, HttpPat asd.set_service_appid_data(chp_final, change_bits, version); if (app_type_flags & APP_TYPE_CLIENT) - asd.set_client_appid_data(chp_final, change_bits, version); + set_client(chp_final, change_bits, "CHP", version); if ( app_type_flags & APP_TYPE_PAYLOAD ) - asd.set_payload_appid_data(chp_final, change_bits, version); + set_payload(chp_final, change_bits, "CHP", version); if ( version ) { @@ -374,9 +374,9 @@ void AppIdHttpSession::process_chp_buffers(AppidChangeBits& change_bits, HttpPat if ( user ) { if (app_type_flags & APP_TYPE_SERVICE) - asd.client.update_user(chp_final, user); + client.update_user(chp_final, user); else - asd.client.update_user(asd.service.get_id(), user); + client.update_user(asd.service.get_id(), user); user = nullptr; asd.set_session_flags(APPID_SESSION_LOGIN_SUCCEEDED); } @@ -398,14 +398,61 @@ void AppIdHttpSession::process_chp_buffers(AppidChangeBits& change_bits, HttpPat } } +void AppIdHttpSession::set_client(AppId app_id, AppidChangeBits& change_bits, const char* type, + const char* version) +{ + if (app_id <= APP_ID_NONE or (app_id == client.get_id())) + return; + + client.set_id(app_id); + change_bits.set(APPID_CLIENT_BIT); + client.set_version(version, change_bits); + + if (appidDebug->is_active()) + { + const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(app_id); + LogMessage("AppIdDbg %s %s is client %s (%d)\n", appidDebug->get_debug_session(), + type, app_name ? app_name : "unknown", app_id); + } +} + +void AppIdHttpSession::set_payload(AppId app_id, AppidChangeBits& change_bits, const char* type, + const char* version) +{ + if (app_id <= APP_ID_NONE or (app_id == payload.get_id())) + return; + + payload.set_id(app_id); + change_bits.set(APPID_PAYLOAD_BIT); + payload.set_version(version, change_bits); + + if (appidDebug->is_active()) + { + const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(app_id); + LogMessage("AppIdDbg %s %s is payload %s (%d)\n", appidDebug->get_debug_session(), + type, app_name ? app_name : "unknown", app_id); + } +} + +void AppIdHttpSession::set_referred_payload(AppId app_id, AppidChangeBits& change_bits) +{ + if (app_id <= APP_ID_NONE or (app_id == referred_payload_app_id)) + return; + + referred_payload_app_id = app_id; + change_bits.set(APPID_REFERRED_BIT); + + if (appidDebug->is_active()) + { + const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(app_id); + LogMessage("AppIdDbg %s URL is referred %s (%d)\n", appidDebug->get_debug_session(), + app_name ? app_name : "unknown", app_id); + } +} + int AppIdHttpSession::process_http_packet(AppidSessionDirection direction, AppidChangeBits& change_bits, HttpPatternMatchers& http_matchers) { - AppId service_id = APP_ID_NONE; - AppId client_id = APP_ID_NONE; - AppId payload_id = APP_ID_NONE; - bool have_tp = asd.tpsession; - const std::string* useragent = meta_data[REQ_AGENT_FID]; const std::string* host = meta_data[REQ_HOST_FID]; const std::string* referer = meta_data[REQ_REFERER_FID]; @@ -413,7 +460,6 @@ int AppIdHttpSession::process_http_packet(AppidSessionDirection direction, // For fragmented HTTP headers, do not process if none of the fields are set. // These fields will get set when the HTTP header is reassembled. - if ( !useragent && !host && !referer && !uri ) { if (!skip_simple_detect) @@ -455,267 +501,207 @@ int AppIdHttpSession::process_http_packet(AppidSessionDirection direction, if (asd.service.get_id() == APP_ID_NONE) { asd.service.set_id(APP_ID_HTTP, asd.ctxt.get_odp_ctxt()); - asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED | APPID_SESSION_HTTP_SESSION); + asd.set_session_flags(APPID_SESSION_SERVICE_DETECTED); asd.service_disco_state = APPID_DISCO_STATE_FINISHED; } if (!chp_finished || chp_hold_flow) process_chp_buffers(change_bits, http_matchers); - if (!skip_simple_detect) // true if processCHP found match + if (skip_simple_detect) // true if process_chp_buffers() found match + return 0; + + if (!asd.get_session_flags(APPID_SESSION_APP_REINSPECT)) { - if (!asd.get_session_flags(APPID_SESSION_APP_REINSPECT)) + // Scan Server Header for Vendor & Version + const std::string* server = meta_data[MISC_SERVER_FID]; + if ( (asd.scan_flags & SCAN_HTTP_VENDOR_FLAG) and server) { - // Scan Server Header for Vendor & Version - // FIXIT-M: Should we be checking the scan_flags even when - // tp_appid_module is off? - const std::string* server = meta_data[MISC_SERVER_FID]; - if ( (have_tp && (asd.scan_flags & SCAN_HTTP_VENDOR_FLAG) && server) - || (!have_tp && server) ) - { - if ( asd.service.get_id() == APP_ID_NONE || asd.service.get_id() == APP_ID_HTTP ) - { - //AppIdServiceSubtype* local_subtype = nullptr; - char* vendorVersion = nullptr; - char* vendor = nullptr; - - http_matchers.get_server_vendor_version(server->c_str(), server->size(), - &vendorVersion, &vendor, &asd.subtype); - if (vendor || vendorVersion) - { - asd.service.set_vendor(vendor); - asd.service.set_version(vendorVersion, change_bits); - asd.scan_flags &= ~SCAN_HTTP_VENDOR_FLAG; - - snort_free(vendor); - snort_free(vendorVersion); - } -#if 0 - if (local_subtype) // FIXIT-W always false - { - AppIdServiceSubtype** tmpSubtype; - - for (tmpSubtype = &asd.subtype; *tmpSubtype; tmpSubtype = - &(*tmpSubtype)->next) - ; - - *tmpSubtype = local_subtype; - } -#endif - } - } - - if (is_webdav) + if ( asd.service.get_id() == APP_ID_NONE || asd.service.get_id() == APP_ID_HTTP ) { - if (appidDebug->is_active() and asd.payload.get_id() != APP_ID_WEBDAV) - LogMessage("AppIdDbg %s Data is webdav\n", appidDebug->get_debug_session()); - asd.set_payload_appid_data(APP_ID_WEBDAV, change_bits); - } - - // Scan User-Agent for Browser types or Skype - if ( (asd.scan_flags & SCAN_HTTP_USER_AGENT_FLAG) - && asd.client.get_id() <= APP_ID_NONE && useragent ) - { - char* version = nullptr; + char* vendorVersion = nullptr; + char* vendor = nullptr; + AppIdServiceSubtype* subtype = nullptr; - http_matchers.identify_user_agent(useragent->c_str(), useragent->size(), - service_id, client_id, &version); - if (appidDebug->is_active()) + http_matchers.get_server_vendor_version(server->c_str(), server->size(), + &vendorVersion, &vendor, &subtype); + if (vendor || vendorVersion) { - if (service_id > APP_ID_NONE and service_id != APP_ID_HTTP and - asd.service.get_id() != service_id) - { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id); - LogMessage("AppIdDbg %s User Agent is service %s (%d)\n", - appidDebug->get_debug_session(), app_name ? app_name : "unknown", service_id); - } - if (client_id > APP_ID_NONE and client_id != APP_ID_HTTP and - asd.client.get_id() != client_id) - { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(client_id); - LogMessage("AppIdDbg %s User Agent is client %s (%d)\n", - appidDebug->get_debug_session(), app_name ? app_name : "unknown", client_id); - } + asd.service.set_vendor(vendor); + asd.service.set_version(vendorVersion, change_bits); + asd.scan_flags &= ~SCAN_HTTP_VENDOR_FLAG; + + snort_free(vendor); + snort_free(vendorVersion); } - asd.set_service_appid_data(service_id, change_bits); - asd.set_client_appid_data(client_id, change_bits, version); - asd.scan_flags &= ~SCAN_HTTP_USER_AGENT_FLAG; - snort_free(version); - } - /* Scan Via Header for squid */ - const std::string* via = meta_data[MISC_VIA_FID]; - if ( !asd.is_payload_appid_set() && (asd.scan_flags & SCAN_HTTP_VIA_FLAG) && via ) - { - payload_id = http_matchers.get_appid_by_pattern(via->c_str(), via->size(), - nullptr); - if (appidDebug->is_active() && payload_id > APP_ID_NONE && - asd.payload.get_id() != payload_id) + if (subtype) { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(payload_id); - LogMessage("AppIdDbg %s VIA is payload %s (%d)\n", appidDebug->get_debug_session(), - app_name ? app_name : "unknown", - payload_id); + AppIdServiceSubtype** tmp_subtype; + + for (tmp_subtype = &asd.subtype; *tmp_subtype; tmp_subtype = &(*tmp_subtype)->next) + ; + + *tmp_subtype = subtype; } - asd.set_payload_appid_data(payload_id, change_bits); - asd.scan_flags &= ~SCAN_HTTP_VIA_FLAG; } } - /* Scan X-Working-With HTTP header */ - // FIXIT-M: Should we be checking the scan_flags even when - // tp_appid_module is off? - const std::string* x_working_with = meta_data[MISC_XWW_FID]; - if ( (have_tp && (asd.scan_flags & SCAN_HTTP_XWORKINGWITH_FLAG) && - x_working_with) || (!have_tp && x_working_with)) + if (is_webdav) + set_payload(APP_ID_WEBDAV, change_bits, "webdav"); + + // Scan User-Agent for Browser types or Skype + if ( (asd.scan_flags & SCAN_HTTP_USER_AGENT_FLAG) + and client.get_id() <= APP_ID_NONE and useragent ) { - AppId appId; char* version = nullptr; + AppId service_id = APP_ID_NONE; + AppId client_id = APP_ID_NONE; - appId = http_matchers.scan_header_x_working_with(x_working_with->c_str(), - x_working_with->size(), &version); - if ( appId ) + http_matchers.identify_user_agent(useragent->c_str(), useragent->size(), + service_id, client_id, &version); + if (appidDebug->is_active()) { - if (direction == APP_ID_FROM_INITIATOR) + if (service_id > APP_ID_NONE and service_id != APP_ID_HTTP and + asd.service.get_id() != service_id) { - if (appidDebug->is_active() && client_id > APP_ID_NONE && client_id != - APP_ID_HTTP && asd.client.get_id() != client_id) - { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(appId); - LogMessage("AppIdDbg %s X is client %s (%d)\n", appidDebug->get_debug_session(), - app_name ? app_name : "unknown", appId); - } - asd.set_client_appid_data(appId, change_bits, version); + const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id); + LogMessage("AppIdDbg %s User Agent is service %s (%d)\n", + appidDebug->get_debug_session(), app_name ? app_name : "unknown", service_id); } - else - { - if (appidDebug->is_active() && service_id > APP_ID_NONE && service_id != - APP_ID_HTTP && asd.service.get_id() != service_id) - { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(appId); - LogMessage("AppIdDbg %s X service %s (%d)\n", appidDebug->get_debug_session(), - app_name ? app_name : "unknown", appId); - } - asd.set_service_appid_data(appId, change_bits, version); - } - asd.scan_flags &= ~SCAN_HTTP_XWORKINGWITH_FLAG; } + asd.set_service_appid_data(service_id, change_bits); + if (client_id != APP_ID_HTTP) + set_client(client_id, change_bits, "User Agent", version); + asd.scan_flags &= ~SCAN_HTTP_USER_AGENT_FLAG; snort_free(version); } - // Scan Content-Type Header for multimedia types and scan contents - // FIXIT-M: Should we be checking the scan_flags even when - // tp_appid_module is off? - const std::string* content_type = meta_data[RSP_CONTENT_TYPE_FID]; - if ( (have_tp && (asd.scan_flags & SCAN_HTTP_CONTENT_TYPE_FLAG) - && content_type && !asd.is_payload_appid_set()) - || (!have_tp && !asd.is_payload_appid_set() && content_type) ) + /* Scan Via Header for squid */ + const std::string* via = meta_data[MISC_VIA_FID]; + if ( !asd.get_tp_payload_app_id() and !payload.get_id() and (asd.scan_flags & SCAN_HTTP_VIA_FLAG) and via ) { - payload_id = http_matchers.get_appid_by_content_type(content_type->c_str(), - content_type->size()); - if (appidDebug->is_active() && payload_id > APP_ID_NONE - && asd.payload.get_id() != payload_id) - { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(payload_id); - LogMessage("AppIdDbg %s Content-Type is payload %s (%d)\n", - appidDebug->get_debug_session(), - app_name ? app_name : "unknown", - payload_id); - } - asd.set_payload_appid_data(payload_id, change_bits); - asd.scan_flags &= ~SCAN_HTTP_CONTENT_TYPE_FLAG; + AppId payload_id = http_matchers.get_appid_by_pattern(via->c_str(), via->size(), + nullptr); + set_payload(payload_id, change_bits, "VIA"); + asd.scan_flags &= ~SCAN_HTTP_VIA_FLAG; } + } + + /* Scan X-Working-With HTTP header */ + const std::string* x_working_with = meta_data[MISC_XWW_FID]; + if ( (asd.scan_flags & SCAN_HTTP_XWORKINGWITH_FLAG) and x_working_with) + { + char* version = nullptr; - if (asd.scan_flags & SCAN_HTTP_HOST_URL_FLAG) + AppId app_id = http_matchers.scan_header_x_working_with(x_working_with->c_str(), + x_working_with->size(), &version); + + if (direction == APP_ID_FROM_INITIATOR) + set_client(app_id, change_bits, "X-working-with", version); + else { - AppId referredPayloadAppId = 0; - char* version = nullptr; - char* my_host = host ? snort_strdup(host->c_str()) : nullptr; - const char* refStr = referer ? referer->c_str() : nullptr; - const std::string* url = meta_data[MISC_URL_FID]; - const char* urlStr = url ? url->c_str() : nullptr; - if ( http_matchers.get_appid_from_url(my_host, urlStr, &version, - refStr, &client_id, &service_id, &payload_id, - &referredPayloadAppId, false, asd.ctxt.get_odp_ctxt()) ) + if (app_id and asd.service.get_id() != app_id) { - // do not overwrite a previously-set client or service - if (asd.client.get_id() <= APP_ID_NONE) + asd.set_service_appid_data(app_id, change_bits, version); + if (appidDebug->is_active()) { - if (appidDebug->is_active() && client_id > APP_ID_NONE && client_id != - APP_ID_HTTP && asd.client.get_id() != client_id) - { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(client_id); - LogMessage("AppIdDbg %s URL is client %s (%d)\n", - appidDebug->get_debug_session(), - app_name ? app_name : "unknown", - client_id); - } - asd.set_client_appid_data(client_id, change_bits); + const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(app_id); + LogMessage("AppIdDbg %s X service %s (%d)\n", appidDebug->get_debug_session(), + app_name ? app_name : "unknown", app_id); } + } + } + asd.scan_flags &= ~SCAN_HTTP_XWORKINGWITH_FLAG; - if (asd.service.get_id() <= APP_ID_NONE) - { - if (appidDebug->is_active() && service_id > APP_ID_NONE && service_id != - APP_ID_HTTP && asd.service.get_id() != service_id) - { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id); - LogMessage("AppIdDbg %s URL is service %s (%d)\n", - appidDebug->get_debug_session(), - app_name ? app_name : "unknown", - service_id); - } - asd.set_service_appid_data(service_id, change_bits); - } + snort_free(version); + } + + // Scan Content-Type Header for multimedia types and scan contents + const std::string* content_type = meta_data[RSP_CONTENT_TYPE_FID]; + if ( (asd.scan_flags & SCAN_HTTP_CONTENT_TYPE_FLAG) + and content_type and !asd.get_tp_payload_app_id() and !payload.get_id()) + { + AppId payload_id = http_matchers.get_appid_by_content_type(content_type->c_str(), + content_type->size()); + set_payload(payload_id, change_bits, "Content-Type"); + asd.scan_flags &= ~SCAN_HTTP_CONTENT_TYPE_FLAG; + } - // DO overwrite a previously-set data - if (appidDebug->is_active() && payload_id > APP_ID_NONE && - asd.payload.get_id() != payload_id) + if (asd.scan_flags & SCAN_HTTP_HOST_URL_FLAG) + { + AppId referredPayloadAppId = APP_ID_NONE; + char* version = nullptr; + char* my_host = host ? snort_strdup(host->c_str()) : nullptr; + const char* refStr = referer ? referer->c_str() : nullptr; + const std::string* url = meta_data[MISC_URL_FID]; + const char* urlStr = url ? url->c_str() : nullptr; + AppId service_id = APP_ID_NONE; + AppId client_id = APP_ID_NONE; + AppId payload_id = APP_ID_NONE; + + if ( http_matchers.get_appid_from_url(my_host, urlStr, &version, + refStr, &client_id, &service_id, &payload_id, + &referredPayloadAppId, false, asd.ctxt.get_odp_ctxt()) ) + { + // do not overwrite a previously-set client or service + if (client.get_id() <= APP_ID_NONE and client_id != APP_ID_HTTP) + set_client(client_id, change_bits, "URL", version); + + if (asd.service.get_id() <= APP_ID_NONE) + { + if (appidDebug->is_active() && service_id > APP_ID_NONE && service_id != + APP_ID_HTTP && asd.service.get_id() != service_id) { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(payload_id); - LogMessage("AppIdDbg %s URL is payload %s (%d)\n", appidDebug->get_debug_session(), + const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(service_id); + LogMessage("AppIdDbg %s URL is service %s (%d)\n", + appidDebug->get_debug_session(), app_name ? app_name : "unknown", - payload_id); + service_id); } - asd.set_payload_appid_data(payload_id, change_bits, version); - asd.set_referred_payload_app_id_data(referredPayloadAppId, change_bits); + asd.set_service_appid_data(service_id, change_bits); } - asd.scan_flags &= ~SCAN_HTTP_HOST_URL_FLAG; - if ( version ) - snort_free(version); - if ( my_host ) - snort_free(my_host); + // DO overwrite a previously-set payload + set_payload(payload_id, change_bits, "URL"); + set_referred_payload(referredPayloadAppId, change_bits); } - if (asd.client.get_id() == APP_ID_APPLE_CORE_MEDIA) + asd.scan_flags &= ~SCAN_HTTP_HOST_URL_FLAG; + if ( version ) + snort_free(version); + if ( my_host ) + snort_free(my_host); + } + + if (client.get_id() == APP_ID_APPLE_CORE_MEDIA) + { + AppInfoTableEntry* entry; + AppId tp_payload_app_id = asd.get_tp_payload_app_id(); + if (tp_payload_app_id > APP_ID_NONE) { - AppInfoTableEntry* entry; - AppId tp_payload_app_id = asd.get_tp_payload_app_id(); - if (tp_payload_app_id > APP_ID_NONE) + entry = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_info_entry(tp_payload_app_id); + // only move tpPayloadAppId to client if client app id is valid + if (entry && entry->clientId > APP_ID_NONE) { - entry = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_info_entry(tp_payload_app_id); - // only move tpPayloadAppId to client if client app id is valid - if (entry && entry->clientId > APP_ID_NONE) - { - asd.misc_app_id = asd.client.get_id(); - asd.client.set_id(tp_payload_app_id); - } + misc_app_id = client.get_id(); + client.set_id(tp_payload_app_id); } - else if (asd.payload.get_id() > APP_ID_NONE) + } + else if (payload.get_id() > APP_ID_NONE) + { + entry = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_info_entry(payload.get_id()); + // only move payload_app_id to client if it has a ClientAppid + if (entry && entry->clientId > APP_ID_NONE) { - entry = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_info_entry(asd.payload.get_id()); - // only move payload_app_id to client if it has a ClientAppid - if (entry && entry->clientId > APP_ID_NONE) - { - asd.misc_app_id = asd.client.get_id(); - asd.client.set_id(asd.payload.get_id()); - } + misc_app_id = client.get_id(); + client.set_id(payload.get_id()); } } + } - asd.clear_http_flags(); - } // end DON'T skip_simple_detect + asd.clear_http_flags(); return 0; } @@ -747,3 +733,109 @@ void AppIdHttpSession::clear_all_fields() } } +void AppIdHttpSession::set_field(HttpFieldIds id, const std::string* str, + AppidChangeBits& change_bits) +{ + delete meta_data[id]; + meta_data[id] = str; + if (str) + { + set_http_change_bits(change_bits, id); + + if (appidDebug->is_active()) + print_field(id, str); + } +} + +void AppIdHttpSession::set_field(HttpFieldIds id, const uint8_t* str, int32_t len, + AppidChangeBits& change_bits) +{ + delete meta_data[id]; + if (str and len) + { + meta_data[id] = new std::string((const char*)str, len); + set_http_change_bits(change_bits, id); + + if (appidDebug->is_active()) + print_field(id, meta_data[id]); + } + else + meta_data[id] = nullptr; +} + +void AppIdHttpSession::print_field(HttpFieldIds id, const std::string* field) +{ + string field_name; + + if (asd.get_session_flags(APPID_SESSION_SPDY_SESSION)) + field_name = "SPDY "; + else if (asd.get_session_flags(APPID_SESSION_HTTP_SESSION)) + field_name = "HTTP "; + else + // This could be RTMP session; not printing RTMP fields for now + return; + + switch (id) + { + case REQ_AGENT_FID: + field_name += "user agent"; + break; + + case REQ_HOST_FID: + field_name += "host"; + break; + + case REQ_REFERER_FID: + field_name += "referer"; + break; + + case REQ_URI_FID: + field_name += "URI"; + break; + + case REQ_COOKIE_FID: + field_name += "cookie"; + break; + + case REQ_BODY_FID: + field_name += "request body"; + break; + + case RSP_CONTENT_TYPE_FID: + field_name += "content type"; + break; + + case RSP_LOCATION_FID: + field_name += "location"; + break; + + case MISC_VIA_FID: + field_name += "via"; + break; + + case MISC_RESP_CODE_FID: + field_name += "response code"; + break; + + case MISC_SERVER_FID: + field_name += "server"; + break; + + case MISC_XWW_FID: + field_name += "x-working-with"; + break; + + // don't print these fields + case MISC_URL_FID: + case RSP_BODY_FID: + default: + return; + } + + if (http2_stream_id > 0) + LogMessage("AppIdDbg %s stream %u: %s is %s\n", appidDebug->get_debug_session(), + http2_stream_id, field_name.c_str(), field->c_str()); + else + LogMessage("AppIdDbg %s %s is %s\n", appidDebug->get_debug_session(), + field_name.c_str(), field->c_str()); +} diff --git a/src/network_inspectors/appid/appid_http_session.h b/src/network_inspectors/appid/appid_http_session.h index bb118c3af..e943c9408 100644 --- a/src/network_inspectors/appid/appid_http_session.h +++ b/src/network_inspectors/appid/appid_http_session.h @@ -60,7 +60,7 @@ class AppIdHttpSession public: typedef std::pair pair_t; - AppIdHttpSession(AppIdSession&); + AppIdHttpSession(AppIdSession&, uint32_t); virtual ~AppIdHttpSession(); ClientAppDescriptor client; PayloadAppDescriptor payload; @@ -71,6 +71,8 @@ public: HttpPatternMatchers& http_matchers); void update_url(AppidChangeBits& change_bits); + void set_field(HttpFieldIds id, const std::string* str, AppidChangeBits& change_bits); + void set_field(HttpFieldIds id, const uint8_t* str, int32_t len, AppidChangeBits& change_bits); const std::string* get_field(HttpFieldIds id) { return meta_data[id]; } @@ -78,26 +80,6 @@ public: const char* get_cfield(HttpFieldIds id) { return meta_data[id] != nullptr ? meta_data[id]->c_str() : nullptr; } - void set_field(HttpFieldIds id, const std::string* str, AppidChangeBits& change_bits) - { - delete meta_data[id]; - meta_data[id] = str; - if (str) - set_http_change_bits(change_bits, id); - } - - void set_field(HttpFieldIds id, const uint8_t* str, int32_t len, AppidChangeBits& change_bits) - { - delete meta_data[id]; - if (str and len) - { - meta_data[id] = new std::string((const char*)str, len); - set_http_change_bits(change_bits, id); - } - else - meta_data[id] = nullptr; - } - bool get_offset(int id, uint16_t& start, uint16_t& end) { if ( REQ_AGENT_FID <= id and id < NUM_HTTP_FIELDS ) @@ -169,6 +151,14 @@ public: virtual void custom_init() { } void clear_all_fields(); + void set_client(AppId, AppidChangeBits&, const char*, const char* version = nullptr); + void set_payload(AppId, AppidChangeBits&, const char*, const char* version = nullptr); + void set_referred_payload(AppId, AppidChangeBits&); + + uint32_t get_http2_stream_id() const + { + return http2_stream_id; + } protected: @@ -177,6 +167,7 @@ protected: void process_chp_buffers(AppidChangeBits&, HttpPatternMatchers&); void free_chp_matches(ChpMatchDescriptor& cmd, unsigned max_matches); void set_http_change_bits(AppidChangeBits& change_bits, HttpFieldIds id); + void print_field(HttpFieldIds id, const std::string* str); AppIdSession& asd; @@ -206,6 +197,7 @@ protected: #if RESPONSE_CODE_PACKET_THRESHHOLD unsigned response_code_packets = 0; #endif + uint32_t http2_stream_id = 0; }; #endif diff --git a/src/network_inspectors/appid/appid_inspector.cc b/src/network_inspectors/appid/appid_inspector.cc index 5a9b58812..1d681293b 100644 --- a/src/network_inspectors/appid/appid_inspector.cc +++ b/src/network_inspectors/appid/appid_inspector.cc @@ -68,7 +68,7 @@ static void add_appid_to_packet_trace(Flow& flow) { AppId service_id, client_id, payload_id, misc_id; const char* service_app_name, * client_app_name, * payload_app_name, * misc_name; - session->get_application_ids(service_id, client_id, payload_id, misc_id); + session->get_first_stream_app_ids(service_id, client_id, payload_id, misc_id); service_app_name = appid_api.get_application_name(service_id, session->ctxt); client_app_name = appid_api.get_application_name(client_id, session->ctxt); payload_app_name = appid_api.get_application_name(payload_id, session->ctxt); diff --git a/src/network_inspectors/appid/appid_session.cc b/src/network_inspectors/appid/appid_session.cc index b06450d8a..7ab5e0cab 100644 --- a/src/network_inspectors/appid/appid_session.cc +++ b/src/network_inspectors/appid/appid_session.cc @@ -147,7 +147,6 @@ AppIdSession::~AppIdSession() free_flow_data(); service_candidates.clear(); client_candidates.clear(); - snort_free(firewall_early_data); } // FIXIT-RC X Move this to somewhere more generally available/appropriate (decode_data.h). @@ -225,7 +224,7 @@ void AppIdSession::reinit_session_data(AppidChangeBits& change_bits) if (is_service_over_ssl(tp_app_id)) { payload.reset(); - referred_payload_app_id = tp_payload_app_id = APP_ID_NONE; + tp_payload_app_id = APP_ID_NONE; clear_session_flags(APPID_SESSION_CONTINUE); if (!hsessions.empty()) hsessions[0]->set_field(MISC_URL_FID, nullptr, change_bits); @@ -265,62 +264,48 @@ void AppIdSession::reinit_session_data(AppidChangeBits& change_bits) void AppIdSession::sync_with_snort_protocol_id(AppId newAppId, Packet* p) { - if (newAppId > APP_ID_NONE && newAppId < SF_APPID_MAX) + if (newAppId <= APP_ID_NONE or newAppId >= SF_APPID_MAX) + return; + + // Certain AppIds are not useful to identifying snort preprocessor choices + switch (newAppId) { - // Certain AppIds are not useful to identifying snort preprocessor choices - switch (newAppId) - { - case APP_ID_FTPS: - case APP_ID_FTPSDATA: - - // These all are variants of HTTPS - case APP_ID_DDM_SSL: - case APP_ID_MSFT_GC_SSL: - case APP_ID_NSIIOPS: - case APP_ID_SF_APPLIANCE_MGMT: - case APP_ID_HTTPS: - - case APP_ID_IMAPS: - case APP_ID_IRCS: - case APP_ID_LDAPS: - case APP_ID_NNTPS: - case APP_ID_POP3S: - case APP_ID_SMTPS: - case APP_ID_SSHELL: - case APP_ID_TELNETS: - return; - - case APP_ID_HTTP: - if (is_http2) - newAppId = APP_ID_HTTP2; - break; - default: + case APP_ID_FTPS: + case APP_ID_FTPSDATA: + + // These all are variants of HTTPS + case APP_ID_DDM_SSL: + case APP_ID_MSFT_GC_SSL: + case APP_ID_NSIIOPS: + case APP_ID_SF_APPLIANCE_MGMT: + case APP_ID_HTTPS: + + case APP_ID_IMAPS: + case APP_ID_IRCS: + case APP_ID_LDAPS: + case APP_ID_NNTPS: + case APP_ID_POP3S: + case APP_ID_SMTPS: + case APP_ID_SSHELL: + case APP_ID_TELNETS: + return; + + default: break; - } + } - AppInfoTableEntry* entry = - ctxt.get_odp_ctxt().get_app_info_mgr().get_app_info_entry(newAppId); - if (entry) - { - SnortProtocolId tmp_snort_protocol_id = entry->snort_protocol_id; - // A particular APP_ID_xxx may not be assigned a service_snort_key value - // in the rna_app.yaml file entry; so ignore the snort_protocol_id == - // UNKNOWN_PROTOCOL_ID case. - if (tmp_snort_protocol_id == UNKNOWN_PROTOCOL_ID && - (newAppId == APP_ID_HTTP2)) - tmp_snort_protocol_id = ctxt.config.snortId_for_http2; - - if (tmp_snort_protocol_id != snort_protocol_id) - { - snort_protocol_id = tmp_snort_protocol_id; - if (appidDebug->is_active() && - tmp_snort_protocol_id == ctxt.config.snortId_for_http2) - LogMessage("AppIdDbg %s Telling Snort that it's HTTP/2\n", - appidDebug->get_debug_session()); + AppInfoTableEntry* entry = ctxt.get_odp_ctxt().get_app_info_mgr().get_app_info_entry(newAppId); + if (!entry) + return; - p->flow->ssn_state.snort_protocol_id = tmp_snort_protocol_id; - } - } + SnortProtocolId tmp_snort_protocol_id = entry->snort_protocol_id; + // A particular APP_ID_xxx may not be assigned a service_snort_key value + // in the rna_app.yaml file entry; so ignore the snort_protocol_id == + // UNKNOWN_PROTOCOL_ID case. + if (tmp_snort_protocol_id != snort_protocol_id) + { + snort_protocol_id = tmp_snort_protocol_id; + p->flow->ssn_state.snort_protocol_id = tmp_snort_protocol_id; } } @@ -340,10 +325,10 @@ void AppIdSession::check_ssl_detection_restart(AppidChangeBits& change_bits) { set_session_flags(APPID_SESSION_DECRYPTED); encrypted.service_id = service_id; - encrypted.payload_id = pick_payload_app_id(); - encrypted.client_id = pick_client_app_id(); - encrypted.misc_id = pick_misc_app_id(); - encrypted.referred_id = pick_referred_payload_app_id(); + encrypted.payload_id = pick_ss_payload_app_id(); + encrypted.client_id = pick_ss_client_app_id(); + encrypted.misc_id = pick_ss_misc_app_id(); + encrypted.referred_id = pick_ss_referred_payload_app_id(); reinit_session_data(change_bits); if (appidDebug->is_active()) LogMessage("AppIdDbg %s SSL decryption is available, restarting app detection\n", @@ -514,13 +499,14 @@ void AppIdSession::examine_rtmp_metadata(AppidChangeBits& change_bits) AppId referred_payload_id = APP_ID_NONE; char* version = nullptr; - if (hsessions.empty()) + AppIdHttpSession* hsession = get_http_session(); + if (!hsession) return; - if (const char* url = hsessions[0]->get_cfield(MISC_URL_FID)) + if (const char* url = hsession->get_cfield(MISC_URL_FID)) { HttpPatternMatchers& http_matchers = ctxt.get_odp_ctxt().get_http_matchers(); - const char* referer = hsessions[0]->get_cfield(REQ_REFERER_FID); + const char* referer = hsession->get_cfield(REQ_REFERER_FID); if (((http_matchers.get_appid_from_url(nullptr, url, &version, referer, &client_id, &service_id, &payload_id, &referred_payload_id, true, ctxt.get_odp_ctxt())) || @@ -528,15 +514,15 @@ void AppIdSession::examine_rtmp_metadata(AppidChangeBits& change_bits) referer, &client_id, &service_id, &payload_id, &referred_payload_id, false, ctxt.get_odp_ctxt())))) { - /* do not overwrite a previously-set client or service */ - if (client.get_id() <= APP_ID_NONE) - set_client_appid_data(payload_id, change_bits); + // do not overwrite a previously-set client or service + if (hsession->client.get_id() <= APP_ID_NONE) + hsession->set_client(client_id, change_bits, "URL"); if (service.get_id() <= APP_ID_NONE) set_service_appid_data(service_id, change_bits); - /* DO overwrite a previously-set data */ - set_payload_appid_data((AppId)payload.get_id(), change_bits); - set_referred_payload_app_id_data(referred_payload_id, change_bits); + // DO overwrite a previously-set payload + hsession->set_payload(payload_id, change_bits, "URL"); + hsession->set_referred_payload(referred_payload_id, change_bits); } } } @@ -558,18 +544,6 @@ void AppIdSession::set_client_appid_data(AppId id, AppidChangeBits& change_bits, client.set_version(version, change_bits); } -void AppIdSession::set_referred_payload_app_id_data(AppId id, AppidChangeBits& change_bits) -{ - if (id <= APP_ID_NONE) - return; - - if (referred_payload_app_id != id) - { - referred_payload_app_id = id; - change_bits.set(APPID_REFERRED_BIT); - } -} - void AppIdSession::set_payload_appid_data(AppId id, AppidChangeBits& change_bits, char* version) { if (id <= APP_ID_NONE) @@ -634,8 +608,7 @@ void AppIdSession::delete_session_data() rna_ss = subtype; } - for (auto* hsession: hsessions) - delete hsession; + delete_all_http_sessions(); free_tls_session_data(); delete dsession; } @@ -779,53 +752,57 @@ AppId AppIdSession::pick_service_app_id() return rval; } -AppId AppIdSession::pick_only_service_app_id() +AppId AppIdSession::pick_ss_misc_app_id() { - if (common.flow_type != APPID_FLOW_TYPE_NORMAL) + if (common.flow_type != APPID_FLOW_TYPE_NORMAL or + service.get_id() == APP_ID_HTTP2) return APP_ID_NONE; - bool deferred = service.get_deferred() || tp_app_id_deferred; - - if (service.get_id() > APP_ID_NONE && !deferred) - return service.get_id(); - - if (tp_app_id > APP_ID_NONE and is_tp_appid_available()) - return tp_app_id; - else if (deferred) - return service.get_id(); - - if (service.get_id() < APP_ID_NONE) - return APP_ID_UNKNOWN_UI; - - return APP_ID_NONE; -} - -AppId AppIdSession::pick_misc_app_id() -{ - if (common.flow_type != APPID_FLOW_TYPE_NORMAL) - return APP_ID_NONE; if (misc_app_id > APP_ID_NONE) return misc_app_id; + + AppId tmp_id = APP_ID_NONE; + if (!hsessions.empty()) + tmp_id = hsessions[0]->misc_app_id; + if (tmp_id > APP_ID_NONE) + return tmp_id; + return encrypted.misc_id; } -AppId AppIdSession::pick_client_app_id() +AppId AppIdSession::pick_ss_client_app_id() { - if (common.flow_type != APPID_FLOW_TYPE_NORMAL) + if (common.flow_type != APPID_FLOW_TYPE_NORMAL or + service.get_id() == APP_ID_HTTP2) return APP_ID_NONE; + + AppId tmp_id = APP_ID_NONE; + if (!hsessions.empty()) + tmp_id = hsessions[0]->client.get_id(); + if (tmp_id > APP_ID_NONE) + return tmp_id; + if (client.get_id() > APP_ID_NONE) return client.get_id(); + return encrypted.client_id; } -AppId AppIdSession::pick_payload_app_id() +AppId AppIdSession::pick_ss_payload_app_id() { - if (common.flow_type != APPID_FLOW_TYPE_NORMAL) + if (common.flow_type != APPID_FLOW_TYPE_NORMAL or + service.get_id() == APP_ID_HTTP2) return APP_ID_NONE; if (tp_payload_app_id_deferred) return tp_payload_app_id; + AppId tmp_id = APP_ID_NONE; + if (!hsessions.empty()) + tmp_id = hsessions[0]->payload.get_id(); + if (tmp_id > APP_ID_NONE) + return tmp_id; + if (payload.get_id() > APP_ID_NONE) return payload.get_id(); @@ -835,24 +812,34 @@ AppId AppIdSession::pick_payload_app_id() if (encrypted.payload_id > APP_ID_NONE) return encrypted.payload_id; - /* APP_ID_UNKNOWN is valid only for HTTP type services */ + // APP_ID_UNKNOWN is valid only for HTTP type services + if (tmp_id == APP_ID_UNKNOWN) + return tmp_id; + + AppId service_id = pick_service_app_id(); if (payload.get_id() == APP_ID_UNKNOWN and - is_svc_http_type(service.get_id())) + is_svc_http_type(service_id)) return APP_ID_UNKNOWN; return APP_ID_NONE; } -AppId AppIdSession::pick_referred_payload_app_id() +AppId AppIdSession::pick_ss_referred_payload_app_id() { - if (common.flow_type != APPID_FLOW_TYPE_NORMAL) + if (common.flow_type != APPID_FLOW_TYPE_NORMAL or + service.get_id() == APP_ID_HTTP2) return APP_ID_NONE; - if (referred_payload_app_id > APP_ID_NONE) - return referred_payload_app_id; + + AppId tmp_id = APP_ID_NONE; + if (!hsessions.empty()) + tmp_id = hsessions[0]->referred_payload_app_id; + if (tmp_id > APP_ID_NONE) + return tmp_id; + return encrypted.referred_id; } -void AppIdSession::set_application_ids(AppId service_id, AppId client_id, +void AppIdSession::set_ss_application_ids(AppId service_id, AppId client_id, AppId payload_id, AppId misc_id, AppidChangeBits& change_bits) { if (application_ids[APP_PROTOID_SERVICE] != service_id) @@ -877,21 +864,58 @@ void AppIdSession::set_application_ids(AppId service_id, AppId client_id, } } -void AppIdSession::get_application_ids(AppId& service_id, AppId& client_id, +void AppIdSession::set_application_ids_service(AppId service_id, AppidChangeBits& change_bits) +{ + if (application_ids[APP_PROTOID_SERVICE] != service_id) + { + application_ids[APP_PROTOID_SERVICE] = service_id; + change_bits.set(APPID_SERVICE_BIT); + } +} + +void AppIdSession::get_first_stream_app_ids(AppId& service_id, AppId& client_id, AppId& payload_id, AppId& misc_id) { service_id = application_ids[APP_PROTOID_SERVICE]; - client_id = application_ids[APP_PROTOID_CLIENT]; - payload_id = application_ids[APP_PROTOID_PAYLOAD]; - misc_id = application_ids[APP_PROTOID_MISC]; + if (service_id != APP_ID_HTTP2) + { + client_id = application_ids[APP_PROTOID_CLIENT]; + payload_id = application_ids[APP_PROTOID_PAYLOAD]; + misc_id = application_ids[APP_PROTOID_MISC]; + } + else if (AppIdHttpSession* hsession = get_http_session(0)) + { + client_id = hsession->client.get_id(); + payload_id = hsession->payload.get_id(); + misc_id = hsession->misc_app_id; + } + else + { + client_id = APP_ID_NONE; + payload_id = APP_ID_NONE; + misc_id = APP_ID_NONE; + } } -void AppIdSession::get_application_ids(AppId& service_id, AppId& client_id, +void AppIdSession::get_first_stream_app_ids(AppId& service_id, AppId& client_id, AppId& payload_id) { service_id = application_ids[APP_PROTOID_SERVICE]; - client_id = application_ids[APP_PROTOID_CLIENT]; - payload_id = application_ids[APP_PROTOID_PAYLOAD]; + if (service_id != APP_ID_HTTP2) + { + client_id = application_ids[APP_PROTOID_CLIENT]; + payload_id = application_ids[APP_PROTOID_PAYLOAD]; + } + else if (AppIdHttpSession* hsession = get_http_session(0)) + { + client_id = hsession->client.get_id(); + payload_id = hsession->payload.get_id(); + } + else + { + client_id = APP_ID_NONE; + payload_id = APP_ID_NONE; + } } AppId AppIdSession::get_application_ids_service() @@ -899,19 +923,49 @@ AppId AppIdSession::get_application_ids_service() return application_ids[APP_PROTOID_SERVICE]; } -AppId AppIdSession::get_application_ids_client() +AppId AppIdSession::get_application_ids_client(uint32_t stream_index) { - return application_ids[APP_PROTOID_CLIENT]; + if (get_application_ids_service() == APP_ID_HTTP2) + { + if (stream_index >= get_hsessions_size()) + return APP_ID_NONE; + else if (AppIdHttpSession* hsession = get_http_session(stream_index)) + return hsession->client.get_id(); + } + else if (stream_index == 0) + return application_ids[APP_PROTOID_CLIENT]; + + return APP_ID_NONE; } -AppId AppIdSession::get_application_ids_payload() +AppId AppIdSession::get_application_ids_payload(uint32_t stream_index) { - return application_ids[APP_PROTOID_PAYLOAD]; + if (get_application_ids_service() == APP_ID_HTTP2) + { + if (stream_index >= get_hsessions_size()) + return APP_ID_NONE; + else if (AppIdHttpSession* hsession = get_http_session(stream_index)) + return hsession->payload.get_id(); + } + else if (stream_index == 0) + return application_ids[APP_PROTOID_PAYLOAD]; + + return APP_ID_NONE; } -AppId AppIdSession::get_application_ids_misc() +AppId AppIdSession::get_application_ids_misc(uint32_t stream_index) { - return application_ids[APP_PROTOID_MISC]; + if (service.get_id() == APP_ID_HTTP2) + { + if (stream_index >= get_hsessions_size()) + return APP_ID_NONE; + else if (AppIdHttpSession* hsession = get_http_session(stream_index)) + return hsession->misc_app_id; + } + else if (stream_index == 0) + return application_ids[APP_PROTOID_MISC]; + + return APP_ID_NONE; } bool AppIdSession::is_ssl_session_decrypted() @@ -955,9 +1009,9 @@ void AppIdSession::clear_http_data() hsessions[0]->clear_all_fields(); } -AppIdHttpSession* AppIdSession::create_http_session() +AppIdHttpSession* AppIdSession::create_http_session(uint32_t stream_id) { - AppIdHttpSession* hsession = new AppIdHttpSession(*this); + AppIdHttpSession* hsession = new AppIdHttpSession(*this, stream_id); hsessions.push_back(hsession); return hsession; } @@ -980,7 +1034,7 @@ bool AppIdSession::is_tp_appid_done() const { if (ctxt.get_tp_appid_ctxt()) { - if (get_session_flags(APPID_SESSION_IGNORE_FLOW)) + if (get_session_flags(APPID_SESSION_FUTURE_FLOW)) return true; if (!tpsession) @@ -1052,19 +1106,24 @@ void AppIdSession::set_tp_payload_app_id(Packet& p, AppidSessionDirection dir, A } } -void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow) +void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow, + bool is_http2, uint32_t http2_stream_index) { if (change_bits.none()) return; - AppidEvent app_event(change_bits); + AppidEvent app_event(change_bits, is_http2, http2_stream_index); DataBus::publish(APPID_EVENT_ANY_CHANGE, app_event, flow); if (appidDebug->is_active()) { std::string str; change_bits_to_string(change_bits, str); - LogMessage("AppIdDbg %s Published event for changes: %s\n", - appidDebug->get_debug_session(), str.c_str()); + if (is_http2) + LogMessage("AppIdDbg %s Published event for changes: %s for HTTP2 stream index %u\n", + appidDebug->get_debug_session(), str.c_str(), http2_stream_index); + else + LogMessage("AppIdDbg %s Published event for changes: %s\n", + appidDebug->get_debug_session(), str.c_str()); } } diff --git a/src/network_inspectors/appid/appid_session.h b/src/network_inspectors/appid/appid_session.h index 7b94f245e..961d96548 100644 --- a/src/network_inspectors/appid/appid_session.h +++ b/src/network_inspectors/appid/appid_session.h @@ -122,7 +122,6 @@ struct CommonAppIdData } snort::APPID_FLOW_TYPE flow_type = snort::APPID_FLOW_TYPE_IGNORE; - unsigned policyId = 0; //flags shared with other preprocessor via session attributes. uint64_t flags = 0; snort::SfIp initiator_ip; @@ -237,8 +236,13 @@ public: snort::AppIdServiceSubtype* subtype = nullptr; std::vector service_candidates; ServiceAppDescriptor service; + + // Following three fields are used only for non-http sessions. For HTTP traffic, + // these fields are maintained inside AppIdHttpSession. + // Note: RTMP traffic is treated like HTTP in AppId ClientAppDescriptor client; PayloadAppDescriptor payload; + AppId misc_app_id = APP_ID_NONE; // AppId matching client side APPID_DISCOVERY_STATE client_disco_state = APPID_DISCO_STATE_NONE; @@ -247,10 +251,6 @@ public: std::map client_candidates; bool tried_reverse_service = false; - AppId referred_payload_app_id = APP_ID_NONE; - AppId misc_app_id = APP_ID_NONE; - - // FIXIT-RC netbios_name is never set to a valid value; set when netbios_domain is set? char* netbios_name = nullptr; char* netbios_domain = nullptr; @@ -284,11 +284,9 @@ public: AppId referred_id; } encrypted = { APP_ID_NONE, APP_ID_NONE, APP_ID_NONE, APP_ID_NONE, APP_ID_NONE }; - void* firewall_early_data = nullptr; AppId past_indicator = APP_ID_NONE; AppId past_forecast = APP_ID_NONE; - bool is_http2 = false; bool in_expected_cache = false; static unsigned inspector_id; static std::mutex inferred_svcs_lock; @@ -317,19 +315,27 @@ public: void free_flow_data(); AppId pick_service_app_id(); - AppId pick_only_service_app_id(); - AppId pick_misc_app_id(); - AppId pick_client_app_id(); - AppId pick_payload_app_id(); - AppId pick_referred_payload_app_id(); - void set_application_ids(AppId service, AppId client, AppId payload, AppId misc, + // pick_ss_* and set_ss_* methods below are for application protocols that support only a single + // stream in a flow. They should not be used for HTTP2 sessions which can have multiple + // streams within a single flow + AppId pick_ss_misc_app_id(); + AppId pick_ss_client_app_id(); + AppId pick_ss_payload_app_id(); + AppId pick_ss_referred_payload_app_id(); + + void set_ss_application_ids(AppId service, AppId client, AppId payload, AppId misc, AppidChangeBits& change_bits); - void get_application_ids(AppId& service, AppId& client, AppId& payload, AppId& misc); - void get_application_ids(AppId& service, AppId& client, AppId& payload); + void set_application_ids_service(AppId service_id, AppidChangeBits& change_bits); + + // For protocols such as HTTP2 which can have multiple streams within a single flow, get_first_stream_* + // methods return the appids in the first stream seen in a packet. + void get_first_stream_app_ids(AppId& service, AppId& client, AppId& payload, AppId& misc); + void get_first_stream_app_ids(AppId& service, AppId& client, AppId& payload); AppId get_application_ids_service(); - AppId get_application_ids_client(); - AppId get_application_ids_payload(); - AppId get_application_ids_misc(); + AppId get_application_ids_client(uint32_t stream_index = 0); + AppId get_application_ids_payload(uint32_t stream_index = 0); + AppId get_application_ids_misc(uint32_t stream_index = 0); + uint32_t get_hsessions_size() { return hsessions.size(); @@ -339,7 +345,6 @@ public: void examine_ssl_metadata(snort::Packet*, AppidChangeBits& change_bits); void set_client_appid_data(AppId, AppidChangeBits& change_bits, char* version = nullptr); void set_service_appid_data(AppId, AppidChangeBits& change_bits, char* version = nullptr); - void set_referred_payload_app_id_data(AppId, AppidChangeBits& change_bits); void set_payload_appid_data(AppId, AppidChangeBits& change_bits, char* version = nullptr); void check_app_detection_restart(AppidChangeBits& change_bits); void check_ssl_detection_restart(AppidChangeBits& change_bits); @@ -354,8 +359,15 @@ public: void clear_http_data(); void reset_session_data(); - AppIdHttpSession* create_http_session(); + AppIdHttpSession* create_http_session(uint32_t stream_id = 0); AppIdHttpSession* get_http_session(uint32_t stream_index = 0); + void delete_all_http_sessions() + { + for (auto hsession : hsessions) + delete hsession; + hsessions.clear(); + } + AppIdDnsSession* get_dns_session(); bool is_tp_appid_done() const; @@ -366,7 +378,8 @@ public: AppidChangeBits& change_bits); void set_tp_payload_app_id(snort::Packet& p, AppidSessionDirection dir, AppId app_id, AppidChangeBits& change_bits); - void publish_appid_event(AppidChangeBits&, snort::Flow*); + void publish_appid_event(AppidChangeBits&, snort::Flow*, bool is_http2 = false, + uint32_t http2_stream_index = 0); inline void set_tp_app_id(AppId app_id) { if (tp_app_id != app_id) @@ -409,9 +422,20 @@ public: inferred_svcs_ver++; } + uint16_t get_prev_http2_raw_packet() const + { + return prev_http2_raw_packet; + } + + void set_prev_http2_raw_packet(uint16_t packet_num) + { + prev_http2_raw_packet = packet_num; + } + private: std::vector hsessions; AppIdDnsSession* dsession = nullptr; + uint16_t prev_http2_raw_packet = 0; void reinit_session_data(AppidChangeBits& change_bits); void delete_session_data(); diff --git a/src/network_inspectors/appid/appid_session_api.cc b/src/network_inspectors/appid/appid_session_api.cc index b2c86f5fc..572aba566 100644 --- a/src/network_inspectors/appid/appid_session_api.cc +++ b/src/network_inspectors/appid/appid_session_api.cc @@ -55,103 +55,80 @@ AppId AppIdSessionApi::get_port_service_app_id() return asd->service.get_port_service_id(); } -AppId AppIdSessionApi::get_only_service_app_id() -{ - return asd->pick_only_service_app_id(); -} - AppId AppIdSessionApi::get_misc_app_id(uint32_t stream_index) { - if (asd->is_http2) - { - if (stream_index >= asd->get_hsessions_size()) - return APP_ID_UNKNOWN; - return asd->get_http_session(stream_index)->misc_app_id; - } - else - return asd->get_application_ids_misc(); + return asd->get_application_ids_misc(stream_index); } AppId AppIdSessionApi::get_client_app_id(uint32_t stream_index) { - if (asd->is_http2) - { - if (stream_index >= asd->get_hsessions_size()) - return APP_ID_UNKNOWN; - return asd->get_http_session(stream_index)->client.get_id(); - } - else - return asd->get_application_ids_client(); + return asd->get_application_ids_client(stream_index); } AppId AppIdSessionApi::get_payload_app_id(uint32_t stream_index) { - if (asd->is_http2) - { - if (stream_index >= asd->get_hsessions_size()) - return APP_ID_UNKNOWN; - return asd->get_http_session(stream_index)->payload.get_id(); - } - else - return asd->get_application_ids_payload(); + return asd->get_application_ids_payload(stream_index); } AppId AppIdSessionApi::get_referred_app_id(uint32_t stream_index) { - if (asd->is_http2) + if (asd->get_application_ids_service() == APP_ID_HTTP2) { - if (stream_index >= asd->get_hsessions_size()) + if ((stream_index != 0) and (stream_index >= asd->get_hsessions_size())) return APP_ID_UNKNOWN; - return asd->get_http_session(stream_index)->referred_payload_app_id; + else if (AppIdHttpSession* hsession = asd->get_http_session(stream_index)) + return hsession->referred_payload_app_id; } - else - return asd->pick_referred_payload_app_id(); + else if (stream_index == 0) + return asd->pick_ss_referred_payload_app_id(); + + return APP_ID_UNKNOWN; } void AppIdSessionApi::get_app_id(AppId& service, AppId& client, AppId& payload, AppId& misc, AppId& referred, uint32_t stream_index) { - if (asd->is_http2) + if (asd->get_application_ids_service() == APP_ID_HTTP2) { - if (stream_index >= asd->get_hsessions_size()) + if ((stream_index != 0) and (stream_index >= asd->get_hsessions_size())) service = client = payload = misc = referred = APP_ID_UNKNOWN; - else + else if (AppIdHttpSession* hsession = asd->get_http_session(stream_index)) { service = asd->get_application_ids_service(); - client = asd->get_http_session(stream_index)->client.get_id(); - payload = asd->get_http_session(stream_index)->payload.get_id(); - misc = asd->get_http_session(stream_index)->misc_app_id; - referred = asd->get_http_session(stream_index)->referred_payload_app_id; + client = hsession->client.get_id(); + payload = hsession->payload.get_id(); + misc = hsession->misc_app_id; + referred = hsession->referred_payload_app_id; } } else { - asd->get_application_ids(service, client, payload, misc); - referred = asd->pick_referred_payload_app_id(); + asd->get_first_stream_app_ids(service, client, payload, misc); + referred = asd->pick_ss_referred_payload_app_id(); } } void AppIdSessionApi::get_app_id(AppId* service, AppId* client, AppId* payload, AppId* misc, AppId* referred, uint32_t stream_index) { - if (asd->is_http2) + if (asd->get_application_ids_service() == APP_ID_HTTP2) { - if (stream_index >= asd->get_hsessions_size()) + if ((stream_index != 0) and (stream_index >= asd->get_hsessions_size())) { - if(service) + if (service) *service = APP_ID_UNKNOWN; - if(client) + if (client) *client = APP_ID_UNKNOWN; - if(payload) + if (payload) *payload = APP_ID_UNKNOWN; - if(misc) + if (misc) *misc = APP_ID_UNKNOWN; - if(referred) + if (referred) *referred = APP_ID_UNKNOWN; + return; } - else + else if (AppIdHttpSession* hsession = asd->get_http_session(stream_index)) { - AppIdHttpSession* hsession = asd->get_http_session(stream_index); if (service) *service = asd->get_application_ids_service(); if (client) @@ -162,6 +139,7 @@ void AppIdSessionApi::get_app_id(AppId* service, AppId* client, *misc = hsession->misc_app_id; if (referred) *referred = hsession->referred_payload_app_id; + return; } } if (service) @@ -173,7 +151,7 @@ void AppIdSessionApi::get_app_id(AppId* service, AppId* client, if (misc) *misc = asd->get_application_ids_misc(); if (referred) - *referred = asd->pick_referred_payload_app_id(); + *referred = asd->pick_ss_referred_payload_app_id(); } bool AppIdSessionApi::is_ssl_session_decrypted() @@ -230,14 +208,17 @@ bool AppIdSessionApi::is_appid_available() const char* AppIdSessionApi::get_client_version(uint32_t stream_index) { - if (asd->is_http2) + if (uint32_t num_hsessions = asd->get_hsessions_size()) { - if (stream_index >= asd->get_hsessions_size()) + if (stream_index >= num_hsessions) return nullptr; - return asd->get_http_session(stream_index)->client.get_version(); + else if (AppIdHttpSession* hsession = asd->get_http_session(stream_index)) + return hsession->client.get_version(); } - else + else if (stream_index == 0) return asd->client.get_version(); + + return nullptr; } uint64_t AppIdSessionApi::get_appid_session_attribute(uint64_t flags) diff --git a/src/network_inspectors/appid/appid_session_api.h b/src/network_inspectors/appid/appid_session_api.h index 98d961fdd..4bd1f348e 100644 --- a/src/network_inspectors/appid/appid_session_api.h +++ b/src/network_inspectors/appid/appid_session_api.h @@ -36,7 +36,7 @@ namespace snort #define APPID_SESSION_RESPONDER_MONITORED (1ULL << 0) #define APPID_SESSION_INITIATOR_MONITORED (1ULL << 1) #define APPID_SESSION_SPECIAL_MONITORED (1ULL << 2) -#define APPID_SESSION_IGNORE_FLOW_LOGGED (1ULL << 3) +#define APPID_SESSION_FUTURE_FLOW (1ULL << 3) #define APPID_SESSION_EXPECTED_EVALUATE (1ULL << 4) #define APPID_SESSION_DISCOVER_USER (1ULL << 5) #define APPID_SESSION_HAS_DHCP_FP (1ULL << 6) @@ -83,15 +83,14 @@ namespace snort #define APPID_SESSION_STICKY_SERVICE (1ULL << 36) #define APPID_SESSION_APP_REINSPECT_SSL (1ULL << 37) #define APPID_SESSION_NO_TPI (1ULL << 38) -#define APPID_SESSION_IGNORE_FLOW (1ULL << 39) -#define APPID_SESSION_IGNORE_FLOW_IDED (1ULL << 40) -#define APPID_SESSION_OOO_CHECK_TP (1ULL << 41) -#define APPID_SESSION_PAYLOAD_SEEN (1ULL << 42) -#define APPID_SESSION_HOST_CACHE_MATCHED (1ULL << 43) -#define APPID_SESSION_DECRYPT_MONITOR (1ULL << 44) -#define APPID_SESSION_HTTP_TUNNEL (1ULL << 45) +#define APPID_SESSION_FUTURE_FLOW_IDED (1ULL << 39) +#define APPID_SESSION_OOO_CHECK_TP (1ULL << 40) +#define APPID_SESSION_PAYLOAD_SEEN (1ULL << 41) +#define APPID_SESSION_HOST_CACHE_MATCHED (1ULL << 42) +#define APPID_SESSION_DECRYPT_MONITOR (1ULL << 43) +#define APPID_SESSION_HTTP_TUNNEL (1ULL << 44) #define APPID_SESSION_IGNORE_ID_FLAGS \ - (APPID_SESSION_IGNORE_FLOW | \ + (APPID_SESSION_FUTURE_FLOW | \ APPID_SESSION_NOT_A_SERVICE | \ APPID_SESSION_NO_TPI | \ APPID_SESSION_SERVICE_DETECTED | \ @@ -151,7 +150,6 @@ public: bool refresh(const Flow& flow); AppId get_service_app_id(); AppId get_port_service_app_id(); - AppId get_only_service_app_id(); AppId get_misc_app_id(uint32_t stream_index = 0); AppId get_client_app_id(uint32_t stream_index = 0); AppId get_payload_app_id(uint32_t stream_index = 0); diff --git a/src/network_inspectors/appid/appid_stats.cc b/src/network_inspectors/appid/appid_stats.cc index 77f25a6b3..0d3590f7a 100644 --- a/src/network_inspectors/appid/appid_stats.cc +++ b/src/network_inspectors/appid/appid_stats.cc @@ -273,7 +273,7 @@ void AppIdStatistics::update(AppIdSession& asd) bucket->totalStats.rxByteCnt += asd.stats.responder_bytes; AppId web_app_id, service_id, client_id; - asd.get_application_ids(service_id, client_id, web_app_id); + asd.get_first_stream_app_ids(service_id, client_id, web_app_id); if ( web_app_id > APP_ID_NONE ) update_stats(asd, web_app_id, bucket); diff --git a/src/network_inspectors/appid/appid_types.h b/src/network_inspectors/appid/appid_types.h index 2e1590413..0e3c1460f 100644 --- a/src/network_inspectors/appid/appid_types.h +++ b/src/network_inspectors/appid/appid_types.h @@ -64,4 +64,5 @@ enum AppidSessionDirection APP_ID_FROM_RESPONDER, APP_ID_APPID_SESSION_DIRECTION_MAX }; + #endif diff --git a/src/network_inspectors/appid/client_plugins/client_discovery.cc b/src/network_inspectors/appid/client_plugins/client_discovery.cc index caaa8465b..c2b655eb0 100644 --- a/src/network_inspectors/appid/client_plugins/client_discovery.cc +++ b/src/network_inspectors/appid/client_plugins/client_discovery.cc @@ -295,7 +295,6 @@ bool ClientDiscovery::do_client_discovery(AppIdSession& asd, Packet* p, bool isTpAppidDiscoveryDone = false; AppInfoTableEntry* entry; uint32_t prevRnaClientState = asd.client_disco_state; - bool was_http2 = asd.is_http2; bool was_service = asd.is_service_detected(); AppId tp_app_id = asd.get_tp_app_id(); @@ -384,10 +383,6 @@ bool ClientDiscovery::do_client_discovery(AppIdSession& asd, Packet* p, } } - if ( appidDebug->is_active() ) - if ( !was_http2 && asd.is_http2 ) - LogMessage("AppIdDbg %s Got a preface for HTTP/2\n", appidDebug->get_debug_session()); - if ( !was_service && asd.is_service_detected() ) asd.sync_with_snort_protocol_id(asd.service.get_id(), p); diff --git a/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h b/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h index 34189df39..c5a4aff60 100644 --- a/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h +++ b/src/network_inspectors/appid/client_plugins/test/client_plugin_mock.h @@ -78,7 +78,6 @@ void ClientDiscovery::initialize() {} int AppIdDetector::initialize(){return 0;} int AppIdDetector::data_add(AppIdSession&, void*, AppIdFreeFCN){return 0;} void* AppIdDetector::data_get(AppIdSession&) {return nullptr;} -void AppIdDetector::add_info(AppIdSession&, const char*, AppidChangeBits&){} void AppIdDetector::add_user(AppIdSession&, const char*, AppId, bool){} void AppIdDetector::add_payload(AppIdSession&, AppId){} void AppIdDetector::add_app(const snort::Packet&, AppIdSession&, AppidSessionDirection, AppId, AppId, const char*, AppidChangeBits&){} diff --git a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h index 5842342e4..1f594dfb2 100644 --- a/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h +++ b/src/network_inspectors/appid/detector_plugins/test/detector_plugins_mock.h @@ -149,8 +149,8 @@ AppIdContext stub_ctxt(stub_config); AppIdSession::AppIdSession(IpProtocol, const SfIp*, uint16_t, AppIdInspector& inspector) : snort::FlowData(inspector_id, (snort::Inspector*)&inspector), ctxt(stub_ctxt) { } AppIdSession::~AppIdSession() = default; -AppIdHttpSession::AppIdHttpSession(AppIdSession& asd) - : asd(asd) +AppIdHttpSession::AppIdHttpSession(AppIdSession& asd, uint32_t http2_stream_id) + : asd(asd), http2_stream_id(http2_stream_id) { for ( int i = 0; i < NUM_METADATA_FIELDS; i++) meta_data[i] = nullptr; diff --git a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc index 29a9e4034..a57f43e4e 100644 --- a/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc +++ b/src/network_inspectors/appid/detector_plugins/test/http_url_patterns_test.cc @@ -39,7 +39,7 @@ static const SfIp* sfip = nullptr; static AppIdModule appid_mod; static AppIdInspector appid_inspector(appid_mod); static AppIdSession session(IpProtocol::IP, sfip, 0, appid_inspector); -static AppIdHttpSession hsession(session); +static AppIdHttpSession mock_hsession(session, 0); static ChpMatchDescriptor cmd_test; static MatchedCHPAction mchp; static CHPAction chpa_test; @@ -55,9 +55,6 @@ OdpContext* AppIdContext::odp_ctxt = &odpctxt; static AppId service_id = APP_ID_NONE; static AppId client_id = APP_ID_NONE; static DetectorHTTPPattern mpattern; -static const char* my_buffer[NUM_HTTP_FIELDS] = { nullptr }; -static uint16_t my_length[NUM_HTTP_FIELDS] = { 0 }; -static CHPAction my_match; void ApplicationDescriptor::set_id(const Packet&, AppIdSession&, AppidSessionDirection, AppId, AppidChangeBits&) { } AppIdDiscovery::AppIdDiscovery() { } @@ -102,18 +99,18 @@ TEST(http_url_patterns_tests, http_field_pattern_match) fp.patternType = REQ_HOST_FID; fpd.payload = (const uint8_t*)"Google"; fpd.length = 6; - fpd.hsession = &hsession; + fpd.hsession = &mock_hsession; test_service_strstr_enabled = false; test_field_offset_set_done = false; - hsession.set_offset(fp.patternType, 0, 5); + mock_hsession.set_offset(fp.patternType, 0, 5); CHECK_EQUAL(1, http_field_pattern_match(&fp, nullptr, 0, &fpd, nullptr)); - hsession.get_offset(fp.patternType, off.first, off.second); + mock_hsession.get_offset(fp.patternType, off.first, off.second); CHECK_EQUAL(5, off.second); // check offset did not change test_service_strstr_enabled = true; CHECK_EQUAL(1, http_field_pattern_match(&fp, nullptr, 0, &fpd, nullptr)); - hsession.get_offset(fp.patternType, off.first, off.second); + mock_hsession.get_offset(fp.patternType, off.first, off.second); CHECK_EQUAL(0, off.second); // if it changed, service_strstr was called } @@ -158,9 +155,9 @@ TEST(http_url_patterns_tests, get_http_offsets) pkt.dsize = 2; pair_t off; - hsession.set_offset(REQ_AGENT_FID, 5, 0); - hm->get_http_offsets(&pkt, &hsession); - hsession.get_offset(REQ_AGENT_FID, off.first, off.second); + mock_hsession.set_offset(REQ_AGENT_FID, 5, 0); + hm->get_http_offsets(&pkt, &mock_hsession); + mock_hsession.get_offset(REQ_AGENT_FID, off.first, off.second); CHECK_EQUAL(0, off.first); // find_all is not called for bigger payload when service_strstr returns nullptr @@ -168,12 +165,12 @@ TEST(http_url_patterns_tests, get_http_offsets) test_find_all_done = false; pkt.data = (const uint8_t*)"GET http://www.w3.org HTTP/1.1"; pkt.dsize = strlen((const char*)pkt.data); - hm->get_http_offsets(&pkt, &hsession); + hm->get_http_offsets(&pkt, &mock_hsession); CHECK_EQUAL(false, test_find_all_done); // find_all is called for bigger payload when service_strstr returns something test_service_strstr_enabled = true; - hm->get_http_offsets(&pkt, &hsession); + hm->get_http_offsets(&pkt, &mock_hsession); CHECK_EQUAL(true, test_find_all_done); } @@ -222,7 +219,7 @@ TEST(http_url_patterns_tests, scan_chp_defer) mchp.mpattern = &chpa_test; cmd_test.chp_matches[RSP_BODY_FID].emplace_back(mchp); cmd_test.cur_ptype = RSP_BODY_FID; - CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &hsession, ctxt) == APP_ID_NONE); + CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &mock_hsession, ctxt) == APP_ID_NONE); CHECK_EQUAL(true, test_find_all_done); } @@ -236,7 +233,7 @@ TEST(http_url_patterns_tests, scan_chp_alt_appid) mchp.mpattern = &chpa_test; cmd_test.chp_matches[RSP_BODY_FID].emplace_back(mchp); cmd_test.cur_ptype = RSP_BODY_FID; - CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &hsession, ctxt) == APP_ID_NONE); + CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &mock_hsession, ctxt) == APP_ID_NONE); CHECK_EQUAL(true, test_find_all_done); } @@ -254,7 +251,7 @@ TEST(http_url_patterns_tests, scan_chp_extract_user) cmd_test.chp_matches[RSP_BODY_FID].emplace_back(mchp); cmd_test.buffer[RSP_BODY_FID] = (const char*)"userid\n\rpassword"; cmd_test.length[RSP_BODY_FID] = strlen(cmd_test.buffer[RSP_BODY_FID]); - CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &hsession, ctxt) == APP_ID_NONE); + CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &mock_hsession, ctxt) == APP_ID_NONE); CHECK_EQUAL(true, test_find_all_done); snort_free(user); user = nullptr; @@ -274,7 +271,7 @@ TEST(http_url_patterns_tests, scan_chp_hold_and_default) mchp.start_match_pos = 0; cmd_test.buffer[RSP_BODY_FID] = my_chp_data; cmd_test.length[RSP_BODY_FID] = strlen(cmd_test.buffer[RSP_BODY_FID]); - CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &hsession, ctxt) == APP_ID_NONE); + CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &mock_hsession, ctxt) == APP_ID_NONE); CHECK_EQUAL(true, test_find_all_done); // testing FUTURE_APPID_SESSION_SIP (default action) @@ -284,7 +281,7 @@ TEST(http_url_patterns_tests, scan_chp_hold_and_default) chpa_test.action = FUTURE_APPID_SESSION_SIP; mchp.mpattern = &chpa_test; cmd_test.chp_matches[RSP_BODY_FID].emplace_back(mchp); - CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &hsession, ctxt) == APP_ID_NONE); + CHECK(hm->scan_chp(cmd_test, &version, &user, &total_found, &mock_hsession, ctxt) == APP_ID_NONE); CHECK_EQUAL(true, test_find_all_done); } diff --git a/src/network_inspectors/appid/ips_appid_option.cc b/src/network_inspectors/appid/ips_appid_option.cc index 77a53b064..9f5606384 100644 --- a/src/network_inspectors/appid/ips_appid_option.cc +++ b/src/network_inspectors/appid/ips_appid_option.cc @@ -46,16 +46,6 @@ using namespace snort; #define s_help \ "detection option for application ids" -// these defs are used during matching when the rule option eval function is called to -// control the order in which the different id types are checked -#define PAYLOAD 0 -#define MISC 1 -#define CP_CLIENT 2 -#define CP_SERVICE 3 -#define SP_CLIENT 3 -#define SP_SERVICE 2 -#define NUM_ID_TYPES 4 - static THREAD_LOCAL ProfileStats ips_appid_perf_stats; class AppIdIpsOption : public IpsOption @@ -104,6 +94,9 @@ bool AppIdIpsOption::operator==(const IpsOption& ips) const bool AppIdIpsOption::match_id_against_rule(OdpContext& odp_ctxt, int32_t id) { + if (id <= APP_ID_NONE) + return false; + const char *app_name_key = odp_ctxt.get_app_info_mgr().get_app_name_key(id); if ( nullptr != app_name_key ) { @@ -129,21 +122,39 @@ IpsOption::EvalStatus AppIdIpsOption::eval(Cursor&, Packet* p) if ( !session ) return NO_MATCH; - AppId app_ids[NUM_ID_TYPES]; + AppId app_ids[APP_PROTOID_MAX]; + AppId service_id = session->get_application_ids_service(); + OdpContext& odp_ctxt = session->ctxt.get_odp_ctxt(); - // id order on stream api call is: service, client, payload, misc - if ( (p->packet_flags & PKT_FROM_CLIENT) ) - session->get_application_ids(app_ids[CP_SERVICE], app_ids[CP_CLIENT], - app_ids[PAYLOAD], app_ids[MISC]); - else - session->get_application_ids(app_ids[SP_SERVICE], app_ids[SP_CLIENT], - app_ids[PAYLOAD], app_ids[MISC]); + if (service_id != APP_ID_HTTP2) + { + // id order on stream api call is: service, client, payload, misc + session->get_first_stream_app_ids(app_ids[APP_PROTOID_SERVICE], app_ids[APP_PROTOID_CLIENT], + app_ids[APP_PROTOID_PAYLOAD], app_ids[APP_PROTOID_MISC]); - for ( unsigned i = 0; i < NUM_ID_TYPES; i++ ) - if ( (app_ids[i] > APP_ID_NONE) and - match_id_against_rule(session->ctxt.get_odp_ctxt(), app_ids[i]) ) + for ( unsigned i = 0; i < APP_PROTOID_MAX; i++ ) + if (match_id_against_rule(odp_ctxt, app_ids[i])) + return MATCH; + } + else + { + if (match_id_against_rule(odp_ctxt, service_id)) return MATCH; + for (uint32_t i = 0; i < session->get_hsessions_size(); i++) + { + AppIdHttpSession* hsession = session->get_http_session(i); + if (!hsession) + return NO_MATCH; + if (match_id_against_rule(odp_ctxt, hsession->client.get_id())) + return MATCH; + if (match_id_against_rule(odp_ctxt, hsession->payload.get_id())) + return MATCH; + if (match_id_against_rule(odp_ctxt, hsession->misc_app_id)) + return MATCH; + } + } + return NO_MATCH; } diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc index 9ea39e67d..d6c4eec87 100644 --- a/src/network_inspectors/appid/lua_detector_api.cc +++ b/src/network_inspectors/appid/lua_detector_api.cc @@ -923,16 +923,10 @@ static int client_add_application(lua_State* L) return 1; } -static int client_add_info(lua_State* L) +static int client_add_info(lua_State*) { - auto& ud = *UserData::check(L, DETECTOR, 1); - // Verify detector user data and that we are in packet context - LuaStateDescriptor* lsd = ud->validate_lua_state(true); - - const char* info = lua_tostring(L, 2); - ud->cd->add_info(*lsd->ldp.asd, info, *lsd->ldp.change_bits); - lua_pushnumber(L, 0); - return 1; + // Deprecated API. But because we are in packet context, silently ignore + return -1; } static int client_add_user(lua_State* L) diff --git a/src/network_inspectors/appid/service_plugins/service_discovery.cc b/src/network_inspectors/appid/service_plugins/service_discovery.cc index 5471396c2..615557d83 100644 --- a/src/network_inspectors/appid/service_plugins/service_discovery.cc +++ b/src/network_inspectors/appid/service_plugins/service_discovery.cc @@ -649,7 +649,7 @@ bool ServiceDiscovery::do_service_discovery(AppIdSession& asd, Packet* p, asd.tpsession->set_state(TP_STATE_TERMINATED); // - Just ignore everything from now on. - asd.set_session_flags(APPID_SESSION_IGNORE_FLOW); + asd.set_session_flags(APPID_SESSION_FUTURE_FLOW); } } diff --git a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h index c0c7b8148..391a053c3 100644 --- a/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h +++ b/src/network_inspectors/appid/service_plugins/test/service_plugin_mock.h @@ -88,7 +88,6 @@ FpSMBData* smb_data = nullptr; int AppIdDetector::initialize(){return 0;} int AppIdDetector::data_add(AppIdSession&, void*, AppIdFreeFCN){return 0;} void* AppIdDetector::data_get(AppIdSession&) {return nullptr;} -void AppIdDetector::add_info(AppIdSession&, const char*, AppidChangeBits&){} void AppIdDetector::add_user(AppIdSession&, const char*, AppId, bool){} void AppIdDetector::add_payload(AppIdSession&, AppId){} void AppIdDetector::add_app(const snort::Packet&, AppIdSession&, AppidSessionDirection, AppId, AppId, const char*, AppidChangeBits&){} diff --git a/src/network_inspectors/appid/test/appid_api_test.cc b/src/network_inspectors/appid/test/appid_api_test.cc index c7621d79c..9cc07e1c5 100644 --- a/src/network_inspectors/appid/test/appid_api_test.cc +++ b/src/network_inspectors/appid/test/appid_api_test.cc @@ -62,9 +62,9 @@ void DataBus::publish(const char*, DataEvent& event, Flow*) mock().actualCall("publish"); } -void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow) +void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow, bool, uint32_t) { - AppidEvent app_event(change_bits); + AppidEvent app_event(change_bits, false, 0); DataBus::publish(APPID_EVENT_ANY_CHANGE, app_event, flow); } diff --git a/src/network_inspectors/appid/test/appid_discovery_test.cc b/src/network_inspectors/appid/test/appid_discovery_test.cc index 5ec1fd726..d3c4fd671 100644 --- a/src/network_inspectors/appid/test/appid_discovery_test.cc +++ b/src/network_inspectors/appid/test/appid_discovery_test.cc @@ -200,9 +200,9 @@ AppIdSession* AppIdSession::allocate_session(const Packet*, IpProtocol, return nullptr; } -void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow) +void AppIdSession::publish_appid_event(AppidChangeBits& change_bits, Flow* flow, bool, uint32_t) { - AppidEvent app_event(change_bits); + AppidEvent app_event(change_bits, false, 0); DataBus::publish(APPID_EVENT_ANY_CHANGE, app_event, flow); } @@ -315,6 +315,7 @@ TEST_GROUP(appid_discovery_tests) mock().clear(); } }; + TEST(appid_discovery_tests, event_published_when_ignoring_flow) { // Testing event from do_pre_discovery() path @@ -334,7 +335,7 @@ TEST(appid_discovery_tests, event_published_when_ignoring_flow) p.flow = flow; asd->common.initiator_port = 21; asd->common.initiator_ip.set("1.2.3.4"); - asd->set_session_flags(APPID_SESSION_IGNORE_FLOW); + asd->set_session_flags(APPID_SESSION_FUTURE_FLOW); AppIdDiscovery::do_application_discovery(&p, ins, nullptr); @@ -461,7 +462,7 @@ TEST(appid_discovery_tests, change_bits_to_string) change_bits.set(); change_bits_to_string(change_bits, str); STRCMP_EQUAL(str.c_str(), "service, client, payload, misc, referred, host," - " tls-host, url, user-agent, response, referrer, client-version"); + " tls-host, url, user-agent, response, referrer, version"); // Failure of this test is a reminder that enum is changed, hence translator needs update CHECK_EQUAL(APPID_MAX_BIT, 12); diff --git a/src/network_inspectors/appid/test/appid_http_event_test.cc b/src/network_inspectors/appid/test/appid_http_event_test.cc index 78631dd84..3b2dd62ab 100644 --- a/src/network_inspectors/appid/test/appid_http_event_test.cc +++ b/src/network_inspectors/appid/test/appid_http_event_test.cc @@ -70,6 +70,23 @@ class FakeHttpMsgHeader FakeHttpMsgHeader* fake_msg_header = nullptr; void AppIdHttpSession::set_http_change_bits(AppidChangeBits&, HttpFieldIds) {} +void AppIdHttpSession::set_field(HttpFieldIds id, const std::string* str, + AppidChangeBits&) +{ + delete meta_data[id]; + meta_data[id] = str; +} + +void AppIdHttpSession::set_field(HttpFieldIds id, const uint8_t* str, int32_t len, + AppidChangeBits&) +{ + delete meta_data[id]; + if (str and len) + meta_data[id] = new std::string((const char*)str, len); + else + meta_data[id] = nullptr; +} +void AppIdHttpSession::print_field(HttpFieldIds, const std::string*) {} const uint8_t* HttpEvent::get_content_type(int32_t& length) { @@ -190,7 +207,7 @@ AppIdSession* AppIdApi::get_appid_session(const Flow&) return mock_session; } -void AppIdSession::publish_appid_event(AppidChangeBits&, Flow*) { } +void AppIdSession::publish_appid_event(AppidChangeBits&, Flow*, bool, uint32_t) { } TEST_GROUP(appid_http_event) { @@ -382,7 +399,7 @@ TEST(appid_http_event, handle_msg_header_content_type) { TestData test_data; test_data.type = HttpEventHandler::RESPONSE_EVENT; - test_data.scan_flags = 0; + test_data.scan_flags = 0x100; test_data.http_flows = 0; // Flows are only counted on request header test_data.content_type = CONTENT_TYPE; @@ -404,7 +421,7 @@ TEST(appid_http_event, handle_msg_header_server) { TestData test_data; test_data.type = HttpEventHandler::RESPONSE_EVENT; - test_data.scan_flags = 0; + test_data.scan_flags = 0x40; test_data.http_flows = 0; // Flows are only counted on request header test_data.server = SERVER; @@ -440,7 +457,7 @@ TEST(appid_http_event, handle_msg_header_all_response_headers) { TestData test_data; test_data.type = HttpEventHandler::RESPONSE_EVENT; - test_data.scan_flags = 1; + test_data.scan_flags = 0x101; test_data.http_flows = 0; // Flows are only counted on request header test_data.response_code = RESPONSE_CODE; test_data.content_type = CONTENT_TYPE; diff --git a/src/network_inspectors/appid/test/appid_http_session_test.cc b/src/network_inspectors/appid/test/appid_http_session_test.cc index cf1270cba..dd4bd407b 100644 --- a/src/network_inspectors/appid/test/appid_http_session_test.cc +++ b/src/network_inspectors/appid/test/appid_http_session_test.cc @@ -132,18 +132,6 @@ bool AppIdSession::is_payload_appid_set() return true; } -void AppIdSession::set_referred_payload_app_id_data(AppId id, AppidChangeBits& change_bits) -{ - if (id <= APP_ID_NONE) - return; - - if (referred_payload_app_id != id) - { - referred_payload_app_id = id; - change_bits.set(APPID_REFERRED_BIT); - } -} - // AppIdDebug mock functions void AppIdDebug::activate(const uint32_t*, const uint32_t*, uint16_t, uint16_t, IpProtocol, const int, uint16_t, const AppIdSession*, bool) @@ -172,7 +160,7 @@ THREAD_LOCAL AppIdDebug* appidDebug = nullptr; const SfIp* sfip = nullptr; AppIdSession session(IpProtocol::IP, sfip, 0, appid_inspector); -AppIdHttpSession hsession(session); +AppIdHttpSession mock_hsession(session, 0); TEST_GROUP(appid_http_session) { @@ -196,49 +184,49 @@ TEST(appid_http_session, http_field_ids_enum_order) // in appid_http_session.h. AppidChangeBits change_bits; - hsession.set_field( (HttpFieldIds)0, new std::string("agent"), change_bits ); - hsession.set_field( (HttpFieldIds)1, new std::string("host"), change_bits ); - hsession.set_field( (HttpFieldIds)2, new std::string("referer"), change_bits ); - hsession.set_field( (HttpFieldIds)3, new std::string("uri"), change_bits ); - hsession.set_field( (HttpFieldIds)4, new std::string("cookie"), change_bits ); - hsession.set_field( (HttpFieldIds)5, new std::string("req_body"), change_bits ); - hsession.set_field( (HttpFieldIds)6, new std::string("content_type"), change_bits ); - hsession.set_field( (HttpFieldIds)7, new std::string("location"), change_bits ); - hsession.set_field( (HttpFieldIds)8, new std::string("rsp_body"), change_bits ); - hsession.set_field( (HttpFieldIds)9, new std::string("via"), change_bits ); - hsession.set_field( (HttpFieldIds)10, new std::string("response_code"), change_bits ); - hsession.set_field( (HttpFieldIds)11, new std::string("server"), change_bits ); - hsession.set_field( (HttpFieldIds)12, new std::string("xww"), change_bits ); - hsession.set_field( (HttpFieldIds)13, new std::string("url"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)0, new std::string("agent"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)1, new std::string("host"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)2, new std::string("referer"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)3, new std::string("uri"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)4, new std::string("cookie"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)5, new std::string("req_body"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)6, new std::string("content_type"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)7, new std::string("location"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)8, new std::string("rsp_body"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)9, new std::string("via"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)10, new std::string("response_code"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)11, new std::string("server"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)12, new std::string("xww"), change_bits ); + mock_hsession.set_field( (HttpFieldIds)13, new std::string("url"), change_bits ); const std::string* field; - field = hsession.get_field(REQ_AGENT_FID); + field = mock_hsession.get_field(REQ_AGENT_FID); STRCMP_EQUAL(field->c_str(), "agent"); - field = hsession.get_field(REQ_HOST_FID); + field = mock_hsession.get_field(REQ_HOST_FID); STRCMP_EQUAL(field->c_str(), "host"); - field = hsession.get_field(REQ_REFERER_FID); + field = mock_hsession.get_field(REQ_REFERER_FID); STRCMP_EQUAL(field->c_str(), "referer"); - field = hsession.get_field(REQ_URI_FID); + field = mock_hsession.get_field(REQ_URI_FID); STRCMP_EQUAL(field->c_str(), "uri"); - field = hsession.get_field(REQ_COOKIE_FID); + field = mock_hsession.get_field(REQ_COOKIE_FID); STRCMP_EQUAL(field->c_str(), "cookie"); - field = hsession.get_field(REQ_BODY_FID); + field = mock_hsession.get_field(REQ_BODY_FID); STRCMP_EQUAL(field->c_str(), "req_body"); - field = hsession.get_field(RSP_CONTENT_TYPE_FID); + field = mock_hsession.get_field(RSP_CONTENT_TYPE_FID); STRCMP_EQUAL(field->c_str(), "content_type"); - field = hsession.get_field(RSP_LOCATION_FID); + field = mock_hsession.get_field(RSP_LOCATION_FID); STRCMP_EQUAL(field->c_str(), "location"); - field = hsession.get_field(RSP_BODY_FID); + field = mock_hsession.get_field(RSP_BODY_FID); STRCMP_EQUAL(field->c_str(), "rsp_body"); - field = hsession.get_field(MISC_VIA_FID); + field = mock_hsession.get_field(MISC_VIA_FID); STRCMP_EQUAL(field->c_str(), "via"); - field = hsession.get_field(MISC_RESP_CODE_FID); + field = mock_hsession.get_field(MISC_RESP_CODE_FID); STRCMP_EQUAL(field->c_str(), "response_code"); - field = hsession.get_field(MISC_SERVER_FID); + field = mock_hsession.get_field(MISC_SERVER_FID); STRCMP_EQUAL(field->c_str(), "server"); - field = hsession.get_field(MISC_XWW_FID); + field = mock_hsession.get_field(MISC_XWW_FID); STRCMP_EQUAL(field->c_str(), "xww"); - field = hsession.get_field(MISC_URL_FID); + field = mock_hsession.get_field(MISC_URL_FID); STRCMP_EQUAL(field->c_str(), "url"); // Detect changes in host, url, user agent, response, and referer fields @@ -256,9 +244,9 @@ TEST(appid_http_session, set_tun_dest) SfIp ipv6; ipv6.set("2001:db8:85a3::8a2e:370:7334"); AppidChangeBits change_bits; - hsession.set_field(REQ_URI_FID, new std::string("[2001:db8:85a3::8a2e:370:7334]:51413"), change_bits); - hsession.set_tun_dest(); - tun_dest = hsession.get_tun_dest(); + mock_hsession.set_field(REQ_URI_FID, new std::string("[2001:db8:85a3::8a2e:370:7334]:51413"), change_bits); + mock_hsession.set_tun_dest(); + tun_dest = mock_hsession.get_tun_dest(); CHECK(tun_dest != nullptr); CHECK_EQUAL(tun_dest->port, 51413); CHECK_EQUAL((ipv6 == tun_dest->ip), true); @@ -268,16 +256,16 @@ TEST(appid_http_session, set_tun_dest_bad_uri) { const TunnelDest* tun_dest = nullptr; AppidChangeBits change_bits; - hsession.set_field(REQ_URI_FID, new std::string("[2001:db8:85a3::8a2e:370:1234]:51413"), change_bits); - hsession.set_tun_dest(); - tun_dest = hsession.get_tun_dest(); + mock_hsession.set_field(REQ_URI_FID, new std::string("[2001:db8:85a3::8a2e:370:1234]:51413"), change_bits); + mock_hsession.set_tun_dest(); + tun_dest = mock_hsession.get_tun_dest(); CHECK(tun_dest != nullptr); // Testing with bad URL - hsession.free_tun_dest(); - hsession.set_field(REQ_URI_FID, new std::string("[2001:db8:85a3::8a2e:370:1235]"), change_bits); - hsession.set_tun_dest(); - tun_dest = hsession.get_tun_dest(); + mock_hsession.free_tun_dest(); + mock_hsession.set_field(REQ_URI_FID, new std::string("[2001:db8:85a3::8a2e:370:1235]"), change_bits); + mock_hsession.set_tun_dest(); + tun_dest = mock_hsession.get_tun_dest(); CHECK(tun_dest == nullptr); } @@ -290,9 +278,9 @@ TEST(appid_http_session, change_bits_for_referred_appid) OdpContext odp_ctxt(config, nullptr); session.service.set_id(APP_ID_HTTP, odp_ctxt); session.scan_flags |= SCAN_HTTP_HOST_URL_FLAG; - hsession.set_skip_simple_detect(false); - hsession.set_field( (HttpFieldIds)2, new std::string("referer"), change_bits ); - hsession.process_http_packet(APP_ID_FROM_INITIATOR, change_bits, odp_ctxt.get_http_matchers()); + mock_hsession.set_skip_simple_detect(false); + mock_hsession.set_field( (HttpFieldIds)2, new std::string("referer"), change_bits ); + mock_hsession.process_http_packet(APP_ID_FROM_INITIATOR, change_bits, odp_ctxt.get_http_matchers()); // Detect changes in referred appid CHECK_EQUAL(change_bits.test(APPID_REFERRED_BIT), true); diff --git a/src/network_inspectors/appid/test/appid_mock_http_session.h b/src/network_inspectors/appid/test/appid_mock_http_session.h index 21809ce71..151020b16 100644 --- a/src/network_inspectors/appid/test/appid_mock_http_session.h +++ b/src/network_inspectors/appid/test/appid_mock_http_session.h @@ -25,8 +25,8 @@ typedef AppIdHttpSession::pair_t pair_t; -AppIdHttpSession::AppIdHttpSession(AppIdSession& session) - : asd(session) +AppIdHttpSession::AppIdHttpSession(AppIdSession& session, uint32_t http2_stream_id) + : asd(session), http2_stream_id(http2_stream_id) { for ( int i = 0; i < NUM_METADATA_FIELDS; i++) meta_data[i] = nullptr; @@ -78,7 +78,7 @@ class MockAppIdHttpSession : public AppIdHttpSession { public: MockAppIdHttpSession(AppIdSession& asd) - : AppIdHttpSession(asd) + : AppIdHttpSession(asd, 0) { meta_data[REQ_AGENT_FID] = new std::string(USERAGENT); meta_data[REQ_HOST_FID] = new std::string(HOST); diff --git a/src/network_inspectors/appid/test/appid_mock_session.h b/src/network_inspectors/appid/test/appid_mock_session.h index dcb79d297..ec222b087 100644 --- a/src/network_inspectors/appid/test/appid_mock_session.h +++ b/src/network_inspectors/appid/test/appid_mock_session.h @@ -175,7 +175,7 @@ void* AppIdSession::remove_flow_data(unsigned type) return data; } -void AppIdSession::set_application_ids(AppId service_id, AppId client_id, +void AppIdSession::set_ss_application_ids(AppId service_id, AppId client_id, AppId payload_id, AppId misc_id, AppidChangeBits& change_bits) { if (application_ids[APP_PROTOID_SERVICE] != service_id) @@ -205,51 +205,68 @@ AppId AppIdSession::pick_service_app_id() return service.get_id(); } -AppId AppIdSession::pick_misc_app_id() +AppId AppIdSession::pick_ss_misc_app_id() { return misc_app_id; } -AppId AppIdSession::pick_client_app_id() +AppId AppIdSession::pick_ss_client_app_id() { return client.get_id(); } -AppId AppIdSession::pick_payload_app_id() +AppId AppIdSession::pick_ss_payload_app_id() { return payload.get_id(); } -AppId AppIdSession::pick_referred_payload_app_id() +AppId AppIdSession::pick_ss_referred_payload_app_id() { return APPID_UT_ID; } -void AppIdSession::get_application_ids(AppId&, AppId&, AppId&, AppId&) { } +void AppIdSession::get_first_stream_app_ids(AppId&, AppId&, AppId&, AppId&) { } -void AppIdSession::get_application_ids(AppId&, AppId&, AppId&) { } +void AppIdSession::get_first_stream_app_ids(AppId&, AppId&, AppId&) { } AppId AppIdSession::get_application_ids_service() { return APPID_UT_ID; } -AppId AppIdSession::get_application_ids_client() { return APPID_UT_ID; } +AppId AppIdSession::get_application_ids_client(uint32_t stream_index) +{ + if (stream_index < hsessions.size() or stream_index == 0) + return APPID_UT_ID; + + return APP_ID_NONE; +} -AppId AppIdSession::get_application_ids_payload() { return APPID_UT_ID; } +AppId AppIdSession::get_application_ids_payload(uint32_t stream_index) +{ + if (stream_index < hsessions.size() or stream_index == 0) + return APPID_UT_ID; -AppId AppIdSession::get_application_ids_misc() { return APPID_UT_ID; } + return APP_ID_NONE; +} -AppId AppIdSession::pick_only_service_app_id() +AppId AppIdSession::get_application_ids_misc(uint32_t stream_index) { - return APPID_UT_ID; + if (stream_index < hsessions.size() or stream_index == 0) + return APPID_UT_ID; + + return APP_ID_NONE; } bool AppIdSession::is_ssl_session_decrypted() { return is_session_decrypted; } -AppIdHttpSession* AppIdSession::create_http_session() + +AppIdHttpSession* AppIdSession::create_http_session(uint32_t) { AppIdHttpSession* hsession = new MockAppIdHttpSession(*this); + AppidChangeBits change_bits; + hsession->client.set_id(APPID_UT_ID); + hsession->client.set_version(APPID_UT_CLIENT_VERSION, change_bits); hsession->payload.set_id(APPID_UT_ID); hsession->misc_app_id = APPID_UT_ID; hsession->referred_payload_app_id = APPID_UT_ID; @@ -283,5 +300,7 @@ bool AppIdSession::is_tp_appid_available() const return true; } +void AppIdSession::set_application_ids_service(int, AppidChangeBits&) { } + #endif diff --git a/src/network_inspectors/appid/test/appid_session_api_test.cc b/src/network_inspectors/appid/test/appid_session_api_test.cc index abddc1396..d33119946 100644 --- a/src/network_inspectors/appid/test/appid_session_api_test.cc +++ b/src/network_inspectors/appid/test/appid_session_api_test.cc @@ -57,12 +57,14 @@ TEST_GROUP(appid_session_api) void setup() override { MemoryLeakWarningPlugin::turnOffNewDeleteOverloads(); + mock_session = new AppIdSession(IpProtocol::TCP, nullptr, 1492, appid_inspector); appid_session_api = new AppIdSessionApi(mock_session); } void teardown() override { delete appid_session_api; + delete mock_session; MemoryLeakWarningPlugin::turnOnNewDeleteOverloads(); } }; @@ -79,58 +81,44 @@ TEST(appid_session_api, get_port_service_app_id) CHECK_EQUAL(id, APPID_UT_ID + 3); } -TEST(appid_session_api, get_only_service_app_id) -{ - AppId id = appid_session_api->get_only_service_app_id(); - CHECK_EQUAL(id, APPID_UT_ID); -} - TEST(appid_session_api, get_misc_app_id) { - mock_session->is_http2 = false; AppId id = appid_session_api->get_misc_app_id(); CHECK_EQUAL(id, APPID_UT_ID); - mock_session->is_http2 = true; - id = appid_session_api->get_client_app_id(0); + id = appid_session_api->get_misc_app_id(0); CHECK_EQUAL(APPID_UT_ID, id); - id = appid_session_api->get_client_app_id(3); - CHECK_EQUAL(APP_ID_UNKNOWN, id); + id = appid_session_api->get_misc_app_id(3); + CHECK_EQUAL(APP_ID_NONE, id); } TEST(appid_session_api, get_client_app_id) { - mock_session->is_http2 = false; AppId id = appid_session_api->get_client_app_id(); CHECK_EQUAL(id, APPID_UT_ID); - mock_session->is_http2 = true; id = appid_session_api->get_client_app_id(0); CHECK_EQUAL(APPID_UT_ID, id); id = appid_session_api->get_client_app_id(3); - CHECK_EQUAL(APP_ID_UNKNOWN, id); + CHECK_EQUAL(APP_ID_NONE, id); } TEST(appid_session_api, get_payload_app_id) { - mock_session->is_http2 = false; AppId id = appid_session_api->get_payload_app_id(); CHECK_EQUAL(id, APPID_UT_ID); - mock_session->is_http2 = true; id = appid_session_api->get_payload_app_id(0); CHECK_EQUAL(APPID_UT_ID, id); id = appid_session_api->get_payload_app_id(2); - CHECK_EQUAL(APP_ID_UNKNOWN, id); + CHECK_EQUAL(APP_ID_NONE, id); } TEST(appid_session_api, get_referred_app_id) { - mock_session->is_http2 = false; AppId id = appid_session_api->get_referred_app_id(); CHECK_EQUAL(id, APPID_UT_ID); - mock_session->is_http2 = true; id = appid_session_api->get_payload_app_id(0); CHECK_EQUAL(APPID_UT_ID, id); id = appid_session_api->get_payload_app_id(2); - CHECK_EQUAL(APP_ID_UNKNOWN, id); + CHECK_EQUAL(APP_ID_NONE, id); } TEST(appid_session_api, get_service_port) @@ -267,15 +255,16 @@ TEST(appid_session_api, get_client_version) const char* val; val = appid_session_api->get_client_version(); STRCMP_EQUAL(val, APPID_UT_CLIENT_VERSION); + mock_session->create_http_session(); val = appid_session_api->get_client_version(0); STRCMP_EQUAL(APPID_UT_CLIENT_VERSION, val); - mock_session->is_http2 = true; val = appid_session_api->get_client_version(2); STRCMP_EQUAL(nullptr, val); } TEST(appid_session_api, get_http_session) { AppIdHttpSession* val; + mock_session->create_http_session(); val = appid_session_api->get_http_session(); CHECK_TRUE(val != nullptr); val = appid_session_api->get_http_session(2); @@ -395,7 +384,7 @@ TEST(appid_session_api, is_http_inspection_done) CHECK_FALSE(val); mock_session->service_disco_state = APPID_DISCO_STATE_STATEFUL; mock_session->set_session_flags(APPID_SESSION_SSL_SESSION); - mock_session->tsession->set_tls_host("www.cisco.com", 13, change_bits); + mock_session->tsession->set_tls_host(APPID_UT_TLS_HOST, 0, change_bits); val = appid_session_api->is_http_inspection_done(); CHECK_TRUE(val); mock_session->service_disco_state = APPID_DISCO_STATE_FINISHED; @@ -407,8 +396,6 @@ TEST(appid_session_api, is_http_inspection_done) int main(int argc, char** argv) { mock_init_appid_pegs(); - mock_session = new AppIdSession(IpProtocol::TCP, nullptr, 1492, appid_inspector); - mock_session->create_http_session(); int rc = CommandLineTestRunner::RunAllTests(argc, argv); mock_cleanup_appid_pegs(); return rc; diff --git a/src/network_inspectors/appid/test/tp_appid_types_test.cc b/src/network_inspectors/appid/test/tp_appid_types_test.cc index e16705cce..33fb33485 100644 --- a/src/network_inspectors/appid/test/tp_appid_types_test.cc +++ b/src/network_inspectors/appid/test/tp_appid_types_test.cc @@ -86,9 +86,7 @@ TEST(tp_appid_types, get_set) SET_GET_OFFSET_MACRO(http_request_cookie); SET_GET_MACRO(http_request_via); SET_GET_MACRO(http_response_via); - SET_GET_MACRO(http_response_upgrade); SET_GET_OFFSET_MACRO(http_request_user_agent); - SET_GET_MACRO(http_response_version); SET_GET_MACRO(http_response_code); SET_GET_MACRO(http_response_content); SET_GET_MACRO(http_response_location); diff --git a/src/network_inspectors/appid/tp_appid_module_api.h b/src/network_inspectors/appid/tp_appid_module_api.h index f2a1d382f..2dee62d28 100644 --- a/src/network_inspectors/appid/tp_appid_module_api.h +++ b/src/network_inspectors/appid/tp_appid_module_api.h @@ -34,8 +34,6 @@ public: bool ftp_userid_disabled = false; bool chp_body_collection_disabled = false; bool tp_allow_probes = false; - bool http_upgrade_reporting_enabled = false; - bool http_response_version_enabled = false; std::string tp_appid_config; bool tp_appid_stats_enable = false; bool tp_appid_config_dump = false; diff --git a/src/network_inspectors/appid/tp_appid_types.h b/src/network_inspectors/appid/tp_appid_types.h index 78b55a0e5..1b0727c04 100644 --- a/src/network_inspectors/appid/tp_appid_types.h +++ b/src/network_inspectors/appid/tp_appid_types.h @@ -110,9 +110,7 @@ class ThirdPartyAppIDAttributeData string* http_request_cookie_buf = nullptr; string* http_request_via_buf = nullptr; string* http_response_via_buf = nullptr; - string* http_response_upgrade_buf = nullptr; string* http_request_user_agent_buf = nullptr; - string* http_response_version_buf = nullptr; string* http_response_code_buf = nullptr; string* http_response_content_buf = nullptr; string* http_response_location_buf = nullptr; @@ -165,9 +163,7 @@ public: if (http_request_cookie_buf) delete http_request_cookie_buf; if (http_request_via_buf) delete http_request_via_buf; if (http_response_via_buf) delete http_response_via_buf; - if (http_response_upgrade_buf) delete http_response_upgrade_buf; if (http_request_user_agent_buf) delete http_request_user_agent_buf; - if (http_response_version_buf) delete http_response_version_buf; if (http_response_code_buf) delete http_response_code_buf; if (http_response_content_buf) delete http_response_content_buf; if (http_response_location_buf) delete http_response_location_buf; @@ -193,9 +189,7 @@ public: TPAD_GET(http_request_cookie) TPAD_GET(http_request_via) TPAD_GET(http_response_via) - TPAD_GET(http_response_upgrade) TPAD_GET(http_request_user_agent) - TPAD_GET(http_response_version) TPAD_GET(http_response_code) TPAD_GET(http_response_content) TPAD_GET(http_response_location) @@ -241,9 +235,7 @@ public: TPAD_SET_OFFSET(http_request_cookie) TPAD_SET(http_request_via) TPAD_SET(http_response_via) - TPAD_SET(http_response_upgrade) TPAD_SET_OFFSET(http_request_user_agent) - TPAD_SET(http_response_version) TPAD_SET(http_response_code) TPAD_SET(http_response_content) TPAD_SET(http_response_location) diff --git a/src/network_inspectors/appid/tp_appid_utils.cc b/src/network_inspectors/appid/tp_appid_utils.cc index 7248e9720..20835b26d 100644 --- a/src/network_inspectors/appid/tp_appid_utils.cc +++ b/src/network_inspectors/appid/tp_appid_utils.cc @@ -129,12 +129,6 @@ static inline void process_http_session(AppIdSession& asd, hsession->set_offset(REQ_HOST_FID, attribute_data.spdy_request_host_begin(), attribute_data.spdy_request_host_end()); - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s SPDY host (%u-%u) is %s\n", - appidDebug->get_debug_session(), - attribute_data.spdy_request_host_begin(), - attribute_data.spdy_request_host_end(), - hsession->get_field(REQ_HOST_FID)->c_str()); asd.scan_flags |= SCAN_HTTP_HOST_URL_FLAG; } @@ -147,11 +141,6 @@ static inline void process_http_session(AppIdSession& asd, hsession->set_offset(REQ_URI_FID, attribute_data.spdy_request_path_begin(), attribute_data.spdy_request_path_end()); - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s SPDY URI (%u-%u) is %s\n", appidDebug->get_debug_session(), - attribute_data.spdy_request_path_begin(), - attribute_data.spdy_request_path_end(), - hsession->get_field(REQ_URI_FID)->c_str()); } } else @@ -166,11 +155,6 @@ static inline void process_http_session(AppIdSession& asd, hsession->set_offset(REQ_HOST_FID, attribute_data.http_request_host_begin(), attribute_data.http_request_host_end()); - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s HTTP host (%u-%u) is %s\n", - appidDebug->get_debug_session(), - attribute_data.http_request_host_begin(), - attribute_data.http_request_host_end(), field->c_str() ); asd.scan_flags |= SCAN_HTTP_HOST_URL_FLAG; } @@ -207,11 +191,6 @@ static inline void process_http_session(AppIdSession& asd, attribute_data.http_request_uri_begin(), attribute_data.http_request_uri_end()); asd.scan_flags |= SCAN_HTTP_URI_FLAG; - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s URI (%u-%u) is %s\n", appidDebug->get_debug_session(), - attribute_data.http_request_uri_begin(), - attribute_data.http_request_uri_end(), - hsession->get_cfield(REQ_URI_FID)); } } @@ -245,35 +224,11 @@ static inline void process_http_session(AppIdSession& asd, hsession->set_offset(REQ_AGENT_FID, attribute_data.http_request_user_agent_begin(), attribute_data.http_request_user_agent_end()); - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s User Agent (%u-%u) is %s\n", - appidDebug->get_debug_session(), - attribute_data.http_request_user_agent_begin(), - attribute_data.http_request_user_agent_end(), - field->c_str()); asd.scan_flags |= SCAN_HTTP_USER_AGENT_FLAG; } - // Check to see if third party discovered HTTP/2. - once it supports it... - if ( (field=attribute_data.http_response_version(false)) != nullptr ) - { - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s HTTP response version is %s\n", - appidDebug->get_debug_session(), field->c_str()); - if (strncmp(field->c_str(), "HTTP/2", 6) == 0) - { - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s 3rd party detected and parsed HTTP/2\n", - appidDebug->get_debug_session()); - asd.is_http2 = true; - } - } - if ( (field=attribute_data.http_response_code(own)) != nullptr ) { - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s HTTP response code is %s\n", - appidDebug->get_debug_session(), field->c_str()); if ( hsession->get_field(MISC_RESP_CODE_FID) ) if (!asd.get_session_flags(APPID_SESSION_APP_REINSPECT)) hsession->set_chp_finished(false); @@ -281,29 +236,6 @@ static inline void process_http_session(AppIdSession& asd, hsession->set_field(MISC_RESP_CODE_FID, field, change_bits); } - // Check to see if we've got an upgrade to HTTP/2 (if enabled). - // - This covers the "without prior knowledge" case (i.e., the client - // asks the server to upgrade to HTTP/2). - if ( (field=attribute_data.http_response_upgrade(false) ) != nullptr ) - { - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s HTTP response upgrade is %s\n", - appidDebug->get_debug_session(),field->c_str()); - - if (asd.ctxt.get_odp_ctxt().http2_detection_enabled) - { - const std::string* rc = hsession->get_field(MISC_RESP_CODE_FID); - if ( rc && *rc == "101" ) - if (strncmp(field->c_str(), "h2c", 3) == 0) - { - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s Got an upgrade to HTTP/2\n", - appidDebug->get_debug_session()); - asd.is_http2 = true; - } - } - } - if ( (field=attribute_data.http_request_referer(own)) != nullptr ) { if ( hsession->get_field(REQ_REFERER_FID) ) @@ -314,11 +246,6 @@ static inline void process_http_session(AppIdSession& asd, hsession->set_offset(REQ_REFERER_FID, attribute_data.http_request_referer_begin(), attribute_data.http_request_referer_end()); - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s Referer (%u-%u) is %s\n", appidDebug->get_debug_session(), - attribute_data.http_request_referer_begin(), - attribute_data.http_request_referer_end(), - hsession->get_cfield(REQ_REFERER_FID) ); } if ( (field=attribute_data.http_request_cookie(own)) != nullptr ) @@ -331,11 +258,6 @@ static inline void process_http_session(AppIdSession& asd, hsession->set_offset(REQ_COOKIE_FID, attribute_data.http_request_cookie_begin(), attribute_data.http_request_cookie_end()); - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s Cookie (%u-%u) is %s\n", appidDebug->get_debug_session(), - attribute_data.http_request_cookie_begin(), - attribute_data.http_request_cookie_end(), - hsession->get_cfield(REQ_COOKIE_FID)); } if ( (field=attribute_data.http_response_content(own)) != nullptr ) @@ -359,9 +281,6 @@ static inline void process_http_session(AppIdSession& asd, if ( (field=attribute_data.http_request_body(own)) != nullptr ) { - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s Got a request body %s\n", - appidDebug->get_debug_session(), field->c_str()); if ( hsession->get_field(REQ_BODY_FID) ) if (!asd.get_session_flags(APPID_SESSION_APP_REINSPECT)) hsession->set_chp_finished(false); @@ -405,7 +324,7 @@ static inline void process_rtmp(AppIdSession& asd, AppId service_id = 0; AppId client_id = 0; AppId payload_id = 0; - AppId referred_payload_app_id = 0; + AppId referred_payload_app_id = APP_ID_NONE; bool own = true; uint16_t size = 0; @@ -452,12 +371,12 @@ static inline void process_rtmp(AppIdSession& asd, HttpPatternMatchers& http_matchers = asd.ctxt.get_odp_ctxt().get_http_matchers(); http_matchers.identify_user_agent(field->c_str(), size, service_id, - client_id, &version); + client_id, &version); - asd.set_client_appid_data(client_id, change_bits, version); + hsession->set_client(client_id, change_bits, "User Agent", version); // do not overwrite a previously-set service - if ( service_id <= APP_ID_NONE ) + if ( asd.service.get_id() <= APP_ID_NONE ) asd.set_service_appid_data(service_id, change_bits); asd.scan_flags |= ~SCAN_HTTP_USER_AGENT_FLAG; @@ -481,14 +400,14 @@ static inline void process_rtmp(AppIdSession& asd, &payload_id, &referred_payload_app_id, false, asd.ctxt.get_odp_ctxt()) ) ) == 1 ) { // do not overwrite a previously-set client or service - if ( client_id <= APP_ID_NONE ) - asd.set_client_appid_data(client_id, change_bits); - if ( service_id <= APP_ID_NONE ) + if ( hsession->client.get_id() <= APP_ID_NONE ) + hsession->set_client(client_id, change_bits, "URL"); + if ( asd.service.get_id() <= APP_ID_NONE ) asd.set_service_appid_data(service_id, change_bits); - // DO overwrite a previously-set data - asd.set_payload_appid_data(payload_id, change_bits); - asd.set_referred_payload_app_id_data(referred_payload_app_id, change_bits); + // DO overwrite a previously-set payload + hsession->set_payload(payload_id, change_bits, "URL"); + hsession->set_referred_payload(referred_payload_app_id, change_bits); } } @@ -640,15 +559,46 @@ static inline void check_terminate_tp_module(AppIdSession& asd, uint16_t tpPktCo if (asd.get_tp_app_id() == APP_ID_NONE) asd.set_tp_app_id(APP_ID_UNKNOWN); - if ( asd.service_disco_state == APPID_DISCO_STATE_FINISHED && asd.payload.get_id() == - APP_ID_NONE ) + if ( !hsession and asd.service_disco_state == APPID_DISCO_STATE_FINISHED and + asd.payload.get_id() == APP_ID_NONE ) asd.payload.set_id(APP_ID_UNKNOWN); + if ( hsession and asd.service_disco_state == APPID_DISCO_STATE_FINISHED and + hsession->payload.get_id() == APP_ID_NONE ) + hsession->payload.set_id(APP_ID_UNKNOWN); + if (asd.tpsession) asd.tpsession->reset(); } } +static void set_tp_reinspect(AppIdSession& asd, const Packet* p, AppidSessionDirection direction) +{ + // restart inspection by 3rd party + if (!asd.tp_reinspect_by_initiator and (direction == APP_ID_FROM_INITIATOR) and + check_reinspect(p, asd) and p->packet_flags & PKT_STREAM_ORDER_OK) + { + asd.tp_reinspect_by_initiator = true; + asd.set_session_flags(APPID_SESSION_APP_REINSPECT); + if (appidDebug->is_active()) + LogMessage("AppIdDbg %s 3rd party allow reinspect http\n", + appidDebug->get_debug_session()); + asd.init_tpPackets = 0; + asd.resp_tpPackets = 0; + asd.clear_http_data(); + } +} + +static void clear_tp_reinspect(AppIdSession& asd, const Packet* p, AppidSessionDirection direction) +{ + if ( asd.tp_reinspect_by_initiator and check_reinspect(p, asd) ) + { + asd.clear_session_flags(APPID_SESSION_APP_REINSPECT); + if (direction == APP_ID_FROM_RESPONDER) + asd.tp_reinspect_by_initiator = false; //toggle at OK response + } +} + bool do_tp_discovery(ThirdPartyAppIdContext& tp_appid_ctxt, AppIdSession& asd, IpProtocol protocol, Packet* p, AppidSessionDirection& direction, AppidChangeBits& change_bits) { @@ -664,247 +614,219 @@ bool do_tp_discovery(ThirdPartyAppIdContext& tp_appid_ctxt, AppIdSession& asd, I if (appidDebug->is_active()) LogMessage("AppIdDbg %s Payload is SFTP\n", appidDebug->get_debug_session()); } + + return true; } - /*** Start of third-party processing. ***/ - bool isTpAppidDiscoveryDone = false; + if (!p->dsize and !asd.ctxt.get_odp_ctxt().tp_allow_probes) + return false; + + bool process_packet = (protocol != IpProtocol::TCP or (p->packet_flags & PKT_STREAM_ORDER_OK) or + asd.ctxt.get_odp_ctxt().tp_allow_probes); - if (p->dsize || asd.ctxt.get_odp_ctxt().tp_allow_probes) + if (!process_packet) + return false; + + set_tp_reinspect(asd, p, direction); + + if (asd.is_tp_processing_done()) { - //restart inspection by 3rd party - if (!asd.tp_reinspect_by_initiator && (direction == APP_ID_FROM_INITIATOR) && - check_reinspect(p, asd) && - p->packet_flags & PKT_STREAM_ORDER_OK) + clear_tp_reinspect(asd, p, direction); + return false; + } + + if (!asd.tpsession) + { + const TPLibHandler* tph = TPLibHandler::get(); + TpAppIdCreateSession tpsf = tph->tpsession_factory(); + if ( !(asd.tpsession = tpsf(tp_appid_ctxt)) ) { - asd.tp_reinspect_by_initiator = true; - asd.set_session_flags(APPID_SESSION_APP_REINSPECT); - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s 3rd party allow reinspect http\n", - appidDebug->get_debug_session()); - asd.init_tpPackets = 0; - asd.resp_tpPackets = 0; - asd.clear_http_data(); + ErrorMessage("Could not allocate asd.tpsession data"); + return false; } + } - if (!asd.is_tp_processing_done()) - { - if (protocol != IpProtocol::TCP || (p->packet_flags & PKT_STREAM_ORDER_OK) - || asd.ctxt.get_odp_ctxt().tp_allow_probes) - { - int tp_confidence; - ThirdPartyAppIDAttributeData tp_attribute_data; - vector tp_proto_list; + int tp_confidence; + ThirdPartyAppIDAttributeData tp_attribute_data; + vector tp_proto_list; - if (!asd.tpsession) - { - const TPLibHandler* tph = TPLibHandler::get(); - TpAppIdCreateSession tpsf = tph->tpsession_factory(); - if ( !(asd.tpsession = tpsf(tp_appid_ctxt)) ) - { - ErrorMessage("Could not allocate asd.tpsession data"); - return false; - } - } + TPState current_tp_state = asd.tpsession->process(*p, direction, + tp_proto_list, tp_attribute_data); + tp_app_id = asd.tpsession->get_appid(tp_confidence); - TPState current_tp_state = asd.tpsession->process(*p, direction, - tp_proto_list, tp_attribute_data); - tp_app_id = asd.tpsession->get_appid(tp_confidence); + // First SSL decrypted packet is now being inspected. Reset the flag so that SSL + // decrypted traffic gets processed like regular traffic from next packet onwards + if (asd.get_session_flags(APPID_SESSION_APP_REINSPECT_SSL)) + asd.clear_session_flags(APPID_SESSION_APP_REINSPECT_SSL); - isTpAppidDiscoveryDone = true; + if (current_tp_state == TP_STATE_CLASSIFIED) + asd.clear_session_flags(APPID_SESSION_APP_REINSPECT); + else if (current_tp_state == TP_STATE_MONITORING) + { + asd.tpsession->disable_flags(TP_SESSION_FLAG_ATTRIBUTE | + TP_SESSION_FLAG_TUNNELING | TP_SESSION_FLAG_FUTUREFLOW); + } - // First SSL decrypted packet is now being inspected. Reset the flag so that SSL - // decrypted - // traffic gets processed like regular traffic from next packet onwards - if (asd.get_session_flags(APPID_SESSION_APP_REINSPECT_SSL)) - asd.clear_session_flags(APPID_SESSION_APP_REINSPECT_SSL); + if (appidDebug->is_active()) + { + const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(tp_app_id); + LogMessage("AppIdDbg %s 3rd party returned %s (%d)\n", + appidDebug->get_debug_session(), app_name ? app_name : "unknown", tp_app_id); + } - if (current_tp_state == TP_STATE_CLASSIFIED) - asd.clear_session_flags(APPID_SESSION_APP_REINSPECT); - else if (current_tp_state == TP_STATE_MONITORING) - { - asd.tpsession->disable_flags(TP_SESSION_FLAG_ATTRIBUTE | - TP_SESSION_FLAG_TUNNELING | TP_SESSION_FLAG_FUTUREFLOW); - } + process_third_party_results(asd, tp_confidence, tp_proto_list, tp_attribute_data, change_bits); - if (appidDebug->is_active()) - { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(tp_app_id); - LogMessage("AppIdDbg %s 3rd party returned %s (%d)\n", - appidDebug->get_debug_session(), - app_name ? app_name : "unknown", - tp_app_id); - } + AppIdHttpSession* hsession = nullptr; + if (asd.get_session_flags(APPID_SESSION_HTTP_SESSION)) + { + hsession = asd.get_http_session(); + assert(hsession); + } - // For now, third party can detect HTTP/2 (w/o metadata) for - // some cases. Treat it like HTTP w/ is_http2 flag set. - if ((tp_app_id == APP_ID_HTTP2) && (tp_confidence == 100)) - { - asd.is_http2 = true; - } - // if the third-party appId must be treated as a client, do it now - unsigned app_info_flags = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_info_flags(tp_app_id, - APPINFO_FLAG_TP_CLIENT | APPINFO_FLAG_IGNORE | APPINFO_FLAG_SSL_SQUELCH); + unsigned app_info_flags = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_info_flags(tp_app_id, + APPINFO_FLAG_TP_CLIENT | APPINFO_FLAG_IGNORE | APPINFO_FLAG_SSL_SQUELCH); - if ( app_info_flags & APPINFO_FLAG_TP_CLIENT ) - asd.client.set_id(*p, asd, direction, tp_app_id, change_bits); + // if the third-party appId must be treated as a client, do it now + if (app_info_flags & APPINFO_FLAG_TP_CLIENT) + { + if (hsession) + hsession->set_client(tp_app_id, change_bits, "Third Party"); + else + asd.client.set_id(*p, asd, direction, tp_app_id, change_bits); + } - process_third_party_results(asd, tp_confidence, tp_proto_list, tp_attribute_data, - change_bits); + if ((app_info_flags & APPINFO_FLAG_SSL_SQUELCH) and + asd.get_session_flags(APPID_SESSION_SSL_SESSION) and + !(asd.scan_flags & SCAN_SSL_HOST_FLAG)) + { + setSSLSquelch(p, 1, tp_app_id, asd.ctxt.get_odp_ctxt()); + } - if ((app_info_flags & APPINFO_FLAG_SSL_SQUELCH) and - asd.get_session_flags(APPID_SESSION_SSL_SESSION) and - !(asd.scan_flags & SCAN_SSL_HOST_FLAG)) - { - setSSLSquelch(p, 1, tp_app_id, asd.ctxt.get_odp_ctxt()); - } + if ( app_info_flags & APPINFO_FLAG_IGNORE ) + { + if (appidDebug->is_active()) + LogMessage("AppIdDbg %s 3rd party ignored\n", + appidDebug->get_debug_session()); - if ( app_info_flags & APPINFO_FLAG_IGNORE ) - { - if (appidDebug->is_active()) - LogMessage("AppIdDbg %s 3rd party ignored\n", - appidDebug->get_debug_session()); - - if (asd.get_session_flags(APPID_SESSION_HTTP_SESSION)) - tp_app_id = APP_ID_HTTP; - else if ( asd.get_session_flags(APPID_SESSION_SSL_SESSION) ) - tp_app_id = APP_ID_SSL; - else - tp_app_id = APP_ID_NONE; - } - if (tp_app_id == APP_ID_SSL && - (Stream::get_snort_protocol_id(p->flow) == asd.ctxt.config.snortId_for_ftp_data)) - { - // If we see SSL on an FTP data channel set tpAppId back - // to APP_ID_NONE so the FTP preprocessor picks up the flow. - tp_app_id = APP_ID_NONE; - } - } - else - { - tp_app_id = APP_ID_NONE; - } + if (asd.get_session_flags(APPID_SESSION_HTTP_SESSION)) + tp_app_id = APP_ID_HTTP; + else if ( asd.get_session_flags(APPID_SESSION_SSL_SESSION) ) + tp_app_id = APP_ID_SSL; + else + tp_app_id = APP_ID_NONE; + } - if ( tp_app_id > APP_ID_NONE - && (!asd.get_session_flags(APPID_SESSION_APP_REINSPECT) - || asd.payload.get_id() > APP_ID_NONE) ) - { - AppId snort_app_id = APP_ID_NONE; + if (tp_app_id == APP_ID_SSL && + (Stream::get_snort_protocol_id(p->flow) == asd.ctxt.config.snortId_for_ftp_data)) + { + // If we see SSL on an FTP data channel set tpAppId back + // to APP_ID_NONE so the FTP preprocessor picks up the flow. + tp_app_id = APP_ID_NONE; + } + + if ( tp_app_id > APP_ID_NONE ) + { + AppId snort_app_id = APP_ID_NONE; + + if ( hsession ) + { + snort_app_id = APP_ID_HTTP; + //data should never be APP_ID_HTTP + if (tp_app_id != APP_ID_HTTP) + asd.set_tp_payload_app_id(*p, direction, tp_app_id, change_bits); + + asd.set_tp_app_id(APP_ID_HTTP); + + // Handle HTTP tunneling and SSL possibly then being used in that tunnel + if (tp_app_id == APP_ID_HTTP_TUNNEL) + hsession->set_payload(APP_ID_HTTP_TUNNEL, change_bits, "3rd party"); + else if (hsession->payload.get_id() == APP_ID_HTTP_TUNNEL and tp_app_id != APP_ID_SSL) + hsession->set_payload(tp_app_id, change_bits, "3rd party"); - // if the packet is HTTP, then search for via pattern - if ( asd.get_session_flags(APPID_SESSION_HTTP_SESSION) ) + hsession->process_http_packet(direction, change_bits, asd.ctxt.get_odp_ctxt().get_http_matchers()); + + if (asd.get_tp_app_id() == APP_ID_HTTP and + !asd.get_session_flags(APPID_SESSION_APP_REINSPECT) and + asd.is_tp_appid_available()) + { + asd.client_disco_state = APPID_DISCO_STATE_FINISHED; + asd.service_disco_state = APPID_DISCO_STATE_FINISHED; + asd.set_session_flags(APPID_SESSION_CLIENT_DETECTED | + APPID_SESSION_SERVICE_DETECTED); + asd.clear_session_flags(APPID_SESSION_CONTINUE); + if (direction == APP_ID_FROM_INITIATOR) { - snort_app_id = APP_ID_HTTP; - //data should never be APP_ID_HTTP - if (tp_app_id != APP_ID_HTTP) - asd.set_tp_payload_app_id(*p, direction, tp_app_id, change_bits); - - asd.set_tp_app_id(APP_ID_HTTP); - - if (tp_app_id == APP_ID_HTTP_TUNNEL) - asd.set_payload_appid_data(APP_ID_HTTP_TUNNEL, change_bits); - else if (asd.payload.get_id() == APP_ID_HTTP_TUNNEL and tp_app_id != APP_ID_SSL) - asd.set_payload_appid_data(tp_app_id, change_bits); - - AppIdHttpSession* hsession = asd.get_http_session(); - if (!hsession) - hsession = asd.create_http_session(); - hsession->process_http_packet(direction, change_bits, asd.ctxt.get_odp_ctxt().get_http_matchers()); - - if (asd.get_tp_app_id() == APP_ID_HTTP and - !asd.get_session_flags(APPID_SESSION_APP_REINSPECT) and - asd.is_tp_appid_available()) - { - asd.client_disco_state = APPID_DISCO_STATE_FINISHED; - asd.service_disco_state = APPID_DISCO_STATE_FINISHED; - asd.set_session_flags(APPID_SESSION_CLIENT_DETECTED | - APPID_SESSION_SERVICE_DETECTED); - asd.clear_session_flags(APPID_SESSION_CONTINUE); - if (direction == APP_ID_FROM_INITIATOR) - { - asd.service_ip = *p->ptrs.ip_api.get_dst(); - asd.service_port = p->ptrs.dp; - } - else - { - asd.service_ip = *p->ptrs.ip_api.get_src(); - asd.service_port = p->ptrs.sp; - } - } + asd.service_ip = *p->ptrs.ip_api.get_dst(); + asd.service_port = p->ptrs.dp; } - else if (asd.get_session_flags(APPID_SESSION_SSL_SESSION) && asd.tsession) + else { - asd.examine_ssl_metadata(p, change_bits); - uint16_t serverPort; - AppId portAppId; - serverPort = (direction == APP_ID_FROM_INITIATOR) ? p->ptrs.dp : p->ptrs.sp; - portAppId = getSslServiceAppId(serverPort); - if (tp_app_id == APP_ID_SSL) - { - tp_app_id = portAppId; - //SSL policy determines IMAPS/POP3S etc before appId sees first server - // packet - asd.service.set_port_service_id(portAppId); - if (appidDebug->is_active()) - { - const char *service_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(tp_app_id); - const char *port_service_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(asd.service.get_port_service_id()); - LogMessage("AppIdDbg %s SSL is service %s (%d), portServiceAppId %s (%d)\n", - appidDebug->get_debug_session(), - service_name ? service_name : "unknown", tp_app_id, - port_service_name ? port_service_name : "unknown", asd.service.get_port_service_id()); - } - } - else - { - asd.set_tp_payload_app_id(*p, direction, tp_app_id, change_bits); - tp_app_id = portAppId; - if (appidDebug->is_active()) - { - const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(tp_app_id); - LogMessage("AppIdDbg %s SSL is %s (%d)\n", appidDebug->get_debug_session(), - app_name ? app_name : "unknown", tp_app_id); - } - } - snort_app_id = APP_ID_SSL; + asd.service_ip = *p->ptrs.ip_api.get_src(); + asd.service_port = p->ptrs.sp; } - else if (asd.service.get_id() == APP_ID_QUIC) - asd.set_tp_payload_app_id(*p, direction, tp_app_id, change_bits); - else + } + } + else if (asd.get_session_flags(APPID_SESSION_SSL_SESSION) && asd.tsession) + { + asd.examine_ssl_metadata(p, change_bits); + uint16_t serverPort; + AppId portAppId; + serverPort = (direction == APP_ID_FROM_INITIATOR) ? p->ptrs.dp : p->ptrs.sp; + portAppId = getSslServiceAppId(serverPort); + if (tp_app_id == APP_ID_SSL) + { + tp_app_id = portAppId; + //SSL policy determines IMAPS/POP3S etc before appId sees first server + // packet + asd.service.set_port_service_id(portAppId); + if (appidDebug->is_active()) { - //for non-http protocols, tp id is treated like serviceId - snort_app_id = tp_app_id; + const char *service_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(tp_app_id); + const char *port_service_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(asd.service.get_port_service_id()); + LogMessage("AppIdDbg %s SSL is service %s (%d), portServiceAppId %s (%d)\n", + appidDebug->get_debug_session(), + service_name ? service_name : "unknown", tp_app_id, + port_service_name ? port_service_name : "unknown", asd.service.get_port_service_id()); } - - asd.set_tp_app_id(*p, direction, tp_app_id, change_bits); - asd.sync_with_snort_protocol_id(snort_app_id, p); } else { - if (protocol != IpProtocol::TCP || - (p->packet_flags & (PKT_STREAM_ORDER_OK | PKT_STREAM_ORDER_BAD))) + asd.set_tp_payload_app_id(*p, direction, tp_app_id, change_bits); + tp_app_id = portAppId; + if (appidDebug->is_active()) { - if (direction == APP_ID_FROM_INITIATOR) - { - asd.init_tpPackets++; - check_terminate_tp_module(asd, asd.init_tpPackets); - } - else - { - asd.resp_tpPackets++; - check_terminate_tp_module(asd, asd.resp_tpPackets); - } + const char *app_name = asd.ctxt.get_odp_ctxt().get_app_info_mgr().get_app_name(tp_app_id); + LogMessage("AppIdDbg %s SSL is %s (%d)\n", appidDebug->get_debug_session(), + app_name ? app_name : "unknown", tp_app_id); } } + snort_app_id = APP_ID_SSL; } - if ( asd.tp_reinspect_by_initiator && check_reinspect(p, asd) ) + else if (asd.service.get_id() == APP_ID_QUIC) + asd.set_tp_payload_app_id(*p, direction, tp_app_id, change_bits); + else { - if (isTpAppidDiscoveryDone) - asd.clear_session_flags(APPID_SESSION_APP_REINSPECT); - if (direction == APP_ID_FROM_RESPONDER) - asd.tp_reinspect_by_initiator = false; //toggle at OK response + //for non-http protocols, tp id is treated like serviceId + snort_app_id = tp_app_id; } + + asd.set_tp_app_id(*p, direction, tp_app_id, change_bits); + asd.sync_with_snort_protocol_id(snort_app_id, p); } - return isTpAppidDiscoveryDone; + if (direction == APP_ID_FROM_INITIATOR) + { + asd.init_tpPackets++; + check_terminate_tp_module(asd, asd.init_tpPackets); + } + else + { + asd.resp_tpPackets++; + check_terminate_tp_module(asd, asd.resp_tpPackets); + } + + clear_tp_reinspect(asd, p, direction); + + return true; } diff --git a/src/network_inspectors/appid/tp_lib_handler.cc b/src/network_inspectors/appid/tp_lib_handler.cc index ade27ab2d..7514b4c30 100644 --- a/src/network_inspectors/appid/tp_lib_handler.cc +++ b/src/network_inspectors/appid/tp_lib_handler.cc @@ -102,11 +102,6 @@ ThirdPartyAppIdContext* TPLibHandler::create_tp_appid_ctxt(const AppIdConfig& co tp_config.chp_body_collection_disabled = odp_ctxt.chp_body_collection_disabled; tp_config.tp_allow_probes = odp_ctxt.tp_allow_probes; - if (odp_ctxt.http2_detection_enabled) - tp_config.http_upgrade_reporting_enabled = true; - else - tp_config.http_upgrade_reporting_enabled = false; - tp_config.http_response_version_enabled = odp_ctxt.http_response_version_enabled; ThirdPartyAppIdContext* tp_appid_ctxt = self->tp_appid_create_ctxt(tp_config); if (tp_appid_ctxt == nullptr) diff --git a/src/pub_sub/appid_events.h b/src/pub_sub/appid_events.h index 8bf3cf410..c809d0515 100644 --- a/src/pub_sub/appid_events.h +++ b/src/pub_sub/appid_events.h @@ -82,7 +82,7 @@ inline void change_bits_to_string(AppidChangeBits& change_bits, std::string& str if (change_bits.test(APPID_REFERER_BIT)) --n? str.append("referrer, ") : str.append("referrer"); if (change_bits.test(APPID_VERSION_BIT)) - --n? str.append("client-version, ") : str.append("client-version"); + --n? str.append("version, ") : str.append("version"); if (n != 0) // make sure all bits from AppidChangeBit enum get translated str.append("change_bits_to_string error!"); } @@ -90,13 +90,22 @@ inline void change_bits_to_string(AppidChangeBits& change_bits, std::string& str class AppidEvent : public snort::DataEvent { public: - AppidEvent(const AppidChangeBits& ac) : ac_bits(ac) {} + AppidEvent(const AppidChangeBits& ac, bool is_http2, uint32_t http2_stream_index) : + ac_bits(ac), is_http2(is_http2), http2_stream_index(http2_stream_index) {} - const AppidChangeBits& get_change_bitset() + const AppidChangeBits& get_change_bitset() const { return ac_bits; } + bool get_is_http2() const + { return is_http2; } + + uint32_t get_http2_stream_index() const + { return http2_stream_index; } + private: const AppidChangeBits& ac_bits; + bool is_http2; + uint32_t http2_stream_index; }; #endif