From: Mike Stepanek (mstepane) Date: Tue, 8 Jun 2021 10:22:25 +0000 (+0000) Subject: Merge pull request #2915 in SNORT/snort3 from ~SVLASIUK/snort3:js_external_script... X-Git-Tag: 3.1.6.0~18 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=425b44c97a372edd59d2057a7f2100685955b2d5;p=thirdparty%2Fsnort3.git Merge pull request #2915 in SNORT/snort3 from ~SVLASIUK/snort3:js_external_script to master Squashed commit of the following: commit ec3d59e7ec908f71cddb89782e6c9c5d76379d2d Author: Serhii Vlasiuk Date: Mon May 24 19:03:11 2021 +0300 http_inspect: add JS normalization for external scripts Support multiple PDUs and partial detection Support existing JS built-in rules Add new built-in rule when script body after script-src attribute is not comments --- diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h index 14a76521b..7b4126c77 100755 --- a/src/service_inspectors/http_inspect/http_enum.h +++ b/src/service_inspectors/http_inspect/http_enum.h @@ -63,7 +63,8 @@ enum PEG_COUNT { PEG_FLOW = 0, PEG_SCAN, PEG_REASSEMBLE, PEG_INSPECT, PEG_REQUES PEG_OTHER_METHOD, PEG_REQUEST_BODY, PEG_CHUNKED, PEG_URI_NORM, PEG_URI_PATH, PEG_URI_CODING, PEG_CONCURRENT_SESSIONS, PEG_MAX_CONCURRENT_SESSIONS, PEG_SCRIPT_DETECTION, PEG_PARTIAL_INSPECT, PEG_EXCESS_PARAMS, PEG_PARAMS, PEG_CUTOVERS, PEG_SSL_SEARCH_ABND_EARLY, - PEG_PIPELINED_FLOWS, PEG_PIPELINED_REQUESTS, PEG_TOTAL_BYTES, PEG_JS_INLINE, PEG_COUNT_MAX }; + PEG_PIPELINED_FLOWS, PEG_PIPELINED_REQUESTS, PEG_TOTAL_BYTES, PEG_JS_INLINE, PEG_JS_EXTERNAL, + PEG_COUNT_MAX }; // Result of scanning by splitter enum ScanResult { SCAN_NOT_FOUND, SCAN_NOT_FOUND_ACCELERATE, SCAN_FOUND, SCAN_FOUND_PIECE, @@ -268,6 +269,7 @@ enum Infraction INF_JS_BAD_TOKEN, INF_JS_OPENING_TAG, INF_JS_CLOSING_TAG, + INF_JS_CODE_IN_EXTERNAL, INF__MAX_VALUE }; @@ -326,9 +328,6 @@ enum EventSid EVENT_PDF_UNSUP_COMP_TYPE = 115, EVENT_PDF_CASC_COMP = 116, EVENT_PDF_PARSE_FAILURE = 117, - EVENT_JS_BAD_TOKEN = 118, - EVENT_JS_OPENING_TAG = 119, - EVENT_JS_CLOSING_TAG = 120, EVENT_LOSS_OF_SYNC = 201, EVENT_CHUNK_ZEROS = 202, @@ -394,6 +393,10 @@ enum EventSid EVENT_LONG_SCHEME = 262, EVENT_HTTP2_UPGRADE_REQUEST = 263, EVENT_HTTP2_UPGRADE_RESPONSE = 264, + EVENT_JS_BAD_TOKEN = 265, + EVENT_JS_OPENING_TAG = 266, + EVENT_JS_CLOSING_TAG = 267, + EVENT_JS_CODE_IN_EXTERNAL = 268, EVENT__MAX_VALUE }; diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h index eb86e790d..d56e23455 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.h +++ b/src/service_inspectors/http_inspect/http_flow_data.h @@ -193,6 +193,7 @@ private: // *** HttpJsNorm snort::JSNormalizer* js_normalizer = nullptr; + bool js_built_in_event = false; snort::JSNormalizer& acquire_js_ctx(); void release_js_ctx(); diff --git a/src/service_inspectors/http_inspect/http_js_norm.cc b/src/service_inspectors/http_inspect/http_js_norm.cc index 90e7666f6..e1ec662a4 100644 --- a/src/service_inspectors/http_inspect/http_js_norm.cc +++ b/src/service_inspectors/http_inspect/http_js_norm.cc @@ -33,6 +33,16 @@ using namespace HttpEnums; using namespace snort; +static inline JSTokenizer::JSRet js_normalize(JSNormalizer& ctx, const char* const end, + const char* dst_end, const char*& ptr, char*& dst) +{ + auto ret = ctx.normalize(ptr, end - ptr, dst, dst_end - dst); + ptr = ctx.get_src_next(); + dst = ctx.get_dst_next(); + + return ret; +} + HttpJsNorm::HttpJsNorm(const HttpParaList::UriParam& uri_param_, int64_t normalization_depth_) : uri_param(uri_param_), normalization_depth(normalization_depth_), @@ -81,7 +91,73 @@ void HttpJsNorm::configure() configure_once = true; } -void HttpJsNorm::enhanced_normalize(const Field& input, Field& output, +void HttpJsNorm::enhanced_external_normalize(const Field& input, Field& output, + HttpInfractions* infractions, HttpFlowData* ssn) const +{ + if (ssn->js_built_in_event) + return; + + const char* ptr = (const char*)input.start(); + const char* const end = ptr + input.length(); + + HttpEventGen* events = ssn->events[HttpCommon::SRC_SERVER]; + + char* buffer = nullptr; + char* dst = nullptr; + const char* dst_end = nullptr; + + if (!alive_ctx(ssn)) + HttpModule::increment_peg_counts(PEG_JS_EXTERNAL); + + while (ptr < end) + { + if (!buffer) + { + auto len = end - ptr; // not more than the remaining raw data + buffer = new char[len]; + dst = buffer; + dst_end = buffer + len; + } + + auto& ctx = ssn->acquire_js_ctx(); + ctx.set_depth(normalization_depth); + auto ret = js_normalize(ctx, end, dst_end, ptr, dst); + + switch (ret) + { + case JSTokenizer::EOS: + case JSTokenizer::SCRIPT_CONTINUE: + break; + case JSTokenizer::SCRIPT_ENDED: + case JSTokenizer::CLOSING_TAG: + *infractions += INF_JS_CLOSING_TAG; + events->create_event(EVENT_JS_CLOSING_TAG); + ssn->js_built_in_event = true; + break; + case JSTokenizer::OPENING_TAG: + *infractions += INF_JS_OPENING_TAG; + events->create_event(EVENT_JS_OPENING_TAG); + ssn->js_built_in_event = true; + break; + case JSTokenizer::BAD_TOKEN: + *infractions += INF_JS_BAD_TOKEN; + events->create_event(EVENT_JS_BAD_TOKEN); + ssn->js_built_in_event = true; + break; + default: + assert(false); + break; + } + + if (ssn->js_built_in_event) + break; + } + + if (buffer) + output.set(dst - buffer, (const uint8_t*)buffer, true); +} + +void HttpJsNorm::enhanced_inline_normalize(const Field& input, Field& output, HttpInfractions* infractions, HttpFlowData* ssn) const { const char* ptr = (const char*)input.start(); @@ -94,6 +170,7 @@ void HttpJsNorm::enhanced_normalize(const Field& input, Field& output, const char* dst_end = nullptr; bool script_continue = alive_ctx(ssn); + bool script_external = false; while (ptr < end) { @@ -115,11 +192,14 @@ void HttpJsNorm::enhanced_normalize(const Field& input, Field& output, ptr = sctx.next; } - if (!sctx.is_javascript || sctx.is_external) + if (!sctx.is_javascript) continue; + script_external = sctx.is_external; + // script found - HttpModule::increment_peg_counts(PEG_JS_INLINE); + if (!script_external) + HttpModule::increment_peg_counts(PEG_JS_INLINE); } if (!buffer) @@ -127,7 +207,7 @@ void HttpJsNorm::enhanced_normalize(const Field& input, Field& output, uint8_t* nbuf = ssn->js_detect_buffer[HttpCommon::SRC_SERVER]; uint32_t nlen = ssn->js_detect_length[HttpCommon::SRC_SERVER]; - auto len = nlen + (end - ptr); // not more then the remaining raw data + auto len = nlen + (end - ptr); // not more than the remaining raw data buffer = new char[len]; if (nbuf) memcpy(buffer, nbuf, nlen); @@ -137,10 +217,8 @@ void HttpJsNorm::enhanced_normalize(const Field& input, Field& output, auto& ctx = ssn->acquire_js_ctx(); ctx.set_depth(normalization_depth); - - auto ret = ctx.normalize(ptr, end - ptr, dst, dst_end - dst); - ptr = ctx.get_src_next(); - dst = ctx.get_dst_next(); + auto dst_before = dst; + auto ret = js_normalize(ctx, end, dst_end, ptr, dst); switch (ret) { @@ -174,6 +252,12 @@ void HttpJsNorm::enhanced_normalize(const Field& input, Field& output, script_continue = false; break; } + + if (script_external && dst_before != dst) + { + *infractions += INF_JS_CODE_IN_EXTERNAL; + events->create_event(EVENT_JS_CODE_IN_EXTERNAL); + } } if (!script_continue) diff --git a/src/service_inspectors/http_inspect/http_js_norm.h b/src/service_inspectors/http_inspect/http_js_norm.h index 385754e16..4fb1d6126 100644 --- a/src/service_inspectors/http_inspect/http_js_norm.h +++ b/src/service_inspectors/http_inspect/http_js_norm.h @@ -41,7 +41,8 @@ public: void legacy_normalize(const Field& input, Field& output, HttpInfractions*, HttpEventGen*, int max_javascript_whitespaces) const; - void enhanced_normalize(const Field& input, Field& output, HttpInfractions*, HttpFlowData*) const; + void enhanced_inline_normalize(const Field& input, Field& output, HttpInfractions*, HttpFlowData*) const; + void enhanced_external_normalize(const Field& input, Field& output, HttpInfractions*, HttpFlowData*) const; void configure(); diff --git a/src/service_inspectors/http_inspect/http_msg_body.cc b/src/service_inspectors/http_inspect/http_msg_body.cc index 2ba5f71a6..869391577 100644 --- a/src/service_inspectors/http_inspect/http_msg_body.cc +++ b/src/service_inspectors/http_inspect/http_msg_body.cc @@ -341,8 +341,13 @@ void HttpMsgBody::do_js_normalization(const Field& input, Field& output, bool pa len = 0; } - params->js_norm_param.js_norm->enhanced_normalize(input, enhanced_js_norm_body, - transaction->get_infractions(source_id), session_data); + auto http_header = get_header(source_id); + if (http_header and http_header->is_external_js()) + params->js_norm_param.js_norm->enhanced_external_normalize(input, enhanced_js_norm_body, + transaction->get_infractions(source_id), session_data); + else + params->js_norm_param.js_norm->enhanced_inline_normalize(input, enhanced_js_norm_body, + transaction->get_infractions(source_id), session_data); const int32_t norm_length = (enhanced_js_norm_body.length() <= session_data->detect_depth_remaining[source_id]) ? diff --git a/src/service_inspectors/http_inspect/http_msg_head_shared.cc b/src/service_inspectors/http_inspect/http_msg_head_shared.cc index 300b5ac1c..45c18d594 100755 --- a/src/service_inspectors/http_inspect/http_msg_head_shared.cc +++ b/src/service_inspectors/http_inspect/http_msg_head_shared.cc @@ -84,6 +84,58 @@ HttpMsgHeadShared::~HttpMsgHeadShared() session_data->update_deallocations(extra_memory_allocations); } +bool HttpMsgHeadShared::is_external_js() +{ + if (js_external != STAT_NOT_COMPUTE) + return js_external; + + const Field& content_type = get_header_value_raw(HEAD_CONTENT_TYPE); + const char* cur = (const char*)content_type.start(); + int len = content_type.length(); + if (SnortStrcasestr(cur, len, "application/")) + { + if (SnortStrcasestr(cur, len, "javascript")) + { + js_external = 1; + return true; + } + + if (SnortStrcasestr(cur, len, "ecmascript")) + { + js_external = 1; + return true; + } + } + else if (SnortStrcasestr(cur, len, "text/")) + { + if (SnortStrcasestr(cur, len, "javascript")) + { + js_external = 1; + return true; + } + + if (SnortStrcasestr(cur, len, "ecmascript")) + { + js_external = 1; + return true; + } + + if (SnortStrcasestr(cur, len, "jscript")) + { + js_external = 1; + return true; + } + + if (SnortStrcasestr(cur, len, "livescript")) + { + js_external = 1; + return true; + } + } + js_external = 0; + return js_external; +} + // All the header processing that is done for every message (i.e. not just-in-time) is done here. void HttpMsgHeadShared::analyze() { diff --git a/src/service_inspectors/http_inspect/http_msg_head_shared.h b/src/service_inspectors/http_inspect/http_msg_head_shared.h index 5a93fb23e..dcbc6e85c 100755 --- a/src/service_inspectors/http_inspect/http_msg_head_shared.h +++ b/src/service_inspectors/http_inspect/http_msg_head_shared.h @@ -58,6 +58,7 @@ public: // verdicts. uint64_t get_file_cache_index(); const Field& get_content_disposition_filename(); + bool is_external_js(); protected: HttpMsgHeadShared(const uint8_t* buffer, const uint16_t buf_size, @@ -133,6 +134,7 @@ private: bool own_msg_buffer; const uint32_t extra_memory_allocations; + int js_external = HttpCommon::STAT_NOT_COMPUTE; }; #endif diff --git a/src/service_inspectors/http_inspect/http_msg_request.cc b/src/service_inspectors/http_inspect/http_msg_request.cc index c59df1f49..3535d93a8 100644 --- a/src/service_inspectors/http_inspect/http_msg_request.cc +++ b/src/service_inspectors/http_inspect/http_msg_request.cc @@ -40,6 +40,7 @@ HttpMsgRequest::HttpMsgRequest(const uint8_t* buffer, const uint16_t buf_size, { transaction->set_request(this); get_related_sections(); + session_data->release_js_ctx(); } HttpMsgRequest::~HttpMsgRequest() diff --git a/src/service_inspectors/http_inspect/http_tables.cc b/src/service_inspectors/http_inspect/http_tables.cc index 26909e2fa..ac671f6a5 100755 --- a/src/service_inspectors/http_inspect/http_tables.cc +++ b/src/service_inspectors/http_inspect/http_tables.cc @@ -357,9 +357,6 @@ const RuleMap HttpModule::http_events[] = { EVENT_PDF_UNSUP_COMP_TYPE, "PDF file unsupported compression type" }, { EVENT_PDF_CASC_COMP, "PDF file cascaded compression" }, { EVENT_PDF_PARSE_FAILURE, "PDF file parse failure" }, - { EVENT_JS_BAD_TOKEN, "bad token in JavaScript" }, - { EVENT_JS_OPENING_TAG, "unexpected script opening tag in JavaScript" }, - { EVENT_JS_CLOSING_TAG, "unexpected script closing tag in JavaScript" }, { EVENT_LOSS_OF_SYNC, "not HTTP traffic" }, { EVENT_CHUNK_ZEROS, "chunk length has excessive leading zeros" }, { EVENT_WS_BETWEEN_MSGS, "white space before or between messages" }, @@ -430,6 +427,10 @@ const RuleMap HttpModule::http_events[] = { EVENT_LONG_SCHEME, "HTTP URI scheme longer than 10 characters" }, { EVENT_HTTP2_UPGRADE_REQUEST, "HTTP/1 client requested HTTP/2 upgrade" }, { EVENT_HTTP2_UPGRADE_RESPONSE, "HTTP/1 server granted HTTP/2 upgrade" }, + { EVENT_JS_BAD_TOKEN, "bad token in JavaScript" }, + { EVENT_JS_OPENING_TAG, "unexpected script opening tag in JavaScript" }, + { EVENT_JS_CLOSING_TAG, "unexpected script closing tag in JavaScript" }, + { EVENT_JS_CODE_IN_EXTERNAL, "JavaScript code under the external script tags" }, { 0, nullptr } }; @@ -467,6 +468,7 @@ const PegInfo HttpModule::peg_names[PEG_COUNT_MAX+1] = { CountType::SUM, "pipelined_requests", "total requests placed in a pipeline" }, { CountType::SUM, "total_bytes", "total HTTP data bytes inspected" }, { CountType::SUM, "js_inline_scripts", "total number of inline JavaScripts processed" }, + { CountType::SUM, "js_external_scripts", "total number of external JavaScripts processed" }, { CountType::END, nullptr, nullptr } };