From: Mike Stepanek (mstepane) Date: Wed, 17 Feb 2021 12:29:35 +0000 (+0000) Subject: Merge pull request #2748 in SNORT/snort3 from ~THOPETER/snort3:nhttp155 to master X-Git-Tag: 3.1.2.0~40 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=91bd0f7bef0d0746a1aa04fbcb330566526c036e;p=thirdparty%2Fsnort3.git Merge pull request #2748 in SNORT/snort3 from ~THOPETER/snort3:nhttp155 to master Squashed commit of the following: commit f6efaf5d3ed10d81275a38931dcaeba00b4564ab Author: Tom Peters Date: Mon Feb 15 17:11:57 2021 -0500 http_inspect: remove detained inspection --- diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.cc b/src/service_inspectors/http2_inspect/http2_data_cutter.cc index e34993a88..b95c24665 100644 --- a/src/service_inspectors/http2_inspect/http2_data_cutter.cc +++ b/src/service_inspectors/http2_inspect/http2_data_cutter.cc @@ -127,7 +127,7 @@ StreamSplitter::Status Http2DataCutter::scan(const uint8_t* data, uint32_t lengt assert(scan_result == StreamSplitter::SEARCH); scan_result = StreamSplitter::FLUSH; if (cur_data > 0) - session_data->hi_ss[source_id]->init_partial_flush(session_data->flow); + session_data->hi_ss[source_id]->prep_partial_flush(session_data->flow, 0); else { session_data->payload_discard[source_id] = true; diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter.cc b/src/service_inspectors/http2_inspect/http2_stream_splitter.cc index 59dfcd1f0..67049c353 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter.cc +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter.cc @@ -129,23 +129,17 @@ const StreamBuffer Http2StreamSplitter::reassemble(Flow* flow, unsigned total, u return http_buf; } bool tcp_close; - bool partial_flush; uint8_t* test_buffer; HttpTestManager::get_test_input_source()->reassemble(&test_buffer, len, total, offset, - flags, source_id, tcp_close, partial_flush); + flags, source_id, tcp_close); if (tcp_close) { finish(flow); } - if (partial_flush) - { - init_partial_flush(flow); - } if (test_buffer == nullptr) { - // Source ID does not match test data, no test data was flushed, preparing for a - // partial flush, preparing for a TCP connection close, or there is no more test - // data + // Source ID does not match test data, no test data was flushed, preparing for a TCP + // connection close, or there is no more test data return http_buf; } data = test_buffer; diff --git a/src/service_inspectors/http_inspect/dev_notes.txt b/src/service_inspectors/http_inspect/dev_notes.txt index d736ea05f..1f066908c 100755 --- a/src/service_inspectors/http_inspect/dev_notes.txt +++ b/src/service_inspectors/http_inspect/dev_notes.txt @@ -7,36 +7,45 @@ It serves several specialized purposes in cases where the HTTP message is trunca unexpectedly). The nature of splitting allows packets to be forwarded before they are aggregated into a message -section and inspected. Detained inspection is a feature that allows the splitter to designate -to Stream packets that are too risky to forward without being inspected. These packets are detained -until such time as inspection is completed. The design is based on the principle that detaining -one packet in a TCP stream effectively blocks all subsequent packets from being reassembled and -delivered to a target application. Once a packet is detained no attempt is made to detain -additional packets. - -The core functions of detained inspection are implemented in the message body cutters. They search -for the beginning of Javascripts by finding the string "". When -one is found an immediate partial inspection is performed. This avoids the adverse network -consequences of detaining packets at the performance and memory cost of doing a much larger number -of partial inspections. Code features that support both approaches are referred to as accelerated -blocking. +section and inspected. This may lead to problems when the target consumes a partial message +body even though the end of the message body was never received because Snort blocked it. + +Script detection is a feature developed to solve this problem for message bodies containing +Javascripts. The stream splitter scan() method searches its input for the end-of-script tag +"". When necessary this requires scan() to unzip the data. This is an extra unzip as +storage limitations preclude saving the unzipped version of the data for subsequent reassembly. + +When the end of a script is found and the normal flush point has not been found, the current TCP +segment and all previous segments for the current message section are flushed using a special +procedure known as partial inspection. From the perspective of Stream (or H2I) a partial inspection +is a regular flush in every respect. + +scan() calls prep_partial_flush() to prepare for the partial inspection. Then it returns a normal +flush point to Stream at the end of the current TCP segment. Partial inspections perform all of the +functions of a regular inspection including forwarding data to file processing and detection. + +The difference between a partial inspection and a regular inspection is reassemble() saves the +input data for future reuse. Eventually there will be a regular full inspection of the entire +message section. reassemble() will accomplish this by combining the input data for the partial +inspection with later data that complete the message section. + +Correct and efficient execution of a full inspection following a partial inspection requires +special handling of certain functions. Unzipping is only done once in reassemble(). The stored +input in reassemble() has already been through dechunking and unzipping. Data is forwarded to file +processing during the partial inspection and duplicate data will not be forwarded again. Some +of the message body normalization steps are done once during partial inspection with work +products saved for reuse. + +It is possible to do more than one partial inspection of a single message section. Each partial +inspection is cumulative, covering the new data and all previous data. + +Compared to just doing a full inspection, a partial inspection followed by a partial inspection +will not miss anything. The benefits of partial inspection are in addition to the benefits of a +full inspection. + +The http_inspect partial inspection mechanism is also used by http2_inspect to manage frame +boundaries. When inspecting HTTP/2, a partial inspection by http_inspect may occur because script +detection triggered it, because H2I wanted it, or both. HttpFlowData is a data class representing all HI information relating to a flow. It serves as persistent memory between invocations of HI by the framework. It also glues together the inspector, @@ -288,9 +297,6 @@ Commands: @tcpclose simulates a half-duplex TCP close. @request and @response set the message direction. Applies to subsequent paragraphs until changed. The initial direction is always request and the break command resets the direction to request. - @partial causes a partial flush, simulating a retransmission of a detained packet. This does not - have any application to script detection or any other feature where the stream splitter is - driving partial inspections instead of stream. @fileset specifies a file from which the tool will read data into the message buffer. This may be used to include a zipped or other binary file into a message body. Data is read beginning at the start of the file. The file is closed automatically whenever a new file is diff --git a/src/service_inspectors/http_inspect/http_cutter.cc b/src/service_inspectors/http_inspect/http_cutter.cc index a07134769..9d33cc55b 100644 --- a/src/service_inspectors/http_inspect/http_cutter.cc +++ b/src/service_inspectors/http_inspect/http_cutter.cc @@ -278,10 +278,10 @@ ScanResult HttpHeaderCutter::cut(const uint8_t* buffer, uint32_t length, return SCAN_NOT_FOUND; } -HttpBodyCutter::HttpBodyCutter(AcceleratedBlocking accelerated_blocking_, CompressId compression_) +HttpBodyCutter::HttpBodyCutter(bool accelerated_blocking_, CompressId compression_) : accelerated_blocking(accelerated_blocking_), compression(compression_) { - if (accelerated_blocking != AB_NONE) + if (accelerated_blocking) { if ((compression == CMP_GZIP) || (compression == CMP_DEFLATE)) { @@ -300,25 +300,13 @@ HttpBodyCutter::HttpBodyCutter(AcceleratedBlocking accelerated_blocking_, Compre } } - static const uint8_t detain_string[] = { '<', 's', 'c', 'r', 'i', 'p', 't' }; - static const uint8_t detain_upper[] = { '<', 'S', 'C', 'R', 'I', 'P', 'T' }; static const uint8_t inspect_string[] = { '<', '/', 's', 'c', 'r', 'i', 'p', 't', '>' }; static const uint8_t inspect_upper[] = { '<', '/', 'S', 'C', 'R', 'I', 'P', 'T', '>' }; - if (accelerated_blocking == AB_DETAIN) - { - match_string = detain_string; - match_string_upper = detain_upper; - string_length = sizeof(detain_string); - HttpModule::get_detain_finder(finder, handle); - } - else - { - match_string = inspect_string; - match_string_upper = inspect_upper; - string_length = sizeof(inspect_string); - HttpModule::get_script_finder(finder, handle); - } + match_string = inspect_string; + match_string_upper = inspect_upper; + string_length = sizeof(inspect_string); + HttpModule::get_script_finder(finder, handle); } } @@ -816,50 +804,17 @@ ScanResult HttpBodyH2Cutter::cut(const uint8_t* buffer, uint32_t length, } // This method searches the input stream looking for a script or other dangerous content that -// requires accelerated blocking. Exactly what we are looking for is encapsulated in dangerous(). +// requires script detection. Exactly what we are looking for is encapsulated in dangerous(). // // Return value true indicates a match and enables the packet that completes the matching sequence -// to be detained (detained inspection) or sent for partial inspection (script detection). -// -// Once detained inspection is activated on a message body it never goes away. The first packet -// of every subsequent message section must be detained (detention_required). Supporting this -// requirement requires that the calling routine submit all data including buffers that are about -// to be flushed. -// -// Script detection (AB_INSPECT) is similar in that the message data must be scanned by dangerous() -// looking for a particular string. It differs in the string being searched for and that difference -// is built into dangerous(). Script detection does not automatically apply to subsequent message -// sections. It only recurs when a new end-of-script tag is found. +// to be sent for partial inspection. // // Any attempt to optimize this code should be mindful that once you skip any part of the message // body, dangerous() loses the ability to unzip subsequent data. bool HttpBodyCutter::need_accelerated_blocking(const uint8_t* data, uint32_t length) { - switch (accelerated_blocking) - { - case AB_DETAIN: - // With detained inspection we have two basic principles here: 1) having detained a packet - // we don't need to detain another one while the first one is still being held and 2) once - // we detain a packet we don't need to keep scanning content. We are always going to detain - // a new packet as soon as we release the previous one. - if (!packet_detained && (detention_required || dangerous(data, length))) - { - packet_detained = true; - detention_required = true; - return true; - } - break; - case AB_INSPECT: - // Script detection requires continuous scanning of the data because every packet is a new - // decision regardless of any previous determinations. - if (dangerous(data, length)) - return true; - break; - case AB_NONE: - break; - } - return false; + return accelerated_blocking && dangerous(data, length); } bool HttpBodyCutter::find_partial(const uint8_t* input_buf, uint32_t input_length, bool end) diff --git a/src/service_inspectors/http_inspect/http_cutter.h b/src/service_inspectors/http_inspect/http_cutter.h index f49e14d8a..8c74e44d0 100644 --- a/src/service_inspectors/http_inspect/http_cutter.h +++ b/src/service_inspectors/http_inspect/http_cutter.h @@ -102,11 +102,9 @@ private: class HttpBodyCutter : public HttpCutter { public: - HttpBodyCutter(HttpEnums::AcceleratedBlocking accelerated_blocking_, - HttpEnums::CompressId compression_); + HttpBodyCutter(bool accelerated_blocking_, HttpEnums::CompressId compression_); ~HttpBodyCutter() override; - void soft_reset() override { octets_seen = 0; packet_detained = false; } - void detain_ended() { packet_detained = false; } + void soft_reset() override { octets_seen = 0; } protected: bool need_accelerated_blocking(const uint8_t* data, uint32_t length); @@ -115,10 +113,8 @@ private: bool dangerous(const uint8_t* data, uint32_t length); bool find_partial(const uint8_t*, uint32_t, bool); - const HttpEnums::AcceleratedBlocking accelerated_blocking; - bool packet_detained = false; + const bool accelerated_blocking; uint8_t partial_match = 0; - bool detention_required = false; HttpEnums::CompressId compression; z_stream* compress_stream = nullptr; bool decompress_failed = false; @@ -133,7 +129,7 @@ class HttpBodyClCutter : public HttpBodyCutter { public: HttpBodyClCutter(int64_t expected_length, - HttpEnums::AcceleratedBlocking accelerated_blocking, + bool accelerated_blocking, HttpEnums::CompressId compression) : HttpBodyCutter(accelerated_blocking, compression), remaining(expected_length) { assert(remaining > 0); } @@ -147,8 +143,7 @@ private: class HttpBodyOldCutter : public HttpBodyCutter { public: - HttpBodyOldCutter(HttpEnums::AcceleratedBlocking accelerated_blocking, - HttpEnums::CompressId compression) : + HttpBodyOldCutter(bool accelerated_blocking, HttpEnums::CompressId compression) : HttpBodyCutter(accelerated_blocking, compression) {} HttpEnums::ScanResult cut(const uint8_t*, uint32_t, HttpInfractions*, HttpEventGen*, @@ -158,8 +153,7 @@ public: class HttpBodyChunkCutter : public HttpBodyCutter { public: - HttpBodyChunkCutter(HttpEnums::AcceleratedBlocking accelerated_blocking, - HttpEnums::CompressId compression) : + HttpBodyChunkCutter(bool accelerated_blocking, HttpEnums::CompressId compression) : HttpBodyCutter(accelerated_blocking, compression) {} HttpEnums::ScanResult cut(const uint8_t* buffer, uint32_t length, @@ -183,7 +177,7 @@ class HttpBodyH2Cutter : public HttpBodyCutter { public: HttpBodyH2Cutter(int64_t expected_length, - HttpEnums::AcceleratedBlocking accelerated_blocking, + bool accelerated_blocking, HttpEnums::CompressId compression) : HttpBodyCutter(accelerated_blocking, compression), expected_body_length(expected_length) {} diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h index f67531bcd..cc39b7fea 100755 --- a/src/service_inspectors/http_inspect/http_enum.h +++ b/src/service_inspectors/http_inspect/http_enum.h @@ -60,7 +60,7 @@ enum HTTP_BUFFER { HTTP_BUFFER_CLIENT_BODY = 1, HTTP_BUFFER_COOKIE, HTTP_BUFFER_ enum PEG_COUNT { PEG_FLOW = 0, PEG_SCAN, PEG_REASSEMBLE, PEG_INSPECT, PEG_REQUEST, PEG_RESPONSE, PEG_GET, PEG_HEAD, PEG_POST, PEG_PUT, PEG_DELETE, PEG_CONNECT, PEG_OPTIONS, PEG_TRACE, PEG_OTHER_METHOD, PEG_REQUEST_BODY, PEG_CHUNKED, PEG_URI_NORM, PEG_URI_PATH, PEG_URI_CODING, - PEG_CONCURRENT_SESSIONS, PEG_MAX_CONCURRENT_SESSIONS, PEG_DETAINED, PEG_SCRIPT_DETECTION, + PEG_CONCURRENT_SESSIONS, PEG_MAX_CONCURRENT_SESSIONS, PEG_SCRIPT_DETECTION, PEG_PARTIAL_INSPECT, PEG_EXCESS_PARAMS, PEG_PARAMS, PEG_CUTOVERS, PEG_SSL_SEARCH_ABND_EARLY, PEG_PIPELINED_FLOWS, PEG_PIPELINED_REQUESTS, PEG_TOTAL_BYTES, PEG_COUNT_MAX }; @@ -72,8 +72,6 @@ enum ScanResult { SCAN_NOT_FOUND, SCAN_NOT_FOUND_ACCELERATE, SCAN_FOUND, SCAN_FO enum ChunkState { CHUNK_NEWLINES, CHUNK_ZEROS, CHUNK_LEADING_WS, CHUNK_NUMBER, CHUNK_TRAILING_WS, CHUNK_OPTIONS, CHUNK_HCRLF, CHUNK_DATA, CHUNK_DCRLF1, CHUNK_DCRLF2, CHUNK_BAD }; -enum AcceleratedBlocking { AB_DETAIN, AB_INSPECT, AB_NONE }; - // List of possible HTTP versions. enum VersionId { VERS__NO_SOURCE=-16, VERS__NOT_COMPUTE=-14, VERS__PROBLEMATIC=-12, VERS__NOT_PRESENT=-11, VERS__OTHER=1, VERS_1_0, VERS_1_1, VERS_2_0, VERS_0_9 }; diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc index 48d1ec522..cc028535a 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.cc +++ b/src/service_inspectors/http_inspect/http_flow_data.cc @@ -130,7 +130,7 @@ void HttpFlowData::half_reset(SourceId source_id) partial_inspected_octets[source_id] = 0; section_size_target[source_id] = 0; stretch_section_to_packet[source_id] = false; - accelerated_blocking[source_id] = AB_NONE; + accelerated_blocking[source_id] = false; file_depth_remaining[source_id] = STAT_NOT_PRESENT; detect_depth_remaining[source_id] = STAT_NOT_PRESENT; detection_status[source_id] = DET_REACTIVATING; diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h index 929528305..4f63e15b2 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.h +++ b/src/service_inspectors/http_inspect/http_flow_data.h @@ -143,8 +143,7 @@ private: HttpEnums::CompressId compression[2] = { HttpEnums::CMP_NONE, HttpEnums::CMP_NONE }; HttpEnums::DetectionStatus detection_status[2] = { HttpEnums::DET_ON, HttpEnums::DET_ON }; bool stretch_section_to_packet[2] = { false, false }; - HttpEnums::AcceleratedBlocking accelerated_blocking[2] = - { HttpEnums::AB_NONE, HttpEnums::AB_NONE }; + bool accelerated_blocking[2] = { false, false }; // *** Inspector's internal data about the current message struct FdCallbackContext diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc index 6f4cb339e..ef2b628b5 100755 --- a/src/service_inspectors/http_inspect/http_inspect.cc +++ b/src/service_inspectors/http_inspect/http_inspect.cc @@ -149,7 +149,6 @@ void HttpInspect::show(const SnortConfig*) const ConfigLogger::log_flag("decompress_pdf", params->decompress_pdf); ConfigLogger::log_flag("decompress_swf", params->decompress_swf); ConfigLogger::log_flag("decompress_zip", params->decompress_zip); - ConfigLogger::log_flag("detained_inspection", params->detained_inspection); ConfigLogger::log_flag("script_detection", params->script_detection); ConfigLogger::log_flag("normalize_javascript", params->js_norm_param.normalize_javascript); ConfigLogger::log_value("max_javascript_whitespaces", diff --git a/src/service_inspectors/http_inspect/http_module.cc b/src/service_inspectors/http_inspect/http_module.cc index 0fbf441d9..7a1d97d82 100755 --- a/src/service_inspectors/http_inspect/http_module.cc +++ b/src/service_inspectors/http_inspect/http_module.cc @@ -35,30 +35,21 @@ using namespace snort; using namespace HttpEnums; LiteralSearch::Handle* s_handle = nullptr; -LiteralSearch* s_detain = nullptr; LiteralSearch* s_script = nullptr; HttpModule::HttpModule() : Module(HTTP_NAME, HTTP_HELP, http_params) { s_handle = LiteralSearch::setup(); - s_detain = LiteralSearch::instantiate(s_handle, (const uint8_t*)"", 9, true, true); } HttpModule::~HttpModule() { delete params; - delete s_detain; delete s_script; LiteralSearch::cleanup(s_handle); } -void HttpModule::get_detain_finder(LiteralSearch*& finder, LiteralSearch::Handle*& handle) -{ - finder = s_detain; - handle = s_handle; -} - void HttpModule::get_script_finder(LiteralSearch*& finder, LiteralSearch::Handle*& handle) { finder = s_script; @@ -88,9 +79,6 @@ const Parameter HttpModule::http_params[] = { "decompress_zip", Parameter::PT_BOOL, nullptr, "false", "decompress zip files in response bodies" }, - { "detained_inspection", Parameter::PT_BOOL, nullptr, "false", - "store-and-forward as necessary to effectively block alerting JavaScript" }, - { "script_detection", Parameter::PT_BOOL, nullptr, "false", "inspect JavaScript immediately upon script end" }, @@ -210,10 +198,6 @@ bool HttpModule::set(const char*, Value& val, SnortConfig*) { params->decompress_zip = val.get_bool(); } - else if (val.is("detained_inspection")) - { - params->detained_inspection = val.get_bool(); - } else if (val.is("script_detection")) { params->script_detection = val.get_bool(); @@ -388,11 +372,6 @@ bool HttpModule::end(const char*, int, SnortConfig*) params->uri_param.utf8_bare_byte = false; } - if (params->detained_inspection && params->script_detection) - { - ParseError("Cannot use detained inspection and script detection together."); - } - if (params->uri_param.iis_unicode) { params->uri_param.unicode_map = new uint8_t[65536]; diff --git a/src/service_inspectors/http_inspect/http_module.h b/src/service_inspectors/http_inspect/http_module.h index b9698667c..4b0adfe7d 100755 --- a/src/service_inspectors/http_inspect/http_module.h +++ b/src/service_inspectors/http_inspect/http_module.h @@ -45,7 +45,6 @@ public: bool decompress_pdf = false; bool decompress_swf = false; bool decompress_zip = false; - bool detained_inspection = false; bool script_detection = false; struct JsNormParam @@ -136,7 +135,6 @@ public: static PegCount get_peg_counts(HttpEnums::PEG_COUNT counter) { return peg_counts[counter]; } - static void get_detain_finder(snort::LiteralSearch*&, snort::LiteralSearch::Handle*&); static void get_script_finder(snort::LiteralSearch*&, snort::LiteralSearch::Handle*&); snort::ProfileStats* get_profile() const override; diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc index a14f5b96c..e7f418588 100755 --- a/src/service_inspectors/http_inspect/http_msg_header.cc +++ b/src/service_inspectors/http_inspect/http_msg_header.cc @@ -400,14 +400,8 @@ void HttpMsgHeader::prepare_body() setup_file_decompression(); update_depth(); - if (source_id == SRC_SERVER) - { - // detained inspection not supported for HTTP/2 - if (params->script_detection) - session_data->accelerated_blocking[source_id] = AB_INSPECT; - else if ((params->detained_inspection) && !session_data->for_http2) - session_data->accelerated_blocking[source_id] = AB_DETAIN; - } + if ((source_id == SRC_SERVER) && (params->script_detection)) + session_data->accelerated_blocking[source_id] = true; if (source_id == SRC_CLIENT) { diff --git a/src/service_inspectors/http_inspect/http_stream_splitter.h b/src/service_inspectors/http_inspect/http_stream_splitter.h index 6610551d0..cf8e33868 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter.h +++ b/src/service_inspectors/http_inspect/http_stream_splitter.h @@ -43,8 +43,7 @@ public: const snort::StreamBuffer reassemble(snort::Flow* flow, unsigned total, unsigned, const uint8_t* data, unsigned len, uint32_t flags, unsigned& copied) override; bool finish(snort::Flow* flow) override; - bool init_partial_flush(snort::Flow* flow) override { return init_partial_flush(flow, 0); } - bool init_partial_flush(snort::Flow* flow, uint32_t num_flush); + bool prep_partial_flush(snort::Flow* flow, uint32_t num_flush); bool is_paf() override { return true; } static StreamSplitter::Status status_value(StreamSplitter::Status ret_val, bool http2 = false); @@ -62,7 +61,6 @@ private: static void decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data, uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream, bool at_start, HttpInfractions* infractions, HttpEventGen* events); - static void detain_packet(snort::Packet* pkt); HttpInspect* const my_inspector; const HttpCommon::SourceId source_id; diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc index a67bfcd35..9d84e47a6 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc @@ -178,19 +178,11 @@ bool HttpStreamSplitter::finish(Flow* flow) return false; } -bool HttpStreamSplitter::init_partial_flush(Flow* flow, uint32_t num_flush) +bool HttpStreamSplitter::prep_partial_flush(Flow* flow, uint32_t num_flush) { Profile profile(HttpModule::get_profile_stats()); HttpFlowData* session_data = HttpInspect::http_get_flow_data(flow); - assert(session_data != nullptr); - - assert(session_data->for_http2 || source_id == SRC_SERVER); - - assert((session_data->type_expected[source_id] == SEC_BODY_CL) || - (session_data->type_expected[source_id] == SEC_BODY_OLD) || - (session_data->type_expected[source_id] == SEC_BODY_CHUNK) || - (session_data->type_expected[source_id] == SEC_BODY_H2)); #ifdef REG_TEST if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP) && @@ -208,7 +200,6 @@ bool HttpStreamSplitter::init_partial_flush(Flow* flow, uint32_t num_flush) session_data->cutter[source_id]->get_is_broken_chunk(), session_data->cutter[source_id]->get_num_good_chunks(), session_data->cutter[source_id]->get_octets_seen() - num_flush); - (static_cast(session_data->cutter[source_id]))->detain_ended(); session_data->partial_flush[source_id] = true; return true; } diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc index d15c6c610..a2bb16c5c 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc @@ -244,24 +244,18 @@ const StreamBuffer HttpStreamSplitter::reassemble(Flow* flow, unsigned total, return http_buf; } bool tcp_close; - bool partial_flush; uint8_t* test_buffer; unsigned unused; HttpTestManager::get_test_input_source()->reassemble(&test_buffer, len, total, unused, - flags, source_id, tcp_close, partial_flush); + flags, source_id, tcp_close); if (tcp_close) { finish(flow); } - if (partial_flush) - { - init_partial_flush(flow); - } if (test_buffer == nullptr) { // Source ID does not match test data, no test data was flushed, preparing for a - // partial flush, preparing for a TCP connection close, or there is no more test - // data + // TCP connection close, or there is no more test data return http_buf; } data = test_buffer; diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc index c0b006063..eee62160b 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc @@ -118,27 +118,6 @@ StreamSplitter::Status HttpStreamSplitter::status_value(StreamSplitter::Status r return ret_val; } -void HttpStreamSplitter::detain_packet(Packet* pkt) -{ -#ifdef REG_TEST - if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP)) - { - fprintf(HttpTestManager::get_output_file(), "Packet detain request\n"); - fflush(HttpTestManager::get_output_file()); - } - - if (!HttpTestManager::use_test_input(HttpTestManager::IN_HTTP)) - { -#endif - pkt->active->hold_packet(pkt); -#ifdef REG_TEST - } -#endif - - // Count attempted detains. - HttpModule::increment_peg_counts(PEG_DETAINED); -} - StreamSplitter::Status HttpStreamSplitter::scan(Packet* pkt, const uint8_t* data, uint32_t length, uint32_t, uint32_t* flush_offset) { @@ -282,19 +261,13 @@ StreamSplitter::Status HttpStreamSplitter::scan(Packet* pkt, const uint8_t* data if (cut_result == SCAN_NOT_FOUND_ACCELERATE) { - if (session_data->accelerated_blocking[source_id] == AB_DETAIN) - detain_packet(pkt); - else - { - assert(session_data->accelerated_blocking[source_id] == AB_INSPECT); - HttpModule::increment_peg_counts(PEG_SCRIPT_DETECTION); - init_partial_flush(flow, length); + HttpModule::increment_peg_counts(PEG_SCRIPT_DETECTION); + prep_partial_flush(flow, length); #ifdef REG_TEST - if (!HttpTestManager::use_test_input(HttpTestManager::IN_HTTP)) + if (!HttpTestManager::use_test_input(HttpTestManager::IN_HTTP)) #endif - *flush_offset = length; - return status_value(StreamSplitter::FLUSH); - } + *flush_offset = length; + return status_value(StreamSplitter::FLUSH); } // Wait patiently for more data diff --git a/src/service_inspectors/http_inspect/http_tables.cc b/src/service_inspectors/http_inspect/http_tables.cc index 86ad08e08..6125af834 100755 --- a/src/service_inspectors/http_inspect/http_tables.cc +++ b/src/service_inspectors/http_inspect/http_tables.cc @@ -435,9 +435,8 @@ const PegInfo HttpModule::peg_names[PEG_COUNT_MAX+1] = { CountType::SUM, "uri_coding", "URIs with character coding problems" }, { CountType::NOW, "concurrent_sessions", "total concurrent http sessions" }, { CountType::MAX, "max_concurrent_sessions", "maximum concurrent http sessions" }, - { CountType::SUM, "detains_requested", "packet hold requests for detained inspection" }, { CountType::SUM, "script_detections", "early inspections of scripts in HTTP responses" }, - { CountType::SUM, "partial_inspections", "pre-inspections for detained inspection" }, + { CountType::SUM, "partial_inspections", "early inspections done for script detection" }, { CountType::SUM, "excess_parameters", "repeat parameters exceeding max" }, { CountType::SUM, "parameters", "HTTP parameters inspected" }, { CountType::SUM, "connect_tunnel_cutovers", "CONNECT tunnel flow cutovers to wizard" }, diff --git a/src/service_inspectors/http_inspect/http_test_input.cc b/src/service_inspectors/http_inspect/http_test_input.cc index 7cea7709a..f80da88e0 100644 --- a/src/service_inspectors/http_inspect/http_test_input.cc +++ b/src/service_inspectors/http_inspect/http_test_input.cc @@ -267,13 +267,6 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u length = 0; return; } - else if ((command_length == strlen("partial")) && !memcmp(command_value, - "partial", strlen("partial"))) - { - partial = true; - length = 0; - return; - } else if ((command_length > strlen("fileset")) && !memcmp(command_value, "fileset", strlen("fileset"))) { @@ -496,11 +489,9 @@ void HttpTestInput::flush(uint32_t num_octets) } void HttpTestInput::reassemble(uint8_t** buffer, unsigned& length, unsigned& total, - unsigned& offset, uint32_t& flags, SourceId source_id, bool& tcp_close, - bool& partial_flush) + unsigned& offset, uint32_t& flags, SourceId source_id, bool& tcp_close) { *buffer = nullptr; - partial_flush = false; tcp_close = false; // Only piggyback on data moving in the same direction. @@ -514,17 +505,9 @@ void HttpTestInput::reassemble(uint8_t** buffer, unsigned& length, unsigned& tot return; } - // Need flushed data unless it's a partial flush. - if (!flushed && !partial) - { - return; - } - - if (partial) + // Need flushed data + if (!flushed) { - // Give the caller a chance to set up for a partial flush before giving him the data - partial_flush = true; - partial = false; return; } diff --git a/src/service_inspectors/http_inspect/http_test_input.h b/src/service_inspectors/http_inspect/http_test_input.h index 00a2aeb7a..034d88b32 100644 --- a/src/service_inspectors/http_inspect/http_test_input.h +++ b/src/service_inspectors/http_inspect/http_test_input.h @@ -37,7 +37,7 @@ public: void scan(uint8_t*& data, uint32_t& length, HttpCommon::SourceId source_id, uint64_t seq_num); void flush(uint32_t num_octets); void reassemble(uint8_t** buffer, unsigned& length, unsigned& total, unsigned& offset, - uint32_t& flags, HttpCommon::SourceId source_id, bool& tcp_close, bool& partial_flush); + uint32_t& flags, HttpCommon::SourceId source_id, bool& tcp_close); bool finish(); private: @@ -66,9 +66,6 @@ private: // TCP connection directional close bool tcp_closed = false; - // partial flush requested, useful for testing detained inspection - bool partial = false; - // number of octets that have been flushed and must be sent by reassemble uint32_t flush_octets = 0;