From: Mike Stepanek (mstepane) Date: Mon, 27 Apr 2020 20:31:26 +0000 (+0000) Subject: Merge pull request #2175 in SNORT/snort3 from ~MDAGON/snort3:multi to master X-Git-Tag: 3.0.1-3~14 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8ff864f348efa8c25db051775b721646daf9106f;p=thirdparty%2Fsnort3.git Merge pull request #2175 in SNORT/snort3 from ~MDAGON/snort3:multi to master Squashed commit of the following: commit 5f3627d7056532a4388cf8a957a2785d28a789ea Author: mdagon Date: Fri Apr 10 13:53:59 2020 -0400 http2_inspect: support stream multiplexing --- diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.cc b/src/service_inspectors/http2_inspect/http2_data_cutter.cc index a9b956c54..cb8ea3a53 100644 --- a/src/service_inspectors/http2_inspect/http2_data_cutter.cc +++ b/src/service_inspectors/http2_inspect/http2_data_cutter.cc @@ -161,8 +161,7 @@ StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t* if (frame_flags & END_STREAM) { - finish_msg_body(session_data, source_id); - session_data->data_processing[source_id] = false; + finish_msg_body(); return StreamSplitter::FLUSH; } else @@ -191,21 +190,32 @@ const StreamBuffer Http2DataCutter::reassemble(const uint8_t* data, unsigned len cur_data = cur_padding = cur_data_offset = 0; unsigned cur_pos = 0; + while (cur_pos < len) { switch (reassemble_state) { case GET_FRAME_HDR: { - const uint32_t missing = FRAME_HEADER_LENGTH - reassemble_hdr_bytes_read; - const uint32_t cur_frame = ((len - cur_pos) < missing) ? (len - cur_pos) : missing; - memcpy(session_data->frame_header[source_id] + - session_data->frame_header_offset[source_id] + - reassemble_hdr_bytes_read, - data + cur_pos, cur_frame); - reassemble_hdr_bytes_read += cur_frame; - - cur_pos += cur_frame; + if (session_data->use_leftover_hdr[source_id]) + { + memcpy(session_data->frame_header[source_id], + session_data->leftover_hdr[source_id], FRAME_HEADER_LENGTH); + reassemble_hdr_bytes_read = FRAME_HEADER_LENGTH; + session_data->use_leftover_hdr[source_id] = false; + } + else + { + const uint32_t missing = FRAME_HEADER_LENGTH - reassemble_hdr_bytes_read; + const uint32_t cur_frame = ((len - cur_pos) < missing) ? (len - cur_pos) : missing; + memcpy(session_data->frame_header[source_id] + + session_data->frame_header_offset[source_id] + + reassemble_hdr_bytes_read, + data + cur_pos, cur_frame); + reassemble_hdr_bytes_read += cur_frame; + + cur_pos += cur_frame; + } if (reassemble_hdr_bytes_read == FRAME_HEADER_LENGTH) { @@ -278,6 +288,18 @@ const StreamBuffer Http2DataCutter::reassemble(const uint8_t* data, unsigned len if (reassemble_state == CLEANUP) { + Http2Stream* const stream = session_data->find_stream( + session_data->current_stream[source_id]); + if (bytes_sent_http == 0 && (reassemble_frame_flags & END_STREAM) && + stream->is_partial_buf_pending(source_id)) + { + // Received end of stream without new data. Flush pending partial buffer. + assert(frame_buf.data == nullptr); + unsigned unused; + frame_buf = session_data->hi_ss[source_id]->reassemble(session_data->flow, + 0, 0, nullptr, 0, PKT_PDU_TAIL, unused); + } + // Done with this packet, cleanup reassemble_state = GET_FRAME_HDR; reassemble_hdr_bytes_read = reassemble_data_bytes_read = reassemble_padding_read = @@ -295,3 +317,18 @@ const StreamBuffer Http2DataCutter::reassemble(const uint8_t* data, unsigned len return frame_buf; } +void Http2DataCutter::finish_msg_body() +{ + uint32_t http_flush_offset = 0; + Http2DummyPacket dummy_pkt; + dummy_pkt.flow = session_data->flow; + uint32_t unused = 0; + session_data->get_current_stream(source_id)->get_hi_flow_data()-> + finish_h2_body(source_id); + const snort::StreamSplitter::Status scan_result = session_data->hi_ss[source_id]->scan( + &dummy_pkt, nullptr, 0, unused, &http_flush_offset); + assert(scan_result == snort::StreamSplitter::FLUSH); + UNUSED(scan_result); + session_data->data_processing[source_id] = false; +} + diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.h b/src/service_inspectors/http2_inspect/http2_data_cutter.h index 69519500b..84d04aa02 100644 --- a/src/service_inspectors/http2_inspect/http2_data_cutter.h +++ b/src/service_inspectors/http2_inspect/http2_data_cutter.h @@ -83,6 +83,7 @@ private: bool http2_scan(const uint8_t* data, uint32_t length, uint32_t* flush_offset, uint32_t frame_len, uint8_t frame_flags, uint32_t& data_offset); snort::StreamSplitter::Status http_scan(const uint8_t* data, uint32_t* flush_offset); + void finish_msg_body(); }; #endif diff --git a/src/service_inspectors/http2_inspect/http2_data_frame.cc b/src/service_inspectors/http2_inspect/http2_data_frame.cc index 01a6c96f6..b3a18b8e6 100644 --- a/src/service_inspectors/http2_inspect/http2_data_frame.cc +++ b/src/service_inspectors/http2_inspect/http2_data_frame.cc @@ -38,15 +38,24 @@ Http2DataFrame::Http2DataFrame(const uint8_t* header_buffer, const int32_t heade HttpCommon::SourceId source_id_) : Http2Frame(header_buffer, header_len, nullptr, 0, session_data_, source_id_) { - Http2DummyPacket dummy_pkt; - dummy_pkt.flow = session_data->flow; - dummy_pkt.packet_flags = (source_id == SRC_CLIENT) ? PKT_FROM_CLIENT : PKT_FROM_SERVER; - dummy_pkt.dsize = data_len; - dummy_pkt.data = data_buffer; - dummy_pkt.xtradata_mask = 0; - session_data->hi->eval(&dummy_pkt); - detection_required = dummy_pkt.is_detection_required(); - xtradata_mask = dummy_pkt.xtradata_mask; + if ((data_len != 0) || !session_data->flushing_data[source_id]) + { + Http2DummyPacket dummy_pkt; + dummy_pkt.flow = session_data->flow; + dummy_pkt.packet_flags = (source_id == SRC_CLIENT) ? PKT_FROM_CLIENT : PKT_FROM_SERVER; + dummy_pkt.dsize = data_len; + dummy_pkt.data = data_buffer; + dummy_pkt.xtradata_mask = 0; + session_data->hi->eval(&dummy_pkt); + detection_required = dummy_pkt.is_detection_required(); + xtradata_mask = dummy_pkt.xtradata_mask; + } + else + { + detection_required = true; + HttpFlowData* const http_flow = (HttpFlowData*)session_data_->get_hi_flow_data(); + http_flow->reset_partial_flush(source_id_); + } } void Http2DataFrame::clear() diff --git a/src/service_inspectors/http2_inspect/http2_headers_frame.cc b/src/service_inspectors/http2_inspect/http2_headers_frame.cc index 78c5a3e9e..8cb71d6ae 100644 --- a/src/service_inspectors/http2_inspect/http2_headers_frame.cc +++ b/src/service_inspectors/http2_inspect/http2_headers_frame.cc @@ -78,7 +78,6 @@ Http2HeadersFrame::Http2HeadersFrame(const uint8_t* header_buffer, const int32_t return; // http_inspect scan() of start line - session_data->stream_in_hi = session_data->current_stream[source_id]; { uint32_t flush_offset; Http2DummyPacket dummy_pkt; diff --git a/src/service_inspectors/http2_inspect/http2_stream.h b/src/service_inspectors/http2_inspect/http2_stream.h index a409772a0..9189bad33 100644 --- a/src/service_inspectors/http2_inspect/http2_stream.h +++ b/src/service_inspectors/http2_inspect/http2_stream.h @@ -55,8 +55,14 @@ public: void set_end_stream(HttpCommon::SourceId source_id) { end_stream_set[source_id] = true; } bool end_stream_is_set(HttpCommon::SourceId source_id) { return end_stream_set[source_id]; } - void set_abort_on_data(HttpCommon::SourceId source_id) { abort_on_data[source_id] = true; } - bool abort_on_data_is_set(HttpCommon::SourceId source_id) { return abort_on_data[source_id]; } + + void set_partial_buf_pending(HttpCommon::SourceId source_id) + { partial_buf_pending[source_id] = true; } + void reset_partial_buf_pending(HttpCommon::SourceId source_id) + { partial_buf_pending[source_id] = false; } + bool is_partial_buf_pending(HttpCommon::SourceId source_id) + { return partial_buf_pending[source_id]; } + #ifdef REG_TEST void print_frame(FILE* output); #endif @@ -69,7 +75,8 @@ private: HttpMsgSection* hi_msg_section = nullptr; Http2DataCutter* data_cutter[2] = { nullptr, nullptr}; bool end_stream_set[2] = { false, false }; - bool abort_on_data[2] = { false, false}; + bool partial_buf_pending[2] = { false, false }; // used to indicate a partial buffer + // is pending from a previous partial flush }; #endif diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter.h b/src/service_inspectors/http2_inspect/http2_stream_splitter.h index d001e1bc3..892ef9a35 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter.h +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter.h @@ -52,8 +52,8 @@ private: static StreamSplitter::Status data_scan(Http2FlowData* session_data, const uint8_t* data, uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, uint32_t frame_length, uint8_t frame_flags, uint32_t& data_offset); - static void flush_data(Http2FlowData* session_data, HttpCommon::SourceId source_id, - uint32_t* flush_offset, uint32_t old_stream); + static void partial_flush_data(Http2FlowData* session_data, HttpCommon::SourceId source_id, + uint32_t* flush_offset, uint32_t data_offset, uint32_t old_stream); static StreamSplitter::Status non_data_scan(Http2FlowData* session_data, uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, uint32_t frame_length, uint8_t type, uint8_t frame_flags, uint32_t& data_offset); diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc index e1a691988..58aa1fdc8 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc @@ -69,8 +69,6 @@ StreamSplitter::Status Http2StreamSplitter::data_scan(Http2FlowData* session_dat uint32_t& data_offset) { Http2Stream* const stream = session_data->find_stream(session_data->current_stream[source_id]); - if (stream && stream->abort_on_data_is_set(source_id)) - return StreamSplitter::ABORT; if (!stream || stream->end_stream_is_set(source_id)) { @@ -132,9 +130,10 @@ StreamSplitter::Status Http2StreamSplitter::non_data_scan(Http2FlowData* session session_data->total_bytes_in_split[source_id] += FRAME_HEADER_LENGTH + frame_length; - // If the stream object exists and the end_stream flag is set, save that state in the stream - // object. If this is the first headers frame in the current stream,the stream object has - // not been created yet. The end_stream flag will be handled in the headers frame processing + // If the stream object exists and the end_stream flag is set, save that state in the + // stream object. If this is the first headers frame in the current stream, the stream + // object has not been created yet. The end_stream flag will be handled in the headers + // frame processing Http2Stream* const stream = session_data->find_stream( session_data->current_stream[source_id]); if (stream and frame_flags & END_STREAM) @@ -174,7 +173,7 @@ StreamSplitter::Status Http2StreamSplitter::non_data_scan(Http2FlowData* session } else { - // FIXIT-M CONTINUATION frames can also follow PUSH_PROMISE frames, which + // FIXIT-E CONTINUATION frames can also follow PUSH_PROMISE frames, which // are not currently supported *session_data->infractions[source_id] += INF_UNEXPECTED_CONTINUATION; session_data->events[source_id]->create_event( @@ -193,24 +192,21 @@ StreamSplitter::Status Http2StreamSplitter::non_data_scan(Http2FlowData* session return status; } -// Flush pending data. Save current non-data header for the next scan/reassemble. -void Http2StreamSplitter::flush_data(Http2FlowData* session_data, HttpCommon::SourceId source_id, - uint32_t* flush_offset, uint32_t old_stream) +// Flush pending data +void Http2StreamSplitter::partial_flush_data(Http2FlowData* session_data, HttpCommon::SourceId source_id, + uint32_t* flush_offset, uint32_t data_offset, uint32_t old_stream) { - session_data->current_stream[source_id] = old_stream; + session_data->current_stream[source_id] = session_data->stream_in_hi = old_stream; session_data->frame_type[source_id] = FT_DATA; Http2Stream* const stream = session_data->find_stream( session_data->current_stream[source_id]); Http2DataCutter* const data_cutter = stream->get_data_cutter(source_id); if (data_cutter->is_flush_required()) - finish_msg_body(session_data, source_id); + session_data->hi_ss[source_id]->init_partial_flush(session_data->flow); session_data->data_processing[source_id] = false; - *flush_offset = FRAME_HEADER_LENGTH; + *flush_offset = data_offset; session_data->flushing_data[source_id] = true; - memcpy(session_data->leftover_hdr[source_id], - session_data->scan_frame_header[source_id], FRAME_HEADER_LENGTH); session_data->num_frame_headers[source_id] -= 1; - stream->set_abort_on_data(source_id); } bool Http2StreamSplitter::read_frame_hdr(Http2FlowData* session_data, const uint8_t* data, @@ -308,19 +304,14 @@ StreamSplitter::Status Http2StreamSplitter::implement_scan(Http2FlowData* sessio const uint8_t frame_flags = get_frame_flags(session_data-> scan_frame_header[source_id]); const uint32_t old_stream = session_data->current_stream[source_id]; - session_data->current_stream[source_id] = + session_data->stream_in_hi = session_data->current_stream[source_id] = get_stream_id(session_data->scan_frame_header[source_id]); - if ((old_stream != session_data->current_stream[source_id]) && - session_data->data_processing[source_id] && type == FT_DATA) + if (session_data->data_processing[source_id] && + ((old_stream != session_data->current_stream[source_id] && type == FT_DATA) + || type != FT_DATA)) { - // FIXIT-E split by stream multiplexing not supported yet - return StreamSplitter::ABORT; - } - - if (session_data->data_processing[source_id] && type != FT_DATA) - { - flush_data(session_data, source_id, flush_offset, old_stream); + partial_flush_data(session_data, source_id, flush_offset, data_offset, old_stream); return StreamSplitter::FLUSH; } @@ -348,6 +339,8 @@ const StreamBuffer Http2StreamSplitter::implement_reassemble(Http2FlowData* sess assert(total <= MAX_OCTETS); StreamBuffer frame_buf { nullptr, 0 }; + Http2Stream* const stream = session_data->find_stream( + session_data->current_stream[source_id]); if (offset == 0) { @@ -361,30 +354,45 @@ const StreamBuffer Http2StreamSplitter::implement_reassemble(Http2FlowData* sess session_data->frame_header_offset[source_id] = 0; } - if (session_data->frame_type[source_id] == FT_DATA) + if (total == 0 && session_data->use_leftover_hdr[source_id]) + { + memcpy(session_data->frame_header[source_id], + session_data->leftover_hdr[source_id], FRAME_HEADER_LENGTH); + session_data->use_leftover_hdr[source_id] = false; + if (session_data->frame_type[source_id] == FT_DATA) + { + // Check if we reached end of stream and have a partial buffer pending + const uint8_t frame_flags = get_frame_flags(session_data->frame_header[source_id]); + if ((frame_flags & END_STREAM) && stream->is_partial_buf_pending(source_id)) + { + unsigned copied; + StreamBuffer http_frame_buf = session_data->hi_ss[source_id]->reassemble( + session_data->flow, + 0, 0, nullptr, 0, PKT_PDU_TAIL, copied); + session_data->frame_data[source_id] = const_cast(http_frame_buf.data); + session_data->frame_data_size[source_id] = http_frame_buf.length; + } + } + } + else if (session_data->frame_type[source_id] == FT_DATA) { if (session_data->flushing_data[source_id] && (flags & PKT_PDU_TAIL)) len -= FRAME_HEADER_LENGTH; if (len != 0) { - Http2Stream* const stream = session_data->find_stream( - session_data->current_stream[source_id]); Http2DataCutter* data_cutter = stream->get_data_cutter(source_id); StreamBuffer http_frame_buf = data_cutter->reassemble(data, len); if (http_frame_buf.data) { session_data->frame_data[source_id] = const_cast(http_frame_buf.data); session_data->frame_data_size[source_id] = http_frame_buf.length; + if (!session_data->flushing_data[source_id] && stream->is_partial_buf_pending( + source_id)) + stream->reset_partial_buf_pending(source_id); } } } - else if (total == 0 && session_data->use_leftover_hdr[source_id]) - { - memcpy(session_data->frame_header[source_id], - session_data->leftover_hdr[source_id], FRAME_HEADER_LENGTH); - session_data->use_leftover_hdr[source_id] = false; - } else { uint32_t data_offset = 0; @@ -504,6 +512,14 @@ const StreamBuffer Http2StreamSplitter::implement_reassemble(Http2FlowData* sess session_data->num_frame_headers[source_id] = 0; session_data->scan_octets_seen[source_id] = 0; + if (session_data->flushing_data[source_id]) + { + stream->set_partial_buf_pending(source_id); + // save current header for next scan/reassemble + memcpy(session_data->leftover_hdr[source_id], + session_data->scan_frame_header[source_id], FRAME_HEADER_LENGTH); + } + // Return 0-length non-null buffer to stream which signals detection required, but don't // create pkt_data buffer frame_buf.data = (const uint8_t*)""; diff --git a/src/service_inspectors/http2_inspect/http2_utils.cc b/src/service_inspectors/http2_inspect/http2_utils.cc index 79a2f0330..f8cb32ae9 100644 --- a/src/service_inspectors/http2_inspect/http2_utils.cc +++ b/src/service_inspectors/http2_inspect/http2_utils.cc @@ -21,9 +21,6 @@ #include -#include "service_inspectors/http_inspect/http_flow_data.h" -#include "service_inspectors/http_inspect/http_stream_splitter.h" - #include "http2_dummy_packet.h" #include "http2_enum.h" @@ -63,17 +60,3 @@ uint8_t get_stream_id(const uint8_t* frame_header_buffer) frame_header_buffer[stream_id_index + 3]; } -void finish_msg_body(Http2FlowData* session_data, HttpCommon::SourceId source_id) -{ - uint32_t http_flush_offset = 0; - Http2DummyPacket dummy_pkt; - dummy_pkt.flow = session_data->flow; - uint32_t unused = 0; - session_data->get_current_stream(source_id)->get_hi_flow_data()-> - finish_h2_body(source_id); - const snort::StreamSplitter::Status scan_result = session_data->hi_ss[source_id]->scan( - &dummy_pkt, nullptr, 0, unused, &http_flush_offset); - assert(scan_result == snort::StreamSplitter::FLUSH); - UNUSED(scan_result); -} - diff --git a/src/service_inspectors/http2_inspect/http2_utils.h b/src/service_inspectors/http2_inspect/http2_utils.h index 3954f69fd..351682c7f 100644 --- a/src/service_inspectors/http2_inspect/http2_utils.h +++ b/src/service_inspectors/http2_inspect/http2_utils.h @@ -37,7 +37,4 @@ uint8_t get_frame_type(const uint8_t* frame_header_buffer); uint8_t get_frame_flags(const uint8_t* frame_header_buffer); uint8_t get_stream_id(const uint8_t* frame_header_buffer); - -void finish_msg_body(Http2FlowData* session_data, HttpCommon::SourceId source_id); - #endif diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h index ef34a5086..d057ae280 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.h +++ b/src/service_inspectors/http_inspect/http_flow_data.h @@ -74,6 +74,9 @@ public: void finish_h2_body(HttpCommon::SourceId source_id) { h2_body_finished[source_id] = true; } + void reset_partial_flush(HttpCommon::SourceId source_id) + { partial_flush[source_id] = false; } + private: // HTTP/2 handling bool for_http2 = false; diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc index 1e36d4f33..ade06bc9f 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc @@ -167,7 +167,8 @@ bool HttpStreamSplitter::init_partial_flush(Flow* flow) assert(session_data != nullptr); assert((session_data->type_expected[source_id] == SEC_BODY_CL) || (session_data->type_expected[source_id] == SEC_BODY_OLD) || - (session_data->type_expected[source_id] == SEC_BODY_CHUNK)); + (session_data->type_expected[source_id] == SEC_BODY_CHUNK) || + (session_data->type_expected[source_id] == SEC_BODY_H2)); #ifdef REG_TEST if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP) &&