From 11cc8af7c1f3b82954aefdcbae7a8f3b852b3f0b Mon Sep 17 00:00:00 2001 From: "Mike Stepanek (mstepane)" Date: Fri, 27 Mar 2020 19:32:59 +0000 Subject: [PATCH] Merge pull request #2101 in SNORT/snort3 from ~MDAGON/snort3:h2i_pt4 to master Squashed commit of the following: commit 9942a2e7ebd578c2c0715646e09f3357026083a7 Author: mdagon Date: Tue Mar 17 10:36:25 2020 -0400 http2_inspect: multiple data frames support --- .../http2_inspect/http2_data_cutter.cc | 109 +++++++-------- .../http2_inspect/http2_data_cutter.h | 18 ++- .../http2_inspect/http2_flow_data.h | 9 +- .../http2_inspect/http2_stream.h | 7 - .../http2_stream_splitter_impl.cc | 132 ++++++++++-------- 5 files changed, 137 insertions(+), 138 deletions(-) diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.cc b/src/service_inspectors/http2_inspect/http2_data_cutter.cc index 6f6852db8..f97e483cf 100644 --- a/src/service_inspectors/http2_inspect/http2_data_cutter.cc +++ b/src/service_inspectors/http2_inspect/http2_data_cutter.cc @@ -40,9 +40,10 @@ Http2DataCutter::Http2DataCutter(Http2FlowData* _session_data, HttpCommon::Sourc // Scan data frame, extract information needed for http scan. // http scan will need the data only, stripped of padding and header. bool Http2DataCutter::http2_scan(const uint8_t* data, uint32_t length, - uint32_t* flush_offset, uint32_t frame_len, uint8_t flags) + uint32_t* flush_offset, uint32_t frame_len, uint8_t flags, uint32_t& data_offset) { - *flush_offset = cur_data_offset = cur_data = cur_padding = 0; + cur_data_offset = data_offset; + cur_data = cur_padding = 0; if (frame_bytes_seen == 0) { @@ -50,13 +51,11 @@ bool Http2DataCutter::http2_scan(const uint8_t* data, uint32_t length, padding_len = data_bytes_read = padding_read = 0; frame_flags = flags; frame_bytes_seen = cur_data_offset = FRAME_HEADER_LENGTH; - length -= FRAME_HEADER_LENGTH; *flush_offset = FRAME_HEADER_LENGTH; data_state = ((frame_flags & PADDED) !=0) ? PADDING_LENGTH : DATA; } - uint32_t cur_pos = leftover_bytes; - + uint32_t cur_pos = data_offset + leftover_bytes; while ((cur_pos < length) && (data_state != FULL_FRAME)) { switch (data_state) @@ -105,10 +104,9 @@ bool Http2DataCutter::http2_scan(const uint8_t* data, uint32_t length, } } - frame_bytes_seen += (cur_pos - leftover_bytes); + frame_bytes_seen += (cur_pos - leftover_bytes - data_offset); + *flush_offset = data_offset = cur_pos; session_data->scan_remaining_frame_octets[source_id] = frame_length - frame_bytes_seen; - *flush_offset += cur_pos; - return true; } @@ -132,7 +130,8 @@ StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t* bytes_sent_http += http_flush_offset; leftover_bytes = cur_data + leftover_bytes - http_flush_offset; *flush_offset -= leftover_bytes; - session_data->mid_packet[source_id] = ( leftover_bytes > 0 ) ? true : false; + if (leftover_bytes || data_state != FULL_FRAME) + session_data->mid_data_frame[source_id] = true; } else if (scan_result == StreamSplitter::SEARCH) { @@ -146,47 +145,40 @@ StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t* { if (leftover_bytes == 0) { - session_data->get_current_stream(source_id)->get_hi_flow_data()-> - set_http2_end_stream(source_id); - scan_result = session_data->hi_ss[source_id]->scan(&dummy_pkt, nullptr, 0, unused, - &http_flush_offset); - assert(scan_result == StreamSplitter::FLUSH); - - // FIXIT-H for now only a single data frame is processed - Http2Stream* const stream = session_data->find_stream( - session_data->current_stream[source_id]); - stream->set_abort_data_processing(source_id); - // Done with this frame, cleanup - session_data->mid_packet[source_id] = false; + session_data->mid_data_frame[source_id] = false; session_data->scan_octets_seen[source_id] = 0; session_data->scan_remaining_frame_octets[source_id] = 0; frame_bytes_seen = 0; + + if (frame_flags & END_STREAM) + { + session_data->get_current_stream(source_id)->get_hi_flow_data()-> + set_http2_end_stream(source_id); + scan_result = session_data->hi_ss[source_id]->scan(&dummy_pkt, nullptr, 0, unused, + &http_flush_offset); + assert(scan_result == StreamSplitter::FLUSH); + } + else + session_data->data_processing[source_id] = true; } } if (scan_result != StreamSplitter::FLUSH) *flush_offset = 0; - return scan_result; } StreamSplitter::Status Http2DataCutter::scan(const uint8_t* data, uint32_t length, - uint32_t* flush_offset, uint32_t frame_len, uint8_t frame_flags) + uint32_t* flush_offset, uint32_t& data_offset, uint32_t frame_len, uint8_t frame_flags) { - // FIXIT-H temporary, until more than 1 data frame sent to http inspect is supported - Http2Stream* const stream = session_data->find_stream(session_data->current_stream[source_id]); - if (stream->get_abort_data_processing(source_id)) - return StreamSplitter::ABORT; - - if (!http2_scan(data, length, flush_offset, frame_len, frame_flags)) + if (!http2_scan(data, length, flush_offset, frame_len, frame_flags, data_offset)) return StreamSplitter::ABORT; return http_scan(data, flush_offset); } -const StreamBuffer Http2DataCutter::reassemble(unsigned, const uint8_t* data, - unsigned len) +const StreamBuffer Http2DataCutter::reassemble(const uint8_t* data, unsigned len) { StreamBuffer frame_buf { nullptr, 0 }; @@ -199,41 +191,40 @@ const StreamBuffer Http2DataCutter::reassemble(unsigned, const uint8_t* data, { case GET_FRAME_HDR: { - if (reassemble_hdr_bytes_read == 0) - { - session_data->frame_header[source_id] = new uint8_t[FRAME_HEADER_LENGTH]; - session_data->frame_header_size[source_id] = FRAME_HEADER_LENGTH; - padding_len = 0; - } - const uint32_t missing = FRAME_HEADER_LENGTH - reassemble_hdr_bytes_read; const uint32_t cur_frame = ((len - cur_pos) < missing) ? (len - cur_pos) : missing; - memcpy(session_data->frame_header[source_id] + reassemble_hdr_bytes_read, data + - cur_pos, - cur_frame); + memcpy(session_data->frame_header[source_id] + + session_data->frame_header_offset[source_id] + + reassemble_hdr_bytes_read, + data + cur_pos, cur_frame); reassemble_hdr_bytes_read += cur_frame; + cur_pos += cur_frame; if (reassemble_hdr_bytes_read == FRAME_HEADER_LENGTH) { - data_len = frame_length = get_frame_length(session_data->frame_header[source_id]); - frame_flags = get_frame_flags(session_data->frame_header[source_id]); + reassemble_data_len = get_frame_length(session_data->frame_header[source_id]+ + session_data->frame_header_offset[source_id]); + reassemble_frame_flags = get_frame_flags(session_data->frame_header[source_id]+ + session_data->frame_header_offset[source_id]); cur_data_offset = cur_pos; - reassemble_state = ((frame_flags & PADDED) !=0) ? GET_PADDING_LEN : SEND_DATA; + reassemble_state = ((reassemble_frame_flags & PADDED) !=0) ? GET_PADDING_LEN : + SEND_DATA; + session_data->frame_header_offset[source_id] += FRAME_HEADER_LENGTH; } break; } case GET_PADDING_LEN: - padding_len = *(data + cur_pos); - data_len -= (padding_len + 1); + reassemble_padding_len = *(data + cur_pos); + reassemble_data_len -= (reassemble_padding_len + 1); cur_pos++; cur_data_offset++; reassemble_state = SEND_DATA; break; case SEND_DATA: { - const uint32_t missing = data_len - reassemble_data_bytes_read; + const uint32_t missing = reassemble_data_len - reassemble_data_bytes_read; cur_data = ((len - cur_pos) >= missing) ? missing : (len - cur_pos); reassemble_data_bytes_read += cur_data; cur_pos += cur_data; @@ -247,33 +238,35 @@ const StreamBuffer Http2DataCutter::reassemble(unsigned, const uint8_t* data, assert(copied == (unsigned)cur_data); reassemble_bytes_sent += copied; - if (reassemble_data_bytes_read == data_len) - reassemble_state = (padding_len) ? SKIP_PADDING : CLEANUP; + if (reassemble_data_bytes_read == reassemble_data_len) + reassemble_state = (reassemble_padding_len) ? SKIP_PADDING : CLEANUP; break; } case SKIP_PADDING: { - const uint32_t missing = padding_len - reassemble_padding_read; + const uint32_t missing = reassemble_padding_len - reassemble_padding_read; cur_padding = ((len - cur_pos) >= missing) ? missing : (len - cur_pos); cur_pos += cur_padding; reassemble_padding_read += cur_padding; - if (reassemble_padding_read == padding_len) + + if (reassemble_padding_read == reassemble_padding_len) reassemble_state = CLEANUP; + break; } - default: break; } - } - if (reassemble_state == CLEANUP) - { - // Done with this packet - reassemble_state = GET_FRAME_HDR; - reassemble_hdr_bytes_read = reassemble_data_bytes_read = reassemble_padding_read = 0; + if (reassemble_state == CLEANUP) + { + // Done with this packet, cleanup + reassemble_state = GET_FRAME_HDR; + reassemble_hdr_bytes_read = reassemble_data_bytes_read = reassemble_padding_read = + reassemble_padding_len = 0; + } } if (frame_buf.data != nullptr) diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.h b/src/service_inspectors/http2_inspect/http2_data_cutter.h index fcdb34bd4..571b4c0fd 100644 --- a/src/service_inspectors/http2_inspect/http2_data_cutter.h +++ b/src/service_inspectors/http2_inspect/http2_data_cutter.h @@ -31,16 +31,16 @@ class Http2DataCutter public: Http2DataCutter(Http2FlowData* flow_data, HttpCommon::SourceId src_id); snort::StreamSplitter::Status scan(const uint8_t* data, uint32_t length, - uint32_t* flush_offset, uint32_t frame_len =0, uint8_t frame_flags =0); - const snort::StreamBuffer reassemble(unsigned total, const uint8_t* data, - unsigned len); + uint32_t* flush_offset, uint32_t& data_offset, uint32_t frame_len =0, + uint8_t frame_flags =0); + const snort::StreamBuffer reassemble(const uint8_t* data, unsigned len); private: Http2FlowData* const session_data; const HttpCommon::SourceId source_id; - // total per frame + // total per frame - scan uint32_t frame_length; uint32_t data_len; uint32_t padding_len = 0; @@ -50,6 +50,12 @@ private: uint32_t bytes_sent_http = 0; uint32_t data_bytes_read = 0; uint32_t padding_read = 0; + // leftover from previous scan call + uint32_t leftover_bytes = 0; + // total per frame - reassemble + uint32_t reassemble_data_len; + uint32_t reassemble_padding_len = 0; + uint8_t reassemble_frame_flags; // accumulating - reassemble uint32_t reassemble_bytes_sent = 0; uint32_t reassemble_hdr_bytes_read = 0; @@ -59,8 +65,6 @@ private: uint32_t cur_data; uint32_t cur_padding; uint32_t cur_data_offset; - // leftover from previous scan call - uint32_t leftover_bytes = 0 ; // // State machines @@ -75,7 +79,7 @@ private: enum ReassembleState reassemble_state = GET_FRAME_HDR; bool http2_scan(const uint8_t* data, uint32_t length, uint32_t* flush_offset, - uint32_t frame_len, uint8_t frame_flags); + uint32_t frame_len, uint8_t frame_flags, uint32_t& data_offset); snort::StreamSplitter::Status http_scan(const uint8_t* data, uint32_t* flush_offset); }; diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.h b/src/service_inspectors/http2_inspect/http2_flow_data.h index b5b2900d8..d57677758 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.h +++ b/src/service_inspectors/http2_inspect/http2_flow_data.h @@ -73,9 +73,9 @@ public: friend class Http2StatusLine; friend class Http2Stream; friend class Http2StreamSplitter; - friend snort::StreamSplitter::Status data_scan(Http2FlowData* session_data, const uint8_t* data, - uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, - uint32_t frame_length, uint8_t frame_flags); + friend snort::StreamSplitter::Status data_scan(Http2FlowData* session_data, const + uint8_t* data, uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, + uint32_t frame_length, uint8_t frame_flags, uint32_t& data_offset); friend const snort::StreamBuffer implement_reassemble(Http2FlowData*, unsigned, unsigned, const uint8_t*, unsigned, uint32_t, HttpCommon::SourceId); friend snort::StreamSplitter::Status implement_scan(Http2FlowData*, const uint8_t*, uint32_t, @@ -136,7 +136,8 @@ protected: uint8_t scan_frame_header[2][Http2Enums::FRAME_HEADER_LENGTH]; uint32_t scan_remaining_frame_octets[2] = { 0, 0 }; uint32_t scan_octets_seen[2] = { 0, 0 }; - bool mid_packet[2] = { false, false }; + bool mid_data_frame[2] = { false, false }; //set for data frame with multiple flushes + bool data_processing[2] = { false, false }; // Scan signals to reassemble() bool payload_discard[2] = { false, false }; diff --git a/src/service_inspectors/http2_inspect/http2_stream.h b/src/service_inspectors/http2_inspect/http2_stream.h index d1215480e..5938614cf 100644 --- a/src/service_inspectors/http2_inspect/http2_stream.h +++ b/src/service_inspectors/http2_inspect/http2_stream.h @@ -53,12 +53,6 @@ public: void set_data_cutter(Http2DataCutter* cutter, HttpCommon::SourceId source_id) { data_cutter[source_id] = cutter; } - bool get_abort_data_processing(HttpCommon::SourceId source_id) const - { return abort_data_processing[source_id]; } - - void set_abort_data_processing(HttpCommon::SourceId source_id) - { abort_data_processing[source_id] = true; } - #ifdef REG_TEST void print_frame(FILE* output); #endif @@ -70,7 +64,6 @@ private: HttpFlowData* hi_flow_data = nullptr; HttpMsgSection* hi_msg_section = nullptr; Http2DataCutter* data_cutter[2] = { nullptr, nullptr}; - bool abort_data_processing[2] = {false, false}; }; #endif diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc index 213f81689..3401d0ecf 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc @@ -39,20 +39,14 @@ using namespace snort; using namespace HttpCommon; using namespace Http2Enums; - StreamSplitter::Status data_scan(Http2FlowData* session_data, const uint8_t* data, uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, - uint32_t frame_length, uint8_t frame_flags) + uint32_t frame_length, uint8_t frame_flags, uint32_t& data_offset) { Http2Stream* const stream = session_data->find_stream(session_data->current_stream[source_id]); HttpFlowData* http_flow = nullptr; if (stream) - { http_flow = (HttpFlowData*)stream->get_hi_flow_data(); - // temporary, till more than 1 data frame sent to http inspect is supported - if (stream->get_abort_data_processing(source_id)) - return StreamSplitter::ABORT; - } if (!stream || !http_flow || (frame_length > 0 and (http_flow->get_type_expected(source_id) != HttpEnums::SEC_BODY_H2))) @@ -66,13 +60,17 @@ StreamSplitter::Status data_scan(Http2FlowData* session_data, const uint8_t* dat return StreamSplitter::ABORT; Http2DataCutter* data_cutter = stream->get_data_cutter(source_id); - return data_cutter->scan(data, length, flush_offset, frame_length, frame_flags); + return data_cutter->scan(data, length, flush_offset, data_offset, frame_length, frame_flags); } StreamSplitter::Status non_data_scan(Http2FlowData* session_data, uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, uint32_t frame_length, uint8_t type, uint8_t frame_flags, uint32_t& data_offset) { + // FIXIT-E - temporary. Will force data frame flush instead + if (session_data->data_processing[source_id]) + return StreamSplitter::ABORT; + // Compute frame section length once per frame if (session_data->scan_remaining_frame_octets[source_id] == 0) { @@ -170,61 +168,66 @@ StreamSplitter::Status implement_scan(Http2FlowData* session_data, const uint8_t return StreamSplitter::SEARCH; } } - else if (session_data->mid_packet[source_id]) - { - // Continuation of ongoing data frame - Http2Stream* const stream = session_data->find_stream( - session_data->current_stream[source_id]); - Http2DataCutter* data_cutter = stream->get_data_cutter(source_id); - return data_cutter->scan(data, length, flush_offset); - } else { - // Frame with header *flush_offset = 0; uint32_t data_offset = 0; session_data->octets_before_first_header[source_id] = 0; - // If there is a header frame followed by a continuation frame in the same tcp segment, - // need to process multiple frames in a single scan + + // If there is a header frame followed by a continuation frame in the same tcp segment or + // several data frames need to process multiple frames in a single scan do { - if (session_data->scan_octets_seen[source_id] == 0) + if (session_data->mid_data_frame[source_id]) { - // Scanning a new frame - session_data->num_frame_headers[source_id] += 1; + // Continuation of ongoing data frame + Http2Stream* const stream = session_data->find_stream( + session_data->current_stream[source_id]); + Http2DataCutter* data_cutter = stream->get_data_cutter(source_id); + status = data_cutter->scan(data, length, flush_offset, data_offset); } - - // The first nine bytes are the frame header. But all nine might not all be present in - // the first TCP segment we receive. - const uint32_t remaining_header = FRAME_HEADER_LENGTH - - session_data->scan_octets_seen[source_id]; - const uint32_t remaining_header_in_data = remaining_header > length - data_offset ? - length - data_offset : remaining_header; - memcpy(session_data->scan_frame_header[source_id] + - session_data->scan_octets_seen[source_id], data + data_offset, - remaining_header_in_data); - session_data->scan_octets_seen[source_id] += remaining_header_in_data; - data_offset += remaining_header_in_data; - - if (session_data->scan_octets_seen[source_id] < FRAME_HEADER_LENGTH) - return StreamSplitter::SEARCH; - - // We have the full frame header, compute some variables - const uint32_t frame_length = get_frame_length(session_data-> - scan_frame_header[source_id]); - const uint8_t type = session_data->frame_type[source_id] = get_frame_type( - session_data->scan_frame_header[source_id]); - const uint8_t frame_flags = get_frame_flags(session_data-> - scan_frame_header[source_id]); - session_data->current_stream[source_id] = - get_stream_id(session_data->scan_frame_header[source_id]); - - if (type == FT_DATA) - return data_scan(session_data, data, length, flush_offset, source_id, - frame_length, frame_flags); else - status = non_data_scan(session_data, length, flush_offset, source_id, - frame_length, type, frame_flags, data_offset); + { + // Frame with header + if (session_data->scan_octets_seen[source_id] == 0) + { + // Scanning a new frame + session_data->num_frame_headers[source_id] += 1; + } + + // The first nine bytes are the frame header. But all nine might not all be present + // in + // the first TCP segment we receive. + const uint32_t remaining_header = FRAME_HEADER_LENGTH - + session_data->scan_octets_seen[source_id]; + const uint32_t remaining_header_in_data = remaining_header > length - data_offset ? + length - data_offset : remaining_header; + memcpy(session_data->scan_frame_header[source_id] + + session_data->scan_octets_seen[source_id], data + data_offset, + remaining_header_in_data); + session_data->scan_octets_seen[source_id] += remaining_header_in_data; + data_offset += remaining_header_in_data; + + if (session_data->scan_octets_seen[source_id] < FRAME_HEADER_LENGTH) + return StreamSplitter::SEARCH; + + // We have the full frame header, compute some variables + const uint32_t frame_length = get_frame_length(session_data-> + scan_frame_header[source_id]); + const uint8_t type = session_data->frame_type[source_id] = get_frame_type( + session_data->scan_frame_header[source_id]); + const uint8_t frame_flags = get_frame_flags(session_data-> + scan_frame_header[source_id]); + session_data->current_stream[source_id] = + get_stream_id(session_data->scan_frame_header[source_id]); + + if (type == FT_DATA) + status = data_scan(session_data, data, length, flush_offset, source_id, + frame_length, frame_flags, data_offset); + else + status = non_data_scan(session_data, length, flush_offset, source_id, + frame_length, type, frame_flags, data_offset); + } } while (status == StreamSplitter::SEARCH && data_offset < length); } @@ -243,12 +246,24 @@ const StreamBuffer implement_reassemble(Http2FlowData* session_data, unsigned to StreamBuffer frame_buf { nullptr, 0 }; + if (offset == 0) + { + // This is the first reassemble() for this frame and we need to allocate some buffers + session_data->frame_header_size[source_id] = FRAME_HEADER_LENGTH * + session_data->num_frame_headers[source_id]; + if (session_data->frame_header_size[source_id] > 0) + session_data->frame_header[source_id] = + new uint8_t[session_data->frame_header_size[source_id]]; + + session_data->frame_header_offset[source_id] = 0; + } + if (session_data->frame_type[source_id] == FT_DATA) { Http2Stream* const stream = session_data->find_stream( session_data->current_stream[source_id]); Http2DataCutter* data_cutter = stream->get_data_cutter(source_id); - StreamBuffer http_frame_buf = data_cutter->reassemble(total, data, len); + StreamBuffer http_frame_buf = data_cutter->reassemble(data, len); if (http_frame_buf.data) { session_data->frame_data[source_id] = const_cast(http_frame_buf.data); @@ -262,20 +277,13 @@ const StreamBuffer implement_reassemble(Http2FlowData* session_data, unsigned to if (offset == 0) { - // This is the first reassemble() for this frame and we need to allocate some buffers - session_data->frame_header_size[source_id] = FRAME_HEADER_LENGTH * - session_data->num_frame_headers[source_id]; - if (session_data->frame_header_size[source_id] > 0) - session_data->frame_header[source_id] = new uint8_t[ - session_data->frame_header_size[source_id]]; - + // This is the first reassemble() for this frame - allocate data buffer session_data->frame_data_size[source_id]= total - session_data->frame_header_size[source_id]; if (session_data->frame_data_size[source_id] > 0) session_data->frame_data[source_id] = new uint8_t[ session_data->frame_data_size[source_id]]; - session_data->frame_header_offset[source_id] = 0; session_data->frame_data_offset[source_id] = 0; session_data->remaining_frame_octets[source_id] = session_data->octets_before_first_header[source_id]; -- 2.47.3