From eaf3d1c296c893c590f5709b36d78292ea83d3bb Mon Sep 17 00:00:00 2001 From: "Mike Stepanek (mstepane)" Date: Wed, 18 Mar 2020 11:41:37 +0000 Subject: [PATCH] Merge pull request #2062 in SNORT/snort3 from ~MDAGON/snort3:h2i_pt3 to master Squashed commit of the following: commit 4ef91cac5ae0967b79a057bbc11828098c55d694 Author: mdagon Date: Wed Feb 26 16:09:04 2020 -0500 http2_inspect: support single data frame sent to http, multiple flushes --- .../http2_inspect/http2_data_cutter.cc | 78 ++++--- .../http2_inspect/http2_data_cutter.h | 12 +- .../http2_inspect/http2_flow_data.h | 15 +- .../http2_stream_splitter_impl.cc | 217 ++++++++---------- 4 files changed, 161 insertions(+), 161 deletions(-) diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.cc b/src/service_inspectors/http2_inspect/http2_data_cutter.cc index b18f77177..87f3b013c 100644 --- a/src/service_inspectors/http2_inspect/http2_data_cutter.cc +++ b/src/service_inspectors/http2_inspect/http2_data_cutter.cc @@ -60,7 +60,7 @@ bool Http2DataCutter::http2_scan(const uint8_t* data, uint32_t length, *flush_offset = FRAME_HEADER_LENGTH; } - uint32_t cur_pos = 0; + uint32_t cur_pos = leftover_bytes; while ((cur_pos < length) && (data_state != FULL_FRAME)) { @@ -110,7 +110,7 @@ bool Http2DataCutter::http2_scan(const uint8_t* data, uint32_t length, } } - frame_bytes_seen += cur_pos; + frame_bytes_seen += (cur_pos - leftover_bytes); session_data->scan_remaining_frame_octets[source_id] = frame_length - frame_bytes_seen; *flush_offset += cur_pos; @@ -142,17 +142,30 @@ StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t* if (scan_result != StreamSplitter::SEARCH) return StreamSplitter::ABORT; } - } // fallthrough + } // fallthrough case HEADER_SENT: { - if (cur_data) + if (cur_data || leftover_bytes) { scan_result = session_data->hi_ss[source_id]->scan(&dummy_pkt, data + cur_data_offset, - cur_data, unused, &http_flush_offset); - bytes_sent_http += cur_data; + cur_data + leftover_bytes, unused, &http_flush_offset); if (scan_result != StreamSplitter::SEARCH) - return StreamSplitter::ABORT; + { + if (scan_result == StreamSplitter::FLUSH) + { + bytes_sent_http += http_flush_offset; + leftover_bytes = cur_data + leftover_bytes - http_flush_offset; + *flush_offset -= leftover_bytes; + session_data->mid_packet[source_id] = true; + return scan_result; + } + else + return StreamSplitter::ABORT; + } + + bytes_sent_http += (cur_data + leftover_bytes); + leftover_bytes = 0; } if (data_state == FULL_FRAME) { @@ -161,7 +174,7 @@ StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t* 5, unused, &http_flush_offset); bytes_sent_http +=5; assert(scan_result == StreamSplitter::FLUSH); - + session_data->mid_packet[source_id] = false; session_data->scan_octets_seen[source_id] = 0; session_data->scan_remaining_frame_octets[source_id] = 0; } @@ -180,18 +193,14 @@ StreamSplitter::Status Http2DataCutter::scan(const uint8_t* data, uint32_t lengt if (!http2_scan(data, length, flush_offset)) return StreamSplitter::ABORT; - return Http2DataCutter::http_scan(data, flush_offset); + return http_scan(data, flush_offset); } -const StreamBuffer Http2DataCutter::reassemble(unsigned total, unsigned offset, const - uint8_t* data, unsigned len) +const StreamBuffer Http2DataCutter::reassemble(unsigned, const uint8_t* data, + unsigned len) { StreamBuffer frame_buf { nullptr, 0 }; - if (offset == 0) - { - padding_read = data_bytes_read = hdr_bytes_read = 0; - } cur_data = cur_padding = cur_data_offset = 0; unsigned cur_pos = 0; @@ -201,18 +210,19 @@ const StreamBuffer Http2DataCutter::reassemble(unsigned total, unsigned offset, { case SKIP_FRAME_HDR: { - if (hdr_bytes_read == 0) + if (reassemble_hdr_bytes_read == 0) { session_data->frame_header[source_id] = new uint8_t[FRAME_HEADER_LENGTH]; session_data->frame_header_size[source_id] = FRAME_HEADER_LENGTH; } - const uint32_t missing = FRAME_HEADER_LENGTH - hdr_bytes_read; + const uint32_t missing = FRAME_HEADER_LENGTH - reassemble_hdr_bytes_read; const uint32_t cur_frame = ((len - cur_pos) < missing) ? (len - cur_pos) : missing; - memcpy(session_data->frame_header[source_id] + hdr_bytes_read, data + cur_pos, + memcpy(session_data->frame_header[source_id] + reassemble_hdr_bytes_read, data + + cur_pos, cur_frame); - hdr_bytes_read += cur_frame; + reassemble_hdr_bytes_read += cur_frame; cur_pos += cur_frame; - if (hdr_bytes_read == FRAME_HEADER_LENGTH) + if (reassemble_hdr_bytes_read == FRAME_HEADER_LENGTH) { cur_data_offset = cur_pos; reassemble_state = (padding_len) ? SKIP_PADDING_LEN : SEND_CHUNK_HDR; @@ -233,34 +243,38 @@ const StreamBuffer Http2DataCutter::reassemble(unsigned total, unsigned offset, bytes_sent_http, 0, (const uint8_t*)chunk_hdr.c_str(), chunk_hdr.length(), 0, copied); assert(copied == (unsigned)chunk_hdr.length()); + reassemble_bytes_sent += copied; reassemble_state = SEND_DATA; - } // fallthrough + } // fallthrough case SEND_DATA: { - const uint32_t missing = data_len - data_bytes_read; + const uint32_t missing = data_len - reassemble_data_bytes_read; cur_data = ((len - cur_pos) >= missing) ? missing : (len - cur_pos); - data_bytes_read += cur_data; + reassemble_data_bytes_read += cur_data; cur_pos += cur_data; unsigned copied; + uint32_t flags = (bytes_sent_http == (cur_data + reassemble_bytes_sent)) ? + PKT_PDU_TAIL : 0; frame_buf = session_data->hi_ss[source_id]->reassemble(session_data->flow, bytes_sent_http, 0, data + cur_data_offset, cur_data, - 0, copied); + flags, copied); assert(copied == (unsigned)cur_data); + reassemble_bytes_sent += copied; - if (data_bytes_read == data_len) + if (reassemble_data_bytes_read == data_len) reassemble_state = (padding_len) ? SKIP_PADDING : SEND_CRLF; break; } case SKIP_PADDING: { - const uint32_t missing = padding_len - padding_read; + const uint32_t missing = padding_len - reassemble_padding_read; cur_padding = ((len - cur_pos) >= missing) ? missing : (len - cur_pos); cur_pos += cur_padding; - padding_read += cur_padding; - if (padding_read == padding_len) + reassemble_padding_read += cur_padding; + if (reassemble_padding_read == padding_len) reassemble_state = SEND_CRLF; break; } @@ -270,19 +284,19 @@ const StreamBuffer Http2DataCutter::reassemble(unsigned total, unsigned offset, } } - if (len + offset == total) - assert(reassemble_state == SEND_CRLF); - if (reassemble_state == SEND_CRLF) { unsigned copied; frame_buf = session_data->hi_ss[source_id]->reassemble(session_data->flow, bytes_sent_http, 0,(const unsigned char*)"\r\n0\r\n", 5, PKT_PDU_TAIL, copied); assert(copied == 5); + } - assert(frame_buf.data != nullptr); + if (frame_buf.data != nullptr) + { session_data->frame_data[source_id] = const_cast (frame_buf.data); session_data->frame_data_size[source_id] = frame_buf.length; + bytes_sent_http = reassemble_bytes_sent = 0; } return frame_buf; diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.h b/src/service_inspectors/http2_inspect/http2_data_cutter.h index 9e010603e..1e79cbfb6 100644 --- a/src/service_inspectors/http2_inspect/http2_data_cutter.h +++ b/src/service_inspectors/http2_inspect/http2_data_cutter.h @@ -33,7 +33,7 @@ public: src_id, bool is_padded); snort::StreamSplitter::Status scan(const uint8_t* data, uint32_t length, uint32_t* flush_offset); - const snort::StreamBuffer reassemble(unsigned total, unsigned offset, const uint8_t* data, + const snort::StreamBuffer reassemble(unsigned total, const uint8_t* data, unsigned len); private: @@ -45,16 +45,22 @@ private: const uint32_t frame_length; uint32_t data_len; uint32_t padding_len = 0; - // accumulating + // accumulating - scan uint32_t frame_bytes_seen = 0; uint32_t bytes_sent_http = 0; - uint32_t hdr_bytes_read = 0; uint32_t data_bytes_read = 0; uint32_t padding_read = 0; + // accumulating - reassemble + uint32_t reassemble_bytes_sent = 0; + uint32_t reassemble_hdr_bytes_read = 0; + uint32_t reassemble_data_bytes_read = 0; + uint32_t reassemble_padding_read = 0; // per call uint32_t cur_data; uint32_t cur_padding; uint32_t cur_data_offset; + // leftover from previous scan call + uint32_t leftover_bytes = 0 ; // // State machines diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.h b/src/service_inspectors/http2_inspect/http2_flow_data.h index ae7c8b3ce..d0b6895cd 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.h +++ b/src/service_inspectors/http2_inspect/http2_flow_data.h @@ -74,11 +74,15 @@ public: friend class Http2Stream; friend class Http2StreamSplitter; friend snort::StreamSplitter::Status data_scan(Http2FlowData* session_data, const uint8_t* data, - uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, uint32_t frame_length, bool is_padded); + uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, + uint32_t frame_length, bool is_padded); friend const snort::StreamBuffer implement_reassemble(Http2FlowData*, unsigned, unsigned, const uint8_t*, unsigned, uint32_t, HttpCommon::SourceId); friend snort::StreamSplitter::Status implement_scan(Http2FlowData*, const uint8_t*, uint32_t, uint32_t*, HttpCommon::SourceId); + friend snort::StreamSplitter::Status non_data_scan(Http2FlowData* session_data, + uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, + uint32_t frame_length, uint8_t type, uint8_t frame_flags, uint32_t& data_offset); size_t size_of() override { return sizeof(*this); } @@ -86,7 +90,7 @@ public: // Stream access class StreamInfo { - public: +public: const uint32_t id; class Http2Stream* stream; @@ -97,9 +101,10 @@ public: uint32_t get_current_stream_id(const HttpCommon::SourceId source_id); Http2HpackDecoder* get_hpack_decoder(const HttpCommon::SourceId source_id) - { return &hpack_decoder[source_id]; } + { return &hpack_decoder[source_id]; } Http2ConnectionSettings* get_connection_settings(const HttpCommon::SourceId source_id) - { return &connection_settings[source_id]; } + { return &connection_settings[source_id]; } + protected: snort::Flow* flow; HttpInspect* const hi; @@ -131,7 +136,7 @@ protected: uint8_t scan_frame_header[2][Http2Enums::FRAME_HEADER_LENGTH]; uint32_t scan_remaining_frame_octets[2] = { 0, 0 }; uint32_t scan_octets_seen[2] = { 0, 0 }; - uint32_t leftover_data[2] = { 0, 0 }; + bool mid_packet[2] = { false, false }; // Scan signals to reassemble() bool payload_discard[2] = { false, false }; diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc index b7cb538f3..16e35dfde 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc @@ -101,6 +101,83 @@ StreamSplitter::Status data_scan(Http2FlowData* session_data, const uint8_t* dat return data_cutter->scan(data, length, flush_offset); } +StreamSplitter::Status non_data_scan(Http2FlowData* session_data, + uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, + uint32_t frame_length, uint8_t type, uint8_t frame_flags, uint32_t& data_offset) +{ + // Compute frame section length once per frame + if (session_data->scan_remaining_frame_octets[source_id] == 0) + { + if (session_data->continuation_expected[source_id] && type != FT_CONTINUATION) + { + *session_data->infractions[source_id] += INF_MISSING_CONTINUATION; + session_data->events[source_id]->create_event(EVENT_MISSING_CONTINUATION); + return StreamSplitter::ABORT; + } + + if (frame_length + FRAME_HEADER_LENGTH > MAX_OCTETS) + { + // FIXIT-M long non-data frame needs to be supported + return StreamSplitter::ABORT; + } + + session_data->scan_remaining_frame_octets[source_id] = frame_length; + session_data->total_bytes_in_split[source_id] += FRAME_HEADER_LENGTH + + frame_length; + } + + // If we don't have the full frame, keep scanning + if (length - data_offset < session_data->scan_remaining_frame_octets[source_id]) + { + session_data->scan_remaining_frame_octets[source_id] -= (length - data_offset); + data_offset = length; + return StreamSplitter::SEARCH; + } + + // Have the full frame + StreamSplitter::Status status = StreamSplitter::FLUSH; + switch (type) + { + case FT_HEADERS: + if (!(frame_flags & END_HEADERS)) + { + session_data->continuation_expected[source_id] = true; + status = StreamSplitter::SEARCH; + } + break; + case FT_CONTINUATION: + if (session_data->continuation_expected[source_id]) + { + if (!(frame_flags & END_HEADERS)) + status = StreamSplitter::SEARCH; + else + { + // continuation frame ending headers + status = StreamSplitter::FLUSH; + session_data->continuation_expected[source_id] = false; + } + } + else + { + // FIXIT-M CONTINUATION frames can also follow PUSH_PROMISE frames, which + // are not currently supported + *session_data->infractions[source_id] += INF_UNEXPECTED_CONTINUATION; + session_data->events[source_id]->create_event( + EVENT_UNEXPECTED_CONTINUATION); + status = StreamSplitter::ABORT; + } + break; + default: + break; + } + + data_offset += session_data->scan_remaining_frame_octets[source_id]; + *flush_offset = data_offset; + session_data->scan_octets_seen[source_id] = 0; + session_data->scan_remaining_frame_octets[source_id] = 0; + return status; +} + StreamSplitter::Status implement_scan(Http2FlowData* session_data, const uint8_t* data, uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id) { @@ -112,7 +189,11 @@ StreamSplitter::Status implement_scan(Http2FlowData* session_data, const uint8_t switch (validate_preface(data, length, session_data->scan_octets_seen[source_id])) { case V_GOOD: - break; + *flush_offset = 24 - session_data->scan_octets_seen[source_id]; + session_data->preface[source_id] = false; + session_data->payload_discard[source_id] = true; + session_data->scan_octets_seen[source_id] = 0; + return StreamSplitter::FLUSH; case V_BAD: session_data->events[source_id]->create_event(EVENT_PREFACE_MATCH_FAILURE); return StreamSplitter::ABORT; @@ -120,46 +201,14 @@ StreamSplitter::Status implement_scan(Http2FlowData* session_data, const uint8_t session_data->scan_octets_seen[source_id] += length; return StreamSplitter::SEARCH; } - - *flush_offset = 24 - session_data->scan_octets_seen[source_id]; - session_data->preface[source_id] = false; - session_data->payload_discard[source_id] = true; - session_data->scan_octets_seen[source_id] = 0; } - //FIXIT-M This should get split points from NHI - else if (session_data->leftover_data[source_id] > 0) + else if (session_data->mid_packet[source_id]) { // Continuation of ongoing data frame - session_data->num_frame_headers[source_id] = 0; - - // If this is a new frame section, update next frame section length - if (session_data->scan_remaining_frame_octets[source_id] == 0) - { - if (session_data->leftover_data[source_id] > DATA_SECTION_SIZE) - session_data->scan_remaining_frame_octets[source_id] = DATA_SECTION_SIZE; - else - session_data->scan_remaining_frame_octets[source_id] = - session_data->leftover_data[source_id]; - session_data->total_bytes_in_split[source_id] = 0; - } - - // Don't have full frame section, keep scanning - if (session_data->scan_remaining_frame_octets[source_id] > length) - { - session_data->scan_remaining_frame_octets[source_id] -= length; - session_data->total_bytes_in_split[source_id] += length; - return status = StreamSplitter::SEARCH; - } - - // Have full frame section, flush and update leftover - session_data->total_bytes_in_split[source_id] += - session_data->scan_remaining_frame_octets[source_id]; - *flush_offset = session_data->scan_remaining_frame_octets[source_id]; - session_data->leftover_data[source_id] -= - session_data->total_bytes_in_split[source_id]; - session_data->octets_before_first_header[source_id] = - session_data->total_bytes_in_split[source_id]; - session_data->scan_remaining_frame_octets[source_id] = 0; + Http2Stream* const stream = session_data->find_stream( + session_data->current_stream[source_id]); + Http2DataCutter* data_cutter = stream->get_data_cutter(source_id); + return data_cutter->scan(data, length, flush_offset); } else { @@ -171,9 +220,11 @@ StreamSplitter::Status implement_scan(Http2FlowData* session_data, const uint8_t // need to process multiple frames in a single scan do { - // Scanning a new frame if (session_data->scan_octets_seen[source_id] == 0) + { + // Scanning a new frame session_data->num_frame_headers[source_id] += 1; + } // The first nine bytes are the frame header. But all nine might not all be present in // the first TCP segment we receive. @@ -188,17 +239,14 @@ StreamSplitter::Status implement_scan(Http2FlowData* session_data, const uint8_t data_offset += remaining_header_in_data; if (session_data->scan_octets_seen[source_id] < FRAME_HEADER_LENGTH) - { - status = StreamSplitter::SEARCH; - break; - } + return StreamSplitter::SEARCH; // We have the full frame header, compute some variables const uint32_t frame_length = get_frame_length(session_data-> scan_frame_header[source_id]); const uint8_t type = session_data->frame_type[source_id] = get_frame_type( session_data->scan_frame_header[source_id]); - uint8_t frame_flags = get_frame_flags(session_data-> + const uint8_t frame_flags = get_frame_flags(session_data-> scan_frame_header[source_id]); session_data->current_stream[source_id] = get_stream_id(session_data->scan_frame_header[source_id]); @@ -206,80 +254,9 @@ StreamSplitter::Status implement_scan(Http2FlowData* session_data, const uint8_t if (type == FT_DATA) return data_scan(session_data, data, length, flush_offset, source_id, frame_length, ((frame_flags & PADDED) !=0)); - - // Compute frame section length once per frame - if (session_data->scan_remaining_frame_octets[source_id] == 0) - { - if (session_data->continuation_expected[source_id] && type != FT_CONTINUATION) - { - *session_data->infractions[source_id] += INF_MISSING_CONTINUATION; - session_data->events[source_id]->create_event(EVENT_MISSING_CONTINUATION); - status = StreamSplitter::ABORT; - break; - } - - if (frame_length + FRAME_HEADER_LENGTH > MAX_OCTETS) - { - // FIXIT-M long non-data frame needs to be supported - status = StreamSplitter::ABORT; - break; - } - else - { - session_data->scan_remaining_frame_octets[source_id] = frame_length; - session_data->total_bytes_in_split[source_id] += FRAME_HEADER_LENGTH + - frame_length; - } - } - - // If we don't have the full frame, keep scanning - if (length - data_offset < session_data->scan_remaining_frame_octets[source_id]) - { - session_data->scan_remaining_frame_octets[source_id] -= (length - data_offset); - status = StreamSplitter::SEARCH; - break; - } - - // Have the full frame - switch (type) - { - case FT_HEADERS: - if (!(frame_flags & END_HEADERS)) - { - session_data->continuation_expected[source_id] = true; - status = StreamSplitter::SEARCH; - } - break; - case FT_CONTINUATION: - if (session_data->continuation_expected[source_id]) - { - if (!(frame_flags & END_HEADERS)) - status = StreamSplitter::SEARCH; - else - { - // continuation frame ending headers - status = StreamSplitter::FLUSH; - session_data->continuation_expected[source_id] = false; - } - } - else - { - // FIXIT-M CONTINUATION frames can also follow PUSH_PROMISE frames, which - // are not currently supported - *session_data->infractions[source_id] += INF_UNEXPECTED_CONTINUATION; - session_data->events[source_id]->create_event( - EVENT_UNEXPECTED_CONTINUATION); - status = StreamSplitter::ABORT; - } - break; - default: - break; - } - - data_offset += session_data->scan_remaining_frame_octets[source_id]; - *flush_offset = data_offset; - session_data->scan_octets_seen[source_id] = 0; - session_data->scan_remaining_frame_octets[source_id] = 0; + else + status = non_data_scan(session_data, length, flush_offset, source_id, + frame_length, type, frame_flags, data_offset); } while (status == StreamSplitter::SEARCH && data_offset < length); } @@ -304,11 +281,9 @@ const StreamBuffer implement_reassemble(Http2FlowData* session_data, unsigned to Http2Stream* const stream = session_data->find_stream( session_data->current_stream[source_id]); Http2DataCutter* data_cutter = stream->get_data_cutter(source_id); - StreamBuffer http_frame_buf = data_cutter->reassemble(total, offset, data, len); + StreamBuffer http_frame_buf = data_cutter->reassemble(total, data, len); if (http_frame_buf.data) { - delete data_cutter; - stream->set_data_cutter(nullptr, source_id); stream->set_abort_data_processing(source_id); session_data->frame_data[source_id] = const_cast(http_frame_buf.data); session_data->frame_data_size[source_id] = http_frame_buf.length; -- 2.47.3