From: Mike Stepanek (mstepane) Date: Wed, 28 Oct 2020 13:56:37 +0000 (+0000) Subject: Merge pull request #2575 in SNORT/snort3 from ~THOPETER/snort3:h2i13 to master X-Git-Tag: 3.0.3-5~29 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=5b5e1c41bd76d1eaf613ccf9f986321ff2a6541e;p=thirdparty%2Fsnort3.git Merge pull request #2575 in SNORT/snort3 from ~THOPETER/snort3:h2i13 to master Squashed commit of the following: commit 0a30ffd77476eb92a410880dbb53769f37496fd1 Author: Tom Peters Date: Thu Oct 8 19:17:09 2020 -0400 http2_inspect: Data frame redesign --- diff --git a/src/payload_injector/test/payload_injector_test.cc b/src/payload_injector/test/payload_injector_test.cc index 6e0c1fccf..a63966390 100644 --- a/src/payload_injector/test/payload_injector_test.cc +++ b/src/payload_injector/test/payload_injector_test.cc @@ -111,6 +111,8 @@ InjectionReturnStatus PayloadInjectorModule::get_http2_payload(InjectionControl, unsigned Http2FlowData::inspector_id = 0; Http2Stream::~Http2Stream() { } HpackDynamicTable::~HpackDynamicTable() { } +Http2DataCutter::Http2DataCutter(Http2FlowData* _session_data, HttpCommon::SourceId src_id) : + session_data(_session_data), source_id(src_id) { } Http2FlowData::Http2FlowData(snort::Flow*) : FlowData(inspector_id), flow(nullptr), @@ -119,10 +121,13 @@ Http2FlowData::Http2FlowData(snort::Flow*) : { Http2HpackDecoder(this, SRC_CLIENT, events[SRC_CLIENT], infractions[SRC_CLIENT]), Http2HpackDecoder(this, SRC_SERVER, events[SRC_SERVER], infractions[SRC_SERVER]) - } + }, + data_cutter { Http2DataCutter(this, SRC_CLIENT), Http2DataCutter(this, SRC_SERVER) } { } Http2FlowData::~Http2FlowData() { } Http2FlowData http2_flow_data(nullptr); +void Http2FlowData::set_mid_frame(bool val) { continuation_expected[SRC_SERVER] = val; } +bool Http2FlowData::is_mid_frame() const { return continuation_expected[SRC_SERVER]; } FlowData* snort::Flow::get_flow_data(unsigned int) const { return &http2_flow_data; } TEST_GROUP(payload_injector_test) @@ -143,8 +148,7 @@ TEST_GROUP(payload_injector_test) control.http_page_len = 4; flow.set_state(Flow::FlowState::INSPECT); translation_status = INJECTION_SUCCESS; - http2_flow_data.set_continuation_expected(SRC_SERVER, false); - http2_flow_data.set_reading_frame(SRC_SERVER, false); + http2_flow_data.set_mid_frame(false); } }; @@ -290,7 +294,7 @@ TEST(payload_injector_test, http2_mid_frame) flow.gadget = new MockInspector(); p.flow = &flow; control.stream_id = 1; - http2_flow_data.set_reading_frame(SRC_SERVER, true); + http2_flow_data.set_mid_frame(true); InjectionReturnStatus status = mod.inject_http_payload(&p, control); CHECK(counts->http2_mid_frame == 1); CHECK(status == ERR_HTTP2_MID_FRAME); @@ -310,7 +314,7 @@ TEST(payload_injector_test, http2_continuation_expected) flow.gadget = new MockInspector(); p.flow = &flow; control.stream_id = 1; - http2_flow_data.set_continuation_expected(SRC_SERVER, true); + http2_flow_data.set_mid_frame(true); InjectionReturnStatus status = mod.inject_http_payload(&p, control); CHECK(counts->http2_mid_frame == 1); CHECK(status == ERR_HTTP2_MID_FRAME); @@ -360,8 +364,7 @@ TEST_GROUP(payload_injector_translate_err_test) control.http_page = (const uint8_t*)"test"; control.http_page_len = 4; flow.set_state(Flow::FlowState::INSPECT); - http2_flow_data.set_continuation_expected(SRC_SERVER, false); - http2_flow_data.set_reading_frame(SRC_SERVER, false); + http2_flow_data.set_mid_frame(false); mod.set_configured(true); mock_api.base.name = "http2_inspect"; flow.gadget = new MockInspector(); diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.cc b/src/service_inspectors/http2_inspect/http2_data_cutter.cc index 5ea64fa58..e8f6a1507 100644 --- a/src/service_inspectors/http2_inspect/http2_data_cutter.cc +++ b/src/service_inspectors/http2_inspect/http2_data_cutter.cc @@ -34,109 +34,64 @@ using namespace HttpCommon; using namespace Http2Enums; Http2DataCutter::Http2DataCutter(Http2FlowData* _session_data, HttpCommon::SourceId src_id) : - session_data(_session_data), source_id(src_id) + session_data(_session_data), source_id(src_id) { } -// Scan data frame, extract information needed for http scan. -// http scan will need the data only, stripped of padding and header. -bool Http2DataCutter::http2_scan(uint32_t length, uint32_t* flush_offset, uint32_t frame_len, - uint8_t flags, uint32_t& data_offset) +void Http2DataCutter::http2_scan(uint32_t length, uint32_t* flush_offset, uint32_t frame_len, + bool padded, uint32_t& data_offset) { cur_data_offset = data_offset; - cur_data = cur_padding = 0; if (frame_bytes_seen == 0) { - frame_length = data_len = frame_len; - padding_len = data_bytes_read = padding_read = 0; - frame_flags = flags; - *flush_offset = frame_bytes_seen = FRAME_HEADER_LENGTH; + data_len = frame_len; + data_bytes_read = 0; + frame_bytes_seen = FRAME_HEADER_LENGTH; - if (frame_flags & PADDED) + if (padded) { - padding_len = session_data->padding_length[source_id]; - data_len -= (padding_len + 1); + data_len -= 1; frame_bytes_seen += 1; } - - if (data_len) - data_state = DATA; - else if (padding_len) - data_state = PADDING; - else - data_state = FULL_FRAME; } - uint32_t cur_pos = data_offset + leftover_bytes + leftover_padding; - while ((cur_pos < length) && (data_state != FULL_FRAME)) - { - switch (data_state) - { - case DATA: - { - const uint32_t missing = data_len - data_bytes_read; - cur_data = ((length - cur_pos) >= missing) ? - missing : (length - cur_pos); - data_bytes_read += cur_data; - cur_pos += cur_data; - if (data_bytes_read == data_len) - data_state = padding_len ? PADDING : FULL_FRAME; - break; - } - case PADDING: - { - const uint32_t missing = padding_len - padding_read; - cur_padding = ((length - cur_pos) >= missing) ? - missing : (length - cur_pos); - cur_pos += cur_padding; - padding_read += cur_padding; - if (padding_read == padding_len) - data_state = FULL_FRAME; - break; - } - default: - break; - } - } - - if (data_state == FULL_FRAME) - session_data->reading_frame[source_id] = false; - - //FIXIT-E shouldn't need both scan_remaining_frame_octets and frame_bytes_seen - frame_bytes_seen += (cur_pos - leftover_bytes - data_offset - leftover_padding); - *flush_offset = data_offset = cur_pos; - session_data->scan_remaining_frame_octets[source_id] = frame_length - frame_bytes_seen; - return true; + uint32_t cur_pos = data_offset; + const uint32_t missing = data_len - data_bytes_read; + cur_data = (missing <= (length - cur_pos)) ? missing : (length - cur_pos); + data_bytes_read += cur_data; + cur_pos += cur_data; + + frame_bytes_seen += cur_pos - data_offset; + // FIXIT-L In theory data_offset should be reduced for any bytes not used by HI scan() later. + // But currently the value is not used in the case where we flush. + data_offset = cur_pos; + *flush_offset = cur_pos; + session_data->scan_remaining_frame_octets[source_id] = frame_len - frame_bytes_seen; } -// Call http scan. After all data in first frame has been sent, set http2_end_stream flag and send -// zero-length buffer to flush through detection -StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t* flush_offset) +StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t* flush_offset, + bool end_stream) { StreamSplitter::Status scan_result = StreamSplitter::SEARCH; - if (cur_data || leftover_bytes) + if (cur_data > 0) { uint32_t http_flush_offset = 0; Http2DummyPacket dummy_pkt; dummy_pkt.flow = session_data->flow; uint32_t unused = 0; scan_result = session_data->hi_ss[source_id]->scan(&dummy_pkt, data + cur_data_offset, - cur_data + leftover_bytes, unused, &http_flush_offset); + cur_data, unused, &http_flush_offset); if (scan_result == StreamSplitter::FLUSH) { bytes_sent_http += http_flush_offset; - leftover_bytes = cur_data + leftover_bytes - http_flush_offset; - if (leftover_bytes != 0 && cur_padding != 0) - leftover_padding = cur_padding; - else - leftover_padding = 0; - *flush_offset -= leftover_bytes + leftover_padding; + const uint32_t unused_input = cur_data - http_flush_offset; + data_bytes_read -= unused_input; + *flush_offset -= unused_input; } else if (scan_result == StreamSplitter::SEARCH) { - bytes_sent_http += (cur_data + leftover_bytes); - leftover_bytes = leftover_padding = 0; + bytes_sent_http += cur_data; } else if (scan_result == StreamSplitter::ABORT) // FIXIT-E eventually need to implement continued processing. We cannot abort just @@ -145,32 +100,36 @@ StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t* // can parse the framing and process most streams it is reasonable to continue. return StreamSplitter::ABORT; } - if (data_state == FULL_FRAME) + + if (data_bytes_read == data_len) { - if (leftover_bytes == 0) - { - // Done with this frame, cleanup - session_data->scan_octets_seen[source_id] = 0; - session_data->scan_remaining_frame_octets[source_id] = 0; - session_data->scan_state[source_id] = SCAN_HEADER; - frame_bytes_seen = 0; + // Done with this frame, cleanup + session_data->scan_octets_seen[source_id] = 0; + session_data->scan_remaining_frame_octets[source_id] = 0; + session_data->scan_state[source_id] = SCAN_FRAME_HEADER; + frame_bytes_seen = 0; - if (frame_flags & END_STREAM) + if (end_stream) + { + Http2Stream* const stream = session_data->find_stream( + session_data->current_stream[source_id]); + stream->finish_msg_body(source_id, false, !bytes_sent_http); + // FIXIT-E this flag seems to mean both END_STREAM and the end of this frame + stream->set_end_stream_on_data_flush(source_id); + return StreamSplitter::FLUSH; + } + else if (scan_result != StreamSplitter::FLUSH) + { + assert(scan_result == StreamSplitter::SEARCH); + scan_result = StreamSplitter::FLUSH; + if (cur_data > 0) + session_data->hi_ss[source_id]->init_partial_flush(session_data->flow); + else { - Http2Stream* const stream = session_data->find_stream( - session_data->current_stream[source_id]); - bool clear_partial_buffers = false; - if (!bytes_sent_http) - clear_partial_buffers = true; - stream->finish_msg_body(source_id, false, clear_partial_buffers); - - // Since there may be multiple frame headers or zero frame headers in the flushed - // data section, remember END_STREAM flag in the stream object - stream->set_end_stream_on_data_flush(source_id); - return StreamSplitter::FLUSH; + session_data->payload_discard[source_id] = true; + assert(!session_data->frame_lengths[source_id].empty()); + session_data->frame_lengths[source_id].pop(); } - else if (scan_result != StreamSplitter::FLUSH) - session_data->data_processing[source_id] = true; } } @@ -182,149 +141,115 @@ StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t* StreamSplitter::Status Http2DataCutter::scan(const uint8_t* data, uint32_t length, uint32_t* flush_offset, uint32_t& data_offset, uint32_t frame_len, uint8_t frame_flags) { - if (!http2_scan(length, flush_offset, frame_len, frame_flags, data_offset)) - return StreamSplitter::ABORT; + http2_scan(length, flush_offset, frame_len, frame_flags & PADDED, data_offset); session_data->stream_in_hi = session_data->current_stream[source_id]; - StreamSplitter::Status status = http_scan(data, flush_offset); + StreamSplitter::Status status = http_scan(data, flush_offset, frame_flags & END_STREAM); session_data->stream_in_hi = NO_STREAM_ID; return status; } -const StreamBuffer Http2DataCutter::reassemble(const uint8_t* data, unsigned len) +void Http2DataCutter::reassemble(const uint8_t* data, unsigned len) { session_data->stream_in_hi = session_data->current_stream[source_id]; - StreamBuffer frame_buf { nullptr, 0 }; - - cur_data = cur_padding = cur_data_offset = 0; + cur_data = cur_data_offset = 0; unsigned cur_pos = 0; - while (cur_pos < len) + while ((cur_pos < len) || (reassemble_state == SEND_EMPTY_DATA)) { switch (reassemble_state) { case GET_FRAME_HDR: { - if (session_data->use_leftover_hdr[source_id]) - { - memcpy(session_data->frame_header[source_id], - session_data->leftover_hdr[source_id], FRAME_HEADER_LENGTH); - reassemble_hdr_bytes_read = FRAME_HEADER_LENGTH; - session_data->use_leftover_hdr[source_id] = false; - cur_pos++; - } - else - { - const uint32_t missing = FRAME_HEADER_LENGTH - reassemble_hdr_bytes_read; - const uint32_t cur_frame = ((len - cur_pos) < missing) ? (len - cur_pos) : missing; - memcpy(session_data->frame_header[source_id] + - session_data->frame_header_offset[source_id] + - reassemble_hdr_bytes_read, - data + cur_pos, cur_frame); - reassemble_hdr_bytes_read += cur_frame; - - cur_pos += cur_frame; - } + const uint32_t missing = FRAME_HEADER_LENGTH - reassemble_hdr_bytes_read; + const uint32_t cur_frame = ((len - cur_pos) < missing) ? (len - cur_pos) : missing; + reassemble_hdr_bytes_read += cur_frame; + cur_pos += cur_frame; if (reassemble_hdr_bytes_read == FRAME_HEADER_LENGTH) { - reassemble_data_len = get_frame_length(session_data->frame_header[source_id]+ - session_data->frame_header_offset[source_id]); - reassemble_frame_flags = get_frame_flags(session_data->frame_header[source_id]+ - session_data->frame_header_offset[source_id]); + assert(!session_data->frame_lengths[source_id].empty()); + reassemble_data_len = session_data->frame_lengths[source_id].front(); + session_data->frame_lengths[source_id].pop(); + reassemble_frame_flags = + get_frame_flags(session_data->lead_frame_header[source_id]); cur_data_offset = cur_pos; - session_data->frame_header_offset[source_id] += FRAME_HEADER_LENGTH; - if (reassemble_data_len == 0) - reassemble_state = CLEANUP; - else if ((reassemble_frame_flags & PADDED) !=0) + if ((reassemble_frame_flags & PADDED) != 0) reassemble_state = GET_PADDING_LEN; - else + else if (reassemble_data_len > 0) reassemble_state = SEND_DATA; + else if (reassemble_frame_flags & END_STREAM) + reassemble_state = SEND_EMPTY_DATA; + else + reassemble_state = DONE; } - break; } case GET_PADDING_LEN: - reassemble_padding_len = *(data + cur_pos); - reassemble_data_len -= (reassemble_padding_len + 1); + { + const uint8_t padding_len = *(data + cur_pos); + reassemble_data_len -= padding_len + 1; cur_pos++; cur_data_offset++; - if (reassemble_data_len) + if (reassemble_data_len > 0) reassemble_state = SEND_DATA; - else if (reassemble_padding_len) - reassemble_state = SKIP_PADDING; + else if (reassemble_frame_flags & END_STREAM) + reassemble_state = SEND_EMPTY_DATA; else - reassemble_state = CLEANUP; + reassemble_state = DONE; break; + } + case SEND_EMPTY_DATA: case SEND_DATA: { + reassemble_state = SEND_DATA; const uint32_t missing = reassemble_data_len - reassemble_data_bytes_read; cur_data = ((len - cur_pos) >= missing) ? missing : (len - cur_pos); reassemble_data_bytes_read += cur_data; cur_pos += cur_data; unsigned copied; - uint32_t flags = (bytes_sent_http == (cur_data + reassemble_bytes_sent)) ? + const uint32_t flags = (bytes_sent_http == (cur_data + reassemble_bytes_sent)) ? PKT_PDU_TAIL : 0; - frame_buf = session_data->hi_ss[source_id]->reassemble(session_data->flow, + StreamBuffer frame_buf = session_data->hi_ss[source_id]->reassemble(session_data->flow, bytes_sent_http, 0, data + cur_data_offset, cur_data, flags, copied); assert(copied == (unsigned)cur_data); - reassemble_bytes_sent += copied; - if (reassemble_data_bytes_read == reassemble_data_len) - reassemble_state = (reassemble_padding_len) ? SKIP_PADDING : CLEANUP; - - break; - } - case SKIP_PADDING: - { - const uint32_t missing = reassemble_padding_len - reassemble_padding_read; - cur_padding = ((len - cur_pos) >= missing) ? - missing : (len - cur_pos); - cur_pos += cur_padding; - reassemble_padding_read += cur_padding; + if (frame_buf.data != nullptr) + { + session_data->frame_data[source_id] = const_cast (frame_buf.data); + session_data->frame_data_size[source_id] = frame_buf.length; + bytes_sent_http = 0; + reassemble_bytes_sent = 0; + } + else + reassemble_bytes_sent += copied; - if (reassemble_padding_read == reassemble_padding_len) - reassemble_state = CLEANUP; + if (reassemble_data_bytes_read == reassemble_data_len) + reassemble_state = DONE; break; } - default: + case DONE: + assert(false); break; } - - if (reassemble_state == CLEANUP) - { - Http2Stream* const stream = session_data->find_stream( - session_data->current_stream[source_id]); - if (bytes_sent_http == 0 && (reassemble_frame_flags & END_STREAM) && - stream->is_partial_buf_pending(source_id)) - { - // Received end of stream without new data. Flush pending partial buffer. - assert(frame_buf.data == nullptr); - unsigned unused; - frame_buf = session_data->hi_ss[source_id]->reassemble(session_data->flow, - 0, 0, nullptr, 0, PKT_PDU_TAIL, unused); - } - - // Done with this packet, cleanup - reassemble_state = GET_FRAME_HDR; - reassemble_hdr_bytes_read = reassemble_data_bytes_read = reassemble_padding_read = - reassemble_padding_len = 0; - } - } - - if (frame_buf.data != nullptr) - { - session_data->frame_data[source_id] = const_cast (frame_buf.data); - session_data->frame_data_size[source_id] = frame_buf.length; - bytes_sent_http = reassemble_bytes_sent = 0; } session_data->stream_in_hi = NO_STREAM_ID; + return; +} - return frame_buf; +void Http2DataCutter::reset() +{ + frame_bytes_seen = 0; + bytes_sent_http = 0; + reassemble_bytes_sent = 0; + reassemble_hdr_bytes_read = 0; + reassemble_data_bytes_read = 0; + reassemble_state = GET_FRAME_HDR; } + diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.h b/src/service_inspectors/http2_inspect/http2_data_cutter.h index 1451e7c23..28d3a8f56 100644 --- a/src/service_inspectors/http2_inspect/http2_data_cutter.h +++ b/src/service_inspectors/http2_inspect/http2_data_cutter.h @@ -24,66 +24,48 @@ #include "stream/stream_splitter.h" #include "http2_enum.h" -#include "http2_flow_data.h" + +class Http2FlowData; class Http2DataCutter { public: Http2DataCutter(Http2FlowData* flow_data, HttpCommon::SourceId src_id); snort::StreamSplitter::Status scan(const uint8_t* data, uint32_t length, - uint32_t* flush_offset, uint32_t& data_offset, uint32_t frame_len =0, - uint8_t frame_flags =0); - const snort::StreamBuffer reassemble(const uint8_t* data, unsigned len); - - bool is_flush_required() { return bytes_sent_http != 0; } + uint32_t* flush_offset, uint32_t& data_offset, uint32_t frame_len, + uint8_t frame_flags); + void reassemble(const uint8_t* data, unsigned len); + void reset(); private: - Http2FlowData* const session_data; const HttpCommon::SourceId source_id; // total per frame - scan - uint32_t frame_length; uint32_t data_len; - uint8_t padding_len; - uint8_t frame_flags; // accumulating - scan uint32_t frame_bytes_seen = 0; uint32_t bytes_sent_http = 0; uint32_t data_bytes_read; - uint32_t padding_read; - // leftover from previous scan call - uint32_t leftover_bytes = 0; - uint32_t leftover_padding = 0; // total per frame - reassemble uint32_t reassemble_data_len; - uint32_t reassemble_padding_len = 0; uint8_t reassemble_frame_flags; // accumulating - reassemble uint32_t reassemble_bytes_sent = 0; uint32_t reassemble_hdr_bytes_read = 0; uint32_t reassemble_data_bytes_read = 0; - uint32_t reassemble_padding_read = 0; // per call uint32_t cur_data; - uint32_t cur_padding; uint32_t cur_data_offset; - // - // State machines - // - - // data scan - enum DataState { DATA, PADDING, FULL_FRAME }; - enum DataState data_state; - // reassemble - enum ReassembleState { GET_FRAME_HDR, GET_PADDING_LEN, SEND_DATA, SKIP_PADDING, CLEANUP }; + enum ReassembleState { GET_FRAME_HDR, GET_PADDING_LEN, SEND_EMPTY_DATA, SEND_DATA, DONE }; enum ReassembleState reassemble_state = GET_FRAME_HDR; - bool http2_scan(uint32_t length, uint32_t* flush_offset, uint32_t frame_len, - uint8_t frame_flags, uint32_t& data_offset); - snort::StreamSplitter::Status http_scan(const uint8_t* data, uint32_t* flush_offset); + void http2_scan(uint32_t length, uint32_t* flush_offset, uint32_t frame_len, bool padded, + uint32_t& data_offset); + snort::StreamSplitter::Status http_scan(const uint8_t* data, uint32_t* flush_offset, + bool end_stream); }; #endif diff --git a/src/service_inspectors/http2_inspect/http2_data_frame.cc b/src/service_inspectors/http2_inspect/http2_data_frame.cc index aa1e40781..759edacd2 100644 --- a/src/service_inspectors/http2_inspect/http2_data_frame.cc +++ b/src/service_inspectors/http2_inspect/http2_data_frame.cc @@ -50,24 +50,16 @@ bool Http2DataFrame::valid_sequence(Http2Enums::StreamState state) void Http2DataFrame::analyze_http1() { - if ((data_length != 0) || !session_data->flushing_data[source_id]) - { - Http2DummyPacket dummy_pkt; - dummy_pkt.flow = session_data->flow; - dummy_pkt.packet_flags = (source_id == SRC_CLIENT) ? PKT_FROM_CLIENT : PKT_FROM_SERVER; - dummy_pkt.dsize = data_length; - dummy_pkt.data = data_buffer; - dummy_pkt.xtradata_mask = 0; - session_data->hi->eval(&dummy_pkt); - detection_required = dummy_pkt.is_detection_required(); - xtradata_mask = dummy_pkt.xtradata_mask; - } - else - { - detection_required = true; - HttpFlowData* const http_flow = (HttpFlowData*)session_data->get_hi_flow_data(); - http_flow->reset_partial_flush(source_id); - } + Http2DummyPacket dummy_pkt; + dummy_pkt.flow = session_data->flow; + dummy_pkt.packet_flags = (source_id == SRC_CLIENT) ? PKT_FROM_CLIENT : PKT_FROM_SERVER; + dummy_pkt.dsize = data_length; + dummy_pkt.data = data_buffer; + dummy_pkt.xtradata_mask = 0; + // FIXIT-E no checks here + session_data->hi->eval(&dummy_pkt); + detection_required = dummy_pkt.is_detection_required(); + xtradata_mask = dummy_pkt.xtradata_mask; } void Http2DataFrame::clear() @@ -104,7 +96,6 @@ void Http2DataFrame::update_stream_state() } break; default: - // FIXIT-E build this out // Stream state is idle or closed - this is caught in scan so should not get here assert(false); } diff --git a/src/service_inspectors/http2_inspect/http2_enum.h b/src/service_inspectors/http2_inspect/http2_enum.h index 6ee57d3ed..3ece39745 100644 --- a/src/service_inspectors/http2_inspect/http2_enum.h +++ b/src/service_inspectors/http2_inspect/http2_enum.h @@ -151,7 +151,7 @@ enum SettingsFrameIds MAX_HEADER_LIST_SIZE, }; -enum ScanState { SCAN_HEADER, SCAN_PADDING_LENGTH, SCAN_DATA, SCAN_EMPTY_DATA }; +enum ScanState { SCAN_FRAME_HEADER, SCAN_PADDING_LENGTH, SCAN_DATA, SCAN_EMPTY_DATA }; } // end namespace Http2Enums #endif diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.cc b/src/service_inspectors/http2_inspect/http2_flow_data.cc index e8cbd0d88..181c99ec8 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.cc +++ b/src/service_inspectors/http2_inspect/http2_flow_data.cc @@ -50,7 +50,8 @@ Http2FlowData::Http2FlowData(Flow* flow_) : hpack_decoder { Http2HpackDecoder(this, SRC_CLIENT, events[SRC_CLIENT], infractions[SRC_CLIENT]), Http2HpackDecoder(this, SRC_SERVER, events[SRC_SERVER], - infractions[SRC_SERVER]) } + infractions[SRC_SERVER]) }, + data_cutter { Http2DataCutter(this, SRC_CLIENT), Http2DataCutter(this, SRC_SERVER) } { if (hi != nullptr) { @@ -92,7 +93,6 @@ Http2FlowData::~Http2FlowData() delete events[k]; delete hi_ss[k]; delete[] frame_data[k]; - delete[] frame_header[k]; } } @@ -152,7 +152,7 @@ class Http2Stream* Http2FlowData::get_hi_stream() const return find_stream(stream_in_hi); } -class Http2Stream* Http2FlowData::get_current_stream(const HttpCommon::SourceId source_id) +class Http2Stream* Http2FlowData::get_current_stream(HttpCommon::SourceId source_id) { return get_stream(current_stream[source_id]); } @@ -187,3 +187,10 @@ void Http2FlowData::deallocate_hi_memory() { update_deallocations(HttpFlowData::get_memory_usage_estimate()); } + +bool Http2FlowData::is_mid_frame() const +{ + return (scan_octets_seen[SRC_SERVER] != 0) || (remaining_data_padding[SRC_SERVER] != 0) || + continuation_expected[SRC_SERVER]; +} + diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.h b/src/service_inspectors/http2_inspect/http2_flow_data.h index a8c75daca..dede05818 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.h +++ b/src/service_inspectors/http2_inspect/http2_flow_data.h @@ -20,6 +20,7 @@ #ifndef HTTP2_FLOW_DATA_H #define HTTP2_FLOW_DATA_H +#include #include #include "main/snort_types.h" @@ -30,6 +31,7 @@ #include "service_inspectors/http_inspect/http_field.h" #include "stream/stream_splitter.h" +#include "http2_data_cutter.h" #include "http2_enum.h" #include "http2_hpack.h" #include "http2_hpack_int_decode.h" @@ -102,14 +104,12 @@ public: Http2ConnectionSettings* get_recipient_connection_settings(const HttpCommon::SourceId source_id) { return &connection_settings[1 - source_id]; } - bool is_mid_frame(const HttpCommon::SourceId source_id = HttpCommon::SRC_SERVER) - { return (continuation_expected[source_id] || reading_frame[source_id]); } + // Used by payload injection to determine whether we are at a safe place to insert our own + // frame into the S2C direction of an HTTP/2 flow. + bool is_mid_frame() const; #ifdef UNIT_TEST - void set_reading_frame(HttpCommon::SourceId source_id, bool val) - { reading_frame[source_id] = val;} - void set_continuation_expected(HttpCommon::SourceId source_id, bool val) - { continuation_expected[source_id] = val;} + void set_mid_frame(bool); // Not implemented outside of unit tests #endif protected: @@ -135,8 +135,7 @@ protected: uint32_t stream_in_hi = Http2Enums::NO_STREAM_ID; // Reassemble() data to eval() - uint8_t* frame_header[2] = { nullptr, nullptr }; - uint32_t frame_header_size[2] = { 0, 0 }; + uint8_t lead_frame_header[2][Http2Enums::FRAME_HEADER_LENGTH]; uint8_t* frame_data[2] = { nullptr, nullptr }; uint32_t frame_data_size[2] = { 0, 0 }; @@ -153,35 +152,32 @@ protected: uint8_t scan_frame_header[2][Http2Enums::FRAME_HEADER_LENGTH]; uint32_t scan_remaining_frame_octets[2] = { 0, 0 }; uint32_t scan_octets_seen[2] = { 0, 0 }; - bool data_processing[2] = { false, false }; uint8_t padding_length[2] = { 0, 0 }; - Http2Enums::ScanState scan_state[2] = { Http2Enums::SCAN_HEADER, Http2Enums::SCAN_HEADER }; + uint8_t remaining_data_padding[2] = { 0, 0 }; + Http2Enums::ScanState scan_state[2] = + { Http2Enums::SCAN_FRAME_HEADER, Http2Enums::SCAN_FRAME_HEADER }; + + // Used by scan() and reassemble() + Http2DataCutter data_cutter[2]; // Scan signals to reassemble() bool payload_discard[2] = { false, false }; - uint32_t num_frame_headers[2] = { 0, 0 }; uint32_t total_bytes_in_split[2] = { 0, 0 }; - bool use_leftover_hdr[2] = { false, false }; - uint8_t leftover_hdr[2][Http2Enums::FRAME_HEADER_LENGTH]; - - // Used by scan, reassemble - bool flushing_data[2] = { false, false }; // Used by scan, reassemble and eval to communicate uint8_t frame_type[2] = { Http2Enums::FT__NONE, Http2Enums::FT__NONE }; bool abort_flow[2] = { false, false }; + std::queue frame_lengths[2]; // Internal to reassemble() uint32_t frame_header_offset[2] = { 0, 0 }; uint32_t frame_data_offset[2] = { 0, 0 }; uint32_t remaining_frame_octets[2] = { 0, 0 }; - uint8_t remaining_padding_octets_in_frame[2] = { 0, 0 }; + uint8_t remaining_padding_reassemble[2] = { 0, 0 }; + bool read_frame_header[2] = { false, false }; + bool continuation_frame[2] = { false, false }; bool read_padding_len[2] = { false, false }; - // used to signal frame wasn't fully read yet, - // currently used by payload injector - bool reading_frame[2] = { false, false }; - #ifdef REG_TEST static uint64_t instance_count; uint64_t seq_num; diff --git a/src/service_inspectors/http2_inspect/http2_frame.cc b/src/service_inspectors/http2_inspect/http2_frame.cc index eb6cfea3d..b18ad1236 100644 --- a/src/service_inspectors/http2_inspect/http2_frame.cc +++ b/src/service_inspectors/http2_inspect/http2_frame.cc @@ -92,10 +92,8 @@ const Field& Http2Frame::get_buf(unsigned id) uint8_t Http2Frame::get_flags() { - if (header.length() > 0) - return header.start()[flags_index]; - else - return 0; + assert(header.length() > 0); + return header.start()[flags_index]; } uint32_t Http2Frame::get_stream_id() diff --git a/src/service_inspectors/http2_inspect/http2_inspect.cc b/src/service_inspectors/http2_inspect/http2_inspect.cc index fb8508782..3311e90e5 100644 --- a/src/service_inspectors/http2_inspect/http2_inspect.cc +++ b/src/service_inspectors/http2_inspect/http2_inspect.cc @@ -123,9 +123,7 @@ void Http2Inspect::eval(Packet* p) return; // FIXIT-E Workaround for unexpected eval() calls - if (session_data->abort_flow[source_id] or - ((session_data->frame_header[source_id] == nullptr ) and - (session_data->frame_data[source_id] == nullptr))) + if (session_data->abort_flow[source_id]) { return; } @@ -135,17 +133,16 @@ void Http2Inspect::eval(Packet* p) assert(stream); session_data->stream_in_hi = stream->get_stream_id(); - stream->eval_frame(session_data->frame_header[source_id], - session_data->frame_header_size[source_id], session_data->frame_data[source_id], - session_data->frame_data_size[source_id], source_id); + uint8_t* const frame_header_copy = new uint8_t[FRAME_HEADER_LENGTH]; + memcpy(frame_header_copy, session_data->lead_frame_header[source_id], FRAME_HEADER_LENGTH); + stream->eval_frame(frame_header_copy, FRAME_HEADER_LENGTH, + session_data->frame_data[source_id], session_data->frame_data_size[source_id], source_id); if (!stream->get_current_frame()->is_detection_required()) DetectionEngine::disable_all(p); p->xtradata_mask |= stream->get_xtradata_mask(); // The current frame now owns these buffers, clear them from the flow data - session_data->frame_header[source_id] = nullptr; - session_data->frame_header_size[source_id] = 0; session_data->frame_data[source_id] = nullptr; session_data->frame_data_size[source_id] = 0; diff --git a/src/service_inspectors/http2_inspect/http2_stream.cc b/src/service_inspectors/http2_inspect/http2_stream.cc index 2f4aa3ee9..599b19f06 100644 --- a/src/service_inspectors/http2_inspect/http2_stream.cc +++ b/src/service_inspectors/http2_inspect/http2_stream.cc @@ -30,6 +30,7 @@ #include "http2_data_cutter.h" #include "http2_dummy_packet.h" +#include "http2_flow_data.h" using namespace HttpCommon; using namespace Http2Enums; @@ -47,8 +48,6 @@ Http2Stream::~Http2Stream() if (hi_flow_data) session_data->deallocate_hi_memory(); delete hi_flow_data; - delete data_cutter[SRC_CLIENT]; - delete data_cutter[SRC_SERVER]; } void Http2Stream::eval_frame(const uint8_t* header_buffer, uint32_t header_len, @@ -116,13 +115,6 @@ void Http2Stream::print_frame(FILE* output) } #endif -Http2DataCutter* Http2Stream::get_data_cutter(HttpCommon::SourceId source_id) -{ - if (!data_cutter[source_id]) - data_cutter[source_id] = new Http2DataCutter(session_data, source_id); - return data_cutter[source_id]; -} - bool Http2Stream::is_open(HttpCommon::SourceId source_id) { return (state[source_id] == STREAM_EXPECT_BODY) || (state[source_id] == STREAM_BODY); @@ -142,5 +134,4 @@ void Http2Stream::finish_msg_body(HttpCommon::SourceId source_id, bool expect_tr &dummy_pkt, nullptr, 0, unused, &http_flush_offset); assert(scan_result == snort::StreamSplitter::FLUSH); UNUSED(scan_result); - session_data->data_processing[source_id] = false; } diff --git a/src/service_inspectors/http2_inspect/http2_stream.h b/src/service_inspectors/http2_inspect/http2_stream.h index abf761939..d107267f6 100644 --- a/src/service_inspectors/http2_inspect/http2_stream.h +++ b/src/service_inspectors/http2_inspect/http2_stream.h @@ -49,17 +49,6 @@ public: current_frame->get_xtradata_mask() : 0; } Http2Frame *get_current_frame() { return current_frame; } - Http2DataCutter* get_data_cutter(HttpCommon::SourceId source_id); - void set_data_cutter(Http2DataCutter* cutter, HttpCommon::SourceId source_id) - { data_cutter[source_id] = cutter; } - - void set_partial_buf_pending(HttpCommon::SourceId source_id) - { partial_buf_pending[source_id] = true; } - void reset_partial_buf_pending(HttpCommon::SourceId source_id) - { partial_buf_pending[source_id] = false; } - bool is_partial_buf_pending(HttpCommon::SourceId source_id) - { return partial_buf_pending[source_id]; } - void set_state(HttpCommon::SourceId source_id, Http2Enums::StreamState new_state); Http2Enums::StreamState get_state(HttpCommon::SourceId source_id) const { return state[source_id]; } @@ -81,8 +70,6 @@ private: Http2Frame* current_frame = nullptr; HttpFlowData* hi_flow_data = nullptr; HttpMsgSection* hi_msg_section = nullptr; - Http2DataCutter* data_cutter[2] = { nullptr, nullptr}; - bool partial_buf_pending[2] = { false, false }; // used to indicate a partial buffer bool end_stream_on_data_flush[2] = { false, false }; Http2Enums::StreamState state[2] = { Http2Enums::STREAM_EXPECT_HEADERS, Http2Enums::STREAM_EXPECT_HEADERS }; diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter.cc b/src/service_inspectors/http2_inspect/http2_stream_splitter.cc index a74f6d897..83c9c81ef 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter.cc +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter.cc @@ -21,6 +21,8 @@ #include "config.h" #endif +#include "http2_stream_splitter.h" + #include #include "framework/data_bus.h" @@ -32,7 +34,6 @@ #include "service_inspectors/http_inspect/http_test_input.h" #include "service_inspectors/http_inspect/http_test_manager.h" -#include "http2_stream_splitter.h" #include "http2_module.h" using namespace snort; @@ -47,7 +48,8 @@ StreamSplitter::Status Http2StreamSplitter::scan(Packet* pkt, const uint8_t* dat // This is the session state information we share with Http2Inspect and store with stream. A // session is defined by a TCP connection. Since scan() is the first to see a new TCP // connection the new flow data object is created here. - Http2FlowData* session_data = (Http2FlowData*)pkt->flow->get_flow_data(Http2FlowData::inspector_id); + Http2FlowData* session_data = + (Http2FlowData*)pkt->flow->get_flow_data(Http2FlowData::inspector_id); if (session_data == nullptr) { @@ -129,8 +131,8 @@ const StreamBuffer Http2StreamSplitter::reassemble(Flow* flow, unsigned total, u bool tcp_close; bool partial_flush; uint8_t* test_buffer; - HttpTestManager::get_test_input_source()->reassemble(&test_buffer, len, source_id, - tcp_close, partial_flush); + HttpTestManager::get_test_input_source()->reassemble(&test_buffer, len, total, offset, + flags, source_id, tcp_close, partial_flush); if (tcp_close) { finish(flow); @@ -147,7 +149,6 @@ const StreamBuffer Http2StreamSplitter::reassemble(Flow* flow, unsigned total, u return http_buf; } data = test_buffer; - total = len; } #endif diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter.h b/src/service_inspectors/http2_inspect/http2_stream_splitter.h index 96c071550..3ce51ea63 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter.h +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter.h @@ -57,9 +57,6 @@ private: static StreamSplitter::Status non_data_scan(Http2FlowData* session_data, uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id, uint8_t type, uint8_t frame_flags, uint32_t& data_offset); - static void partial_flush_data(Http2FlowData* session_data, - HttpCommon::SourceId source_id, uint32_t* flush_offset, uint32_t data_offset, - Http2Stream* const stream); static snort::StreamSplitter::Status implement_scan(Http2FlowData* session_data, const uint8_t* data, uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id); static const snort::StreamBuffer implement_reassemble(Http2FlowData* session_data, unsigned total, diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc index 131e711e9..bb6412e28 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc @@ -91,7 +91,7 @@ StreamSplitter::Status Http2StreamSplitter::data_frame_header_checks(Http2FlowDa if (frame_length == 0) { *flush_offset = data_offset; - session_data->scan_state[source_id] = SCAN_HEADER; + session_data->scan_state[source_id] = SCAN_FRAME_HEADER; return StreamSplitter::FLUSH; } @@ -146,92 +146,40 @@ StreamSplitter::Status Http2StreamSplitter::non_data_scan(Http2FlowData* session } // Have the full frame - session_data->reading_frame[source_id] = false; StreamSplitter::Status status = StreamSplitter::FLUSH; - switch (type) + session_data->continuation_expected[source_id] = false; + if (((type == FT_HEADERS) || (type == FT_PUSH_PROMISE) || (type == FT_CONTINUATION)) && + !(frame_flags & END_HEADERS)) { - case FT_HEADERS: - case FT_PUSH_PROMISE: - if (!(frame_flags & END_HEADERS)) - { - session_data->continuation_expected[source_id] = true; - status = StreamSplitter::SEARCH; - } - break; - case FT_CONTINUATION: - if (!(frame_flags & END_HEADERS)) - status = StreamSplitter::SEARCH; - else - { - // continuation frame ending headers - session_data->continuation_expected[source_id] = false; - } - break; - default: - break; + session_data->continuation_expected[source_id] = true; + status = StreamSplitter::SEARCH; } data_offset += session_data->scan_remaining_frame_octets[source_id]; *flush_offset = data_offset; session_data->scan_octets_seen[source_id] = 0; session_data->scan_remaining_frame_octets[source_id] = 0; - session_data->scan_state[source_id] = SCAN_HEADER; + session_data->scan_state[source_id] = SCAN_FRAME_HEADER; return status; } -// Flush pending data -void Http2StreamSplitter::partial_flush_data(Http2FlowData* session_data, - HttpCommon::SourceId source_id, uint32_t* flush_offset, uint32_t data_offset, - Http2Stream* const stream) -{ - session_data->frame_type[source_id] = FT_DATA; - Http2DataCutter* const data_cutter = stream->get_data_cutter(source_id); - if (data_cutter->is_flush_required()) - session_data->hi_ss[source_id]->init_partial_flush(session_data->flow); - session_data->data_processing[source_id] = false; - assert(data_offset != 0); - *flush_offset = data_offset - 1; - session_data->flushing_data[source_id] = true; - session_data->num_frame_headers[source_id] -= 1; -} - bool Http2StreamSplitter::read_frame_hdr(Http2FlowData* session_data, const uint8_t* data, uint32_t length, HttpCommon::SourceId source_id, uint32_t& data_offset) { - if (!session_data->flushing_data[source_id]) - { - // Frame with header - if (session_data->scan_octets_seen[source_id] == 0) - { - // Scanning a new frame - session_data->num_frame_headers[source_id] += 1; - session_data->reading_frame[source_id] = true; - } - - // The first nine bytes are the frame header. But all nine might not all be - // present in the first TCP segment we receive. - const uint32_t remaining_header = FRAME_HEADER_LENGTH - - session_data->scan_octets_seen[source_id]; - const uint32_t remaining_header_in_data = remaining_header > length - data_offset ? - length - data_offset : remaining_header; - memcpy(session_data->scan_frame_header[source_id] + - session_data->scan_octets_seen[source_id], data + data_offset, - remaining_header_in_data); - session_data->scan_octets_seen[source_id] += remaining_header_in_data; - data_offset += remaining_header_in_data; - - if (session_data->scan_octets_seen[source_id] < FRAME_HEADER_LENGTH) - return false; - } - else - { - // Just finished flushing data. Use saved header, skip first byte. - session_data->num_frame_headers[source_id] = 1; - session_data->flushing_data[source_id] = false; - session_data->use_leftover_hdr[source_id] = true; - session_data->scan_octets_seen[source_id] = FRAME_HEADER_LENGTH; - data_offset++; - } + // The first nine bytes are the frame header. But all nine might not all be + // present in the first TCP segment we receive. + const uint32_t remaining_header = FRAME_HEADER_LENGTH - + session_data->scan_octets_seen[source_id]; + const uint32_t remaining_header_in_data = remaining_header > length - data_offset ? + length - data_offset : remaining_header; + memcpy(session_data->scan_frame_header[source_id] + + session_data->scan_octets_seen[source_id], data + data_offset, + remaining_header_in_data); + session_data->scan_octets_seen[source_id] += remaining_header_in_data; + data_offset += remaining_header_in_data; + + if (session_data->scan_octets_seen[source_id] < FRAME_HEADER_LENGTH) + return false; return true; } @@ -239,7 +187,6 @@ bool Http2StreamSplitter::read_frame_hdr(Http2FlowData* session_data, const uint StreamSplitter::Status Http2StreamSplitter::implement_scan(Http2FlowData* session_data, const uint8_t* data, uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id) { - StreamSplitter::Status status = StreamSplitter::SEARCH; if (session_data->preface[source_id]) { // 24-byte preface, not a real frame, no frame header @@ -262,149 +209,155 @@ StreamSplitter::Status Http2StreamSplitter::implement_scan(Http2FlowData* sessio session_data->payload_discard[source_id] = true; return StreamSplitter::FLUSH; } + assert(false); } - else - { - *flush_offset = 0; - uint32_t data_offset = 0; - // Need to process multiple frames in a single scan() if a single TCP segment has - // 1) multiple header and continuation frames or 2) multiple data frames. - while ((status == StreamSplitter::SEARCH) && - ((data_offset < length) or (session_data->scan_state[source_id] == SCAN_EMPTY_DATA))) + StreamSplitter::Status status = StreamSplitter::SEARCH; + *flush_offset = 0; + uint32_t data_offset = 0; + + // Need to process multiple frames in a single scan() if a single TCP segment has multiple + // header and continuation frames + while ((status == StreamSplitter::SEARCH) && + ((data_offset < length) or (session_data->scan_state[source_id] == SCAN_EMPTY_DATA))) + { + switch(session_data->scan_state[source_id]) { - switch(session_data->scan_state[source_id]) + case SCAN_FRAME_HEADER: { - case SCAN_HEADER: + // Discard padding that trails previous Data frame + if (session_data->remaining_data_padding[source_id] > 0) { - if (!read_frame_hdr(session_data, data, length, source_id, data_offset)) - return StreamSplitter::SEARCH; - - // We have the full frame header, compute some variables - const uint8_t type = get_frame_type( - session_data->scan_frame_header[source_id]); - // Continuation frames are collapsed into the preceding Headers or Push Promise - // frame - if (type != FT_CONTINUATION) - session_data->frame_type[source_id] = type; - const uint32_t frame_length = get_frame_length(session_data-> - scan_frame_header[source_id]); - const uint8_t frame_flags = get_frame_flags(session_data-> - scan_frame_header[source_id]); - const uint32_t old_stream = session_data->current_stream[source_id]; - session_data->current_stream[source_id] = - get_stream_id_from_header(session_data->scan_frame_header[source_id]); - - if (session_data->continuation_expected[source_id] && type != FT_CONTINUATION) - { - *session_data->infractions[source_id] += INF_MISSING_CONTINUATION; - session_data->events[source_id]->create_event(EVENT_MISSING_CONTINUATION); - return StreamSplitter::ABORT; - } + const uint8_t avail = + session_data->remaining_data_padding[source_id] <= (length - data_offset) ? + session_data->remaining_data_padding[source_id] : (length - data_offset); + session_data->remaining_data_padding[source_id] -= avail; + session_data->payload_discard[source_id] = true; + *flush_offset = avail; + return StreamSplitter::FLUSH; + } - if (session_data->data_processing[source_id] && - ((type != FT_DATA) || (old_stream != session_data->current_stream[source_id]))) - { - // When there is unflushed data in stream we cannot bypass it to work on some - // other frame. Partial flush gets it out of stream while retaining control of - // message body section sizes. It also avoids extreme delays in inspecting the - // data that could occur if we put this aside indefinitely while processing - // other streams. - const uint32_t next_stream = session_data->current_stream[source_id]; - session_data->current_stream[source_id] = session_data->stream_in_hi = - old_stream; - Http2Stream* const stream = session_data->find_stream( - session_data->current_stream[source_id]); - partial_flush_data(session_data, source_id, flush_offset, data_offset, stream); - - if ((type == FT_HEADERS) and - (session_data->current_stream[source_id]) == next_stream) - { - stream->finish_msg_body(source_id, true, false); - } - session_data->stream_in_hi = NO_STREAM_ID; - return StreamSplitter::FLUSH; - } + if (!read_frame_hdr(session_data, data, length, source_id, data_offset)) + return StreamSplitter::SEARCH; - assert(session_data->scan_remaining_frame_octets[source_id] == 0); - session_data->scan_remaining_frame_octets[source_id] = frame_length; + // We have the full frame header, compute some variables + const uint8_t type = get_frame_type(session_data->scan_frame_header[source_id]); + const uint32_t old_stream_id = session_data->current_stream[source_id]; + session_data->current_stream[source_id] = + get_stream_id_from_header(session_data->scan_frame_header[source_id]); - if (frame_flags & PADDED) + if (session_data->continuation_expected[source_id] && + ((type != FT_CONTINUATION) || + (old_stream_id != session_data->current_stream[source_id]))) + { + *session_data->infractions[source_id] += INF_MISSING_CONTINUATION; + session_data->events[source_id]->create_event(EVENT_MISSING_CONTINUATION); + return StreamSplitter::ABORT; + } + + if (type != FT_CONTINUATION) + { + session_data->frame_type[source_id] = type; + memcpy(session_data->lead_frame_header[source_id], + session_data->scan_frame_header[source_id], FRAME_HEADER_LENGTH); + } + const uint32_t frame_length = get_frame_length(session_data-> + scan_frame_header[source_id]); + session_data->frame_lengths[source_id].push(frame_length); + const uint8_t frame_flags = get_frame_flags(session_data-> + scan_frame_header[source_id]); + + assert(session_data->scan_remaining_frame_octets[source_id] == 0); + session_data->scan_remaining_frame_octets[source_id] = frame_length; + + if (frame_flags & PADDED) + { + if (!(type == FT_DATA || type == FT_HEADERS || type == FT_PUSH_PROMISE)) + { + *session_data->infractions[source_id] += INF_PADDING_ON_INVALID_FRAME; + session_data->events[source_id]->create_event( + EVENT_PADDING_ON_INVALID_FRAME); + // FIXIT-E this is not a sufficient reason to abort + return StreamSplitter::ABORT; + } + if (frame_length == 0) { - if (!(type == FT_DATA || type == FT_HEADERS || type == FT_PUSH_PROMISE)) - { - *session_data->infractions[source_id] += INF_PADDING_ON_INVALID_FRAME; - session_data->events[source_id]->create_event( - EVENT_PADDING_ON_INVALID_FRAME); - return StreamSplitter::ABORT; - } - if (frame_length == 0) - { - *session_data->infractions[source_id] += INF_PADDING_ON_EMPTY_FRAME; - session_data->events[source_id]->create_event( - EVENT_PADDING_ON_EMPTY_FRAME); - return StreamSplitter::ABORT; - } - session_data->scan_state[source_id] = SCAN_PADDING_LENGTH; + *session_data->infractions[source_id] += INF_PADDING_ON_EMPTY_FRAME; + session_data->events[source_id]->create_event( + EVENT_PADDING_ON_EMPTY_FRAME); + return StreamSplitter::ABORT; } - else if (frame_length == 0) + session_data->scan_state[source_id] = SCAN_PADDING_LENGTH; + } + else + { + session_data->padding_length[source_id] = 0; + if (frame_length == 0) session_data->scan_state[source_id] = SCAN_EMPTY_DATA; else session_data->scan_state[source_id] = SCAN_DATA; + } - if (type == FT_DATA) - status = data_frame_header_checks(session_data, flush_offset, source_id, - frame_length, data_offset); - else - status = non_data_frame_header_checks(session_data, source_id, - frame_length, type); + if (type == FT_DATA) + { + status = data_frame_header_checks(session_data, flush_offset, source_id, + frame_length, data_offset); + session_data->data_cutter[source_id].reset(); + } + else + status = non_data_frame_header_checks(session_data, source_id, frame_length, + type); - break; + break; + } + case SCAN_PADDING_LENGTH: + assert(session_data->scan_remaining_frame_octets[source_id] > 0); + session_data->padding_length[source_id] = *(data + data_offset); + if (session_data->frame_type[source_id] == FT_DATA) + { + session_data->remaining_data_padding[source_id] = + session_data->padding_length[source_id]; } - case SCAN_PADDING_LENGTH: - assert(session_data->scan_remaining_frame_octets[source_id] > 0); - session_data->padding_length[source_id] = *(data + data_offset); - session_data->scan_remaining_frame_octets[source_id] -= 1; - if (session_data->padding_length[source_id] > - get_frame_length(session_data->scan_frame_header[source_id]) - 1) - { - *session_data->infractions[source_id] += INF_PADDING_LEN; - session_data->events[source_id]->create_event(EVENT_PADDING_LEN); - return StreamSplitter::ABORT; - } - data_offset++; + session_data->scan_remaining_frame_octets[source_id] -= 1; + if (session_data->padding_length[source_id] > + session_data->frame_lengths[source_id].back() - 1) + { + *session_data->infractions[source_id] += INF_PADDING_LEN; + session_data->events[source_id]->create_event(EVENT_PADDING_LEN); + return StreamSplitter::ABORT; + } + data_offset++; - if (session_data->scan_remaining_frame_octets[source_id] == 0) - { - assert(session_data->padding_length[source_id] == 0); - session_data->scan_state[source_id] = SCAN_EMPTY_DATA; - } - else - session_data->scan_state[source_id] = SCAN_DATA; - break; - case SCAN_DATA: - case SCAN_EMPTY_DATA: + if (session_data->scan_remaining_frame_octets[source_id] == 0) { - const uint32_t frame_length = get_frame_length(session_data-> - scan_frame_header[source_id]); - const uint8_t type = get_frame_type( - session_data->scan_frame_header[source_id]); - const uint8_t frame_flags = get_frame_flags(session_data-> - scan_frame_header[source_id]); - if (session_data->frame_type[source_id] == FT_DATA) - { - Http2Stream* const stream = session_data->find_stream( - session_data->current_stream[source_id]); - Http2DataCutter* data_cutter = stream->get_data_cutter(source_id); - status = data_cutter->scan(data, length, flush_offset, data_offset, frame_length, frame_flags); - } - else - status = non_data_scan(session_data, length, flush_offset, source_id, - type, frame_flags, data_offset); - assert(status != StreamSplitter::SEARCH or - session_data->scan_state[source_id] != SCAN_EMPTY_DATA); - break; + assert(session_data->padding_length[source_id] == 0); + session_data->scan_state[source_id] = SCAN_EMPTY_DATA; + } + else + session_data->scan_state[source_id] = SCAN_DATA; + break; + case SCAN_DATA: + case SCAN_EMPTY_DATA: + { + const uint32_t frame_length = session_data->frame_lengths[source_id].back(); + const uint8_t type = get_frame_type( + session_data->scan_frame_header[source_id]); + const uint8_t frame_flags = get_frame_flags(session_data-> + scan_frame_header[source_id]); + if (session_data->frame_type[source_id] == FT_DATA) + { + status = session_data->data_cutter[source_id].scan( + data, length, flush_offset, data_offset, + frame_length - session_data->padding_length[source_id], frame_flags); + } + else + { + status = non_data_scan(session_data, length, flush_offset, source_id, type, + frame_flags, data_offset); } + assert(status != StreamSplitter::SEARCH or + session_data->scan_state[source_id] != SCAN_EMPTY_DATA); + break; } } } @@ -412,8 +365,6 @@ StreamSplitter::Status Http2StreamSplitter::implement_scan(Http2FlowData* sessio return status; } -// FIXIT-M If there are any errors in header decoding, this currently tells stream not to send -// headers to detection. This behavior may need to be changed. const StreamBuffer Http2StreamSplitter::implement_reassemble(Http2FlowData* session_data, unsigned total, unsigned offset, const uint8_t* data, unsigned len, uint32_t flags, HttpCommon::SourceId source_id) @@ -422,106 +373,90 @@ const StreamBuffer Http2StreamSplitter::implement_reassemble(Http2FlowData* sess assert(total <= MAX_OCTETS); StreamBuffer frame_buf { nullptr, 0 }; - Http2Stream* const stream = session_data->find_stream( - session_data->current_stream[source_id]); - - if (offset == 0) - { - // This is the first reassemble() for this frame and we need to allocate some buffers - session_data->frame_header_size[source_id] = FRAME_HEADER_LENGTH * - session_data->num_frame_headers[source_id]; - session_data->frame_header[source_id] = - new uint8_t[session_data->frame_header_size[source_id]]; - - session_data->frame_header_offset[source_id] = 0; - } if (session_data->frame_type[source_id] == FT_DATA) { - if (session_data->flushing_data[source_id]) - { - assert(total > (FRAME_HEADER_LENGTH - 1)); - const uint32_t total_data = total - (FRAME_HEADER_LENGTH - 1); - if (offset+len > total_data) - { - // frame header that caused the flush is included in current data - if (offset > total_data) - len = 0; // only header bytes - else - { - const uint32_t frame_hdr_bytes = offset + len - total_data; - assert(len >= frame_hdr_bytes); - len -= frame_hdr_bytes; - } - } - } - if (len != 0) { - Http2DataCutter* const data_cutter = stream->get_data_cutter(source_id); - StreamBuffer http_frame_buf = data_cutter->reassemble(data, len); - if (http_frame_buf.data) - { - // FIXIT-L this use of const_cast is worrisome - session_data->frame_data[source_id] = const_cast(http_frame_buf.data); - session_data->frame_data_size[source_id] = http_frame_buf.length; - if (!session_data->flushing_data[source_id] && stream->is_partial_buf_pending( - source_id)) - stream->reset_partial_buf_pending(source_id); - } + session_data->data_cutter[source_id].reassemble(data, len); } } else { - uint32_t data_offset = 0; - if (offset == 0) { - // This is the first reassemble() for this frame - allocate data buffer - if (session_data->use_leftover_hdr[source_id]) - { - // total has 1 byte of header, missing 8 bytes of first header - session_data->frame_data_size[source_id] = - total - 1 - (session_data->frame_header_size[source_id] - FRAME_HEADER_LENGTH); - } - else - { - session_data->frame_data_size[source_id] = - total - session_data->frame_header_size[source_id]; - } + session_data->frame_header_offset[source_id] = 0; + // This is the first reassemble() for this frame - allocate data buffer + session_data->frame_data_size[source_id] = + total - (session_data->frame_lengths[source_id].size() * FRAME_HEADER_LENGTH); if (session_data->frame_data_size[source_id] > 0) session_data->frame_data[source_id] = new uint8_t[ session_data->frame_data_size[source_id]]; session_data->frame_data_offset[source_id] = 0; session_data->remaining_frame_octets[source_id] = 0; - session_data->remaining_padding_octets_in_frame[source_id] = 0; + session_data->remaining_padding_reassemble[source_id] = 0; + session_data->read_frame_header[source_id] = true; + session_data->continuation_frame[source_id] = false; } - do + uint32_t data_offset = 0; + while (data_offset < len) { - uint32_t octets_to_copy; + // Skip frame header + if (session_data->read_frame_header[source_id]) + { + + const uint32_t remaining_frame_header = FRAME_HEADER_LENGTH - + session_data->frame_header_offset[source_id]; + const uint32_t octets_to_skip = remaining_frame_header > len - data_offset ? + len - data_offset : remaining_frame_header; + session_data->frame_header_offset[source_id] += octets_to_skip; + data_offset += octets_to_skip; + + if (session_data->frame_header_offset[source_id] != FRAME_HEADER_LENGTH) + break; + session_data->read_frame_header[source_id] = false; + session_data->frame_header_offset[source_id] = 0; + + // Just passed a header: parse and update frame variables + assert(!session_data->frame_lengths[source_id].empty()); + session_data->remaining_frame_octets[source_id] = + session_data->frame_lengths[source_id].front(); + session_data->frame_lengths[source_id].pop(); + + const uint8_t frame_flags = + get_frame_flags(session_data->lead_frame_header[source_id]); + if ((frame_flags & PADDED) && !session_data->continuation_frame[source_id]) + session_data->read_padding_len[source_id] = true; + + if (data_offset == len) + break; + } // Read the padding length if necessary if (session_data->read_padding_len[source_id]) { session_data->read_padding_len[source_id] = false; - session_data->remaining_padding_octets_in_frame[source_id] = *(data + data_offset); + session_data->remaining_padding_reassemble[source_id] = *(data + data_offset); data_offset += 1; session_data->remaining_frame_octets[source_id] -= 1; // Subtract the padding and padding length from the frame data size session_data->frame_data_size[source_id] -= - (session_data->remaining_padding_octets_in_frame[source_id] + 1); + (session_data->remaining_padding_reassemble[source_id] + 1); + + if (data_offset == len) + break; } // Copy data into the frame buffer until we run out of data or reach the end of the // current frame's data const uint32_t remaining_frame_payload = session_data->remaining_frame_octets[source_id] - - session_data->remaining_padding_octets_in_frame[source_id]; - octets_to_copy = remaining_frame_payload > len - data_offset ? len - data_offset : - remaining_frame_payload; + session_data->remaining_padding_reassemble[source_id]; + const uint32_t octets_to_copy = remaining_frame_payload < len - data_offset ? + remaining_frame_payload : len - data_offset; if (octets_to_copy > 0) { memcpy(session_data->frame_data[source_id] + @@ -536,75 +471,28 @@ const StreamBuffer Http2StreamSplitter::implement_reassemble(Http2FlowData* sess break; // Skip over any padding - uint32_t padding_bytes_to_skip = - session_data->remaining_padding_octets_in_frame[source_id] > len - data_offset ? - len - data_offset : session_data->remaining_padding_octets_in_frame[source_id]; + const uint32_t padding_bytes_to_skip = + session_data->remaining_padding_reassemble[source_id] > len - data_offset ? + len - data_offset : session_data->remaining_padding_reassemble[source_id]; session_data->remaining_frame_octets[source_id] -= padding_bytes_to_skip; - session_data->remaining_padding_octets_in_frame[source_id] -= padding_bytes_to_skip; + session_data->remaining_padding_reassemble[source_id] -= padding_bytes_to_skip; data_offset += padding_bytes_to_skip; if (data_offset == len) break; - assert(session_data->remaining_padding_octets_in_frame[source_id] == 0); - - // Copy headers - if (session_data->use_leftover_hdr[source_id]) - { - assert(session_data->frame_header_offset[source_id] == 0); - memcpy(session_data->frame_header[source_id], - session_data->leftover_hdr[source_id], FRAME_HEADER_LENGTH); - session_data->frame_header_offset[source_id] += FRAME_HEADER_LENGTH; - session_data->use_leftover_hdr[source_id] = false; - data_offset++; - } - else - { - const uint32_t remaining_frame_header = FRAME_HEADER_LENGTH - - (session_data->frame_header_offset[source_id] % FRAME_HEADER_LENGTH); - octets_to_copy = remaining_frame_header > len - data_offset ? len - data_offset : - remaining_frame_header; - memcpy(session_data->frame_header[source_id] + - session_data->frame_header_offset[source_id], - data + data_offset, octets_to_copy); - session_data->frame_header_offset[source_id] += octets_to_copy; - data_offset += octets_to_copy; - - if (session_data->frame_header_offset[source_id] % FRAME_HEADER_LENGTH != 0) - break; - } - - // If we just finished copying a header, parse and update frame variables - session_data->remaining_frame_octets[source_id] = - get_frame_length(session_data->frame_header[source_id] + - session_data->frame_header_offset[source_id] - FRAME_HEADER_LENGTH); - - // Get the most recent frame header type - assert(session_data->frame_header_offset[source_id] >= FRAME_HEADER_LENGTH); - assert(session_data->frame_header_offset[source_id] % FRAME_HEADER_LENGTH == 0); - const uint8_t frame_flags = get_frame_flags(session_data->frame_header[source_id] + - session_data->frame_header_offset[source_id] - FRAME_HEADER_LENGTH); - - if (frame_flags & PADDED) - session_data->read_padding_len[source_id] = true; + session_data->read_frame_header[source_id] = true; + session_data->continuation_frame[source_id] = true; + assert(session_data->remaining_padding_reassemble[source_id] == 0); } - while (data_offset < len); + assert(data_offset == len); } if (flags & PKT_PDU_TAIL) { session_data->total_bytes_in_split[source_id] = 0; - session_data->num_frame_headers[source_id] = 0; session_data->scan_octets_seen[source_id] = 0; - if (session_data->flushing_data[source_id]) - { - stream->set_partial_buf_pending(source_id); - // save current header for next scan/reassemble - memcpy(session_data->leftover_hdr[source_id], - session_data->scan_frame_header[source_id], FRAME_HEADER_LENGTH); - } - // Return 0-length non-null buffer to stream which signals detection required, but don't // create pkt_data buffer frame_buf.data = (const uint8_t*)""; diff --git a/src/service_inspectors/http_inspect/dev_notes.txt b/src/service_inspectors/http_inspect/dev_notes.txt index 5833d8003..07a62cb57 100755 --- a/src/service_inspectors/http_inspect/dev_notes.txt +++ b/src/service_inspectors/http_inspect/dev_notes.txt @@ -211,8 +211,8 @@ the same paragraph (splitter must split) or continuing a section in the next par must search and reassemble). Lines beginning with # are comments. Lines beginning with @ are commands. These do not apply to -lines in the middle of a paragraph. Lines that begin with $ are insert commands - a special class of -commands that may be used within a paragraph to insert data into the message buffer. +lines in the middle of a paragraph. Lines that begin with $ are insert commands - a special class +of commands that may be used within a paragraph to insert data into the message buffer. Commands: @break resets HTTP Inspect data structures and begins a new test. Use it liberally to prevent @@ -220,7 +220,9 @@ Commands: @tcpclose simulates a half-duplex TCP close. @request and @response set the message direction. Applies to subsequent paragraphs until changed. The initial direction is always request and the break command resets the direction to request. - @partial causes a partial flush, simulating a retransmission of a detained packet + @partial causes a partial flush, simulating a retransmission of a detained packet. This does not + have any application to script detection or any other feature where the stream splitter is + driving partial inspections instead of stream. @fileset specifies a file from which the tool will read data into the message buffer. This may be used to include a zipped or other binary file into a message body. Data is read beginning at the start of the file. The file is closed automatically whenever a new file is diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc index 2a462d41c..f505e02dc 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.cc +++ b/src/service_inspectors/http_inspect/http_flow_data.cc @@ -267,6 +267,7 @@ void HttpFlowData::finish_h2_body(HttpCommon::SourceId source_id, HttpEnums::H2B partial_buffer_length[source_id] = 0; delete[] partial_buffer[source_id]; partial_buffer[source_id] = nullptr; + body_octets[source_id] += partial_inspected_octets[source_id]; partial_inspected_octets[source_id] = 0; partial_detect_length[source_id] = 0; delete[] partial_detect_buffer[source_id]; diff --git a/src/service_inspectors/http_inspect/http_stream_splitter.h b/src/service_inspectors/http_inspect/http_stream_splitter.h index 75cd78bbe..6610551d0 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter.h +++ b/src/service_inspectors/http_inspect/http_stream_splitter.h @@ -43,7 +43,8 @@ public: const snort::StreamBuffer reassemble(snort::Flow* flow, unsigned total, unsigned, const uint8_t* data, unsigned len, uint32_t flags, unsigned& copied) override; bool finish(snort::Flow* flow) override; - bool init_partial_flush(snort::Flow* flow) override; + bool init_partial_flush(snort::Flow* flow) override { return init_partial_flush(flow, 0); } + bool init_partial_flush(snort::Flow* flow, uint32_t num_flush); bool is_paf() override { return true; } static StreamSplitter::Status status_value(StreamSplitter::Status ret_val, bool http2 = false); diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc index c53993e6d..9aa85c24c 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc @@ -169,7 +169,7 @@ bool HttpStreamSplitter::finish(Flow* flow) return session_data->section_type[source_id] != SEC__NOT_COMPUTE; } -bool HttpStreamSplitter::init_partial_flush(Flow* flow) +bool HttpStreamSplitter::init_partial_flush(Flow* flow, uint32_t num_flush) { Profile profile(HttpModule::get_profile_stats()); @@ -195,10 +195,10 @@ bool HttpStreamSplitter::init_partial_flush(Flow* flow) // Set up to process partial message section uint32_t not_used; - prepare_flush(session_data, ¬_used, session_data->type_expected[source_id], 0, 0, 0, + prepare_flush(session_data, ¬_used, session_data->type_expected[source_id], num_flush, 0, 0, session_data->cutter[source_id]->get_is_broken_chunk(), session_data->cutter[source_id]->get_num_good_chunks(), - session_data->cutter[source_id]->get_octets_seen()); + session_data->cutter[source_id]->get_octets_seen() - num_flush); (static_cast(session_data->cutter[source_id]))->detain_ended(); session_data->partial_flush[source_id] = true; return true; diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc index 2dd2732c5..478dab8bf 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc @@ -246,8 +246,9 @@ const StreamBuffer HttpStreamSplitter::reassemble(Flow* flow, unsigned total, bool tcp_close; bool partial_flush; uint8_t* test_buffer; - HttpTestManager::get_test_input_source()->reassemble(&test_buffer, len, source_id, - tcp_close, partial_flush); + unsigned unused; + HttpTestManager::get_test_input_source()->reassemble(&test_buffer, len, total, unused, + flags, source_id, tcp_close, partial_flush); if (tcp_close) { finish(flow); @@ -264,7 +265,6 @@ const StreamBuffer HttpStreamSplitter::reassemble(Flow* flow, unsigned total, return http_buf; } data = test_buffer; - total = len; } else { diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc index 10edb1693..150ee4061 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc @@ -288,13 +288,9 @@ StreamSplitter::Status HttpStreamSplitter::scan(Packet* pkt, const uint8_t* data { assert(session_data->accelerated_blocking[source_id] == AB_INSPECT); HttpModule::increment_peg_counts(PEG_SCRIPT_DETECTION); - init_partial_flush(flow); + init_partial_flush(flow, length); #ifdef REG_TEST - if (HttpTestManager::use_test_input(HttpTestManager::IN_HTTP)) - { - HttpTestManager::get_test_input_source()->flush(length); - } - else + if (!HttpTestManager::use_test_input(HttpTestManager::IN_HTTP)) #endif *flush_offset = length; return status_value(StreamSplitter::FLUSH); diff --git a/src/service_inspectors/http_inspect/http_test_input.cc b/src/service_inspectors/http_inspect/http_test_input.cc index a5cde3483..7cea7709a 100644 --- a/src/service_inspectors/http_inspect/http_test_input.cc +++ b/src/service_inspectors/http_inspect/http_test_input.cc @@ -25,6 +25,8 @@ #include "http_test_input.h" +#include "protocols/packet.h" + #include "http_common.h" #include "http_enum.h" #include "http_module.h" @@ -96,10 +98,11 @@ void HttpTestInput::reset() { flushed = false; last_source_id = SRC_CLIENT; - just_flushed = true; + just_flushed = false; tcp_closed = false; flush_octets = 0; need_break = false; + reassembled_octets = 0; for (int k = 0; k <= 1; k++) { @@ -110,6 +113,10 @@ void HttpTestInput::reset() fclose(include_file[k]); include_file[k] = nullptr; } + while (!segments[k].empty()) + { + segments[k].pop(); + } } // Each test needs separate peg counts @@ -139,14 +146,14 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u if (just_flushed) { - // Beginning of a new test or StreamSplitter just flushed and it has all been sent by - // reassemble(). There may or may not be leftover data from the last paragraph that was not - // flushed. + // StreamSplitter just flushed and it has all been sent by reassemble(). There may or may + // not be leftover data from the last paragraph that was not flushed. just_flushed = false; data = msg_buf[last_source_id]; + assert(segments[last_source_id].empty()); // compute the leftover data - end_offset[last_source_id] = (flush_octets <= end_offset[last_source_id]) ? - (end_offset[last_source_id] - flush_octets) : 0; + assert(flush_octets <= end_offset[last_source_id]); + end_offset[last_source_id] = (end_offset[last_source_id] - flush_octets); previous_offset[last_source_id] = 0; if (end_offset[last_source_id] > 0) { @@ -165,6 +172,10 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u else { // The data we gave StreamSplitter last time was not flushed + const uint32_t last_seg_length = + end_offset[last_source_id] - previous_offset[last_source_id]; + if (last_seg_length > 0) + segments[last_source_id].push(last_seg_length); previous_offset[last_source_id] = end_offset[last_source_id]; data = msg_buf[last_source_id] + previous_offset[last_source_id]; } @@ -370,23 +381,10 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u memcpy(msg_buf[last_source_id] + end_offset[last_source_id], preface, sizeof(preface) - 1); end_offset[last_source_id] += sizeof(preface) - 1; } - else if (command_length > 0) + else { - // Look for a test number - if (is_number(command_value, command_length)) - { - int64_t test_number = 0; - for (unsigned j=0; j < command_length; j++) - { - test_number = test_number * 10 + (command_value[j] - '0'); - } - HttpTestManager::update_test_number(test_number); - } - else - { - // Bad command in test file - assert(false); - } + // Bad command in test file + assert(false); } } else @@ -485,14 +483,21 @@ void HttpTestInput::scan(uint8_t*& data, uint32_t& length, SourceId source_id, u void HttpTestInput::flush(uint32_t num_octets) { + if ((num_octets > 0) || (segments[last_source_id].size() == 0)) + { + segments[last_source_id].push(num_octets); + } + flush_octets = previous_offset[last_source_id] + num_octets; + reassembled_octets = 0; assert(flush_octets <= end_offset[last_source_id]); assert(flush_octets <= MAX_OCTETS); flushed = true; } -void HttpTestInput::reassemble(uint8_t** buffer, unsigned& length, SourceId source_id, - bool& tcp_close, bool& partial_flush) +void HttpTestInput::reassemble(uint8_t** buffer, unsigned& length, unsigned& total, + unsigned& offset, uint32_t& flags, SourceId source_id, bool& tcp_close, + bool& partial_flush) { *buffer = nullptr; partial_flush = false; @@ -523,10 +528,29 @@ void HttpTestInput::reassemble(uint8_t** buffer, unsigned& length, SourceId sour return; } - *buffer = msg_buf[last_source_id]; - length = flush_octets; - just_flushed = true; - flushed = false; + total = flush_octets; + assert(!segments[last_source_id].empty()); + const uint32_t segment_length = segments[last_source_id].front(); + segments[last_source_id].pop(); + + length = segment_length; + offset = reassembled_octets; + *buffer = msg_buf[last_source_id] + reassembled_octets; + reassembled_octets += length; + if (!segments[last_source_id].empty()) + { + // Not the final TCP segment to be reassembled + flags &= ~PKT_PDU_TAIL; + } + else + { + // Final segment split at flush point + assert(total == reassembled_octets); + just_flushed = true; + flushed = false; + } + + return; } static uint8_t parse_frame_type(const char buffer[], const unsigned bytes_remaining, diff --git a/src/service_inspectors/http_inspect/http_test_input.h b/src/service_inspectors/http_inspect/http_test_input.h index 0ba06ac10..00a2aeb7a 100644 --- a/src/service_inspectors/http_inspect/http_test_input.h +++ b/src/service_inspectors/http_inspect/http_test_input.h @@ -23,6 +23,7 @@ #ifdef REG_TEST #include +#include #include "http_common.h" #include "http_enum.h" @@ -35,14 +36,15 @@ public: ~HttpTestInput(); void scan(uint8_t*& data, uint32_t& length, HttpCommon::SourceId source_id, uint64_t seq_num); void flush(uint32_t num_octets); - void reassemble(uint8_t** buffer, unsigned& length, HttpCommon::SourceId source_id, - bool& tcp_close, bool& partial_flush); + void reassemble(uint8_t** buffer, unsigned& length, unsigned& total, unsigned& offset, + uint32_t& flags, HttpCommon::SourceId source_id, bool& tcp_close, bool& partial_flush); bool finish(); private: FILE* test_data_file; - // FIXIT-L Figure out how big this buf needs to be and revise value + // FIXIT-E Figure out how big this buf needs to be and revise value uint8_t msg_buf[2][2 * HttpEnums::MAX_OCTETS]; + std::queue segments[2]; FILE* include_file[2] = { nullptr, nullptr }; // break command has been read and we are waiting for a new underlying flow to start @@ -59,7 +61,7 @@ private: HttpCommon::SourceId last_source_id = HttpCommon::SRC_CLIENT; // reassemble() just completed and all flushed octets forwarded, time to resume scan() - bool just_flushed = true; + bool just_flushed = false; // TCP connection directional close bool tcp_closed = false; @@ -70,6 +72,9 @@ private: // number of octets that have been flushed and must be sent by reassemble uint32_t flush_octets = 0; + // Number of octets sent in previous calls to reassemble() + uint32_t reassembled_octets = 0; + // number of characters in the buffer previously shown to splitter but not flushed yet uint32_t previous_offset[2] = { 0, 0 };