From: Mike Stepanek (mstepane) Date: Mon, 14 Oct 2019 16:19:18 +0000 (-0400) Subject: Merge pull request #1796 in SNORT/snort3 from ~THOPETER/snort3:h2i1 to master X-Git-Tag: 3.0.0-263~24 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cb0188f1bcd7c7dd67067cc56dcec91d716c0deb;p=thirdparty%2Fsnort3.git Merge pull request #1796 in SNORT/snort3 from ~THOPETER/snort3:h2i1 to master Squashed commit of the following: commit 96da272489408884f09cff1c6c7960b19dcc5a4a Author: Tom Peters Date: Wed Oct 9 17:15:58 2019 -0400 http2_inspect: Move HPACK decompression out of stream splitter into a separate class. --- diff --git a/src/service_inspectors/http2_inspect/CMakeLists.txt b/src/service_inspectors/http2_inspect/CMakeLists.txt index 4c54912d7..19ce99abd 100644 --- a/src/service_inspectors/http2_inspect/CMakeLists.txt +++ b/src/service_inspectors/http2_inspect/CMakeLists.txt @@ -5,6 +5,8 @@ set (FILE_LIST http2_enum.h http2_flow_data.cc http2_flow_data.h + http2_hpack.cc + http2_hpack.h http2_hpack_int_decode.cc http2_hpack_int_decode.h http2_hpack_string_decode.cc diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.h b/src/service_inspectors/http2_inspect/http2_flow_data.h index 3393851f8..db207517f 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.h +++ b/src/service_inspectors/http2_inspect/http2_flow_data.h @@ -31,6 +31,7 @@ #include "stream/stream_splitter.h" #include "http2_enum.h" +#include "http2_hpack.h" #include "http2_hpack_int_decode.h" #include "http2_hpack_string_decode.h" @@ -49,42 +50,13 @@ public: friend class Http2Inspect; friend class Http2StreamSplitter; + friend class Http2Hpack; friend const snort::StreamBuffer implement_reassemble(Http2FlowData*, unsigned, unsigned, const uint8_t*, unsigned, uint32_t, HttpCommon::SourceId); friend snort::StreamSplitter::Status implement_scan(Http2FlowData*, const uint8_t*, uint32_t, uint32_t*, HttpCommon::SourceId); friend bool implement_get_buf(unsigned id, Http2FlowData*, HttpCommon::SourceId, snort::InspectionBuffer&); - friend bool decode_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* raw_header_buffer, const uint32_t header_length); - friend bool decode_header_line(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t encoded_header_buffer_length, - uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, - const uint32_t decoded_header_buffer_length, uint32_t& bytes_written); - friend bool handle_dynamic_size_update(Http2FlowData* session_data, - HttpCommon::SourceId source_id, const uint8_t* encoded_header_buffer, - const uint32_t encoded_header_length, const Http2HpackIntDecode &decode_int, - uint32_t &bytes_consumed, uint32_t &bytes_written); - friend bool decode_literal_header_line(Http2FlowData* session_data, - HttpCommon::SourceId source_id, const uint8_t* encoded_header_buffer, - const uint32_t encoded_header_buffer_length, - const uint8_t name_index_mask, const Http2HpackIntDecode &decode_int, - uint32_t &bytes_consumed, uint8_t* decoded_header_buffer, - const uint32_t decoded_header_buffer_length, uint32_t &bytes_written); - friend bool decode_index(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, - const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t &bytes_written); - friend bool decode_string_literal(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, - const Http2HpackStringDecode &decode_string, bool is_field_name, uint32_t &bytes_consumed, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_buffer_length, - uint32_t &bytes_written); - friend bool write_decoded_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* in_buffer, const uint32_t in_length, - uint8_t* decoded_header_buffer, uint32_t decoded_header_buffer_length, - uint32_t &bytes_written); size_t size_of() override { return sizeof(*this); } @@ -100,22 +72,23 @@ protected: uint32_t http2_decoded_header_size[2] = { 0, 0 }; bool frame_in_detection = false; - // Internal to scan + // Internal to scan() bool continuation_expected[2] = { false, false }; uint8_t currently_processing_frame_header[2][Http2Enums::FRAME_HEADER_LENGTH]; uint32_t inspection_section_length[2] = { 0, 0 }; uint32_t leftover_data[2] = { 0, 0 }; - // Used internally by scan and reassemble + // Used internally by scan() and reassemble() uint32_t octets_seen[2] = { 0, 0 }; uint8_t header_octets_seen[2] = { 0, 0 }; - // Scan signals to reassemble + // Scan signals to reassemble() bool header_coming[2] = { false, false }; bool payload_discard[2] = { false, false }; uint32_t frames_aggregated[2] = { 0, 0 }; - // Internal to reassemble + // Internal to reassemble() + Http2Hpack hpack[2]; uint32_t remaining_octets_to_next_header[2] = { 0, 0 }; uint32_t remaining_frame_data_octets[2] = { 0, 0 }; uint32_t remaining_frame_data_offset[2] = { 0, 0 }; diff --git a/src/service_inspectors/http2_inspect/http2_hpack.cc b/src/service_inspectors/http2_inspect/http2_hpack.cc new file mode 100644 index 000000000..d40d25089 --- /dev/null +++ b/src/service_inspectors/http2_inspect/http2_hpack.cc @@ -0,0 +1,320 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http2_hpack.cc author Katura Harvey + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "http2_hpack.h" + +#include "service_inspectors/http_inspect/http_field.h" +#include "service_inspectors/http_inspect/http_test_manager.h" + +#include "http2_enum.h" +#include "http2_flow_data.h" + +using namespace HttpCommon; +using namespace Http2Enums; + +Http2HpackIntDecode Http2Hpack::decode_int7(7); +Http2HpackIntDecode Http2Hpack::decode_int6(6); +Http2HpackIntDecode Http2Hpack::decode_int5(5); +Http2HpackIntDecode Http2Hpack::decode_int4(4); +Http2HpackStringDecode Http2Hpack::decode_string; + +bool Http2Hpack::write_decoded_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* in_buffer, const uint32_t in_length, + uint8_t* decoded_header_buffer, uint32_t decoded_header_length, + uint32_t &bytes_written) +{ + bool ret = true; + uint32_t length = in_length; + bytes_written = 0; + + if (in_length > decoded_header_length) + { + length = MAX_OCTETS - session_data->http2_decoded_header_size[source_id]; + *session_data->infractions[source_id] += INF_DECODED_HEADER_BUFF_OUT_OF_SPACE; + session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); + ret = false; + } + + memcpy((void*)decoded_header_buffer, (void*) in_buffer, length); + bytes_written = length; + return ret; +} + +bool Http2Hpack::decode_string_literal(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, + bool is_field_name, uint32_t &bytes_consumed, + uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, + uint32_t &bytes_written) +{ + uint32_t decoded_bytes_written; + uint32_t encoded_bytes_consumed; + uint32_t encoded_header_offset = 0; + bytes_written = 0; + bytes_consumed = 0; + + if (is_field_name) + { + // skip over parsed pattern and zeroed index + encoded_header_offset++; + bytes_consumed++; + } + + if (!decode_string.translate(encoded_header_buffer + encoded_header_offset, + encoded_header_length, encoded_bytes_consumed, decoded_header_buffer, + decoded_header_length, decoded_bytes_written, session_data->events[source_id], + session_data->infractions[source_id])) + { + return false; + } + + bytes_consumed += encoded_bytes_consumed; + bytes_written += decoded_bytes_written; + + if (is_field_name) + { + if (!Http2Hpack::write_decoded_headers(session_data, source_id, (const uint8_t*)": ", 2, + decoded_header_buffer + bytes_written, decoded_header_length - + bytes_written, decoded_bytes_written)) + return false; + } + else + { + if (!Http2Hpack::write_decoded_headers(session_data, source_id, (const uint8_t*)"\r\n", 2, + decoded_header_buffer + bytes_written, decoded_header_length - + bytes_written, decoded_bytes_written)) + return false; + } + + bytes_written += decoded_bytes_written; + + return true; +} + +// FIXIT-H Will be incrementally updated to actually decode indexes. For now just copies encoded +// index directly to decoded_header_buffer +bool Http2Hpack::decode_index(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, + const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, + uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, + uint32_t &bytes_written) +{ + uint64_t index; + bytes_written = 0; + bytes_consumed = 0; + + if (!decode_int.translate(encoded_header_buffer, encoded_header_length, + bytes_consumed, index, session_data->events[source_id], + session_data->infractions[source_id])) + { + return false; + } + + if (index <= STATIC_TABLE_MAX_INDEX) + decode_static_table_index(); + else + decode_dynamic_table_index(); + + if (!Http2Hpack::write_decoded_headers(session_data, source_id, encoded_header_buffer, + bytes_consumed, decoded_header_buffer, decoded_header_length, bytes_written)) + return false; + + return true; +} + +bool Http2Hpack::decode_literal_header_line(Http2FlowData* session_data, + HttpCommon::SourceId source_id, const uint8_t* encoded_header_buffer, + const uint32_t encoded_header_length, const uint8_t name_index_mask, + const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, + uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, uint32_t &bytes_written) +{ + bytes_written = 0; + bytes_consumed = 0; + uint32_t partial_bytes_consumed; + uint32_t partial_bytes_written; + + // indexed field name + if (encoded_header_buffer[0] & name_index_mask) + { + if (!Http2Hpack::decode_index(session_data, source_id, encoded_header_buffer, + encoded_header_length, decode_int, partial_bytes_consumed, + decoded_header_buffer, decoded_header_length, partial_bytes_written)) + return false; + } + // literal field name + else + { + if (!Http2Hpack::decode_string_literal(session_data, source_id, encoded_header_buffer, + encoded_header_length, true, + partial_bytes_consumed, decoded_header_buffer, decoded_header_length, + partial_bytes_written)) + return false; + } + + bytes_consumed += partial_bytes_consumed; + bytes_written += partial_bytes_written; + + // value is always literal + if (!Http2Hpack::decode_string_literal(session_data, source_id, encoded_header_buffer + + partial_bytes_consumed, encoded_header_length - partial_bytes_consumed, + false, partial_bytes_consumed, + decoded_header_buffer + partial_bytes_written, decoded_header_length - + partial_bytes_written, partial_bytes_written)) + return false; + + bytes_consumed += partial_bytes_consumed; + bytes_written += partial_bytes_written; + + return true; +} + +// FIXIT-M Will be updated to actually update dynamic table size. For now just skips over +bool Http2Hpack::handle_dynamic_size_update(Http2FlowData* session_data, + HttpCommon::SourceId source_id, const uint8_t* encoded_header_buffer, + const uint32_t encoded_header_length, const Http2HpackIntDecode &decode_int, + uint32_t &bytes_consumed, uint32_t &bytes_written) +{ + uint64_t decoded_int; + uint32_t encoded_bytes_consumed; + bytes_consumed = 0; + bytes_written = 0; + + if (!decode_int.translate(encoded_header_buffer, encoded_header_length, + encoded_bytes_consumed, decoded_int, session_data->events[source_id], + session_data->infractions[source_id])) + { + return false; + } +#ifdef REG_TEST + //FIXIT-M remove when dynamic size updates are handled + if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP2)) + { + fprintf(HttpTestManager::get_output_file(), + "Skipping HPACK dynamic size update: %lu\n", decoded_int); + } +#endif + bytes_consumed += encoded_bytes_consumed; + + return true; +} + +bool Http2Hpack::decode_header_line(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, + uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, + const uint32_t decoded_header_length, uint32_t& bytes_written) +{ + const uint8_t index_mask = 0x80; + const uint8_t literal_index_mask = 0x40; + const uint8_t literal_index_name_index_mask = 0x3f; + const uint8_t literal_no_index_mask = 0xf0; + const uint8_t literal_never_index_pattern = 0x10; + const uint8_t literal_no_index_name_index_mask = 0x0f; + + // indexed header representation + if (encoded_header_buffer[0] & index_mask) + return Http2Hpack::decode_index(session_data, source_id, encoded_header_buffer, + encoded_header_length, decode_int7, bytes_consumed, + decoded_header_buffer, decoded_header_length, bytes_written); + + // literal header representation to be added to dynamic table + else if (encoded_header_buffer[0] & literal_index_mask) + return Http2Hpack::decode_literal_header_line(session_data, source_id, + encoded_header_buffer, encoded_header_length, literal_index_name_index_mask, + decode_int6, bytes_consumed, decoded_header_buffer, + decoded_header_length, bytes_written); + + // literal header field representation not to be added to dynamic table + // Note that this includes two representation types from the RFC - literal without index and + // literal never index. From a decoding standpoint these are identical. + else if ((encoded_header_buffer[0] & literal_no_index_mask) == 0 or + (encoded_header_buffer[0] & literal_no_index_mask) == literal_never_index_pattern) + return Http2Hpack::decode_literal_header_line(session_data, source_id, + encoded_header_buffer, encoded_header_length, literal_no_index_name_index_mask, + decode_int4, bytes_consumed, decoded_header_buffer, + decoded_header_length, bytes_written); + else + // FIXIT-M dynamic table size update not yet supported, just skip + return handle_dynamic_size_update(session_data, source_id, encoded_header_buffer, + encoded_header_length, decode_int5, bytes_consumed, bytes_written); +} + +// FIXIT-H This will eventually be the decoded header buffer. For now only string literals are +// decoded +bool Http2Hpack::decode_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* encoded_header_buffer, const uint32_t header_length) +{ + uint32_t total_bytes_consumed = 0; + uint32_t line_bytes_consumed = 0; + uint32_t line_bytes_written = 0; + bool success = true; + session_data->http2_decoded_header[source_id] = new uint8_t[MAX_OCTETS]; + session_data->http2_decoded_header_size[source_id] = 0; + + while (total_bytes_consumed < header_length) + { + if (!Http2Hpack::decode_header_line(session_data, source_id, + encoded_header_buffer + total_bytes_consumed, header_length - total_bytes_consumed, + line_bytes_consumed, session_data->http2_decoded_header[source_id] + + session_data->http2_decoded_header_size[source_id], MAX_OCTETS - + session_data->http2_decoded_header_size[source_id], line_bytes_written)) + { + success = false; + break; + } + total_bytes_consumed += line_bytes_consumed; + session_data->http2_decoded_header_size[source_id] += line_bytes_written; + } + + if (!success) + { +#ifdef REG_TEST + if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP2)) + { + fprintf(HttpTestManager::get_output_file(), "Error decoding headers. "); + if (session_data->http2_decoded_header_size[source_id] > 0) + Field(session_data->http2_decoded_header_size[source_id], + session_data->http2_decoded_header[source_id]).print( + HttpTestManager::get_output_file(), "Partially Decoded Header"); + } +#endif + return false; + } + + // write the last CRLF to end the header + if (!Http2Hpack::write_decoded_headers(session_data, source_id, (const uint8_t*)"\r\n", 2, + session_data->http2_decoded_header[source_id] + + session_data->http2_decoded_header_size[source_id], MAX_OCTETS - + session_data->http2_decoded_header_size[source_id], line_bytes_written)) + return false; + session_data->http2_decoded_header_size[source_id] += line_bytes_written; + +#ifdef REG_TEST + if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP2)) + { + Field(session_data->http2_decoded_header_size[source_id], + session_data->http2_decoded_header[source_id]). + print(HttpTestManager::get_output_file(), "Decoded Header"); + } +#endif + + return success; +} diff --git a/src/service_inspectors/http2_inspect/http2_hpack.h b/src/service_inspectors/http2_inspect/http2_hpack.h new file mode 100644 index 000000000..e268419cb --- /dev/null +++ b/src/service_inspectors/http2_inspect/http2_hpack.h @@ -0,0 +1,81 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http2_hpack.h author Tom Peters + +#ifndef HTTP2_HPACK_H +#define HTTP2_HPACK_H + +#include "service_inspectors/http_inspect/http_common.h" + +#include "http2_hpack_int_decode.h" +#include "http2_hpack_string_decode.h" + +class Http2FlowData; + +// This class implements HPACK decompression. One instance is required in each direction for each +// HTTP/2 flow +class Http2Hpack +{ +public: + Http2Hpack() {} + static bool decode_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* raw_header_buffer, const uint32_t header_length); + static bool write_decoded_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* in_buffer, const uint32_t in_length, uint8_t* decoded_header_buffer, + uint32_t decoded_header_length, uint32_t &bytes_written); + static bool decode_header_line(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, + uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, + const uint32_t decoded_header_length, uint32_t& bytes_written); + static bool decode_literal_header_line(Http2FlowData* session_data, + HttpCommon::SourceId source_id, const uint8_t* encoded_header_buffer, + const uint32_t encoded_header_length, const uint8_t name_index_mask, + const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, + uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, + uint32_t &bytes_written); + static bool decode_string_literal(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, + bool is_field_name, uint32_t &bytes_consumed, + uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, + uint32_t &bytes_written); + static bool decode_index(Http2FlowData* session_data, HttpCommon::SourceId source_id, + const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, + const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, + uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, + uint32_t &bytes_written); + static bool handle_dynamic_size_update(Http2FlowData* session_data, + HttpCommon::SourceId source_id, const uint8_t* encoded_header_buffer, + const uint32_t encoded_header_length, const Http2HpackIntDecode &decode_int, + uint32_t &bytes_consumed, uint32_t &bytes_written); + static bool decode_static_table_index(void) { return false; } + static bool decode_dynamic_table_index(void) { return false; } + + static const int STATIC_TABLE_MAX_INDEX = 61; + +private: + static Http2HpackIntDecode decode_int7; + static Http2HpackIntDecode decode_int6; + static Http2HpackIntDecode decode_int5; + static Http2HpackIntDecode decode_int4; + static Http2HpackStringDecode decode_string; + +// FIXIT-H Dictionary class and object go here +}; + +#endif + diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter.h b/src/service_inspectors/http2_inspect/http2_stream_splitter.h index 2a351e7ae..49b2f4f07 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter.h +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter.h @@ -25,16 +25,16 @@ #include "http2_enum.h" #include "http2_flow_data.h" -#include "http2_hpack_int_decode.h" -#include "http2_hpack_string_decode.h" class Http2Inspect; class Http2StreamSplitter : public snort::StreamSplitter { public: - Http2StreamSplitter(bool is_client_to_server) : snort::StreamSplitter(is_client_to_server), - source_id(is_client_to_server ? HttpCommon::SRC_CLIENT : HttpCommon::SRC_SERVER) { } + Http2StreamSplitter(bool is_client_to_server) : + snort::StreamSplitter(is_client_to_server), + source_id(is_client_to_server ? HttpCommon::SRC_CLIENT : HttpCommon::SRC_SERVER) + { } Status scan(snort::Packet* pkt, const uint8_t* data, uint32_t length, uint32_t not_used, uint32_t* flush_offset) override; const snort::StreamBuffer reassemble(snort::Flow* flow, unsigned total, unsigned offset, const @@ -46,24 +46,8 @@ public: // FIXIT-M should return actual packet buffer size unsigned max(snort::Flow*) override { return Http2Enums::MAX_OCTETS; } - friend bool decode_literal_header_line(Http2FlowData* session_data, - HttpCommon::SourceId source_id, const uint8_t* encoded_header_buffer, - const uint32_t encoded_header_length, const uint8_t name_index_mask, - const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t &bytes_written); - friend bool decode_header_line(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, - uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, - const uint32_t decoded_header_length, uint32_t& bytes_written); - private: const HttpCommon::SourceId source_id; - static Http2HpackIntDecode decode_int7; - static Http2HpackIntDecode decode_int6; - static Http2HpackIntDecode decode_int5; - static Http2HpackIntDecode decode_int4; - static Http2HpackStringDecode decode_string; }; snort::StreamSplitter::Status implement_scan(Http2FlowData* session_data, const uint8_t* data, @@ -71,8 +55,6 @@ snort::StreamSplitter::Status implement_scan(Http2FlowData* session_data, const const snort::StreamBuffer implement_reassemble(Http2FlowData* session_data, unsigned total, unsigned offset, const uint8_t* data, unsigned len, uint32_t flags, HttpCommon::SourceId source_id); -bool decode_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* raw_header_buffer, const uint32_t header_length, bool field_name); enum ValidationResult { V_GOOD, V_BAD, V_TBD }; diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc index cca653eba..919219a98 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc @@ -21,36 +21,20 @@ #include "config.h" #endif +#include "http2_stream_splitter.h" + #include -#include "protocols/packet.h" #include "service_inspectors/http_inspect/http_common.h" -#include "service_inspectors/http_inspect/http_field.h" #include "service_inspectors/http_inspect/http_test_input.h" #include "service_inspectors/http_inspect/http_test_manager.h" #include "http2_flow_data.h" -#include "http2_hpack_int_decode.h" -#include "http2_hpack_string_decode.h" -#include "http2_stream_splitter.h" using namespace snort; using namespace HttpCommon; using namespace Http2Enums; -#define STATIC_TABLE_MAX_INDEX 61 - -// FIXIT-H remove these declarations once implemented, for some reason this makes the compiler -// happy for build alt -bool decode_static_table_index(void); -bool decode_dynamic_table_index(void); - -Http2HpackIntDecode Http2StreamSplitter::decode_int7(7); -Http2HpackIntDecode Http2StreamSplitter::decode_int6(6); -Http2HpackIntDecode Http2StreamSplitter::decode_int5(5); -Http2HpackIntDecode Http2StreamSplitter::decode_int4(4); -Http2HpackStringDecode Http2StreamSplitter::decode_string; - static uint32_t get_frame_length(const uint8_t* frame_buffer) { return (frame_buffer[0] << 16) + (frame_buffer[1] << 8) + frame_buffer[2]; @@ -413,8 +397,8 @@ const StreamBuffer implement_reassemble(Http2FlowData* session_data, unsigned to assert(session_data->http2_decoded_header[source_id] == nullptr); // FIXIT-H This will eventually be the decoded header buffer. Under development. - if (!decode_headers(session_data, source_id, session_data->frame_data[source_id], - session_data->frame_data_size[source_id])) + if (!Http2Hpack::decode_headers(session_data, source_id, + session_data->frame_data[source_id], session_data->frame_data_size[source_id])) return frame_buf; } } @@ -435,7 +419,8 @@ ValidationResult validate_preface(const uint8_t* data, const uint32_t length, assert(octets_seen < preface_length); - const uint32_t count = (octets_seen + length) < preface_length ? length : (preface_length - octets_seen); + const uint32_t count = (octets_seen + length) < preface_length ? length : + (preface_length - octets_seen); if (memcmp(data, connection_prefix + octets_seen, count)) return V_BAD; @@ -446,295 +431,3 @@ ValidationResult validate_preface(const uint8_t* data, const uint32_t length, return V_GOOD; } -bool write_decoded_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* in_buffer, const uint32_t in_length, - uint8_t* decoded_header_buffer, uint32_t decoded_header_length, - uint32_t &bytes_written) -{ - bool ret = true; - uint32_t length = in_length; - bytes_written = 0; - - if (in_length > decoded_header_length) - { - length = MAX_OCTETS - session_data->http2_decoded_header_size[source_id]; - *session_data->infractions[source_id] += INF_DECODED_HEADER_BUFF_OUT_OF_SPACE; - session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); - ret = false; - } - - memcpy((void*)decoded_header_buffer, (void*) in_buffer, length); - bytes_written = length; - return ret; -} - -bool decode_string_literal(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, - const Http2HpackStringDecode &decode_string, bool is_field_name, uint32_t &bytes_consumed, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t &bytes_written) -{ - uint32_t decoded_bytes_written; - uint32_t encoded_bytes_consumed; - uint32_t encoded_header_offset = 0; - bytes_written = 0; - bytes_consumed = 0; - - if (is_field_name) - { - // skip over parsed pattern and zeroed index - encoded_header_offset++; - bytes_consumed++; - } - - if (!decode_string.translate(encoded_header_buffer + encoded_header_offset, - encoded_header_length, encoded_bytes_consumed, decoded_header_buffer, - decoded_header_length, decoded_bytes_written, session_data->events[source_id], - session_data->infractions[source_id])) - { - return false; - } - - bytes_consumed += encoded_bytes_consumed; - bytes_written += decoded_bytes_written; - - if (is_field_name) - { - if (!write_decoded_headers(session_data, source_id, (const uint8_t*)": ", 2, - decoded_header_buffer + bytes_written, decoded_header_length - - bytes_written, decoded_bytes_written)) - return false; - } - else - { - if (!write_decoded_headers(session_data, source_id, (const uint8_t*)"\r\n", 2, - decoded_header_buffer + bytes_written, decoded_header_length - - bytes_written, decoded_bytes_written)) - return false; - } - - bytes_written += decoded_bytes_written; - - return true; -} - -// FIXIT-H implement -bool decode_static_table_index(void) -{ - return false; -} - -// FIXIT-H implement -bool decode_dynamic_table_index(void) -{ - return false; -} - -// FIXIT-H Will be incrementally updated to actually decode indexes. For now just copies encoded -// index directly to decoded_header_buffer -bool decode_index(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, - const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t &bytes_written) -{ - uint64_t index; - bytes_written = 0; - bytes_consumed = 0; - - if (!decode_int.translate(encoded_header_buffer, encoded_header_length, - bytes_consumed, index, session_data->events[source_id], - session_data->infractions[source_id])) - { - return false; - } - - if (index <= STATIC_TABLE_MAX_INDEX) - decode_static_table_index(); - else - decode_dynamic_table_index(); - - if (!write_decoded_headers(session_data, source_id, encoded_header_buffer, bytes_consumed, - decoded_header_buffer, decoded_header_length, bytes_written)) - return false; - - return true; -} - -bool decode_literal_header_line(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, - const uint8_t name_index_mask, const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t &bytes_written) -{ - bytes_written = 0; - bytes_consumed = 0; - uint32_t partial_bytes_consumed; - uint32_t partial_bytes_written; - - // indexed field name - if (encoded_header_buffer[0] & name_index_mask) - { - if (!decode_index(session_data, source_id, encoded_header_buffer, - encoded_header_length, decode_int, partial_bytes_consumed, - decoded_header_buffer, decoded_header_length, partial_bytes_written)) - return false; - } - // literal field name - else - { - if (!decode_string_literal(session_data, source_id, encoded_header_buffer, - encoded_header_length, Http2StreamSplitter::decode_string, true, - partial_bytes_consumed, decoded_header_buffer, decoded_header_length, - partial_bytes_written)) - return false; - } - - bytes_consumed += partial_bytes_consumed; - bytes_written += partial_bytes_written; - - // value is always literal - if (!decode_string_literal(session_data, source_id, encoded_header_buffer + - partial_bytes_consumed, encoded_header_length - partial_bytes_consumed, - Http2StreamSplitter::decode_string, false, partial_bytes_consumed, - decoded_header_buffer + partial_bytes_written, decoded_header_length - - partial_bytes_written, partial_bytes_written)) - return false; - - bytes_consumed += partial_bytes_consumed; - bytes_written += partial_bytes_written; - - return true; -} - -// FIXIT-M Will be updated to actually update dynamic table size. For now just skips over -bool handle_dynamic_size_update(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, - const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, uint32_t &bytes_written) -{ - uint64_t decoded_int; - uint32_t encoded_bytes_consumed; - bytes_consumed = 0; - bytes_written = 0; - - if (!decode_int.translate(encoded_header_buffer, encoded_header_length, - encoded_bytes_consumed, decoded_int, session_data->events[source_id], - session_data->infractions[source_id])) - { - return false; - } -#ifdef REG_TEST - //FIXIT-M remove when dynamic size updates are handled - if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP2)) - { - fprintf(HttpTestManager::get_output_file(), - "Skipping HPACK dynamic size update: %lu\n", decoded_int); - } -#endif - bytes_consumed += encoded_bytes_consumed; - - return true; -} - -bool decode_header_line(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, - uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, - const uint32_t decoded_header_length, uint32_t& bytes_written) -{ - const uint8_t index_mask = 0x80; - const uint8_t literal_index_mask = 0x40; - const uint8_t literal_index_name_index_mask = 0x3f; - const uint8_t literal_no_index_mask = 0xf0; - const uint8_t literal_never_index_pattern = 0x10; - const uint8_t literal_no_index_name_index_mask = 0x0f; - - // indexed header representation - if (encoded_header_buffer[0] & index_mask) - return decode_index(session_data, source_id, encoded_header_buffer, encoded_header_length, - Http2StreamSplitter::decode_int7, bytes_consumed, decoded_header_buffer, - decoded_header_length, bytes_written); - - // literal header representation to be added to dynamic table - else if (encoded_header_buffer[0] & literal_index_mask) - return decode_literal_header_line(session_data, source_id, encoded_header_buffer, - encoded_header_length, literal_index_name_index_mask, - Http2StreamSplitter::decode_int6, bytes_consumed, decoded_header_buffer, - decoded_header_length, bytes_written); - - // literal header field representation not to be added to dynamic table - // Note that this includes two representation types from the RFC - literal without index and - // literal never index. From a decoding standpoint these are identical. - else if ((encoded_header_buffer[0] & literal_no_index_mask) == 0 or - (encoded_header_buffer[0] & literal_no_index_mask) == literal_never_index_pattern) - return decode_literal_header_line(session_data, source_id, encoded_header_buffer, - encoded_header_length, literal_no_index_name_index_mask, - Http2StreamSplitter::decode_int4, bytes_consumed, decoded_header_buffer, - decoded_header_length, bytes_written); - else - // FIXIT-M dynamic table size update not yet supported, just skip - return handle_dynamic_size_update(session_data, source_id, encoded_header_buffer, - encoded_header_length, Http2StreamSplitter::decode_int5, bytes_consumed, bytes_written); -} - -// FIXIT-H This will eventually be the decoded header buffer. For now only string literals are -// decoded -bool decode_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, - const uint8_t* encoded_header_buffer, const uint32_t header_length) -{ - - uint32_t total_bytes_consumed = 0; - uint32_t line_bytes_consumed = 0; - uint32_t line_bytes_written = 0; - bool success = true; - session_data->http2_decoded_header[source_id] = new uint8_t[MAX_OCTETS]; - session_data->http2_decoded_header_size[source_id] = 0; - - while (total_bytes_consumed < header_length) - { - if (!decode_header_line(session_data, source_id, - encoded_header_buffer + total_bytes_consumed, header_length - total_bytes_consumed, - line_bytes_consumed, session_data->http2_decoded_header[source_id] + - session_data->http2_decoded_header_size[source_id], MAX_OCTETS - - session_data->http2_decoded_header_size[source_id], line_bytes_written)) - { - success = false; - break; - } - total_bytes_consumed += line_bytes_consumed; - session_data->http2_decoded_header_size[source_id] += line_bytes_written; - } - - if (!success) - { -#ifdef REG_TEST - if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP2)) - { - fprintf(HttpTestManager::get_output_file(), "Error decoding headers. "); - if (session_data->http2_decoded_header_size[source_id] > 0) - Field(session_data->http2_decoded_header_size[source_id], - session_data->http2_decoded_header[source_id]).print( - HttpTestManager::get_output_file(), "Partially Decoded Header"); - } -#endif - return false; - } - - // write the last CRLF to end the header - if (!write_decoded_headers(session_data, source_id, (const uint8_t*)"\r\n", 2, - session_data->http2_decoded_header[source_id] + - session_data->http2_decoded_header_size[source_id], MAX_OCTETS - - session_data->http2_decoded_header_size[source_id], line_bytes_written)) - return false; - session_data->http2_decoded_header_size[source_id] += line_bytes_written; - -#ifdef REG_TEST - if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP2)) - { - Field(session_data->http2_decoded_header_size[source_id], - session_data->http2_decoded_header[source_id]). - print(HttpTestManager::get_output_file(), "Decoded Header"); - } -#endif - - return success; -} diff --git a/src/service_inspectors/http2_inspect/test/CMakeLists.txt b/src/service_inspectors/http2_inspect/test/CMakeLists.txt index 0b312e509..91e066796 100644 --- a/src/service_inspectors/http2_inspect/test/CMakeLists.txt +++ b/src/service_inspectors/http2_inspect/test/CMakeLists.txt @@ -13,6 +13,7 @@ add_cpputest( http2_stream_splitter_impl_test ../http2_hpack_string_decode.cc ../http2_huffman_state_machine.cc ../http2_stream_splitter_impl.cc + ../http2_hpack.cc ../http2_module.cc ../http2_tables.cc ../../../framework/module.cc diff --git a/src/service_inspectors/http_inspect/dev_notes.txt b/src/service_inspectors/http_inspect/dev_notes.txt index c51fdd6f1..4a733690c 100644 --- a/src/service_inspectors/http_inspect/dev_notes.txt +++ b/src/service_inspectors/http_inspect/dev_notes.txt @@ -212,6 +212,13 @@ Escape sequences begin with '\'. They may be used within a paragraph or to begin Data is separated into segments for presentation to the splitter whenever a paragraph ends (blank line). +When the inspector aborts the connection (scan() returns StreamSplitter::ABORT) it does not expect +to receive any more input from stream on that connection in that direction. Accordingly the test +tool should not send it any more input. A paragraph of test input expected to result in an abort +should be the last paragraph. The developer should either start a new test (@break, etc.) or at +least reverse the direction and not send any more data in the orignal direction. Sending more data +after an abort is likely to lead to confusing output that has no bearing on the test. + This test tool does not implement the feature of being hardened against bad input. If you write a badly formatted or improper test case the program may assert or crash. The responsibility is on the developer to get it right.