From: Mike Stepanek (mstepane) Date: Wed, 27 Nov 2019 18:16:22 +0000 (+0000) Subject: Merge pull request #1865 in SNORT/snort3 from ~KATHARVE/snort3:h2i_dynamic_2 to master X-Git-Tag: 3.0.0-266~10 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=51d1a4b504b73aff3a573c97e52c44704dd89ca1;p=thirdparty%2Fsnort3.git Merge pull request #1865 in SNORT/snort3 from ~KATHARVE/snort3:h2i_dynamic_2 to master Squashed commit of the following: commit 8f4efe3e017be5036c368e2bd4fbdd70b9c3a025 Author: Katura Harvey Date: Mon Nov 25 14:56:43 2019 -0500 http2_inspect: implement hpack dynamic index lookups --- diff --git a/src/service_inspectors/http2_inspect/CMakeLists.txt b/src/service_inspectors/http2_inspect/CMakeLists.txt index 9f8357e38..db5d9f060 100644 --- a/src/service_inspectors/http2_inspect/CMakeLists.txt +++ b/src/service_inspectors/http2_inspect/CMakeLists.txt @@ -11,12 +11,14 @@ set (FILE_LIST http2_headers_frame.h http2_hpack.cc http2_hpack.h + http2_hpack_dynamic_table.cc + http2_hpack_dynamic_table.h http2_hpack_int_decode.cc http2_hpack_int_decode.h - http2_hpack_table.cc - http2_hpack_table.h http2_hpack_string_decode.cc http2_hpack_string_decode.h + http2_hpack_table.cc + http2_hpack_table.h http2_huffman_state_machine.cc http2_huffman_state_machine.h http2_inspect.cc diff --git a/src/service_inspectors/http2_inspect/http2_enum.h b/src/service_inspectors/http2_inspect/http2_enum.h index 9a0306fdf..f7e03ede1 100644 --- a/src/service_inspectors/http2_inspect/http2_enum.h +++ b/src/service_inspectors/http2_inspect/http2_enum.h @@ -82,6 +82,7 @@ enum Infraction INF_PSEUDO_HEADER_AFTER_REGULAR_HEADER = 14, INF_PSEUDO_HEADER_URI_FORM_MISMATCH = 15, INF_RESPONSE_WITHOUT_STATUS = 16, + INF_HPACK_INDEX_OUT_OF_BOUNDS = 17, INF__MAX_VALUE }; diff --git a/src/service_inspectors/http2_inspect/http2_hpack.cc b/src/service_inspectors/http2_inspect/http2_hpack.cc index bd1ec8858..cc3d213a3 100644 --- a/src/service_inspectors/http2_inspect/http2_hpack.cc +++ b/src/service_inspectors/http2_inspect/http2_hpack.cc @@ -37,10 +37,9 @@ Http2HpackIntDecode Http2HpackDecoder::decode_int6(6); Http2HpackIntDecode Http2HpackDecoder::decode_int5(5); Http2HpackIntDecode Http2HpackDecoder::decode_int4(4); Http2HpackStringDecode Http2HpackDecoder::decode_string; -Http2HpackTable Http2HpackDecoder::table; bool Http2HpackDecoder::write_decoded_headers(const uint8_t* in_buffer, const uint32_t in_length, - uint8_t* decoded_header_buffer, uint32_t decoded_header_length, uint32_t &bytes_written) + uint8_t* decoded_header_buffer, uint32_t decoded_header_length, uint32_t& bytes_written) { bool ret = true; uint32_t length = in_length; @@ -60,71 +59,100 @@ bool Http2HpackDecoder::write_decoded_headers(const uint8_t* in_buffer, const ui } bool Http2HpackDecoder::decode_string_literal(const uint8_t* encoded_header_buffer, - const uint32_t encoded_header_length, bool is_field_name, uint32_t &bytes_consumed, + const uint32_t encoded_header_length, uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t &bytes_written) + uint32_t& bytes_written, Field& field) { - uint32_t decoded_bytes_written; - uint32_t encoded_bytes_consumed; - uint32_t encoded_header_offset = 0; bytes_written = 0; bytes_consumed = 0; - if (is_field_name) + if (!decode_string.translate(encoded_header_buffer, encoded_header_length, bytes_consumed, + decoded_header_buffer, decoded_header_length, bytes_written, events, infractions)) { - // skip over parsed pattern and zeroed index - encoded_header_offset++; - bytes_consumed++; + return false; } - if (!decode_string.translate(encoded_header_buffer + encoded_header_offset, - encoded_header_length - encoded_header_offset, encoded_bytes_consumed, - decoded_header_buffer, decoded_header_length, decoded_bytes_written, - events, infractions)) - { + field.set(bytes_written, decoded_header_buffer, false); + + return true; +} + +bool Http2HpackDecoder::decode_indexed_name(const uint8_t* encoded_header_buffer, + const uint32_t encoded_header_length, const Http2HpackIntDecode& decode_int, + uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, + uint32_t& bytes_written, Field& name) +{ + uint64_t index; + bytes_written = 0; + bytes_consumed = 0; + + if (!decode_int.translate(encoded_header_buffer, encoded_header_length, bytes_consumed, + index, events, infractions)) return false; - } - bytes_consumed += encoded_bytes_consumed; - bytes_written += decoded_bytes_written; + const HpackTableEntry* const entry = decode_table.lookup(index); - if (is_field_name) + if (!entry) { - if (!write_decoded_headers((const uint8_t*)": ", 2, - decoded_header_buffer + bytes_written, decoded_header_length - - bytes_written, decoded_bytes_written)) - return false; + infractions += INF_HPACK_INDEX_OUT_OF_BOUNDS; + events->create_event(EVENT_MISFORMATTED_HTTP2); + return false; } - else + + // If not a pseudo-header, write to the decoded_header buffer + if (index > HpackIndexTable::PSEUDO_HEADER_MAX_STATIC_INDEX) { - if (!write_decoded_headers((const uint8_t*)"\r\n", 2, - decoded_header_buffer + bytes_written, decoded_header_length - - bytes_written, decoded_bytes_written)) + if (!write_decoded_headers(entry->name.start(), entry->name.length(), decoded_header_buffer, + decoded_header_length, bytes_written)) return false; } - bytes_written += decoded_bytes_written; - + name.set(entry->name); return true; } -bool Http2HpackDecoder::decode_static_table_index(const uint64_t index, const bool decode_full_line, +bool Http2HpackDecoder::decode_literal_header_line(const uint8_t* encoded_header_buffer, + const uint32_t encoded_header_length, const uint8_t name_index_mask, + const Http2HpackIntDecode& decode_int, bool with_indexing, uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, uint32_t& bytes_written) { - uint32_t local_bytes_written = 0; - const Http2HpackTable::TableEntry* const entry = table.lookup(index); bytes_written = 0; + bytes_consumed = 0; + uint32_t partial_bytes_consumed; + uint32_t partial_bytes_written; + Field name, value; - // Index should never be 0 - zeroed index means string literal - assert(index > 0); + // Indexed field name + if (encoded_header_buffer[0] & name_index_mask) + { + if (!decode_indexed_name(encoded_header_buffer, encoded_header_length, decode_int, + partial_bytes_consumed, decoded_header_buffer, decoded_header_length, + partial_bytes_written, name)) + return false; + } - // If this is a pseudo-header, pass it to the start line - if (index < PSEUDO_HEADER_MAX_INDEX) + // Literal field name + else { - start_line->process_pseudo_header_name(index); + // Skip over the byte wth the parsed pattern and zeroed index + bytes_consumed++; + + if (!decode_string_literal(encoded_header_buffer + bytes_consumed, encoded_header_length - + bytes_consumed, partial_bytes_consumed, decoded_header_buffer, + decoded_header_length, partial_bytes_written, name)) + return false; } + bytes_consumed += partial_bytes_consumed; + bytes_written += partial_bytes_written; - // If this is a regular header, write header name + ': ' to decoded headers + // If this was a pseudo-header value, give it to the start-line. + if (start_line->is_pseudo_name(name.start())) + { + start_line->process_pseudo_header_name(name.start(), name.length()); + // Pseudo_header after regular header not allowed + if (start_line->is_finalized()) + bytes_written -= partial_bytes_written; + } else { if (!start_line->is_finalized()) @@ -133,174 +161,100 @@ bool Http2HpackDecoder::decode_static_table_index(const uint64_t index, const bo return false; } - if (!write_decoded_headers((const uint8_t*) entry->name, - strlen(entry->name), decoded_header_buffer, - decoded_header_length, local_bytes_written)) + if (!write_decoded_headers((const uint8_t*)": ", 2, decoded_header_buffer + bytes_written, + decoded_header_length - bytes_written, partial_bytes_written)) return false; - bytes_written += local_bytes_written; - if (!write_decoded_headers((const uint8_t*)": ", 2, - decoded_header_buffer + bytes_written, - decoded_header_length - bytes_written, - local_bytes_written)) - return false; - bytes_written += local_bytes_written; + bytes_written += partial_bytes_written; } - if (decode_full_line) - { - if (strlen(entry->value) == 0) - { - *infractions += INF_LOOKUP_EMPTY_VALUE; - events->create_event(EVENT_MISFORMATTED_HTTP2); - return false; - } + // Value is always a string literal + if (!decode_string_literal(encoded_header_buffer + bytes_consumed, encoded_header_length - + bytes_consumed, partial_bytes_consumed, decoded_header_buffer + bytes_written, + decoded_header_length - bytes_written, partial_bytes_written, value)) + return false; + bytes_written += partial_bytes_written; + bytes_consumed += partial_bytes_consumed; - if (index < PSEUDO_HEADER_MAX_INDEX) - { - start_line->process_pseudo_header_value( - (const uint8_t*)entry->value, strlen(entry->value)); - } - else - { - if (!write_decoded_headers((const uint8_t*)entry->value, - strlen(entry->value), decoded_header_buffer + bytes_written, - decoded_header_length - bytes_written, local_bytes_written)) - return false; - bytes_written += local_bytes_written; - if (!write_decoded_headers((const uint8_t*)"\r\n", 2, - decoded_header_buffer + bytes_written, decoded_header_length - - bytes_written, local_bytes_written)) - return false; - bytes_written += local_bytes_written; - } + // If this was a pseudo-header, give it to the start-line + if (start_line->is_pseudo_value()) + { + start_line->process_pseudo_header_value(value.start(), value.length()); + // Pseudo_header after regular header not allowed + if (start_line->is_finalized()) + bytes_written -= partial_bytes_written; } - - return true; -} - -// FIXIT-H Implement dynamic table. Currently copies encoded index to decoded headers -bool Http2HpackDecoder::decode_dynamic_table_index(const uint64_t index, - const bool decode_full_line, uint32_t &bytes_consumed, const uint8_t* encoded_header_buffer, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, uint32_t& bytes_written) -{ - UNUSED(index); - UNUSED(decode_full_line); - - //FIXIT-H finalize start_line only for regular headers - if (!start_line->is_finalized()) + else { - if (!finalize_start_line()) + if (!write_decoded_headers((const uint8_t*)"\r\n", 2, decoded_header_buffer + bytes_written, + decoded_header_length - bytes_written, partial_bytes_written)) return false; + bytes_written += partial_bytes_written; } - if(!write_decoded_headers(encoded_header_buffer, - bytes_consumed, decoded_header_buffer + bytes_written, decoded_header_length, - bytes_written)) - return false; - return true; + if (with_indexing) + decode_table.add_index(name, value); + return true; } -// FIXIT-H Will be incrementally updated to actually decode indexes. For now just copies encoded -// index directly to decoded_header_buffer -bool Http2HpackDecoder::decode_index(const uint8_t* encoded_header_buffer, - const uint32_t encoded_header_length, const Http2HpackIntDecode &decode_int, - const bool decode_full_line, uint32_t &bytes_consumed, uint8_t* decoded_header_buffer, - const uint32_t decoded_header_length, uint32_t &bytes_written) +bool Http2HpackDecoder::decode_indexed_header(const uint8_t* encoded_header_buffer, + const uint32_t encoded_header_length, const Http2HpackIntDecode& decode_int, + uint32_t &bytes_consumed, uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, + uint32_t& bytes_written) { uint64_t index; bytes_written = 0; bytes_consumed = 0; - if (!decode_int.translate(encoded_header_buffer, encoded_header_length, - bytes_consumed, index, events, infractions)) - { + if (!decode_int.translate(encoded_header_buffer, encoded_header_length, bytes_consumed, + index, events, infractions)) return false; - } - if (index <= STATIC_TABLE_MAX_INDEX) - return decode_static_table_index(index, decode_full_line, - decoded_header_buffer, decoded_header_length, bytes_written); - else - return decode_dynamic_table_index(index, decode_full_line, - bytes_consumed, encoded_header_buffer, decoded_header_buffer, - decoded_header_length, bytes_written); -} + const HpackTableEntry* const entry = decode_table.lookup(index); -bool Http2HpackDecoder::decode_literal_header_line(const uint8_t* encoded_header_buffer, - const uint32_t encoded_header_length, const uint8_t name_index_mask, - const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, uint32_t &bytes_written) -{ - bytes_written = 0; - bytes_consumed = 0; - uint32_t partial_bytes_consumed; - uint32_t partial_bytes_written; - - // indexed field name - if (encoded_header_buffer[0] & name_index_mask) + if (!entry) { - if (!Http2HpackDecoder::decode_index(encoded_header_buffer, - encoded_header_length, decode_int, false, partial_bytes_consumed, - decoded_header_buffer, decoded_header_length, partial_bytes_written)) - { - bytes_written += partial_bytes_written; - return false; - } - } - // literal field name - else - { - if (!Http2HpackDecoder::decode_string_literal(encoded_header_buffer, - encoded_header_length, true, - partial_bytes_consumed, decoded_header_buffer, decoded_header_length, - partial_bytes_written)) - { - bytes_written += partial_bytes_written; - return false; - } - // If this was a pseudo-header value, give it to the start-line. - if (start_line->is_pseudo_name( - (const char*) decoded_header_buffer)) - { - // don't include the ': ' that was written following the header name - start_line->process_pseudo_header_name( - decoded_header_buffer, partial_bytes_written - 2); - } - // If not a pseudo-header value, keep it in the decoded headers - else - { - if (!start_line->is_finalized()) - { - if (!finalize_start_line()) - return false; - } - } + infractions += INF_HPACK_INDEX_OUT_OF_BOUNDS; + events->create_event(EVENT_MISFORMATTED_HTTP2); + return false; } - bytes_written += partial_bytes_written; - bytes_consumed += partial_bytes_consumed; - // value is always literal - if (!Http2HpackDecoder::decode_string_literal(encoded_header_buffer + - partial_bytes_consumed, encoded_header_length - partial_bytes_consumed, - false, partial_bytes_consumed, - decoded_header_buffer + bytes_written, decoded_header_length - - bytes_written, partial_bytes_written)) + if (entry->value.length() <= 0) { - bytes_written += partial_bytes_written; - return false; + infractions += INF_LOOKUP_EMPTY_VALUE; + events->create_event(EVENT_MISFORMATTED_HTTP2); } // If this was a pseudo-header value, give it to the start-line. - if (start_line->is_pseudo_value()) + if (start_line->is_pseudo_name(entry->name.start())) { - // Subtract 2 from the length to remove the trailing CRLF before passing to the start line - start_line->process_pseudo_header_value( - decoded_header_buffer + bytes_written, partial_bytes_written - 2); + start_line->process_pseudo_header_name(entry->name.start(), entry->name.length()); + start_line->process_pseudo_header_value(entry->value.start(), entry->value.length()); + } + else + { + uint32_t local_bytes_written = 0; + if (!start_line->is_finalized()) + if (!finalize_start_line()) + return false; + if (!write_decoded_headers(entry->name.start(), entry->name.length(), decoded_header_buffer, + decoded_header_length, local_bytes_written)) + return false; + bytes_written += local_bytes_written; + if (!write_decoded_headers((const uint8_t*) ": ", 2, decoded_header_buffer + bytes_written, + decoded_header_length - bytes_written, local_bytes_written)) + return false; + bytes_written += local_bytes_written; + if (!write_decoded_headers(entry->value.start(), entry->value.length(), + decoded_header_buffer + bytes_written, decoded_header_length - bytes_written, + local_bytes_written)) + return false; + bytes_written += local_bytes_written; + if (!write_decoded_headers((const uint8_t*) "\r\n", 2, decoded_header_buffer + + bytes_written, decoded_header_length - bytes_written, local_bytes_written)) + return false; + bytes_written += local_bytes_written; } - bytes_written += partial_bytes_written; - bytes_consumed += partial_bytes_consumed; - return true; } @@ -336,37 +290,44 @@ bool Http2HpackDecoder::decode_header_line(const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, uint32_t& bytes_written) { - // indexed header representation - if (encoded_header_buffer[0] & index_mask) - return decode_index(encoded_header_buffer, - encoded_header_length, decode_int7, true, bytes_consumed, + const static uint8_t INDEX_MASK = 0x80; + const static uint8_t LITERAL_INDEX_MASK = 0x40; + const static uint8_t LITERAL_INDEX_NAME_INDEX_MASK = 0x3f; + const static uint8_t LITERAL_NO_INDEX_MASK = 0xf0; + const static uint8_t LITERAL_NEVER_INDEX_PATTERN = 0x10; + const static uint8_t LITERAL_NO_INDEX_NAME_INDEX_MASK = 0x0f; + + // Indexed header representation + if (encoded_header_buffer[0] & INDEX_MASK) + return decode_indexed_header(encoded_header_buffer, + encoded_header_length, decode_int7, bytes_consumed, decoded_header_buffer, decoded_header_length, bytes_written); - // literal header representation to be added to dynamic table - else if (encoded_header_buffer[0] & literal_index_mask) - return decode_literal_header_line( - encoded_header_buffer, encoded_header_length, literal_index_name_index_mask, - decode_int6, bytes_consumed, decoded_header_buffer, + // Literal header representation to be added to dynamic table + else if (encoded_header_buffer[0] & LITERAL_INDEX_MASK) + return decode_literal_header_line(encoded_header_buffer, encoded_header_length, + LITERAL_INDEX_NAME_INDEX_MASK, decode_int6, true, bytes_consumed, decoded_header_buffer, decoded_header_length, bytes_written); - // literal header field representation not to be added to dynamic table + // Literal header field representation not to be added to dynamic table // Note that this includes two representation types from the RFC - literal without index and // literal never index. From a decoding standpoint these are identical. - else if ((encoded_header_buffer[0] & literal_no_index_mask) == 0 or - (encoded_header_buffer[0] & literal_no_index_mask) == literal_never_index_pattern) - return decode_literal_header_line( - encoded_header_buffer, encoded_header_length, literal_no_index_name_index_mask, - decode_int4, bytes_consumed, decoded_header_buffer, - decoded_header_length, bytes_written); + else if ((encoded_header_buffer[0] & LITERAL_NO_INDEX_MASK) == 0 or + (encoded_header_buffer[0] & LITERAL_NO_INDEX_MASK) == LITERAL_NEVER_INDEX_PATTERN) + return decode_literal_header_line(encoded_header_buffer, encoded_header_length, + LITERAL_NO_INDEX_NAME_INDEX_MASK, decode_int4, false, bytes_consumed, + decoded_header_buffer, decoded_header_length, bytes_written); else // FIXIT-M dynamic table size update not yet supported, just skip return handle_dynamic_size_update(encoded_header_buffer, encoded_header_length, decode_int5, bytes_consumed, bytes_written); } -// FIXIT-H This will eventually be the decoded header buffer. String literals and static table -// indexes are decoded. Dynamic table indexes are not yet decoded. Both the start-line and -// http2_decoded_header need to be sent to NHI +// Entry point to decode an HPACK-encoded header block. This function returns true on successful +// decode and false on an unrecoverable decode error. Note that alerts may still be generated for +// recoverable errors while the function returns true. This function performs all decoding, but does +// not output the start line or decoded headers - this function must be followed by calls to +// get_start_line() and get_decoded_headers() to generate and obtain these fields. bool Http2HpackDecoder::decode_headers(const uint8_t* encoded_headers, const uint32_t encoded_headers_length, uint8_t* decoded_headers, uint32_t* decoded_headers_len, Http2StartLine *start_line_generator, Http2EventGen* stream_events, @@ -394,9 +355,7 @@ bool Http2HpackDecoder::decode_headers(const uint8_t* encoded_headers, // If there were only pseudo-headers, finalize never got called, so create the start-line if (!start_line->is_finalized()) - { success &= finalize_start_line(); - } // write the last CRLF to end the header if (success) @@ -410,7 +369,7 @@ bool Http2HpackDecoder::decode_headers(const uint8_t* encoded_headers, return success; } -bool Http2HpackDecoder::finalize_start_line()//const uint32_t decoded_headers_size) +bool Http2HpackDecoder::finalize_start_line() { // Save the current position in the decoded buffer so we can set the pointer to the start // of the regular headers diff --git a/src/service_inspectors/http2_inspect/http2_hpack.h b/src/service_inspectors/http2_inspect/http2_hpack.h index 97cea5972..97cc2e34b 100644 --- a/src/service_inspectors/http2_inspect/http2_hpack.h +++ b/src/service_inspectors/http2_inspect/http2_hpack.h @@ -26,8 +26,8 @@ #include "http2_hpack_string_decode.h" #include "http2_hpack_table.h" -class Http2FlowData; class Field; +class Http2FlowData; class Http2StartLine; // This class implements HPACK decompression. One instance is required in each direction for each @@ -40,40 +40,35 @@ public: uint8_t* decoded_headers, uint32_t* decoded_headers_size, Http2StartLine* start_line, Http2EventGen* stream_events, Http2Infractions* stream_infractions); bool write_decoded_headers(const uint8_t* in_buffer, const uint32_t in_length, - uint8_t* decoded_header_buffer, uint32_t decoded_header_length, uint32_t &bytes_written); + uint8_t* decoded_header_buffer, uint32_t decoded_header_length, uint32_t& bytes_written); bool decode_header_line(const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, uint32_t& bytes_written); bool decode_literal_header_line(const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, const uint8_t name_index_mask, - const Http2HpackIntDecode &decode_int, uint32_t &bytes_consumed, + const Http2HpackIntDecode& decode_int, bool with_indexing, uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t &bytes_written); + uint32_t& bytes_written); bool decode_string_literal(const uint8_t* encoded_header_buffer, - const uint32_t encoded_header_length, bool is_field_name, uint32_t &bytes_consumed, + const uint32_t encoded_header_length, uint32_t &bytes_consumed, uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t &bytes_written); - bool decode_index(const uint8_t* encoded_header_buffer, - const uint32_t encoded_header_length, const Http2HpackIntDecode &decode_int, - const bool decode_full_line, uint32_t &bytes_consumed, uint8_t* decoded_header_buffer, - const uint32_t decoded_header_length, uint32_t &bytes_written); + uint32_t &bytes_written, Field& field); + bool decode_indexed_name(const uint8_t* encoded_header_buffer, + const uint32_t encoded_header_length, const Http2HpackIntDecode& decode_int, + uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, + const uint32_t decoded_header_length, uint32_t& bytes_written, Field& name); + bool decode_indexed_header(const uint8_t* encoded_header_buffer, + const uint32_t encoded_header_length, const Http2HpackIntDecode& decode_int, + uint32_t& bytes_consumed, uint8_t* decoded_header_buffer, + const uint32_t decoded_header_length, uint32_t& bytes_written); bool handle_dynamic_size_update(const uint8_t* encoded_header_buffer, const uint32_t encoded_header_length, const Http2HpackIntDecode &decode_int, - uint32_t &bytes_consumed, uint32_t &bytes_written); - bool decode_static_table_index(const uint64_t index, const bool decode_full_line, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t& bytes_written); - bool decode_dynamic_table_index(const uint64_t index, const bool decode_full_line, - uint32_t &bytes_consumed, const uint8_t* encoded_header_buffer, - uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, - uint32_t& bytes_written); + uint32_t& bytes_consumed, uint32_t& bytes_written); bool finalize_start_line(); const Field* get_start_line(); const Field* get_decoded_headers(const uint8_t* const decoded_headers); - static const int STATIC_TABLE_MAX_INDEX = 61; - private: Http2StartLine* start_line = nullptr; uint32_t* decoded_headers_size; @@ -88,15 +83,7 @@ private: static Http2HpackIntDecode decode_int4; static Http2HpackStringDecode decode_string; - static Http2HpackTable table; //static until dynamic table is implemented - - const static uint8_t index_mask = 0x80; - const static uint8_t literal_index_mask = 0x40; - const static uint8_t literal_index_name_index_mask = 0x3f; - const static uint8_t literal_no_index_mask = 0xf0; - const static uint8_t literal_never_index_pattern = 0x10; - const static uint8_t literal_no_index_name_index_mask = 0x0f; - + HpackIndexTable decode_table; }; #endif diff --git a/src/service_inspectors/http2_inspect/http2_hpack_dynamic_table.cc b/src/service_inspectors/http2_inspect/http2_hpack_dynamic_table.cc new file mode 100644 index 000000000..2643fb0ee --- /dev/null +++ b/src/service_inspectors/http2_inspect/http2_hpack_dynamic_table.cc @@ -0,0 +1,105 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http2_hpack_dynamic_table.cc author Katura Harvey + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "http2_hpack_dynamic_table.h" + +#include + +#include "http2_hpack_table.h" + +HpackDynamicTable::~HpackDynamicTable() +{ + assert(num_entries <= array_capacity); + const uint32_t end_index = (start + num_entries) % array_capacity; + for (uint32_t i = 0; i < array_capacity; i++) + { + if ((start <= end_index and (i >= start and i < end_index)) or + (start > end_index and (i >= start or i < end_index))) + { + delete circular_array[i]; + circular_array[i] = nullptr; + } + } + delete[] circular_array; +} + +void HpackDynamicTable::add_entry(Field name, Field value) +{ + const uint32_t new_entry_size = name.length() + value.length() + RFC_ENTRY_OVERHEAD; + + // As per the RFC, attempting to add an entry that is larger than the max size of the table is + // not an error, it causes the table to be cleared + if (new_entry_size > max_size) + { + prune_to_size(0); + return; + } + + // If add entry would exceed max table size, evict old entries + prune_to_size(max_size - new_entry_size); + + // Add new entry to the front of the table (newest entry = lowest index) + HpackTableEntry *new_entry = new HpackTableEntry(name, value); + + start = (start + array_capacity - 1) % array_capacity; + + // FIXIT-P May want to initially allocate small circular array and expand as needed. For now + // array big enough to support hardcoded max table size of 4096 bytes + assert(num_entries < array_capacity); + circular_array[start] = new_entry; + + num_entries++; + rfc_table_size += new_entry_size; +} + +const HpackTableEntry* HpackDynamicTable::get_entry(uint32_t virtual_index) const +{ + const uint32_t dyn_index = virtual_index - HpackIndexTable::STATIC_MAX_INDEX - 1; + + if (num_entries == 0 or dyn_index > num_entries - 1) + return nullptr; + + const uint32_t arr_index = (start + dyn_index) % array_capacity; + return circular_array[arr_index]; +} + +/* This is called when adding a new entry and when receiving a dynamic table size update. + * If adding the new entry would make the table size exceed the max size, entries are pruned + * until the new entry fits. If the dynamic size update is smaller than the current table size, + * entries are pruned until the table is no larger than the max size. Entries are pruned least + * recently added first. + * Note: dynamic size updates not yet implemented + */ +void HpackDynamicTable::prune_to_size(uint32_t new_max_size) +{ + while (rfc_table_size > new_max_size) + { + const uint32_t last_index = (start + num_entries - 1 + array_capacity) % array_capacity; + HpackTableEntry *last_entry = circular_array[last_index]; + num_entries--; + rfc_table_size -= last_entry->name.length() + last_entry->value.length() + + RFC_ENTRY_OVERHEAD; + delete last_entry; + circular_array[last_index] = nullptr; + } +} diff --git a/src/service_inspectors/http2_inspect/http2_hpack_dynamic_table.h b/src/service_inspectors/http2_inspect/http2_hpack_dynamic_table.h new file mode 100644 index 000000000..29770e2ef --- /dev/null +++ b/src/service_inspectors/http2_inspect/http2_hpack_dynamic_table.h @@ -0,0 +1,54 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http2_hpack_dynamic_table.h author Katura Harvey + +#ifndef HTTP2_HPACK_DYNAMIC_TABLE_H +#define HTTP2_HPACK_DYNAMIC_TABLE_H + +#include "service_inspectors/http_inspect/http_field.h" +#include "main/snort_types.h" + +#include "http2_enum.h" + +struct HpackTableEntry; + +class HpackDynamicTable +{ +public: + // FIXIT-M allocate array based on actual max_size from settings + HpackDynamicTable() : circular_array(new HpackTableEntry*[DEFAULT_NUM_ENTRIES]()) { } + ~HpackDynamicTable(); + const HpackTableEntry* get_entry(uint32_t index) const; + void add_entry(Field name, Field value); + void prune_to_size(uint32_t new_max_size); + // FIXIT-M implement handle_dynamic_size_update function +private: + const static uint32_t RFC_ENTRY_OVERHEAD = 32; + + // FIXIT-H set/update these parameters dynamically. For now hardcoded + const static uint32_t DEFAULT_MAX_SIZE = 4096; + const static uint32_t DEFAULT_NUM_ENTRIES = DEFAULT_MAX_SIZE / RFC_ENTRY_OVERHEAD; + uint32_t array_capacity = DEFAULT_NUM_ENTRIES; + uint32_t max_size = DEFAULT_MAX_SIZE; + + uint32_t start = 0; + uint32_t num_entries = 0; + uint32_t rfc_table_size = 0; + HpackTableEntry** circular_array; +}; +#endif diff --git a/src/service_inspectors/http2_inspect/http2_hpack_table.cc b/src/service_inspectors/http2_inspect/http2_hpack_table.cc index 2d62023d2..dcb350007 100644 --- a/src/service_inspectors/http2_inspect/http2_hpack_table.cc +++ b/src/service_inspectors/http2_inspect/http2_hpack_table.cc @@ -19,76 +19,92 @@ #include "http2_hpack_table.h" -#include +#include -const Http2HpackTable::TableEntry Http2HpackTable::table[STATIC_MAX_INDEX + 1] = +#define MAKE_TABLE_ENTRY(name, value) \ + HpackTableEntry(strlen(name), (const uint8_t*)name, strlen(value), (const uint8_t*)value) + +HpackTableEntry::HpackTableEntry(Field& copy_name, Field& copy_value) +{ + uint8_t* new_name = new uint8_t[copy_name.length()]; + uint8_t* new_value = new uint8_t[copy_value.length()]; + memcpy(new_value, copy_value.start(), copy_value.length()); + memcpy(new_name, copy_name.start(), copy_name.length()); + + name.set(copy_name.length(), new_name, true); + value.set(copy_value.length(), new_value, true); +} + +const HpackTableEntry HpackIndexTable::static_table[STATIC_MAX_INDEX + 1] = { - {"", ""}, - {":authority", ""}, - {":method", "GET"}, - {":method", "POST"}, - {":path", "/"}, - {":path", "/index.html"}, - {":scheme", "http"}, - {":scheme", "https"}, - {":status", "200"}, - {":status", "204"}, - {":status", "206"}, - {":status", "304"}, - {":status", "400"}, - {":status", "404"}, - {":status", "500"}, - {"accept-charset", ""}, - {"accept-encoding", "gzip, deflate"}, - {"accept-language", ""}, - {"accept-ranges", ""}, - {"accept", ""}, - {"access-control-allow-origin", ""}, - {"age", ""}, - {"allow", ""}, - {"authorization", ""}, - {"cache-control", ""}, - {"content-disposition", ""}, - {"content-encoding", ""}, - {"content-language", ""}, - {"content-length", ""}, - {"content-location", ""}, - {"content-range", ""}, - {"content-type", ""}, - {"cookie", ""}, - {"date", ""}, - {"etag", ""}, - {"expect", ""}, - {"expires", ""}, - {"from", ""}, - {"host", ""}, - {"if-match", ""}, - {"if-modified-since", ""}, - {"if-none-match", ""}, - {"if-range", ""}, - {"if-unmodified-since", ""}, - {"last-modified", ""}, - {"link", ""}, - {"location", ""}, - {"max-forwards", ""}, - {"proxy-authenticate", ""}, - {"proxy-authorization", ""}, - {"range", ""}, - {"referer", ""}, - {"refresh", ""}, - {"retry-after", ""}, - {"server", ""}, - {"set-cookie", ""}, - {"strict-transport-security", ""}, - {"transfer-encoding", ""}, - {"user-agent", ""}, - {"vary", ""}, - {"via", ""}, - {"www-authenticate", ""}, + MAKE_TABLE_ENTRY("", ""), + MAKE_TABLE_ENTRY(":authority", ""), + MAKE_TABLE_ENTRY(":method", "GET"), + MAKE_TABLE_ENTRY(":method", "POST"), + MAKE_TABLE_ENTRY(":path", "/"), + MAKE_TABLE_ENTRY(":path", "/index.html"), + MAKE_TABLE_ENTRY(":scheme", "http"), + MAKE_TABLE_ENTRY(":scheme", "https"), + MAKE_TABLE_ENTRY(":status", "200"), + MAKE_TABLE_ENTRY(":status", "204"), + MAKE_TABLE_ENTRY(":status", "206"), + MAKE_TABLE_ENTRY(":status", "304"), + MAKE_TABLE_ENTRY(":status", "400"), + MAKE_TABLE_ENTRY(":status", "404"), + MAKE_TABLE_ENTRY(":status", "500"), + MAKE_TABLE_ENTRY("accept-charset", ""), + MAKE_TABLE_ENTRY("accept-encoding", "gzip, deflate"), + MAKE_TABLE_ENTRY("accept-language", ""), + MAKE_TABLE_ENTRY("accept-ranges", ""), + MAKE_TABLE_ENTRY("accept", ""), + MAKE_TABLE_ENTRY("access-control-allow-origin", ""), + MAKE_TABLE_ENTRY("age", ""), + MAKE_TABLE_ENTRY("allow", ""), + MAKE_TABLE_ENTRY("authorization", ""), + MAKE_TABLE_ENTRY("cache-control", ""), + MAKE_TABLE_ENTRY("content-disposition", ""), + MAKE_TABLE_ENTRY("content-encoding", ""), + MAKE_TABLE_ENTRY("content-language", ""), + MAKE_TABLE_ENTRY("content-length", ""), + MAKE_TABLE_ENTRY("content-location", ""), + MAKE_TABLE_ENTRY("content-range", ""), + MAKE_TABLE_ENTRY("content-type", ""), + MAKE_TABLE_ENTRY("cookie", ""), + MAKE_TABLE_ENTRY("date", ""), + MAKE_TABLE_ENTRY("etag", ""), + MAKE_TABLE_ENTRY("expect", ""), + MAKE_TABLE_ENTRY("expires", ""), + MAKE_TABLE_ENTRY("from", ""), + MAKE_TABLE_ENTRY("host", ""), + MAKE_TABLE_ENTRY("if-match", ""), + MAKE_TABLE_ENTRY("if-modified-since", ""), + MAKE_TABLE_ENTRY("if-none-match", ""), + MAKE_TABLE_ENTRY("if-range", ""), + MAKE_TABLE_ENTRY("if-unmodified-since", ""), + MAKE_TABLE_ENTRY("last-modified", ""), + MAKE_TABLE_ENTRY("link", ""), + MAKE_TABLE_ENTRY("location", ""), + MAKE_TABLE_ENTRY("max-forwards", ""), + MAKE_TABLE_ENTRY("proxy-authenticate", ""), + MAKE_TABLE_ENTRY("proxy-authorization", ""), + MAKE_TABLE_ENTRY("range", ""), + MAKE_TABLE_ENTRY("referer", ""), + MAKE_TABLE_ENTRY("refresh", ""), + MAKE_TABLE_ENTRY("retry-after", ""), + MAKE_TABLE_ENTRY("server", ""), + MAKE_TABLE_ENTRY("set-cookie", ""), + MAKE_TABLE_ENTRY("strict-transport-security", ""), + MAKE_TABLE_ENTRY("transfer-encoding", ""), + MAKE_TABLE_ENTRY("user-agent", ""), + MAKE_TABLE_ENTRY("vary", ""), + MAKE_TABLE_ENTRY("via", ""), + MAKE_TABLE_ENTRY("www-authenticate", ""), }; -const Http2HpackTable::TableEntry* Http2HpackTable::lookup(uint64_t index) +const HpackTableEntry* HpackIndexTable::lookup(uint64_t index) const { - assert(index <= STATIC_MAX_INDEX); - return &table[index]; + if (index <= STATIC_MAX_INDEX) + return &static_table[index]; + else + return dynamic_table.get_entry(index); } diff --git a/src/service_inspectors/http2_inspect/http2_hpack_table.h b/src/service_inspectors/http2_inspect/http2_hpack_table.h index 806e62136..a08fd09c4 100644 --- a/src/service_inspectors/http2_inspect/http2_hpack_table.h +++ b/src/service_inspectors/http2_inspect/http2_hpack_table.h @@ -23,24 +23,29 @@ #include "main/snort_types.h" #include "http2_enum.h" +#include "http2_hpack_dynamic_table.h" -#define STATIC_MAX_INDEX 61 -#define PSEUDO_HEADER_MAX_INDEX 14 +struct HpackTableEntry +{ + HpackTableEntry(uint32_t name_len, const uint8_t* _name, uint32_t value_len, + const uint8_t* _value) : name { static_cast(name_len), _name }, + value { static_cast(value_len), _value } { } + HpackTableEntry(Field& copy_name, Field& copy_value); + Field name; + Field value; +}; -// Only static table is implemented. lookup() will be extended to support dynamic table -// lookups once dynamic table is implemented -class Http2HpackTable +class HpackIndexTable { public: - struct TableEntry - { - const char* name; - const char* value; - }; + const HpackTableEntry* lookup(uint64_t index) const; + void add_index(Field name, Field value) { dynamic_table.add_entry(name, value); } - const static TableEntry* lookup(uint64_t index); + const static uint8_t STATIC_MAX_INDEX = 61; + const static uint8_t PSEUDO_HEADER_MAX_STATIC_INDEX = 14; private: - const static TableEntry table[STATIC_MAX_INDEX + 1]; + const static HpackTableEntry static_table[STATIC_MAX_INDEX + 1]; + HpackDynamicTable dynamic_table; }; #endif diff --git a/src/service_inspectors/http2_inspect/http2_request_line.cc b/src/service_inspectors/http2_inspect/http2_request_line.cc index 6e602dbb1..4d11dd5a1 100644 --- a/src/service_inspectors/http2_inspect/http2_request_line.cc +++ b/src/service_inspectors/http2_inspect/http2_request_line.cc @@ -42,26 +42,6 @@ const char* Http2RequestLine::SCHEME_NAME = ":scheme"; const char* Http2RequestLine::OPTIONS = "OPTIONS"; const char* Http2RequestLine::CONNECT = "CONNECT"; -void Http2RequestLine::process_pseudo_header_name(const uint64_t index) -{ - process_pseudo_header_precheck(); - - if (index <= AUTHORITY and authority.length() <= 0) - value_coming = AUTHORITY; - else if (index <= METHOD and method.length() <= 0) - value_coming = METHOD; - else if (index <= PATH and path.length() <= 0) - value_coming = PATH; - else if (index <= SCHEME and scheme.length() <= 0) - value_coming = SCHEME; - else - { - infractions += INF_INVALID_PSEUDO_HEADER; - events->create_event(EVENT_INVALID_HEADER); - value_coming = HEADER__INVALID; - } -} - void Http2RequestLine::process_pseudo_header_name(const uint8_t* const& name, uint32_t length) { process_pseudo_header_precheck(); diff --git a/src/service_inspectors/http2_inspect/http2_request_line.h b/src/service_inspectors/http2_inspect/http2_request_line.h index 22693fc94..280b78afb 100644 --- a/src/service_inspectors/http2_inspect/http2_request_line.h +++ b/src/service_inspectors/http2_inspect/http2_request_line.h @@ -29,7 +29,6 @@ class Http2RequestLine : public Http2StartLine { public: - void process_pseudo_header_name(const uint64_t index) override; void process_pseudo_header_name(const uint8_t* const& name, uint32_t length) override; void process_pseudo_header_value(const uint8_t* const& value, const uint32_t length) override; bool generate_start_line() override; diff --git a/src/service_inspectors/http2_inspect/http2_start_line.h b/src/service_inspectors/http2_inspect/http2_start_line.h index f48a9ba99..3cd143c5c 100644 --- a/src/service_inspectors/http2_inspect/http2_start_line.h +++ b/src/service_inspectors/http2_inspect/http2_start_line.h @@ -44,13 +44,12 @@ public: friend class Http2Hpack; const Field* get_start_line(); - virtual void process_pseudo_header_name(const uint64_t index) = 0; virtual void process_pseudo_header_name(const uint8_t* const& name, uint32_t length) = 0; virtual void process_pseudo_header_value(const uint8_t* const& value, const uint32_t length) = 0; bool finalize(); bool is_finalized() { return finalized; } bool is_pseudo_value() { return value_coming != Http2Enums::HEADER__NONE; } - bool is_pseudo_name(const char* const& name) { return name[0] == ':'; } + bool is_pseudo_name(const uint8_t* const& name) { return name[0] == ':'; } protected: Http2StartLine(Http2EventGen* events, Http2Infractions* infractions) : events(events), diff --git a/src/service_inspectors/http2_inspect/http2_status_line.cc b/src/service_inspectors/http2_inspect/http2_status_line.cc index fa6951862..28e4b8500 100644 --- a/src/service_inspectors/http2_inspect/http2_status_line.cc +++ b/src/service_inspectors/http2_inspect/http2_status_line.cc @@ -36,20 +36,6 @@ using namespace Http2Enums; const char* Http2StatusLine::STATUS_NAME = ":status"; -void Http2StatusLine::process_pseudo_header_name(const uint64_t index) -{ - process_pseudo_header_precheck(); - - if (index >= RESPONSE_PSEUDO_MIN_INDEX and index <= STATUS and status.length() <= 0) - value_coming = STATUS; - else - { - infractions += INF_INVALID_PSEUDO_HEADER; - events->create_event(EVENT_INVALID_HEADER); - value_coming = HEADER__INVALID; - } -} - void Http2StatusLine::process_pseudo_header_name(const uint8_t* const& name, uint32_t length) { process_pseudo_header_precheck(); diff --git a/src/service_inspectors/http2_inspect/http2_status_line.h b/src/service_inspectors/http2_inspect/http2_status_line.h index e0a22addb..76e24e959 100644 --- a/src/service_inspectors/http2_inspect/http2_status_line.h +++ b/src/service_inspectors/http2_inspect/http2_status_line.h @@ -28,7 +28,6 @@ class Http2StatusLine : public Http2StartLine { public: - void process_pseudo_header_name(const uint64_t index) override; void process_pseudo_header_name(const uint8_t* const& name, uint32_t length) override; void process_pseudo_header_value(const uint8_t* const& value, const uint32_t length) override; bool generate_start_line() override; diff --git a/src/service_inspectors/http2_inspect/test/CMakeLists.txt b/src/service_inspectors/http2_inspect/test/CMakeLists.txt index 87b644e8f..b2e42b223 100644 --- a/src/service_inspectors/http2_inspect/test/CMakeLists.txt +++ b/src/service_inspectors/http2_inspect/test/CMakeLists.txt @@ -1,6 +1,8 @@ add_cpputest( http2_stream_splitter_impl_test SOURCES ../http2_flow_data.cc + ../http2_hpack_dynamic_table.cc + ../http2_hpack_table.cc ../http2_stream_splitter_impl.cc ../http2_module.cc ../http2_tables.cc