From: Mike Stepanek (mstepane) Date: Wed, 30 Oct 2019 16:22:01 +0000 (-0400) Subject: Merge pull request #1815 in SNORT/snort3 from ~KATHARVE/snort3:h2i_request_start_line... X-Git-Tag: 3.0.0-263~2 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=a992613dfa210f2b26b6a63c77ecccfab9d93bbf;p=thirdparty%2Fsnort3.git Merge pull request #1815 in SNORT/snort3 from ~KATHARVE/snort3:h2i_request_start_line to master Squashed commit of the following: commit 2efd67923bc0de65e7282e3a1387884f39279c7b Author: Katura Harvey Date: Mon Oct 21 09:37:41 2019 -0400 http2_inspect: generate request start line from pseudo-headers --- diff --git a/src/service_inspectors/http2_inspect/CMakeLists.txt b/src/service_inspectors/http2_inspect/CMakeLists.txt index b38260976..f115997ed 100644 --- a/src/service_inspectors/http2_inspect/CMakeLists.txt +++ b/src/service_inspectors/http2_inspect/CMakeLists.txt @@ -20,6 +20,10 @@ set (FILE_LIST http2_inspect.h http2_module.cc http2_module.h + http2_request_line.cc + http2_request_line.h + http2_start_line.cc + http2_start_line.h http2_stream_splitter.cc http2_stream_splitter_impl.cc http2_stream_splitter.h diff --git a/src/service_inspectors/http2_inspect/dev_notes.txt b/src/service_inspectors/http2_inspect/dev_notes.txt index 204ab473f..60b474047 100644 --- a/src/service_inspectors/http2_inspect/dev_notes.txt +++ b/src/service_inspectors/http2_inspect/dev_notes.txt @@ -10,7 +10,33 @@ buffers. As in NHI, long data frames are split into 16kb chunks for inspection. is followed by continuation frames, all the header frames are flushed together for inspection. The frame headers from each frame are stored contiguously in the frame_header buffer. After cutting out the frame headers, the frame data is stored as a single block, consisting of the HPACK encoded -HTTP/2 headers. HPACK decoding is under ongoing development. Decoded headers will be stored in the -http2_decoded_header buffer. +HTTP/2 headers. + +HPACK decoding is under ongoing development. In the current implementation, reassemble() makes a +first copy of the encoded headers, which is stored in the frame_data buffer. The frame_data buffer +is passed to the function decode_headers(), which is the main loop driving HPACK decoding. The +function allocates a second buffer, raw_decoded_header, to which the decoding routine will +progressively write. As part of decoding the pseudo-headers (described below), data that must not be +sent to NHI may be written to the start of the buffer. In order to avoid making any extra copies, +the decoding routine sets a pointer to the start of the regular headers inside raw_decoded_header +that will be processed by NHI, called http2_decoded_header. + +The main loop in decode_headers() finds the cut point for a single header line. The line is is +passed to decode_header_line(), which parses the line and calls the appropriate decoding function +based on the header representation type. If the type is indexed, the full header line is looked up +in the table and copied to the decoded header buffer. The index may belong to either the static or +the dynamic table. The static table is 61-elements defined in the HPACK RFC. The dynamic table, +which starts at index 62, is specific to each direction of each flow. For the second type, literal +to be indexed, the header name may be indexed or a string literal, while the value is always a +literal. The resulting header line will then be added to the dynamic table (not yet implemented). +The third representation type is literal not to be indexed, which is the same as literal to be +indexed, except the header line is not added to the dynamic table. + +HTTP/2 uses pseudo-headers to convey the information included in an HTTP/1.1 start-line. +Pseudo-headers are not valid HTTP/1.1 headers, so must be translated into a start-line before the +decoded headers can be passed to NHI. The two Http2StartLine subclasses, Http2RequestLine and +Http2ResponseLine (not yet implemented) perform this translation and generate the start-line, which +is stored in a new buffer inside the Http2StartLine object. The start-line buffer must be passed to +NHI before the http2_decoded_header buffer, which contains the regular HTTP/1.1 headers. H2I supports the NHI test tool. See ../http_inspect/dev_notes.txt for usage instructions. diff --git a/src/service_inspectors/http2_inspect/http2_enum.h b/src/service_inspectors/http2_inspect/http2_enum.h index 4401cb142..76a86ce3e 100644 --- a/src/service_inspectors/http2_inspect/http2_enum.h +++ b/src/service_inspectors/http2_inspect/http2_enum.h @@ -55,7 +55,6 @@ enum EventSid EVENT_UNEXPECTED_CONTINUATION = 5, EVENT_MISFORMATTED_HTTP2 = 6, EVENT_PREFACE_MATCH_FAILURE = 7, - EVENT_HPACK_INDEX_DECODE_FAILURE = 8, EVENT__MAX_VALUE }; @@ -75,7 +74,10 @@ enum Infraction INF_HUFFMAN_INCOMPLETE_CODE_PADDING = 9, INF_MISSING_CONTINUATION = 10, INF_UNEXPECTED_CONTINUATION = 11, - INF_STATIC_TABLE_LOOKUP_ERROR = 12, + INF_LOOKUP_EMPTY_VALUE = 12, + INF_INVALID_PSEUDO_HEADER = 13, + INF_PSEUDO_HEADER_AFTER_REGULAR_HEADER = 14, + INF_PSEUDO_HEADER_URI_FORM_MISMATCH = 15, INF__MAX_VALUE }; @@ -88,7 +90,16 @@ enum HeaderFrameFlags NO_HEADER = 0x80, //No valid flags use this bit }; - +enum PseudoHeaders +{ + HEADER_NONE = 0, + AUTHORITY = 1, + METHOD = 3, + PATH = 5, + SCHEME = 7, + STATUS = 14, +}; + } // end namespace Http2Enums #endif diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.cc b/src/service_inspectors/http2_inspect/http2_flow_data.cc index 7080b82b0..438c85b3b 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.cc +++ b/src/service_inspectors/http2_inspect/http2_flow_data.cc @@ -27,6 +27,7 @@ #include "http2_enum.h" #include "http2_module.h" +#include "http2_start_line.h" using namespace snort; using namespace Http2Enums; @@ -72,9 +73,10 @@ Http2FlowData::~Http2FlowData() { delete[] frame_header[k]; delete[] frame_data[k]; - delete[] http2_decoded_header[k]; + delete[] raw_decoded_header[k]; delete infractions[k]; delete events[k]; + delete http2_decoded_header[k]; } } @@ -85,9 +87,13 @@ void Http2FlowData::clear_frame_data(HttpCommon::SourceId source_id) delete[] frame_data[source_id]; frame_data[source_id] = nullptr; frame_in_detection = false; - delete[] http2_decoded_header[source_id]; - http2_decoded_header[source_id] = nullptr; + delete[] raw_decoded_header[source_id]; + raw_decoded_header[source_id] = nullptr; continuation_expected[source_id] = false; frames_aggregated[source_id] = 0; scan_header_octets_seen[source_id] = 0; + delete header_start_line[source_id]; + header_start_line[source_id] = nullptr; + delete http2_decoded_header[source_id]; + http2_decoded_header[source_id] = nullptr; } diff --git a/src/service_inspectors/http2_inspect/http2_flow_data.h b/src/service_inspectors/http2_inspect/http2_flow_data.h index 503ec856d..9cfa258a8 100644 --- a/src/service_inspectors/http2_inspect/http2_flow_data.h +++ b/src/service_inspectors/http2_inspect/http2_flow_data.h @@ -51,6 +51,8 @@ public: friend class Http2Inspect; friend class Http2StreamSplitter; friend class Http2Hpack; + friend class Http2StartLine; + friend class Http2RequestLine; friend const snort::StreamBuffer implement_reassemble(Http2FlowData*, unsigned, unsigned, const uint8_t*, unsigned, uint32_t, HttpCommon::SourceId); friend snort::StreamSplitter::Status implement_scan(Http2FlowData*, const uint8_t*, uint32_t, @@ -68,8 +70,10 @@ protected: uint32_t frame_header_size[2] = { 0, 0 }; uint8_t* frame_data[2] = { nullptr, nullptr }; uint32_t frame_data_size[2] = { 0, 0 }; - uint8_t* http2_decoded_header[2] = { nullptr, nullptr }; - uint32_t http2_decoded_header_size[2] = { 0, 0 }; + uint8_t* raw_decoded_header[2] = { nullptr, nullptr }; + uint32_t raw_decoded_header_size[2] = { 0, 0 }; + uint32_t pseudo_header_fragment_size[2] = { 0, 0 }; + Field* http2_decoded_header[2] = { nullptr, nullptr }; bool frame_in_detection = false; // Internal to scan() @@ -90,6 +94,7 @@ protected: // Internal to reassemble() Http2Hpack hpack[2]; + class Http2StartLine* header_start_line[2] = { nullptr, nullptr }; uint32_t remaining_octets_to_next_header[2] = { 0, 0 }; uint32_t remaining_frame_data_octets[2] = { 0, 0 }; uint32_t remaining_frame_data_offset[2] = { 0, 0 }; diff --git a/src/service_inspectors/http2_inspect/http2_hpack.cc b/src/service_inspectors/http2_inspect/http2_hpack.cc index e51d73d85..25b576d98 100644 --- a/src/service_inspectors/http2_inspect/http2_hpack.cc +++ b/src/service_inspectors/http2_inspect/http2_hpack.cc @@ -28,6 +28,7 @@ #include "http2_enum.h" #include "http2_flow_data.h" +#include "http2_request_line.h" using namespace HttpCommon; using namespace Http2Enums; @@ -50,7 +51,7 @@ bool Http2Hpack::write_decoded_headers(Http2FlowData* session_data, HttpCommon:: if (in_length > decoded_header_length) { - length = MAX_OCTETS - session_data->http2_decoded_header_size[source_id]; + length = MAX_OCTETS - session_data->raw_decoded_header_size[source_id]; *session_data->infractions[source_id] += INF_DECODED_HEADER_BUFF_OUT_OF_SPACE; session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); ret = false; @@ -120,45 +121,97 @@ bool Http2Hpack::decode_static_table_index(Http2FlowData* session_data, const Http2HpackTable::TableEntry* const entry = table.lookup(index); bytes_written = 0; - // FIXIT-H check if header is part of start-line, if so pass to start-line generating - // object and don't write to decoded header buffer + // Index should never be 0 - zeroed index means string literal + assert(index > 0); - // Write header name + ': ' to decoded headers - // FIXIT-H For now pseudo-headers are also copied. Need to be converted to start-line - if (!write_decoded_headers(session_data, source_id, (const uint8_t*) entry->name, - strlen(entry->name), decoded_header_buffer, decoded_header_length, - local_bytes_written)) - return false; - bytes_written += local_bytes_written; - if (!write_decoded_headers(session_data, source_id, (const uint8_t*)": ", 2, - decoded_header_buffer + bytes_written, decoded_header_length - bytes_written, - local_bytes_written)) - return false; - bytes_written += local_bytes_written; + // If this is a pseudo-header, pass it to the start line + // Remove second condition after response start line implemented + if (index < PSEUDO_HEADER_MAX_INDEX and session_data->header_start_line[source_id]) + { + if (!session_data->header_start_line[source_id]->process_pseudo_header_name(index)) + return false; + } - if (decode_full_line) + // If this is a regular header, write header name + ': ' to decoded headers + else { - if (strlen(entry->value) == 0) + if (session_data->header_start_line[source_id] and + !session_data->header_start_line[source_id]->is_finalized()) { - *session_data->infractions[source_id] += INF_STATIC_TABLE_LOOKUP_ERROR; - session_data->events[source_id]->create_event(EVENT_HPACK_INDEX_DECODE_FAILURE); - return false; + if (!session_data->header_start_line[source_id]->finalize()) + return false; } - if (!write_decoded_headers(session_data, source_id, (const uint8_t*)entry->value, - strlen(entry->value), decoded_header_buffer + bytes_written, - decoded_header_length - bytes_written, local_bytes_written)) + + if (!write_decoded_headers(session_data, source_id, (const uint8_t*) entry->name, + strlen(entry->name), decoded_header_buffer, + decoded_header_length, local_bytes_written)) return false; bytes_written += local_bytes_written; - if (!write_decoded_headers(session_data, source_id, (const uint8_t*)"\r\n", 2, - decoded_header_buffer + bytes_written, decoded_header_length - - bytes_written, local_bytes_written)) + if (!write_decoded_headers(session_data, source_id, (const uint8_t*)": ", 2, + decoded_header_buffer + bytes_written, + decoded_header_length - bytes_written, + local_bytes_written)) return false; bytes_written += local_bytes_written; } + if (decode_full_line) + { + if (strlen(entry->value) == 0) + { + *session_data->infractions[source_id] += INF_LOOKUP_EMPTY_VALUE; + session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); + return false; + } + + // Remove second condition after response start line implemented + if (index < PSEUDO_HEADER_MAX_INDEX and session_data->header_start_line[source_id]) + { + session_data->header_start_line[source_id]->process_pseudo_header_value( + (const uint8_t*)entry->value, strlen(entry->value)); + } + else + { + if (!write_decoded_headers(session_data, source_id, (const uint8_t*)entry->value, + strlen(entry->value), decoded_header_buffer + bytes_written, + decoded_header_length - bytes_written, local_bytes_written)) + return false; + bytes_written += local_bytes_written; + if (!write_decoded_headers(session_data, source_id, (const uint8_t*)"\r\n", 2, + decoded_header_buffer + bytes_written, decoded_header_length - + bytes_written, local_bytes_written)) + return false; + bytes_written += local_bytes_written; + } + } + return true; } +// FIXIT-H Implement dynamic table. Currently copies encoded index to decoded headers +bool Http2Hpack::decode_dynamic_table_index(Http2FlowData* session_data, + HttpCommon::SourceId source_id, const uint64_t index, const bool decode_full_line, + uint32_t &bytes_consumed, const uint8_t* encoded_header_buffer, + uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, uint32_t& bytes_written) +{ + UNUSED(index); + UNUSED(decode_full_line); + + //FIXIT-H finalize header_start_line only for regular headers + if (session_data->header_start_line[source_id] and + !session_data->header_start_line[source_id]->is_finalized()) + { + if (!session_data->header_start_line[source_id]->finalize()) + return false; + } + + if(!Http2Hpack::write_decoded_headers(session_data, source_id, encoded_header_buffer, + bytes_consumed, decoded_header_buffer + bytes_written, decoded_header_length, + bytes_written)) + return false; + return true; + +} // FIXIT-H Will be incrementally updated to actually decode indexes. For now just copies encoded // index directly to decoded_header_buffer @@ -182,8 +235,9 @@ bool Http2Hpack::decode_index(Http2FlowData* session_data, HttpCommon::SourceId return decode_static_table_index(session_data, source_id, index, decode_full_line, decoded_header_buffer, decoded_header_length, bytes_written); else - return Http2Hpack::write_decoded_headers(session_data, source_id, encoded_header_buffer, - bytes_consumed, decoded_header_buffer, decoded_header_length, bytes_written); + return decode_dynamic_table_index(session_data, source_id, index, decode_full_line, + bytes_consumed, encoded_header_buffer, decoded_header_buffer, + decoded_header_length, bytes_written); } bool Http2Hpack::decode_literal_header_line(Http2FlowData* session_data, @@ -213,21 +267,49 @@ bool Http2Hpack::decode_literal_header_line(Http2FlowData* session_data, partial_bytes_consumed, decoded_header_buffer, decoded_header_length, partial_bytes_written)) return false; - } - bytes_consumed += partial_bytes_consumed; + // If this was a pseudo-header value, give it to the start-line. + if (session_data->header_start_line[source_id] and + session_data->header_start_line[source_id]->is_pseudo_name( + (const char*) decoded_header_buffer)) + { + // don't include the ': ' that was written following the header name + if (!session_data->header_start_line[source_id]->process_pseudo_header_name( + decoded_header_buffer, partial_bytes_written - 2)) + return false; + } + // If not a pseudo-header value, keep it in the decoded headers + else + { + if (session_data->header_start_line[source_id] and + !session_data->header_start_line[source_id]->is_finalized()) + { + if (!session_data->header_start_line[source_id]->finalize()) + return false; + } + } + } bytes_written += partial_bytes_written; + bytes_consumed += partial_bytes_consumed; // value is always literal if (!Http2Hpack::decode_string_literal(session_data, source_id, encoded_header_buffer + partial_bytes_consumed, encoded_header_length - partial_bytes_consumed, false, partial_bytes_consumed, - decoded_header_buffer + partial_bytes_written, decoded_header_length - - partial_bytes_written, partial_bytes_written)) + decoded_header_buffer + bytes_written, decoded_header_length - + bytes_written, partial_bytes_written)) return false; - bytes_consumed += partial_bytes_consumed; + // If this was a pseudo-header value, give it to the start-line. + if (session_data->header_start_line[source_id] and + session_data->header_start_line[source_id]->is_pseudo_value()) + { + // Subtract 2 from the length to remove the trailing CRLF before passing to the start line + session_data->header_start_line[source_id]->process_pseudo_header_value( + decoded_header_buffer + bytes_written, partial_bytes_written - 2); + } bytes_written += partial_bytes_written; + bytes_consumed += partial_bytes_consumed; return true; } @@ -302,8 +384,9 @@ bool Http2Hpack::decode_header_line(Http2FlowData* session_data, HttpCommon::Sou encoded_header_length, decode_int5, bytes_consumed, bytes_written); } -// FIXIT-H This will eventually be the decoded header buffer. For now only string literals are -// decoded +// FIXIT-H This will eventually be the decoded header buffer. String literals and static table +// indexes are decoded. Dynamic table indexes are not yet decoded. Both the start-line and +// http2_decoded_header need to be sent to NHI bool Http2Hpack::decode_headers(Http2FlowData* session_data, HttpCommon::SourceId source_id, const uint8_t* encoded_header_buffer, const uint32_t header_length) { @@ -311,22 +394,26 @@ bool Http2Hpack::decode_headers(Http2FlowData* session_data, HttpCommon::SourceI uint32_t line_bytes_consumed = 0; uint32_t line_bytes_written = 0; bool success = true; - session_data->http2_decoded_header[source_id] = new uint8_t[MAX_OCTETS]; - session_data->http2_decoded_header_size[source_id] = 0; + session_data->raw_decoded_header[source_id] = new uint8_t[MAX_OCTETS]; + session_data->raw_decoded_header_size[source_id] = 0; + + // FIXIT-H Implement response start line + if (source_id == SRC_CLIENT) + session_data->header_start_line[source_id] = new Http2RequestLine(session_data, source_id); while (total_bytes_consumed < header_length) { if (!Http2Hpack::decode_header_line(session_data, source_id, encoded_header_buffer + total_bytes_consumed, header_length - total_bytes_consumed, - line_bytes_consumed, session_data->http2_decoded_header[source_id] + - session_data->http2_decoded_header_size[source_id], MAX_OCTETS - - session_data->http2_decoded_header_size[source_id], line_bytes_written)) + line_bytes_consumed, session_data->raw_decoded_header[source_id] + + session_data->raw_decoded_header_size[source_id], MAX_OCTETS - + session_data->raw_decoded_header_size[source_id], line_bytes_written)) { success = false; break; } total_bytes_consumed += line_bytes_consumed; - session_data->http2_decoded_header_size[source_id] += line_bytes_written; + session_data->raw_decoded_header_size[source_id] += line_bytes_written; } if (!success) @@ -334,30 +421,51 @@ bool Http2Hpack::decode_headers(Http2FlowData* session_data, HttpCommon::SourceI #ifdef REG_TEST if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP2)) { - fprintf(HttpTestManager::get_output_file(), "Error decoding headers. "); - if (session_data->http2_decoded_header_size[source_id] > 0) - Field(session_data->http2_decoded_header_size[source_id], - session_data->http2_decoded_header[source_id]).print( - HttpTestManager::get_output_file(), "Partially Decoded Header"); + fprintf(HttpTestManager::get_output_file(), "Error decoding headers.\n"); + if (session_data->header_start_line[source_id] and + session_data->header_start_line[source_id]->get_start_line().length() > 0) + session_data->header_start_line[source_id]->get_start_line(). + print(HttpTestManager::get_output_file(), "Decoded start-line"); + if (session_data->raw_decoded_header_size[source_id] > 0) + Field(session_data->raw_decoded_header_size[source_id], + session_data->raw_decoded_header[source_id]).print( + HttpTestManager::get_output_file(), "Partially decoded raw header"); } #endif return false; } + // If there were only pseudo-headers, finalize never got called, so create the start-line + if (session_data->header_start_line[source_id] and + !session_data->header_start_line[source_id]->is_finalized()) + { + if (!session_data->header_start_line[source_id]->finalize()) + return false; + } + // write the last CRLF to end the header if (!Http2Hpack::write_decoded_headers(session_data, source_id, (const uint8_t*)"\r\n", 2, - session_data->http2_decoded_header[source_id] + - session_data->http2_decoded_header_size[source_id], MAX_OCTETS - - session_data->http2_decoded_header_size[source_id], line_bytes_written)) + session_data->raw_decoded_header[source_id] + + session_data->raw_decoded_header_size[source_id], MAX_OCTETS - + session_data->raw_decoded_header_size[source_id], line_bytes_written)) return false; - session_data->http2_decoded_header_size[source_id] += line_bytes_written; + session_data->raw_decoded_header_size[source_id] += line_bytes_written; + + // set http2_decoded_header to send to NHI + session_data->http2_decoded_header[source_id] = new Field( + session_data->raw_decoded_header_size[source_id] - + session_data->pseudo_header_fragment_size[source_id], + session_data->raw_decoded_header[source_id] + + session_data->pseudo_header_fragment_size[source_id], false); #ifdef REG_TEST if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP2)) { - Field(session_data->http2_decoded_header_size[source_id], - session_data->http2_decoded_header[source_id]). - print(HttpTestManager::get_output_file(), "Decoded Header"); + if (session_data->header_start_line[source_id]) + session_data->header_start_line[source_id]->get_start_line(). + print(HttpTestManager::get_output_file(), "Decoded start-line"); + session_data->http2_decoded_header[source_id]-> + print(HttpTestManager::get_output_file(), "Decoded header"); } #endif diff --git a/src/service_inspectors/http2_inspect/http2_hpack.h b/src/service_inspectors/http2_inspect/http2_hpack.h index c5a853814..09c1e31ce 100644 --- a/src/service_inspectors/http2_inspect/http2_hpack.h +++ b/src/service_inspectors/http2_inspect/http2_hpack.h @@ -67,7 +67,11 @@ public: HttpCommon::SourceId source_id, const uint64_t index, const bool decode_full_line, uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, uint32_t& bytes_written); - static bool decode_dynamic_table_index(void) { return true; } + static bool decode_dynamic_table_index(Http2FlowData* session_data, + HttpCommon::SourceId source_id, const uint64_t index, const bool decode_full_line, + uint32_t &bytes_consumed, const uint8_t* encoded_header_buffer, + uint8_t* decoded_header_buffer, const uint32_t decoded_header_length, + uint32_t& bytes_written); static const int STATIC_TABLE_MAX_INDEX = 61; diff --git a/src/service_inspectors/http2_inspect/http2_hpack_table.h b/src/service_inspectors/http2_inspect/http2_hpack_table.h index d1e2a279b..806e62136 100644 --- a/src/service_inspectors/http2_inspect/http2_hpack_table.h +++ b/src/service_inspectors/http2_inspect/http2_hpack_table.h @@ -20,9 +20,12 @@ #ifndef HTTP2_HPACK_TABLE_H #define HTTP2_HPACK_TABLE_H -#include +#include "main/snort_types.h" + +#include "http2_enum.h" #define STATIC_MAX_INDEX 61 +#define PSEUDO_HEADER_MAX_INDEX 14 // Only static table is implemented. lookup() will be extended to support dynamic table // lookups once dynamic table is implemented @@ -31,8 +34,8 @@ class Http2HpackTable public: struct TableEntry { - const char *name; - const char *value; + const char* name; + const char* value; }; const static TableEntry* lookup(uint64_t index); @@ -40,5 +43,4 @@ public: private: const static TableEntry table[STATIC_MAX_INDEX + 1]; }; - #endif diff --git a/src/service_inspectors/http2_inspect/http2_inspect_impl.cc b/src/service_inspectors/http2_inspect/http2_inspect_impl.cc index e70564085..9109b7b07 100644 --- a/src/service_inspectors/http2_inspect/http2_inspect_impl.cc +++ b/src/service_inspectors/http2_inspect/http2_inspect_impl.cc @@ -48,10 +48,11 @@ bool implement_get_buf(unsigned id, Http2FlowData* session_data, SourceId source b.len = session_data->frame_data_size[source_id]; break; case HTTP2_BUFFER_DECODED_HEADER: - if (session_data->http2_decoded_header[source_id] == nullptr) + if (!session_data->http2_decoded_header[source_id] or + session_data->http2_decoded_header[source_id]->length() <= 0) return false; - b.data = session_data->http2_decoded_header[source_id]; - b.len = session_data->http2_decoded_header_size[source_id]; + b.data = session_data->http2_decoded_header[source_id]->start(); + b.len = session_data->http2_decoded_header[source_id]->length(); break; default: return false; diff --git a/src/service_inspectors/http2_inspect/http2_request_line.cc b/src/service_inspectors/http2_inspect/http2_request_line.cc new file mode 100644 index 000000000..1db8d2d35 --- /dev/null +++ b/src/service_inspectors/http2_inspect/http2_request_line.cc @@ -0,0 +1,243 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http2_request_line.cc author Katura Harvey + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "http2_request_line.h" + +#include +#include + +#include "service_inspectors/http_inspect/http_common.h" +#include "service_inspectors/http_inspect/http_field.h" + +#include "http2_enum.h" +#include "http2_flow_data.h" + +using namespace HttpCommon; +using namespace Http2Enums; + +const char* Http2RequestLine::AUTHORITY_NAME = ":authority"; +const char* Http2RequestLine::METHOD_NAME = ":method"; +const char* Http2RequestLine::PATH_NAME = ":path"; +const char* Http2RequestLine::SCHEME_NAME = ":scheme"; +const char* Http2RequestLine::OPTIONS = "OPTIONS"; +const char* Http2RequestLine::CONNECT = "CONNECT"; + +Http2RequestLine::Http2RequestLine(Http2FlowData* _session_data, SourceId _source_id) + : Http2StartLine(_session_data, _source_id) +{ } + +Http2RequestLine::~Http2RequestLine() +{ } + +bool Http2RequestLine::process_pseudo_header_name(const uint64_t index) +{ + if (!process_pseudo_header_precheck()) + return false; + + if (index <= AUTHORITY and authority.length() <= 0) + value_coming = AUTHORITY; + else if (index <= METHOD and method.length() <= 0) + value_coming = METHOD; + else if (index <= PATH and path.length() <= 0) + value_coming = PATH; + else if (index <= SCHEME and scheme.length() <= 0) + value_coming = SCHEME; + else + { + *session_data->infractions[source_id] += INF_INVALID_PSEUDO_HEADER; + session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); + return false; + } + return true; +} + +bool Http2RequestLine::process_pseudo_header_name(const uint8_t* const& name, uint32_t length) +{ + if (!process_pseudo_header_precheck()) + return false; + + if (length == AUTHORITY_NAME_LENGTH and memcmp(name, AUTHORITY_NAME, length) == 0 and + authority.length() <= 0) + value_coming = AUTHORITY; + else if (length == METHOD_NAME_LENGTH and memcmp(name, METHOD_NAME, length) == 0 and + method.length() <= 0) + value_coming = METHOD; + else if (length == PATH_NAME_LENGTH and memcmp(name, PATH_NAME, length) == 0 and + path.length() <= 0) + value_coming = PATH; + else if (length == SCHEME_NAME_LENGTH and memcmp(name, SCHEME_NAME, length) == 0 and + scheme.length() <= 0) + value_coming = SCHEME; + else + { + *session_data->infractions[source_id] += INF_INVALID_PSEUDO_HEADER; + session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); + return false; + } + return true; +} + +void Http2RequestLine::process_pseudo_header_value(const uint8_t* const& value, const uint32_t length) +{ + switch (value_coming) + { + case AUTHORITY: + authority.set(length, value); + break; + case METHOD: + method.set(length, value); + break; + case PATH: + path.set(length, value); + break; + case SCHEME: + scheme.set(length, value); + break; + default: + // should never get here + assert(false); + } + value_coming = HEADER_NONE; +} + +// This is called on the first non-pseudo-header. Select the appropriate URI form based on the +// provided pseudo-headers and generate the start line +bool Http2RequestLine::generate_start_line() +{ + uint8_t *start_line_buffer; + uint32_t bytes_written = 0; + + // Asterisk form - used for OPTIONS requests + if (path.length() > 0 and path.start()[0] == '*') + { + if (method.length() <= 0) + { + *session_data->infractions[source_id] += INF_PSEUDO_HEADER_URI_FORM_MISMATCH; + session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); + return false; + } + start_line_length = method.length() + path.length() + http_version_length + + start_line_extra_chars; + start_line_buffer = new uint8_t[start_line_length]; + + memcpy(start_line_buffer, method.start(), method.length()); + bytes_written += method.length(); + memcpy(start_line_buffer + bytes_written, " ", 1); + bytes_written += 1; + memcpy(start_line_buffer + bytes_written, path.start(), path.length()); + bytes_written += path.length(); + memcpy(start_line_buffer + bytes_written, " ", 1); + bytes_written += 1; + memcpy(start_line_buffer + bytes_written, http_version_string, http_version_length); + bytes_written += http_version_length; + } + // Authority form - used for CONNECT requests + else if (method.length() == CONNECT_LENGTH and memcmp(method.start(), + CONNECT, method.length()) == 0) + { + // Must have an authority and not have a scheme or path + // FIXIT-L May want to be more lenient than RFC on generating start line + if ( scheme.length() > 0 or path.length() > 0 or authority.length() <= 0) + { + *session_data->infractions[source_id] += INF_PSEUDO_HEADER_URI_FORM_MISMATCH; + session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); + return false; + } + start_line_length = method.length() + authority.length() + http_version_length + + start_line_extra_chars; + start_line_buffer = new uint8_t[start_line_length]; + + memcpy(start_line_buffer, method.start(), method.length()); + bytes_written += method.length(); + memcpy(start_line_buffer + bytes_written, " ", 1); + bytes_written += 1; + memcpy(start_line_buffer + bytes_written, authority.start(), authority.length()); + bytes_written += authority.length(); + memcpy(start_line_buffer + bytes_written, " ", 1); + bytes_written += 1; + memcpy(start_line_buffer + bytes_written, http_version_string, http_version_length); + bytes_written += http_version_length; + } + // HTTP/2 requests with URIs in absolute or origin form must have a method, scheme, and length + else if (method.length() > 0 and scheme.length() > 0 and path.length() > 0) + { + // If there is an authority, the URI is in absolute form + if (authority.length() > 0) + { + start_line_length = method.length() + scheme.length() + authority.length() + + path.length() + http_version_length + start_line_extra_chars + + absolute_form_extra_chars_num; + start_line_buffer = new uint8_t[start_line_length]; + + memcpy(start_line_buffer, method.start(), method.length()); + bytes_written += method.length(); + memcpy(start_line_buffer + bytes_written, " ", 1); + bytes_written += 1; + memcpy(start_line_buffer + bytes_written, scheme.start(), scheme.length()); + bytes_written += scheme.length(); + memcpy(start_line_buffer + bytes_written, "://", 3); + bytes_written += 3; + memcpy(start_line_buffer + bytes_written, authority.start(), authority.length()); + bytes_written += authority.length(); + memcpy(start_line_buffer + bytes_written, path.start(), path.length()); + bytes_written += path.length(); + memcpy(start_line_buffer + bytes_written, " ", 1); + bytes_written += 1; + memcpy(start_line_buffer + bytes_written, http_version_string, http_version_length); + bytes_written += http_version_length; + } + // If there is no authority, the URI is in origin form + else + { + start_line_length = method.length() + path.length() + http_version_length + + start_line_extra_chars; + start_line_buffer = new uint8_t[start_line_length]; + + memcpy(start_line_buffer, method.start(), method.length()); + bytes_written += method.length(); + memcpy(start_line_buffer + bytes_written, " ", 1); + bytes_written += 1; + memcpy(start_line_buffer + bytes_written, path.start(), path.length()); + bytes_written += path.length(); + memcpy(start_line_buffer + bytes_written, " ", 1); + bytes_written += 1; + memcpy(start_line_buffer + bytes_written, http_version_string, http_version_length); + bytes_written += http_version_length; + } + } + else + { + // FIXIT-L May want to be more lenient than RFC on generating start line + *session_data->infractions[source_id] += INF_PSEUDO_HEADER_URI_FORM_MISMATCH; + session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); + return false; + } + + memcpy(start_line_buffer + bytes_written, "\r\n", 2); + bytes_written += 2; + assert(bytes_written == start_line_length); + + start_line.set(start_line_length, start_line_buffer, true); + + return true; +} diff --git a/src/service_inspectors/http2_inspect/http2_request_line.h b/src/service_inspectors/http2_inspect/http2_request_line.h new file mode 100644 index 000000000..088f41ad2 --- /dev/null +++ b/src/service_inspectors/http2_inspect/http2_request_line.h @@ -0,0 +1,65 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http2_request_line.h author Katura Harvey + +#ifndef HTTP2_REQUEST_LINE_H +#define HTTP2_REQUEST_LINE_H + +#include "service_inspectors/http_inspect/http_common.h" +#include "service_inspectors/http_inspect/http_field.h" + +#include "http2_enum.h" +#include "http2_start_line.h" + +class Http2RequestLine : public Http2StartLine +{ +public: + Http2RequestLine(Http2FlowData* session_data, HttpCommon::SourceId source_id); + ~Http2RequestLine() override; + + bool process_pseudo_header_name(const uint64_t index) override; + bool process_pseudo_header_name(const uint8_t* const& name, uint32_t length) override; + void process_pseudo_header_value(const uint8_t* const& value, const uint32_t length) override; + bool generate_start_line() override; + +private: + Field method; + Field path; + Field scheme; + Field authority; + + static const char* AUTHORITY_NAME; + static const uint32_t AUTHORITY_NAME_LENGTH = 10; + static const char* METHOD_NAME; + static const uint32_t METHOD_NAME_LENGTH = 7; + static const char* PATH_NAME; + static const uint32_t PATH_NAME_LENGTH = 5; + static const char* SCHEME_NAME; + static const uint32_t SCHEME_NAME_LENGTH = 7; + + // Methods that have special URI forms. Subtract 1 from the sizeof for the terminating null byte + static const char* OPTIONS; + static const int32_t OPTIONS_LENGTH = 7; + static const char* CONNECT; + static const int32_t CONNECT_LENGTH = 7; + + // absolute form adds '://' between scheme and authority + static const uint32_t absolute_form_extra_chars_num = 3; +}; + +#endif diff --git a/src/service_inspectors/http2_inspect/http2_start_line.cc b/src/service_inspectors/http2_inspect/http2_start_line.cc new file mode 100644 index 000000000..2515e1342 --- /dev/null +++ b/src/service_inspectors/http2_inspect/http2_start_line.cc @@ -0,0 +1,62 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http2_start_line.cc author Katura Harvey + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "http2_start_line.h" + +#include "service_inspectors/http_inspect/http_common.h" +#include "service_inspectors/http_inspect/http_field.h" + +#include "http2_enum.h" +#include "http2_flow_data.h" + +using namespace HttpCommon; +using namespace Http2Enums; + +const char* Http2StartLine::http_version_string = "HTTP/1.1"; + +Http2StartLine::Http2StartLine(Http2FlowData* _session_data, HttpCommon::SourceId _source_id) : + session_data(_session_data), source_id(_source_id) +{ } + +bool Http2StartLine::process_pseudo_header_precheck() +{ + if (finalized) + { + *session_data->infractions[source_id] += INF_PSEUDO_HEADER_AFTER_REGULAR_HEADER; + session_data->events[source_id]->create_event(EVENT_MISFORMATTED_HTTP2); + return false; + } + return true; +} + +bool Http2StartLine::finalize() +{ + finalized = true; + + // Save the current position in the raw decoded buffer so we can set the pointer to the start + // of the regular headers + session_data->pseudo_header_fragment_size[source_id] = + session_data->raw_decoded_header_size[source_id]; + + return generate_start_line(); +} diff --git a/src/service_inspectors/http2_inspect/http2_start_line.h b/src/service_inspectors/http2_inspect/http2_start_line.h new file mode 100644 index 000000000..df5a92f2c --- /dev/null +++ b/src/service_inspectors/http2_inspect/http2_start_line.h @@ -0,0 +1,66 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2019-2019 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http2_start_line.h author Katura Harvey + +#ifndef HTTP2_START_LINE_H +#define HTTP2_START_LINE_H + +#include "service_inspectors/http_inspect/http_common.h" +#include "service_inspectors/http_inspect/http_field.h" + +#include "http2_enum.h" + +class Http2FlowData; + +class Http2StartLine +{ +public: + Http2StartLine(Http2FlowData* _session_data, HttpCommon::SourceId _source_id); + virtual ~Http2StartLine() = default; + + friend class Http2Hpack; + + const Field& get_start_line() { return start_line; } + virtual bool process_pseudo_header_name(const uint64_t index) = 0; + virtual bool process_pseudo_header_name(const uint8_t* const& name, uint32_t length) = 0; + virtual void process_pseudo_header_value(const uint8_t* const& value, const uint32_t length) = 0; + bool finalize(); + bool is_finalized() { return finalized; } + uint32_t get_start_line_length() { return start_line_length; } + bool is_pseudo_value() { return value_coming != Http2Enums::HEADER_NONE; } + bool is_pseudo_name(const char* const& name) { return name[0] == ':'; } + +protected: + bool process_pseudo_header_precheck(); + virtual bool generate_start_line() = 0; + + Field start_line; + Http2FlowData* session_data; + HttpCommon::SourceId source_id; + bool finalized = false; + uint32_t start_line_length = 0; + Http2Enums::PseudoHeaders value_coming = Http2Enums::HEADER_NONE; + + // Version string is HTTP/1.1 + static const char* http_version_string; + static const uint8_t http_version_length = 8; + // Account for two spaces, and trailing crlf + static const uint8_t start_line_extra_chars = 4; +}; + +#endif diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc index 481d47c09..cd250c3c6 100644 --- a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc +++ b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc @@ -393,7 +393,7 @@ const StreamBuffer implement_reassemble(Http2FlowData* session_data, unsigned to { if (get_frame_type(session_data->frame_header[source_id]) == FT_HEADERS) { - assert(session_data->http2_decoded_header[source_id] == nullptr); + assert(session_data->raw_decoded_header[source_id] == nullptr); // FIXIT-H This will eventually be the decoded header buffer. Under development. if (!Http2Hpack::decode_headers(session_data, source_id, diff --git a/src/service_inspectors/http2_inspect/http2_tables.cc b/src/service_inspectors/http2_inspect/http2_tables.cc index fc1a4a2dd..04a44eb70 100644 --- a/src/service_inspectors/http2_inspect/http2_tables.cc +++ b/src/service_inspectors/http2_inspect/http2_tables.cc @@ -38,7 +38,6 @@ const RuleMap Http2Module::http2_events[] = { EVENT_UNEXPECTED_CONTINUATION, "unexpected continuation frame"}, { EVENT_MISFORMATTED_HTTP2, "misformatted HTTP/2 traffic"}, { EVENT_PREFACE_MATCH_FAILURE, "HTTP/2 connection preface does not match"}, - { EVENT_HPACK_INDEX_DECODE_FAILURE, "error in decoding HPACK indexed field"}, { 0, nullptr } }; diff --git a/src/service_inspectors/http2_inspect/test/CMakeLists.txt b/src/service_inspectors/http2_inspect/test/CMakeLists.txt index 3903bdb7b..c3882abd7 100644 --- a/src/service_inspectors/http2_inspect/test/CMakeLists.txt +++ b/src/service_inspectors/http2_inspect/test/CMakeLists.txt @@ -16,6 +16,8 @@ add_cpputest( http2_stream_splitter_impl_test ../http2_stream_splitter_impl.cc ../http2_hpack.cc ../http2_module.cc + ../http2_request_line.cc + ../http2_start_line.cc ../http2_tables.cc ../../../framework/module.cc ) diff --git a/src/service_inspectors/http2_inspect/test/http2_stream_splitter_impl_test.cc b/src/service_inspectors/http2_inspect/test/http2_stream_splitter_impl_test.cc index 41577742e..a8ce23b87 100644 --- a/src/service_inspectors/http2_inspect/test/http2_stream_splitter_impl_test.cc +++ b/src/service_inspectors/http2_inspect/test/http2_stream_splitter_impl_test.cc @@ -50,6 +50,7 @@ unsigned HttpTestManager::test_output = IN_NONE; FILE* HttpTestManager::test_out = nullptr ; int DetectionEngine::queue_event(unsigned int, unsigned int, Actions::Type) { return 0; } void Field::print(_IO_FILE*, char const*) const {} +void Field::set(int, unsigned char const*, bool) {} TEST_GROUP(http2_scan_test) {