From: Tom Peters (thopeter) Date: Tue, 8 Feb 2022 00:38:02 +0000 (+0000) Subject: Pull request #3258: http_inspect: HttpStreamSplitter::reassemble verifies gzip file... X-Git-Tag: 3.1.23.0~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=ad2faf2c19dfbda13942c843f8eb7994a2571b1e;p=thirdparty%2Fsnort3.git Pull request #3258: http_inspect: HttpStreamSplitter::reassemble verifies gzip file magic and checks for FEXTRA flag Merge in SNORT/snort3 from ~KATHARVE/snort3:http_gzip_fextra to master Squashed commit of the following: commit 63e64d99166c241f253be1c1ce088dbf3e2d4e23 Author: Katura Harvey Date: Wed Jan 26 12:02:52 2022 -0500 http_inspect: HttpStreamSplitter::reassemble verifies gzip file magic and checks for FEXTRA flag --- diff --git a/doc/reference/builtin_stubs.txt b/doc/reference/builtin_stubs.txt index 0254995dd..eb68855cf 100644 --- a/doc/reference/builtin_stubs.txt +++ b/doc/reference/builtin_stubs.txt @@ -1295,6 +1295,10 @@ The HTTP version in the start line has a valid format but the version is 0. The HTTP version in the start line has a valid format but the version is higher than 1. This alert does not apply to HTTP/2 or HTTP/3 traffic. +119:277 + +The HTTP message body is gzip encoded and the FEXTRA flag is set in the gzip header. + 121:1 Invalid flag set on HTTP/2 frame header diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h index 4c87c9657..beb11bc03 100755 --- a/src/service_inspectors/http_inspect/http_enum.h +++ b/src/service_inspectors/http_inspect/http_enum.h @@ -37,6 +37,8 @@ static const int REQUEST_PUBLISH_DEPTH = 2000; static const uint32_t HTTP_GID = 119; static const int GZIP_WINDOW_BITS = 31; +static const uint8_t GZIP_HEADER_FLAG_OFFSET = 3; +static const uint8_t GZIP_FLAG_FEXTRA = 0x4; static const int DEFLATE_WINDOW_BITS = 15; static const int MAX_FIELD_NAME_LENGTH = 100; // Plan to support max 8 xff headers @@ -113,6 +115,9 @@ enum UriType { URI__NOT_COMPUTE=-14, URI__PROBLEMATIC=-12, URI_ASTERISK = 2, URI // Body compression types enum CompressId { CMP_NONE=2, CMP_GZIP, CMP_DEFLATE }; +// GZIP magic verification state +enum GzipVerificationState { GZIP_TBD, GZIP_MAGIC_BAD, GZIP_MAGIC_GOOD, GZIP_FLAGS_PROCESSED }; + // Message section in which an IPS option provides the buffer enum InspectSection { IS_NONE, IS_HEADER, IS_FLEX_HEADER, IS_FIRST_BODY, IS_BODY, IS_TRAILER }; @@ -289,6 +294,7 @@ enum Infraction INF_JS_SCOPE_NEST_OVERFLOW = 132, INF_INVALID_SUBVERSION = 133, INF_VERSION_0 = 134, + INF_GZIP_FEXTRA = 135, INF__MAX_VALUE }; @@ -426,6 +432,7 @@ enum EventSid EVENT_INVALID_SUBVERSION = 275, EVENT_VERSION_0 = 276, EVENT_VERSION_HIGHER_THAN_1 = 277, + EVENT_GZIP_FEXTRA = 278, EVENT__MAX_VALUE }; diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc index cc16c897b..c5b98df68 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.cc +++ b/src/service_inspectors/http_inspect/http_flow_data.cc @@ -161,6 +161,8 @@ void HttpFlowData::half_reset(SourceId source_id) detection_status[source_id] = DET_REACTIVATING; compression[source_id] = CMP_NONE; + gzip_state[source_id] = GZIP_TBD; + gzip_header_bytes_processed[source_id] = 0; if (compress_stream[source_id] != nullptr) { inflateEnd(compress_stream[source_id]); diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h index 23f4d2377..2dbea4dbe 100644 --- a/src/service_inspectors/http_inspect/http_flow_data.h +++ b/src/service_inspectors/http_inspect/http_flow_data.h @@ -113,6 +113,9 @@ private: uint32_t partial_raw_bytes[2] = { 0, 0 }; uint8_t* partial_buffer[2] = { nullptr, nullptr }; uint32_t partial_buffer_length[2] = { 0, 0 }; + uint32_t gzip_header_bytes_processed[2] = { 0, 0 }; + HttpEnums::GzipVerificationState gzip_state[2] = { HttpEnums::GZIP_TBD, HttpEnums::GZIP_TBD }; + bool gzip_header_check_done(); // *** StreamSplitter internal data - scan() => reassemble() uint32_t num_excess[2] = { 0, 0 }; diff --git a/src/service_inspectors/http_inspect/http_stream_splitter.h b/src/service_inspectors/http_inspect/http_stream_splitter.h index 8c756e6e2..0f6781a8e 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter.h +++ b/src/service_inspectors/http_inspect/http_stream_splitter.h @@ -58,10 +58,13 @@ private: HttpCutter* get_cutter(HttpEnums::SectionType type, HttpFlowData* session) const; void chunk_spray(HttpFlowData* session_data, uint8_t* buffer, const uint8_t* data, unsigned length) const; - static void decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data, + void decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data, uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream, bool at_start, HttpInfractions* infractions, HttpEventGen* events, - HttpFlowData* session_data); + HttpFlowData* session_data) const; + void process_gzip_header(const uint8_t* data, + uint32_t length, HttpFlowData* session_data) const; + bool gzip_header_check_done(HttpFlowData* session_data) const; HttpInspect* const my_inspector; const HttpCommon::SourceId source_id; diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc index 1bdc60c36..e06d67097 100644 --- a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc +++ b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc @@ -139,12 +139,53 @@ void HttpStreamSplitter::chunk_spray(HttpFlowData* session_data, uint8_t* buffer } } +void HttpStreamSplitter::process_gzip_header(const uint8_t* data, + uint32_t length, HttpFlowData* session_data) const +{ + uint32_t& header_bytes_processed = session_data->gzip_header_bytes_processed[source_id]; + uint32_t input_bytes_processed = 0; + if (session_data->gzip_state[source_id] == GZIP_TBD) + { + static const uint8_t gzip_magic[] = {0x1f, 0x8b, 0x08}; + static const uint8_t magic_length = 3; + const uint32_t magic_cmp_len = (magic_length - header_bytes_processed) < length ? + (magic_length - header_bytes_processed) : length; + + if (memcmp(data, gzip_magic + header_bytes_processed, magic_cmp_len)) + session_data->gzip_state[source_id] = GZIP_MAGIC_BAD; + else if (header_bytes_processed + length >= magic_length) + session_data->gzip_state[source_id] = GZIP_MAGIC_GOOD; + header_bytes_processed += magic_cmp_len; + input_bytes_processed += magic_cmp_len; + } + if (session_data->gzip_state[source_id] == GZIP_MAGIC_GOOD and length > input_bytes_processed) + { + const uint8_t gzip_flags = data[input_bytes_processed]; + if (gzip_flags & GZIP_FLAG_FEXTRA) + { + *session_data->get_infractions(source_id) += INF_GZIP_FEXTRA; + session_data->events[source_id]->create_event(EVENT_GZIP_FEXTRA); + } + header_bytes_processed++; + session_data->gzip_state[source_id] = GZIP_FLAGS_PROCESSED; + } +} + +bool HttpStreamSplitter::gzip_header_check_done(HttpFlowData* session_data) const +{ + return session_data->gzip_state[source_id] == HttpEnums::GZIP_MAGIC_BAD or + session_data->gzip_state[source_id] == HttpEnums::GZIP_FLAGS_PROCESSED; +} + void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data, uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream, - bool at_start, HttpInfractions* infractions, HttpEventGen* events, HttpFlowData* session_data) + bool at_start, HttpInfractions* infractions, HttpEventGen* events, HttpFlowData* session_data) const { if ((compression == CMP_GZIP) || (compression == CMP_DEFLATE)) { + if (compression == CMP_GZIP and !gzip_header_check_done(session_data)) + process_gzip_header(data, length, session_data); + compress_stream->next_in = const_cast(data); compress_stream->avail_in = length; compress_stream->next_out = buffer + offset; @@ -179,6 +220,8 @@ void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, cons inflateEnd(compress_stream); delete compress_stream; compress_stream = nullptr; + // FIXIT-E - Will need to clear gzip header processing state here when we implement + // processing multiple gzip members in a message section } return; } diff --git a/src/service_inspectors/http_inspect/http_tables.cc b/src/service_inspectors/http_inspect/http_tables.cc index 2a72f096c..39f1dda20 100755 --- a/src/service_inspectors/http_inspect/http_tables.cc +++ b/src/service_inspectors/http_inspect/http_tables.cc @@ -337,6 +337,7 @@ const RuleMap HttpModule::http_events[] = { EVENT_INVALID_SUBVERSION, "HTTP/1 version other than 1.0 or 1.1" }, { EVENT_VERSION_0, "HTTP version in start line is 0" }, { EVENT_VERSION_HIGHER_THAN_1, "HTTP version in start line is higher than 1" }, + { EVENT_GZIP_FEXTRA, "HTTP gzip body with the FEXTRA flag set" }, { 0, nullptr } };