Pull request #3258: http_inspect: HttpStreamSplitter::reassemble verifies gzip file...

Merge in SNORT/snort3 from ~KATHARVE/snort3:http_gzip_fextra to master

Squashed commit of the following:

commit 63e64d99166c241f253be1c1ce088dbf3e2d4e23
Author: Katura Harvey <katharve@cisco.com>
Date:   Wed Jan 26 12:02:52 2022 -0500

    http_inspect: HttpStreamSplitter::reassemble verifies gzip file magic and checks for FEXTRA flag

diff --git a/doc/reference/builtin_stubs.txt b/doc/reference/builtin_stubs.txt
index 0254995dd8f229ca37d43a521109336fa718b8ec..eb68855cf4e58a2b9d07df1551170157541395f8 100644 (file)
--- a/doc/reference/builtin_stubs.txt
+++ b/doc/reference/builtin_stubs.txt
@@ -1295,6 +1295,10 @@ The HTTP version in the start line has a valid format but the version is 0.
 The HTTP version in the start line has a valid format but the version is higher than 1. This alert
 does not apply to HTTP/2 or HTTP/3 traffic.
 
+119:277
+
+The HTTP message body is gzip encoded and the FEXTRA flag is set in the gzip header.
+
 121:1
 
 Invalid flag set on HTTP/2 frame header

diff --git a/src/service_inspectors/http_inspect/http_enum.h b/src/service_inspectors/http_inspect/http_enum.h
index 4c87c9657e1b8cbaf18a97e882cd5dee41271302..beb11bc039d5a096392f79de725b547ff733b3b4 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_enum.h
+++ b/src/service_inspectors/http_inspect/http_enum.h
@@ -37,6 +37,8 @@ static const int REQUEST_PUBLISH_DEPTH = 2000;
 
 static const uint32_t HTTP_GID = 119;
 static const int GZIP_WINDOW_BITS = 31;
+static const uint8_t GZIP_HEADER_FLAG_OFFSET = 3;
+static const uint8_t GZIP_FLAG_FEXTRA = 0x4;
 static const int DEFLATE_WINDOW_BITS = 15;
 static const int MAX_FIELD_NAME_LENGTH = 100;
 // Plan to support max 8 xff headers
@@ -113,6 +115,9 @@ enum UriType { URI__NOT_COMPUTE=-14, URI__PROBLEMATIC=-12, URI_ASTERISK = 2, URI
 // Body compression types
 enum CompressId { CMP_NONE=2, CMP_GZIP, CMP_DEFLATE };
 
+// GZIP magic verification state
+enum GzipVerificationState { GZIP_TBD, GZIP_MAGIC_BAD, GZIP_MAGIC_GOOD, GZIP_FLAGS_PROCESSED };
+
 // Message section in which an IPS option provides the buffer
 enum InspectSection { IS_NONE, IS_HEADER, IS_FLEX_HEADER, IS_FIRST_BODY, IS_BODY, IS_TRAILER };
 
@@ -289,6 +294,7 @@ enum Infraction
     INF_JS_SCOPE_NEST_OVERFLOW = 132,
     INF_INVALID_SUBVERSION = 133,
     INF_VERSION_0 = 134,
+    INF_GZIP_FEXTRA = 135,
     INF__MAX_VALUE
 };
 
@@ -426,6 +432,7 @@ enum EventSid
     EVENT_INVALID_SUBVERSION = 275,
     EVENT_VERSION_0 = 276,
     EVENT_VERSION_HIGHER_THAN_1 = 277,
+    EVENT_GZIP_FEXTRA = 278,
     EVENT__MAX_VALUE
 };
 

diff --git a/src/service_inspectors/http_inspect/http_flow_data.cc b/src/service_inspectors/http_inspect/http_flow_data.cc
index cc16c897ba63d240449d419f99dd53da8b387b02..c5b98df68bf2685e1c3a7032ebffa7b7a5e47b85 100644 (file)
--- a/src/service_inspectors/http_inspect/http_flow_data.cc
+++ b/src/service_inspectors/http_inspect/http_flow_data.cc
@@ -161,6 +161,8 @@ void HttpFlowData::half_reset(SourceId source_id)
     detection_status[source_id] = DET_REACTIVATING;
 
     compression[source_id] = CMP_NONE;
+    gzip_state[source_id] = GZIP_TBD;
+    gzip_header_bytes_processed[source_id] = 0;
     if (compress_stream[source_id] != nullptr)
     {
         inflateEnd(compress_stream[source_id]);

diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h
index 23f4d237714f6bdca8f1a42d841d510056eac142..2dbea4dbeb48fb9572afc8c928783a28d1c3d5ce 100644 (file)
--- a/src/service_inspectors/http_inspect/http_flow_data.h
+++ b/src/service_inspectors/http_inspect/http_flow_data.h
@@ -113,6 +113,9 @@ private:
     uint32_t partial_raw_bytes[2] = { 0, 0 };
     uint8_t* partial_buffer[2] = { nullptr, nullptr };
     uint32_t partial_buffer_length[2] = { 0, 0 };
+    uint32_t gzip_header_bytes_processed[2] = { 0, 0 };
+    HttpEnums::GzipVerificationState gzip_state[2] = { HttpEnums::GZIP_TBD, HttpEnums::GZIP_TBD };
+    bool gzip_header_check_done();
 
     // *** StreamSplitter internal data - scan() => reassemble()
     uint32_t num_excess[2] = { 0, 0 };

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter.h b/src/service_inspectors/http_inspect/http_stream_splitter.h
index 8c756e6e2d70b29755bbf3533c0e66c8dad122ba..0f6781a8eb65bc9e3dc634cf3763f669c58221bb 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter.h
+++ b/src/service_inspectors/http_inspect/http_stream_splitter.h
@@ -58,10 +58,13 @@ private:
     HttpCutter* get_cutter(HttpEnums::SectionType type, HttpFlowData* session) const;
     void chunk_spray(HttpFlowData* session_data, uint8_t* buffer, const uint8_t* data,
         unsigned length) const;
-    static void decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data,
+    void decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data,
         uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream,
         bool at_start, HttpInfractions* infractions, HttpEventGen* events,
-        HttpFlowData* session_data);
+        HttpFlowData* session_data) const;
+    void process_gzip_header(const uint8_t* data,
+        uint32_t length, HttpFlowData* session_data) const;
+    bool gzip_header_check_done(HttpFlowData* session_data) const;
 
     HttpInspect* const my_inspector;
     const HttpCommon::SourceId source_id;

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
index 1bdc60c366f2068fcf2fbfef95b8a72158767efe..e06d67097503fbfb3490b1b871cda5f768e0d909 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
@@ -139,12 +139,53 @@ void HttpStreamSplitter::chunk_spray(HttpFlowData* session_data, uint8_t* buffer
     }
 }
 
+void HttpStreamSplitter::process_gzip_header(const uint8_t* data,
+    uint32_t length, HttpFlowData* session_data) const
+{
+    uint32_t& header_bytes_processed = session_data->gzip_header_bytes_processed[source_id];
+    uint32_t input_bytes_processed = 0;
+    if (session_data->gzip_state[source_id] == GZIP_TBD)
+    {
+        static const uint8_t gzip_magic[] = {0x1f, 0x8b, 0x08};
+        static const uint8_t magic_length = 3;
+        const uint32_t magic_cmp_len = (magic_length - header_bytes_processed) < length ?
+            (magic_length - header_bytes_processed) : length;
+
+        if (memcmp(data, gzip_magic + header_bytes_processed, magic_cmp_len))
+            session_data->gzip_state[source_id] = GZIP_MAGIC_BAD; 
+        else if (header_bytes_processed + length >= magic_length)
+            session_data->gzip_state[source_id] = GZIP_MAGIC_GOOD;
+        header_bytes_processed += magic_cmp_len;
+        input_bytes_processed += magic_cmp_len;
+    }
+    if (session_data->gzip_state[source_id] == GZIP_MAGIC_GOOD and length > input_bytes_processed)
+    {
+        const uint8_t gzip_flags = data[input_bytes_processed];
+        if (gzip_flags & GZIP_FLAG_FEXTRA)
+        {
+            *session_data->get_infractions(source_id) += INF_GZIP_FEXTRA;
+            session_data->events[source_id]->create_event(EVENT_GZIP_FEXTRA);
+        }
+        header_bytes_processed++;
+        session_data->gzip_state[source_id] = GZIP_FLAGS_PROCESSED;
+    }
+}
+
+bool HttpStreamSplitter::gzip_header_check_done(HttpFlowData* session_data) const
+{ 
+    return session_data->gzip_state[source_id] == HttpEnums::GZIP_MAGIC_BAD or
+        session_data->gzip_state[source_id] == HttpEnums::GZIP_FLAGS_PROCESSED;
+}
+
 void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data,
     uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream,
-    bool at_start, HttpInfractions* infractions, HttpEventGen* events, HttpFlowData* session_data)
+    bool at_start, HttpInfractions* infractions, HttpEventGen* events, HttpFlowData* session_data) const
 {
     if ((compression == CMP_GZIP) || (compression == CMP_DEFLATE))
     {
+        if (compression == CMP_GZIP and !gzip_header_check_done(session_data))
+            process_gzip_header(data, length, session_data);
+
         compress_stream->next_in = const_cast<Bytef*>(data);
         compress_stream->avail_in = length;
         compress_stream->next_out = buffer + offset;
@@ -179,6 +220,8 @@ void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, cons
                 inflateEnd(compress_stream);
                 delete compress_stream;
                 compress_stream = nullptr;
+                // FIXIT-E - Will need to clear gzip header processing state here when we implement
+                // processing multiple gzip members in a message section
             }
             return;
         }

diff --git a/src/service_inspectors/http_inspect/http_tables.cc b/src/service_inspectors/http_inspect/http_tables.cc
index 2a72f096cb63322d1d25f90e10e7e0f5fc11afd7..39f1dda20b784d6f68f17544d6526e7c784bd487 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_tables.cc
+++ b/src/service_inspectors/http_inspect/http_tables.cc
@@ -337,6 +337,7 @@ const RuleMap HttpModule::http_events[] =
     { EVENT_INVALID_SUBVERSION,         "HTTP/1 version other than 1.0 or 1.1" },
     { EVENT_VERSION_0,                  "HTTP version in start line is 0" },
     { EVENT_VERSION_HIGHER_THAN_1,      "HTTP version in start line is higher than 1" },
+    { EVENT_GZIP_FEXTRA,                "HTTP gzip body with the FEXTRA flag set" },
     { 0, nullptr }
 };