Pull request #4855: http_inspect: partial inspection for headers

Merge in SNORT/snort3 from ~ADMAMOLE/snort3:part_header2 to master

Squashed commit of the following:

commit f75941d810813f2aba755e0b6acfd11d377f3387
Author: Adrian Mamolea <admamole@cisco.com>
Date:   Fri Jun 20 14:58:10 2025 -0400

    http_inspect: partial inspection for headers

diff --git a/doc/user/http_inspect.txt b/doc/user/http_inspect.txt
index e80f6ea998b8c41539088634b21f64f9ee79fafd..66c319e0ed9cc0bf2c0ab3c0fb22588bc0aff5fc 100755 (executable)
--- a/doc/user/http_inspect.txt
+++ b/doc/user/http_inspect.txt
@@ -148,20 +148,25 @@ more of the sensor's resources.
 
 This feature is off by default. script_detection = true will activate it.
 
-===== partial_depth_body
-
-Partial depth detection is a feature that enables Snort to more quickly detect
-and block malicious requests. It is configured by the partial_depth_body parameter
-which can take values in the range -1-16384 bytes. The feature is enabled by
-setting partial_depth_body to some non zero value. When the feature is enabled and
-either, the number of bytes received in the request body is below the value
-specified by partial_depth_body, or partial_depth_body is set to -1, unlimited; it
-immediately forwards the available part of the message body for early detection.
-This enables earlier threat detection but consumes somewhat more of the sensor's
-resources.
-
-This feature is turned off by default by setting partial_depth_body = 0. To activate
-it, set partial_depth_body to the desired value.
+===== partial_depth_body and partial_depth_header
+
+Partial depth detection enables faster threat detection by immediately forwarding
+partial message data to the detection engine before the complete message arrives.
+This feature can be configured independently for HTTP request bodies
+(partial_depth_body) and headers (partial_depth_header).
+
+    Configuration options:
+    0 (default):    Feature disabled
+    -1 (unlimited): Enable for all partial messages regardless of size
+    Positive value: Enable only when received bytes are below the specified threshold
+
+For HTTP request bodies only, the maximum configurable value is 16,384 bytes. Use
+partial_depth_body = -1 when early detection is needed beyond this limit. HTTP
+headers have no such limitation.
+
+This feature is turned off by default by setting partial_depth_body = 0
+and partial_depth_header = 0. To activate it, set the corresponding parameter to
+the desired value.
 
 ===== gzip
 

diff --git a/src/pub_sub/test/pub_sub_http_transaction_end_event_test.cc b/src/pub_sub/test/pub_sub_http_transaction_end_event_test.cc
index fe54bbc5c1602db2f287e72c44e80e8f4f26d089..116265a0cb6b4e924e74f5ed0596e63c748c5bbd 100644 (file)
--- a/src/pub_sub/test/pub_sub_http_transaction_end_event_test.cc
+++ b/src/pub_sub/test/pub_sub_http_transaction_end_event_test.cc
@@ -103,7 +103,7 @@ const snort::StreamBuffer HttpStreamSplitter::reassemble(snort::Flow*, unsigned,
     return buf;
 }
 bool HttpStreamSplitter::finish(snort::Flow*) { return false; }
-void HttpStreamSplitter::prep_partial_flush(snort::Flow*, uint32_t) { }
+void HttpStreamSplitter::prep_partial_flush(snort::Flow*, uint32_t, uint32_t, uint32_t) { }
 
 HttpMsgHeader::HttpMsgHeader(const uint8_t* buffer, const uint16_t buf_size,
     HttpFlowData* session_data_, SourceId source_id_, bool buf_owner, Flow* flow_,

diff --git a/src/service_inspectors/http_inspect/dev_notes_partial_inspection.txt b/src/service_inspectors/http_inspect/dev_notes_partial_inspection.txt
index 7ac69a63abb312297cc300d9b03636f9cfd23acf..b25574acf45d5b401375687af128e1ef715655c8 100644 (file)
--- a/src/service_inspectors/http_inspect/dev_notes_partial_inspection.txt
+++ b/src/service_inspectors/http_inspect/dev_notes_partial_inspection.txt
@@ -22,6 +22,10 @@ Compared to just doing a full inspection, a partial inspection followed by a ful
 will not miss anything. The benefits of partial inspection are in addition to the benefits of a
 full inspection.
 
+Header inspections, partial or not, are done on the latest reassembled buffer. The result of
+a previous partial inspection is ignored by deleting the previous header from HttpTransaction.
+Partial header inspections ignore truncated fields at the end of the header fragment.
+
 Partial inspection is used in multiple scenarios, described below, and in combinations of them.
 
 Script detection uses partial inspection for message bodies containing Javascripts. The stream

diff --git a/src/service_inspectors/http_inspect/http_cutter.h b/src/service_inspectors/http_inspect/http_cutter.h
index 4430f3593182436c24490422c1ab25ea89f42b6e..7e6016477ac449cf6db32f7a0396111353498b91 100644 (file)
--- a/src/service_inspectors/http_inspect/http_cutter.h
+++ b/src/service_inspectors/http_inspect/http_cutter.h
@@ -113,7 +113,7 @@ public:
 private:
     enum LineEndState { ZERO, HALF, ONE, THREEHALF };
     LineEndState state = ONE;
-    int32_t num_head_lines = 0;
+    uint32_t num_head_lines = 0;
 };
 
 class HttpBodyCutter : public HttpCutter

diff --git a/src/service_inspectors/http_inspect/http_inspect.cc b/src/service_inspectors/http_inspect/http_inspect.cc
index 8901d06cc380058a5d0b6969e293773e1d9144a0..b5f51c4403ae7095bffbc1c8753f2c4018834b7d 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_inspect.cc
+++ b/src/service_inspectors/http_inspect/http_inspect.cc
@@ -158,6 +158,7 @@ void HttpInspect::show(const SnortConfig*) const
 
     ConfigLogger::log_limit("request_depth", params->request_depth, -1);
     ConfigLogger::log_limit("response_depth", params->response_depth, -1);
+    ConfigLogger::log_limit("partial_depth_header", params->partial_depth_header, -1, 0);
     ConfigLogger::log_limit("partial_depth_body", params->partial_depth_body, -1, 0);
     ConfigLogger::log_flag("unzip", params->unzip);
     ConfigLogger::log_flag("normalize_utf", params->normalize_utf);

diff --git a/src/service_inspectors/http_inspect/http_module.cc b/src/service_inspectors/http_inspect/http_module.cc
index 551d29f597ec78e50c77fd241c28ff398ceb8349..6d6f18424b0c6cca110f295b2c657d73e2939bde 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_module.cc
+++ b/src/service_inspectors/http_inspect/http_module.cc
@@ -52,6 +52,9 @@ const Parameter HttpModule::http_params[] =
     { "response_depth", Parameter::PT_INT, "-1:max53", "-1",
       "maximum response message body bytes to examine (-1 no limit)" },
 
+    { "partial_depth_header", Parameter::PT_INT, "-1:max53", "0",
+      "maximum request header to send to early detection (0 disabled, -1 no limit)" },
+
     { "partial_depth_body", Parameter::PT_INT, "-1:16384", "0",
       "maximum request body to send to early detection (0 disabled, -1 no limit)" },
 
@@ -212,6 +215,10 @@ bool HttpModule::set(const char*, Value& val, SnortConfig*)
     {
         params->response_depth = val.get_int64();
     }
+    else if (val.is("partial_depth_header"))
+    {
+        params->partial_depth_header = val.get_int64();
+    }
     else if (val.is("partial_depth_body"))
     {
         params->partial_depth_body = val.get_int64();

diff --git a/src/service_inspectors/http_inspect/http_module.h b/src/service_inspectors/http_inspect/http_module.h
index 546ce481bacf7bb558d1a77e812943e804e8aeb4..fe8db282e74fbd8794ab45bfa1d31240b916d999 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_module.h
+++ b/src/service_inspectors/http_inspect/http_module.h
@@ -49,6 +49,7 @@ public:
     ~HttpParaList();
     int64_t request_depth = -1;
     int64_t response_depth = -1;
+    int64_t partial_depth_header = 0;
     int64_t partial_depth_body = 0;
 
     bool unzip = true;

diff --git a/src/service_inspectors/http_inspect/http_msg_head_shared.cc b/src/service_inspectors/http_inspect/http_msg_head_shared.cc
index 2ab0535a32dcf489686e20665d84df449fd4a043..92eedea363420503786127a414f744979501aed5 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_msg_head_shared.cc
+++ b/src/service_inspectors/http_inspect/http_msg_head_shared.cc
@@ -129,9 +129,12 @@ void HttpMsgHeadShared::parse_header_block()
     // calculated correctly.
     while (bytes_used < msg_text.length())
     {
-        assert(num_headers < session_data->num_head_lines[source_id]);
         const int32_t header_length = find_next_header(msg_text.start() + bytes_used,
             msg_text.length() - bytes_used, num_seps);
+        if (header_length == 0)
+            break;
+
+        assert(num_headers < session_data->num_head_lines[source_id]);
         if (header_length >  max_header_line)
         {
             max_header_line = header_length;
@@ -203,6 +206,8 @@ int32_t HttpMsgHeadShared::find_next_header(const uint8_t* buffer, int32_t lengt
             }
         }
     }
+    if (session_data->partial_flush[source_id] && session_data->num_excess[source_id] == 0)
+        return 0;
     return length - num_seps;
 }
 

diff --git a/src/service_inspectors/http_inspect/http_msg_header.cc b/src/service_inspectors/http_inspect/http_msg_header.cc
index 48726ca850fe8528f278c62c50d243e0ddffe763..aa268a5cc3bc446c610985b3318b293d86f45276 100755 (executable)
--- a/src/service_inspectors/http_inspect/http_msg_header.cc
+++ b/src/service_inspectors/http_inspect/http_msg_header.cc
@@ -61,6 +61,9 @@ HttpMsgHeader::HttpMsgHeader(const uint8_t* buffer, const uint16_t buf_size,
 
 void HttpMsgHeader::publish(unsigned pub_id)
 {
+    if (session_data->partial_flush[source_id])
+        return;
+
     const int64_t stream_id = session_data->get_hx_stream_id();
 
     HttpEvent http_header_event(this, session_data->for_httpx, stream_id);

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter.h b/src/service_inspectors/http_inspect/http_stream_splitter.h
index ad8ec563afa81c1e55832844b2e04265058aa5ad..c821b4dde23e0a6c8f6f9310a8e7ffd8f5624a49 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter.h
+++ b/src/service_inspectors/http_inspect/http_stream_splitter.h
@@ -43,7 +43,10 @@ public:
     const snort::StreamBuffer reassemble(snort::Flow* flow, unsigned total, unsigned, const
         uint8_t* data, unsigned len, uint32_t flags, unsigned& copied) override;
     bool finish(snort::Flow* flow) override;
-    void prep_partial_flush(snort::Flow* flow, uint32_t num_flush) override;
+    void prep_partial_flush(snort::Flow* flow, uint32_t num_flush) override
+    { prep_partial_flush(flow, num_flush, 0, 0); }
+    void prep_partial_flush(snort::Flow* flow, uint32_t num_flush, uint32_t num_excess,
+        uint32_t num_head_lines);
     bool is_paf() override { return true; }
     static StreamSplitter::Status status_value(StreamSplitter::Status ret_val, bool http2 = false);
 

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
index 1be14e162a405926db3a10957f581a34c42005e7..5b0dd4b2f1db1ad2691a6c6d9e75b1305cba2c3f 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
@@ -241,7 +241,8 @@ bool HttpStreamSplitter::finish(Flow* flow)
     return false;
 }
 
-void HttpStreamSplitter::prep_partial_flush(Flow* flow, uint32_t num_flush)
+void HttpStreamSplitter::prep_partial_flush(Flow* flow, uint32_t num_flush,
+    uint32_t num_excess, uint32_t num_head_lines)
 {
     // cppcheck-suppress unreadVariable
     Profile profile(HttpModule::get_profile_stats());
@@ -261,7 +262,8 @@ void HttpStreamSplitter::prep_partial_flush(Flow* flow, uint32_t num_flush)
 
     // Set up to process partial message section
     uint32_t not_used;
-    prepare_flush(session_data, &not_used, session_data->type_expected[source_id], num_flush, 0, 0,
+    prepare_flush(session_data, &not_used, session_data->type_expected[source_id], num_flush,
+        num_excess, num_head_lines,
         session_data->cutter[source_id]->get_is_broken_chunk(),
         session_data->cutter[source_id]->get_num_good_chunks(),
         session_data->cutter[source_id]->get_octets_seen() - num_flush);

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
index eb7c07a532d62f51daf721f758e04b1abfc020c9..3e0ec3c64a9704596fed77113b2d79cbff583466 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_reassemble.cc
@@ -429,7 +429,8 @@ const StreamBuffer HttpStreamSplitter::reassemble(Flow* flow, unsigned total,
             buffer = new uint8_t[MAX_OCTETS];
         else
         {
-            const uint32_t buffer_size = (total > 0) ? total : 1;
+            uint32_t buffer_size = (total > 0) ? total : 1;
+            buffer_size += partial_buffer_length;
             buffer = new uint8_t[buffer_size];
         }
     }
@@ -477,12 +478,12 @@ const StreamBuffer HttpStreamSplitter::reassemble(Flow* flow, unsigned total,
         {
             // It's possible we're doing a partial flush but there is no actual data to flush after
             // decompression.
-            if (buf_size > 0)
+            if (session_data->section_offset[source_id] > 0)
             {
                 // Store the data from a partial flush for reuse
-                partial_buffer = new uint8_t[buf_size];
-                memcpy(partial_buffer, buffer, buf_size);
-                partial_buffer_length = buf_size;
+                partial_buffer = new uint8_t[session_data->section_offset[source_id]];
+                memcpy(partial_buffer, buffer, session_data->section_offset[source_id]);
+                partial_buffer_length = session_data->section_offset[source_id];
             }
             partial_raw_bytes += total;
         }

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc
index 2462a3bcf4daff510e00c0df219fbc30ba0e71be..0a9df8095f8f0e81d6b02ce3cd2fa33924fa5952 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_scan.cc
@@ -217,19 +217,35 @@ StreamSplitter::Status HttpStreamSplitter::call_cutter(Flow* flow, HttpFlowData*
             return status_value(StreamSplitter::ABORT);
         }
 
-        if (is_body(type) && source_id == SRC_CLIENT &&
-            (my_inspector->params->partial_depth_body == -1 ||
-             (cutter->get_octets_seen() < my_inspector->params->partial_depth_body && cutter->get_num_flush() == 0)))
+        if (source_id == SRC_CLIENT)
         {
-            static const uint64_t MAX_PARTIAL_FLUSH_COUNTER = 20;
-            if (++session_data->partial_flush_counter == MAX_PARTIAL_FLUSH_COUNTER)
-                session_data->events[source_id]->create_event(HttpEnums::EVENT_MAX_PARTIAL_FLUSH);
-            cut_result = SCAN_NOT_FOUND_ACCELERATE;
+            int64_t partial_depth = 0;
+            auto params = my_inspector->params;
+
+            if (type == SEC_HEADER)
+            {
+                if (params->partial_depth_header != 0 && !session_data->for_httpx)
+                    partial_depth = params->partial_depth_header;
+            }
+            else if (is_body(type))
+            {
+                if (params->partial_depth_body != 0 &&
+                    (params->partial_depth_body == -1 || cutter->get_num_flush() == 0))
+                    partial_depth = params->partial_depth_body;
+            }
+
+            if (partial_depth == -1 || cutter->get_octets_seen() < partial_depth)
+            {
+                static const uint64_t MAX_PARTIAL_FLUSH_COUNTER = 20;
+                if (++session_data->partial_flush_counter == MAX_PARTIAL_FLUSH_COUNTER)
+                    session_data->events[source_id]->create_event(HttpEnums::EVENT_MAX_PARTIAL_FLUSH);
+                cut_result = SCAN_NOT_FOUND_ACCELERATE;
+            }
         }
 
         if (cut_result == SCAN_NOT_FOUND_ACCELERATE)
         {
-            prep_partial_flush(flow, length);
+            prep_partial_flush(flow, length, cutter->get_num_excess(), cutter->get_num_head_lines());
 #ifdef REG_TEST
             if (!HttpTestManager::use_test_input(HttpTestManager::IN_HTTP))
 #endif

diff --git a/src/service_inspectors/http_inspect/http_transaction.cc b/src/service_inspectors/http_inspect/http_transaction.cc
index b5951c232161bddf49d0f603be90109e7bd34535..ace0fcbbdc42c3eadb911ac6f0a498ba630b6791 100644 (file)
--- a/src/service_inspectors/http_inspect/http_transaction.cc
+++ b/src/service_inspectors/http_inspect/http_transaction.cc
@@ -278,6 +278,12 @@ void HttpTransaction::delete_transaction(HttpTransaction* transaction, HttpFlowD
     }
 }
 
+void HttpTransaction::set_header(HttpMsgHeader* header_, HttpCommon::SourceId source_id)
+{
+    delete (header[source_id]);
+    header[source_id] = header_;
+}
+
 void HttpTransaction::set_body(HttpMsgBody* latest_body)
 {
     latest_body->next = body_list;

diff --git a/src/service_inspectors/http_inspect/http_transaction.h b/src/service_inspectors/http_inspect/http_transaction.h
index bc44c578d51eff122b9c9c10b86644867af2242b..70944b59069d256eb96340f44a864518103c0753 100644 (file)
--- a/src/service_inspectors/http_inspect/http_transaction.h
+++ b/src/service_inspectors/http_inspect/http_transaction.h
@@ -48,8 +48,7 @@ public:
     void set_status(HttpMsgStatus* status_) { status = status_; }
 
     HttpMsgHeader* get_header(HttpCommon::SourceId source_id) const { return header[source_id]; }
-    void set_header(HttpMsgHeader* header_, HttpCommon::SourceId source_id)
-    { header[source_id] = header_; }
+    void set_header(HttpMsgHeader* header_, HttpCommon::SourceId source_id);
 
     HttpMsgTrailer* get_trailer(HttpCommon::SourceId source_id) const
     { return trailer[source_id]; }

diff --git a/src/service_inspectors/http_inspect/test/http_transaction_test.cc b/src/service_inspectors/http_inspect/test/http_transaction_test.cc
index 58f560593a00da21e96992cacb13e549f309cd3d..96f36ec55c82b72a7da6cc49884f45dd17704236 100644 (file)
--- a/src/service_inspectors/http_inspect/test/http_transaction_test.cc
+++ b/src/service_inspectors/http_inspect/test/http_transaction_test.cc
@@ -111,7 +111,7 @@ const snort::StreamBuffer HttpStreamSplitter::reassemble(snort::Flow*, unsigned,
     return buf;
 }
 bool HttpStreamSplitter::finish(snort::Flow*) { return false; }
-void HttpStreamSplitter::prep_partial_flush(snort::Flow*, uint32_t) { }
+void HttpStreamSplitter::prep_partial_flush(snort::Flow*, uint32_t, uint32_t, uint32_t) { }
 
 THREAD_LOCAL PegCount HttpModule::peg_counts[PEG_COUNT_MAX] = { };
 const Field Field::FIELD_NULL { STAT_NO_SOURCE };