Merge pull request #2175 in SNORT/snort3 from ~MDAGON/snort3:multi to master

Squashed commit of the following:

commit 5f3627d7056532a4388cf8a957a2785d28a789ea
Author: mdagon <mdagon@cisco.com>
Date:   Fri Apr 10 13:53:59 2020 -0400

    http2_inspect: support stream multiplexing

diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.cc b/src/service_inspectors/http2_inspect/http2_data_cutter.cc
index a9b956c549c414c63d2883a138707613c686de65..cb8ea3a5342424044213316af227f1a0129b7ceb 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_data_cutter.cc
+++ b/src/service_inspectors/http2_inspect/http2_data_cutter.cc
@@ -161,8 +161,7 @@ StreamSplitter::Status Http2DataCutter::http_scan(const uint8_t* data, uint32_t*
 
             if (frame_flags & END_STREAM)
             {
-                finish_msg_body(session_data, source_id);
-                session_data->data_processing[source_id] = false;
+                finish_msg_body();
                 return StreamSplitter::FLUSH;
             }
             else
@@ -191,21 +190,32 @@ const StreamBuffer Http2DataCutter::reassemble(const uint8_t* data, unsigned len
     cur_data = cur_padding = cur_data_offset = 0;
 
     unsigned cur_pos = 0;
+
     while (cur_pos < len)
     {
         switch (reassemble_state)
         {
         case GET_FRAME_HDR:
           {
-            const uint32_t missing = FRAME_HEADER_LENGTH - reassemble_hdr_bytes_read;
-            const uint32_t cur_frame = ((len - cur_pos) < missing) ? (len - cur_pos) : missing;
-            memcpy(session_data->frame_header[source_id] +
-                session_data->frame_header_offset[source_id] +
-                reassemble_hdr_bytes_read,
-                data + cur_pos, cur_frame);
-            reassemble_hdr_bytes_read += cur_frame;
-
-            cur_pos += cur_frame;
+            if (session_data->use_leftover_hdr[source_id])
+            {
+                memcpy(session_data->frame_header[source_id],
+                    session_data->leftover_hdr[source_id], FRAME_HEADER_LENGTH);
+                reassemble_hdr_bytes_read = FRAME_HEADER_LENGTH;
+                session_data->use_leftover_hdr[source_id] = false;
+            }
+            else
+            {
+                const uint32_t missing = FRAME_HEADER_LENGTH - reassemble_hdr_bytes_read;
+                const uint32_t cur_frame = ((len - cur_pos) < missing) ? (len - cur_pos) : missing;
+                memcpy(session_data->frame_header[source_id] +
+                    session_data->frame_header_offset[source_id] +
+                    reassemble_hdr_bytes_read,
+                    data + cur_pos, cur_frame);
+                reassemble_hdr_bytes_read += cur_frame;
+
+                cur_pos += cur_frame;
+            }
 
             if (reassemble_hdr_bytes_read == FRAME_HEADER_LENGTH)
             {
@@ -278,6 +288,18 @@ const StreamBuffer Http2DataCutter::reassemble(const uint8_t* data, unsigned len
 
         if (reassemble_state == CLEANUP)
         {
+            Http2Stream* const stream = session_data->find_stream(
+                session_data->current_stream[source_id]);
+            if (bytes_sent_http == 0 && (reassemble_frame_flags & END_STREAM) &&
+                stream->is_partial_buf_pending(source_id))
+            {
+                // Received end of stream without new data. Flush pending partial buffer.
+                assert(frame_buf.data == nullptr);
+                unsigned unused;
+                frame_buf = session_data->hi_ss[source_id]->reassemble(session_data->flow,
+                    0, 0, nullptr, 0, PKT_PDU_TAIL, unused);
+            }
+
             // Done with this packet, cleanup
             reassemble_state = GET_FRAME_HDR;
             reassemble_hdr_bytes_read = reassemble_data_bytes_read = reassemble_padding_read =
@@ -295,3 +317,18 @@ const StreamBuffer Http2DataCutter::reassemble(const uint8_t* data, unsigned len
     return frame_buf;
 }
 
+void Http2DataCutter::finish_msg_body()
+{
+    uint32_t http_flush_offset = 0;
+    Http2DummyPacket dummy_pkt;
+    dummy_pkt.flow = session_data->flow;
+    uint32_t unused = 0;
+    session_data->get_current_stream(source_id)->get_hi_flow_data()->
+    finish_h2_body(source_id);
+    const snort::StreamSplitter::Status scan_result = session_data->hi_ss[source_id]->scan(
+        &dummy_pkt, nullptr, 0, unused, &http_flush_offset);
+    assert(scan_result == snort::StreamSplitter::FLUSH);
+    UNUSED(scan_result);
+    session_data->data_processing[source_id] = false;
+}
+

diff --git a/src/service_inspectors/http2_inspect/http2_data_cutter.h b/src/service_inspectors/http2_inspect/http2_data_cutter.h
index 69519500b50b84daa894ce00237b0874e23feca6..84d04aa029391f46748f02b1e58508702e44fc99 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_data_cutter.h
+++ b/src/service_inspectors/http2_inspect/http2_data_cutter.h
@@ -83,6 +83,7 @@ private:
     bool http2_scan(const uint8_t* data, uint32_t length, uint32_t* flush_offset,
         uint32_t frame_len, uint8_t frame_flags, uint32_t& data_offset);
     snort::StreamSplitter::Status http_scan(const uint8_t* data, uint32_t* flush_offset);
+    void finish_msg_body();
 };
 
 #endif

diff --git a/src/service_inspectors/http2_inspect/http2_data_frame.cc b/src/service_inspectors/http2_inspect/http2_data_frame.cc
index 01a6c96f6a7264ab74d6d781eb0bb5a0432357ad..b3a18b8e64c0bc374c8291c427fae715aa5dc9c1 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_data_frame.cc
+++ b/src/service_inspectors/http2_inspect/http2_data_frame.cc
@@ -38,15 +38,24 @@ Http2DataFrame::Http2DataFrame(const uint8_t* header_buffer, const int32_t heade
     HttpCommon::SourceId source_id_) :
     Http2Frame(header_buffer, header_len, nullptr, 0, session_data_, source_id_)
 {
-    Http2DummyPacket dummy_pkt;
-    dummy_pkt.flow = session_data->flow;
-    dummy_pkt.packet_flags = (source_id == SRC_CLIENT) ? PKT_FROM_CLIENT : PKT_FROM_SERVER;
-    dummy_pkt.dsize = data_len;
-    dummy_pkt.data = data_buffer;
-    dummy_pkt.xtradata_mask = 0;
-    session_data->hi->eval(&dummy_pkt);
-    detection_required = dummy_pkt.is_detection_required();
-    xtradata_mask = dummy_pkt.xtradata_mask;
+    if ((data_len != 0) || !session_data->flushing_data[source_id])
+    {
+        Http2DummyPacket dummy_pkt;
+        dummy_pkt.flow = session_data->flow;
+        dummy_pkt.packet_flags = (source_id == SRC_CLIENT) ? PKT_FROM_CLIENT : PKT_FROM_SERVER;
+        dummy_pkt.dsize = data_len;
+        dummy_pkt.data = data_buffer;
+        dummy_pkt.xtradata_mask = 0;
+        session_data->hi->eval(&dummy_pkt);
+        detection_required = dummy_pkt.is_detection_required();
+        xtradata_mask = dummy_pkt.xtradata_mask;
+    }
+    else
+    {
+        detection_required = true;
+        HttpFlowData* const http_flow = (HttpFlowData*)session_data_->get_hi_flow_data();
+        http_flow->reset_partial_flush(source_id_);
+    }
 }
 
 void Http2DataFrame::clear()

diff --git a/src/service_inspectors/http2_inspect/http2_headers_frame.cc b/src/service_inspectors/http2_inspect/http2_headers_frame.cc
index 78c5a3e9e3b6dfbff1fb932b90cae3c594d52f0b..8cb71d6ae9faf19990944e28ea66a4044da1abd2 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_headers_frame.cc
+++ b/src/service_inspectors/http2_inspect/http2_headers_frame.cc
@@ -78,7 +78,6 @@ Http2HeadersFrame::Http2HeadersFrame(const uint8_t* header_buffer, const int32_t
         return;
 
     // http_inspect scan() of start line
-    session_data->stream_in_hi = session_data->current_stream[source_id];
     {
         uint32_t flush_offset;
         Http2DummyPacket dummy_pkt;

diff --git a/src/service_inspectors/http2_inspect/http2_stream.h b/src/service_inspectors/http2_inspect/http2_stream.h
index a409772a0a288aaab6f050f5ebb3bd7426700f9c..9189bad33e11474af60b96e88eb7adc5eae4aa69 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_stream.h
+++ b/src/service_inspectors/http2_inspect/http2_stream.h
@@ -55,8 +55,14 @@ public:
 
     void set_end_stream(HttpCommon::SourceId source_id) { end_stream_set[source_id] = true; }
     bool end_stream_is_set(HttpCommon::SourceId source_id) { return end_stream_set[source_id]; }
-    void set_abort_on_data(HttpCommon::SourceId source_id) { abort_on_data[source_id] = true; }
-    bool abort_on_data_is_set(HttpCommon::SourceId source_id) { return abort_on_data[source_id]; }
+
+    void set_partial_buf_pending(HttpCommon::SourceId source_id)
+    { partial_buf_pending[source_id] = true; }
+    void reset_partial_buf_pending(HttpCommon::SourceId source_id)
+    { partial_buf_pending[source_id] = false; }
+    bool is_partial_buf_pending(HttpCommon::SourceId source_id)
+    { return partial_buf_pending[source_id]; }
+
 #ifdef REG_TEST
     void print_frame(FILE* output);
 #endif
@@ -69,7 +75,8 @@ private:
     HttpMsgSection* hi_msg_section = nullptr;
     Http2DataCutter* data_cutter[2] = { nullptr, nullptr};
     bool end_stream_set[2] = { false, false };
-    bool abort_on_data[2] = { false, false};
+    bool partial_buf_pending[2] = { false, false }; // used to indicate a partial buffer
+                                                    // is pending from a previous partial flush
 };
 
 #endif

diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter.h b/src/service_inspectors/http2_inspect/http2_stream_splitter.h
index d001e1bc31b539cf620c0f2387ec7599ff80455c..892ef9a35afbcac5456fd1a3fff8b5a4357010c8 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_stream_splitter.h
+++ b/src/service_inspectors/http2_inspect/http2_stream_splitter.h
@@ -52,8 +52,8 @@ private:
     static StreamSplitter::Status data_scan(Http2FlowData* session_data, const uint8_t* data,
         uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id,
         uint32_t frame_length, uint8_t frame_flags, uint32_t& data_offset);
-    static void flush_data(Http2FlowData* session_data, HttpCommon::SourceId source_id,
-        uint32_t* flush_offset, uint32_t old_stream);
+    static void partial_flush_data(Http2FlowData* session_data, HttpCommon::SourceId source_id,
+        uint32_t* flush_offset, uint32_t data_offset, uint32_t old_stream);
     static StreamSplitter::Status non_data_scan(Http2FlowData* session_data,
         uint32_t length, uint32_t* flush_offset, HttpCommon::SourceId source_id,
         uint32_t frame_length, uint8_t type, uint8_t frame_flags, uint32_t& data_offset);

diff --git a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc
index e1a691988e70e42ca883e2ea1bdf2e75fd48e5cb..58aa1fdc867681ed34f4e5f9bc62409b2d9430e4 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc
+++ b/src/service_inspectors/http2_inspect/http2_stream_splitter_impl.cc
@@ -69,8 +69,6 @@ StreamSplitter::Status Http2StreamSplitter::data_scan(Http2FlowData* session_dat
     uint32_t& data_offset)
 {
     Http2Stream* const stream = session_data->find_stream(session_data->current_stream[source_id]);
-    if (stream && stream->abort_on_data_is_set(source_id))
-        return StreamSplitter::ABORT;
 
     if (!stream || stream->end_stream_is_set(source_id))
     {
@@ -132,9 +130,10 @@ StreamSplitter::Status Http2StreamSplitter::non_data_scan(Http2FlowData* session
         session_data->total_bytes_in_split[source_id] += FRAME_HEADER_LENGTH +
             frame_length;
 
-        // If the stream object exists and the end_stream flag is set, save that state in the stream
-        // object. If this is the first headers frame in the current stream,the stream object has
-        // not been created yet. The end_stream flag will be handled in the headers frame processing
+        // If the stream object exists and the end_stream flag is set, save that state in the
+        // stream object. If this is the first headers frame in the current stream, the stream
+        // object has not been created yet. The end_stream flag will be handled in the headers
+        // frame processing
         Http2Stream* const stream = session_data->find_stream(
             session_data->current_stream[source_id]);
         if (stream and frame_flags & END_STREAM)
@@ -174,7 +173,7 @@ StreamSplitter::Status Http2StreamSplitter::non_data_scan(Http2FlowData* session
         }
         else
         {
-            // FIXIT-M CONTINUATION frames can also follow PUSH_PROMISE frames, which
+            // FIXIT-E CONTINUATION frames can also follow PUSH_PROMISE frames, which
             // are not currently supported
             *session_data->infractions[source_id] += INF_UNEXPECTED_CONTINUATION;
             session_data->events[source_id]->create_event(
@@ -193,24 +192,21 @@ StreamSplitter::Status Http2StreamSplitter::non_data_scan(Http2FlowData* session
     return status;
 }
 
-// Flush pending data. Save current non-data header for the next scan/reassemble.
-void Http2StreamSplitter::flush_data(Http2FlowData* session_data, HttpCommon::SourceId source_id,
-    uint32_t* flush_offset, uint32_t old_stream)
+// Flush pending data
+void Http2StreamSplitter::partial_flush_data(Http2FlowData* session_data, HttpCommon::SourceId source_id,
+    uint32_t* flush_offset, uint32_t data_offset, uint32_t old_stream)
 {
-    session_data->current_stream[source_id] = old_stream;
+    session_data->current_stream[source_id] = session_data->stream_in_hi = old_stream;
     session_data->frame_type[source_id] = FT_DATA;
     Http2Stream* const stream = session_data->find_stream(
         session_data->current_stream[source_id]);
     Http2DataCutter* const data_cutter = stream->get_data_cutter(source_id);
     if (data_cutter->is_flush_required())
-        finish_msg_body(session_data, source_id);
+        session_data->hi_ss[source_id]->init_partial_flush(session_data->flow);
     session_data->data_processing[source_id] = false;
-    *flush_offset = FRAME_HEADER_LENGTH;
+    *flush_offset = data_offset;
     session_data->flushing_data[source_id] = true;
-    memcpy(session_data->leftover_hdr[source_id],
-        session_data->scan_frame_header[source_id], FRAME_HEADER_LENGTH);
     session_data->num_frame_headers[source_id] -= 1;
-    stream->set_abort_on_data(source_id);
 }
 
 bool Http2StreamSplitter::read_frame_hdr(Http2FlowData* session_data, const uint8_t* data,
@@ -308,19 +304,14 @@ StreamSplitter::Status Http2StreamSplitter::implement_scan(Http2FlowData* sessio
                 const uint8_t frame_flags = get_frame_flags(session_data->
                     scan_frame_header[source_id]);
                 const uint32_t old_stream = session_data->current_stream[source_id];
-                session_data->current_stream[source_id] =
+                session_data->stream_in_hi = session_data->current_stream[source_id] =
                     get_stream_id(session_data->scan_frame_header[source_id]);
 
-                if ((old_stream != session_data->current_stream[source_id]) &&
-                    session_data->data_processing[source_id] && type == FT_DATA)
+                if (session_data->data_processing[source_id] &&
+                    ((old_stream != session_data->current_stream[source_id] && type == FT_DATA)
+                    || type != FT_DATA))
                 {
-                    // FIXIT-E split by stream multiplexing not supported yet
-                    return StreamSplitter::ABORT;
-                }
-
-                if (session_data->data_processing[source_id] && type != FT_DATA)
-                {
-                    flush_data(session_data, source_id, flush_offset, old_stream);
+                    partial_flush_data(session_data, source_id, flush_offset, data_offset, old_stream);
                     return StreamSplitter::FLUSH;
                 }
 
@@ -348,6 +339,8 @@ const StreamBuffer Http2StreamSplitter::implement_reassemble(Http2FlowData* sess
     assert(total <= MAX_OCTETS);
 
     StreamBuffer frame_buf { nullptr, 0 };
+    Http2Stream* const stream = session_data->find_stream(
+        session_data->current_stream[source_id]);
 
     if (offset == 0)
     {
@@ -361,30 +354,45 @@ const StreamBuffer Http2StreamSplitter::implement_reassemble(Http2FlowData* sess
         session_data->frame_header_offset[source_id] = 0;
     }
 
-    if (session_data->frame_type[source_id] == FT_DATA)
+    if (total == 0 && session_data->use_leftover_hdr[source_id])
+    {
+        memcpy(session_data->frame_header[source_id],
+            session_data->leftover_hdr[source_id], FRAME_HEADER_LENGTH);
+        session_data->use_leftover_hdr[source_id] = false;
+        if (session_data->frame_type[source_id] == FT_DATA)
+        {
+            // Check if we reached end of stream and have a partial buffer pending
+            const uint8_t frame_flags = get_frame_flags(session_data->frame_header[source_id]);
+            if ((frame_flags & END_STREAM) && stream->is_partial_buf_pending(source_id))
+            {
+                unsigned copied;
+                StreamBuffer http_frame_buf = session_data->hi_ss[source_id]->reassemble(
+                    session_data->flow,
+                    0, 0, nullptr, 0, PKT_PDU_TAIL, copied);
+                session_data->frame_data[source_id] = const_cast<uint8_t*>(http_frame_buf.data);
+                session_data->frame_data_size[source_id] = http_frame_buf.length;
+            }
+        }
+    }
+    else if (session_data->frame_type[source_id] == FT_DATA)
     {
         if (session_data->flushing_data[source_id] && (flags & PKT_PDU_TAIL))
             len -= FRAME_HEADER_LENGTH;
 
         if (len != 0)
         {
-            Http2Stream* const stream = session_data->find_stream(
-                session_data->current_stream[source_id]);
             Http2DataCutter* data_cutter = stream->get_data_cutter(source_id);
             StreamBuffer http_frame_buf = data_cutter->reassemble(data, len);
             if (http_frame_buf.data)
             {
                 session_data->frame_data[source_id] = const_cast<uint8_t*>(http_frame_buf.data);
                 session_data->frame_data_size[source_id] = http_frame_buf.length;
+                if (!session_data->flushing_data[source_id] && stream->is_partial_buf_pending(
+                    source_id))
+                    stream->reset_partial_buf_pending(source_id);
             }
         }
     }
-    else if (total == 0 && session_data->use_leftover_hdr[source_id])
-    {
-        memcpy(session_data->frame_header[source_id],
-            session_data->leftover_hdr[source_id], FRAME_HEADER_LENGTH);
-        session_data->use_leftover_hdr[source_id] = false;
-    }
     else
     {
         uint32_t data_offset = 0;
@@ -504,6 +512,14 @@ const StreamBuffer Http2StreamSplitter::implement_reassemble(Http2FlowData* sess
         session_data->num_frame_headers[source_id] = 0;
         session_data->scan_octets_seen[source_id] = 0;
 
+        if (session_data->flushing_data[source_id])
+        {
+            stream->set_partial_buf_pending(source_id);
+            // save current header for next scan/reassemble
+            memcpy(session_data->leftover_hdr[source_id],
+                session_data->scan_frame_header[source_id], FRAME_HEADER_LENGTH);
+        }
+
         // Return 0-length non-null buffer to stream which signals detection required, but don't
         // create pkt_data buffer
         frame_buf.data = (const uint8_t*)"";

diff --git a/src/service_inspectors/http2_inspect/http2_utils.cc b/src/service_inspectors/http2_inspect/http2_utils.cc
index 79a2f0330ce0bd75a8f28faa9af46bac162f2126..f8cb32ae98d4952f2bfd1aba7c66c21c57455c82 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_utils.cc
+++ b/src/service_inspectors/http2_inspect/http2_utils.cc
@@ -21,9 +21,6 @@
 
 #include <cassert>
 
-#include "service_inspectors/http_inspect/http_flow_data.h"
-#include "service_inspectors/http_inspect/http_stream_splitter.h"
-
 #include "http2_dummy_packet.h"
 #include "http2_enum.h"
 
@@ -63,17 +60,3 @@ uint8_t get_stream_id(const uint8_t* frame_header_buffer)
            frame_header_buffer[stream_id_index + 3];
 }
 
-void finish_msg_body(Http2FlowData* session_data, HttpCommon::SourceId source_id)
-{
-    uint32_t http_flush_offset = 0;
-    Http2DummyPacket dummy_pkt;
-    dummy_pkt.flow = session_data->flow;
-    uint32_t unused = 0;
-    session_data->get_current_stream(source_id)->get_hi_flow_data()->
-        finish_h2_body(source_id);
-    const snort::StreamSplitter::Status scan_result = session_data->hi_ss[source_id]->scan(
-        &dummy_pkt, nullptr, 0, unused, &http_flush_offset);
-    assert(scan_result == snort::StreamSplitter::FLUSH);
-    UNUSED(scan_result);
-}
-

diff --git a/src/service_inspectors/http2_inspect/http2_utils.h b/src/service_inspectors/http2_inspect/http2_utils.h
index 3954f69fdb808ae0bc02ef5ca2d5493b3a67ebf2..351682c7f559e99b769872a95778b4db91daa6cc 100644 (file)
--- a/src/service_inspectors/http2_inspect/http2_utils.h
+++ b/src/service_inspectors/http2_inspect/http2_utils.h
@@ -37,7 +37,4 @@ uint8_t get_frame_type(const uint8_t* frame_header_buffer);
 uint8_t get_frame_flags(const uint8_t* frame_header_buffer);
 uint8_t get_stream_id(const uint8_t* frame_header_buffer);
 
-
-void finish_msg_body(Http2FlowData* session_data, HttpCommon::SourceId source_id);
-
 #endif

diff --git a/src/service_inspectors/http_inspect/http_flow_data.h b/src/service_inspectors/http_inspect/http_flow_data.h
index ef34a50865fb363b19e6730ad748144e2597da8f..d057ae2800fea8fdbcc0f7cc0ca2a9dfc5661a01 100644 (file)
--- a/src/service_inspectors/http_inspect/http_flow_data.h
+++ b/src/service_inspectors/http_inspect/http_flow_data.h
@@ -74,6 +74,9 @@ public:
     void finish_h2_body(HttpCommon::SourceId source_id)
     { h2_body_finished[source_id] = true; }
 
+    void reset_partial_flush(HttpCommon::SourceId source_id)
+    { partial_flush[source_id] = false; }
+
 private:
     // HTTP/2 handling
     bool for_http2 = false;

diff --git a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
index 1e36d4f336453e72b107534cff76301a53156632..ade06bc9f250dc52e0317e778c75f4f4e6ba4e51 100644 (file)
--- a/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
+++ b/src/service_inspectors/http_inspect/http_stream_splitter_finish.cc
@@ -167,7 +167,8 @@ bool HttpStreamSplitter::init_partial_flush(Flow* flow)
     assert(session_data != nullptr);
     assert((session_data->type_expected[source_id] == SEC_BODY_CL)      ||
            (session_data->type_expected[source_id] == SEC_BODY_OLD)     ||
-           (session_data->type_expected[source_id] == SEC_BODY_CHUNK));
+           (session_data->type_expected[source_id] == SEC_BODY_CHUNK)   ||
+           (session_data->type_expected[source_id] == SEC_BODY_H2));
 
 #ifdef REG_TEST
     if (HttpTestManager::use_test_output(HttpTestManager::IN_HTTP) &&