]> git.ipfire.org Git - thirdparty/snort3.git/commitdiff
Merge pull request #2734 in SNORT/snort3 from ~BRASTULT/snort3:zip_data_desc to master
authorMichael Altizer (mialtize) <mialtize@cisco.com>
Tue, 9 Mar 2021 03:56:50 +0000 (03:56 +0000)
committerMichael Altizer (mialtize) <mialtize@cisco.com>
Tue, 9 Mar 2021 03:56:50 +0000 (03:56 +0000)
Squashed commit of the following:

commit 142372710cf9717980b1e2ab14f11c2f7ea5a18d
Author: Brandon Stultz <brastult@cisco.com>
Date:   Wed Feb 3 00:23:10 2021 -0500

    decompress: add support for streaming ZIPs

src/decompress/file_decomp.h
src/decompress/file_decomp_zip.cc
src/decompress/file_decomp_zip.h

index 384c5133b92a73215f336c22b6b069fcc8d53706..bb8bf9d3fc808ade8c8b0afda37673650108d03c 100644 (file)
@@ -192,7 +192,7 @@ inline bool Get_1(fd_session_t* SessionPtr, uint8_t* c)
 
 /* If available, get N bytes from the input queue.  All N must be
    available for this call to succeed. */
-inline bool Get_N(fd_session_t* SessionPtr, const uint8_t** c, uint16_t N)
+inline bool Get_N(fd_session_t* SessionPtr, const uint8_t** c, uint32_t N)
 {
     if ( (SessionPtr->Next_In != nullptr) && (SessionPtr->Avail_In >= N) )
     {
@@ -222,7 +222,7 @@ inline bool Put_1(fd_session_t* SessionPtr, uint8_t c)
 
 /* If the output queue has room available, place N bytes onto the queue.
    The output queue must have space for N bytes for this call to succeed. */
-inline bool Put_N(fd_session_t* SessionPtr, const uint8_t* c, uint16_t N)
+inline bool Put_N(fd_session_t* SessionPtr, const uint8_t* c, uint32_t N)
 {
     if ( (SessionPtr->Next_Out != nullptr) && (SessionPtr->Avail_Out >= N) )
     {
@@ -244,8 +244,8 @@ inline bool Move_1(fd_session_t* SessionPtr)
         (SessionPtr->Next_In != nullptr) && (SessionPtr->Avail_In > 0) )
     {
         *(SessionPtr->Next_Out) = *(SessionPtr->Next_In);
-        SessionPtr->Next_Out += 1;
         SessionPtr->Next_In += 1;
+        SessionPtr->Next_Out += 1;
         SessionPtr->Avail_In -= 1;
         SessionPtr->Avail_Out -= 1;
         SessionPtr->Total_In += 1;
@@ -258,14 +258,14 @@ inline bool Move_1(fd_session_t* SessionPtr)
 
 /* If the input queue has at least N bytes available AND there's at
    space for at least N bytes in the output queue, then move all N bytes. */
-inline bool Move_N(fd_session_t* SessionPtr, uint16_t N)
+inline bool Move_N(fd_session_t* SessionPtr, uint32_t N)
 {
     if ( (SessionPtr->Next_Out != nullptr) && (SessionPtr->Avail_Out >= N) &&
         (SessionPtr->Next_In != nullptr) && (SessionPtr->Avail_In >= N) )
     {
         memcpy( (char*)SessionPtr->Next_Out, (const char*)SessionPtr->Next_In, N);
-        SessionPtr->Next_Out += N;
         SessionPtr->Next_In += N;
+        SessionPtr->Next_Out += N;
         SessionPtr->Avail_In -= N;
         SessionPtr->Avail_Out -= N;
         SessionPtr->Total_In += N;
index c880900d9452845a56be388e64f33e1b0e20d908..6d2603c65239791c017a7e781e7f667dfae6707e 100644 (file)
@@ -15,7 +15,6 @@
 // with this program; if not, write to the Free Software Foundation, Inc.,
 // 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 //--------------------------------------------------------------------------
-
 // file_decomp_zip.cc author Brandon Stultz <brastult@cisco.com>
 
 #ifdef HAVE_CONFIG_H
@@ -23,6 +22,8 @@
 #endif
 
 #include "file_decomp_zip.h"
+
+#include "helpers/boyer_moore_search.h"
 #include "utils/util.h"
 
 using namespace snort;
@@ -98,12 +99,12 @@ fd_status_t File_Decomp_Init_ZIP(fd_session_t* SessionPtr)
     SessionPtr->ZIP = (fd_ZIP_t*)snort_calloc(sizeof(fd_ZIP_t));
 
     // file_decomp.cc already matched the local header
-    // skip the version and bitflag (4 bytes)
+    // skip the version (2 bytes)
     SessionPtr->ZIP->State = ZIP_STATE_SKIP;
-    SessionPtr->ZIP->Length = 4;
+    SessionPtr->ZIP->Length = 2;
 
-    // land on compression method
-    SessionPtr->ZIP->Next = ZIP_STATE_METHOD;
+    // land on bitflag
+    SessionPtr->ZIP->Next = ZIP_STATE_BITFLAG;
     SessionPtr->ZIP->Next_Length = 2;
 
     return File_Decomp_OK;
@@ -119,6 +120,12 @@ fd_status_t File_Decomp_End_ZIP(fd_session_t* SessionPtr)
     if ( SessionPtr->ZIP->State == ZIP_STATE_INFLATE )
         Inflate_End(SessionPtr);
 
+    if ( SessionPtr->ZIP->header_searcher )
+    {
+        delete SessionPtr->ZIP->header_searcher;
+        SessionPtr->ZIP->header_searcher = nullptr;
+    }
+
     // File_Decomp_Free() will free SessionPtr->ZIP
     return File_Decomp_OK;
 }
@@ -155,6 +162,8 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
 
                 // reset ZIP fields
                 parser->local_header = 0;
+                parser->bitflag = 0;
+                parser->data_descriptor = false;
                 parser->method = 0;
                 parser->compressed_size = 0;
                 parser->filename_length = 0;
@@ -163,18 +172,39 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
                 // reset decompression progress
                 parser->progress = 0;
 
-                // skip the version and bitflag (4 bytes)
+                // skip the version (2 bytes)
                 parser->State = ZIP_STATE_SKIP;
-                parser->Length = 4;
+                parser->Length = 2;
 
-                // land on compression method
-                parser->Next = ZIP_STATE_METHOD;
+                // land on bitflag
+                parser->Next = ZIP_STATE_BITFLAG;
                 parser->Next_Length = 2;
                 continue;
             }
             // read the local header
             byte = *SessionPtr->Next_In;
-            parser->local_header |= byte << parser->Index*8;
+            parser->local_header |= byte << (parser->Index * 8);
+            break;
+        // bitflag
+        case ZIP_STATE_BITFLAG:
+            // check if we are done with the bitflag
+            if ( parser->Index == parser->Length )
+            {
+                // read the bitflag, reset the index
+                parser->Index = 0;
+
+                // check the data descriptor bit
+                if ( parser->bitflag & DATA_DESC_BIT )
+                    parser->data_descriptor = true;
+
+                // read the compression method next
+                parser->State = ZIP_STATE_METHOD;
+                parser->Length = 2;
+                continue;
+            }
+            // read the bitflag
+            byte = *SessionPtr->Next_In;
+            parser->bitflag |= byte << (parser->Index * 8);
             break;
         // compression method
         case ZIP_STATE_METHOD:
@@ -196,7 +226,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
             }
             // read the method
             byte = *SessionPtr->Next_In;
-            parser->method |= byte << parser->Index*8;
+            parser->method |= byte << (parser->Index * 8);
             break;
         // compressed size
         case ZIP_STATE_COMPSIZE:
@@ -217,7 +247,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
             }
             // read the compressed size
             byte = *SessionPtr->Next_In;
-            parser->compressed_size |= byte << parser->Index*8;
+            parser->compressed_size |= byte << (parser->Index * 8);
             break;
         // filename length
         case ZIP_STATE_FILENAMELEN:
@@ -234,7 +264,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
             }
             // read the filename length
             byte = *SessionPtr->Next_In;
-            parser->filename_length |= byte << parser->Index*8;
+            parser->filename_length |= byte << (parser->Index * 8);
             break;
         // extra length
         case ZIP_STATE_EXTRALEN:
@@ -254,11 +284,20 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
                     // the compression type is deflate (8),
                     // land on the compressed stream, init zlib
                     parser->Next = ZIP_STATE_INFLATE_INIT;
-                    parser->Next_Length = parser->compressed_size;
+                    parser->Next_Length = 0;
                     continue;
                 }
 
-                // no output space or compression type isn't deflate, skip the stream
+                // no output space or compression type isn't deflate
+                if ( parser->data_descriptor )
+                {
+                    // search for the next file
+                    parser->Next = ZIP_STATE_SEARCH;
+                    parser->Next_Length = 0;
+                    continue;
+                }
+
+                // skip the stream
                 parser->Length += parser->compressed_size;
 
                 // land on another local header
@@ -268,7 +307,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
             }
             // read the extra length
             byte = *SessionPtr->Next_In;
-            parser->extra_length |= byte << parser->Index*8;
+            parser->extra_length |= byte << (parser->Index * 8);
             break;
         // initialize zlib inflate
         case ZIP_STATE_INFLATE_INIT:
@@ -295,6 +334,17 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
             if ( status == File_Decomp_BlockOut )
             {
                 // ran out of output space
+                if ( parser->data_descriptor )
+                {
+                    // close the inflate stream
+                    Inflate_End(SessionPtr);
+
+                    // search for next file
+                    parser->State = ZIP_STATE_SEARCH;
+                    parser->Length = 0;
+                    continue;
+                }
+
                 // progress should be < compressed_size
                 if ( parser->progress >= parser->compressed_size )
                     return File_Decomp_Error;
@@ -318,6 +368,14 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
                 // close the inflate stream
                 Inflate_End(SessionPtr);
 
+                if ( parser->data_descriptor )
+                {
+                    // search for the next file
+                    parser->State = ZIP_STATE_SEARCH;
+                    parser->Length = 0;
+                    continue;
+                }
+
                 // parse next local header
                 parser->State = ZIP_STATE_LH;
                 parser->Length = 4;
@@ -328,12 +386,47 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
             // circle back for more input
             return File_Decomp_OK;
         }
+        // search state
+        case ZIP_STATE_SEARCH:
+        {
+            if ( parser->header_searcher == nullptr )
+            {
+                // initialize local file header searcher
+                parser->header_searcher = new BoyerMooreSearchCase(
+                    header_pattern, sizeof(header_pattern));
+            }
+
+            // search for the next local file header
+            int pos = parser->header_searcher->search(
+                SessionPtr->Next_In, SessionPtr->Avail_In);
+
+            if ( pos < 0 )
+            {
+                // not found, skip the rest of this flush
+                parser->State = ZIP_STATE_SKIP;
+                parser->Length = SessionPtr->Avail_In;
+
+                // search for next file
+                parser->Next = ZIP_STATE_SEARCH;
+                parser->Next_Length = 0;
+                continue;
+            }
+
+            // found, skip the rest of this file
+            parser->State = ZIP_STATE_SKIP;
+            parser->Length = pos;
+
+            // land on local header
+            parser->Next = ZIP_STATE_LH;
+            parser->Next_Length = 4;
+            continue;
+        }
         // skip state
         case ZIP_STATE_SKIP:
             // check if we need to skip
             if ( parser->Index < parser->Length )
             {
-                unsigned skip = parser->Length - parser->Index;
+                uint32_t skip = parser->Length - parser->Index;
 
                 // check if we can skip within this flush
                 if ( SessionPtr->Avail_In < skip )
@@ -341,7 +434,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr)
                     // the available input is < skip
                     parser->Index += SessionPtr->Avail_In;
 
-                    unsigned min = SessionPtr->Avail_In < SessionPtr->Avail_Out ?
+                    uint32_t min = SessionPtr->Avail_In < SessionPtr->Avail_Out ?
                                    SessionPtr->Avail_In : SessionPtr->Avail_Out;
 
                     // copy what we can
index 09b6351afa912f80cdc4425f243e3c12a537f204..4d87edf74b52220ef2b099bffd0a105454bdd689 100644 (file)
 // with this program; if not, write to the Free Software Foundation, Inc.,
 // 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 //--------------------------------------------------------------------------
-
 // file_decomp_zip.h author Brandon Stultz <brastult@cisco.com>
 
 #ifndef FILE_DECOMP_ZIP_H
 #define FILE_DECOMP_ZIP_H
 
+#include <zlib.h>
+
 #include "file_decomp.h"
 
-#include <zlib.h>
+namespace snort
+{
+class BoyerMooreSearchCase;
+}
 
 static const uint32_t ZIP_LOCAL_HEADER = 0x04034B50;
+static const uint8_t header_pattern[4] = { 0x50, 0x4B, 0x03, 0x04 };
+static const uint8_t DATA_DESC_BIT = 0x08;
 
 enum fd_ZIP_states
 {
@@ -33,8 +39,8 @@ enum fd_ZIP_states
 
     // skipped:
     // ZIP_STATE_VER,         // version (2 bytes)
-    // ZIP_STATE_BITFLAG,     // bitflag (2 bytes)
 
+    ZIP_STATE_BITFLAG,        // bitflag (2 bytes)
     ZIP_STATE_METHOD,         // compression method (2 bytes)
 
     // skipped:
@@ -57,6 +63,7 @@ enum fd_ZIP_states
 
     ZIP_STATE_INFLATE_INIT,   // initialize zlib inflate
     ZIP_STATE_INFLATE,        // perform zlib inflate
+    ZIP_STATE_SEARCH,         // search for local header
     ZIP_STATE_SKIP            // skip state
 };
 
@@ -66,25 +73,30 @@ struct fd_ZIP_t
     z_stream Stream;
 
     // decompression progress
-    unsigned progress;
+    uint32_t progress;
 
     // ZIP fields
     uint32_t local_header;
+    uint16_t bitflag;
+    bool data_descriptor;
     uint16_t method;
     uint32_t compressed_size;
     uint16_t filename_length;
     uint16_t extra_length;
 
     // field index
-    unsigned Index;
+    uint32_t Index;
 
     // current parser state
     fd_ZIP_states State;
-    unsigned Length;
+    uint32_t Length;
 
     // next parser state
     fd_ZIP_states Next;
-    unsigned Next_Length;
+    uint32_t Next_Length;
+
+    // local file header searcher
+    snort::BoyerMooreSearchCase* header_searcher;
 };
 
 // allocate and set initial ZIP state