From: Michael Altizer (mialtize) Date: Tue, 9 Mar 2021 03:56:50 +0000 (+0000) Subject: Merge pull request #2734 in SNORT/snort3 from ~BRASTULT/snort3:zip_data_desc to master X-Git-Tag: 3.1.2.0~7 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=4175382d9eb3db86137b54ced611d088d2ceca5d;p=thirdparty%2Fsnort3.git Merge pull request #2734 in SNORT/snort3 from ~BRASTULT/snort3:zip_data_desc to master Squashed commit of the following: commit 142372710cf9717980b1e2ab14f11c2f7ea5a18d Author: Brandon Stultz Date: Wed Feb 3 00:23:10 2021 -0500 decompress: add support for streaming ZIPs --- diff --git a/src/decompress/file_decomp.h b/src/decompress/file_decomp.h index 384c5133b..bb8bf9d3f 100644 --- a/src/decompress/file_decomp.h +++ b/src/decompress/file_decomp.h @@ -192,7 +192,7 @@ inline bool Get_1(fd_session_t* SessionPtr, uint8_t* c) /* If available, get N bytes from the input queue. All N must be available for this call to succeed. */ -inline bool Get_N(fd_session_t* SessionPtr, const uint8_t** c, uint16_t N) +inline bool Get_N(fd_session_t* SessionPtr, const uint8_t** c, uint32_t N) { if ( (SessionPtr->Next_In != nullptr) && (SessionPtr->Avail_In >= N) ) { @@ -222,7 +222,7 @@ inline bool Put_1(fd_session_t* SessionPtr, uint8_t c) /* If the output queue has room available, place N bytes onto the queue. The output queue must have space for N bytes for this call to succeed. */ -inline bool Put_N(fd_session_t* SessionPtr, const uint8_t* c, uint16_t N) +inline bool Put_N(fd_session_t* SessionPtr, const uint8_t* c, uint32_t N) { if ( (SessionPtr->Next_Out != nullptr) && (SessionPtr->Avail_Out >= N) ) { @@ -244,8 +244,8 @@ inline bool Move_1(fd_session_t* SessionPtr) (SessionPtr->Next_In != nullptr) && (SessionPtr->Avail_In > 0) ) { *(SessionPtr->Next_Out) = *(SessionPtr->Next_In); - SessionPtr->Next_Out += 1; SessionPtr->Next_In += 1; + SessionPtr->Next_Out += 1; SessionPtr->Avail_In -= 1; SessionPtr->Avail_Out -= 1; SessionPtr->Total_In += 1; @@ -258,14 +258,14 @@ inline bool Move_1(fd_session_t* SessionPtr) /* If the input queue has at least N bytes available AND there's at space for at least N bytes in the output queue, then move all N bytes. */ -inline bool Move_N(fd_session_t* SessionPtr, uint16_t N) +inline bool Move_N(fd_session_t* SessionPtr, uint32_t N) { if ( (SessionPtr->Next_Out != nullptr) && (SessionPtr->Avail_Out >= N) && (SessionPtr->Next_In != nullptr) && (SessionPtr->Avail_In >= N) ) { memcpy( (char*)SessionPtr->Next_Out, (const char*)SessionPtr->Next_In, N); - SessionPtr->Next_Out += N; SessionPtr->Next_In += N; + SessionPtr->Next_Out += N; SessionPtr->Avail_In -= N; SessionPtr->Avail_Out -= N; SessionPtr->Total_In += N; diff --git a/src/decompress/file_decomp_zip.cc b/src/decompress/file_decomp_zip.cc index c880900d9..6d2603c65 100644 --- a/src/decompress/file_decomp_zip.cc +++ b/src/decompress/file_decomp_zip.cc @@ -15,7 +15,6 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. //-------------------------------------------------------------------------- - // file_decomp_zip.cc author Brandon Stultz #ifdef HAVE_CONFIG_H @@ -23,6 +22,8 @@ #endif #include "file_decomp_zip.h" + +#include "helpers/boyer_moore_search.h" #include "utils/util.h" using namespace snort; @@ -98,12 +99,12 @@ fd_status_t File_Decomp_Init_ZIP(fd_session_t* SessionPtr) SessionPtr->ZIP = (fd_ZIP_t*)snort_calloc(sizeof(fd_ZIP_t)); // file_decomp.cc already matched the local header - // skip the version and bitflag (4 bytes) + // skip the version (2 bytes) SessionPtr->ZIP->State = ZIP_STATE_SKIP; - SessionPtr->ZIP->Length = 4; + SessionPtr->ZIP->Length = 2; - // land on compression method - SessionPtr->ZIP->Next = ZIP_STATE_METHOD; + // land on bitflag + SessionPtr->ZIP->Next = ZIP_STATE_BITFLAG; SessionPtr->ZIP->Next_Length = 2; return File_Decomp_OK; @@ -119,6 +120,12 @@ fd_status_t File_Decomp_End_ZIP(fd_session_t* SessionPtr) if ( SessionPtr->ZIP->State == ZIP_STATE_INFLATE ) Inflate_End(SessionPtr); + if ( SessionPtr->ZIP->header_searcher ) + { + delete SessionPtr->ZIP->header_searcher; + SessionPtr->ZIP->header_searcher = nullptr; + } + // File_Decomp_Free() will free SessionPtr->ZIP return File_Decomp_OK; } @@ -155,6 +162,8 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) // reset ZIP fields parser->local_header = 0; + parser->bitflag = 0; + parser->data_descriptor = false; parser->method = 0; parser->compressed_size = 0; parser->filename_length = 0; @@ -163,18 +172,39 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) // reset decompression progress parser->progress = 0; - // skip the version and bitflag (4 bytes) + // skip the version (2 bytes) parser->State = ZIP_STATE_SKIP; - parser->Length = 4; + parser->Length = 2; - // land on compression method - parser->Next = ZIP_STATE_METHOD; + // land on bitflag + parser->Next = ZIP_STATE_BITFLAG; parser->Next_Length = 2; continue; } // read the local header byte = *SessionPtr->Next_In; - parser->local_header |= byte << parser->Index*8; + parser->local_header |= byte << (parser->Index * 8); + break; + // bitflag + case ZIP_STATE_BITFLAG: + // check if we are done with the bitflag + if ( parser->Index == parser->Length ) + { + // read the bitflag, reset the index + parser->Index = 0; + + // check the data descriptor bit + if ( parser->bitflag & DATA_DESC_BIT ) + parser->data_descriptor = true; + + // read the compression method next + parser->State = ZIP_STATE_METHOD; + parser->Length = 2; + continue; + } + // read the bitflag + byte = *SessionPtr->Next_In; + parser->bitflag |= byte << (parser->Index * 8); break; // compression method case ZIP_STATE_METHOD: @@ -196,7 +226,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) } // read the method byte = *SessionPtr->Next_In; - parser->method |= byte << parser->Index*8; + parser->method |= byte << (parser->Index * 8); break; // compressed size case ZIP_STATE_COMPSIZE: @@ -217,7 +247,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) } // read the compressed size byte = *SessionPtr->Next_In; - parser->compressed_size |= byte << parser->Index*8; + parser->compressed_size |= byte << (parser->Index * 8); break; // filename length case ZIP_STATE_FILENAMELEN: @@ -234,7 +264,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) } // read the filename length byte = *SessionPtr->Next_In; - parser->filename_length |= byte << parser->Index*8; + parser->filename_length |= byte << (parser->Index * 8); break; // extra length case ZIP_STATE_EXTRALEN: @@ -254,11 +284,20 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) // the compression type is deflate (8), // land on the compressed stream, init zlib parser->Next = ZIP_STATE_INFLATE_INIT; - parser->Next_Length = parser->compressed_size; + parser->Next_Length = 0; continue; } - // no output space or compression type isn't deflate, skip the stream + // no output space or compression type isn't deflate + if ( parser->data_descriptor ) + { + // search for the next file + parser->Next = ZIP_STATE_SEARCH; + parser->Next_Length = 0; + continue; + } + + // skip the stream parser->Length += parser->compressed_size; // land on another local header @@ -268,7 +307,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) } // read the extra length byte = *SessionPtr->Next_In; - parser->extra_length |= byte << parser->Index*8; + parser->extra_length |= byte << (parser->Index * 8); break; // initialize zlib inflate case ZIP_STATE_INFLATE_INIT: @@ -295,6 +334,17 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) if ( status == File_Decomp_BlockOut ) { // ran out of output space + if ( parser->data_descriptor ) + { + // close the inflate stream + Inflate_End(SessionPtr); + + // search for next file + parser->State = ZIP_STATE_SEARCH; + parser->Length = 0; + continue; + } + // progress should be < compressed_size if ( parser->progress >= parser->compressed_size ) return File_Decomp_Error; @@ -318,6 +368,14 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) // close the inflate stream Inflate_End(SessionPtr); + if ( parser->data_descriptor ) + { + // search for the next file + parser->State = ZIP_STATE_SEARCH; + parser->Length = 0; + continue; + } + // parse next local header parser->State = ZIP_STATE_LH; parser->Length = 4; @@ -328,12 +386,47 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) // circle back for more input return File_Decomp_OK; } + // search state + case ZIP_STATE_SEARCH: + { + if ( parser->header_searcher == nullptr ) + { + // initialize local file header searcher + parser->header_searcher = new BoyerMooreSearchCase( + header_pattern, sizeof(header_pattern)); + } + + // search for the next local file header + int pos = parser->header_searcher->search( + SessionPtr->Next_In, SessionPtr->Avail_In); + + if ( pos < 0 ) + { + // not found, skip the rest of this flush + parser->State = ZIP_STATE_SKIP; + parser->Length = SessionPtr->Avail_In; + + // search for next file + parser->Next = ZIP_STATE_SEARCH; + parser->Next_Length = 0; + continue; + } + + // found, skip the rest of this file + parser->State = ZIP_STATE_SKIP; + parser->Length = pos; + + // land on local header + parser->Next = ZIP_STATE_LH; + parser->Next_Length = 4; + continue; + } // skip state case ZIP_STATE_SKIP: // check if we need to skip if ( parser->Index < parser->Length ) { - unsigned skip = parser->Length - parser->Index; + uint32_t skip = parser->Length - parser->Index; // check if we can skip within this flush if ( SessionPtr->Avail_In < skip ) @@ -341,7 +434,7 @@ fd_status_t File_Decomp_ZIP(fd_session_t* SessionPtr) // the available input is < skip parser->Index += SessionPtr->Avail_In; - unsigned min = SessionPtr->Avail_In < SessionPtr->Avail_Out ? + uint32_t min = SessionPtr->Avail_In < SessionPtr->Avail_Out ? SessionPtr->Avail_In : SessionPtr->Avail_Out; // copy what we can diff --git a/src/decompress/file_decomp_zip.h b/src/decompress/file_decomp_zip.h index 09b6351af..4d87edf74 100644 --- a/src/decompress/file_decomp_zip.h +++ b/src/decompress/file_decomp_zip.h @@ -15,17 +15,23 @@ // with this program; if not, write to the Free Software Foundation, Inc., // 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. //-------------------------------------------------------------------------- - // file_decomp_zip.h author Brandon Stultz #ifndef FILE_DECOMP_ZIP_H #define FILE_DECOMP_ZIP_H +#include + #include "file_decomp.h" -#include +namespace snort +{ +class BoyerMooreSearchCase; +} static const uint32_t ZIP_LOCAL_HEADER = 0x04034B50; +static const uint8_t header_pattern[4] = { 0x50, 0x4B, 0x03, 0x04 }; +static const uint8_t DATA_DESC_BIT = 0x08; enum fd_ZIP_states { @@ -33,8 +39,8 @@ enum fd_ZIP_states // skipped: // ZIP_STATE_VER, // version (2 bytes) - // ZIP_STATE_BITFLAG, // bitflag (2 bytes) + ZIP_STATE_BITFLAG, // bitflag (2 bytes) ZIP_STATE_METHOD, // compression method (2 bytes) // skipped: @@ -57,6 +63,7 @@ enum fd_ZIP_states ZIP_STATE_INFLATE_INIT, // initialize zlib inflate ZIP_STATE_INFLATE, // perform zlib inflate + ZIP_STATE_SEARCH, // search for local header ZIP_STATE_SKIP // skip state }; @@ -66,25 +73,30 @@ struct fd_ZIP_t z_stream Stream; // decompression progress - unsigned progress; + uint32_t progress; // ZIP fields uint32_t local_header; + uint16_t bitflag; + bool data_descriptor; uint16_t method; uint32_t compressed_size; uint16_t filename_length; uint16_t extra_length; // field index - unsigned Index; + uint32_t Index; // current parser state fd_ZIP_states State; - unsigned Length; + uint32_t Length; // next parser state fd_ZIP_states Next; - unsigned Next_Length; + uint32_t Next_Length; + + // local file header searcher + snort::BoyerMooreSearchCase* header_searcher; }; // allocate and set initial ZIP state