From: Tom Peters (thopeter) <thopeter@cisco.com>
Date: Wed, 27 Mar 2019 17:21:20 +0000 (-0400)
Subject: Merge pull request #1550 in SNORT/snort3 from ~BRASTULT/snort3:mime_decomp to master
X-Git-Tag: 3.0.0-251~7
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e9e92ede5c621a352197f20a4f706d73e6ba3ec7;p=thirdparty%2Fsnort3.git

Merge pull request #1550 in SNORT/snort3 from ~BRASTULT/snort3:mime_decomp to master

Squashed commit of the following:

commit 8c90afe003ccdf8367cfdc75bb10b9bac6d0d396
Author: Brandon Stultz <brastult@cisco.com>
Date:   Mon Mar 4 19:59:41 2019 -0500

    mime: add file decompression
---

diff --git a/src/mime/file_mime_config.cc b/src/mime/file_mime_config.cc
index 44156db81..153c96647 100644
--- a/src/mime/file_mime_config.cc
+++ b/src/mime/file_mime_config.cc
@@ -81,6 +81,36 @@ int DecodeConfig::get_uu_depth()
     return uu_depth;
 }
 
+void DecodeConfig::set_decompress_pdf(bool enabled)
+{
+    decompress_pdf = enabled;
+}
+
+bool DecodeConfig::is_decompress_pdf()
+{
+    return decompress_pdf;
+}
+
+void DecodeConfig::set_decompress_swf(bool enabled)
+{
+    decompress_swf = enabled;
+}
+
+bool DecodeConfig::is_decompress_swf()
+{
+    return decompress_swf;
+}
+
+void DecodeConfig::set_decompress_zip(bool enabled)
+{
+    decompress_zip = enabled;
+}
+
+bool DecodeConfig::is_decompress_zip()
+{
+    return decompress_zip;
+}
+
 int64_t DecodeConfig::get_file_depth()
 {
     return file_depth;
diff --git a/src/mime/file_mime_config.h b/src/mime/file_mime_config.h
index 3811063ac..677f5e05e 100644
--- a/src/mime/file_mime_config.h
+++ b/src/mime/file_mime_config.h
@@ -53,6 +53,15 @@ public:
     void set_uu_depth(int);
     int get_uu_depth();
 
+    void set_decompress_pdf(bool);
+    bool is_decompress_pdf();
+
+    void set_decompress_swf(bool);
+    bool is_decompress_swf();
+
+    void set_decompress_zip(bool);
+    bool is_decompress_zip();
+
     int64_t get_file_depth();
     bool is_decoding_enabled();
     void sync_all_depths();
@@ -65,6 +74,9 @@ private:
     int qp_depth = DEFAULT_DEPTH;
     int bitenc_depth = DEFAULT_DEPTH;
     int uu_depth = DEFAULT_DEPTH;
+    bool decompress_pdf = false;
+    bool decompress_swf = false;
+    bool decompress_zip = false;
     int64_t file_depth = MIN_DEPTH;
     bool decode_enabled = true;
 };
diff --git a/src/mime/file_mime_context_data.cc b/src/mime/file_mime_context_data.cc
index cbf7edd82..4e88734cf 100644
--- a/src/mime/file_mime_context_data.cc
+++ b/src/mime/file_mime_context_data.cc
@@ -34,11 +34,15 @@ unsigned MimeDecodeContextData::mime_ips_id = 0;
 MimeDecodeContextData::MimeDecodeContextData()
 {
     decode_buf = (uint8_t*)snort_alloc(MAX_DEPTH);
+    decompress_buf = (uint8_t*)snort_alloc(MAX_DEPTH);
 }
+
 MimeDecodeContextData::~MimeDecodeContextData()
 {
     snort_free(decode_buf);
+    snort_free(decompress_buf);
     decode_buf = nullptr;
+    decompress_buf = nullptr;
 }
 
 void MimeDecodeContextData::init()
@@ -51,3 +55,10 @@ uint8_t* MimeDecodeContextData::get_decode_buf()
     return data->decode_buf;
 }
 
+uint8_t* MimeDecodeContextData::get_decompress_buf()
+{
+    MimeDecodeContextData* data = IpsContextData::get<MimeDecodeContextData>(mime_ips_id);
+
+    return data->decompress_buf;
+}
+
diff --git a/src/mime/file_mime_context_data.h b/src/mime/file_mime_context_data.h
index dc0e18582..7afde22b0 100644
--- a/src/mime/file_mime_context_data.h
+++ b/src/mime/file_mime_context_data.h
@@ -31,9 +31,11 @@ public:
     static unsigned mime_ips_id;
 
     uint8_t* decode_buf = nullptr;
+    uint8_t* decompress_buf = nullptr;
 
     static void init();
     static uint8_t* get_decode_buf();
+    static uint8_t* get_decompress_buf();
 };
 
 #endif
diff --git a/src/mime/file_mime_decode.cc b/src/mime/file_mime_decode.cc
index 1082d4611..4e5558bc6 100644
--- a/src/mime/file_mime_decode.cc
+++ b/src/mime/file_mime_decode.cc
@@ -138,13 +138,74 @@ DecodeType MimeDecode::get_decode_type()
     return decode_type;
 }
 
+DecodeResult MimeDecode::decompress_data(const uint8_t* buf_in, uint32_t size_in,
+                                         const uint8_t*& buf_out, uint32_t& size_out)
+{
+    DecodeResult result = DECODE_SUCCESS;
+
+    if ( fd_state == nullptr )
+    {
+        buf_out = buf_in;
+        size_out = size_in;
+        return result;
+    }
+
+    uint8_t* decompress_buf = MimeDecodeContextData::get_decompress_buf();
+    fd_state->Next_In = buf_in;
+    fd_state->Avail_In = size_in;
+    fd_state->Next_Out = decompress_buf;
+    fd_state->Avail_Out = MAX_DEPTH;
+
+    const fd_status_t status = File_Decomp(fd_state);
+
+    switch ( status )
+    {
+    case File_Decomp_DecompError:
+        result = DECODE_FAIL;
+        // fallthrough
+    case File_Decomp_NoSig:
+    case File_Decomp_Error:
+        buf_out = buf_in;
+        size_out = size_in;
+        break;
+    default:
+        buf_out = decompress_buf;
+        size_out = fd_state->Next_Out - decompress_buf;
+        break;
+    }
+
+    return result;
+}
+
 MimeDecode::MimeDecode(snort::DecodeConfig* conf)
 {
     config = conf;
+
+    bool decompress_pdf = config->is_decompress_pdf();
+    bool decompress_swf = config->is_decompress_swf();
+    bool decompress_zip = config->is_decompress_zip();
+
+    if ( !decompress_pdf && !decompress_swf && !decompress_zip )
+        return;
+
+    fd_state = File_Decomp_New();
+    fd_state->Modes =
+        (decompress_pdf ? FILE_PDF_DEFL_BIT : 0) |
+        (decompress_swf ? (FILE_SWF_ZLIB_BIT | FILE_SWF_LZMA_BIT) : 0) |
+        (decompress_zip ? FILE_ZIP_DEFL_BIT : 0);
+    fd_state->Alert_Callback = nullptr;
+    fd_state->Alert_Context = nullptr;
+    fd_state->Compr_Depth = 0;
+    fd_state->Decompr_Depth = 0;
+
+    (void)File_Decomp_Init(fd_state);
 }
 
 MimeDecode::~MimeDecode()
 {
+    if (fd_state)
+        File_Decomp_StopFree(fd_state);
+
     if (decoder)
         delete decoder;
 }
diff --git a/src/mime/file_mime_decode.h b/src/mime/file_mime_decode.h
index b95f09bb2..356801c34 100644
--- a/src/mime/file_mime_decode.h
+++ b/src/mime/file_mime_decode.h
@@ -22,6 +22,7 @@
 
 // Email attachment decoder, supports Base64, QP, UU, and Bit7/8
 
+#include "decompress/file_decomp.h"
 #include "framework/counts.h"
 #include "main/snort_types.h"
 #include "mime/decode_base.h"
@@ -75,12 +76,16 @@ public:
 
     DecodeType get_decode_type();
 
+    DecodeResult decompress_data(const uint8_t* buf_in, uint32_t size_in,
+                                 const uint8_t*& buf_out, uint32_t& size_out);
+
     static void init();
 
 private:
     DecodeType decode_type = DECODE_NONE;
     snort::DecodeConfig* config;
     DataDecode* decoder = nullptr;
+    fd_session_t* fd_state = nullptr;
 };
 
 } // namespace snort
diff --git a/src/mime/file_mime_process.cc b/src/mime/file_mime_process.cc
index 65e730591..fbfb6646d 100644
--- a/src/mime/file_mime_process.cc
+++ b/src/mime/file_mime_process.cc
@@ -562,7 +562,7 @@ const uint8_t* MimeSession::process_mime_data_paf(
 
     if ((decode_state) != nullptr)
     {
-        DecodeConfig* conf= decode_conf;
+        DecodeConfig* conf = decode_conf;
         const uint8_t* buffer = nullptr;
         uint32_t buf_size = 0;
 
@@ -570,8 +570,19 @@ const uint8_t* MimeSession::process_mime_data_paf(
 
         if (conf)
         {
-            int detection_size = decode_state->get_detection_depth();
-            set_file_data(buffer, (uint16_t)detection_size);
+            const uint8_t* decomp_buffer = nullptr;
+            uint32_t detection_size, decomp_buf_size = 0;
+
+            detection_size = (uint32_t)decode_state->get_detection_depth();
+
+            DecodeResult result = decode_state->decompress_data(
+                buffer, detection_size, decomp_buffer, decomp_buf_size
+            );
+
+            if ( result != DECODE_SUCCESS )
+                decompress_alert();
+
+            set_file_data(decomp_buffer, decomp_buf_size);
         }
 
         /*Process file type/file signature*/
diff --git a/src/mime/file_mime_process.h b/src/mime/file_mime_process.h
index 609268f5c..5097b6212 100644
--- a/src/mime/file_mime_process.h
+++ b/src/mime/file_mime_process.h
@@ -88,6 +88,7 @@ private:
     virtual int handle_header_line(const uint8_t*, const uint8_t*, int) { return 0; }
     virtual int normalize_data(const uint8_t*, const uint8_t*) { return 0; }
     virtual void decode_alert() { }
+    virtual void decompress_alert() { }
     virtual void reset_state(Flow*) { }
     virtual bool is_end_of_data(Flow*) { return false; }
 
diff --git a/src/service_inspectors/imap/imap.cc b/src/service_inspectors/imap/imap.cc
index 2e75a90aa..756fe6736 100644
--- a/src/service_inspectors/imap/imap.cc
+++ b/src/service_inspectors/imap/imap.cc
@@ -680,6 +680,11 @@ void ImapMime::decode_alert()
     }
 }
 
+void ImapMime::decompress_alert()
+{
+    DetectionEngine::queue_event(GID_IMAP, IMAP_FILE_DECOMP_FAILED);
+}
+
 void ImapMime::reset_state(Flow* ssn)
 {
     IMAP_ResetState(ssn);
diff --git a/src/service_inspectors/imap/imap.h b/src/service_inspectors/imap/imap.h
index 0aa5ebd85..51c645d9a 100644
--- a/src/service_inspectors/imap/imap.h
+++ b/src/service_inspectors/imap/imap.h
@@ -149,6 +149,7 @@ class ImapMime : public snort::MimeSession
     using snort::MimeSession::MimeSession;
 private:
     void decode_alert() override;
+    void decompress_alert() override;
     void reset_state(snort::Flow* ssn) override;
     bool is_end_of_data(snort::Flow* ssn) override;
 };
diff --git a/src/service_inspectors/imap/imap_module.cc b/src/service_inspectors/imap/imap_module.cc
index a7a78a9c4..a3031a110 100644
--- a/src/service_inspectors/imap/imap_module.cc
+++ b/src/service_inspectors/imap/imap_module.cc
@@ -39,6 +39,15 @@ static const Parameter s_params[] =
     { "bitenc_decode_depth", Parameter::PT_INT, "-1:65535", "1460",
       "non-Encoded MIME attachment extraction depth (-1 no limit)" },
 
+    { "decompress_pdf", Parameter::PT_BOOL, nullptr, "false",
+      "decompress pdf files in MIME attachments" },
+
+    { "decompress_swf", Parameter::PT_BOOL, nullptr, "false",
+      "decompress swf files in MIME attachments" },
+
+    { "decompress_zip", Parameter::PT_BOOL, nullptr, "false",
+      "decompress zip files in MIME attachments" },
+
     { "qp_decode_depth", Parameter::PT_INT, "-1:65535", "1460",
       "quoted Printable decoding depth (-1 no limit)" },
 
@@ -55,6 +64,7 @@ static const RuleMap imap_rules[] =
     { IMAP_B64_DECODING_FAILED, "base64 decoding failed" },
     { IMAP_QP_DECODING_FAILED, "quoted-printable decoding failed" },
     { IMAP_UU_DECODING_FAILED, "Unix-to-Unix decoding failed" },
+    { IMAP_FILE_DECOMP_FAILED, "file decompression failed" },
 
     { 0, nullptr }
 };
@@ -97,6 +107,15 @@ bool ImapModule::set(const char*, Value& v, SnortConfig*)
     else if ( v.is("bitenc_decode_depth") )
         config->decode_conf.set_bitenc_depth(mime_value);
 
+    else if ( v.is("decompress_pdf") )
+        config->decode_conf.set_decompress_pdf(v.get_bool());
+
+    else if ( v.is("decompress_swf") )
+        config->decode_conf.set_decompress_swf(v.get_bool());
+
+    else if ( v.is("decompress_zip") )
+        config->decode_conf.set_decompress_zip(v.get_bool());
+
     else if ( v.is("qp_decode_depth") )
         config->decode_conf.set_qp_depth(mime_value);
 
diff --git a/src/service_inspectors/imap/imap_module.h b/src/service_inspectors/imap/imap_module.h
index 3f45b2725..dfcd87a2a 100644
--- a/src/service_inspectors/imap/imap_module.h
+++ b/src/service_inspectors/imap/imap_module.h
@@ -33,6 +33,7 @@
 #define IMAP_B64_DECODING_FAILED    4
 #define IMAP_QP_DECODING_FAILED     5
 #define IMAP_UU_DECODING_FAILED     7
+#define IMAP_FILE_DECOMP_FAILED     8
 
 #define IMAP_NAME "imap"
 #define IMAP_HELP "imap inspection"
diff --git a/src/service_inspectors/pop/pop.cc b/src/service_inspectors/pop/pop.cc
index a9222c4be..17f76164d 100644
--- a/src/service_inspectors/pop/pop.cc
+++ b/src/service_inspectors/pop/pop.cc
@@ -619,6 +619,11 @@ void PopMime::decode_alert()
     }
 }
 
+void PopMime::decompress_alert()
+{
+    DetectionEngine::queue_event(GID_POP, POP_FILE_DECOMP_FAILED);
+}
+
 void PopMime::reset_state(Flow* ssn)
 {
     POP_ResetState(ssn);
diff --git a/src/service_inspectors/pop/pop.h b/src/service_inspectors/pop/pop.h
index 4e4bc2e03..d69655a26 100644
--- a/src/service_inspectors/pop/pop.h
+++ b/src/service_inspectors/pop/pop.h
@@ -104,6 +104,7 @@ class PopMime : public snort::MimeSession
     using snort::MimeSession::MimeSession;
 private:
     void decode_alert() override;
+    void decompress_alert() override;
     void reset_state(snort::Flow* ssn) override;
     bool is_end_of_data(snort::Flow* ssn) override;
 };
diff --git a/src/service_inspectors/pop/pop_module.cc b/src/service_inspectors/pop/pop_module.cc
index 50728e1f7..0f74dc439 100644
--- a/src/service_inspectors/pop/pop_module.cc
+++ b/src/service_inspectors/pop/pop_module.cc
@@ -39,6 +39,15 @@ static const Parameter s_params[] =
     { "bitenc_decode_depth", Parameter::PT_INT, "-1:65535", "1460",
       "Non-Encoded MIME attachment extraction depth (-1 no limit)" },
 
+    { "decompress_pdf", Parameter::PT_BOOL, nullptr, "false",
+      "decompress pdf files in MIME attachments" },
+
+    { "decompress_swf", Parameter::PT_BOOL, nullptr, "false",
+      "decompress swf files in MIME attachments" },
+
+    { "decompress_zip", Parameter::PT_BOOL, nullptr, "false",
+      "decompress zip files in MIME attachments" },
+
     { "qp_decode_depth", Parameter::PT_INT, "-1:65535", "1460",
       "Quoted Printable decoding depth (-1 no limit)" },
 
@@ -55,6 +64,7 @@ static const RuleMap pop_rules[] =
     { POP_B64_DECODING_FAILED, "base64 decoding failed" },
     { POP_QP_DECODING_FAILED, "quoted-printable decoding failed" },
     { POP_UU_DECODING_FAILED, "Unix-to-Unix decoding failed" },
+    { POP_FILE_DECOMP_FAILED, "file decompression failed" },
     { 0, nullptr }
 };
 
@@ -96,6 +106,15 @@ bool PopModule::set(const char*, Value& v, SnortConfig*)
     else if ( v.is("bitenc_decode_depth") )
         config->decode_conf.set_bitenc_depth(mime_value);
 
+    else if ( v.is("decompress_pdf") )
+        config->decode_conf.set_decompress_pdf(v.get_bool());
+
+    else if ( v.is("decompress_swf") )
+        config->decode_conf.set_decompress_swf(v.get_bool());
+
+    else if ( v.is("decompress_zip") )
+        config->decode_conf.set_decompress_zip(v.get_bool());
+
     else if ( v.is("qp_decode_depth") )
         config->decode_conf.set_qp_depth(mime_value);
 
diff --git a/src/service_inspectors/pop/pop_module.h b/src/service_inspectors/pop/pop_module.h
index 37e462b95..4daac862f 100644
--- a/src/service_inspectors/pop/pop_module.h
+++ b/src/service_inspectors/pop/pop_module.h
@@ -33,6 +33,7 @@
 #define POP_B64_DECODING_FAILED    4
 #define POP_QP_DECODING_FAILED     5
 #define POP_UU_DECODING_FAILED     7
+#define POP_FILE_DECOMP_FAILED     8
 
 #define POP_NAME "pop"
 #define POP_HELP "pop inspection"
diff --git a/src/service_inspectors/smtp/smtp.cc b/src/service_inspectors/smtp/smtp.cc
index 1aaef0436..943d49fc2 100644
--- a/src/service_inspectors/smtp/smtp.cc
+++ b/src/service_inspectors/smtp/smtp.cc
@@ -1368,6 +1368,11 @@ void SmtpMime::decode_alert()
     }
 }
 
+void SmtpMime::decompress_alert()
+{
+    DetectionEngine::queue_event(GID_SMTP, SMTP_FILE_DECOMP_FAILED);
+}
+
 void SmtpMime::reset_state(Flow* ssn)
 {
     SMTP_ResetState(ssn);
diff --git a/src/service_inspectors/smtp/smtp.h b/src/service_inspectors/smtp/smtp.h
index 6a75a3cc5..031b65fc8 100644
--- a/src/service_inspectors/smtp/smtp.h
+++ b/src/service_inspectors/smtp/smtp.h
@@ -148,6 +148,7 @@ private:
         int max_header_len) override;
     int normalize_data(const uint8_t* ptr, const uint8_t* data_end) override;
     void decode_alert() override;
+    void decompress_alert() override;
     void reset_state(snort::Flow* ssn) override;
     bool is_end_of_data(snort::Flow* ssn) override;
 };
diff --git a/src/service_inspectors/smtp/smtp_module.cc b/src/service_inspectors/smtp/smtp_module.cc
index 27e655abe..444de65fb 100644
--- a/src/service_inspectors/smtp/smtp_module.cc
+++ b/src/service_inspectors/smtp/smtp_module.cc
@@ -82,6 +82,15 @@ static const Parameter s_params[] =
     { "data_cmds", Parameter::PT_STRING, nullptr, nullptr,
       "commands that initiate sending of data with an end of data delimiter" },
 
+    { "decompress_pdf", Parameter::PT_BOOL, nullptr, "false",
+      "decompress pdf files in MIME attachments" },
+
+    { "decompress_swf", Parameter::PT_BOOL, nullptr, "false",
+      "decompress swf files in MIME attachments" },
+
+    { "decompress_zip", Parameter::PT_BOOL, nullptr, "false",
+      "decompress zip files in MIME attachments" },
+
     { "email_hdrs_log_depth", Parameter::PT_INT, "0:20480", "1464",
       "depth for logging email headers" },
 
@@ -155,6 +164,7 @@ static const RuleMap smtp_rules[] =
     { SMTP_UU_DECODING_FAILED, "Unix-to-Unix decoding failed" },
     { SMTP_AUTH_ABORT_AUTH, "Cyrus SASL authentication attack" },
     { SMTP_AUTH_COMMAND_OVERFLOW, "attempted authentication command buffer overflow" },
+    { SMTP_FILE_DECOMP_FAILED, "file decompression failed" },
 
     { 0, nullptr }
 };
@@ -247,6 +257,15 @@ bool SmtpModule::set(const char*, Value& v, SnortConfig*)
     else if ( v.is("data_cmds"))
         add_commands(v, PCMD_DATA);
 
+    else if ( v.is("decompress_pdf") )
+        config->decode_conf.set_decompress_pdf(v.get_bool());
+
+    else if ( v.is("decompress_swf") )
+        config->decode_conf.set_decompress_swf(v.get_bool());
+
+    else if ( v.is("decompress_zip") )
+        config->decode_conf.set_decompress_zip(v.get_bool());
+
     else if ( v.is("email_hdrs_log_depth") )
         config->log_config.email_hdrs_log_depth = v.get_uint16();
 
diff --git a/src/service_inspectors/smtp/smtp_module.h b/src/service_inspectors/smtp/smtp_module.h
index 3c7dd08eb..303e911d9 100644
--- a/src/service_inspectors/smtp/smtp_module.h
+++ b/src/service_inspectors/smtp/smtp_module.h
@@ -44,6 +44,7 @@
 #define SMTP_UU_DECODING_FAILED     13
 #define SMTP_AUTH_ABORT_AUTH        14
 #define SMTP_AUTH_COMMAND_OVERFLOW  15
+#define SMTP_FILE_DECOMP_FAILED     16
 
 #define SMTP_NAME "smtp"
 #define SMTP_HELP "smtp inspection"