From: Giuseppe Longo Date: Thu, 23 Jul 2015 08:39:35 +0000 (+0200) Subject: util-file-decompression: add swf decompression API X-Git-Tag: suricata-4.1.0-beta1~227 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7b23d305423ecc6eeb6508f9f9309dcdd0bc2b23;p=thirdparty%2Fsuricata.git util-file-decompression: add swf decompression API This adds a new module that permits to decompress swf file compressed with zlib or lzma algorithms. The API that performs decompression will take a compressed buffer and build a new decompressed buffer following the FWS format which represents an uncompressed file. The maximum buffer that can be created is up to 50mb. --- diff --git a/src/Makefile.am b/src/Makefile.am index 88750d012c..e24254e513 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -398,6 +398,8 @@ util-device.c util-device.h \ util-enum.c util-enum.h \ util-error.c util-error.h \ util-file.c util-file.h \ +util-file-decompression.c util-file-decompression.h \ +util-file-swf-decompression.c util-file-swf-decompression.h \ util-fix_checksum.c util-fix_checksum.h \ util-fmemopen.c util-fmemopen.h \ util-hash.c util-hash.h \ diff --git a/src/detect.h b/src/detect.h index 89a6f0d527..1c38ac493a 100644 --- a/src/detect.h +++ b/src/detect.h @@ -790,8 +790,10 @@ enum { typedef struct HttpReassembledBody_ { const uint8_t *buffer; + uint8_t *decompressed_buffer; uint32_t buffer_size; /**< size of the buffer itself */ uint32_t buffer_len; /**< data len in the buffer */ + uint32_t decompressed_buffer_len; uint64_t offset; /**< data offset */ } HttpReassembledBody; diff --git a/src/util-file-decompression.c b/src/util-file-decompression.c new file mode 100644 index 0000000000..9ce8158684 --- /dev/null +++ b/src/util-file-decompression.c @@ -0,0 +1,214 @@ +/* Copyright (C) 2017 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** \file + * + * \author Giuseppe Longo + * + * \brief Decompress files transfered via HTTP corresponding to file_data + * keyword. + * + */ + +#include "suricata-common.h" +#include "suricata.h" + +#include "app-layer-htp.h" + +#include "util-file-decompression.h" +#include "util-file-swf-decompression.h" +#include "util-misc.h" +#include "util-print.h" + +#define SWF_ZLIB_MIN_VERSION 0x06 +#define SWF_LZMA_MIN_VERSION 0x0D + +int FileIsSwfFile(const uint8_t *buffer, uint32_t buffer_len) +{ + if (buffer_len >= 3 && buffer[1] == 'W' && buffer[2] == 'S') { + if (buffer[0] == 'F') + return FILE_SWF_NO_COMPRESSION; + else if (buffer[0] == 'C') + return FILE_SWF_ZLIB_COMPRESSION; + else if (buffer[0] == 'Z') + return FILE_SWF_LZMA_COMPRESSION; + else + return FILE_IS_NOT_SWF; + } + + return FILE_IS_NOT_SWF; +} + +/** + * \brief This function decompresses a buffer with zlib/lzma algorithm + * + * \param buffer compressed buffer + * \param buffer_len compressed buffer length + * \param decompressed_buffer buffer that store decompressed data + * \param decompressed_buffer_len decompressesd data length + * \param swf_type decompression algorithm to use + * \param decompress_depth how much decompressed data we want to store + * \param compress_depth how much compressed data we want to decompress + * + * \retval 1 if decompression works + * \retval 0 an error occured, and event set + */ +int FileSwfDecompression(const uint8_t *buffer, uint32_t buffer_len, + DetectEngineThreadCtx *det_ctx, + int index, + int swf_type, + uint32_t decompress_depth, + uint32_t compress_depth) +{ + int r = 0; + + int compression_type = FileIsSwfFile(buffer, buffer_len); + if (compression_type == FILE_SWF_NO_COMPRESSION) { + return 0; + } + + uint32_t offset = 0; + if (compression_type == FILE_SWF_ZLIB_COMPRESSION) { + /* compressed data start from the 4th bytes */ + offset = 8; + } else if (compression_type == FILE_SWF_LZMA_COMPRESSION) { + /* compressed data start from the 17th bytes */ + offset = 17; + } + + if (buffer_len <= offset) { + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_INVALID_SWF_LENGTH); + return 0; + } + + uint32_t compressed_data_len = 0; + if (buffer_len > offset && compress_depth == 0) { + compressed_data_len = buffer_len - offset; + } else if (compress_depth > 0 && compress_depth <= buffer_len) { + compressed_data_len = compress_depth; + } else if (compress_depth > 0 && compress_depth > buffer_len) { + compressed_data_len = buffer_len; + } + + /* get swf version */ + uint8_t swf_version = FileGetSwfVersion(buffer, buffer_len); + if (compression_type == FILE_SWF_ZLIB_COMPRESSION && + swf_version < SWF_ZLIB_MIN_VERSION) + { + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_INVALID_SWF_VERSION); + return 0; + } + if (compression_type == FILE_SWF_LZMA_COMPRESSION && + swf_version < SWF_LZMA_MIN_VERSION) + { + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_INVALID_SWF_VERSION); + return 0; + } + + /* get flash decompressed file length */ + uint32_t decompressed_swf_len = FileGetSwfDecompressedLen(buffer, buffer_len); + if (decompressed_swf_len == 0) { + decompressed_swf_len = MIN_SWF_LEN; + } + + /* if decompress_depth is 0, keep the flash file length */ + uint32_t decompressed_data_len = (decompress_depth == 0) ? decompressed_swf_len : decompress_depth; + decompressed_data_len += 8; + + if (det_ctx->hsbd[index].decompressed_buffer_len == 0 || + det_ctx->hsbd[index].decompressed_buffer_len < decompressed_data_len) { + void *ptmp = NULL; + ptmp = SCRealloc(det_ctx->hsbd[index].decompressed_buffer, + decompressed_data_len); + if (ptmp == NULL) { + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_NO_MEM); + return 0; + } + det_ctx->hsbd[index].decompressed_buffer = ptmp; + } + + det_ctx->hsbd[index].decompressed_buffer_len = decompressed_data_len; + memset(det_ctx->hsbd[index].decompressed_buffer, 0x00, + det_ctx->hsbd[index].decompressed_buffer_len); + + /* + * FWS format + * | 4 bytes | 4 bytes | n bytes | + * | 'FWS' + version | script len | data | + */ + det_ctx->hsbd[index].decompressed_buffer[0] = 'F'; + det_ctx->hsbd[index].decompressed_buffer[1] = 'W'; + det_ctx->hsbd[index].decompressed_buffer[2] = 'S'; + det_ctx->hsbd[index].decompressed_buffer[3] = swf_version; + memcpy(det_ctx->hsbd[index].decompressed_buffer + 4, &decompressed_swf_len, 4); + + if ((swf_type == HTTP_SWF_COMPRESSION_ZLIB || swf_type == HTTP_SWF_COMPRESSION_BOTH) && + compression_type == FILE_SWF_ZLIB_COMPRESSION) + { + /* the first 8 bytes represents the fws header, see 'FWS format' above. + * data will start from 8th bytes + */ + r = FileSwfZlibDecompression(det_ctx, + (uint8_t *)buffer + offset, compressed_data_len, + det_ctx->hsbd[index].decompressed_buffer + 8, + det_ctx->hsbd[index].decompressed_buffer_len - 8); + if (r == 0) + goto error; + + } else if ((swf_type == HTTP_SWF_COMPRESSION_LZMA || swf_type == HTTP_SWF_COMPRESSION_BOTH) && + compression_type == FILE_SWF_LZMA_COMPRESSION) + { + /* we need to setup the lzma header */ + /* + * | 5 bytes | 8 bytes | n bytes | + * | LZMA properties | Uncompressed length | Compressed data | + */ + compressed_data_len += 13; + uint8_t *compressed_data = SCMalloc(compressed_data_len); + if (compressed_data == NULL) { + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_NO_MEM); + goto error; + } + /* put lzma properties */ + memcpy(compressed_data, buffer + 12, 5); + /* put lzma end marker */ + memset(compressed_data + 5, 0xFF, 8); + /* put compressed data */ + memcpy(compressed_data + 13, buffer + offset, compressed_data_len - 13); + + /* the first 8 bytes represents the fws header, see 'FWS format' above. + * data will start from 8th bytes + */ + r = FileSwfLzmaDecompression(det_ctx, + compressed_data, compressed_data_len, + det_ctx->hsbd[index].decompressed_buffer + 8, + det_ctx->hsbd[index].decompressed_buffer_len - 8); + SCFree(compressed_data); + if (r == 0) + goto error; + } else { + goto error; + } + + return 1; + +error: + det_ctx->hsbd[index].decompressed_buffer_len = 0; + memset(det_ctx->hsbd[index].decompressed_buffer, 0x00, + det_ctx->hsbd[index].decompressed_buffer_len); + return 0; +} diff --git a/src/util-file-decompression.h b/src/util-file-decompression.h new file mode 100644 index 0000000000..420be04343 --- /dev/null +++ b/src/util-file-decompression.h @@ -0,0 +1,44 @@ +/* Copyright (C) 2017 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** \file + * + * \author Giuseppe Longo + * + * + */ + +#ifndef __UTIL_FILE_DECOMPRESSION_H__ +#define __UTIL_FILE_DECOMPRESSION_H__ + +#include "detect.h" + +enum { + FILE_IS_NOT_SWF = 0, + FILE_SWF_NO_COMPRESSION, + FILE_SWF_ZLIB_COMPRESSION, + FILE_SWF_LZMA_COMPRESSION, +}; + +int FileIsSwfFile(const uint8_t *buffer, uint32_t buffer_len); +int FileSwfDecompression(const uint8_t *buffer, uint32_t buffer_len, + DetectEngineThreadCtx *det_ctx, + int hsbd_index, + int swf_type, + uint32_t decompress_depth, uint32_t compress_depth); + +#endif /* __UTIL_FILE_DECOMPRESSION_H__ */ diff --git a/src/util-file-swf-decompression.c b/src/util-file-swf-decompression.c new file mode 100644 index 0000000000..81b07d8163 --- /dev/null +++ b/src/util-file-swf-decompression.c @@ -0,0 +1,183 @@ +/* Copyright (C) 2017 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** \file + * + * \author Giuseppe Longo + * + */ + + +#include "suricata.h" +#include "suricata-common.h" + +#include "app-layer-htp.h" + +#include "util-file-decompression.h" +#include "util-file-swf-decompression.h" +#include "util-misc.h" +#include "util-print.h" + +#include + +#ifdef HAVE_LIBLZMA +#include +#endif + +#define MAX_SWF_DECOMPRESSED_LEN 50000000 +/* + * Return uncompressed file length + * in little-endian order + */ +uint32_t FileGetSwfDecompressedLen(const uint8_t *buffer, + const uint32_t buffer_len) +{ + if (buffer_len < 8) { + return 0; + } + + int a = buffer[4]; + int b = buffer[5]; + int c = buffer[6]; + int d = buffer[7]; + + uint32_t value = (((a & 0xff) << 24) | ((b & 0xff) << 16) | ((c & 0xff) << 8) | (d & 0xff)); + + uint32_t len = (((value >> 24) & 0x000000FF) | ((value >> 8) & 0x0000FF00) | + ((value << 8) & 0x00FF0000) | ((value << 24) & 0xFF000000)); + + return MIN(MAX_SWF_DECOMPRESSED_LEN, len); +} + +uint8_t FileGetSwfVersion(const uint8_t *buffer, const uint32_t buffer_len) +{ + if (buffer_len > 3) + return buffer[3]; + + return 0; +} + +/* CWS format */ +/* + * | 4 bytes | 4 bytes | n bytes | + * | 'CWS' + version | script len | compressed data | + */ +int FileSwfZlibDecompression(DetectEngineThreadCtx *det_ctx, + uint8_t *compressed_data, uint32_t compressed_data_len, + uint8_t *decompressed_data, uint32_t decompressed_data_len) +{ + int ret = 1; + z_stream infstream; + infstream.zalloc = Z_NULL; + infstream.zfree = Z_NULL; + infstream.opaque = Z_NULL; + + infstream.avail_in = (uInt)compressed_data_len; + infstream.next_in = (Bytef *)compressed_data; + infstream.avail_out = (uInt)decompressed_data_len; + infstream.next_out = (Bytef *)decompressed_data; + + inflateInit(&infstream); + int result = inflate(&infstream, Z_NO_FLUSH); + switch(result) { + case Z_STREAM_END: + break; + case Z_OK: + break; + case Z_DATA_ERROR: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_Z_DATA_ERROR); + ret = 0; + break; + case Z_STREAM_ERROR: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_Z_STREAM_ERROR); + ret = 0; + break; + case Z_BUF_ERROR: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_Z_BUF_ERROR); + ret = 0; + break; + default: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_Z_UNKNOWN_ERROR); + ret = 0; + break; + } + inflateEnd(&infstream); + + return ret; +} + +/* ZWS format */ +/* + * | 4 bytes | 4 bytes | 4 bytes | 5 bytes | n bytes | 6 bytes | + * | 'ZWS' + version | script len | compressed len | LZMA props | LZMA data | LZMA end marker | + */ +int FileSwfLzmaDecompression(DetectEngineThreadCtx *det_ctx, + uint8_t *compressed_data, uint32_t compressed_data_len, + uint8_t *decompressed_data, uint32_t decompressed_data_len) +{ +#ifdef HAVE_LIBLZMA + int ret = 1; + lzma_stream strm = LZMA_STREAM_INIT; + lzma_ret result = lzma_alone_decoder(&strm, UINT64_MAX /* memlimit */); + if (result != LZMA_OK) { + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_DECODER_ERROR); + return 0; + } + + strm.avail_in = compressed_data_len; + strm.next_in = compressed_data; + strm.avail_out = decompressed_data_len; + strm.next_out = decompressed_data; + + result = lzma_code(&strm, LZMA_RUN); + switch(result) { + case LZMA_STREAM_END: + break; + case LZMA_OK: + break; + case LZMA_MEMLIMIT_ERROR: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_MEMLIMIT_ERROR); + ret = 0; + break; + case LZMA_OPTIONS_ERROR: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_OPTIONS_ERROR); + ret = 0; + break; + case LZMA_FORMAT_ERROR: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_FORMAT_ERROR); + ret = 0; + break; + case LZMA_DATA_ERROR: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_DATA_ERROR); + ret = 0; + break; + case LZMA_BUF_ERROR: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_BUF_ERROR); + ret = 0; + break; + default: + DetectEngineSetEvent(det_ctx, FILE_DECODER_EVENT_LZMA_UNKNOWN_ERROR); + ret = 0; + break; + } + + lzma_end(&strm); + return ret; +#else + return 0; +#endif /* HAVE_LIBLZMA */ +} diff --git a/src/util-file-swf-decompression.h b/src/util-file-swf-decompression.h new file mode 100644 index 0000000000..aa2af4b106 --- /dev/null +++ b/src/util-file-swf-decompression.h @@ -0,0 +1,42 @@ +/* Copyright (C) 2017 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** \file + * + * \author Giuseppe Longo + * + * + */ + +#ifndef __UTIL_FILE_SWF_DECOMPRESSION_H__ +#define __UTIL_FILE_SWF_DECOMPRESSION_H__ + +/* If we don't have the decompressed data len, + * we use a default value. + */ +#define MIN_SWF_LEN 2920 + +uint8_t FileGetSwfVersion(const uint8_t *buffer, const uint32_t buffer_len); +uint32_t FileGetSwfDecompressedLen(const uint8_t *buffer, uint32_t buffr_len); +int FileSwfZlibDecompression(DetectEngineThreadCtx *det_ctx, + uint8_t *compressed_data, uint32_t compressed_data_len, + uint8_t *decompressed_data, uint32_t decompressed_data_len); +int FileSwfLzmaDecompression(DetectEngineThreadCtx *det_ctx, + uint8_t *compressed_data, uint32_t compressed_data_len, + uint8_t *decompressed_data, uint32_t decompressed_data_len); + +#endif /* __UTIL_FILE_SWF_DECOMPRESSION_H__ */