../../service_inspectors/http_inspect/http_test_manager.cc
../../service_inspectors/http_inspect/http_test_input.cc
../../service_inspectors/http_inspect/http_field.cc
+ ../../service_inspectors/http_inspect/http_compress_stream.cc
LIBS ${ZLIB_LIBRARIES}
)
add_cpputest( pub_sub_ftp_events_test
return buf;
}
unsigned StreamSplitter::max(snort::Flow*) { return 0; }
+uint8_t TraceApi::get_constraints_generation() { return 0; }
+void TraceApi::filter(const Packet&) { }
+void trace_vprintf(const char*, TraceLevel, const char*, const snort::Packet*, const char*, va_list) { }
}
+THREAD_LOCAL const snort::Trace* http_trace = nullptr;
+
HttpParaList::UriParam::UriParam() { }
HttpParaList::JsNormParam::~JsNormParam() { }
HttpParaList::~HttpParaList() { }
http_module.h
http_test_input.cc
http_test_input.h
+ http_compress_stream.cc
+ http_compress_stream.h
http_flow_data.cc
http_flow_data.h
http_context_data.cc
zlib are very large. It would save a lot of memory and some processing time for script detection
to unzip one time in scan() and store the result for eventual use by reassemble(). The memory
lost by storing partial message sections in HI while waiting for reassemble() would be more than
-compensated for by not having two instances of zlib.
+compensated for by not having two instances of zlib. Decompression now always happens
+once during scan() for all HTTP bodies, regardless of whether script detection is enabled.
+Doing decompression in scan() changes the order of events, so all decompression events are
+triggered after scan().
For request bodies, when partial_depth_body parameter is set to a non zero value, a partial body
will be subjected to partial inspection if its length is below partial_depth_body value. When
enum HXBodyState { HX_BODY_NOT_COMPLETE, HX_BODY_LAST_SEG, HX_BODY_COMPLETE,
HX_BODY_COMPLETE_EXPECT_TRAILERS, HX_BODY_NO_BODY };
+enum
+{
+ TRACE_COMPRESS,
+};
+
} // end namespace HttpCommon
#endif
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2026-2026 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_compress_stream.cc author Oleksandr Fedorych <ofedoryc@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "http_compress_stream.h"
+
+#include "http_common.h"
+#include "http_module.h"
+
+using namespace HttpEnums;
+using namespace HttpCommon;
+
+HttpCompressStream::HttpCompressStream()
+ : compress_stream { nullptr }
+ , gzip_header_bytes_processed { 0 }
+ , compression_id { CMP_NONE }
+ , gzip_state { GZIP_TBD }
+{ }
+
+HttpCompressStream::~HttpCompressStream()
+{
+ if ( compress_stream == nullptr )
+ return;
+
+ inflateEnd(compress_stream);
+ delete compress_stream;
+
+ debug_logf(http_trace, TRACE_COMPRESS, nullptr, "Compress: zlib cleared\n");
+}
+
+bool HttpCompressStream::setup(HttpEnums::CompressId compression)
+{
+ if ( compress_stream != nullptr )
+ {
+ assert(false);
+
+ compression_id = CMP_NONE;
+ delete compress_stream;
+ compress_stream = nullptr;
+
+ return false;
+ }
+
+ compression_id = compression;
+
+ switch ( compression )
+ {
+ case CMP_DEFLATE:
+ case CMP_GZIP:
+ compress_stream = new z_stream;
+
+ compress_stream->zalloc = Z_NULL;
+ compress_stream->zfree = Z_NULL;
+ compress_stream->opaque = Z_NULL;
+ compress_stream->next_in = Z_NULL;
+ compress_stream->avail_in = 0;
+
+ if ( const int window_bits = compression == CMP_GZIP ? GZIP_WINDOW_BITS : DEFLATE_WINDOW_BITS;
+ inflateInit2(compress_stream, window_bits) == Z_OK )
+ {
+ debug_logf(http_trace, TRACE_COMPRESS, nullptr, "Compress: zlib setup is successful\n");
+
+ return true;
+ }
+
+ assert(false);
+
+ compression_id = CMP_NONE;
+ delete compress_stream;
+ compress_stream = nullptr;
+
+ return false;
+ default:
+ assert(false);
+
+ compression_id = CMP_NONE;
+
+ return false;
+ }
+}
+
+std::optional<uint32_t> HttpCompressStream::decompress(const uint8_t* src, uint32_t src_size,
+ uint8_t* dst, uint32_t& dst_size, bool at_start,
+ HttpInfractions* infractions, HttpEventGen* const events)
+{
+ switch ( get_compression_id() )
+ {
+ case CMP_NONE:
+ return std::nullopt;
+ case CMP_DEFLATE:
+ case CMP_GZIP:
+ return decompress_zlib(src, src_size, dst, dst_size,
+ at_start, infractions, events);
+ default:
+ assert(false);
+ return std::nullopt;
+ }
+}
+
+uint8_t* HttpCompressStream::process_gzip_header(const uint8_t* data, uint32_t length,
+ HttpInfractions* const infractions, HttpEventGen* const events)
+{
+ uint32_t input_bytes_processed = 0;
+ uint8_t* modified_data = nullptr;
+
+ if ( gzip_state == GZIP_TBD )
+ {
+ static constexpr uint8_t gzip_magic[] = { 0x1f, 0x8b, 0x08 };
+ static constexpr uint8_t magic_length = 3;
+
+ const uint32_t magic_cmp_len = (magic_length - gzip_header_bytes_processed) < length ?
+ (magic_length - gzip_header_bytes_processed) : length;
+
+ if ( memcmp(data, gzip_magic + gzip_header_bytes_processed, magic_cmp_len) != 0 )
+ gzip_state = GZIP_MAGIC_BAD;
+ else if ( gzip_header_bytes_processed + length >= magic_length )
+ gzip_state = GZIP_MAGIC_GOOD;
+
+ gzip_header_bytes_processed += magic_cmp_len;
+ input_bytes_processed += magic_cmp_len;
+ }
+
+ if ( gzip_state == GZIP_MAGIC_GOOD and length > input_bytes_processed )
+ {
+ const uint8_t gzip_flags = data[input_bytes_processed];
+
+ if ( gzip_flags & GZIP_FLAG_FEXTRA )
+ {
+ *infractions += INF_GZIP_FEXTRA;
+ events->create_event(EVENT_GZIP_FEXTRA);
+ }
+
+ if ( gzip_flags & GZIP_RESERVED_FLAGS )
+ {
+ *infractions += INF_GZIP_RESERVED_FLAGS;
+ events->create_event(EVENT_GZIP_RESERVED_FLAGS);
+
+ modified_data = new uint8_t[length];
+ memcpy(modified_data, data, length);
+ modified_data[input_bytes_processed] &= ~GZIP_RESERVED_FLAGS;
+ }
+
+ gzip_header_bytes_processed++;
+ gzip_state = GZIP_FLAGS_PROCESSED;
+ }
+
+ return modified_data;
+}
+
+bool HttpCompressStream::gzip_header_check_done() const
+{
+ return gzip_state == HttpEnums::GZIP_MAGIC_BAD or
+ gzip_state == HttpEnums::GZIP_FLAGS_PROCESSED;
+}
+
+std::optional<std::uint32_t> HttpCompressStream::decompress_zlib(const uint8_t* src, uint32_t src_size,
+ uint8_t* dst, uint32_t& dst_size, bool at_start,
+ HttpInfractions* const infractions, HttpEventGen* const events)
+{
+ uint8_t* data_w_updated_hdr = nullptr;
+
+ if ( get_compression_id() == CMP_GZIP and !gzip_header_check_done() )
+ data_w_updated_hdr = process_gzip_header(src, src_size, infractions, events);
+
+ if ( data_w_updated_hdr != nullptr )
+ compress_stream->next_in = const_cast<Bytef*>(data_w_updated_hdr);
+ else
+ compress_stream->next_in = const_cast<Bytef*>(src);
+
+ compress_stream->avail_in = src_size;
+ compress_stream->next_out = dst + dst_size;
+ compress_stream->avail_out = MAX_OCTETS - dst_size;
+
+ const int result = inflate(compress_stream, Z_SYNC_FLUSH);
+
+ delete[] data_w_updated_hdr;
+
+ switch ( result )
+ {
+ case Z_OK:
+ case Z_STREAM_END:
+ dst_size = MAX_OCTETS - compress_stream->avail_out;
+
+ debug_logf(http_trace, TRACE_COMPRESS, nullptr, "Compress: decompressed %u/%u, used %u/%u\n",
+ src_size - compress_stream->avail_in, src_size, dst_size, MAX_OCTETS);
+
+ if ( compress_stream->avail_in > 0 )
+ {
+ if ( result == Z_STREAM_END )
+ {
+ // The zipped data stream ended but there is more input data
+ if ( get_compression_id() == CMP_GZIP )
+ {
+ *infractions += INF_GZIP_EARLY_END;
+ events->create_event(EVENT_GZIP_EARLY_END);
+ }
+ else
+ {
+ *infractions += INF_DEFLATE_EARLY_END;
+ events->create_event(EVENT_DEFLATE_EARLY_END);
+ }
+
+ const uInt num_copy = (compress_stream->avail_in <= compress_stream->avail_out) ?
+ compress_stream->avail_in : compress_stream->avail_out;
+
+ memcpy(dst + dst_size, src + (src_size - compress_stream->avail_in), num_copy);
+ dst_size += num_copy;
+
+ debug_logf(http_trace, TRACE_COMPRESS, nullptr,
+ "Compress: compressed data ended, copied %u, used %u/%u\n", num_copy, dst_size, MAX_OCTETS);
+
+ compress_stream->avail_in -= num_copy;
+ compression_id = CMP_NONE;
+ }
+ else
+ {
+ assert(compress_stream->avail_out == 0);
+
+ // The data expanded too much
+ debug_logf(http_trace, TRACE_COMPRESS, nullptr, "Compress: data caused buffer to overrun\n");
+ }
+
+ // FIXIT-E - Will need to clear gzip header processing state here when we implement
+ // processing multiple gzip members in a message section
+ }
+
+ return compress_stream->avail_in;
+ case Z_DATA_ERROR:
+ if ( get_compression_id() == CMP_DEFLATE and at_start )
+ {
+ // Some incorrect implementations of deflate don't use the expected header. Feed a
+ // dummy header to zlib and retry the inflate.
+ static constexpr uint8_t zlib_header[2] = { 0x78, 0x01 };
+
+ inflateReset(compress_stream);
+
+ compress_stream->next_in = const_cast<Bytef*>(zlib_header);
+ compress_stream->avail_in = sizeof(zlib_header);
+
+ const int ret = inflate(compress_stream, Z_SYNC_FLUSH);
+
+ if ( ret == Z_OK or ret == Z_STREAM_END )
+ {
+ debug_log(http_trace, TRACE_COMPRESS, nullptr, "Compress: deflate header substituted\n");
+
+ // Start over at the beginning
+ const auto decompressed = decompress_zlib(src, src_size, dst, dst_size, false, infractions, events);
+
+ if ( decompressed )
+ HttpModule::increment_peg_counts(PEG_INCORRECT_DEFLATE_HEADER);
+
+ return decompressed;
+ }
+
+ assert(false);
+ return std::nullopt;
+ }
+
+ [[fallthrough]];
+ default:
+ if ( get_compression_id() == CMP_GZIP )
+ {
+ *infractions += INF_GZIP_FAILURE;
+ events->create_event(EVENT_GZIP_FAILURE);
+ HttpModule::increment_peg_counts(PEG_COMPRESSED_GZIP_FAILED);
+ }
+ else
+ {
+ *infractions += INF_DEFLATE_FAILURE;
+ events->create_event(EVENT_DEFLATE_FAILURE);
+ HttpModule::increment_peg_counts(PEG_COMPRESSED_DEFLATE_FAILED);
+ }
+
+ compression_id = CMP_NONE;
+
+ debug_logf(http_trace, TRACE_COMPRESS, nullptr, "Compress: decompress failed, %s\n",
+ compress_stream->msg ? compress_stream->msg : "unknown error");
+
+ return std::nullopt;
+ }
+}
+
+void HttpCompressStream::copy_compressed(const uint8_t* src, uint32_t src_size, uint8_t* dst, uint32_t& dst_size)
+{
+ copy_raw(src, src_size, dst, dst_size);
+
+ debug_logf(http_trace, TRACE_COMPRESS, nullptr, "Compress: data copied, used %u/%u\n", dst_size, MAX_OCTETS);
+}
+
+void HttpCompressStream::copy_raw(const uint8_t* src, uint32_t src_size, uint8_t* dst, uint32_t& dst_size)
+{
+ // The following precaution is necessary because mixed compressed and uncompressed data can
+ // cause the buffer to overrun even though we are not decompressing right now
+
+ if ( src_size > MAX_OCTETS - dst_size )
+ src_size = MAX_OCTETS - dst_size;
+
+ memcpy(dst + dst_size, src, src_size);
+ dst_size += src_size;
+}
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2026-2026 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_compress_stream.h author Oleksandr Fedorych <ofedoryc@cisco.com>
+
+#ifndef HTTP_COMPRESS_STREAM_H
+#define HTTP_COMPRESS_STREAM_H
+
+#include <optional>
+#include <zlib.h>
+
+#include "http_enum.h"
+#include "http_event.h"
+
+class HttpCompressStream
+{
+public:
+ HttpCompressStream();
+ HttpCompressStream(const HttpCompressStream& other) = delete;
+ HttpCompressStream& operator=(const HttpCompressStream& other) = delete;
+ HttpCompressStream(HttpCompressStream&& other) = delete;
+ HttpCompressStream& operator=(HttpCompressStream&& other) = delete;
+ ~HttpCompressStream();
+
+ bool setup(HttpEnums::CompressId compression);
+
+ std::optional<std::uint32_t> decompress(const uint8_t* src, uint32_t src_size,
+ uint8_t* dst, uint32_t& dst_size, bool at_start,
+ HttpInfractions* const infractions, HttpEventGen* const events);
+
+ void copy_compressed(const uint8_t* src, uint32_t src_size, uint8_t* dst, uint32_t& dst_size);
+ static void copy_raw(const uint8_t* src, uint32_t src_size, uint8_t* dst, uint32_t& dst_size);
+
+ HttpEnums::CompressId get_compression_id() const
+ { return compression_id; }
+
+private:
+ std::optional<std::uint32_t> decompress_zlib(const uint8_t* src, uint32_t src_size,
+ uint8_t* dst, uint32_t& dst_size, bool at_start,
+ HttpInfractions* const infractions, HttpEventGen* const events);
+
+ uint8_t* process_gzip_header(const uint8_t* data, uint32_t length,
+ HttpInfractions* const infractions, HttpEventGen* const events);
+ bool gzip_header_check_done() const;
+
+ z_stream* compress_stream;
+ uint32_t gzip_header_bytes_processed;
+ HttpEnums::CompressId compression_id;
+ HttpEnums::GzipVerificationState gzip_state;
+};
+
+#endif
#include "http_cutter.h"
#include "http_common.h"
+#include "http_compress_stream.h"
#include "http_enum.h"
#include "http_flow_data.h"
#include "http_module.h"
return SCAN_NOT_FOUND;
}
+HttpBodyCutter::DecompressOutput HttpBodyCutter::decompress(const uint8_t* src, uint32_t src_size,
+ HttpInfractions* infractions, HttpEventGen* events)
+{
+ auto* compress = session_data->compress[source_id];
+
+ if ( compress == nullptr )
+ return { Buffer { src, src_size }, src_size };
+
+ auto& dst = session_data->partial_buffer[source_id];
+ auto& dst_size = session_data->partial_buffer_length[source_id];
+
+ if ( dst == nullptr )
+ dst = new uint8_t[MAX_OCTETS];
+
+ const bool at_start = (octets_seen + session_data->partial_raw_bytes[source_id]) == 0 and dst_size == 0;
+
+ const auto prev_size = dst_size;
+
+ if ( const auto ret = compress->decompress(src, src_size, dst, dst_size, at_start, infractions, events) )
+ return { Buffer { dst + prev_size, dst_size - prev_size }, src_size - *ret };
+
+ compress->copy_compressed(src, src_size, dst, dst_size);
+
+ return { std::nullopt, src_size };
+}
+
HttpBodyCutter::HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder_,
- CompressId compression_)
- : accelerated_blocking(accelerated_blocking_), compression(compression_), finder(finder_)
+ HttpFlowData* const session_data, SourceId source_id)
+ : accelerated_blocking(accelerated_blocking_)
+ , finder(finder_)
+ , session_data(session_data)
+ , source_id(source_id)
{
- if (accelerated_blocking)
+ if ( accelerated_blocking )
{
- if ((compression == CMP_GZIP) || (compression == CMP_DEFLATE))
- {
- compress_stream = new z_stream();
- compress_stream->zalloc = Z_NULL;
- compress_stream->zfree = Z_NULL;
- compress_stream->next_in = Z_NULL;
- compress_stream->avail_in = 0;
- const int window_bits = (compression == CMP_GZIP) ?
- GZIP_WINDOW_BITS : DEFLATE_WINDOW_BITS;
- if (inflateInit2(compress_stream, window_bits) != Z_OK)
- {
- assert(false);
- compression = CMP_NONE;
- delete compress_stream;
- compress_stream = nullptr;
- }
- }
-
static const uint8_t inspect_string[] = { '<', '/', 's', 'c', 'r', 'i', 'p', 't', '>' };
static const uint8_t inspect_upper[] = { '<', '/', 'S', 'C', 'R', 'I', 'P', 'T', '>' };
}
}
-HttpBodyCutter::~HttpBodyCutter()
-{
- if (compress_stream != nullptr)
- {
- inflateEnd(compress_stream);
- delete compress_stream;
- }
-}
-
-ScanResult HttpBodyClCutter::cut(const uint8_t* buffer, uint32_t length, HttpInfractions*,
- HttpEventGen*, uint32_t flow_target, bool stretch, HXBodyState)
+ScanResult HttpBodyClCutter::cut(const uint8_t* buffer, uint32_t length,
+ HttpInfractions* infractions, HttpEventGen* events, uint32_t flow_target, bool stretch,
+ HXBodyState)
{
assert(remaining > octets_seen);
if (octets_seen + length < flow_target)
{
+ const auto [accelerate, consumed] = analyze_body(buffer, length, infractions, events);
+
+ if ( consumed < length )
+ {
+ num_flush = consumed;
+ remaining -= octets_seen + num_flush;
+ return SCAN_FOUND_PIECE;
+ }
+
octets_seen += length;
- return need_accelerated_blocking(buffer, length) ?
- SCAN_NOT_FOUND_ACCELERATE : SCAN_NOT_FOUND;
+ return accelerate ? SCAN_NOT_FOUND_ACCELERATE : SCAN_NOT_FOUND;
}
if (!stretch)
{
- remaining -= flow_target;
- num_flush = flow_target - octets_seen;
- if (remaining > 0)
- {
- need_accelerated_blocking(buffer, num_flush);
- return SCAN_FOUND_PIECE;
- }
- else
- return SCAN_FOUND;
+ assert(flow_target >= octets_seen);
+ const uint32_t planned_flush = flow_target - octets_seen;
+ const auto [_, consumed] = analyze_body(buffer, planned_flush, infractions, events);
+
+ num_flush = consumed;
+ remaining -= octets_seen + num_flush;
+
+ return remaining > 0 ? SCAN_FOUND_PIECE : SCAN_FOUND;
}
if (octets_seen + length < remaining)
{
// The message body continues beyond this segment
// Stretch the section to include this entire segment provided it is not too big
+ uint32_t planned_flush = 0;
if (octets_seen + length <= flow_target + MAX_SECTION_STRETCH)
- num_flush = length;
+ planned_flush = length;
else
- num_flush = flow_target - octets_seen;
+ planned_flush = flow_target - octets_seen;
+
+ const auto [_, consumed] = analyze_body(buffer, planned_flush, infractions, events);
+
+ num_flush = consumed;
remaining -= octets_seen + num_flush;
- need_accelerated_blocking(buffer, num_flush);
+
return SCAN_FOUND_PIECE;
}
if (remaining - flow_target <= MAX_SECTION_STRETCH)
{
// Stretch the section to finish the message body
- num_flush = remaining - octets_seen;
- remaining = 0;
- return SCAN_FOUND;
+ const uint32_t planned_flush = remaining - octets_seen;
+ const auto [_, consumed] = decompress(buffer, planned_flush, infractions, events);
+
+ num_flush = consumed;
+ remaining -= octets_seen + num_flush;
+
+ return remaining > 0 ? SCAN_FOUND_PIECE : SCAN_FOUND;
}
// Cannot stretch to the end of the message body. Cut at the original target.
- num_flush = flow_target - octets_seen;
- remaining -= flow_target;
- need_accelerated_blocking(buffer, num_flush);
+ const uint32_t planned_flush = flow_target - octets_seen;
+ const auto [_, consumed] = decompress(buffer, planned_flush, infractions, events);
+
+ num_flush = consumed;
+ remaining -= octets_seen + num_flush;
+
return SCAN_FOUND_PIECE;
}
-ScanResult HttpBodyOldCutter::cut(const uint8_t* buffer, uint32_t length, HttpInfractions*,
- HttpEventGen*, uint32_t flow_target, bool stretch, HXBodyState)
+ScanResult HttpBodyOldCutter::cut(const uint8_t* buffer, uint32_t length,
+ HttpInfractions* infractions, HttpEventGen* events, uint32_t flow_target, bool stretch,
+ HXBodyState)
{
if (flow_target == 0)
{
if (octets_seen + length < flow_target)
{
+ const auto [accelerate, consumed] = analyze_body(buffer, length, infractions, events);
+
+ if ( consumed < length )
+ {
+ num_flush = consumed;
+ return SCAN_FOUND_PIECE;
+ }
+
// Not enough data yet to create a message section
octets_seen += length;
- return need_accelerated_blocking(buffer, length) ?
- SCAN_NOT_FOUND_ACCELERATE : SCAN_NOT_FOUND;
+ return accelerate ? SCAN_NOT_FOUND_ACCELERATE : SCAN_NOT_FOUND;
}
else if (stretch && (octets_seen + length <= flow_target + MAX_SECTION_STRETCH))
{
// Cut the section at the end of this TCP segment to avoid splitting a packet
- num_flush = length;
- need_accelerated_blocking(buffer, num_flush);
+ const uint32_t planned_flush = length;
+ const auto [_, consumed] = analyze_body(buffer, planned_flush, infractions, events);
+
+ num_flush = consumed;
+
return SCAN_FOUND_PIECE;
}
else
{
// Cut the section at the target length. Either stretching is not allowed or the end of
// the segment is too far away.
- num_flush = flow_target - octets_seen;
- need_accelerated_blocking(buffer, num_flush);
+ const uint32_t planned_flush = flow_target - octets_seen;
+ const auto [_, consumed] = analyze_body(buffer, planned_flush, infractions, events);
+
+ num_flush = consumed;
+
return SCAN_FOUND_PIECE;
}
}
skip_amount = adjusted_target-data_seen;
}
- accelerate_this_packet = need_accelerated_blocking(buffer+k, skip_amount) ||
- accelerate_this_packet;
+ if ( accelerated_blocking or !discard_mode )
+ {
+ const auto [accelerate, consumed] = analyze_body(buffer + k, skip_amount,
+ infractions, events);
+ accelerate_this_packet = accelerate or accelerate_this_packet;
+
+ if ( consumed < skip_amount and !discard_mode )
+ {
+ expected -= consumed;
+
+ assert(expected != 0);
+
+ num_flush = k + consumed;
+ data_seen = 0;
+ return SCAN_FOUND_PIECE;
+ }
+ }
k += skip_amount - 1;
if ((expected -= skip_amount) == 0)
{
data_seen = 0;
num_flush = k+1;
- return SCAN_FOUND_PIECE;
+ return discard_mode ? SCAN_DISCARD_PIECE : SCAN_FOUND_PIECE;
}
break;
}
// planned to delete them during reassembly. Because they are not part of a valid chunk
// they will be reassembled after all. This will overrun the adjusted_target making the
// message section a little bigger than planned. It's not important.
+ {
uint32_t skip_amount = length-k;
skip_amount = (skip_amount <= adjusted_target-data_seen) ? skip_amount :
adjusted_target-data_seen;
- accelerate_this_packet = need_accelerated_blocking(buffer+k, skip_amount) ||
- accelerate_this_packet;
+
+ const auto [accelerate, consumed] = analyze_body(buffer + k,
+ skip_amount, infractions, events);
+ accelerate_this_packet = accelerate or accelerate_this_packet;
+
+ if ( consumed < skip_amount )
+ {
+ num_flush = k + consumed;
+ data_seen = 0;
+ return SCAN_FOUND_PIECE;
+ }
+
k += skip_amount - 1;
if ((data_seen += skip_amount) == adjusted_target)
{
return SCAN_FOUND_PIECE;
}
break;
+ }
}
}
if (discard_mode)
return SCAN_NOT_FOUND;
}
-ScanResult HttpBodyHXCutter::cut(const uint8_t* buffer, uint32_t length,
- HttpInfractions* infractions, HttpEventGen* events, uint32_t flow_target, bool stretch,
- HXBodyState state)
+ScanResult HttpBodyHXCutter::cut(const uint8_t* buffer, uint32_t length, HttpInfractions* infractions,
+ HttpEventGen* events, uint32_t flow_target, bool stretch, HXBodyState state)
{
// If the headers included a content length header (expected length >= 0), check it against the
// actual message body length. Alert if it does not match at the end of the message body or if
if (octets_seen + length < flow_target)
{
// Not enough data yet to create a message section
+ const auto [accelerate, consumed] = analyze_body(buffer, length, infractions, events);
+
+ if ( consumed < length )
+ {
+ num_flush = consumed;
+ total_octets_scanned += num_flush;
+ return SCAN_FOUND_PIECE;
+ }
+
octets_seen += length;
total_octets_scanned += length;
- return need_accelerated_blocking(buffer, length) ?
- SCAN_NOT_FOUND_ACCELERATE : SCAN_NOT_FOUND;
+ return accelerate ? SCAN_NOT_FOUND_ACCELERATE : SCAN_NOT_FOUND;
}
else
{
+ uint32_t planned_flush = 0;
if (stretch && (octets_seen + length <= flow_target + MAX_SECTION_STRETCH))
- num_flush = length;
+ planned_flush = length;
else
- num_flush = flow_target - octets_seen;
+ planned_flush = flow_target - octets_seen;
+
+ const auto [_, consumed] = analyze_body(buffer, planned_flush, infractions, events);
+
+ num_flush = consumed;
total_octets_scanned += num_flush;
- need_accelerated_blocking(buffer, num_flush);
+
return SCAN_FOUND_PIECE;
}
}
else if (state == HX_BODY_LAST_SEG)
{
const uint32_t adjusted_target = stretch ? MAX_SECTION_STRETCH + flow_target : flow_target;
+ uint32_t planned_flush = 0;
if (octets_seen + length <= adjusted_target)
- num_flush = length;
+ planned_flush = length;
else
- num_flush = flow_target - octets_seen;
+ planned_flush = flow_target - octets_seen;
+
+ const auto [_, consumed] = decompress(buffer, planned_flush, infractions, events);
+ num_flush = consumed;
total_octets_scanned += num_flush;
+
if (num_flush == length)
return SCAN_FOUND;
else
// This method searches the input stream looking for a script or other dangerous content that
// requires script detection. Exactly what we are looking for is encapsulated in dangerous().
//
-// Return value true indicates a match and enables the packet that completes the matching sequence
-// to be sent for partial inspection.
+// Return value FlushContext contains:
+// accelerate: true indicates a match and enables the packet that completes the matching sequence
+// to be sent for partial inspection.
+// consumed: how many input bytes the decompressor actually processed. When less than the
+// input length the decompressor buffer is full and the caller should flush early.
//
// Any attempt to optimize this code should be mindful that once you skip any part of the message
// body, dangerous() loses the ability to unzip subsequent data.
-bool HttpBodyCutter::need_accelerated_blocking(const uint8_t* data, uint32_t length)
+HttpBodyCutter::FlushContext HttpBodyCutter::analyze_body(const uint8_t* data, uint32_t length,
+ HttpInfractions* infractions, HttpEventGen* events)
{
- const bool need_accelerated_blocking = accelerated_blocking && dangerous(data, length);
- if (need_accelerated_blocking)
+ if ( !accelerated_blocking )
+ {
+ const auto [_, consumed] = decompress(data, length, infractions, events);
+ assert(consumed <= length);
+ return { false, consumed };
+ }
+
+ const auto result = dangerous(data, length, infractions, events);
+ assert(result.consumed <= length);
+
+ if ( result.accelerate )
HttpModule::increment_peg_counts(PEG_SCRIPT_DETECTION);
- return need_accelerated_blocking;
+
+ return result;
}
bool HttpBodyCutter::find_partial(const uint8_t* input_buf, uint32_t input_length, bool end)
}
// Currently we do accelerated blocking when we see a javascript
-bool HttpBodyCutter::dangerous(const uint8_t* data, uint32_t length)
+HttpBodyCutter::FlushContext HttpBodyCutter::dangerous(const uint8_t* data, uint32_t length,
+ HttpInfractions* infractions, HttpEventGen* events)
{
- const uint8_t* input_buf = data;
- uint32_t input_length = length;
- uint8_t* decomp_output = nullptr;
-
- // Zipped flows must be decompressed before we can check them. Unzipping for accelerated
- // blocking is completely separate from the unzipping done later in reassemble().
- if ((compression == CMP_GZIP) || (compression == CMP_DEFLATE))
- {
- // Previous decompression failures make it impossible to search for scripts
- if (decompress_failed)
- return true;
+ const auto [decompressed, consumed] = decompress(data, length, infractions, events);
- const uint32_t decomp_buffer_size = MAX_OCTETS;
- decomp_output = new uint8_t[decomp_buffer_size];
+ if ( !decompressed )
+ return { true, consumed };
- compress_stream->next_in = const_cast<Bytef*>(data);
- compress_stream->avail_in = length;
- compress_stream->next_out = decomp_output;
- compress_stream->avail_out = decomp_buffer_size;
-
- int ret_val = inflate(compress_stream, Z_SYNC_FLUSH);
-
- // Not going to be subtle about this and try to fix decompression problems. If it doesn't
- // work out we assume it could be dangerous.
- if (((ret_val != Z_OK) && (ret_val != Z_STREAM_END)) || (compress_stream->avail_in > 0))
- {
- decompress_failed = true;
- delete[] decomp_output;
- return true;
- }
-
- input_buf = decomp_output;
- input_length = decomp_buffer_size - compress_stream->avail_out;
- }
-
- std::unique_ptr<uint8_t[]> uniq(decomp_output);
+ auto [input_buf, input_length] = *decompressed;
if ( input_length > string_length )
{
if ( partial_match and find_partial(input_buf, input_length, true) )
- return true;
+ return { true, consumed };
if ( finder->search(input_buf, input_length) >= 0 )
- return true;
+ return { true, consumed };
uint32_t delta = input_length - string_length + 1;
input_buf += delta;
}
if ( find_partial(input_buf, input_length, false) )
- return true;
+ return { true, consumed };
- return false;
+ return { false, consumed };
}
uint8_t HttpZeroNineCutter::match[] = { 'H', 'T', 'T', 'P', '/' };
#define HTTP_CUTTER_H
#include <cassert>
-#include <zlib.h>
+#include <optional>
#include "http_common.h"
#include "http_enum.h"
class HttpBodyCutter : public HttpCutter
{
public:
+ using Buffer = std::pair<const uint8_t*, uint32_t>;
+
HttpBodyCutter(bool accelerated_blocking_, ScriptFinder* finder,
- HttpEnums::CompressId compression_);
- ~HttpBodyCutter() override;
+ HttpFlowData* const session_data, HttpCommon::SourceId source_id);
void soft_reset() override { octets_seen = 0; }
+private:
+ struct FlushContext
+ {
+ bool accelerate = false;
+ uint32_t consumed = 0;
+ };
+
+ struct DecompressOutput
+ {
+ std::optional<Buffer> decompressed;
+ uint32_t consumed = 0;
+ };
+
protected:
- bool need_accelerated_blocking(const uint8_t* data, uint32_t length);
+ FlushContext analyze_body(const uint8_t* data, uint32_t length,
+ HttpInfractions* infractions, HttpEventGen* events);
+ DecompressOutput decompress(const uint8_t* src, uint32_t src_size,
+ HttpInfractions* infractions, HttpEventGen* events);
+ const bool accelerated_blocking;
private:
- bool dangerous(const uint8_t* data, uint32_t length);
+ FlushContext dangerous(const uint8_t* data, uint32_t length,
+ HttpInfractions* infractions, HttpEventGen* events);
bool find_partial(const uint8_t*, uint32_t, bool);
- const bool accelerated_blocking;
uint8_t partial_match = 0;
- HttpEnums::CompressId compression = HttpEnums::CompressId::CMP_NONE;
- bool decompress_failed = false;
uint8_t string_length = 0;
- z_stream* compress_stream = nullptr;
ScriptFinder* const finder;
const uint8_t* match_string = nullptr;
const uint8_t* match_string_upper = nullptr;
+ HttpFlowData* const session_data;
+ const HttpCommon::SourceId source_id;
};
class HttpBodyClCutter : public HttpBodyCutter
{
public:
- HttpBodyClCutter(int64_t expected_length,
- bool accelerated_blocking,
- ScriptFinder* finder,
- HttpEnums::CompressId compression) :
- HttpBodyCutter(accelerated_blocking, finder, compression),
- remaining(expected_length)
- { assert(remaining > 0); }
+ HttpBodyClCutter(int64_t expected_length, bool accelerated_blocking_, ScriptFinder* finder,
+ HttpFlowData* const session_data, HttpCommon::SourceId source_id)
+ : HttpBodyCutter(accelerated_blocking_, finder, session_data, source_id)
+ , remaining(expected_length)
+ { assert(remaining > 0); }
+
HttpEnums::ScanResult cut(const uint8_t*, uint32_t length, HttpInfractions*, HttpEventGen*,
uint32_t flow_target, bool stretch, HttpCommon::HXBodyState) override;
class HttpBodyOldCutter : public HttpBodyCutter
{
public:
- HttpBodyOldCutter(bool accelerated_blocking, ScriptFinder* finder,
- HttpEnums::CompressId compression) :
- HttpBodyCutter(accelerated_blocking, finder, compression)
- {}
+ HttpBodyOldCutter(bool accelerated_blocking_, ScriptFinder* finder,
+ HttpFlowData* const session_data, HttpCommon::SourceId source_id)
+ : HttpBodyCutter(accelerated_blocking_, finder, session_data, source_id)
+ { }
HttpEnums::ScanResult cut(const uint8_t*, uint32_t, HttpInfractions*, HttpEventGen*,
uint32_t flow_target, bool stretch, HttpCommon::HXBodyState) override;
};
class HttpBodyChunkCutter : public HttpBodyCutter
{
public:
- HttpBodyChunkCutter(int64_t maximum_chunk_length_, bool accelerated_blocking,
- ScriptFinder* finder, HttpEnums::CompressId compression) :
- HttpBodyCutter(accelerated_blocking, finder, compression),
- maximum_chunk_length(maximum_chunk_length_)
- {}
+ HttpBodyChunkCutter(int64_t maximum_chunk_length_, bool accelerated_blocking_, ScriptFinder* finder,
+ HttpFlowData* const session_data, HttpCommon::SourceId source_id)
+ : HttpBodyCutter(accelerated_blocking_, finder, session_data, source_id)
+ , maximum_chunk_length(maximum_chunk_length_)
+ { }
+
HttpEnums::ScanResult cut(const uint8_t* buffer, uint32_t length,
HttpInfractions* infractions, HttpEventGen* events, uint32_t flow_target, bool stretch,
HttpCommon::HXBodyState) override;
class HttpBodyHXCutter : public HttpBodyCutter
{
public:
- HttpBodyHXCutter(int64_t expected_length, bool accelerated_blocking, ScriptFinder* finder,
- HttpEnums::CompressId compression) :
- HttpBodyCutter(accelerated_blocking, finder, compression),
- expected_body_length(expected_length)
- {}
+ HttpBodyHXCutter(int64_t expected_length, bool accelerated_blocking_, ScriptFinder* finder,
+ HttpFlowData* const session_data, HttpCommon::SourceId source_id)
+ : HttpBodyCutter(accelerated_blocking_, finder, session_data, source_id)
+ , expected_body_length(expected_length)
+ { }
HttpEnums::ScanResult cut(const uint8_t* buffer, uint32_t length, HttpInfractions*,
HttpEventGen*, uint32_t flow_target, bool stretch, HttpCommon::HXBodyState state) override;
private:
PEG_CONCURRENT_SESSIONS, PEG_MAX_CONCURRENT_SESSIONS, PEG_SCRIPT_DETECTION,
PEG_PARTIAL_INSPECT, PEG_EXCESS_PARAMS, PEG_PARAMS, PEG_CUTOVERS, PEG_SSL_SEARCH_ABND_EARLY,
PEG_PIPELINED_FLOWS, PEG_PIPELINED_REQUESTS, PEG_TOTAL_BYTES, PEG_JS_INLINE, PEG_JS_EXTERNAL,
- PEG_JS_PDF, PEG_SKIP_MIME_ATTACH, PEG_COMPRESSED_GZIP, PEG_COMPRESSED_NOT_SUPPORTED,
+ PEG_JS_PDF, PEG_SKIP_MIME_ATTACH, PEG_COMPRESSED_GZIP, PEG_COMPRESSED_GZIP_FAILED, PEG_COMPRESSED_DEFLATE,
+ PEG_INCORRECT_DEFLATE_HEADER, PEG_COMPRESSED_DEFLATE_FAILED, PEG_COMPRESSED_NOT_SUPPORTED,
PEG_COMPRESSED_UNKNOWN, PEG_MAX_PUBLISH_DEPTH_HITS, PEG_COUNT_MAX};
// Result of scanning by splitter
INF_PARTIAL_START = 51,
INF_CHUNK_WHITESPACE = 52,
INF_HEAD_NAME_WHITESPACE = 53,
- INF_GZIP_OVERRUN = 54,
+ // INF_GZIP_OVERRUN = 54, // Retired. Do not reuse this number
INF_GZIP_FAILURE = 55,
INF_GZIP_EARLY_END = 56,
INF_URI_NEED_NORM_PATH = 57,
INF_METHOD_ON_DISALLOWED_LIST = 137,
INF_PIPELINE_MAX = 138,
INF_GZIP_RESERVED_FLAGS = 139,
+ INF_DEFLATE_EARLY_END = 140,
+ INF_DEFLATE_FAILURE = 141,
INF__MAX_VALUE
};
EVENT_BROKEN_CHUNK = 213,
EVENT_CHUNK_WHITESPACE = 214,
EVENT_HEAD_NAME_WHITESPACE = 215,
- EVENT_GZIP_OVERRUN = 216,
+ // EVENT_GZIP_OVERRUN = 216, // Retired. Do not reuse this number
EVENT_GZIP_FAILURE = 217,
EVENT_ZERO_NINE_CONTINUE = 218,
EVENT_ZERO_NINE_NOT_FIRST = 219,
EVENT_DISALLOWED_METHOD = 287,
EVENT_GZIP_RESERVED_FLAGS = 288,
EVENT_MAX_PARTIAL_FLUSH = 289,
+ EVENT_DEFLATE_EARLY_END = 290,
+ EVENT_DEFLATE_FAILURE = 291,
EVENT__MAX_VALUE
};
#include "http_cutter.h"
#include "http_common.h"
+#include "http_compress_stream.h"
#include "http_enum.h"
#include "http_js_norm.h"
#include "http_module.h"
delete[] section_buffer[k];
delete[] partial_buffer[k];
delete[] partial_detect_buffer[k];
+ delete compress[k];
delete partial_mime_bufs[k];
HttpTransaction::delete_transaction(transaction[k], nullptr);
delete cutter[k];
- if (compress_stream[k] != nullptr)
- {
- inflateEnd(compress_stream[k]);
- delete compress_stream[k];
- }
delete mime_state[k];
delete utf_state[k];
if (fd_state[k] != nullptr)
detect_depth_remaining[source_id] = STAT_NOT_PRESENT;
publish_depth_remaining[source_id] = STAT_NOT_PRESENT;
- compression[source_id] = CMP_NONE;
- gzip_state[source_id] = GZIP_TBD;
- gzip_header_bytes_processed[source_id] = 0;
- if (compress_stream[source_id] != nullptr)
- {
- inflateEnd(compress_stream[source_id]);
- delete compress_stream[source_id];
- compress_stream[source_id] = nullptr;
- }
+ delete compress[source_id];
+ compress[source_id] = nullptr;
delete mime_state[source_id];
mime_state[source_id] = nullptr;
delete utf_state[source_id];
void HttpFlowData::trailer_prep(SourceId source_id)
{
type_expected[source_id] = SEC_TRAILER;
- compression[source_id] = CMP_NONE;
- if (compress_stream[source_id] != nullptr)
- {
- inflateEnd(compress_stream[source_id]);
- delete compress_stream[source_id];
- compress_stream[source_id] = nullptr;
- }
+ delete compress[source_id];
+ compress[source_id] = nullptr;
delete mime_state[source_id];
mime_state[source_id] = nullptr;
delete utf_state[source_id];
class HttpMsgSection;
class HttpCutter;
class HttpQueryParser;
+class HttpCompressStream;
namespace snort
{
uint32_t partial_raw_bytes[2] = { 0, 0 };
uint8_t* partial_buffer[2] = { nullptr, nullptr };
uint32_t partial_buffer_length[2] = { 0, 0 };
- uint32_t gzip_header_bytes_processed[2] = { 0, 0 };
- HttpEnums::GzipVerificationState gzip_state[2] = { HttpEnums::GZIP_TBD, HttpEnums::GZIP_TBD };
- bool gzip_header_check_done();
// *** StreamSplitter internal data - scan() => reassemble()
uint32_t num_excess[2] = { 0, 0 };
bool partial_flush[2] = { false, false };
uint64_t last_connect_trans_w_early_traffic = 0;
+ HttpCompressStream* compress[2] = { nullptr, nullptr };
HttpInfractions* infractions[2] = { new HttpInfractions, new HttpInfractions };
HttpEventGen* events[2] = { new HttpEventGen, new HttpEventGen };
bool last_request_was_connect = false;
bool stretch_section_to_packet[2] = { false, false };
bool accelerated_blocking[2] = { false, false };
- z_stream* compress_stream[2] = { nullptr, nullptr };
uint64_t zero_nine_expected = 0;
// length of the data from Content-Length field
int64_t data_length[2] = { HttpCommon::STAT_NOT_PRESENT, HttpCommon::STAT_NOT_PRESENT };
uint32_t section_size_target[2] = { 0, 0 };
- HttpEnums::CompressId compression[2] = { HttpEnums::CMP_NONE, HttpEnums::CMP_NONE };
// *** Inspector's internal data about the current message
struct FdCallbackContext
using namespace snort;
using namespace HttpEnums;
+using namespace HttpCommon;
+
+THREAD_LOCAL const Trace* http_trace = nullptr;
+
+#ifdef DEBUG_MSGS
+static const TraceOption http_trace_options[] =
+{
+ { "compress", TRACE_COMPRESS, "enable compress trace logging" },
+ { nullptr, 0, nullptr }
+};
+#endif
HttpModule::HttpModule() : Module(HTTP_NAME, HTTP_HELP, http_params),
script_detection_handle(LiteralSearch::setup())
delete mpse_attr;
}
+void HttpModule::set_trace(const snort::Trace* trace) const
+{ http_trace = trace; }
+
+#ifdef DEBUG_MSGS
+const snort::TraceOption* HttpModule::get_trace_options() const
+{ return http_trace_options; }
+#endif
+
void HttpParaList::JsNormParam::configure() const
{
mpse_otag = js_create_mpse_open_tag();
#include "mime/file_mime_config.h"
#include "profiler/profiler.h"
#include "search_engines/search_tool.h"
+#include "trace/trace_api.h"
#include "http_enum.h"
#include "http_str_to_code.h"
#define HTTP_NAME "http_inspect"
#define HTTP_HELP "HTTP inspector"
+extern THREAD_LOCAL const snort::Trace* http_trace;
+
namespace snort
{
class Trace;
static snort::ProfileStats& get_profile_stats()
{ return http_profile; }
+ void set_trace(const snort::Trace*) const override;
+
+#ifdef DEBUG_MSGS
+ const snort::TraceOption* get_trace_options() const override;
+#endif
+
Usage get_usage() const override
{ return INSPECT; }
decode_infs += INF_STACKED_ENCODINGS;
decode_infs += INF_CONTENT_ENCODING_CHUNKED;
decode_infs += INF_GZIP_FAILURE;
- decode_infs += INF_GZIP_OVERRUN;
}
static int _init_decode_infs __attribute__((unused)) = (static_cast<void>(init_decode_infs()), 0);
#include "http_api.h"
#include "http_common.h"
+#include "http_compress_stream.h"
#include "http_enum.h"
#include "http_inspect.h"
#include "http_js_norm.h"
if (!params->unzip)
return;
- CompressId& compression = session_data->compression[source_id];
+ CompressId compression = CMP_NONE;
// Search the Content-Encoding header to find the type of compression used. We detect and alert
// on multiple layers of compression but we only decompress the outermost layer. Thus the last
compression = CMP_GZIP;
break;
case CONTENTCODE_DEFLATE:
+ HttpModule::increment_peg_counts(PEG_COMPRESSED_DEFLATE);
compression = CMP_DEFLATE;
break;
case CONTENTCODE_IDENTITY:
}
}
- if (compression == CMP_NONE)
+ if ( compression == CMP_NONE )
return;
- session_data->compress_stream[source_id] = new z_stream;
- session_data->compress_stream[source_id]->zalloc = Z_NULL;
- session_data->compress_stream[source_id]->zfree = Z_NULL;
- session_data->compress_stream[source_id]->next_in = Z_NULL;
- session_data->compress_stream[source_id]->avail_in = 0;
- const int window_bits = (compression == CMP_GZIP) ? GZIP_WINDOW_BITS : DEFLATE_WINDOW_BITS;
- if (inflateInit2(session_data->compress_stream[source_id], window_bits) != Z_OK)
- {
- assert(false);
- session_data->compression[source_id] = CMP_NONE;
- delete session_data->compress_stream[source_id];
- session_data->compress_stream[source_id] = nullptr;
- }
+ session_data->compress[source_id] = new HttpCompressStream;
+
+ if ( session_data->compress[source_id]->setup(compression) )
+ return;
+
+ assert(false);
+ delete session_data->compress[source_id];
+ session_data->compress[source_id] = nullptr;
}
void HttpMsgHeader::setup_utf_decoding()
#include "http_context_data.h"
#include "http_common.h"
+#include "http_compress_stream.h"
#include "http_enum.h"
#include "http_module.h"
#include "http_msg_body.h"
const int64_t& detect_depth_remaining = session_data->detect_depth_remaining[source_id];
const int32_t& publish_depth_remaining = session_data->publish_depth_remaining[source_id];
- const unsigned target_size = (session_data->compression[source_id] == CMP_NONE) ?
- SnortConfig::get_conf()->max_pdu : GZIP_BLOCK_SIZE;
+ unsigned target_size = SnortConfig::get_conf()->max_pdu;
+
+ if ( session_data->compress[source_id] != nullptr and
+ session_data->compress[source_id]->get_compression_id() != CMP_NONE )
+ target_size = GZIP_BLOCK_SIZE;
if (detect_depth_remaining <= 0)
{
HttpCutter* get_cutter(HttpCommon::SectionType type, HttpFlowData* session) const;
void chunk_spray(HttpFlowData* session_data, uint8_t* buffer, const uint8_t* data,
unsigned length) const;
- void decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data,
- uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream,
- bool at_start, HttpInfractions* infractions, HttpEventGen* events,
- HttpFlowData* session_data) const;
- uint8_t* process_gzip_header(const uint8_t* data,
- uint32_t length, HttpFlowData* session_data) const;
- bool gzip_header_check_done(HttpFlowData* session_data) const;
+ void decompress_copy(const uint8_t* src, uint32_t src_size,
+ uint8_t* dst, uint32_t& dst_size, HttpFlowData* const session_data) const;
StreamSplitter::Status handle_zero_nine(snort::Flow*, HttpFlowData*, const uint8_t* data,
uint32_t length, uint32_t* flush_offset, HttpCommon::SectionType&, HttpCutter*&);
StreamSplitter::Status call_cutter(snort::Flow*, HttpFlowData*, const uint8_t* data,
#include "protocols/packet.h"
+#include "http_compress_stream.h"
#include "http_inspect.h"
#include "http_module.h"
#include "http_test_input.h"
case CHUNK_DATA:
{
const uint32_t skip_amount = (length-k <= expected) ? length-k : expected;
- const bool at_start = (session_data->body_octets[source_id] == 0) &&
- (session_data->section_offset[source_id] == 0);
- decompress_copy(buffer, session_data->section_offset[source_id], data+k, skip_amount,
- session_data->compression[source_id], session_data->compress_stream[source_id],
- at_start, session_data->get_infractions(source_id),
- session_data->events[source_id], session_data);
+ decompress_copy(data+k, skip_amount, buffer,
+ session_data->section_offset[source_id], session_data);
if ((expected -= skip_amount) == 0)
curr_state = CHUNK_DCRLF1;
k += skip_amount-1;
case CHUNK_BAD:
{
const uint32_t skip_amount = length-k;
- const bool at_start = (session_data->body_octets[source_id] == 0) &&
- (session_data->section_offset[source_id] == 0);
- decompress_copy(buffer, session_data->section_offset[source_id], data+k, skip_amount,
- session_data->compression[source_id], session_data->compress_stream[source_id],
- at_start, session_data->get_infractions(source_id),
- session_data->events[source_id], session_data);
+ decompress_copy(data+k, skip_amount, buffer,
+ session_data->section_offset[source_id], session_data);
k += skip_amount-1;
break;
}
}
}
-uint8_t* HttpStreamSplitter::process_gzip_header(const uint8_t* data,
- uint32_t length, HttpFlowData* session_data) const
+void HttpStreamSplitter::decompress_copy(const uint8_t* src, uint32_t src_size,
+ uint8_t* dst, uint32_t& dst_size, HttpFlowData* const session_data) const
{
- uint32_t& header_bytes_processed = session_data->gzip_header_bytes_processed[source_id];
- uint32_t input_bytes_processed = 0;
- uint8_t* modified_data = nullptr;
- if (session_data->gzip_state[source_id] == GZIP_TBD)
- {
- static const uint8_t gzip_magic[] = {0x1f, 0x8b, 0x08};
- static const uint8_t magic_length = 3;
- const uint32_t magic_cmp_len = (magic_length - header_bytes_processed) < length ?
- (magic_length - header_bytes_processed) : length;
-
- if (memcmp(data, gzip_magic + header_bytes_processed, magic_cmp_len))
- session_data->gzip_state[source_id] = GZIP_MAGIC_BAD;
- else if (header_bytes_processed + length >= magic_length)
- session_data->gzip_state[source_id] = GZIP_MAGIC_GOOD;
- header_bytes_processed += magic_cmp_len;
- input_bytes_processed += magic_cmp_len;
- }
- if (session_data->gzip_state[source_id] == GZIP_MAGIC_GOOD and length > input_bytes_processed)
- {
- const uint8_t gzip_flags = data[input_bytes_processed];
- if (gzip_flags & GZIP_FLAG_FEXTRA)
- {
- *session_data->get_infractions(source_id) += INF_GZIP_FEXTRA;
- session_data->events[source_id]->create_event(EVENT_GZIP_FEXTRA);
- }
- if (gzip_flags & GZIP_RESERVED_FLAGS)
- {
- *session_data->get_infractions(source_id) += INF_GZIP_RESERVED_FLAGS;
- session_data->events[source_id]->create_event(EVENT_GZIP_RESERVED_FLAGS);
- modified_data = new uint8_t[length];
- memcpy(modified_data, data, length);
- modified_data[input_bytes_processed] &= ~GZIP_RESERVED_FLAGS;
- }
- header_bytes_processed++;
- session_data->gzip_state[source_id] = GZIP_FLAGS_PROCESSED;
- }
- return modified_data;
-}
-
-bool HttpStreamSplitter::gzip_header_check_done(HttpFlowData* session_data) const
-{
- return session_data->gzip_state[source_id] == HttpEnums::GZIP_MAGIC_BAD or
- session_data->gzip_state[source_id] == HttpEnums::GZIP_FLAGS_PROCESSED;
-}
-
-void HttpStreamSplitter::decompress_copy(uint8_t* buffer, uint32_t& offset, const uint8_t* data,
- uint32_t length, HttpEnums::CompressId& compression, z_stream*& compress_stream,
- bool at_start, HttpInfractions* infractions, HttpEventGen* events, HttpFlowData* session_data) const
-{
- if ((compression == CMP_GZIP) || (compression == CMP_DEFLATE))
- {
- uint8_t* data_w_updated_hdr = nullptr;
- if (compression == CMP_GZIP and !gzip_header_check_done(session_data))
- data_w_updated_hdr = process_gzip_header(data, length, session_data);
-
- if (data_w_updated_hdr != nullptr)
- compress_stream->next_in = const_cast<Bytef*>(data_w_updated_hdr);
- else
- compress_stream->next_in = const_cast<Bytef*>(data);
- compress_stream->avail_in = length;
- compress_stream->next_out = buffer + offset;
- compress_stream->avail_out = MAX_OCTETS - offset;
- int ret_val = inflate(compress_stream, Z_SYNC_FLUSH);
-
- delete[] data_w_updated_hdr;
-
- if ((ret_val == Z_OK) || (ret_val == Z_STREAM_END))
- {
- offset = MAX_OCTETS - compress_stream->avail_out;
- if (compress_stream->avail_in > 0)
- {
- // There are two ways not to consume all the input
- if (ret_val == Z_STREAM_END)
- {
- // The zipped data stream ended but there is more input data
- *infractions += INF_GZIP_EARLY_END;
- events->create_event(EVENT_GZIP_EARLY_END);
- const uInt num_copy =
- (compress_stream->avail_in <= compress_stream->avail_out) ?
- compress_stream->avail_in : compress_stream->avail_out;
- memcpy(buffer + offset, data + (length - compress_stream->avail_in), num_copy);
- offset += num_copy;
- }
- else
- {
- assert(compress_stream->avail_out == 0);
- // The data expanded too much
- *infractions += INF_GZIP_OVERRUN;
- events->create_event(EVENT_GZIP_OVERRUN);
- }
- compression = CMP_NONE;
- inflateEnd(compress_stream);
- delete compress_stream;
- compress_stream = nullptr;
- // FIXIT-E - Will need to clear gzip header processing state here when we implement
- // processing multiple gzip members in a message section
- }
- return;
- }
- else if ((compression == CMP_DEFLATE) && at_start && (ret_val == Z_DATA_ERROR))
- {
- // Some incorrect implementations of deflate don't use the expected header. Feed a
- // dummy header to zlib and retry the inflate.
- static constexpr uint8_t zlib_header[2] = { 0x78, 0x01 };
-
- inflateReset(compress_stream);
- compress_stream->next_in = const_cast<Bytef*>(zlib_header);
- compress_stream->avail_in = sizeof(zlib_header);
- int ret = inflate(compress_stream, Z_SYNC_FLUSH);
- if ( ret == Z_OK or ret == Z_STREAM_END)
- {
- // Start over at the beginning
- decompress_copy(buffer, offset, data, length, compression, compress_stream, false,
- infractions, events, session_data);
- }
- return;
- }
- else
- {
- *infractions += INF_GZIP_FAILURE;
- events->create_event(EVENT_GZIP_FAILURE);
- compression = CMP_NONE;
- inflateEnd(compress_stream);
- delete compress_stream;
- compress_stream = nullptr;
- // Since we failed to uncompress the data, fall through
- }
- }
+ if ( session_data->compress[source_id] != nullptr and
+ is_body(session_data->section_type[source_id]) )
+ return;
- // The following precaution is necessary because mixed compressed and uncompressed data can
- // cause the buffer to overrun even though we are not decompressing right now
- if (length > MAX_OCTETS - offset)
- {
- length = MAX_OCTETS - offset;
- *infractions += INF_GZIP_OVERRUN;
- events->create_event(EVENT_GZIP_OVERRUN);
- }
- memcpy(buffer + offset, data, length);
- offset += length;
+ HttpCompressStream::copy_raw(src, src_size, dst, dst_size);
}
const StreamBuffer HttpStreamSplitter::reassemble(Flow* flow, unsigned total,
HttpModule::increment_peg_counts(PEG_REASSEMBLE);
uint8_t*& buffer = session_data->section_buffer[source_id];
- if (buffer == nullptr)
+ if ( buffer == nullptr )
{
// Body sections need extra space to accommodate unzipping
- if (is_body(session_data->section_type[source_id]))
- buffer = new uint8_t[MAX_OCTETS];
+ if ( is_body(session_data->section_type[source_id]) )
+ if ( session_data->compress[source_id] != nullptr and partial_buffer_length > 0 )
+ {
+ assert(partial_buffer != nullptr);
+
+ buffer = partial_buffer;
+ session_data->section_offset[source_id] = partial_buffer_length;
+
+ partial_buffer = nullptr;
+ partial_buffer_length = 0;
+ }
+ else
+ buffer = new uint8_t[MAX_OCTETS];
else
{
uint32_t buffer_size = (total > 0) ? total : 1;
}
if (session_data->section_type[source_id] != SEC_BODY_CHUNK)
- {
- const bool at_start = (session_data->body_octets[source_id] == 0) &&
- (session_data->section_offset[source_id] == 0);
- decompress_copy(buffer, session_data->section_offset[source_id], data, len,
- session_data->compression[source_id], session_data->compress_stream[source_id],
- at_start, session_data->get_infractions(source_id),
- session_data->events[source_id], session_data);
- }
+ decompress_copy(data, len, buffer, session_data->section_offset[source_id], session_data);
else
- {
chunk_spray(session_data, buffer, data, len);
- }
StreamBuffer http_buf { nullptr, 0 };
if (session_data->section_offset[source_id] > 0)
{
// Store the data from a partial flush for reuse
- partial_buffer = new uint8_t[session_data->section_offset[source_id]];
+ if ( is_body(session_data->section_type[source_id]) )
+ partial_buffer = new uint8_t[MAX_OCTETS];
+ else
+ partial_buffer = new uint8_t[session_data->section_offset[source_id]];
+
memcpy(partial_buffer, buffer, session_data->section_offset[source_id]);
partial_buffer_length = session_data->section_offset[source_id];
}
#include "protocols/packet.h"
#include "http_common.h"
+#include "http_compress_stream.h"
#include "http_context_data.h"
#include "http_cutter.h"
#include "http_enum.h"
session_data->num_good_chunks[source_id] = num_good_chunks;
session_data->octets_expected[source_id] = octets_seen + num_flushed;
+ if ( session_data->compress[source_id] != nullptr and section_type == SEC_DISCARD )
+ {
+ delete[] session_data->partial_buffer[source_id];
+
+ session_data->partial_buffer[source_id] = nullptr;
+ session_data->partial_buffer_length[source_id] = 0;
+ }
+
if (flush_offset != nullptr)
{
#ifdef REG_TEST
switch (type)
{
case SEC_REQUEST:
- return (HttpCutter*)new HttpRequestCutter;
+ return new HttpRequestCutter;
case SEC_STATUS:
if (session_data->expected_trans_num[SRC_SERVER] == session_data->zero_nine_expected)
- return (HttpCutter*)new HttpZeroNineCutter;
+ return new HttpZeroNineCutter;
- return (HttpCutter*)new HttpStatusCutter;
+ return new HttpStatusCutter;
case SEC_HEADER:
case SEC_TRAILER:
- return (HttpCutter*)new HttpHeaderCutter;
+ return new HttpHeaderCutter;
case SEC_BODY_CL:
- return (HttpCutter*)new HttpBodyClCutter(
- session_data->data_length[source_id],
+ return new HttpBodyClCutter(session_data->data_length[source_id],
session_data->accelerated_blocking[source_id],
my_inspector->script_finder,
- session_data->compression[source_id]);
+ session_data, source_id);
case SEC_BODY_CHUNK:
- return (HttpCutter*)new HttpBodyChunkCutter(
- my_inspector->params->maximum_chunk_length,
+ return new HttpBodyChunkCutter(my_inspector->params->maximum_chunk_length,
session_data->accelerated_blocking[source_id],
my_inspector->script_finder,
- session_data->compression[source_id]);
+ session_data, source_id);
case SEC_BODY_OLD:
- return (HttpCutter*)new HttpBodyOldCutter(
- session_data->accelerated_blocking[source_id],
- my_inspector->script_finder,
- session_data->compression[source_id]);
+ return new HttpBodyOldCutter(session_data->accelerated_blocking[source_id],
+ my_inspector->script_finder, session_data, source_id);
case SEC_BODY_HX:
- return (HttpCutter*)new HttpBodyHXCutter(
- session_data->data_length[source_id],
+ return new HttpBodyHXCutter(session_data->data_length[source_id],
session_data->accelerated_blocking[source_id],
- my_inspector->script_finder,
- session_data->compression[source_id]);
+ my_inspector->script_finder, session_data, source_id);
default:
assert(false);
return nullptr;
{ EVENT_BROKEN_CHUNK, "HTTP chunk misformatted" },
{ EVENT_CHUNK_WHITESPACE, "white space adjacent to chunk length" },
{ EVENT_HEAD_NAME_WHITESPACE, "white space within header name" },
- { EVENT_GZIP_OVERRUN, "excessive gzip compression" },
{ EVENT_GZIP_FAILURE, "gzip decompression failed" },
{ EVENT_ZERO_NINE_CONTINUE, "HTTP 0.9 requested followed by another request" },
{ EVENT_ZERO_NINE_NOT_FIRST, "HTTP 0.9 request following a normal request" },
"disallowed methods list" },
{ EVENT_GZIP_RESERVED_FLAGS, "HTTP gzip body with reserved flag set" },
{ EVENT_MAX_PARTIAL_FLUSH, "Too many partial flushes" },
+ { EVENT_DEFLATE_EARLY_END, "deflate compressed data followed by unexpected non-deflate data" },
+ { EVENT_DEFLATE_FAILURE, "deflate decompression failed" },
{ 0, nullptr }
};
{ CountType::SUM, "js_pdf_scripts", "total number of PDF files processed" },
{ CountType::SUM, "skip_mime_attach", "total number of HTTP requests with too many MIME attachments to inspect" },
{ CountType::SUM, "compressed_gzip", "total number of HTTP bodies compressed with GZIP" },
+ { CountType::SUM, "compressed_gzip_failed", "total number of HTTP bodies with failed GZIP decompression" },
+ { CountType::SUM, "compressed_deflate", "total number of HTTP bodies compressed with Deflate" },
+ { CountType::SUM, "incorrect_deflate_header", "total number of HTTP bodies compressed with Deflate that had incorrect header" },
+ { CountType::SUM, "compressed_deflate_failed", "total number of HTTP bodies with failed Deflate decompression" },
{ CountType::SUM, "compressed_not_supported", "total number of HTTP bodies compressed with known but not supported methods" },
{ CountType::SUM, "compressed_unknown", "total number of HTTP bodies compressed with unknown methods" },
{ CountType::SUM, "max_publish_depth_hits", "total number of times the maximum publish depth was exceeded" },
+add_cpputest( http_decompression_test
+ SOURCES
+ ../http_compress_stream.cc
+ ../http_cutter.cc
+ ../http_flow_data.cc
+ ../http_tables.cc
+ ../http_test_input.cc
+ ../http_test_manager.cc
+ LIBS ${ZLIB_LIBRARIES}
+)
+
add_cpputest( http_module_test
SOURCES
../http_module.cc
../http_flow_data.cc
../http_test_manager.cc
../http_test_input.cc
+ ../http_compress_stream.cc
LIBS ${ZLIB_LIBRARIES}
)
--- /dev/null
+//--------------------------------------------------------------------------
+// Copyright (C) 2026-2026 Cisco and/or its affiliates. All rights reserved.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License Version 2 as published
+// by the Free Software Foundation. You may not use, modify or distribute
+// this program under any other version of the GNU General Public License.
+//
+// This program is distributed in the hope that it will be useful, but
+// WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program; if not, write to the Free Software Foundation, Inc.,
+// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+//--------------------------------------------------------------------------
+// http_decompression_test.cc author Oleksandr Fedorych <ofedoryc@cisco.com>
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <cstring>
+
+#include "pub_sub/http_transaction_end_event.h"
+#include "pub_sub/http_form_data_event.h"
+#include "service_inspectors/http_inspect/http_common.h"
+#include "service_inspectors/http_inspect/http_compress_stream.h"
+#include "service_inspectors/http_inspect/http_cutter.h"
+#include "service_inspectors/http_inspect/http_enum.h"
+#include "service_inspectors/http_inspect/http_event.h"
+#include "service_inspectors/http_inspect/http_flow_data.h"
+#include "service_inspectors/http_inspect/http_inspect.h"
+#include "service_inspectors/http_inspect/http_module.h"
+#include "service_inspectors/http_inspect/http_msg_section.h"
+#include "service_inspectors/http_inspect/http_transaction.h"
+#include "service_inspectors/http2_inspect/http2_flow_data.h"
+
+#include "http_unit_test_helpers.h"
+
+#include <CppUTest/CommandLineTestRunner.h>
+#include <CppUTest/TestHarness.h>
+#include <CppUTestExt/MockSupport.h>
+
+using namespace snort;
+using namespace HttpCommon;
+using namespace HttpEnums;
+
+static const uint8_t deflate_compressed[] = {
+ 0x78, 0x9c, 0x63, 0x60, 0xc0, 0x07, 0x00, 0x00, 0x1e, 0x00, 0x01
+};
+static constexpr uint32_t deflate_compressed_size = sizeof(deflate_compressed);
+
+static const uint8_t gzip_compressed[] = {
+ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff,
+ 0x63, 0x60, 0xc0, 0x07, 0x00, 0xb5, 0x3e, 0x04, 0x00, 0x1e,
+ 0x00, 0x00, 0x00
+};
+static constexpr uint32_t gzip_compressed_size = sizeof(gzip_compressed);
+
+static constexpr uint32_t scan_buf_size = 2000;
+
+namespace snort
+{
+unsigned FlowData::flow_data_id = 0;
+FlowData::FlowData(unsigned) : id(0) { }
+FlowData::~FlowData() = default;
+FlowDataStore::~FlowDataStore() = default;
+int DetectionEngine::queue_event(unsigned int, unsigned int) { return 0; }
+fd_status_t File_Decomp_StopFree(fd_session_t*) { return File_Decomp_OK; }
+uint32_t str_to_hash(const uint8_t*, size_t) { return 0; }
+FlowData* FlowDataStore::get(unsigned) const { return nullptr; }
+void FlowDataStore::set(FlowData*) { }
+Flow::~Flow() = default;
+unsigned DataBus::get_id(PubKey const&) { return 0; }
+void DataBus::publish(unsigned int, unsigned int, DataEvent&, Flow*) { }
+HttpTransactionEndEvent::HttpTransactionEndEvent(const HttpTransaction* const trans)
+ : transaction(trans) { }
+Inspector::Inspector() : ref_count(nullptr) { }
+Inspector::~Inspector() = default;
+bool Inspector::likes(Packet*) { return true; }
+bool Inspector::get_buf(const char*, Packet*, InspectionBuffer&) { return false; }
+class StreamSplitter* Inspector::get_splitter(bool) { return nullptr; }
+const StreamBuffer StreamSplitter::reassemble(snort::Flow*, unsigned int, unsigned int,
+ unsigned char const*, unsigned int, unsigned int, unsigned int&)
+{
+ StreamBuffer buf { nullptr, 0 };
+ return buf;
+}
+
+unsigned StreamSplitter::max(snort::Flow*) { return 0; }
+uint8_t TraceApi::get_constraints_generation() { return 0; }
+void TraceApi::filter(const Packet&) { }
+void trace_vprintf(const char*, TraceLevel, const char*, const snort::Packet*, const char*,
+ va_list) { }
+}
+
+THREAD_LOCAL const snort::Trace* http_trace = nullptr;
+
+HttpParaList::UriParam::UriParam() : uri_char{} { }
+HttpParaList::JsNormParam::~JsNormParam() { }
+HttpParaList::~HttpParaList() { }
+
+void HttpFormDataEvent::format_as_uri() const { }
+
+unsigned Http2FlowData::inspector_id = 0;
+uint32_t Http2FlowData::get_processing_stream_id() const { return 0; }
+HttpInspect::HttpInspect(const HttpParaList* para)
+ : params(para), xtra_trueip_id(0), xtra_uri_id(0), xtra_host_id(0), xtra_jsnorm_id(0) { }
+HttpInspect::~HttpInspect() = default;
+bool HttpInspect::configure(SnortConfig*) { return true; }
+void HttpInspect::show(const SnortConfig*) const { }
+bool HttpInspect::get_buf(unsigned, snort::Packet*, snort::InspectionBuffer&) { return true; }
+HttpCommon::SectionType HttpInspect::get_type_expected(snort::Flow*, HttpCommon::SourceId) const
+{ return SEC_DISCARD; }
+void HttpInspect::finish_hx_body(snort::Flow*, HttpCommon::SourceId, HttpCommon::HXBodyState,
+ bool) const { }
+void HttpInspect::set_hx_body_state(snort::Flow*, HttpCommon::SourceId,
+ HttpCommon::HXBodyState) const { }
+bool HttpInspect::get_fp_buf(snort::InspectionBuffer::Type, snort::Packet*,
+ snort::InspectionBuffer&) { return false; }
+void HttpInspect::eval(snort::Packet*) { }
+void HttpInspect::eval(snort::Packet*, HttpCommon::SourceId, const uint8_t*, uint16_t) { }
+void HttpInspect::clear(snort::Packet*) { }
+bool HttpInspect::get_buf(snort::InspectionBuffer::Type, snort::Packet*,
+ snort::InspectionBuffer&) { return false; }
+const uint8_t* HttpInspect::adjust_log_packet(snort::Packet*, uint16_t&) { return nullptr; }
+StreamSplitter::Status HttpStreamSplitter::scan(snort::Packet*, const uint8_t*, uint32_t,
+ uint32_t, uint32_t*) { return StreamSplitter::FLUSH; }
+StreamSplitter::Status HttpStreamSplitter::scan(snort::Flow*, const uint8_t*, uint32_t,
+ uint32_t*, snort::Packet*) { return StreamSplitter::FLUSH; }
+const snort::StreamBuffer HttpStreamSplitter::reassemble(snort::Flow*, unsigned, unsigned,
+ const uint8_t*, unsigned, uint32_t, unsigned&)
+{
+ StreamBuffer buf { nullptr, 0 };
+ return buf;
+}
+
+bool HttpStreamSplitter::finish(snort::Flow*) { return false; }
+void HttpStreamSplitter::prep_partial_flush(snort::Flow*, uint32_t, uint32_t, uint32_t) { }
+void HttpMsgSection::clear_tmp_buffers() { }
+
+HttpTransaction::~HttpTransaction() = default;
+void HttpTransaction::delete_transaction(HttpTransaction*, HttpFlowData*) { }
+HttpInfractions* HttpTransaction::get_infractions(HttpCommon::SourceId) { return nullptr; }
+
+THREAD_LOCAL PegCount HttpModule::peg_counts[PEG_COUNT_MAX] = { };
+const Field Field::FIELD_NULL { STAT_NO_SOURCE };
+
+struct HttpDecompressionFixture : public Utest
+{
+ Flow* const flow = new Flow;
+ HttpParaList params;
+ // cppcheck-suppress constVariablePointer
+ HttpFlowData* session_data = nullptr;
+
+ void setup() override
+ {
+ flow->gadget = new HttpInspect(¶ms);
+ // cppcheck-suppress unreadVariable
+ session_data = new HttpFlowData(flow, ¶ms);
+ }
+
+ void teardown() override
+ {
+ delete session_data;
+ session_data = nullptr;
+ delete flow->gadget;
+ flow->gadget = nullptr;
+ delete flow;
+ }
+};
+
+TEST_GROUP_BASE(cl_cutter_decompression_test, HttpDecompressionFixture)
+{
+};
+
+TEST(cl_cutter_decompression_test, long_body_deflate_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_DEFLATE);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_len = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS];
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_len);
+
+ const uint32_t flow_target = deflate_compressed_size;
+ const uint32_t remaining = scan_buf_size;
+
+ uint8_t scan_buf[scan_buf_size];
+ memcpy(scan_buf, deflate_compressed, deflate_compressed_size);
+
+ HttpBodyClCutter cutter(static_cast<int64_t>(remaining), false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ const auto result = cutter.cut(scan_buf, scan_buf_size, infractions, &events,
+ flow_target, true, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(0, cutter.get_octets_seen());
+ CHECK_EQUAL(7, cutter.get_num_flush());
+ // cppcheck-suppress syntaxError
+ CHECK_COMPARE(flow_target, >, cutter.get_num_flush());
+}
+
+TEST(cl_cutter_decompression_test, long_body_gzip_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_GZIP);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_size = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS];
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_size);
+
+ const uint32_t flow_target = gzip_compressed_size;
+ const uint32_t remaining = scan_buf_size;
+
+ uint8_t scan_buf[scan_buf_size];
+ memcpy(scan_buf, gzip_compressed, gzip_compressed_size);
+
+ HttpBodyClCutter cutter(static_cast<int64_t>(remaining), false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ const auto result = cutter.cut(scan_buf, scan_buf_size, infractions, &events,
+ flow_target, true, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(0, cutter.get_octets_seen());
+ CHECK_EQUAL(15, cutter.get_num_flush());
+ CHECK_COMPARE(flow_target, >, cutter.get_num_flush());
+}
+
+TEST(cl_cutter_decompression_test, octets_seen_long_section_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_DEFLATE);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_len = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS];
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_len);
+
+ const uint32_t flow_target = 10;
+
+ uint8_t scan_buf[scan_buf_size];
+ memcpy(scan_buf, deflate_compressed, deflate_compressed_size);
+
+ HttpBodyClCutter cutter(10, false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ auto result = cutter.cut(scan_buf, 1, infractions, &events, flow_target, true, HX_BODY_NOT_COMPLETE);
+ CHECK_EQUAL(SCAN_NOT_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(0, cutter.get_num_flush());
+ CHECK_COMPARE(flow_target, >, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf + 1, 7, infractions, &events, flow_target, true, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(6, cutter.get_num_flush());
+ CHECK_COMPARE(flow_target, >, cutter.get_num_flush());
+
+ // remaining after the two cuts above is 2 bytes, so when we send the next 3 bytes, only 2 are
+ // flushed.
+ result = cutter.cut(scan_buf + 8, 3, infractions, &events, flow_target, false, HX_BODY_NOT_COMPLETE);
+ CHECK_EQUAL(SCAN_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(2, cutter.get_num_flush());
+ CHECK_COMPARE(flow_target, >, cutter.get_num_flush());
+}
+
+TEST(cl_cutter_decompression_test, octets_seen_no_stretch_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_GZIP);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_size = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS];
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_size);
+
+ const uint32_t flow_target = gzip_compressed_size;
+ const uint32_t remaining = scan_buf_size;
+
+ uint8_t scan_buf[scan_buf_size];
+ memcpy(scan_buf, gzip_compressed, gzip_compressed_size);
+
+ HttpBodyClCutter cutter(static_cast<int64_t>(remaining), false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ auto result = cutter.cut(scan_buf, 1, infractions, &events, flow_target, true, HX_BODY_NOT_COMPLETE);
+ CHECK_EQUAL(SCAN_NOT_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(0, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf + 1, scan_buf_size - 1, infractions, &events,
+ flow_target, true, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(14, cutter.get_num_flush());
+ CHECK_COMPARE(flow_target, >, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf, scan_buf_size, infractions, &events, 2000, false, HX_BODY_NOT_COMPLETE);
+ // If the cutter computed remaining correctly in the previous cut() call, it will return 1984
+ // bytes to be flushed.
+ // Otherwise, it will return 1985 bytes because we haven't accounted for octets_seen in the
+ // previous cut() call where remaining was computed.
+ CHECK_EQUAL(SCAN_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(1984, cutter.get_num_flush());
+ CHECK_COMPARE(2000, >, cutter.get_num_flush());
+}
+
+TEST(cl_cutter_decompression_test, octets_seen_mid_segment_stretch_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_DEFLATE);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_len = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS];
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_len);
+
+ const uint32_t flow_target = 240;
+
+ uint8_t scan_buf[scan_buf_size] = { };
+ memcpy(scan_buf, deflate_compressed, deflate_compressed_size);
+
+ HttpBodyClCutter cutter(1800, false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ auto result = cutter.cut(scan_buf, 1, infractions, &events, flow_target, true, HX_BODY_NOT_COMPLETE);
+ CHECK_EQUAL(SCAN_NOT_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(0, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf + 1, 1700, infractions, &events, flow_target, true, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(6, cutter.get_num_flush());
+ CHECK_COMPARE(flow_target - cutter.get_octets_seen(), >, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf, scan_buf_size, infractions, &events, 2000, false, HX_BODY_NOT_COMPLETE);
+ // If the cutter computed remaining correctly in the previous cut() call, it will return 1792
+ // bytes to be flushed.
+ // Otherwise, it will return 1793 bytes because we haven't accounted for octets_seen in the
+ // previous cut() call where remaining was computed.
+ CHECK_EQUAL(SCAN_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(1792, cutter.get_num_flush());
+ CHECK_COMPARE(2000, >, cutter.get_num_flush());
+}
+
+TEST(cl_cutter_decompression_test, octets_seen_last_segment_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_DEFLATE);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_len = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS];
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_len);
+
+ const uint32_t flow_target = 240;
+
+ uint8_t scan_buf[scan_buf_size] = { };
+ memcpy(scan_buf, deflate_compressed, deflate_compressed_size);
+
+ HttpBodyClCutter cutter(10, false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ auto result = cutter.cut(scan_buf, 1, infractions, &events, flow_target, true, HX_BODY_NOT_COMPLETE);
+ CHECK_EQUAL(SCAN_NOT_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(0, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf + 1, 9, infractions, &events, flow_target, false, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(6, cutter.get_num_flush());
+ CHECK_COMPARE((flow_target - cutter.get_octets_seen()), >, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf + 10, 10, infractions, &events, flow_target, false, HX_BODY_NOT_COMPLETE);
+ // Here we verify that remaining is computed correctly when stretching is not allowed.
+ // If remaining is computed correctly, we will receive SCAN_FOUND. Otherwise, SCAN_NOT_FOUND.
+ CHECK_EQUAL(SCAN_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(2, cutter.get_num_flush());
+ CHECK_COMPARE((flow_target - cutter.get_octets_seen()), >, cutter.get_num_flush());
+}
+
+TEST_GROUP_BASE(old_cutter_decompression_test, HttpDecompressionFixture)
+{
+};
+
+TEST(old_cutter_decompression_test, no_stretch_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_DEFLATE);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_len = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS];
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_len);
+
+ const uint32_t flow_target = 8;
+
+ uint8_t scan_buf[scan_buf_size];
+ memcpy(scan_buf, deflate_compressed, deflate_compressed_size);
+
+ HttpBodyOldCutter cutter(false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ auto result = cutter.cut(scan_buf, 1, infractions, &events, flow_target, true, HX_BODY_NOT_COMPLETE);
+ CHECK_EQUAL(SCAN_NOT_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(0, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf + 1, 7, infractions, &events, flow_target, false, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(6, cutter.get_num_flush());
+}
+
+TEST(old_cutter_decompression_test, large_body_deflate_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_DEFLATE);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_size = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS]();
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_size);
+
+ // flow_target = compressed size; send a larger buffer so length >= flow_target
+ const uint32_t flow_target = deflate_compressed_size;
+
+ uint8_t scan_buf[scan_buf_size] = { };
+ memcpy(scan_buf, deflate_compressed, deflate_compressed_size);
+
+ HttpBodyOldCutter cutter(false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ const auto result = cutter.cut(scan_buf, scan_buf_size, infractions, &events,
+ flow_target, false, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(0, cutter.get_octets_seen());
+ CHECK_EQUAL(7, cutter.get_num_flush());
+ CHECK_COMPARE(flow_target, >, cutter.get_num_flush());
+}
+
+TEST(old_cutter_decompression_test, large_body_gzip_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_GZIP);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_size = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS]();
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_size);
+
+ const uint32_t flow_target = gzip_compressed_size;
+
+ uint8_t scan_buf[scan_buf_size] = { };
+ memcpy(scan_buf, gzip_compressed, gzip_compressed_size);
+
+ HttpBodyOldCutter cutter(false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ const auto result = cutter.cut(scan_buf, scan_buf_size, infractions, &events,
+ flow_target, false, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(0, cutter.get_octets_seen());
+ CHECK_EQUAL(15, cutter.get_num_flush());
+ CHECK_COMPARE(flow_target, >, cutter.get_num_flush());
+}
+
+TEST_GROUP_BASE(hx_cutter_decompression_test, HttpDecompressionFixture)
+{
+};
+
+TEST(hx_cutter_decompression_test, octets_seen_section_not_complete_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_DEFLATE);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_len = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS];
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_len);
+
+ const uint32_t flow_target = 8;
+
+ uint8_t scan_buf[scan_buf_size];
+ memcpy(scan_buf, deflate_compressed, deflate_compressed_size);
+
+ HttpBodyHXCutter cutter(20, false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ auto result = cutter.cut(scan_buf, 1, infractions, &events, flow_target, true, HX_BODY_NOT_COMPLETE);
+ CHECK_EQUAL(SCAN_NOT_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(0, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf + 1, 7, infractions, &events, flow_target, false, HX_BODY_NOT_COMPLETE);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(6, cutter.get_num_flush());
+}
+
+TEST(hx_cutter_decompression_test, octets_seen_last_segment_overrun)
+{
+ auto* compress_stream = new HttpCompressStream;
+ compress_stream->setup(CMP_DEFLATE);
+ HttpUnitTestSetup::set_compress_stream(session_data, SRC_SERVER, compress_stream);
+
+ const uint32_t pre_filled_len = MAX_OCTETS - 5;
+ auto* pre_filled_buf = new uint8_t[MAX_OCTETS];
+ HttpUnitTestSetup::set_partial_buffer(session_data, SRC_SERVER, pre_filled_buf, pre_filled_len);
+
+ const uint32_t flow_target = 8;
+
+ uint8_t scan_buf[scan_buf_size];
+ memcpy(scan_buf, deflate_compressed, deflate_compressed_size);
+
+ HttpBodyHXCutter cutter(20, false, nullptr, session_data, SRC_SERVER);
+
+ auto* infractions = HttpUnitTestSetup::get_infractions(session_data, SRC_SERVER);
+ HttpEventGen events;
+
+ auto result = cutter.cut(scan_buf, 1, infractions, &events, flow_target, true, HX_BODY_NOT_COMPLETE);
+ CHECK_EQUAL(SCAN_NOT_FOUND, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(0, cutter.get_num_flush());
+
+ result = cutter.cut(scan_buf + 1, 8, infractions, &events, flow_target, false, HX_BODY_LAST_SEG);
+ // num_flush is reduced because decompress ran out of space
+ CHECK_EQUAL(SCAN_FOUND_PIECE, result);
+ CHECK_EQUAL(1, cutter.get_octets_seen());
+ CHECK_EQUAL(6, cutter.get_num_flush());
+}
+
+int main(int argc, char** argv)
+{
+ return CommandLineTestRunner::RunAllTests(argc, argv);
+}
+
return buf;
}
unsigned StreamSplitter::max(snort::Flow*) { return 0; }
+uint8_t TraceApi::get_constraints_generation() { return 0; }
+void TraceApi::filter(const Packet&) { }
+void trace_vprintf(const char*, TraceLevel, const char*, const snort::Packet*, const char*, va_list) { }
}
+THREAD_LOCAL const snort::Trace* http_trace = nullptr;
+
HttpParaList::UriParam::UriParam() {}
HttpParaList::JsNormParam::~JsNormParam() {}
HttpParaList::~HttpParaList() {}
#define HTTP_UNIT_TEST_HELPERS_H
#include "service_inspectors/http_inspect/http_common.h"
+#include "service_inspectors/http_inspect/http_compress_stream.h"
+#include "service_inspectors/http_inspect/http_event.h"
#include "service_inspectors/http_inspect/http_flow_data.h"
class HttpUnitTestSetup
{ assert(flow_data!=nullptr); return flow_data->type_expected; }
static void half_reset(HttpFlowData* flow_data, HttpCommon::SourceId source_id)
{ assert(flow_data!=nullptr); flow_data->half_reset(source_id); }
+ static void set_compress_stream(HttpFlowData* flow_data, HttpCommon::SourceId source_id,
+ HttpCompressStream* stream)
+ { assert(flow_data!=nullptr); flow_data->compress[source_id] = stream; }
+ static void set_partial_buffer(HttpFlowData* flow_data, HttpCommon::SourceId source_id,
+ uint8_t* buf, uint32_t length)
+ {
+ assert(flow_data!=nullptr);
+ flow_data->partial_buffer[source_id] = buf;
+ flow_data->partial_buffer_length[source_id] = length;
+ }
+ static HttpInfractions* get_infractions(HttpFlowData* flow_data, HttpCommon::SourceId source_id)
+ { assert(flow_data!=nullptr); return flow_data->infractions[source_id]; }
};
#endif
projects / thirdparty / snort3.git / commitdiff
