From: Oleksii Shumeiko -X (oshumeik - SOFTSERVE INC at Cisco) Date: Fri, 10 Feb 2023 09:33:19 +0000 (+0000) Subject: Pull request #3753: sd_pattern: keep obfuscation blocks per buffer X-Git-Tag: 3.1.56.0~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c4e5855cf77f71245c9c2e06571c0f11535c5558;p=thirdparty%2Fsnort3.git Pull request #3753: sd_pattern: keep obfuscation blocks per buffer Merge in SNORT/snort3 from ~ASERBENI/snort3:sd_obfuscation to master Squashed commit of the following: commit 0db98b656216676553096952d7df2d815e073627 Author: Andrii Serbeniuk Date: Tue Jan 31 11:12:57 2023 +0200 doc: update sd_pattern docs after obfuscation changes commit 7699a8338c6d7ec534d648d16cae8fde7947fd3a Author: Andrii Serbeniuk Date: Mon Jan 23 11:26:18 2023 +0200 sd_pattern: keep obfuscation blocks per buffer --- diff --git a/doc/user/sensitive_data.txt b/doc/user/sensitive_data.txt index c2f002b10..82c969be2 100644 --- a/doc/user/sensitive_data.txt +++ b/doc/user/sensitive_data.txt @@ -10,7 +10,7 @@ library from Intel. It provides a regex grammar which is mostly PCRE compatible. To learn more about Hyperscan see https://intel.github.io/hyperscan/dev-reference/ -==== Syntax +==== Syntax Snort provides `sd_pattern` as IPS rule option with no additional inspector overhead. The Rule option takes the following syntax. @@ -90,7 +90,7 @@ Snort provides discreet logging for the built-in patterns "credit_card", was matched by the patterns. This configuration is enabled by default. ips = - { + { obfuscate_pii = true } @@ -164,12 +164,12 @@ Doesn't match a rule like this. in the config. So, Snort must be built and run with Hyperscan to have sd_pattern IPS option available. -2. Log obfuscation is only applicable to CMG and Unified2 logging formats. +2. Log obfuscation is only applicable to the buffer that the pattern was found in. -3. Log obfuscation doesn't support user defined PII patterns. It is -currently only supported for the built-in patterns for Credit Cards and U.S. -Social Security numbers. +3. Log obfuscation is only applicable to CMG and Unified2 logging formats. +Unified2 logger only supports obfuscation of http extra data. -4. Log obfuscation doesn't work with stream rebuilt packet payloads. (This -is a known bug). +4. Log obfuscation doesn't support user defined PII patterns. It is +currently only supported for the built-in patterns for Credit Cards and U.S. +Social Security numbers, emails and U.S. phone numbers. diff --git a/src/ips_options/ips_sd_pattern.cc b/src/ips_options/ips_sd_pattern.cc index e1a4e6695..c4ea15370 100644 --- a/src/ips_options/ips_sd_pattern.cc +++ b/src/ips_options/ips_sd_pattern.cc @@ -232,8 +232,9 @@ bool SdPatternOption::operator==(const IpsOption& ips) const struct hsContext { hsContext(const SdPatternConfig& c_, Packet* p_, const uint8_t* const start_, - const uint8_t* _buf, unsigned int _buflen ) - : config(c_), packet(p_), start(start_), buf(_buf), buflen(_buflen) { } + const uint8_t* _buf, unsigned int _buf_len, const char* _buf_name) + : config(c_), packet(p_), start(start_), buf(_buf), buf_name(_buf_name), buf_len(_buf_len) + { } bool has_valid_bounds(unsigned long long from, unsigned long long len) { @@ -249,9 +250,9 @@ struct hsContext // validate the right side - if ( from+len == buflen ) + if ( from+len == buf_len ) right = true; - else if ( from + len < buflen && !::isdigit((int)buf[from+len]) ) + else if ( from + len < buf_len && !::isdigit((int)buf[from+len]) ) right = true; return left and right; @@ -265,7 +266,9 @@ struct hsContext Packet* packet = nullptr; const uint8_t* const start = nullptr; const uint8_t* buf = nullptr; - unsigned int buflen = 0; + const char* buf_name; + unsigned int buf_len = 0; + bool buf_set = false; }; static int hs_match(unsigned int /*id*/, unsigned long long from, @@ -304,6 +307,12 @@ static int hs_match(unsigned int /*id*/, unsigned long long from, if ( !ctx->packet->obfuscator ) ctx->packet->obfuscator = new Obfuscator(); + if ( !ctx->buf_set ) + { + ctx->packet->obfuscator->set_buffer(ctx->buf_name); + ctx->buf_set = true; + } + // FIXIT-L Make configurable or don't show any PII partials (0 for user defined??) uint32_t off = ctx->buf + from - ctx->start; ctx->packet->obfuscator->push(off, len - 4); @@ -316,11 +325,11 @@ unsigned SdPatternOption::SdSearch(const Cursor& c, Packet* p) { const uint8_t* const start = c.buffer(); const uint8_t* buf = c.start(); - unsigned int buflen = c.length(); + unsigned int buf_len = c.length(); - hsContext ctx(config, p, start, buf, buflen); + hsContext ctx(config, p, start, buf, buf_len, c.get_name()); - hs_error_t stat = hs_scan(config.db, (const char*)buf, buflen, 0, + hs_error_t stat = hs_scan(config.db, (const char*)buf, buf_len, 0, scratcher->get(), hs_match, (void*)&ctx); if ( stat == HS_SCAN_TERMINATED ) diff --git a/src/log/obfuscator.cc b/src/log/obfuscator.cc index 1d4348d04..b19573f5b 100644 --- a/src/log/obfuscator.cc +++ b/src/log/obfuscator.cc @@ -28,20 +28,24 @@ using namespace snort; bool Obfuscator::first(ObfuscatorBlock &b) { - if ( blocks.empty() ) + if (buffer_blocks.empty()) return false; - it = blocks.begin(); + if (cur_buf->second.empty()) + return false; + it = cur_buf->second.begin(); b = *it; return true; } bool Obfuscator::next(ObfuscatorBlock &b) { - if ( blocks.empty() ) + if (buffer_blocks.empty()) + return false; + if (cur_buf->second.empty()) return false; - if ( it == blocks.end() ) + if (it == cur_buf->second.end()) return false; - if ( ++it == blocks.end() ) + if (++it == cur_buf->second.end()) return false; b = *it; return true; diff --git a/src/log/obfuscator.h b/src/log/obfuscator.h index 9e6cc400c..130086ef6 100644 --- a/src/log/obfuscator.h +++ b/src/log/obfuscator.h @@ -24,6 +24,8 @@ #include #include #include +#include +#include #include "main/snort_types.h" @@ -53,25 +55,53 @@ public: }; using ObSet = std::set; + using BufBlocks = std::unordered_map; using const_iterator = ObSet::const_iterator; using iterator = ObSet::iterator; + Obfuscator() + { + cur_buf = buffer_blocks.begin(); + } + void push(uint32_t offset, uint32_t length) { - const auto push_res = blocks.emplace(offset, length); - + if (cur_buf == buffer_blocks.end()) + set_buffer(""); + const auto push_res = cur_buf->second.emplace(offset, length); + if (!push_res.second and length > push_res.first->length) { - blocks.erase(push_res.first); - blocks.emplace(offset, length); + cur_buf->second.erase(push_res.first); + cur_buf->second.emplace(offset, length); } } + bool select_buffer(const char* buf_key) + { + if (!buf_key) + return false; + + auto buf = buffer_blocks.find(buf_key); + if (buf == buffer_blocks.end()) + return false; + cur_buf = buf; + return true; + } + + void set_buffer(const char* buf_key) + { + if (!buf_key) + return; + + cur_buf = buffer_blocks.emplace(buf_key, ObSet()).first; + } + const_iterator begin() const - { return blocks.cbegin(); } + { return cur_buf->second.cbegin(); } const_iterator end() const - { return blocks.cend(); } + { return cur_buf->second.cend(); } bool first(ObfuscatorBlock &b); bool next(ObfuscatorBlock &b); @@ -80,9 +110,10 @@ public: { return mask_char; } private: - ObSet blocks; + BufBlocks buffer_blocks; + BufBlocks::iterator cur_buf; iterator it; - static const char mask_char = 'X'; + static constexpr char mask_char = 'X'; }; } diff --git a/src/loggers/alert_fast.cc b/src/loggers/alert_fast.cc index 7c09121d0..4e328a4c0 100644 --- a/src/loggers/alert_fast.cc +++ b/src/loggers/alert_fast.cc @@ -134,6 +134,20 @@ static void load_buf_ids( } } +static void ObfuscateLogNetData(TextLog* log, const uint8_t* data, const int len, + Packet* p, const char* buf_name, const char* buf_key, const char* ins_name) +{ + // FIXIT-P avoid string copy + std::string buf((const char*)data, len); + auto obf = p->obfuscator; + + if ( obf and obf->select_buffer(buf_key) ) + for ( const auto& b : *obf ) + buf.replace(b.offset, b.length, b.length, obf->get_mask_char()); + + LogNetData(log, (const uint8_t*)buf.c_str(), len, p, buf_name, ins_name); +} + using BufferIds = std::vector; //------------------------------------------------------------------------- @@ -283,7 +297,7 @@ void FastLogger::log_data(Packet* p, const Event& event) InspectionBuffer buf; if ( gadget->get_buf(id, p, buf) ) - LogNetData(fast_log, buf.data, buf.len, p, buffers[id-1], ins_name); + ObfuscateLogNetData(fast_log, buf.data, buf.len, p, buffers[id-1], buffers[id-1], ins_name); log_pkt = (idv == rsp_ids); } @@ -303,19 +317,8 @@ void FastLogger::log_data(Packet* p, const Event& event) } if (p->has_ip()) LogIPPkt(fast_log, p); - - else if ( log_pkt and p->obfuscator ) - { - // FIXIT-P avoid string copy - std::string buf((const char*)p->data, p->dsize); - - for ( const auto& b : *p->obfuscator ) - buf.replace(b.offset, b.length, b.length, p->obfuscator->get_mask_char()); - - LogNetData(fast_log, (const uint8_t*)buf.c_str(), p->dsize, p, nullptr, ins_name); - } else if ( log_pkt ) - LogNetData(fast_log, p->data, p->dsize, p, nullptr, ins_name); + ObfuscateLogNetData(fast_log, p->data, p->dsize, p, nullptr, "pkt_data", ins_name); DataBuffer& buf = DetectionEngine::get_alt_buffer(p); diff --git a/src/loggers/unified2.cc b/src/loggers/unified2.cc index 10c6aa304..9ca5a3921 100644 --- a/src/loggers/unified2.cc +++ b/src/loggers/unified2.cc @@ -33,6 +33,7 @@ #include "detection/signature.h" #include "detection/detection_util.h" +#include "detection/detection_engine.h" #include "events/event.h" #include "framework/logger.h" #include "framework/module.h" @@ -244,7 +245,30 @@ static void alert_event(Packet* p, const char*, Unified2Config* config, const Ev Unified2Write(write_pkt_buffer, write_len, config); } +static void apply_mask(Obfuscator* obf, uint8_t* buf, const char* buf_key) +{ + if ( obf->select_buffer(buf_key) ) + for ( const auto& b : *obf ) + memset(buf + b.offset, obf->get_mask_char(), b.length); +} + +static void obfuscate(uint8_t* buf, Obfuscator* obf, uint32_t type) +{ + switch (type) + { + case EVENT_INFO_HTTP_URI: + apply_mask(obf, buf, "http_uri"); + break; + case EVENT_INFO_JSNORM_DATA: + apply_mask(obf, buf, "file_data"); + break; + default: + break; + } +} + static void _WriteExtraData(Unified2Config* config, + Obfuscator* obf, uint32_t event_id, uint32_t event_second, const uint8_t* buffer, @@ -297,6 +321,9 @@ static void _WriteExtraData(Unified2Config* config, memcpy_s(ptr + offset, sizeof(write_buffer) - offset, buffer, len); + if (obf) + obfuscate(ptr + offset, obf, type); + Unified2Write(write_buffer, write_len, config); } @@ -314,6 +341,9 @@ static void AlertExtraData( xid = ffs(xtradata_mask); + const IpsContext* c = DetectionEngine::get_context(); + Obfuscator* obf = (c and c->packet) ? c->packet->obfuscator : nullptr; + while ( xid && (xid <= max_count) ) { uint32_t len = 0; @@ -323,7 +353,7 @@ static void AlertExtraData( if ( log_func(flow, &write_buffer, &len, &type) && (len > 0) ) { - _WriteExtraData(config, event_id, event_second, write_buffer, len, type); + _WriteExtraData(config, obf, event_id, event_second, write_buffer, len, type); } xtradata_mask ^= BIT(xid); xid = ffs(xtradata_mask); @@ -388,8 +418,9 @@ static void _Unified2LogPacketAlert( if ( p->is_data() ) off = 0; - for ( const auto& b : *p->obfuscator ) - memset(&start[ off + b.offset ], p->obfuscator->get_mask_char(), b.length); + if ( p->obfuscator->select_buffer("pkt_data") ) + for ( const auto& b : *p->obfuscator ) + memset(&start[ off + b.offset ], p->obfuscator->get_mask_char(), b.length); } } @@ -902,10 +933,10 @@ void U2Logger::alert_legacy(Packet* p, const char* msg, const Event& event) if (p->ptrs.ip_api.is_ip6()) { const SfIp* ip = p->ptrs.ip_api.get_src(); - _WriteExtraData(&config, event.get_event_id(), event.ref_time.tv_sec, + _WriteExtraData(&config, p->obfuscator, event.get_event_id(), event.ref_time.tv_sec, (const uint8_t*) ip->get_ip6_ptr(), sizeof(struct in6_addr), EVENT_INFO_IPV6_SRC); ip = p->ptrs.ip_api.get_dst(); - _WriteExtraData(&config, event.get_event_id(), event.ref_time.tv_sec, + _WriteExtraData(&config, p->obfuscator, event.get_event_id(), event.ref_time.tv_sec, (const uint8_t*) ip->get_ip6_ptr(), sizeof(struct in6_addr), EVENT_INFO_IPV6_DST); } }