From: Anna Norokh -X (anorokh - SOFTSERVE INC at Cisco) Date: Mon, 5 May 2025 14:25:35 +0000 (+0000) Subject: Pull request #4657: extractor: add ips events logging X-Git-Tag: 3.7.4.0~2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=561cf8692e27166cd4ba626080ef769052952bbf;p=thirdparty%2Fsnort3.git Pull request #4657: extractor: add ips events logging Merge in SNORT/snort3 from ~ANOROKH/snort3:extr_detection to master Squashed commit of the following: commit 582e912a61e0993915ed83d84e77f1841f4e3423 Author: anorokh Date: Thu Feb 20 02:28:16 2025 +0200 extractor: add weird and notice logging --- diff --git a/doc/user/extractor.txt b/doc/user/extractor.txt index 8bb10c2b5..5d36e36ca 100644 --- a/doc/user/extractor.txt +++ b/doc/user/extractor.txt @@ -1,4 +1,4 @@ -Snort 3 can log IPS events with some meta data and dump packets. The Data +Snort 3 can log IPS events with some meta data and dump packets. Advanced Logging feature extends that ability to log protocol-specific data, sniffing traffic alongside with normal inspection. @@ -20,7 +20,7 @@ The module's configuration consists of two parts: ** `fields` - data fields to log (if a field is not supported it will be ignored) Configuration from different bindings do not interfere. Among other -things it allows tenants to get independent data logging configurations. +things it allows tenants to get independent logging configurations. extractor = { @@ -34,6 +34,8 @@ things it allows tenants to get independent data logging configurations. { service = 'http', on_events = 'eot', fields = 'ts, uri' }, { service = 'conn', on_events = 'eof', fields = 'ts, uid, service' }, { service = 'dns', on_events = 'response', fields = 'ts, uid, query, answers' } + { service = 'weird', on_events = 'builtin', fields = 'ts, msg, gid, sid' } + { service = 'notice', on_events = 'ips_logging', fields = 'msg, sid, refs' } } } @@ -61,6 +63,10 @@ Services and their events: ** `response` * connection (conn) ** `eof` (end of flow) +* internal built-in checks which failed (weird) + ** 'builtin' (internally-detected infraction is queued for further processing) +* triggered IPS rule, whether built-in or text or SO (notice) + ** `ips_logging` (matched rules sent to IPS logging) Common fields available for every service: @@ -151,6 +157,20 @@ Fields supported for connection: For TCP orig_bytes and resp_bytes are calculated using first seen sequence number and next expected sequence number. These are reset during TCP flow restart. For this case only bytes seen following the restart will be reported. +Fields supported for 'weird' and 'notice' logs: + +* `sid` - unique signature number of the rule +* `gid` - component ID which generated the event +* `msg` - rule message +* `proto` - transport protocol +* `source` - assigned inspector + +'notice' events for text rules also get the following fields: + +* `action` - action of triggered event +* `refs` - references mentioned in a rule +* `rev` - particular revision number of the rule + ==== Example Adding the following lines to a default snort configuration (which supports FTP diff --git a/src/detection/detect.cc b/src/detection/detect.cc index fcc3f55b2..3eda1489c 100644 --- a/src/detection/detect.cc +++ b/src/detection/detect.cc @@ -96,7 +96,7 @@ void CallLogFuncs(Packet* p, const OptTreeNode* otn, ListHead* head) p->dsize = dsize; } - IpsRuleEvent data_event(event, p); + IpsRuleEvent data_event(event); DataBus::publish(DetectionEngine::get_pub_id(), DetectionEventIds::IPS_LOGGING, data_event, p->flow); OutputSet* idx = head ? head->LogList : nullptr; diff --git a/src/detection/detection_engine.cc b/src/detection/detection_engine.cc index ea5d25f54..28b6a3a0e 100644 --- a/src/detection/detection_engine.cc +++ b/src/detection/detection_engine.cc @@ -715,6 +715,13 @@ int DetectionEngine::queue_event(unsigned gid, unsigned sid) if ( !otn ) return 0; + const Packet* p = get_current_packet(); + if ( p ) + { + IpsQueuingEvent data_event(otn->sigInfo); + DataBus::publish(get_pub_id(), DetectionEventIds::BUILTIN, data_event, p->flow); + } + SF_EVENTQ* pq = get_event_queue(); EventNode* en = (EventNode*)sfeventq_event_alloc(pq); diff --git a/src/network_inspectors/extractor/CMakeLists.txt b/src/network_inspectors/extractor/CMakeLists.txt index 756cceffb..9cfd0b271 100644 --- a/src/network_inspectors/extractor/CMakeLists.txt +++ b/src/network_inspectors/extractor/CMakeLists.txt @@ -12,6 +12,8 @@ set( FILE_LIST extractor_conn.cc extractor_csv_logger.cc extractor_csv_logger.h + extractor_detection.cc + extractor_detection.h extractor_dns.cc extractor_enums.h extractor_flow_data.cc @@ -22,11 +24,11 @@ set( FILE_LIST extractor_json_logger.h extractor_logger.cc extractor_logger.h + extractor_null_conn.h extractor_service.cc extractor_service.h extractors.cc extractors.h - extractor_null_conn.h ) add_library(extractor OBJECT ${FILE_LIST}) diff --git a/src/network_inspectors/extractor/extractor.cc b/src/network_inspectors/extractor/extractor.cc index 292e2c2c7..15ab87359 100644 --- a/src/network_inspectors/extractor/extractor.cc +++ b/src/network_inspectors/extractor/extractor.cc @@ -50,7 +50,7 @@ THREAD_LOCAL ExtractorLogger* Extractor::logger = nullptr; static const Parameter extractor_proto_params[] = { - { "service", Parameter::PT_ENUM, "http | ftp | conn | dns", nullptr, + { "service", Parameter::PT_ENUM, "http | ftp | conn | dns | weird | notice", nullptr, "service to extract from" }, { "tenant_id", Parameter::PT_INT, "0:max32", "0", diff --git a/src/network_inspectors/extractor/extractor.h b/src/network_inspectors/extractor/extractor.h index 1b234d547..c54b05563 100644 --- a/src/network_inspectors/extractor/extractor.h +++ b/src/network_inspectors/extractor/extractor.h @@ -64,7 +64,7 @@ static const PegInfo extractor_pegs[] = struct ExtractorStats { - PegCount total_event; + PegCount total_events; }; class ExtractorReloadSwapper; diff --git a/src/network_inspectors/extractor/extractor_conn.cc b/src/network_inspectors/extractor/extractor_conn.cc index 969d72a94..129a0f122 100644 --- a/src/network_inspectors/extractor/extractor_conn.cc +++ b/src/network_inspectors/extractor/extractor_conn.cc @@ -165,7 +165,7 @@ void ConnExtractor::handle(DataEvent& event, Flow* flow) if (flow->pkt_type < PktType::IP or flow->pkt_type > PktType::ICMP or !filter(flow)) return; - extractor_stats.total_event++; + extractor_stats.total_events++; logger->open_record(); log(nts_fields, &event, flow); diff --git a/src/network_inspectors/extractor/extractor_csv_logger.cc b/src/network_inspectors/extractor/extractor_csv_logger.cc index e204aa8dc..83feecef2 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.cc +++ b/src/network_inspectors/extractor/extractor_csv_logger.cc @@ -278,8 +278,75 @@ void CsvExtractorLogger::ts_usec(const struct timeval& v) record.append(to_string(sec * 1000000 + usec)); } +void CsvExtractorLogger::add_array_separator() +{ + record.push_back(' '); +} + +void CsvExtractorLogger::add_field(const char*, const std::vector& v) +{ + record.push_back(delimiter); + + if (v.empty()) + { + if (!isprint(delimiter)) + record.append("-"); + return; + } + + if (v[0] && v[0][0]) + add_escaped(v[0], strlen(v[0])); + + for (size_t i = 1; i < v.size(); ++i) + { + add_array_separator(); + if (v[i] && v[i][0]) + add_escaped(v[i], strlen(v[i])); + } +} + +void CsvExtractorLogger::add_field(const char*, const std::vector& v) +{ + record.push_back(delimiter); + + if (v.empty()) + { + if (!isprint(delimiter)) + record.append("-"); + return; + } + + record.append(to_string(v[0])); + for (size_t i = 1; i < v.size(); ++i) + { + add_array_separator(); + record.append(to_string(v[i])); + } +} + +void CsvExtractorLogger::add_field(const char*, const std::vector& v) +{ + record.push_back(delimiter); + + if (v.empty()) + { + if (!isprint(delimiter)) + record.append("-"); + return; + } + + record.append(v[0] ? "true" : "false"); + for (size_t i = 1; i < v.size(); ++i) + { + add_array_separator(); + record.append(v[i] ? "true" : "false"); + } +} + #ifdef UNIT_TEST +#include + #include "catch/snort_catch.h" class CsvExtractorLoggerHelper : public CsvExtractorLogger @@ -293,6 +360,30 @@ public: add_escaped(input, i_len); CHECK(record == expected); } + + void check(const std::vector& v, const std::string& expected) + { + record.clear(); + add_field(nullptr, v); + auto data = record.substr(1); + CHECK(data == expected); + } + + void check(const std::vector& v, const std::string& expected) + { + record.clear(); + add_field(nullptr, v); + auto data = record.substr(1); + CHECK(data == expected); + } + + void check(const std::vector& v, const std::string& expected) + { + record.clear(); + add_field(nullptr, v); + auto data = record.substr(1); + CHECK(data == expected); + } }; class CsvExtractorLoggerTest @@ -305,6 +396,24 @@ public: void check_tsv(const char* input, size_t i_len, const std::string& expected) { tsv.check(input, i_len, expected); } + void check_csv_vec(const std::vector& v, const std::string& expected) + { csv.check(v, expected); } + + void check_csv_vec(const std::vector& v, const std::string& expected) + { csv.check(v, expected); } + + void check_csv_vec(const std::vector& v, const std::string& expected) + { csv.check(v, expected); } + + void check_tsv_vec(const std::vector& v, const std::string& expected) + { tsv.check(v, expected); } + + void check_tsv_vec(const std::vector& v, const std::string& expected) + { tsv.check(v, expected); } + + void check_tsv_vec(const std::vector& v, const std::string& expected) + { tsv.check(v, expected); } + private: CsvExtractorLoggerHelper csv{','}; CsvExtractorLoggerHelper tsv{'\t'}; @@ -435,4 +544,46 @@ TEST_CASE_METHOD(CsvExtractorLoggerTest, "escape: single whitespace", "[extracto check_tsv(input, strlen(input), " "); } +TEST_CASE_METHOD(CsvExtractorLoggerTest, "csv/tsv vector bool: empty", "[extractor]") +{ + const std::vector bool_vec = {}; + check_csv_vec(bool_vec, ""); + check_tsv_vec(bool_vec, "-"); +} + +TEST_CASE_METHOD(CsvExtractorLoggerTest, "csv/tsv vector bool: 3 items", "[extractor]") +{ + const std::vector bool_vec = {true, false, true}; + check_csv_vec(bool_vec, "true false true"); + check_tsv_vec(bool_vec, "true false true"); +} + +TEST_CASE_METHOD(CsvExtractorLoggerTest, "csv/tsv vector uint64_t: empty", "[extractor]") +{ + const std::vector unum_vec = {}; + check_csv_vec(unum_vec, ""); + check_tsv_vec(unum_vec, "-"); +} + +TEST_CASE_METHOD(CsvExtractorLoggerTest, "csv/tsv vector uint64_t: 3 items", "[extractor]") +{ + const std::vector unum_vec = {1,2,3}; + check_csv_vec(unum_vec, "1 2 3"); + check_tsv_vec(unum_vec, "1 2 3"); +} + +TEST_CASE_METHOD(CsvExtractorLoggerTest, "csv/tsv vector str: empty", "[extractor]") +{ + const std::vector char_vec = {}; + check_csv_vec(char_vec, ""); + check_tsv_vec(char_vec, "-"); +} + +TEST_CASE_METHOD(CsvExtractorLoggerTest, "csv/tsv vector str: 3 items", "[extractor]") +{ + const std::vector char_vec = {"exe", "pdf", "txt"}; + check_csv_vec(char_vec, "exe pdf txt"); + check_tsv_vec(char_vec, "exe pdf txt"); +} + #endif diff --git a/src/network_inspectors/extractor/extractor_csv_logger.h b/src/network_inspectors/extractor/extractor_csv_logger.h index 4b1fa942a..3f612dc82 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.h +++ b/src/network_inspectors/extractor/extractor_csv_logger.h @@ -37,10 +37,16 @@ public: void add_field(const char*, struct timeval) override; void add_field(const char*, const snort::SfIp&) override; void add_field(const char*, bool) override; + + void add_field(const char*, const std::vector&) override; + void add_field(const char*, const std::vector&) override; + void add_field(const char*, const std::vector&) override; + void open_record() override; void close_record(const snort::Connector::ID&) override; protected: + void add_array_separator(); void add_escaped(const char*, size_t); void ts_snort(const struct timeval&); void ts_snort_yy(const struct timeval&); diff --git a/src/network_inspectors/extractor/extractor_detection.cc b/src/network_inspectors/extractor/extractor_detection.cc new file mode 100644 index 000000000..a2798ccd5 --- /dev/null +++ b/src/network_inspectors/extractor/extractor_detection.cc @@ -0,0 +1,252 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// extractor_detection.cc author Anna Norokh + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "extractor_detection.h" + +#include "profiler/profiler.h" +#include "pub_sub/detection_events.h" + +#include "extractor.h" + +using namespace snort; +using namespace std; + +namespace builtin +{ +static const char* get_msg(const DataEvent* event, const Flow*) +{ + return ((const IpsQueuingEvent*)event)->get_stripped_msg().c_str(); +} + +static const char* get_proto(const DataEvent*, const Flow*) +{ + const Packet* p = ExtractorEvent::get_packet(); + + if (p != nullptr) + return p->get_type(); + + return nullptr; +} + +static const char* get_source(const DataEvent*, const Flow* flow) +{ + if (flow->gadget) + return flow->gadget->get_name(); + + return ""; +} + +static uint64_t get_sid(const DataEvent* event, const Flow*) +{ + return (uint64_t)((const IpsQueuingEvent*)event)->get_sid(); +} + +static uint64_t get_gid(const DataEvent* event, const Flow*) +{ + return (uint64_t)((const IpsQueuingEvent*)event)->get_gid(); +} + +static const map sub_buf_getters = +{ + {"msg", get_msg}, + {"source", get_source}, + {"proto", get_proto}, +}; + +static const map gid_sid_getters = +{ + {"gid", get_gid}, + {"sid", get_sid}, +}; +} + +THREAD_LOCAL const snort::Connector::ID* BuiltinExtractor::log_id = nullptr; + +BuiltinExtractor::BuiltinExtractor(Extractor& i, uint32_t t, const vector& fields) + : ExtractorEvent(ServiceType::IPS_BUILTIN, i, t) +{ + for (const auto& f : fields) + { + if (append(nts_fields, nts_getters, f)) + continue; + if (append(sip_fields, sip_getters, f)) + continue; + if (append(num_fields, num_getters, f)) + continue; + if (append(num_fields, builtin::gid_sid_getters, f)) + continue; + if (append(buf_fields, builtin::sub_buf_getters, f)) + continue; + } + + DataBus::subscribe_global(de_pub_key, DetectionEventIds::BUILTIN, new IpsBuiltin(*this, S_NAME), i.get_snort_config()); +} + +void BuiltinExtractor::internal_tinit(const snort::Connector::ID* service_id) +{ log_id = service_id; } + +void BuiltinExtractor::handle(DataEvent& event, Flow* flow) +{ + // cppcheck-suppress unreadVariable + Profile profile(extractor_perf_stats); + + if (!filter(flow)) + return; + + extractor_stats.total_events++; + + logger->open_record(); + log(nts_fields, &event, flow); + log(sip_fields, &event, flow); + log(num_fields, &event, flow); + log(buf_fields, &event, flow); + logger->close_record(*log_id); +} + +namespace ips +{ +static const char* get_msg(const DataEvent* event, const Flow*) +{ + return ((const IpsRuleEvent*)event)->get_stripped_msg().c_str(); +} + +static const char* get_action(const DataEvent* event, const Flow*) +{ + return ((const IpsRuleEvent*)event)->get_action(); +} + +static const vector& get_refs(const DataEvent* event, const Flow*) +{ + return ((const IpsRuleEvent*)event)->get_references(); +} + +static const char* get_proto(const DataEvent*, const Flow*) +{ + const Packet* p = ExtractorEvent::get_packet(); + + if (p != nullptr) + return p->get_type(); + + return nullptr; +} + +static const char* get_source(const DataEvent*, const Flow* flow) +{ + if (flow->gadget) + return flow->gadget->get_name(); + + return ""; +} + +static uint64_t get_sid(const DataEvent* event, const Flow*) +{ + return (uint64_t)((const IpsRuleEvent*)event)->get_sid(); +} + +static uint64_t get_gid(const DataEvent* event, const Flow*) +{ + return (uint64_t)((const IpsRuleEvent*)event)->get_gid(); +} + +static uint64_t get_rev(const DataEvent* event, const Flow*) +{ + return (uint64_t)((const IpsRuleEvent*)event)->get_rev(); +} + +static const map sub_buf_getters = +{ + {"msg", get_msg}, + {"action", get_action}, + {"source", get_source}, + {"proto", get_proto}, +}; + +static const map gid_sid_rev_getters = +{ + {"gid", get_gid}, + {"sid", get_sid}, + {"rev", get_rev}, +}; + +static const map vec_getters = +{ + {"refs", get_refs} +}; +} + +THREAD_LOCAL const snort::Connector::ID* IpsUserExtractor::log_id = nullptr; + +IpsUserExtractor::IpsUserExtractor(Extractor& i, uint32_t t, const vector& fields) + : ExtractorEvent(ServiceType::IPS_USER, i, t) +{ + for (const auto& f : fields) + { + if (append(nts_fields, nts_getters, f)) + continue; + if (append(sip_fields, sip_getters, f)) + continue; + if (append(num_fields, num_getters, f)) + continue; + if (append(num_fields, ips::gid_sid_rev_getters, f)) + continue; + if (append(buf_fields, ips::sub_buf_getters, f)) + continue; + if (append(vec_fields, ips::vec_getters, f)) + continue; + } + + DataBus::subscribe_global(de_pub_key, DetectionEventIds::IPS_LOGGING, new IpsUser(*this, S_NAME), i.get_snort_config()); +} + +void IpsUserExtractor::internal_tinit(const snort::Connector::ID* service_id) +{ log_id = service_id; } + +void IpsUserExtractor::handle(DataEvent& event, Flow* flow) +{ + // cppcheck-suppress unreadVariable + Profile profile(extractor_perf_stats); + + if (!filter(flow)) + return; + + extractor_stats.total_events++; + + logger->open_record(); + log(nts_fields, &event, flow); + log(sip_fields, &event, flow); + log(num_fields, &event, flow); + log(buf_fields, &event, flow); + log(vec_fields, &event, flow); + logger->close_record(*log_id); +} + +vector IpsUserExtractor::get_field_names() const +{ + vector res = ExtractorEvent::get_field_names(); + + for (const auto& f : vec_fields) + res.push_back(f.name); + + return res; +} + diff --git a/src/network_inspectors/extractor/extractor_detection.h b/src/network_inspectors/extractor/extractor_detection.h new file mode 100644 index 000000000..61ca8cfcb --- /dev/null +++ b/src/network_inspectors/extractor/extractor_detection.h @@ -0,0 +1,60 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// extractor_detection.h author Anna Norokh + +#ifndef EXTRACTOR_RULES_H +#define EXTRACTOR_RULES_H + +#include "extractors.h" + +class BuiltinExtractor : public ExtractorEvent +{ +public: + BuiltinExtractor(Extractor&, uint32_t tenant, const std::vector& fields); + + void handle(DataEvent&, Flow*); + +private: + using IpsBuiltin = Handler; + + void internal_tinit(const snort::Connector::ID*) override; + + static THREAD_LOCAL const snort::Connector::ID* log_id; +}; + +class IpsUserExtractor : public ExtractorEvent +{ +public: + using VecGetFn = const std::vector& (*) (const DataEvent*, const Flow*); + using VecField = DataField&, const DataEvent*, const Flow*>; + + IpsUserExtractor(Extractor&, uint32_t tenant, const std::vector& fields); + + std::vector get_field_names() const override; + void handle(DataEvent&, Flow*); + +private: + using IpsUser = Handler; + + void internal_tinit(const snort::Connector::ID*) override; + + std::vector vec_fields; + static THREAD_LOCAL const snort::Connector::ID* log_id; +}; + +#endif diff --git a/src/network_inspectors/extractor/extractor_dns.cc b/src/network_inspectors/extractor/extractor_dns.cc index 49bad0625..0ed35d931 100644 --- a/src/network_inspectors/extractor/extractor_dns.cc +++ b/src/network_inspectors/extractor/extractor_dns.cc @@ -198,7 +198,7 @@ void DnsResponseExtractor::handle(DataEvent& event, Flow* flow) if (!filter(flow)) return; - extractor_stats.total_event++; + extractor_stats.total_events++; logger->open_record(); log(nts_fields, &event, flow); diff --git a/src/network_inspectors/extractor/extractor_enums.h b/src/network_inspectors/extractor/extractor_enums.h index 3b2d99118..22961a23b 100644 --- a/src/network_inspectors/extractor/extractor_enums.h +++ b/src/network_inspectors/extractor/extractor_enums.h @@ -31,6 +31,8 @@ public: FTP, CONN, DNS, + IPS_BUILTIN, + IPS_USER, ANY, MAX }; @@ -54,6 +56,10 @@ public: return "conn"; case DNS: return "dns"; + case IPS_BUILTIN: + return "weird"; + case IPS_USER: + return "notice"; case ANY: // fallthrough case MAX: // fallthrough default: diff --git a/src/network_inspectors/extractor/extractor_ftp.cc b/src/network_inspectors/extractor/extractor_ftp.cc index 3a63e85dd..7d6360a24 100644 --- a/src/network_inspectors/extractor/extractor_ftp.cc +++ b/src/network_inspectors/extractor/extractor_ftp.cc @@ -107,7 +107,7 @@ void FtpRequestExtractor::handle(DataEvent& event, Flow* flow) if (!filter(flow)) return; - extractor_stats.total_event++; + extractor_stats.total_events++; logger->open_record(); log(nts_fields, &event, flow); @@ -267,7 +267,7 @@ void FtpResponseExtractor::handle(DataEvent& event, Flow* flow) if (!filter(flow)) return; - extractor_stats.total_event++; + extractor_stats.total_events++; logger->open_record(); log(nts_fields, &event, flow); @@ -526,7 +526,7 @@ void FtpExtractor::Req::handle(DataEvent& event, Flow* flow) if (!owner.filter(flow)) return; - extractor_stats.total_event++; + extractor_stats.total_events++; auto fd = ExtractorFlowData::get(flow); @@ -582,7 +582,7 @@ void FtpExtractor::Resp::handle(DataEvent& event, Flow* flow) if (!owner.filter(flow)) return; - extractor_stats.total_event++; + extractor_stats.total_events++; auto fd = ExtractorFlowData::get(flow); diff --git a/src/network_inspectors/extractor/extractor_http.cc b/src/network_inspectors/extractor/extractor_http.cc index d30e9692c..b4e7fd29c 100644 --- a/src/network_inspectors/extractor/extractor_http.cc +++ b/src/network_inspectors/extractor/extractor_http.cc @@ -216,7 +216,7 @@ void HttpExtractor::handle(DataEvent& event, Flow* flow) if (!filter(flow)) return; - extractor_stats.total_event++; + extractor_stats.total_events++; logger->open_record(); log(nts_fields, &event, flow); diff --git a/src/network_inspectors/extractor/extractor_json_logger.cc b/src/network_inspectors/extractor/extractor_json_logger.cc index cdc095749..32ae08c38 100644 --- a/src/network_inspectors/extractor/extractor_json_logger.cc +++ b/src/network_inspectors/extractor/extractor_json_logger.cc @@ -145,3 +145,110 @@ void JsonExtractorLogger::ts_usec(const char* f, const struct timeval& v) js.uput(f, sec * 1000000 + usec); } + +void JsonExtractorLogger::add_field(const char* f, const std::vector& v) +{ + if (v.empty()) + return; + + js.open_array(f); + for (const auto& val : v) + js.put(nullptr, val); + + js.close_array(); +} + +void JsonExtractorLogger::add_field(const char* f, const std::vector& v) +{ + if (v.empty()) + return; + + js.open_array(f); + for (const auto unum : v) + js.put(nullptr, unum); + + js.close_array(); +} + +void JsonExtractorLogger::add_field(const char* f, const std::vector& v) +{ + if (v.empty()) + return; + + js.open_array(f); + for (bool b : v) + b ? js.put_true(nullptr) : js.put_false(nullptr); + + js.close_array(); +} + +#ifdef UNIT_TEST + +#include + +#include "catch/snort_catch.h" + +class JsonExtractorLoggerTest : public JsonExtractorLogger +{ +public: + JsonExtractorLoggerTest() : JsonExtractorLogger(nullptr, TimeType::MAX) { } + + void check(const char* f, const std::vector& v, const std::string& expected) + { + oss.str(std::string()); + add_field(f, v); + CHECK(oss.str() == expected); + } + + void check(const char* f, const std::vector& v, const std::string& expected) + { + oss.str(std::string()); + add_field(f, v); + CHECK(oss.str() == expected); + } + + void check(const char* f, const std::vector& v, const std::string& expected) + { + oss.str(std::string()); + add_field(f, v); + CHECK(oss.str() == expected); + } +}; + +TEST_CASE_METHOD(JsonExtractorLoggerTest, "json vector bool: empty", "[extractor]") +{ + const std::vector bool_vec = {}; + check("bool", bool_vec, ""); +} + +TEST_CASE_METHOD(JsonExtractorLoggerTest, "json vector bool: 3 items", "[extractor]") +{ + const std::vector bool_vec = {true, false, true}; + check("bool", bool_vec, "\"bool\": [ true, false, true ]\n"); +} + +TEST_CASE_METHOD(JsonExtractorLoggerTest, "json vector uint64_t: empty", "[extractor]") +{ + const std::vector num_vec = {}; + check("num", num_vec, ""); +} + +TEST_CASE_METHOD(JsonExtractorLoggerTest, "json vector uint64_t: 3 items", "[extractor]") +{ + const std::vector num_vec = {1,2,3}; + check("num", num_vec, "\"num\": [ 1, 2, 3 ]\n"); +} + +TEST_CASE_METHOD(JsonExtractorLoggerTest, "json vector str: empty", "[extractor]") +{ + const std::vector char_vec = {}; + check("str", char_vec, ""); +} + +TEST_CASE_METHOD(JsonExtractorLoggerTest, "json vector str: 3 items", "[extractor]") +{ + const std::vector num_vec = {"exe", "pdf", "txt"}; + check("str", num_vec, "\"str\": [ \"exe\", \"pdf\", \"txt\" ]\n"); +} + +#endif diff --git a/src/network_inspectors/extractor/extractor_json_logger.h b/src/network_inspectors/extractor/extractor_json_logger.h index c511201d5..811d1300a 100644 --- a/src/network_inspectors/extractor/extractor_json_logger.h +++ b/src/network_inspectors/extractor/extractor_json_logger.h @@ -37,9 +37,17 @@ public: void add_field(const char*, struct timeval) override; void add_field(const char*, const snort::SfIp&) override; void add_field(const char*, bool) override; + + void add_field(const char*, const std::vector&) override; + void add_field(const char*, const std::vector&) override; + void add_field(const char*, const std::vector&) override; + void open_record() override; void close_record(const snort::Connector::ID&) override; +protected: + std::ostringstream oss; + private: void ts_snort(const char*, const struct timeval&); void ts_snort_yy(const char*, const struct timeval&); @@ -47,7 +55,6 @@ private: void ts_sec(const char*, const struct timeval&); void ts_usec(const char*, const struct timeval&); - std::ostringstream oss; snort::JsonStream js; void (JsonExtractorLogger::*add_ts)(const char*, const struct timeval&); }; diff --git a/src/network_inspectors/extractor/extractor_logger.h b/src/network_inspectors/extractor/extractor_logger.h index 3f0d54426..3734adf77 100644 --- a/src/network_inspectors/extractor/extractor_logger.h +++ b/src/network_inspectors/extractor/extractor_logger.h @@ -55,6 +55,10 @@ public: virtual void add_field(const char*, const snort::SfIp&) {} virtual void add_field(const char*, bool) {} + virtual void add_field(const char*, const std::vector&) {} + virtual void add_field(const char*, const std::vector&) {} + virtual void add_field(const char*, const std::vector&) {} + const snort::Connector::ID get_id(const char* service_name) const { return output_conn->get_id(service_name); } diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc index b52922a00..aab7ae3e1 100644 --- a/src/network_inspectors/extractor/extractor_service.cc +++ b/src/network_inspectors/extractor/extractor_service.cc @@ -27,6 +27,7 @@ #include "extractor.h" #include "extractor_conn.h" +#include "extractor_detection.h" #include "extractor_dns.h" #include "extractor_ftp.h" #include "extractor_http.h" @@ -129,6 +130,14 @@ ExtractorService* ExtractorService::make_service(Extractor& ins, const ServiceCo srv = new DnsExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, ins); break; + case ServiceType::IPS_BUILTIN: + srv = new BuiltinExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, ins); + break; + + case ServiceType::IPS_USER: + srv = new IpsUserExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, ins); + break; + case ServiceType::ANY: // fallthrough default: ErrorMessage("Extractor: '%s' service is not supported\n", cfg.service.c_str()); @@ -226,6 +235,16 @@ void ExtractorService::validate(const ServiceConfig& cfg) validate_fields(DnsExtractorService::blueprint, cfg.fields); break; + case ServiceType::IPS_BUILTIN: + validate_fields(BuiltinExtractorService::blueprint, cfg.fields); + validate_events(BuiltinExtractorService::blueprint, cfg.on_events); + break; + + case ServiceType::IPS_USER: + validate_fields(IpsUserExtractorService::blueprint, cfg.fields); + validate_events(IpsUserExtractorService::blueprint, cfg.on_events); + break; + case ServiceType::ANY: // fallthrough default: ParseError("'%s' service is not supported", cfg.service.c_str()); @@ -429,6 +448,87 @@ const snort::Connector::ID& DnsExtractorService::internal_tinit() const snort::Connector::ID& DnsExtractorService::get_log_id() { return log_id; } +//------------------------------------------------------------------------- +// IpsUserExtractorService +//------------------------------------------------------------------------- + +const ServiceBlueprint IpsUserExtractorService::blueprint = +{ + // events + { + "ips_logging", + }, + // fields + { + "action", + "sid", + "gid", + "rev", + "msg", + "refs", + "proto", + "source", + }, +}; + +THREAD_LOCAL Connector::ID IpsUserExtractorService::log_id; + +IpsUserExtractorService::IpsUserExtractorService(uint32_t tenant, const std::vector& srv_fields, + const std::vector& srv_events, ServiceType s_type, Extractor& ins) + : ExtractorService(tenant, srv_fields, srv_events, blueprint, s_type, ins) +{ + for (const auto& event : get_events()) + { + if (!strcmp("ips_logging", event.c_str())) + handlers.push_back(new IpsUserExtractor(ins, tenant_id, get_fields())); + } +} + +const snort::Connector::ID& IpsUserExtractorService::internal_tinit() +{ return log_id = logger->get_id(type.c_str()); } + +const snort::Connector::ID& IpsUserExtractorService::get_log_id() +{ return log_id; } + +//------------------------------------------------------------------------- +// BuiltinExtractorService +//------------------------------------------------------------------------- + +const ServiceBlueprint BuiltinExtractorService::blueprint = +{ + // events + { + "builtin", + }, + // fields + { + "sid", + "gid", + "msg", + "proto", + "source", + }, +}; + +THREAD_LOCAL Connector::ID BuiltinExtractorService::log_id; + +BuiltinExtractorService::BuiltinExtractorService(uint32_t tenant, const std::vector& srv_fields, + const std::vector& srv_events, ServiceType s_type, Extractor& ins) + : ExtractorService(tenant, srv_fields, srv_events, blueprint, s_type, ins) +{ + for (const auto& event : get_events()) + { + if (!strcmp("builtin", event.c_str())) + handlers.push_back(new BuiltinExtractor(ins, tenant_id, get_fields())); + } +} + +const snort::Connector::ID& BuiltinExtractorService::internal_tinit() +{ return log_id = logger->get_id(type.c_str()); } + +const snort::Connector::ID& BuiltinExtractorService::get_log_id() +{ return log_id; } + //------------------------------------------------------------------------- // Unit Tests //------------------------------------------------------------------------- @@ -447,6 +547,8 @@ TEST_CASE("Service Type", "[extractor]") ServiceType ftp = ServiceType::FTP; ServiceType conn = ServiceType::CONN; ServiceType dns = ServiceType::DNS; + ServiceType weird = ServiceType::IPS_BUILTIN; + ServiceType notice = ServiceType::IPS_USER; ServiceType any = ServiceType::ANY; ServiceType max = ServiceType::MAX; @@ -454,6 +556,8 @@ TEST_CASE("Service Type", "[extractor]") CHECK_FALSE(strcmp("ftp", ftp.c_str())); CHECK_FALSE(strcmp("conn", conn.c_str())); CHECK_FALSE(strcmp("dns", dns.c_str())); + CHECK_FALSE(strcmp("weird", weird.c_str())); + CHECK_FALSE(strcmp("notice", notice.c_str())); CHECK_FALSE(strcmp("(not set)", any.c_str())); CHECK_FALSE(strcmp("(not set)", max.c_str())); } diff --git a/src/network_inspectors/extractor/extractor_service.h b/src/network_inspectors/extractor/extractor_service.h index c0bd0896e..e93164a7e 100644 --- a/src/network_inspectors/extractor/extractor_service.h +++ b/src/network_inspectors/extractor/extractor_service.h @@ -152,5 +152,35 @@ private: static THREAD_LOCAL snort::Connector::ID log_id; }; +class BuiltinExtractorService : public ExtractorService +{ +public: + static const ServiceBlueprint blueprint; + + BuiltinExtractorService(uint32_t tenant, const std::vector& fields, + const std::vector& events, ServiceType, Extractor&); + +private: + const snort::Connector::ID& internal_tinit() override; + const snort::Connector::ID& get_log_id() override; + + static THREAD_LOCAL snort::Connector::ID log_id; +}; + +class IpsUserExtractorService : public ExtractorService +{ +public: + static const ServiceBlueprint blueprint; + + IpsUserExtractorService(uint32_t tenant, const std::vector& fields, + const std::vector& events, ServiceType, Extractor&); + +private: + const snort::Connector::ID& internal_tinit() override; + const snort::Connector::ID& get_log_id() override; + + static THREAD_LOCAL snort::Connector::ID log_id; +}; + #endif diff --git a/src/pub_sub/CMakeLists.txt b/src/pub_sub/CMakeLists.txt index 7c8e05c3d..ca205d5ea 100644 --- a/src/pub_sub/CMakeLists.txt +++ b/src/pub_sub/CMakeLists.txt @@ -40,6 +40,7 @@ add_library( pub_sub OBJECT ${PUB_SUB_INCLUDES} cip_events.cc http_events.cc + detection_events.cc dns_events.cc http_request_body_event.cc http_body_event.cc diff --git a/src/pub_sub/detection_events.cc b/src/pub_sub/detection_events.cc new file mode 100644 index 000000000..f5e2ead9c --- /dev/null +++ b/src/pub_sub/detection_events.cc @@ -0,0 +1,87 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// detection_events.cc author Anna Norokh + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include +#include +#include + +#include "detection_events.h" + +using namespace snort; + +static const char* merge_with_colon(const char* first, const char* second) +{ + size_t length = std::strlen(first) + std::strlen(second) + 2; + char* result = new char[length]; + + std::sprintf(result, "%s:%s", first, second); + + return result; +} + +static std::string strip_msg(const char* msg) +{ + std::string str(msg); + if (str.front() == '"' and str.back() == '"') + str = str.substr(1, str.length() - 2); + + return str; +} + +const std::vector& IpsRuleEvent::get_references() const +{ + if (!references.empty()) + return references; + + unsigned idx = 0; + const char* name = nullptr; + const char* id = nullptr; + const char* url = nullptr; + + while (get_reference(idx++, name, id, url)) + { + if (url and *url) + references.push_back(merge_with_colon(url, id)); + else + references.push_back(merge_with_colon(name, id)); + } + + return references; +} + +const std::string& IpsRuleEvent::get_stripped_msg() const +{ + if (stripped_msg.empty()) + stripped_msg = strip_msg(get_msg()); + + return stripped_msg; +} + +const std::string& IpsQueuingEvent::get_stripped_msg() const +{ + if (stripped_msg.empty()) + stripped_msg = strip_msg(get_msg()); + + return stripped_msg; +} diff --git a/src/pub_sub/detection_events.h b/src/pub_sub/detection_events.h index bd6742cad..33dadb897 100644 --- a/src/pub_sub/detection_events.h +++ b/src/pub_sub/detection_events.h @@ -22,6 +22,7 @@ #include "events/event.h" #include "framework/data_bus.h" +#include "protocols/packet.h" namespace snort { @@ -31,6 +32,7 @@ struct DetectionEventIds enum : unsigned { IPS_LOGGING, + BUILTIN, MAX }; }; @@ -40,13 +42,33 @@ const PubKey de_pub_key { "detection", DetectionEventIds::MAX }; class IpsRuleEvent : public DataEvent, public Event { public: - IpsRuleEvent(const Event& e, const Packet* p) : Event(e), p(p) {} + IpsRuleEvent(const Event& e) : Event(e) {} + ~IpsRuleEvent() override + { + for (const char* ref : references) + delete[] ref; + } + + const std::string& get_stripped_msg() const; + + const std::vector& get_references() const; + +protected: + mutable std::vector references; + +private: + mutable std::string stripped_msg; +}; + +class IpsQueuingEvent : public DataEvent, public Event +{ +public: + IpsQueuingEvent(const SigInfo& sig_info) : Event(0, 0, sig_info, nullptr, "") {} - const snort::Packet* get_packet() const override - { return p; } + const std::string& get_stripped_msg() const; private: - const Packet* p; + mutable std::string stripped_msg; }; } diff --git a/src/pub_sub/test/CMakeLists.txt b/src/pub_sub/test/CMakeLists.txt index 9bb74a5af..e1047026e 100644 --- a/src/pub_sub/test/CMakeLists.txt +++ b/src/pub_sub/test/CMakeLists.txt @@ -36,3 +36,7 @@ add_cpputest( pub_sub_ftp_events_test ../ftp_events.h $ ) +add_cpputest( pub_sub_detection_events_test + SOURCES + ../detection_events.cc +) diff --git a/src/pub_sub/test/pub_sub_detection_events_test.cc b/src/pub_sub/test/pub_sub_detection_events_test.cc new file mode 100644 index 000000000..8f96590ec --- /dev/null +++ b/src/pub_sub/test/pub_sub_detection_events_test.cc @@ -0,0 +1,111 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2025 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// http_transaction_test.cc author Anna Norokh + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "detection/signature.h" +#include "pub_sub/detection_events.h" + +#include +#include + +using namespace snort; + +static SigInfo s_dummy; +Event::Event() : sig_info(s_dummy) { } +Event::Event(unsigned int, unsigned int, SigInfo const&, char const**, char const*) : sig_info(s_dummy) { } +bool Event::get_reference(unsigned int, const char*&, const char*&, const char*&) const +{ + return true; +} +const char* Event::get_msg() const +{ + return "\"mock message\""; +} + +class MockIpsRuleEvent : public IpsRuleEvent +{ +public: + MockIpsRuleEvent() : IpsRuleEvent({}) {} + void mock_references() + { + const char* url = new char[strlen("https://example.com") + 1]; + strcpy(const_cast(url), "https://example.com"); + references.push_back(url); + } +}; +class MockIpsQueuingEvent : public IpsQueuingEvent +{ +public: + MockIpsQueuingEvent() : IpsQueuingEvent(s_dummy) {} +}; + +TEST_GROUP(detection_events_test) +{ + MockIpsRuleEvent* mock_ips_rule_event = new MockIpsRuleEvent(); + + void teardown() override + { + delete mock_ips_rule_event; + } +}; + +TEST(detection_events_test, get_references_is_not_empty) +{ + mock_ips_rule_event->mock_references(); + + auto vec = mock_ips_rule_event->get_references(); + + CHECK_FALSE(vec.empty()); + CHECK(1 == vec.size()); + CHECK(strcmp("https://example.com", vec[0]) == 0); +} + +TEST(detection_events_test, get_ips_rule_message_is_not_empty) +{ + // first call initialize the message + auto stripped_msg1 = mock_ips_rule_event->get_stripped_msg(); + CHECK("mock message" == stripped_msg1); + + // check that we got cached message + auto stripped_msg2 = mock_ips_rule_event->get_stripped_msg(); + CHECK("mock message" == stripped_msg2); +} + +TEST(detection_events_test, get_ips_queued_message_is_not_empty) +{ + MockIpsQueuingEvent* ips_queuing_event = new MockIpsQueuingEvent(); + + // first call initialize the message + auto stripped_msg1 = ips_queuing_event->get_stripped_msg(); + CHECK("mock message" == stripped_msg1); + + // check that we got cached message + auto stripped_msg2 = ips_queuing_event->get_stripped_msg(); + CHECK("mock message" == stripped_msg2); + + delete ips_queuing_event; +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +}