From: Oleksii Shumeiko -X (oshumeik - SOFTSERVE INC at Cisco) Date: Tue, 4 Feb 2025 21:29:01 +0000 (+0000) Subject: Pull request #4587: Data log filtering X-Git-Tag: 3.7.0.0~2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=411d6b1dacb4821c9b19e80f9241e8e578f51bff;p=thirdparty%2Fsnort3.git Pull request #4587: Data log filtering Merge in SNORT/snort3 from ~OSHUMEIK/snort3:data_log_filtering to master Squashed commit of the following: commit 5d73e7676db2bb678860ba07607cb840ea6ab516 Author: Oleksii Shumeiko Date: Fri Jan 31 15:12:28 2025 +0200 extractor: rework parsing messages commit 653b4570e28aff6a62fb71dc4d83bc11f881a7c3 Author: Oleksii Shumeiko Date: Fri Jan 31 12:07:38 2025 +0200 extractor: fix subscription to be global As the inspector itself is global, it ought to get events from all policies. commit d60c29383e5c0841f09659dc226dc57e29fe56a8 Author: Oleksii Shumeiko Date: Tue Jan 28 14:51:23 2025 +0200 extractor: add default filter commit 450ba51ae1f3833b7c8f80a38fcf633a768dd319 Author: Oleksii Shumeiko Date: Thu Jan 23 14:41:57 2025 +0200 extractor: export service types Exported ServiceType value renamed to make it more clear for an external module. commit be29879348a0ed24cad06618fe6ec59d62c53bcf Author: Oleksii Shumeiko Date: Mon Nov 18 15:42:46 2024 +0200 extractor: add logging constraints Being a part of logging filtering Tenant ID is not cached. In absence of native filters the extractor sets filtering unconditionally. --- diff --git a/src/flow/flow.h b/src/flow/flow.h index fd17d6454..e6c44be13 100644 --- a/src/flow/flow.h +++ b/src/flow/flow.h @@ -27,6 +27,7 @@ // state. Inspector state is stored in FlowData, and Flow manages a list // of FlowData items. +#include #include #include @@ -534,6 +535,8 @@ public: // FIXIT-M privatize if possible DAQ_Verdict last_verdict = MAX_DAQ_VERDICT; + std::bitset<64> data_log_filtering_state; + private: void clean(); std::atomic_ullong inspection_duration{0}; diff --git a/src/framework/plugins.h b/src/framework/plugins.h index 3694bc540..325782f59 100644 --- a/src/framework/plugins.h +++ b/src/framework/plugins.h @@ -37,7 +37,7 @@ #include "framework/policy_selector.h" #include "framework/so_rule.h" -// forward decls we must explicitly include here to +// forward decls we must explicitly include here to // generate the complete set of API dependencies: #include "flow/flow.h" diff --git a/src/network_inspectors/extractor/CMakeLists.txt b/src/network_inspectors/extractor/CMakeLists.txt index cd6dbe0aa..23316a73b 100644 --- a/src/network_inspectors/extractor/CMakeLists.txt +++ b/src/network_inspectors/extractor/CMakeLists.txt @@ -1,3 +1,11 @@ +set ( INCLUDES + extractor_enums.h +) + +install ( FILES ${INCLUDES} + DESTINATION "${INCLUDE_INSTALL_PATH}/network_inspectors/extractor" +) + set( FILE_LIST extractor.cc extractor.h diff --git a/src/network_inspectors/extractor/dev_notes.txt b/src/network_inspectors/extractor/dev_notes.txt index 2472c97a4..40f17ebc5 100644 --- a/src/network_inspectors/extractor/dev_notes.txt +++ b/src/network_inspectors/extractor/dev_notes.txt @@ -116,9 +116,18 @@ Filtering helps to decrease the amount of traffic being logged. The goal is to keep performance overhead low. The check action is performed as early as possible, at the beginning of each event handling function. -Currently, filtering by tenant ID is supported. +Filtering may be performed by external modules or by extractor itself. +The module stores filtering results on the flow, so it becomes +cached for the flow. However, tenant ID filter is not cached. -*(Filtering by IP and port ranges yet to be implemented)* +To override extractor's filtering, an external module sets `ServiceType::ANY` +bit in the filtering item. + +For `extractor.protocols` entries which don't set filtering items, +`extractor.default_filter` action is applied (until an external module +re-computes filtering for the flow). + +*(IP and port filtering by extractor yet to be implemented)* ==== Extracting Data diff --git a/src/network_inspectors/extractor/extractor.cc b/src/network_inspectors/extractor/extractor.cc index f7050b7b3..c63a94b4e 100644 --- a/src/network_inspectors/extractor/extractor.cc +++ b/src/network_inspectors/extractor/extractor.cc @@ -73,6 +73,9 @@ static const Parameter s_params[] = { "connector", Parameter::PT_STRING, nullptr, nullptr, "output destination for extractor" }, + { "default_filter", Parameter::PT_ENUM, "pick | skip", "pick", + "default action for protocol with no filter provided" }, + { "protocols", Parameter::PT_LIST, extractor_proto_params, nullptr, "protocols to extract data" }, @@ -81,7 +84,7 @@ static const Parameter s_params[] = void ServiceConfig::clear() { - service = ServiceType::UNDEFINED; + service = ServiceType::ANY; on_events.clear(); tenant_id = 0; fields.clear(); @@ -138,6 +141,9 @@ bool ExtractorModule::set(const char*, Value& v, SnortConfig*) else if (v.is("connector")) extractor_config.output_conn = v.get_string(); + if (v.is("default_filter")) + extractor_config.pick_by_default = v.get_uint8() == 0; + else if (v.is("service")) service_config.service = (ServiceType)(v.get_uint8()); @@ -184,7 +190,7 @@ public: inspector.logger->flush(); delete inspector.logger; - inspector.logger = ExtractorLogger::make_logger(inspector.format, inspector.output_conn); + inspector.logger = ExtractorLogger::make_logger(inspector.cfg.formatting, inspector.cfg.output_conn); for (auto& s : inspector.services) s->tinit(inspector.logger); @@ -195,19 +201,10 @@ private: }; Extractor::Extractor(ExtractorModule* m) + : cfg(m->get_config()) { - auto& cfg = m->get_config(); - - format = cfg.formatting; - output_conn = cfg.output_conn; - for (const auto& p : cfg.protocols) - { - auto s = ExtractorService::make_service(*this, p); - - if (s) - services.push_back(s); - } + ExtractorService::validate(p); } Extractor::~Extractor() @@ -216,14 +213,25 @@ Extractor::~Extractor() delete s; } -bool Extractor::configure(SnortConfig*) +bool Extractor::configure(SnortConfig* sc) { - Connector::Direction mode = ConnectorManager::is_instantiated(output_conn); + assert(sc); + snort_config = sc; + + for (const auto& p : cfg.protocols) + { + auto s = ExtractorService::make_service(*this, p); + + if (s) + services.push_back(s); + } + + Connector::Direction mode = ConnectorManager::is_instantiated(cfg.output_conn); if (mode != Connector::CONN_TRANSMIT and mode != Connector::CONN_DUPLEX) { ParseError("can't initialize extractor, cannot find Connector \"%s\" in transmit mode.\n", - output_conn.c_str()); + cfg.output_conn.c_str()); return false; } @@ -232,8 +240,9 @@ bool Extractor::configure(SnortConfig*) void Extractor::show(const SnortConfig*) const { - ConfigLogger::log_value("formatting", format.c_str()); - ConfigLogger::log_value("connector", output_conn.c_str()); + ConfigLogger::log_value("formatting", cfg.formatting.c_str()); + ConfigLogger::log_value("connector", cfg.output_conn.c_str()); + ConfigLogger::log_value("pick_by_default", cfg.pick_by_default ? "pick" : "skip"); bool log_header = true; for (const auto& s : services) @@ -252,7 +261,7 @@ void Extractor::show(const SnortConfig*) const void Extractor::tinit() { - logger = ExtractorLogger::make_logger(format, output_conn); + logger = ExtractorLogger::make_logger(cfg.formatting, cfg.output_conn); for (auto& s : services) s->tinit(logger); diff --git a/src/network_inspectors/extractor/extractor.h b/src/network_inspectors/extractor/extractor.h index 8098b34b1..e55d6d77f 100644 --- a/src/network_inspectors/extractor/extractor.h +++ b/src/network_inspectors/extractor/extractor.h @@ -38,7 +38,7 @@ class ServiceConfig { public: - ServiceConfig() : service(ServiceType::UNDEFINED), tenant_id(0) {} + ServiceConfig() : service(ServiceType::ANY), tenant_id(0) {} void clear(); ServiceType service; @@ -51,6 +51,7 @@ struct ExtractorConfig { FormatType formatting = FormatType::CSV; std::string output_conn; + bool pick_by_default = true; std::vector protocols; }; @@ -116,10 +117,16 @@ public: void tterm() override; void install_reload_handler(snort::SnortConfig*) override; + snort::SnortConfig& get_snort_config() const + { return snort_config ? *snort_config : *snort::SnortConfig::get_main_conf(); } + + bool get_default_filter() const + { return cfg.pick_by_default; } + private: + snort::SnortConfig* snort_config = nullptr; + ExtractorConfig cfg; std::vector services; - FormatType format; - std::string output_conn; static THREAD_LOCAL ExtractorLogger* logger; friend class ExtractorReloadSwapper; diff --git a/src/network_inspectors/extractor/extractor_conn.cc b/src/network_inspectors/extractor/extractor_conn.cc index 64c6ec7fb..8d7c4fd3a 100644 --- a/src/network_inspectors/extractor/extractor_conn.cc +++ b/src/network_inspectors/extractor/extractor_conn.cc @@ -88,7 +88,7 @@ static const map sub_buf_getters = THREAD_LOCAL const snort::Connector::ID* ConnExtractor::log_id = nullptr; ConnExtractor::ConnExtractor(Extractor& i, uint32_t t, const vector& fields) - : ExtractorEvent(i, t) + : ExtractorEvent(ServiceType::CONN, i, t) { for (const auto& f : fields) { @@ -104,7 +104,8 @@ ConnExtractor::ConnExtractor(Extractor& i, uint32_t t, const vector& fie continue; } - DataBus::subscribe(intrinsic_pub_key, IntrinsicEventIds::FLOW_END, new Eof(*this, S_NAME)); + DataBus::subscribe_global(intrinsic_pub_key, IntrinsicEventIds::FLOW_END, + new Eof(*this, S_NAME), i.get_snort_config()); } void ConnExtractor::internal_tinit(const snort::Connector::ID* service_id) @@ -115,16 +116,7 @@ void ConnExtractor::handle(DataEvent& event, Flow* flow) // cppcheck-suppress unreadVariable Profile profile(extractor_perf_stats); - uint32_t tid = 0; - - if ((flow->pkt_type < PktType::IP) or (flow->pkt_type > PktType::ICMP)) - return; - -#ifndef DISABLE_TENANT_ID - tid = flow->key->tenant_id; -#endif - - if (tenant_id != tid) + if (flow->pkt_type < PktType::IP or flow->pkt_type > PktType::ICMP or !filter(flow)) return; Packet* packet = (DetectionEngine::get_context()) ? DetectionEngine::get_current_packet() : nullptr; @@ -156,7 +148,7 @@ TEST_CASE("Conn Proto", "[extractor]") set_inspection_policy(&ins); NetworkPolicy net; set_network_policy(&net); - + SECTION("unknown") { flow->pkt_type = PktType::NONE; diff --git a/src/network_inspectors/extractor/extractor_enums.h b/src/network_inspectors/extractor/extractor_enums.h index c53838713..95deb8128 100644 --- a/src/network_inspectors/extractor/extractor_enums.h +++ b/src/network_inspectors/extractor/extractor_enums.h @@ -30,7 +30,7 @@ public: HTTP, FTP, CONN, - UNDEFINED, + ANY, MAX }; @@ -51,15 +51,15 @@ public: return "ftp"; case CONN: return "conn"; - case UNDEFINED: // fallthrough - case MAX: // fallthrough + case ANY: // fallthrough + case MAX: // fallthrough default: return "(not set)"; } } private: - Value v = UNDEFINED; + Value v = ANY; }; class FormatType diff --git a/src/network_inspectors/extractor/extractor_ftp.cc b/src/network_inspectors/extractor/extractor_ftp.cc index 06b0f8587..ac9d61370 100644 --- a/src/network_inspectors/extractor/extractor_ftp.cc +++ b/src/network_inspectors/extractor/extractor_ftp.cc @@ -78,7 +78,7 @@ static const map sub_str_getters = THREAD_LOCAL const snort::Connector::ID* FtpRequestExtractor::log_id = nullptr; FtpRequestExtractor::FtpRequestExtractor(Extractor& i, uint32_t t, const vector& fields) : - ExtractorEvent(i, t) + ExtractorEvent(ServiceType::FTP, i, t) { for (const auto& f : fields) { @@ -92,7 +92,8 @@ FtpRequestExtractor::FtpRequestExtractor(Extractor& i, uint32_t t, const vector< continue; } - DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_REQUEST, new Req(*this, S_NAME)); + DataBus::subscribe_global(ftp_pub_key, FtpEventIds::FTP_REQUEST, + new Req(*this, S_NAME), i.get_snort_config()); } void FtpRequestExtractor::internal_tinit(const snort::Connector::ID* service_id) @@ -103,14 +104,8 @@ void FtpRequestExtractor::handle(DataEvent& event, Flow* flow) // cppcheck-suppress unreadVariable Profile profile(extractor_perf_stats); - uint32_t tid = 0; - -#ifndef DISABLE_TENANT_ID - tid = flow->key->tenant_id; -#endif - - if (tenant_id != tid) - return; + if (!filter(flow)) + return; extractor_stats.total_event++; @@ -225,7 +220,7 @@ static const map sub_getters = THREAD_LOCAL const snort::Connector::ID* FtpResponseExtractor::log_id = nullptr; FtpResponseExtractor::FtpResponseExtractor(Extractor& i, uint32_t t, const vector& fields) : - ExtractorEvent(i, t) + ExtractorEvent(ServiceType::FTP, i, t) { for (const auto& f : fields) { @@ -245,7 +240,8 @@ FtpResponseExtractor::FtpResponseExtractor(Extractor& i, uint32_t t, const vecto continue; } - DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_RESPONSE, new Resp(*this, S_NAME)); + DataBus::subscribe_global(ftp_pub_key, FtpEventIds::FTP_RESPONSE, + new Resp(*this, S_NAME), i.get_snort_config()); } void FtpResponseExtractor::internal_tinit(const snort::Connector::ID* service_id) @@ -270,14 +266,8 @@ void FtpResponseExtractor::handle(DataEvent& event, Flow* flow) // cppcheck-suppress unreadVariable Profile profile(extractor_perf_stats); - uint32_t tid = 0; - -#ifndef DISABLE_TENANT_ID - tid = flow->key->tenant_id; -#endif - - if (tenant_id != tid) - return; + if (!filter(flow)) + return; extractor_stats.total_event++; @@ -419,7 +409,7 @@ static const map fd_sub_getters = THREAD_LOCAL const snort::Connector::ID* FtpExtractor::log_id = nullptr; FtpExtractor::FtpExtractor(Extractor& i, uint32_t t, const vector& fields) : - ExtractorEvent(i, t) + ExtractorEvent(ServiceType::FTP, i, t) { for (const auto& f : fields) { @@ -439,8 +429,10 @@ FtpExtractor::FtpExtractor(Extractor& i, uint32_t t, const vector& field continue; } - DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_REQUEST, new Req(*this, S_NAME)); - DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_RESPONSE, new Resp(*this, S_NAME)); + DataBus::subscribe_global(ftp_pub_key, FtpEventIds::FTP_REQUEST, + new Req(*this, S_NAME), i.get_snort_config()); + DataBus::subscribe_global(ftp_pub_key, FtpEventIds::FTP_RESPONSE, + new Resp(*this, S_NAME), i.get_snort_config()); } void FtpExtractor::internal_tinit(const snort::Connector::ID* service_id) @@ -535,13 +527,7 @@ void FtpExtractor::Req::handle(DataEvent& event, Flow* flow) // cppcheck-suppress unreadVariable Profile profile(extractor_perf_stats); - uint32_t tid = 0; - -#ifndef DISABLE_TENANT_ID - tid = flow->key->tenant_id; -#endif - - if (owner.tenant_id != tid) + if (!owner.filter(flow)) return; extractor_stats.total_event++; @@ -592,13 +578,7 @@ void FtpExtractor::Resp::handle(DataEvent& event, Flow* flow) // cppcheck-suppress unreadVariable Profile profile(extractor_perf_stats); - uint32_t tid = 0; - -#ifndef DISABLE_TENANT_ID - tid = flow->key->tenant_id; -#endif - - if (owner.tenant_id != tid) + if (!owner.filter(flow)) return; extractor_stats.total_event++; diff --git a/src/network_inspectors/extractor/extractor_http.cc b/src/network_inspectors/extractor/extractor_http.cc index 883c43d80..6ae7d3c4d 100644 --- a/src/network_inspectors/extractor/extractor_http.cc +++ b/src/network_inspectors/extractor/extractor_http.cc @@ -157,7 +157,7 @@ static const map sub_getters = THREAD_LOCAL const snort::Connector::ID* HttpExtractor::log_id = nullptr; HttpExtractor::HttpExtractor(Extractor& i, uint32_t t, const vector& fields) - : ExtractorEvent(i, t) + : ExtractorEvent(ServiceType::HTTP, i, t) { for (const auto& f : fields) { @@ -175,7 +175,8 @@ HttpExtractor::HttpExtractor(Extractor& i, uint32_t t, const vector& fie continue; } - DataBus::subscribe(http_pub_key, HttpEventIds::END_OF_TRANSACTION, new Eot(*this, S_NAME)); + DataBus::subscribe_global(http_pub_key, HttpEventIds::END_OF_TRANSACTION, + new Eot(*this, S_NAME), i.get_snort_config()); } void HttpExtractor::internal_tinit(const snort::Connector::ID* service_id) @@ -200,13 +201,7 @@ void HttpExtractor::handle(DataEvent& event, Flow* flow) // cppcheck-suppress unreadVariable Profile profile(extractor_perf_stats); - uint32_t tid = 0; - -#ifndef DISABLE_TENANT_ID - tid = flow->key->tenant_id; -#endif - - if (tenant_id != tid) + if (!filter(flow)) return; extractor_stats.total_event++; diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc index 85746507a..19658404d 100644 --- a/src/network_inspectors/extractor/extractor_service.cc +++ b/src/network_inspectors/extractor/extractor_service.cc @@ -87,8 +87,6 @@ void ExtractorService::add_events(const std::vector& vals) { if (find_event(val)) events.push_back(val); - else - ParseWarning(WARN_CONF_STRICT, "unsupported '%s' event in protocols.on_events", val.c_str()); } } @@ -98,8 +96,6 @@ void ExtractorService::add_fields(const std::vector& vals) { if (find_field(val)) fields.push_back(val); - else - ParseWarning(WARN_CONF_STRICT, "unsupported '%s' field in protocols.fields", val.c_str()); } } @@ -107,7 +103,7 @@ ExtractorService* ExtractorService::make_service(Extractor& ins, const ServiceCo { if (cfg.on_events.empty()) { - ParseError("%s service misses on_events field", cfg.service.c_str()); + ErrorMessage("Extractor: %s service misses on_events field\n", cfg.service.c_str()); return nullptr; } @@ -127,9 +123,9 @@ ExtractorService* ExtractorService::make_service(Extractor& ins, const ServiceCo srv = new ConnExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, ins); break; - case ServiceType::UNDEFINED: // fallthrough + case ServiceType::ANY: // fallthrough default: - ParseError("'%s' service is not supported", cfg.service.c_str()); + ErrorMessage("Extractor: '%s' service is not supported\n", cfg.service.c_str()); } return srv; @@ -137,15 +133,12 @@ ExtractorService* ExtractorService::make_service(Extractor& ins, const ServiceCo bool ExtractorService::find_event(const std::string& event) const { - return std::find(sbp.supported_events.begin(), sbp.supported_events.end(), event) - != sbp.supported_events.end(); + return find_event(sbp, event); } bool ExtractorService::find_field(const std::string& field) const { - return ((std::find(common_fields.begin(), common_fields.end(), field) != common_fields.end()) or - (std::find(sbp.supported_fields.begin(), sbp.supported_fields.end(),field) - != sbp.supported_fields.end())); + return find_field(sbp, field); } void ExtractorService::show(std::string& str) const @@ -169,11 +162,70 @@ void ExtractorService::show(std::string& str) const str += " }"; } +bool ExtractorService::find_event(const ServiceBlueprint& sbp, const std::string& event) +{ + return std::find(sbp.supported_events.begin(), sbp.supported_events.end(), event) + != sbp.supported_events.end(); +} + +bool ExtractorService::find_field(const ServiceBlueprint& sbp, const std::string& field) +{ + return ((std::find(common_fields.begin(), common_fields.end(), field) != common_fields.end()) + or (std::find(sbp.supported_fields.begin(), sbp.supported_fields.end(),field) + != sbp.supported_fields.end())); +} + +void ExtractorService::validate_events(const ServiceBlueprint& sbp, const std::vector& vals) +{ + for (const auto& val : vals) + { + if (!find_event(sbp, val)) + ParseWarning(WARN_CONF_STRICT, "unsupported '%s' event in protocols.on_events", val.c_str()); + } +} + +void ExtractorService::validate_fields(const ServiceBlueprint& sbp, const std::vector& vals) +{ + for (auto& val : vals) + { + if (!find_field(sbp, val)) + ParseWarning(WARN_CONF_STRICT, "unsupported '%s' field in protocols.fields\n", val.c_str()); + } +} + +void ExtractorService::validate(const ServiceConfig& cfg) +{ + if (cfg.on_events.empty()) + ParseError("%s service misses on_events field", cfg.service.c_str()); + + switch (cfg.service) + { + case ServiceType::HTTP: + validate_events(HttpExtractorService::blueprint, cfg.on_events); + validate_fields(HttpExtractorService::blueprint, cfg.fields); + break; + + case ServiceType::FTP: + validate_events(FtpExtractorService::blueprint, cfg.on_events); + validate_fields(FtpExtractorService::blueprint, cfg.fields); + break; + + case ServiceType::CONN: + validate_events(ConnExtractorService::blueprint, cfg.on_events); + validate_fields(ConnExtractorService::blueprint, cfg.fields); + break; + + case ServiceType::ANY: // fallthrough + default: + ParseError("'%s' service is not supported", cfg.service.c_str()); + } +} + //------------------------------------------------------------------------- // HttpExtractorService //------------------------------------------------------------------------- -ServiceBlueprint HttpExtractorService::blueprint = +const ServiceBlueprint HttpExtractorService::blueprint = { // events { @@ -224,7 +276,7 @@ const snort::Connector::ID& HttpExtractorService::get_log_id() // FtpExtractorService //------------------------------------------------------------------------- -ServiceBlueprint FtpExtractorService::blueprint = +const ServiceBlueprint FtpExtractorService::blueprint = { // events { @@ -274,7 +326,7 @@ const snort::Connector::ID& FtpExtractorService::get_log_id() // ConnExtractorService //------------------------------------------------------------------------- -ServiceBlueprint ConnExtractorService::blueprint = +const ServiceBlueprint ConnExtractorService::blueprint = { // events { @@ -326,13 +378,13 @@ TEST_CASE("Service Type", "[extractor]") ServiceType http = ServiceType::HTTP; ServiceType ftp = ServiceType::FTP; ServiceType conn = ServiceType::CONN; - ServiceType undef = ServiceType::UNDEFINED; + ServiceType any = ServiceType::ANY; ServiceType max = ServiceType::MAX; CHECK_FALSE(strcmp("http", http.c_str())); CHECK_FALSE(strcmp("ftp", ftp.c_str())); CHECK_FALSE(strcmp("conn", conn.c_str())); - CHECK_FALSE(strcmp("(not set)", undef.c_str())); + CHECK_FALSE(strcmp("(not set)", any.c_str())); CHECK_FALSE(strcmp("(not set)", max.c_str())); } } diff --git a/src/network_inspectors/extractor/extractor_service.h b/src/network_inspectors/extractor/extractor_service.h index 91a0625b9..2e4ebc205 100644 --- a/src/network_inspectors/extractor/extractor_service.h +++ b/src/network_inspectors/extractor/extractor_service.h @@ -43,6 +43,7 @@ struct ServiceBlueprint class ExtractorService { public: + static void validate(const ServiceConfig&); static ExtractorService* make_service(Extractor&, const ServiceConfig&); ExtractorService() = delete; @@ -83,11 +84,19 @@ protected: const ServiceBlueprint& sbp; const ServiceType type; + +private: + static void validate_events(const ServiceBlueprint&, const std::vector& vals); + static void validate_fields(const ServiceBlueprint&, const std::vector& vals); + static bool find_event(const ServiceBlueprint&, const std::string&); + static bool find_field(const ServiceBlueprint&, const std::string&); }; class HttpExtractorService : public ExtractorService { public: + static const ServiceBlueprint blueprint; + HttpExtractorService(uint32_t tenant, const std::vector& fields, const std::vector& events, ServiceType, Extractor&); @@ -95,13 +104,14 @@ private: const snort::Connector::ID& internal_tinit() override; const snort::Connector::ID& get_log_id() override; - static ServiceBlueprint blueprint; static THREAD_LOCAL snort::Connector::ID log_id; }; class FtpExtractorService : public ExtractorService { public: + static const ServiceBlueprint blueprint; + FtpExtractorService(uint32_t tenant, const std::vector& fields, const std::vector& events, ServiceType, Extractor&); @@ -109,13 +119,14 @@ private: const snort::Connector::ID& internal_tinit() override; const snort::Connector::ID& get_log_id() override; - static ServiceBlueprint blueprint; static THREAD_LOCAL snort::Connector::ID log_id; }; class ConnExtractorService : public ExtractorService { public: + static const ServiceBlueprint blueprint; + ConnExtractorService(uint32_t tenant, const std::vector& fields, const std::vector& events, ServiceType, Extractor&); @@ -123,7 +134,6 @@ private: const snort::Connector::ID& internal_tinit() override; const snort::Connector::ID& get_log_id() override; - static ServiceBlueprint blueprint; static THREAD_LOCAL snort::Connector::ID log_id; }; diff --git a/src/network_inspectors/extractor/extractors.h b/src/network_inspectors/extractor/extractors.h index 3e344cd90..dd46ad8f2 100644 --- a/src/network_inspectors/extractor/extractors.h +++ b/src/network_inspectors/extractor/extractors.h @@ -30,10 +30,10 @@ #include "sfip/sf_ip.h" #include "time/packet_time.h" +#include "extractor.h" +#include "extractor_enums.h" #include "extractor_logger.h" -class Extractor; - template struct DataField { @@ -144,14 +144,19 @@ protected: return it != map.end(); } - ExtractorEvent(Extractor& i, uint32_t tid) : tenant_id(tid), inspector(i) - { } + inline bool filter(Flow*); + + ExtractorEvent(ServiceType st, Extractor& i, uint32_t tid) + : service_type(st), pick_by_default(i.get_default_filter()), tenant_id(tid), inspector(i) { } virtual void internal_tinit(const snort::Connector::ID*) = 0; + static THREAD_LOCAL ExtractorLogger* logger; + + ServiceType service_type; + bool pick_by_default; uint32_t tenant_id; Extractor& inspector; - static THREAD_LOCAL ExtractorLogger* logger; std::vector nts_fields; std::vector sip_fields; @@ -164,4 +169,31 @@ protected: static const std::map num_getters; }; +bool ExtractorEvent::filter(Flow* flow) +{ + if (!flow) + return false; + + auto& filter = flow->data_log_filtering_state; + assert(filter.size() >= ServiceType::MAX); + +#ifdef DISABLE_TENANT_ID + uint32_t tid = 0; +#else + uint32_t tid = flow->key->tenant_id; +#endif + + // computed by external filter + if (filter.test(ServiceType::ANY)) + return filter.test(service_type) and tenant_id == tid; + + if (!pick_by_default) + return false; + + // extractor sets targeted filtering + filter.set(service_type); + + return filter.test(service_type) and tenant_id == tid; +} + #endif