// state. Inspector state is stored in FlowData, and Flow manages a list
// of FlowData items.
+#include <bitset>
#include <string>
#include <sys/time.h>
DAQ_Verdict last_verdict = MAX_DAQ_VERDICT;
+ std::bitset<64> data_log_filtering_state;
+
private:
void clean();
std::atomic_ullong inspection_duration{0};
#include "framework/policy_selector.h"
#include "framework/so_rule.h"
-// forward decls we must explicitly include here to
+// forward decls we must explicitly include here to
// generate the complete set of API dependencies:
#include "flow/flow.h"
+set ( INCLUDES
+ extractor_enums.h
+)
+
+install ( FILES ${INCLUDES}
+ DESTINATION "${INCLUDE_INSTALL_PATH}/network_inspectors/extractor"
+)
+
set( FILE_LIST
extractor.cc
extractor.h
keep performance overhead low. The check action is performed as early as
possible, at the beginning of each event handling function.
-Currently, filtering by tenant ID is supported.
+Filtering may be performed by external modules or by extractor itself.
+The module stores filtering results on the flow, so it becomes
+cached for the flow. However, tenant ID filter is not cached.
-*(Filtering by IP and port ranges yet to be implemented)*
+To override extractor's filtering, an external module sets `ServiceType::ANY`
+bit in the filtering item.
+
+For `extractor.protocols` entries which don't set filtering items,
+`extractor.default_filter` action is applied (until an external module
+re-computes filtering for the flow).
+
+*(IP and port filtering by extractor yet to be implemented)*
==== Extracting Data
{ "connector", Parameter::PT_STRING, nullptr, nullptr,
"output destination for extractor" },
+ { "default_filter", Parameter::PT_ENUM, "pick | skip", "pick",
+ "default action for protocol with no filter provided" },
+
{ "protocols", Parameter::PT_LIST, extractor_proto_params, nullptr,
"protocols to extract data" },
void ServiceConfig::clear()
{
- service = ServiceType::UNDEFINED;
+ service = ServiceType::ANY;
on_events.clear();
tenant_id = 0;
fields.clear();
else if (v.is("connector"))
extractor_config.output_conn = v.get_string();
+ if (v.is("default_filter"))
+ extractor_config.pick_by_default = v.get_uint8() == 0;
+
else if (v.is("service"))
service_config.service = (ServiceType)(v.get_uint8());
inspector.logger->flush();
delete inspector.logger;
- inspector.logger = ExtractorLogger::make_logger(inspector.format, inspector.output_conn);
+ inspector.logger = ExtractorLogger::make_logger(inspector.cfg.formatting, inspector.cfg.output_conn);
for (auto& s : inspector.services)
s->tinit(inspector.logger);
};
Extractor::Extractor(ExtractorModule* m)
+ : cfg(m->get_config())
{
- auto& cfg = m->get_config();
-
- format = cfg.formatting;
- output_conn = cfg.output_conn;
-
for (const auto& p : cfg.protocols)
- {
- auto s = ExtractorService::make_service(*this, p);
-
- if (s)
- services.push_back(s);
- }
+ ExtractorService::validate(p);
}
Extractor::~Extractor()
delete s;
}
-bool Extractor::configure(SnortConfig*)
+bool Extractor::configure(SnortConfig* sc)
{
- Connector::Direction mode = ConnectorManager::is_instantiated(output_conn);
+ assert(sc);
+ snort_config = sc;
+
+ for (const auto& p : cfg.protocols)
+ {
+ auto s = ExtractorService::make_service(*this, p);
+
+ if (s)
+ services.push_back(s);
+ }
+
+ Connector::Direction mode = ConnectorManager::is_instantiated(cfg.output_conn);
if (mode != Connector::CONN_TRANSMIT and mode != Connector::CONN_DUPLEX)
{
ParseError("can't initialize extractor, cannot find Connector \"%s\" in transmit mode.\n",
- output_conn.c_str());
+ cfg.output_conn.c_str());
return false;
}
void Extractor::show(const SnortConfig*) const
{
- ConfigLogger::log_value("formatting", format.c_str());
- ConfigLogger::log_value("connector", output_conn.c_str());
+ ConfigLogger::log_value("formatting", cfg.formatting.c_str());
+ ConfigLogger::log_value("connector", cfg.output_conn.c_str());
+ ConfigLogger::log_value("pick_by_default", cfg.pick_by_default ? "pick" : "skip");
bool log_header = true;
for (const auto& s : services)
void Extractor::tinit()
{
- logger = ExtractorLogger::make_logger(format, output_conn);
+ logger = ExtractorLogger::make_logger(cfg.formatting, cfg.output_conn);
for (auto& s : services)
s->tinit(logger);
class ServiceConfig
{
public:
- ServiceConfig() : service(ServiceType::UNDEFINED), tenant_id(0) {}
+ ServiceConfig() : service(ServiceType::ANY), tenant_id(0) {}
void clear();
ServiceType service;
{
FormatType formatting = FormatType::CSV;
std::string output_conn;
+ bool pick_by_default = true;
std::vector<ServiceConfig> protocols;
};
void tterm() override;
void install_reload_handler(snort::SnortConfig*) override;
+ snort::SnortConfig& get_snort_config() const
+ { return snort_config ? *snort_config : *snort::SnortConfig::get_main_conf(); }
+
+ bool get_default_filter() const
+ { return cfg.pick_by_default; }
+
private:
+ snort::SnortConfig* snort_config = nullptr;
+ ExtractorConfig cfg;
std::vector<ExtractorService*> services;
- FormatType format;
- std::string output_conn;
static THREAD_LOCAL ExtractorLogger* logger;
friend class ExtractorReloadSwapper;
THREAD_LOCAL const snort::Connector::ID* ConnExtractor::log_id = nullptr;
ConnExtractor::ConnExtractor(Extractor& i, uint32_t t, const vector<string>& fields)
- : ExtractorEvent(i, t)
+ : ExtractorEvent(ServiceType::CONN, i, t)
{
for (const auto& f : fields)
{
continue;
}
- DataBus::subscribe(intrinsic_pub_key, IntrinsicEventIds::FLOW_END, new Eof(*this, S_NAME));
+ DataBus::subscribe_global(intrinsic_pub_key, IntrinsicEventIds::FLOW_END,
+ new Eof(*this, S_NAME), i.get_snort_config());
}
void ConnExtractor::internal_tinit(const snort::Connector::ID* service_id)
// cppcheck-suppress unreadVariable
Profile profile(extractor_perf_stats);
- uint32_t tid = 0;
-
- if ((flow->pkt_type < PktType::IP) or (flow->pkt_type > PktType::ICMP))
- return;
-
-#ifndef DISABLE_TENANT_ID
- tid = flow->key->tenant_id;
-#endif
-
- if (tenant_id != tid)
+ if (flow->pkt_type < PktType::IP or flow->pkt_type > PktType::ICMP or !filter(flow))
return;
Packet* packet = (DetectionEngine::get_context()) ? DetectionEngine::get_current_packet() : nullptr;
set_inspection_policy(&ins);
NetworkPolicy net;
set_network_policy(&net);
-
+
SECTION("unknown")
{
flow->pkt_type = PktType::NONE;
HTTP,
FTP,
CONN,
- UNDEFINED,
+ ANY,
MAX
};
return "ftp";
case CONN:
return "conn";
- case UNDEFINED: // fallthrough
- case MAX: // fallthrough
+ case ANY: // fallthrough
+ case MAX: // fallthrough
default:
return "(not set)";
}
}
private:
- Value v = UNDEFINED;
+ Value v = ANY;
};
class FormatType
THREAD_LOCAL const snort::Connector::ID* FtpRequestExtractor::log_id = nullptr;
FtpRequestExtractor::FtpRequestExtractor(Extractor& i, uint32_t t, const vector<string>& fields) :
- ExtractorEvent(i, t)
+ ExtractorEvent(ServiceType::FTP, i, t)
{
for (const auto& f : fields)
{
continue;
}
- DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_REQUEST, new Req(*this, S_NAME));
+ DataBus::subscribe_global(ftp_pub_key, FtpEventIds::FTP_REQUEST,
+ new Req(*this, S_NAME), i.get_snort_config());
}
void FtpRequestExtractor::internal_tinit(const snort::Connector::ID* service_id)
// cppcheck-suppress unreadVariable
Profile profile(extractor_perf_stats);
- uint32_t tid = 0;
-
-#ifndef DISABLE_TENANT_ID
- tid = flow->key->tenant_id;
-#endif
-
- if (tenant_id != tid)
- return;
+ if (!filter(flow))
+ return;
extractor_stats.total_event++;
THREAD_LOCAL const snort::Connector::ID* FtpResponseExtractor::log_id = nullptr;
FtpResponseExtractor::FtpResponseExtractor(Extractor& i, uint32_t t, const vector<string>& fields) :
- ExtractorEvent(i, t)
+ ExtractorEvent(ServiceType::FTP, i, t)
{
for (const auto& f : fields)
{
continue;
}
- DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_RESPONSE, new Resp(*this, S_NAME));
+ DataBus::subscribe_global(ftp_pub_key, FtpEventIds::FTP_RESPONSE,
+ new Resp(*this, S_NAME), i.get_snort_config());
}
void FtpResponseExtractor::internal_tinit(const snort::Connector::ID* service_id)
// cppcheck-suppress unreadVariable
Profile profile(extractor_perf_stats);
- uint32_t tid = 0;
-
-#ifndef DISABLE_TENANT_ID
- tid = flow->key->tenant_id;
-#endif
-
- if (tenant_id != tid)
- return;
+ if (!filter(flow))
+ return;
extractor_stats.total_event++;
THREAD_LOCAL const snort::Connector::ID* FtpExtractor::log_id = nullptr;
FtpExtractor::FtpExtractor(Extractor& i, uint32_t t, const vector<string>& fields) :
- ExtractorEvent(i, t)
+ ExtractorEvent(ServiceType::FTP, i, t)
{
for (const auto& f : fields)
{
continue;
}
- DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_REQUEST, new Req(*this, S_NAME));
- DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_RESPONSE, new Resp(*this, S_NAME));
+ DataBus::subscribe_global(ftp_pub_key, FtpEventIds::FTP_REQUEST,
+ new Req(*this, S_NAME), i.get_snort_config());
+ DataBus::subscribe_global(ftp_pub_key, FtpEventIds::FTP_RESPONSE,
+ new Resp(*this, S_NAME), i.get_snort_config());
}
void FtpExtractor::internal_tinit(const snort::Connector::ID* service_id)
// cppcheck-suppress unreadVariable
Profile profile(extractor_perf_stats);
- uint32_t tid = 0;
-
-#ifndef DISABLE_TENANT_ID
- tid = flow->key->tenant_id;
-#endif
-
- if (owner.tenant_id != tid)
+ if (!owner.filter(flow))
return;
extractor_stats.total_event++;
// cppcheck-suppress unreadVariable
Profile profile(extractor_perf_stats);
- uint32_t tid = 0;
-
-#ifndef DISABLE_TENANT_ID
- tid = flow->key->tenant_id;
-#endif
-
- if (owner.tenant_id != tid)
+ if (!owner.filter(flow))
return;
extractor_stats.total_event++;
THREAD_LOCAL const snort::Connector::ID* HttpExtractor::log_id = nullptr;
HttpExtractor::HttpExtractor(Extractor& i, uint32_t t, const vector<string>& fields)
- : ExtractorEvent(i, t)
+ : ExtractorEvent(ServiceType::HTTP, i, t)
{
for (const auto& f : fields)
{
continue;
}
- DataBus::subscribe(http_pub_key, HttpEventIds::END_OF_TRANSACTION, new Eot(*this, S_NAME));
+ DataBus::subscribe_global(http_pub_key, HttpEventIds::END_OF_TRANSACTION,
+ new Eot(*this, S_NAME), i.get_snort_config());
}
void HttpExtractor::internal_tinit(const snort::Connector::ID* service_id)
// cppcheck-suppress unreadVariable
Profile profile(extractor_perf_stats);
- uint32_t tid = 0;
-
-#ifndef DISABLE_TENANT_ID
- tid = flow->key->tenant_id;
-#endif
-
- if (tenant_id != tid)
+ if (!filter(flow))
return;
extractor_stats.total_event++;
{
if (find_event(val))
events.push_back(val);
- else
- ParseWarning(WARN_CONF_STRICT, "unsupported '%s' event in protocols.on_events", val.c_str());
}
}
{
if (find_field(val))
fields.push_back(val);
- else
- ParseWarning(WARN_CONF_STRICT, "unsupported '%s' field in protocols.fields", val.c_str());
}
}
{
if (cfg.on_events.empty())
{
- ParseError("%s service misses on_events field", cfg.service.c_str());
+ ErrorMessage("Extractor: %s service misses on_events field\n", cfg.service.c_str());
return nullptr;
}
srv = new ConnExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, ins);
break;
- case ServiceType::UNDEFINED: // fallthrough
+ case ServiceType::ANY: // fallthrough
default:
- ParseError("'%s' service is not supported", cfg.service.c_str());
+ ErrorMessage("Extractor: '%s' service is not supported\n", cfg.service.c_str());
}
return srv;
bool ExtractorService::find_event(const std::string& event) const
{
- return std::find(sbp.supported_events.begin(), sbp.supported_events.end(), event)
- != sbp.supported_events.end();
+ return find_event(sbp, event);
}
bool ExtractorService::find_field(const std::string& field) const
{
- return ((std::find(common_fields.begin(), common_fields.end(), field) != common_fields.end()) or
- (std::find(sbp.supported_fields.begin(), sbp.supported_fields.end(),field)
- != sbp.supported_fields.end()));
+ return find_field(sbp, field);
}
void ExtractorService::show(std::string& str) const
str += " }";
}
+bool ExtractorService::find_event(const ServiceBlueprint& sbp, const std::string& event)
+{
+ return std::find(sbp.supported_events.begin(), sbp.supported_events.end(), event)
+ != sbp.supported_events.end();
+}
+
+bool ExtractorService::find_field(const ServiceBlueprint& sbp, const std::string& field)
+{
+ return ((std::find(common_fields.begin(), common_fields.end(), field) != common_fields.end())
+ or (std::find(sbp.supported_fields.begin(), sbp.supported_fields.end(),field)
+ != sbp.supported_fields.end()));
+}
+
+void ExtractorService::validate_events(const ServiceBlueprint& sbp, const std::vector<std::string>& vals)
+{
+ for (const auto& val : vals)
+ {
+ if (!find_event(sbp, val))
+ ParseWarning(WARN_CONF_STRICT, "unsupported '%s' event in protocols.on_events", val.c_str());
+ }
+}
+
+void ExtractorService::validate_fields(const ServiceBlueprint& sbp, const std::vector<std::string>& vals)
+{
+ for (auto& val : vals)
+ {
+ if (!find_field(sbp, val))
+ ParseWarning(WARN_CONF_STRICT, "unsupported '%s' field in protocols.fields\n", val.c_str());
+ }
+}
+
+void ExtractorService::validate(const ServiceConfig& cfg)
+{
+ if (cfg.on_events.empty())
+ ParseError("%s service misses on_events field", cfg.service.c_str());
+
+ switch (cfg.service)
+ {
+ case ServiceType::HTTP:
+ validate_events(HttpExtractorService::blueprint, cfg.on_events);
+ validate_fields(HttpExtractorService::blueprint, cfg.fields);
+ break;
+
+ case ServiceType::FTP:
+ validate_events(FtpExtractorService::blueprint, cfg.on_events);
+ validate_fields(FtpExtractorService::blueprint, cfg.fields);
+ break;
+
+ case ServiceType::CONN:
+ validate_events(ConnExtractorService::blueprint, cfg.on_events);
+ validate_fields(ConnExtractorService::blueprint, cfg.fields);
+ break;
+
+ case ServiceType::ANY: // fallthrough
+ default:
+ ParseError("'%s' service is not supported", cfg.service.c_str());
+ }
+}
+
//-------------------------------------------------------------------------
// HttpExtractorService
//-------------------------------------------------------------------------
-ServiceBlueprint HttpExtractorService::blueprint =
+const ServiceBlueprint HttpExtractorService::blueprint =
{
// events
{
// FtpExtractorService
//-------------------------------------------------------------------------
-ServiceBlueprint FtpExtractorService::blueprint =
+const ServiceBlueprint FtpExtractorService::blueprint =
{
// events
{
// ConnExtractorService
//-------------------------------------------------------------------------
-ServiceBlueprint ConnExtractorService::blueprint =
+const ServiceBlueprint ConnExtractorService::blueprint =
{
// events
{
ServiceType http = ServiceType::HTTP;
ServiceType ftp = ServiceType::FTP;
ServiceType conn = ServiceType::CONN;
- ServiceType undef = ServiceType::UNDEFINED;
+ ServiceType any = ServiceType::ANY;
ServiceType max = ServiceType::MAX;
CHECK_FALSE(strcmp("http", http.c_str()));
CHECK_FALSE(strcmp("ftp", ftp.c_str()));
CHECK_FALSE(strcmp("conn", conn.c_str()));
- CHECK_FALSE(strcmp("(not set)", undef.c_str()));
+ CHECK_FALSE(strcmp("(not set)", any.c_str()));
CHECK_FALSE(strcmp("(not set)", max.c_str()));
}
}
class ExtractorService
{
public:
+ static void validate(const ServiceConfig&);
static ExtractorService* make_service(Extractor&, const ServiceConfig&);
ExtractorService() = delete;
const ServiceBlueprint& sbp;
const ServiceType type;
+
+private:
+ static void validate_events(const ServiceBlueprint&, const std::vector<std::string>& vals);
+ static void validate_fields(const ServiceBlueprint&, const std::vector<std::string>& vals);
+ static bool find_event(const ServiceBlueprint&, const std::string&);
+ static bool find_field(const ServiceBlueprint&, const std::string&);
};
class HttpExtractorService : public ExtractorService
{
public:
+ static const ServiceBlueprint blueprint;
+
HttpExtractorService(uint32_t tenant, const std::vector<std::string>& fields,
const std::vector<std::string>& events, ServiceType, Extractor&);
const snort::Connector::ID& internal_tinit() override;
const snort::Connector::ID& get_log_id() override;
- static ServiceBlueprint blueprint;
static THREAD_LOCAL snort::Connector::ID log_id;
};
class FtpExtractorService : public ExtractorService
{
public:
+ static const ServiceBlueprint blueprint;
+
FtpExtractorService(uint32_t tenant, const std::vector<std::string>& fields,
const std::vector<std::string>& events, ServiceType, Extractor&);
const snort::Connector::ID& internal_tinit() override;
const snort::Connector::ID& get_log_id() override;
- static ServiceBlueprint blueprint;
static THREAD_LOCAL snort::Connector::ID log_id;
};
class ConnExtractorService : public ExtractorService
{
public:
+ static const ServiceBlueprint blueprint;
+
ConnExtractorService(uint32_t tenant, const std::vector<std::string>& fields,
const std::vector<std::string>& events, ServiceType, Extractor&);
const snort::Connector::ID& internal_tinit() override;
const snort::Connector::ID& get_log_id() override;
- static ServiceBlueprint blueprint;
static THREAD_LOCAL snort::Connector::ID log_id;
};
#include "sfip/sf_ip.h"
#include "time/packet_time.h"
+#include "extractor.h"
+#include "extractor_enums.h"
#include "extractor_logger.h"
-class Extractor;
-
template <typename Ret, class... Context>
struct DataField
{
return it != map.end();
}
- ExtractorEvent(Extractor& i, uint32_t tid) : tenant_id(tid), inspector(i)
- { }
+ inline bool filter(Flow*);
+
+ ExtractorEvent(ServiceType st, Extractor& i, uint32_t tid)
+ : service_type(st), pick_by_default(i.get_default_filter()), tenant_id(tid), inspector(i) { }
virtual void internal_tinit(const snort::Connector::ID*) = 0;
+ static THREAD_LOCAL ExtractorLogger* logger;
+
+ ServiceType service_type;
+ bool pick_by_default;
uint32_t tenant_id;
Extractor& inspector;
- static THREAD_LOCAL ExtractorLogger* logger;
std::vector<NtsField> nts_fields;
std::vector<SipField> sip_fields;
static const std::map<std::string, ExtractorEvent::NumGetFn> num_getters;
};
+bool ExtractorEvent::filter(Flow* flow)
+{
+ if (!flow)
+ return false;
+
+ auto& filter = flow->data_log_filtering_state;
+ assert(filter.size() >= ServiceType::MAX);
+
+#ifdef DISABLE_TENANT_ID
+ uint32_t tid = 0;
+#else
+ uint32_t tid = flow->key->tenant_id;
+#endif
+
+ // computed by external filter
+ if (filter.test(ServiceType::ANY))
+ return filter.test(service_type) and tenant_id == tid;
+
+ if (!pick_by_default)
+ return false;
+
+ // extractor sets targeted filtering
+ filter.set(service_type);
+
+ return filter.test(service_type) and tenant_id == tid;
+}
+
#endif
projects / thirdparty / snort3.git / commitdiff
