From: Oleksii Shumeiko -X (oshumeik - SOFTSERVE INC at Cisco) Date: Fri, 20 Sep 2024 09:51:31 +0000 (+0000) Subject: Pull request #4441: Extractor Refactoring X-Git-Tag: 3.3.7.0~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=43737fdab569bf156e1f66e718aab35044f6fb29;p=thirdparty%2Fsnort3.git Pull request #4441: Extractor Refactoring Merge in SNORT/snort3 from ~OSHUMEIK/snort3:ext_hare to master Squashed commit of the following: commit 21382cc49cc74bfd0f9c375eca7904bc221fbfe1 Author: Oleksii Shumeiko Date: Tue Sep 10 16:51:00 2024 +0300 extractor: notify handler whether it is a fixed-width formatting commit a6bc7ff8501415b727fa09f4c5e62eadb232519b Author: Oleksii Shumeiko Date: Tue Sep 10 16:18:44 2024 +0300 extractor: update logger with an internal set of fields for logging commit 6b8452f585d60ef6cf2215f6c7e3696894db392e Author: Oleksii Shumeiko Date: Fri Sep 6 13:47:27 2024 +0300 extractor: refactor data pipe between an inspector and extractor's logger --- diff --git a/src/network_inspectors/extractor/CMakeLists.txt b/src/network_inspectors/extractor/CMakeLists.txt index 20bbc9775..a79097759 100644 --- a/src/network_inspectors/extractor/CMakeLists.txt +++ b/src/network_inspectors/extractor/CMakeLists.txt @@ -3,6 +3,7 @@ set( FILE_LIST extractor.h extractor_csv_logger.cc extractor_csv_logger.h + extractor_event.cc extractor_event_handlers.h extractor_http_event_handler.cc extractor_json_logger.cc diff --git a/src/network_inspectors/extractor/extractor_csv_logger.cc b/src/network_inspectors/extractor/extractor_csv_logger.cc index d497b9cd6..a4f95f722 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.cc +++ b/src/network_inspectors/extractor/extractor_csv_logger.cc @@ -26,19 +26,22 @@ #include #include +#include "utils/util_cstring.h" + static THREAD_LOCAL bool first_write; void CsvExtractorLogger::add_header() { std::string header; + char d = '#'; - header += "#"; - header += fields_name[0]; - for (size_t i = 1; i < fields_name.size(); ++i) + for (auto n : field_names) { - header += ","; - header += fields_name[i]; + header += d; + header += n; + d = ','; } + header += "\n"; writer->write(header.c_str()); @@ -56,31 +59,43 @@ void CsvExtractorLogger::close_record() writer->unlock(); } -void CsvExtractorLogger::add_field(const char*, const snort::Value& v) +void CsvExtractorLogger::add_field(const char*, const char* v) { - switch (v.get_type()) - { - case snort::Value::ValueType::VT_UNUM: - { - first_write ? []() { first_write = false; } () : writer->write(","); - writer->write(std::to_string(v.get_uint64()).c_str()); - break; - } + first_write ? []() { first_write = false; } () : writer->write(","); + writer->write(v); +} - case snort::Value::ValueType::VT_STR: - { - first_write ? []() { first_write = false; } () : writer->write(","); - writer->write(v.get_string()); - break; - } +void CsvExtractorLogger::add_field(const char*, const char* v, size_t len) +{ + first_write ? []() { first_write = false; } () : writer->write(","); + writer->write(v, len); +} - case snort::Value::ValueType::VT_BOOL: // fallthrough - case snort::Value::ValueType::VT_NUM: // fallthrough - case snort::Value::ValueType::VT_REAL: // fallthrough - default: - assert(false); - break; - } +void CsvExtractorLogger::add_field(const char*, uint64_t v) +{ + first_write ? []() { first_write = false; } () : writer->write(","); + writer->write(v); +} + +void CsvExtractorLogger::add_field(const char*, struct timeval v) +{ + first_write ? []() { first_write = false; } () : writer->write(","); + + char u_sec[8]; + snort::SnortSnprintf(u_sec, sizeof(u_sec), ".%06d", (unsigned)v.tv_usec); + + writer->write(v.tv_sec); + writer->write(u_sec); +} + +void CsvExtractorLogger::add_field(const char*, const snort::SfIp& v) +{ + first_write ? []() { first_write = false; } () : writer->write(","); + + snort::SfIpString buf; + + v.ntop(buf); + writer->write(buf); } CsvExtractorLogger::~CsvExtractorLogger() diff --git a/src/network_inspectors/extractor/extractor_csv_logger.h b/src/network_inspectors/extractor/extractor_csv_logger.h index 76f5e4fc4..9a104cedd 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.h +++ b/src/network_inspectors/extractor/extractor_csv_logger.h @@ -28,17 +28,20 @@ class CsvExtractorLogger : public ExtractorLogger { public: - CsvExtractorLogger(OutputType o_type, const std::vector& fields) - : ExtractorLogger(fields), writer(ExtractorWriter::make_writer(o_type)) - { - if (writer) - CsvExtractorLogger::add_header(); - } + CsvExtractorLogger(OutputType o_type) + : writer(ExtractorWriter::make_writer(o_type)) {} ~CsvExtractorLogger() override; + virtual bool is_strict() const override + { return true; } + void add_header() override; - void add_field(const char*, const snort::Value&) override; + void add_field(const char*, const char*) override; + void add_field(const char*, const char*, size_t) override; + void add_field(const char*, uint64_t) override; + void add_field(const char*, struct timeval) override; + void add_field(const char*, const snort::SfIp&) override; void open_record() override; void close_record() override; diff --git a/src/network_inspectors/extractor/extractor_event.cc b/src/network_inspectors/extractor/extractor_event.cc new file mode 100644 index 000000000..f9353e305 --- /dev/null +++ b/src/network_inspectors/extractor/extractor_event.cc @@ -0,0 +1,46 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// extractor_event.cc author Cisco + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "extractor_event_handlers.h" + +using namespace snort; +using namespace std; + +vector ExtractorEvent::get_field_names() const +{ + vector res; + + for (auto& f : nts_fields) + res.push_back(f.name); + + for (auto& f : sip_fields) + res.push_back(f.name); + + for (auto& f : num_fields) + res.push_back(f.name); + + for (auto& f : str_fields) + res.push_back(f.name); + + return res; +} diff --git a/src/network_inspectors/extractor/extractor_event_handlers.h b/src/network_inspectors/extractor/extractor_event_handlers.h index 038db1176..5bc2ff21a 100644 --- a/src/network_inspectors/extractor/extractor_event_handlers.h +++ b/src/network_inspectors/extractor/extractor_event_handlers.h @@ -20,41 +20,85 @@ #ifndef EXTRACTOR_EVENT_HANDLERS_H #define EXTRACTOR_EVENT_HANDLERS_H +#include +#include + #include "flow/flow_key.h" #include "framework/data_bus.h" +#include "sfip/sf_ip.h" #include "extractor.h" #include "extractor_logger.h" +template +struct DataField +{ + DataField(const char* name, Ret (*get)(Context...)) : name(name), get(get) { } + + const char* name; + Ret (*get)(Context...); +}; + +class Field; + namespace snort { class ExtractorEvent { public: + using StrGetFn = const char* (*) (const DataEvent*, const Packet*, const Flow*); + using StrField = DataField; + using SipGetFn = const SfIp& (*) (const DataEvent*, const Packet*, const Flow*); + using SipField = DataField; + using NumGetFn = uint64_t (*) (const DataEvent*, const Packet*, const Flow*); + using NumField = DataField; + using NtsGetFn = struct timeval (*) (const DataEvent*, const Packet*, const Flow*); + using NtsField = DataField; + static FlowHashKeyOps& get_hash() { static thread_local FlowHashKeyOps flow_key_ops(0); return flow_key_ops; } + virtual std::vector get_field_names() const; + protected: - ExtractorEvent(uint32_t tid, const std::vector& flds, ExtractorLogger& l) - : tenant_id(tid), fields(flds), logger(l) {} + ExtractorEvent(uint32_t tid, ExtractorLogger& l) + : tenant_id(tid), logger(l) {} + + template + void log(const T& fields, Context... context) + { + for (const auto& f : fields) + logger.add_field(f.name, f.get(context...)); + } uint32_t tenant_id; - const std::vector fields; ExtractorLogger& logger; + + std::vector nts_fields; + std::vector sip_fields; + std::vector num_fields; + std::vector str_fields; }; class HttpExtractorEventHandler : public DataHandler, public ExtractorEvent { public: - HttpExtractorEventHandler(uint32_t tenant, const std::vector& flds, - ExtractorLogger& l) : DataHandler(S_NAME), ExtractorEvent(tenant, flds, l) {} + using SubGetFn = const Field& (*) (const DataEvent*, const Packet*, const Flow*); + using SubField = DataField; + + HttpExtractorEventHandler(uint32_t tenant, const std::vector& flds, ExtractorLogger& l); void handle(DataEvent&, Flow*) override; + std::vector get_field_names() const override; + +private: + std::vector sub_fields; }; } + #endif diff --git a/src/network_inspectors/extractor/extractor_http_event_handler.cc b/src/network_inspectors/extractor/extractor_http_event_handler.cc index dd3acd320..0d489e8cb 100644 --- a/src/network_inspectors/extractor/extractor_http_event_handler.cc +++ b/src/network_inspectors/extractor/extractor_http_event_handler.cc @@ -25,7 +25,6 @@ #include "detection/detection_engine.h" #include "flow/flow_key.h" -#include "framework/value.h" #include "profiler/profiler.h" #include "pub_sub/http_transaction_end_event.h" #include "service_inspectors/http_inspect/http_transaction.h" @@ -34,189 +33,178 @@ #include "utils/util_net.h" using namespace snort; +using namespace std; -// FIXIT-P: inspector's data passes many functions before getting to the logger - -typedef Value* (*GetFunc) (DataEvent*, Packet*, Flow*); - -// HttpTransactionEnd specific -Value* get_method(DataEvent*, Packet*, Flow*); -Value* get_host(DataEvent*, Packet*, Flow*); -Value* get_user_agent(DataEvent*, Packet*, Flow*); -Value* get_uri(DataEvent*, Packet*, Flow*); -Value* get_referrer(DataEvent*, Packet*, Flow*); -Value* get_origin(DataEvent*, Packet*, Flow*); -Value* get_version(DataEvent*, Packet*, Flow*); -Value* get_stat_code(DataEvent*, Packet*, Flow*); -Value* get_stat_msg(DataEvent*, Packet*, Flow*); -Value* get_trans_depth(DataEvent*, Packet*, Flow*); - -// Common -Value* get_timestamp(DataEvent*, Packet*, Flow*); -Value* get_ip_src(DataEvent*, Packet*, Flow*); -Value* get_ip_dst(DataEvent*, Packet*, Flow*); -Value* get_ip_src_port(DataEvent*, Packet*, Flow*); -Value* get_ip_dst_port(DataEvent*, Packet*, Flow*); -Value* get_pkt_num(DataEvent*, Packet*, Flow*); -Value* get_uid(DataEvent*, Packet*, Flow*); - -static void field_to_string(const Field& field, std::string& value) -{ - if (field.length() > 0) - value.assign((const char*)field.start(), field.length()); -} - -Value* get_method(DataEvent* event, Packet*, Flow*) +static const Field& get_method(const DataEvent* event, const Packet*, const Flow*) { - const Field& field = ((HttpTransactionEndEvent*)event)->get_method(); - std::string str; - field_to_string(field, str); - return new Value(str.c_str()); + return ((const HttpTransactionEndEvent*)event)->get_method(); } -Value* get_host(DataEvent* event, Packet*, Flow*) +static const Field& get_host(const DataEvent* event, const Packet*, const Flow*) { - const Field& field = ((HttpTransactionEndEvent*)event)->get_host_hdr(); - std::string str; - field_to_string(field, str); - return new Value(str.c_str()); + return ((const HttpTransactionEndEvent*)event)->get_host_hdr(); } -Value* get_user_agent(DataEvent* event, Packet*, Flow*) +static const Field& get_user_agent(const DataEvent* event, const Packet*, const Flow*) { - const Field& field = ((HttpTransactionEndEvent*)event)->get_user_agent(); - std::string str; - field_to_string(field, str); - return new Value(str.c_str()); + return ((const HttpTransactionEndEvent*)event)->get_user_agent(); } -Value* get_uri(DataEvent* event, Packet*, Flow*) +static const Field& get_uri(const DataEvent* event, const Packet*, const Flow*) { - const Field& field = ((HttpTransactionEndEvent*)event)->get_uri(); - std::string str; - field_to_string(field, str); - return new Value(str.c_str()); + return ((const HttpTransactionEndEvent*)event)->get_uri(); } -Value* get_referrer(DataEvent* event, Packet*, Flow*) +static const Field& get_referrer(const DataEvent* event, const Packet*, const Flow*) { - const Field& field = ((HttpTransactionEndEvent*)event)->get_referer_hdr(); - std::string str; - field_to_string(field, str); - return new Value(str.c_str()); + return ((const HttpTransactionEndEvent*)event)->get_referer_hdr(); } -Value* get_origin(DataEvent* event, Packet*, Flow*) +static const Field& get_origin(const DataEvent* event, const Packet*, const Flow*) { - const Field& field = ((HttpTransactionEndEvent*)event)->get_origin_hdr(); - std::string str; - field_to_string(field, str); - return new Value(str.c_str()); + return ((const HttpTransactionEndEvent*)event)->get_origin_hdr(); } -Value* get_version(DataEvent* event, Packet*, Flow*) +static const char* get_version(const DataEvent* event, const Packet*, const Flow*) { - HttpEnums::VersionId version = ((HttpTransactionEndEvent*)event)->get_version(); + HttpEnums::VersionId version = ((const HttpTransactionEndEvent*)event)->get_version(); const auto& iter = HttpEnums::VersionEnumToStr.find(version); - if (iter != HttpEnums::VersionEnumToStr.end()) - return new Value(iter->second); - return new Value(""); + return iter != HttpEnums::VersionEnumToStr.end() ? iter->second : ""; } -Value* get_stat_code(DataEvent* event, Packet*, Flow*) +static const Field& get_stat_code(const DataEvent* event, const Packet*, const Flow*) { - const Field& field = ((HttpTransactionEndEvent*)event)->get_stat_code(); - std::string str; - field_to_string(field, str); - - return new Value((uint64_t)atoi(str.c_str())); + return ((const HttpTransactionEndEvent*)event)->get_stat_code(); } -Value* get_stat_msg(DataEvent* event, Packet*, Flow*) +static const Field& get_stat_msg(const DataEvent* event, const Packet*, const Flow*) { - const Field& field = ((HttpTransactionEndEvent*)event)->get_stat_msg(); - std::string str; - field_to_string(field, str); - return new Value(str.c_str()); + return ((const HttpTransactionEndEvent*)event)->get_stat_msg(); } -Value* get_trans_depth(DataEvent* event, Packet*, Flow*) +static uint64_t get_trans_depth(const DataEvent* event, const Packet*, const Flow*) { - const uint64_t trans_depth = ((HttpTransactionEndEvent*)event)->get_trans_depth(); - return new Value(trans_depth); + return ((const HttpTransactionEndEvent*)event)->get_trans_depth(); } -Value* get_timestamp(DataEvent*, Packet* p, Flow*) +static struct timeval get_timestamp(const DataEvent*, const Packet* p, const Flow*) { - char u_sec[8]; - SnortSnprintf(u_sec, sizeof(u_sec),".%06d",(unsigned)p->pkth->ts.tv_usec); - auto str = std::to_string(p->pkth->ts.tv_sec) + u_sec; - - return new Value(str.c_str()); + return p->pkth->ts; } -Value* get_ip_src(DataEvent*, Packet*, Flow* flow) +static const SfIp& get_ip_src(const DataEvent*, const Packet*, const Flow* flow) { - InetBuf src; - const SfIp& flow_srcip = flow->flags.client_initiated ? flow->client_ip : flow->server_ip; - sfip_ntop(&flow_srcip, src, sizeof(src)); - std::string str = src; - return new Value(str.c_str()); + return flow->flags.client_initiated ? flow->client_ip : flow->server_ip; } -Value* get_ip_dst(DataEvent*, Packet*, Flow* flow) +static const SfIp& get_ip_dst(const DataEvent*, const Packet*, const Flow* flow) { - InetBuf dst; - const SfIp& flow_dstip = flow->flags.client_initiated ? flow->server_ip : flow->client_ip; - sfip_ntop(&flow_dstip, dst, sizeof(dst)); - std::string str = dst; - return new Value(str.c_str()); + return flow->flags.client_initiated ? flow->server_ip : flow->client_ip; } -Value* get_ip_src_port(DataEvent*, Packet*, Flow* flow) +static uint64_t get_ip_src_port(const DataEvent*, const Packet*, const Flow* flow) { - return new Value((uint64_t)flow->client_port); + return flow->client_port; } -Value* get_ip_dst_port(DataEvent*, Packet*, Flow* flow) +static uint64_t get_ip_dst_port(const DataEvent*, const Packet*, const Flow* flow) { - return new Value((uint64_t)flow->server_port); + return flow->server_port; } -Value* get_pkt_num(DataEvent*, Packet* p, Flow*) +static uint64_t get_pkt_num(const DataEvent*, const Packet* p, const Flow*) { - return new Value(p->context->packet_number); + return p->context->packet_number; } -Value* get_uid(DataEvent*, Packet*, Flow* f) +static uint64_t get_uid(const DataEvent*, const Packet*, const Flow* flow) { - unsigned key = ExtractorEvent::get_hash().do_hash((const unsigned char*)f->key, 0); - - return new Value((uint64_t)key); + return ExtractorEvent::get_hash().do_hash((const unsigned char*)flow->key, 0); } -static std::map event_getters = +static const map nts_getters = { {"ts", get_timestamp}, - {"uid", get_uid}, +}; + +static const map sip_getters = +{ {"id.orig_h", get_ip_src}, - {"id.resp_h", get_ip_dst}, + {"id.resp_h", get_ip_dst} +}; + +static const map str_getters = +{ + {"version", get_version} +}; + +static const map num_getters = +{ {"id.orig_p", get_ip_src_port}, {"id.resp_p", get_ip_dst_port}, + {"uid", get_uid}, {"pkt_num", get_pkt_num}, + {"trans_depth", get_trans_depth} +}; + +static const map sub_getters = +{ {"method", get_method}, {"host", get_host}, {"uri", get_uri}, {"user_agent", get_user_agent}, {"referrer", get_referrer}, {"origin", get_origin}, - {"version", get_version}, {"status_code", get_stat_code}, - {"status_msg", get_stat_msg}, - {"trans_depth", get_trans_depth} + {"status_msg", get_stat_msg} }; +template +static inline bool append(T& cont, const U& map, const V& key) +{ + auto it = map.find(key); + + if (it == map.end()) + return false; + + cont.emplace_back(it->first.c_str(), it->second); + + return true; +} + +HttpExtractorEventHandler::HttpExtractorEventHandler(uint32_t t, const vector& fields, ExtractorLogger& l) + : DataHandler(S_NAME), ExtractorEvent(t, l) +{ + for (const auto& f : fields) + { + if (append(nts_fields, nts_getters, f)) + continue; + if (append(sip_fields, sip_getters, f)) + continue; + if (append(num_fields, num_getters, f)) + continue; + if (append(str_fields, str_getters, f)) + continue; + if (append(sub_fields, sub_getters, f)) + continue; + } +} + +template<> +void ExtractorEvent::log, DataEvent*, Packet*, Flow*, bool>( + const vector& fields, DataEvent* event, Packet* pkt, Flow* flow, bool strict) +{ + for (const auto& f : fields) + { + const auto& d = f.get(event, pkt, flow); + if (d.length() > 0) + logger.add_field(f.name, (const char*)d.start(), d.length()); + else if (strict) + logger.add_field(f.name, ""); + } +} + void HttpExtractorEventHandler::handle(DataEvent& event, Flow* flow) { // cppcheck-suppress unreadVariable @@ -232,16 +220,25 @@ void HttpExtractorEventHandler::handle(DataEvent& event, Flow* flow) if (tenant_id != tid) return; - Packet* p = DetectionEngine::get_current_packet(); + Packet* packet = DetectionEngine::get_current_packet(); logger.open_record(); - for (const auto& field : fields) - { - // FIXIT-P: this is way too slow (a map with a string key type) - auto val = std::unique_ptr(event_getters[field](&event, p, flow)); - logger.add_field(field.c_str(), *val.get()); - } + log(nts_fields, &event, packet, flow); + log(sip_fields, &event, packet, flow); + log(num_fields, &event, packet, flow); + log(str_fields, &event, packet, flow); + log(sub_fields, &event, packet, flow, logger.is_strict()); logger.close_record(); extractor_stats.total_event++; } + +vector HttpExtractorEventHandler::get_field_names() const +{ + vector res = ExtractorEvent::get_field_names(); + + for (const auto& f : sub_fields) + res.push_back(f.name); + + return res; +} diff --git a/src/network_inspectors/extractor/extractor_json_logger.cc b/src/network_inspectors/extractor/extractor_json_logger.cc index a286bc87c..68ec7c147 100644 --- a/src/network_inspectors/extractor/extractor_json_logger.cc +++ b/src/network_inspectors/extractor/extractor_json_logger.cc @@ -24,6 +24,9 @@ #include "extractor_json_logger.h" #include +#include + +#include "utils/util_cstring.h" void JsonExtractorLogger::open_record() { @@ -40,23 +43,36 @@ void JsonExtractorLogger::close_record() writer->unlock(); } -void JsonExtractorLogger::add_field(const char* f, const snort::Value& v) +void JsonExtractorLogger::add_field(const char* f, const char* v) +{ + js.put(f, v); +} + +void JsonExtractorLogger::add_field(const char* f, const char* v, size_t len) +{ + std::string s(v, len); + + js.put(f, s); +} + +void JsonExtractorLogger::add_field(const char* f, uint64_t v) { - switch (v.get_type()) - { - case snort::Value::ValueType::VT_UNUM: - js.uput(f, v.get_uint64()); - break; - - case snort::Value::ValueType::VT_STR: - js.put(f, v.get_string()); - break; - - case snort::Value::ValueType::VT_BOOL: // fallthrough - case snort::Value::ValueType::VT_NUM: // fallthrough - case snort::Value::ValueType::VT_REAL: // fallthrough - default: - assert(false); - break; - } + js.uput(f, v); +} + +void JsonExtractorLogger::add_field(const char* f, struct timeval v) +{ + char u_sec[8]; + snort::SnortSnprintf(u_sec, sizeof(u_sec), ".%06d",(unsigned)v.tv_usec); + + auto str = std::to_string(v.tv_sec) + u_sec; + js.put(f, str); +} + +void JsonExtractorLogger::add_field(const char* f, const snort::SfIp& v) +{ + snort::SfIpString buf; + + v.ntop(buf); + js.put(f, buf); } diff --git a/src/network_inspectors/extractor/extractor_json_logger.h b/src/network_inspectors/extractor/extractor_json_logger.h index ca98925c3..857bed8d6 100644 --- a/src/network_inspectors/extractor/extractor_json_logger.h +++ b/src/network_inspectors/extractor/extractor_json_logger.h @@ -31,14 +31,17 @@ class JsonExtractorLogger : public ExtractorLogger { public: - JsonExtractorLogger(OutputType o_type, const std::vector& fields) - : ExtractorLogger(fields), writer(ExtractorWriter::make_writer(o_type)), oss(), js(oss) - { } + JsonExtractorLogger(OutputType o_type) + : writer(ExtractorWriter::make_writer(o_type)), oss(), js(oss) {} ~JsonExtractorLogger() override { delete writer; } - void add_field(const char*, const snort::Value&) override; + void add_field(const char*, const char*) override; + void add_field(const char*, const char*, size_t) override; + void add_field(const char*, uint64_t) override; + void add_field(const char*, struct timeval) override; + void add_field(const char*, const snort::SfIp&) override; void open_record() override; void close_record() override; diff --git a/src/network_inspectors/extractor/extractor_logger.cc b/src/network_inspectors/extractor/extractor_logger.cc index 9cfccd9a4..a96642ae3 100644 --- a/src/network_inspectors/extractor/extractor_logger.cc +++ b/src/network_inspectors/extractor/extractor_logger.cc @@ -28,21 +28,17 @@ #include "extractor_csv_logger.h" #include "extractor_json_logger.h" -ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, OutputType o_type, - const std::vector& fields) +ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, OutputType o_type) { - if (fields.empty()) - return nullptr; - ExtractorLogger* logger = nullptr; switch (f_type) { case FormatType::CSV: - logger = new CsvExtractorLogger(o_type, fields); + logger = new CsvExtractorLogger(o_type); break; case FormatType::JSON: - logger = new JsonExtractorLogger(o_type, fields); + logger = new JsonExtractorLogger(o_type); break; case FormatType::MAX: // fallthrough default: diff --git a/src/network_inspectors/extractor/extractor_logger.h b/src/network_inspectors/extractor/extractor_logger.h index 68e4e35e3..f5b10869f 100644 --- a/src/network_inspectors/extractor/extractor_logger.h +++ b/src/network_inspectors/extractor/extractor_logger.h @@ -20,10 +20,10 @@ #ifndef EXTRACTOR_LOGGER_H #define EXTRACTOR_LOGGER_H -#include +#include #include -#include "framework/value.h" +#include "sfip/sf_ip.h" #include "extractor_writer.h" @@ -65,27 +65,33 @@ private: class ExtractorLogger { public: - static ExtractorLogger* make_logger(FormatType, OutputType, const std::vector&); + static ExtractorLogger* make_logger(FormatType, OutputType); - ExtractorLogger() = delete; + ExtractorLogger() = default; ExtractorLogger(const ExtractorLogger&) = delete; ExtractorLogger& operator=(const ExtractorLogger&) = delete; ExtractorLogger(ExtractorLogger&&) = delete; - virtual ~ExtractorLogger() = default; + virtual bool is_strict() const + { return false; } + virtual void set_fields(std::vector& names) + { field_names = names; } + virtual void add_header() {} virtual void add_footer() {} - // FIXIT-P: replace Value type designed for parsing with a better type - virtual void add_field(const char*, const snort::Value&) {} + + virtual void add_field(const char*, const char*) {} + virtual void add_field(const char*, const char*, size_t) {} + virtual void add_field(const char*, uint64_t) {} + virtual void add_field(const char*, struct timeval) {} + virtual void add_field(const char*, const snort::SfIp&) {} virtual void open_record() {} virtual void close_record() {} protected: - ExtractorLogger(const std::vector& fns) : fields_name(fns) {} - - const std::vector& fields_name; + std::vector field_names; }; #endif diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc index 3c771e60a..e7cacb0d1 100644 --- a/src/network_inspectors/extractor/extractor_service.cc +++ b/src/network_inspectors/extractor/extractor_service.cc @@ -55,7 +55,7 @@ ExtractorService::ExtractorService(uint32_t tenant, const std::vector& vals) @@ -174,8 +174,13 @@ HttpExtractorService::HttpExtractorService(uint32_t tenant, const std::vectorget_field_names(); + logger->set_fields(names_set); + logger->add_header(); } } } diff --git a/src/network_inspectors/extractor/extractor_writer.cc b/src/network_inspectors/extractor/extractor_writer.cc index 7f402a31f..2e642f29b 100644 --- a/src/network_inspectors/extractor/extractor_writer.cc +++ b/src/network_inspectors/extractor/extractor_writer.cc @@ -23,6 +23,8 @@ #include "extractor_writer.h" +using namespace snort; + ExtractorWriter* ExtractorWriter::make_writer(OutputType o_type) { switch (o_type) @@ -35,17 +37,27 @@ ExtractorWriter* ExtractorWriter::make_writer(OutputType o_type) } } -StdExtractorWriter::StdExtractorWriter() : ExtractorWriter(), extr_std_log(snort::TextLog_Init("stdout")) +StdExtractorWriter::StdExtractorWriter() : ExtractorWriter(), extr_std_log(TextLog_Init("stdout")) {} StdExtractorWriter::~StdExtractorWriter() { - snort::TextLog_Term(extr_std_log); + TextLog_Term(extr_std_log); } void StdExtractorWriter::write(const char* ss) { - snort::TextLog_Print(extr_std_log, "%s", ss); + TextLog_Print(extr_std_log, "%s", ss); +} + +void StdExtractorWriter::write(const char* ss, size_t len) +{ + TextLog_Print(extr_std_log, "%.*s", (int)len, ss); +} + +void StdExtractorWriter::write(uint64_t n) +{ + TextLog_Print(extr_std_log, STDu64, n); } #ifdef UNIT_TEST diff --git a/src/network_inspectors/extractor/extractor_writer.h b/src/network_inspectors/extractor/extractor_writer.h index a30c912b2..d2abfac0d 100644 --- a/src/network_inspectors/extractor/extractor_writer.h +++ b/src/network_inspectors/extractor/extractor_writer.h @@ -24,6 +24,7 @@ #include #include "log/text_log.h" +#include "main/snort_types.h" class OutputType { @@ -69,6 +70,8 @@ public: virtual ~ExtractorWriter() = default; virtual void write(const char*) = 0; + virtual void write(const char*, size_t) = 0; + virtual void write(uint64_t) = 0; virtual void lock() { } virtual void unlock() { } @@ -83,6 +86,8 @@ public: ~StdExtractorWriter() override; void write(const char* ss) override; + void write(const char* ss, size_t len) override; + void write(uint64_t n) override; void lock() override { write_mutex.lock(); }