From: Oleksii Shumeiko -X (oshumeik - SOFTSERVE INC at Cisco) Date: Mon, 10 Mar 2025 14:47:19 +0000 (+0000) Subject: Pull request #4655: Extractor timestamp field X-Git-Tag: 3.7.1.0~4 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=288a6eeea9d2e09a12b53e342910dac09ae4a667;p=thirdparty%2Fsnort3.git Pull request #4655: Extractor timestamp field Merge in SNORT/snort3 from ~OSHUMEIK/snort3:extr_field_types to master Squashed commit of the following: commit 22aae83d1edfaa22a7145501068a29954370d38d Author: Oleksii Shumeiko Date: Thu Mar 6 17:37:52 2025 +0200 extractor: add time formatting in loggers commit bdd2f2ac6ccf9f7aa2984bc22455a5959bc6745c Author: Oleksii Shumeiko Date: Wed Mar 5 17:36:14 2025 +0200 extractor: add configuration option for time formatting commit df147998fd47b5e3813e909328748e85e254c8b0 Author: Oleksii Shumeiko Date: Wed Mar 5 17:10:53 2025 +0200 extractor: remove obsolete includes --- diff --git a/doc/user/extractor.txt b/doc/user/extractor.txt index 7a0143e98..1686ff9d9 100644 --- a/doc/user/extractor.txt +++ b/doc/user/extractor.txt @@ -2,7 +2,6 @@ Snort 3 can log IPS events with some meta data and dump packets. The Data Logging feature extends that ability to log protocol-specific data, sniffing traffic alongside with normal inspection. - ==== Configurations The module's configuration consists of two parts: @@ -11,6 +10,7 @@ The module's configuration consists of two parts: ** `formatting` - log record format ** `connector` - Connector object through which logs will be sent. See Connectors page for more details. + ** `time` - timestamp format * protocol-targeted parameters bind the targeted service and events with filters and a set of fields to log ** `service` - protocol name @@ -29,28 +29,38 @@ things it allows tenants to get independent data logging configurations. protocols = { - { service = 'http', tenant_id = 1, on_events = 'eot', fields = 'ts, uri, host, method' }, - { service = 'ftp', tenant_id = 1, on_events = 'request', fields = 'ts, command, arg' }, - { service = 'http', tenant_id = 2, on_events = 'eot', fields = 'ts, uri' }, - { service = 'conn', tenant_id = 1, on_events = 'eof', fields = 'ts, uid, service' }, - { service = 'dns', tenant_id = 1, on_events = 'response', fields = 'ts, uid, query, answers' } + { service = 'http', on_events = 'eot', fields = 'ts, uri, host, method' }, + { service = 'ftp', on_events = 'request', fields = 'ts, command, arg' }, + { service = 'http', on_events = 'eot', fields = 'ts, uri' }, + { service = 'conn', on_events = 'eof', fields = 'ts, uid, service' }, + { service = 'dns', on_events = 'response', fields = 'ts, uid, query, answers' } } } ==== Supported Parameters +Timestamp formats: + +* `snort` prints timestamp as in IPS events (see snort command line options + `-U` and `-y`) (string `ts` field) +* `snort_yy` same as above, but using YYYY-MM-DD format (string `ts` field) +* `unix` prints UTC time in seconds (integer part) and microseconds (fractional part) + (floating `ts` field) +* `unix_s` prints UTC time in seconds (integer `ts` field) +* `unix_us` prints UTC time in microseconds (integer `ts` field) + Services and their events: * HTTP, HTTP2 - ** eot (request-response pair) + ** `eot` (request-response pair) * FTP - ** request - ** response - ** eot (a session defined by the following commands: APPE, DELE, RETR, STOR, STOU, ACCT, PORT, PASV, EPRT, EPSV) + ** `request` + ** `response` + ** `eot` (a session defined by the following commands: APPE, DELE, RETR, STOR, STOU, ACCT, PORT, PASV, EPRT, EPSV) * DNS - ** response + ** `response` * connection (conn) - ** eof (end of flow) + ** `eof` (end of flow) Common fields available for every service: diff --git a/src/network_inspectors/extractor/extractor.cc b/src/network_inspectors/extractor/extractor.cc index 9e11eeb0b..e943040de 100644 --- a/src/network_inspectors/extractor/extractor.cc +++ b/src/network_inspectors/extractor/extractor.cc @@ -73,6 +73,9 @@ static const Parameter s_params[] = { "connector", Parameter::PT_STRING, nullptr, nullptr, "output destination for extractor" }, + { "time", Parameter::PT_ENUM, "snort | snort_yy | unix | unix_s | unix_us", "unix", + "output format for timestamp values" }, + { "default_filter", Parameter::PT_ENUM, "pick | skip", "pick", "default action for protocol with no filter provided" }, @@ -141,6 +144,9 @@ bool ExtractorModule::set(const char*, Value& v, SnortConfig*) else if (v.is("connector")) extractor_config.output_conn = v.get_string(); + else if (v.is("time")) + extractor_config.time_formatting = (TimeType)(v.get_uint8()); + if (v.is("default_filter")) extractor_config.pick_by_default = v.get_uint8() == 0; @@ -190,7 +196,8 @@ public: inspector.logger->flush(); delete inspector.logger; - inspector.logger = ExtractorLogger::make_logger(inspector.cfg.formatting, inspector.cfg.output_conn); + inspector.logger = ExtractorLogger::make_logger( + inspector.cfg.formatting, inspector.cfg.output_conn, inspector.cfg.time_formatting); for (auto& s : inspector.services) s->tinit(inspector.logger); @@ -242,6 +249,7 @@ void Extractor::show(const SnortConfig*) const { ConfigLogger::log_value("formatting", cfg.formatting.c_str()); ConfigLogger::log_value("connector", cfg.output_conn.c_str()); + ConfigLogger::log_value("time", cfg.time_formatting.c_str()); ConfigLogger::log_value("pick_by_default", cfg.pick_by_default ? "pick" : "skip"); bool log_header = true; @@ -261,7 +269,7 @@ void Extractor::show(const SnortConfig*) const void Extractor::tinit() { - logger = ExtractorLogger::make_logger(cfg.formatting, cfg.output_conn); + logger = ExtractorLogger::make_logger(cfg.formatting, cfg.output_conn, cfg.time_formatting); for (auto& s : services) s->tinit(logger); @@ -337,3 +345,34 @@ const BaseApi* nin_extractor[] = nullptr }; +//------------------------------------------------------------------------- +// Unit Tests +//------------------------------------------------------------------------- + +#ifdef UNIT_TEST + +#include "catch/snort_catch.h" + +#include + +TEST_CASE("Time Type", "[extractor]") +{ + SECTION("to string") + { + TimeType a = TimeType::SNORT; + TimeType b = TimeType::SNORT_YY; + TimeType c = TimeType::UNIX; + TimeType d = TimeType::UNIX_S; + TimeType e = TimeType::UNIX_US; + TimeType f = TimeType::MAX; + + CHECK_FALSE(strcmp("snort", a.c_str())); + CHECK_FALSE(strcmp("snort_yy", b.c_str())); + CHECK_FALSE(strcmp("unix", c.c_str())); + CHECK_FALSE(strcmp("unix_s", d.c_str())); + CHECK_FALSE(strcmp("unix_us", e.c_str())); + CHECK_FALSE(strcmp("(not set)", f.c_str())); + } +} + +#endif diff --git a/src/network_inspectors/extractor/extractor.h b/src/network_inspectors/extractor/extractor.h index c54169c29..1b234d547 100644 --- a/src/network_inspectors/extractor/extractor.h +++ b/src/network_inspectors/extractor/extractor.h @@ -51,6 +51,7 @@ struct ExtractorConfig { FormatType formatting = FormatType::CSV; std::string output_conn; + TimeType time_formatting = TimeType::UNIX; bool pick_by_default = true; std::vector protocols; }; diff --git a/src/network_inspectors/extractor/extractor_csv_logger.cc b/src/network_inspectors/extractor/extractor_csv_logger.cc index bd7ba9596..c8d92a506 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.cc +++ b/src/network_inspectors/extractor/extractor_csv_logger.cc @@ -29,6 +29,7 @@ #include #include +#include "utils/util.h" #include "utils/util_cstring.h" using namespace snort; @@ -36,6 +37,33 @@ using namespace std; static THREAD_LOCAL bool first_write; +CsvExtractorLogger::CsvExtractorLogger(snort::Connector* conn, TimeType ts_type) + : ExtractorLogger(conn) +{ + switch (ts_type) + { + case TimeType::SNORT: + add_ts = &CsvExtractorLogger::ts_snort; + break; + case TimeType::SNORT_YY: + add_ts = &CsvExtractorLogger::ts_snort_yy; + break; + case TimeType::UNIX: + add_ts = &CsvExtractorLogger::ts_unix; + break; + case TimeType::UNIX_S: + add_ts = &CsvExtractorLogger::ts_sec; + break; + case TimeType::UNIX_US: + add_ts = &CsvExtractorLogger::ts_usec; + break; + case TimeType::MAX: // fallthrough + default: + add_ts = &CsvExtractorLogger::ts_snort; + break; + } +} + void CsvExtractorLogger::add_header(const vector& field_names, const Connector::ID& service_id) { string header; @@ -83,16 +111,6 @@ void CsvExtractorLogger::add_field(const char*, uint64_t v) buffer.append(to_string(v)); } -void CsvExtractorLogger::add_field(const char*, struct timeval v) -{ - first_write ? []() { first_write = false; } () : buffer.push_back(','); - - char time_str[numeric_limits::digits10 + 8]; - snort::SnortSnprintf(time_str, sizeof(time_str), "%" PRIu64 ".%06d", (uint64_t)v.tv_sec, (unsigned)v.tv_usec); - - buffer.append(time_str); -} - void CsvExtractorLogger::add_field(const char*, const snort::SfIp& v) { first_write ? []() { first_write = false; } () : buffer.push_back(','); @@ -159,6 +177,51 @@ void CsvExtractorLogger::add_escaped(const char* v, size_t len) buffer.push_back('"'); } +void CsvExtractorLogger::add_field(const char*, struct timeval v) +{ + first_write ? []() { first_write = false; } () : buffer.push_back(','); + (this->*add_ts)(v); +} + +void CsvExtractorLogger::ts_snort(const struct timeval& v) +{ + char ts[TIMEBUF_SIZE]; + ts_print(&v, ts, false); + + buffer.append(ts); +} + +void CsvExtractorLogger::ts_snort_yy(const struct timeval& v) +{ + char ts[TIMEBUF_SIZE]; + ts_print(&v, ts, true); + + buffer.append(ts); +} + +void CsvExtractorLogger::ts_unix(const struct timeval& v) +{ + char ts[numeric_limits::digits10 + 8]; + + snort::SnortSnprintf(ts, sizeof(ts), "%" PRIu64 ".%06d", (uint64_t)v.tv_sec, (unsigned)v.tv_usec); + buffer.append(ts); +} + +void CsvExtractorLogger::ts_sec(const struct timeval& v) +{ + uint64_t sec = (uint64_t)v.tv_sec; + + buffer.append(to_string(sec)); +} + +void CsvExtractorLogger::ts_usec(const struct timeval& v) +{ + uint64_t sec = (uint64_t)v.tv_sec; + uint64_t usec = (uint64_t)v.tv_usec; + + buffer.append(to_string(sec * 1000000 + usec)); +} + #ifdef UNIT_TEST #include "catch/snort_catch.h" @@ -166,7 +229,7 @@ void CsvExtractorLogger::add_escaped(const char* v, size_t len) class CsvExtractorLoggerTest : public CsvExtractorLogger { public: - CsvExtractorLoggerTest() : CsvExtractorLogger(nullptr) {} + CsvExtractorLoggerTest() : CsvExtractorLogger(nullptr, TimeType::MAX) {} void check_escaping(const char* input, size_t i_len, const std::string& expected) { diff --git a/src/network_inspectors/extractor/extractor_csv_logger.h b/src/network_inspectors/extractor/extractor_csv_logger.h index 4c74fc981..655686ac0 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.h +++ b/src/network_inspectors/extractor/extractor_csv_logger.h @@ -20,15 +20,12 @@ #ifndef EXTRACTOR_CSV_LOGGER_H #define EXTRACTOR_CSV_LOGGER_H -#include "framework/value.h" - #include "extractor_logger.h" class CsvExtractorLogger : public ExtractorLogger { public: - CsvExtractorLogger(snort::Connector* conn) : ExtractorLogger(conn) - { } + CsvExtractorLogger(snort::Connector*, TimeType); virtual bool is_strict() const override { return true; } @@ -45,8 +42,14 @@ public: protected: void add_escaped(const char*, size_t); + void ts_snort(const struct timeval&); + void ts_snort_yy(const struct timeval&); + void ts_unix(const struct timeval&); + void ts_sec(const struct timeval&); + void ts_usec(const struct timeval&); std::string buffer; + void (CsvExtractorLogger::*add_ts)(const struct timeval&); }; #endif diff --git a/src/network_inspectors/extractor/extractor_enums.h b/src/network_inspectors/extractor/extractor_enums.h index 9d7bd15cd..566cb057a 100644 --- a/src/network_inspectors/extractor/extractor_enums.h +++ b/src/network_inspectors/extractor/extractor_enums.h @@ -100,4 +100,48 @@ private: Value v = CSV; }; +class TimeType +{ +public: + enum Value : uint8_t + { + SNORT, + SNORT_YY, + UNIX, + UNIX_S, + UNIX_US, + MAX + }; + + TimeType() = default; + constexpr TimeType(Value a) : v(a) {} + template constexpr TimeType(T a) : v((Value)a) {} + + constexpr operator Value() const { return v; } + explicit operator bool() const = delete; + + const char* c_str() const + { + switch (v) + { + case SNORT: + return "snort"; + case SNORT_YY: + return "snort_yy"; + case UNIX: + return "unix"; + case UNIX_S: + return "unix_s"; + case UNIX_US: + return "unix_us"; + case MAX: // fallthrough + default: + return "(not set)"; + } + } + +private: + Value v = UNIX; +}; + #endif diff --git a/src/network_inspectors/extractor/extractor_json_logger.cc b/src/network_inspectors/extractor/extractor_json_logger.cc index cf79c1e25..cdc095749 100644 --- a/src/network_inspectors/extractor/extractor_json_logger.cc +++ b/src/network_inspectors/extractor/extractor_json_logger.cc @@ -26,8 +26,36 @@ #include #include +#include "utils/util.h" #include "utils/util_cstring.h" +JsonExtractorLogger::JsonExtractorLogger(snort::Connector* conn, TimeType ts_type) + : ExtractorLogger(conn), oss(), js(oss) +{ + switch (ts_type) + { + case TimeType::SNORT: + add_ts = &JsonExtractorLogger::ts_snort; + break; + case TimeType::SNORT_YY: + add_ts = &JsonExtractorLogger::ts_snort_yy; + break; + case TimeType::UNIX: + add_ts = &JsonExtractorLogger::ts_unix; + break; + case TimeType::UNIX_S: + add_ts = &JsonExtractorLogger::ts_sec; + break; + case TimeType::UNIX_US: + add_ts = &JsonExtractorLogger::ts_usec; + break; + case TimeType::MAX: // fallthrough + default: + add_ts = &JsonExtractorLogger::ts_snort; + break; + } +} + void JsonExtractorLogger::open_record() { oss.str(""); @@ -61,15 +89,6 @@ void JsonExtractorLogger::add_field(const char* f, uint64_t v) js.uput(f, v); } -void JsonExtractorLogger::add_field(const char* f, struct timeval v) -{ - char u_sec[8]; - snort::SnortSnprintf(u_sec, sizeof(u_sec), ".%06d",(unsigned)v.tv_usec); - - auto str = std::to_string(v.tv_sec) + u_sec; - js.put(f, str); -} - void JsonExtractorLogger::add_field(const char* f, const snort::SfIp& v) { snort::SfIpString buf; @@ -82,3 +101,47 @@ void JsonExtractorLogger::add_field(const char* f, bool v) { v ? js.put_true(f) : js.put_false(f); } + +void JsonExtractorLogger::add_field(const char* f, struct timeval v) +{ + (this->*add_ts)(f, v); +} + +void JsonExtractorLogger::ts_snort(const char* f, const struct timeval& v) +{ + char ts[TIMEBUF_SIZE]; + snort::ts_print(&v, ts, false); + + js.put(f, ts); +} + +void JsonExtractorLogger::ts_snort_yy(const char* f, const struct timeval& v) +{ + char ts[TIMEBUF_SIZE]; + snort::ts_print(&v, ts, true); + + js.put(f, ts); +} + +void JsonExtractorLogger::ts_unix(const char* f, const struct timeval& v) +{ + double sec = (uint64_t)v.tv_sec; + double usec = (uint64_t)v.tv_usec; + + js.put(f, sec + usec / 1000000.0, 6); +} + +void JsonExtractorLogger::ts_sec(const char* f, const struct timeval& v) +{ + uint64_t sec = (uint64_t)v.tv_sec; + + js.uput(f, sec); +} + +void JsonExtractorLogger::ts_usec(const char* f, const struct timeval& v) +{ + uint64_t sec = (uint64_t)v.tv_sec; + uint64_t usec = (uint64_t)v.tv_usec; + + js.uput(f, sec * 1000000 + usec); +} diff --git a/src/network_inspectors/extractor/extractor_json_logger.h b/src/network_inspectors/extractor/extractor_json_logger.h index 282fded65..c511201d5 100644 --- a/src/network_inspectors/extractor/extractor_json_logger.h +++ b/src/network_inspectors/extractor/extractor_json_logger.h @@ -22,7 +22,6 @@ #include -#include "framework/value.h" #include "helpers/json_stream.h" #include "extractor_logger.h" @@ -30,8 +29,7 @@ class JsonExtractorLogger : public ExtractorLogger { public: - JsonExtractorLogger(snort::Connector* conn) : ExtractorLogger(conn), oss(), js(oss) - { } + JsonExtractorLogger(snort::Connector*, TimeType); void add_field(const char*, const char*) override; void add_field(const char*, const char*, size_t) override; @@ -43,8 +41,15 @@ public: void close_record(const snort::Connector::ID&) override; private: + void ts_snort(const char*, const struct timeval&); + void ts_snort_yy(const char*, const struct timeval&); + void ts_unix(const char*, const struct timeval&); + void ts_sec(const char*, const struct timeval&); + void ts_usec(const char*, const struct timeval&); + std::ostringstream oss; snort::JsonStream js; + void (JsonExtractorLogger::*add_ts)(const char*, const struct timeval&); }; #endif diff --git a/src/network_inspectors/extractor/extractor_logger.cc b/src/network_inspectors/extractor/extractor_logger.cc index 07c992d09..b32ce1ac5 100644 --- a/src/network_inspectors/extractor/extractor_logger.cc +++ b/src/network_inspectors/extractor/extractor_logger.cc @@ -63,7 +63,7 @@ Connector* ExtractorLogger::get_connector(const std::string& conn_name) return nullptr; } -ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, const std::string& conn_name) +ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, const std::string& conn_name, TimeType ts_type) { ExtractorLogger* logger = nullptr; @@ -74,10 +74,10 @@ ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, const std::stri switch (f_type) { case FormatType::CSV: - logger = new CsvExtractorLogger(output_conn); + logger = new CsvExtractorLogger(output_conn, ts_type); break; case FormatType::JSON: - logger = new JsonExtractorLogger(output_conn); + logger = new JsonExtractorLogger(output_conn, ts_type); break; case FormatType::MAX: // fallthrough default: diff --git a/src/network_inspectors/extractor/extractor_logger.h b/src/network_inspectors/extractor/extractor_logger.h index 59a9f3e1e..3f0d54426 100644 --- a/src/network_inspectors/extractor/extractor_logger.h +++ b/src/network_inspectors/extractor/extractor_logger.h @@ -32,7 +32,7 @@ class ExtractorLogger { public: - static ExtractorLogger* make_logger(FormatType, const std::string&); + static ExtractorLogger* make_logger(FormatType, const std::string&, TimeType); ExtractorLogger(snort::Connector* conn) : output_conn(conn) { }