From: Oleksii Shumeiko -X (oshumeik - SOFTSERVE INC at Cisco) Date: Wed, 4 Sep 2024 10:00:40 +0000 (+0000) Subject: Pull request #4431: JSON formatting for data logging X-Git-Tag: 3.3.5.0~2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b1d96796a924d42dd9fe0b89b567ea415b82ae59;p=thirdparty%2Fsnort3.git Pull request #4431: JSON formatting for data logging Merge in SNORT/snort3 from ~OSHUMEIK/snort3:extr_json to master Squashed commit of the following: commit 2bd2c95c4d5ab3b10dc1600f79e77e0c5fe2ab9e Author: Oleksii Shumeiko Date: Wed Sep 4 09:56:08 2024 +0300 extractor: mention a field in initialization list Despite that initialization order is defined by class fields only, an explicit initializer in the list will add cross-validation. commit 7eeaf9b8814aab9331a0e6d507c7e356ee8a9582 Author: Oleksii Shumeiko Date: Tue Sep 3 13:50:18 2024 +0300 extractor: add unit tests for enum types commit 08928ef3bddf3f61878fe2a936d6be96f2a410e1 Author: Oleksii Shumeiko Date: Tue Sep 3 10:49:41 2024 +0300 extractor: remove unused headers commit 4a6f42fd87d00ce27d5b8b27add094921c85fd5e Author: Oleksii Shumeiko Date: Wed Aug 28 12:13:52 2024 +0300 extractor: add json logger commit b783608bc41247bd3d45e8acedfc87c5dea445c2 Author: Oleksii Shumeiko Date: Wed Aug 28 12:00:25 2024 +0300 extractor: fix local variable Declare a local variable to be static to not interfere at linking stage. commit 077f8133e94079ec86ee5e053b96f2b45afa71de Author: Oleksii Shumeiko Date: Wed Aug 28 11:55:23 2024 +0300 extractor: take a note of FIXIT-P in key points commit 08ed77f484fa19a4cd384af4ee62565a5ea81976 Author: Oleksii Shumeiko Date: Wed Aug 28 11:47:49 2024 +0300 extractor: add field name to logging function Field order is not preset for all formattings. Thus, a caller must pass a field name for proper logging. commit 7a81360468ffe1431c4f93d384b295f659c7b93e Author: Oleksii Shumeiko Date: Tue Aug 27 15:21:38 2024 +0300 extractor: fix guard-macro names --- diff --git a/src/network_inspectors/extractor/CMakeLists.txt b/src/network_inspectors/extractor/CMakeLists.txt index f6451cc7c..20bbc9775 100644 --- a/src/network_inspectors/extractor/CMakeLists.txt +++ b/src/network_inspectors/extractor/CMakeLists.txt @@ -5,6 +5,8 @@ set( FILE_LIST extractor_csv_logger.h extractor_event_handlers.h extractor_http_event_handler.cc + extractor_json_logger.cc + extractor_json_logger.h extractor_logger.cc extractor_logger.h extractor_service.cc diff --git a/src/network_inspectors/extractor/extractor.cc b/src/network_inspectors/extractor/extractor.cc index b569f5005..f2f6fddff 100644 --- a/src/network_inspectors/extractor/extractor.cc +++ b/src/network_inspectors/extractor/extractor.cc @@ -65,7 +65,7 @@ static const Parameter extractor_proto_params[] = static const Parameter s_params[] = { - { "formatting", Parameter::PT_ENUM, "csv", "csv", + { "formatting", Parameter::PT_ENUM, "csv | json", "csv", "output format for extractor" }, { "output", Parameter::PT_ENUM, "stdout", "stdout", diff --git a/src/network_inspectors/extractor/extractor_csv_logger.cc b/src/network_inspectors/extractor/extractor_csv_logger.cc index 4b9c96960..d497b9cd6 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.cc +++ b/src/network_inspectors/extractor/extractor_csv_logger.cc @@ -23,10 +23,10 @@ #include "extractor_csv_logger.h" -#include #include +#include -THREAD_LOCAL bool first_write; +static THREAD_LOCAL bool first_write; void CsvExtractorLogger::add_header() { @@ -56,7 +56,7 @@ void CsvExtractorLogger::close_record() writer->unlock(); } -void CsvExtractorLogger::add_field(const snort::Value& v) +void CsvExtractorLogger::add_field(const char*, const snort::Value& v) { switch (v.get_type()) { diff --git a/src/network_inspectors/extractor/extractor_csv_logger.h b/src/network_inspectors/extractor/extractor_csv_logger.h index e996ec441..76f5e4fc4 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.h +++ b/src/network_inspectors/extractor/extractor_csv_logger.h @@ -17,8 +17,8 @@ //-------------------------------------------------------------------------- // csv_logger.h author Anna Norokh -#ifndef CSV_LOGGER_H -#define CSV_LOGGER_H +#ifndef EXTRACTOR_CSV_LOGGER_H +#define EXTRACTOR_CSV_LOGGER_H #include "framework/value.h" @@ -38,7 +38,7 @@ public: ~CsvExtractorLogger() override; void add_header() override; - void add_field(const snort::Value&) override; + void add_field(const char*, const snort::Value&) override; void open_record() override; void close_record() override; diff --git a/src/network_inspectors/extractor/extractor_http_event_handler.cc b/src/network_inspectors/extractor/extractor_http_event_handler.cc index b76c50f46..dd3acd320 100644 --- a/src/network_inspectors/extractor/extractor_http_event_handler.cc +++ b/src/network_inspectors/extractor/extractor_http_event_handler.cc @@ -35,6 +35,7 @@ using namespace snort; +// FIXIT-P: inspector's data passes many functions before getting to the logger typedef Value* (*GetFunc) (DataEvent*, Packet*, Flow*); @@ -236,8 +237,9 @@ void HttpExtractorEventHandler::handle(DataEvent& event, Flow* flow) logger.open_record(); for (const auto& field : fields) { + // FIXIT-P: this is way too slow (a map with a string key type) auto val = std::unique_ptr(event_getters[field](&event, p, flow)); - logger.add_field(*val.get()); + logger.add_field(field.c_str(), *val.get()); } logger.close_record(); diff --git a/src/network_inspectors/extractor/extractor_json_logger.cc b/src/network_inspectors/extractor/extractor_json_logger.cc new file mode 100644 index 000000000..a286bc87c --- /dev/null +++ b/src/network_inspectors/extractor/extractor_json_logger.cc @@ -0,0 +1,62 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// json_logger.cc author Cisco + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "extractor_json_logger.h" + +#include + +void JsonExtractorLogger::open_record() +{ + oss.str(""); + js.open(); +} + +void JsonExtractorLogger::close_record() +{ + js.close(); + + writer->lock(); + writer->write(oss.str().c_str()); + writer->unlock(); +} + +void JsonExtractorLogger::add_field(const char* f, const snort::Value& v) +{ + switch (v.get_type()) + { + case snort::Value::ValueType::VT_UNUM: + js.uput(f, v.get_uint64()); + break; + + case snort::Value::ValueType::VT_STR: + js.put(f, v.get_string()); + break; + + case snort::Value::ValueType::VT_BOOL: // fallthrough + case snort::Value::ValueType::VT_NUM: // fallthrough + case snort::Value::ValueType::VT_REAL: // fallthrough + default: + assert(false); + break; + } +} diff --git a/src/network_inspectors/extractor/extractor_json_logger.h b/src/network_inspectors/extractor/extractor_json_logger.h new file mode 100644 index 000000000..ca98925c3 --- /dev/null +++ b/src/network_inspectors/extractor/extractor_json_logger.h @@ -0,0 +1,52 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- +// json_logger.h author Cisco + +#ifndef EXTRACTOR_JSON_LOGGER_H +#define EXTRACTOR_JSON_LOGGER_H + +#include + +#include "framework/value.h" +#include "helpers/json_stream.h" + +#include "extractor_logger.h" +#include "extractor_writer.h" + +class JsonExtractorLogger : public ExtractorLogger +{ +public: + JsonExtractorLogger(OutputType o_type, const std::vector& fields) + : ExtractorLogger(fields), writer(ExtractorWriter::make_writer(o_type)), oss(), js(oss) + { } + + ~JsonExtractorLogger() override + { delete writer; } + + void add_field(const char*, const snort::Value&) override; + void open_record() override; + void close_record() override; + +private: + ExtractorWriter* const writer; + std::ostringstream oss; + snort::JsonStream js; + +}; + +#endif diff --git a/src/network_inspectors/extractor/extractor_logger.cc b/src/network_inspectors/extractor/extractor_logger.cc index fa8d7cae1..9cfccd9a4 100644 --- a/src/network_inspectors/extractor/extractor_logger.cc +++ b/src/network_inspectors/extractor/extractor_logger.cc @@ -26,6 +26,7 @@ #include #include "extractor_csv_logger.h" +#include "extractor_json_logger.h" ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, OutputType o_type, const std::vector& fields) @@ -40,9 +41,39 @@ ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, OutputType o_ty case FormatType::CSV: logger = new CsvExtractorLogger(o_type, fields); break; + case FormatType::JSON: + logger = new JsonExtractorLogger(o_type, fields); + break; + case FormatType::MAX: // fallthrough + default: + break; } assert(logger); return logger; } + +#ifdef UNIT_TEST + +#include "catch/snort_catch.h" + +#include + +using namespace snort; + +TEST_CASE("Format Type", "[extractor]") +{ + SECTION("to string") + { + FormatType csv = FormatType::CSV; + FormatType json = FormatType::JSON; + FormatType max = FormatType::MAX; + + CHECK_FALSE(strcmp("csv", csv.c_str())); + CHECK_FALSE(strcmp("json", json.c_str())); + CHECK_FALSE(strcmp("(not set)", max.c_str())); + } +} + +#endif diff --git a/src/network_inspectors/extractor/extractor_logger.h b/src/network_inspectors/extractor/extractor_logger.h index 359a183ff..68e4e35e3 100644 --- a/src/network_inspectors/extractor/extractor_logger.h +++ b/src/network_inspectors/extractor/extractor_logger.h @@ -32,7 +32,9 @@ class FormatType public: enum Value : uint8_t { - CSV + CSV, + JSON, + MAX }; FormatType() = default; @@ -48,6 +50,9 @@ public: { case CSV: return "csv"; + case JSON: + return "json"; + case MAX: // fallthrough default: return "(not set)"; } @@ -71,7 +76,8 @@ public: virtual void add_header() {} virtual void add_footer() {} - virtual void add_field(const snort::Value&) {} + // FIXIT-P: replace Value type designed for parsing with a better type + virtual void add_field(const char*, const snort::Value&) {} virtual void open_record() {} virtual void close_record() {} diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc index e9e1fabe5..3c771e60a 100644 --- a/src/network_inspectors/extractor/extractor_service.cc +++ b/src/network_inspectors/extractor/extractor_service.cc @@ -179,3 +179,27 @@ HttpExtractorService::HttpExtractorService(uint32_t tenant, const std::vector + +using namespace snort; + +TEST_CASE("Service Type", "[extractor]") +{ + SECTION("to string") + { + ServiceType http = ServiceType::HTTP; + ServiceType undef = ServiceType::UNDEFINED; + ServiceType max = ServiceType::MAX; + + CHECK_FALSE(strcmp("http", http.c_str())); + CHECK_FALSE(strcmp("(not set)", undef.c_str())); + CHECK_FALSE(strcmp("(not set)", max.c_str())); + } +} + +#endif diff --git a/src/network_inspectors/extractor/extractor_service.h b/src/network_inspectors/extractor/extractor_service.h index 2bb560ea1..6725f462f 100644 --- a/src/network_inspectors/extractor/extractor_service.h +++ b/src/network_inspectors/extractor/extractor_service.h @@ -17,8 +17,8 @@ //-------------------------------------------------------------------------- // extractor_service.h author Maya Dagon -#ifndef EXTRACTOR_SERVICES_H -#define EXTRACTOR_SERVICES_H +#ifndef EXTRACTOR_SERVICE_H +#define EXTRACTOR_SERVICE_H #include #include @@ -68,6 +68,7 @@ struct ServiceBlueprint std::vector supported_fields; }; +// FIXIT-P: make a template with Logger and Writer as parameters class ExtractorService { public: diff --git a/src/network_inspectors/extractor/extractor_writer.cc b/src/network_inspectors/extractor/extractor_writer.cc index 11e521674..227b65f6c 100644 --- a/src/network_inspectors/extractor/extractor_writer.cc +++ b/src/network_inspectors/extractor/extractor_writer.cc @@ -34,3 +34,25 @@ ExtractorWriter* ExtractorWriter::make_writer(OutputType o_type) return nullptr; } } + +#ifdef UNIT_TEST + +#include "catch/snort_catch.h" + +#include + +using namespace snort; + +TEST_CASE("Output Type", "[extractor]") +{ + SECTION("to string") + { + OutputType std = OutputType::STD; + OutputType max = OutputType::MAX; + + CHECK_FALSE(strcmp("stdout", std.c_str())); + CHECK_FALSE(strcmp("(not set)", max.c_str())); + } +} + +#endif