From: Vitalii Serhiiovych Horbatov -X (vhorbato - SOFTSERVE INC at Cisco) Date: Tue, 26 Nov 2024 14:01:43 +0000 (+0000) Subject: Pull request #4514: extractor: replace Writer with Connector X-Git-Tag: 3.6.0.0~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=58dff7d37e6974a12cc2719150ad7c7352344085;p=thirdparty%2Fsnort3.git Pull request #4514: extractor: replace Writer with Connector Merge in SNORT/snort3 from ~VHORBATO/snort3:extractor_conn to master Squashed commit of the following: commit 471be3fed9f5dd10ed724fdb10d338a5d6a9466e Author: vhorbato Date: Wed Nov 20 18:41:45 2024 +0200 extractor: update thread initialization commit 3e87e761431d77f39abd4c1ea6183a49f3c0b18b Author: vhorbato Date: Thu Nov 7 17:13:43 2024 +0200 build: update docs about the bump of C++ compiler supported feature set requirement commit 1cc99e4e1d7784beb046449697b33324c0ba622d Author: Yehor Velykozhon Date: Wed Nov 13 11:56:40 2024 +0200 connectors: update config transition commit 190e9bb3ce86ee9cdc43414ab2e592b334d83c2e Author: vhorbato Date: Tue Nov 5 19:08:51 2024 +0200 connectors: add metadata support to Connector API commit 3df7a97195f51463bd881b6decd378d6a32b18b6 Author: vhorbato Date: Wed Oct 30 14:44:24 2024 +0200 connectors: add std I/O connector commit e5aa4bf71a73cc42824b37448f7a62eca2abea52 Author: vhorbato Date: Wed Oct 30 14:40:57 2024 +0200 extractor: replace writer with connector commit 4a43a077933c59c88677cdb827a72ab77919b7a9 Author: vhorbato Date: Mon Oct 28 18:03:45 2024 +0200 extractor: make csv formatter call writer only once commit b2c5a3e2075af951c3554b09e8d087b0979f557e Author: vhorbato Date: Wed Oct 23 14:07:58 2024 +0300 extractor: make logger thread_local --- diff --git a/README.md b/README.md index 3197109b0..c731ce60a 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ the latest: * daq from https://github.com/snort3/libdaq for packet IO * dnet from https://github.com/dugsong/libdnet.git for network utility functions * flex >= 2.6.0 from https://github.com/westes/flex for JavaScript syntax parser -* g++ >= 5 or other C++14 compiler +* g++ >= 7 or other C++17 compiler * hwloc from https://www.open-mpi.org/projects/hwloc/ for CPU affinity management * LuaJIT from http://luajit.org for configuration and scripting * OpenSSL from https://www.openssl.org/source/ for SHA and MD5 file signatures, diff --git a/cmake/compiler_features.cmake b/cmake/compiler_features.cmake index 23d447936..cf997e07a 100644 --- a/cmake/compiler_features.cmake +++ b/cmake/compiler_features.cmake @@ -1,5 +1,5 @@ set ( _SAVE_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS} ) -set ( CMAKE_REQUIRED_FLAGS "-std=c++14 -fPIC -shared -Wl,-undefined,dynamic_lookup" ) +set ( CMAKE_REQUIRED_FLAGS "-std=c++17 -fPIC -shared -Wl,-undefined,dynamic_lookup" ) unset ( HAVE_EXTERN_GNU_TLS ) check_cxx_source_compiles ( diff --git a/doc/user/connectors.txt b/doc/user/connectors.txt index 3be255e0d..20c0bcb93 100644 --- a/doc/user/connectors.txt +++ b/doc/user/connectors.txt @@ -18,12 +18,14 @@ element for client module configuration. The 'direction' element may have a default value, for instance TcpConnector is 'duplex'. -There are currently two implementations of Connectors: +Currently there are the following implementations of the Connector: * TcpConnector - Exchange messages over a tcp channel. * FileConnector - Write messages to files and read messages from files. +* StdConnector - Exchange messages over a standard input/output. + ===== TcpConnector @@ -97,3 +99,30 @@ An example segment of FileConnector configuration: }, } + +===== StdConnector + +StdConnector is an implementation of a Connector that has the capability to +read from the standard input and write to the standard output. Its principal +purpose is testing. + +All messages transmitted through the stdout are automatically appended with a new +line. Therefore, it is expected that each message read from the stdin will also be +separated by a newline. + +The std_connector module automatically sets-up three default connectors on startup: + +* `stdout`: the default transmit connector +* `stdin`: the default receive connector +* `stdio`: the default duplex connector. + +An example segment of a duplex StdConnector configuration: + +std_connector = +{ + { + connector = 'std_in_out', + direction = 'duplex' + } +} + diff --git a/doc/user/extractor.txt b/doc/user/extractor.txt index 3a4da61a2..872324ca5 100644 --- a/doc/user/extractor.txt +++ b/doc/user/extractor.txt @@ -9,7 +9,8 @@ The module's configuration consists of two parts: * global parameters ** `formatting` - log record format - ** `output` - where to write logs + ** `connector` - Connector object through which logs will be sent. See Connectors page + for more details. * protocol-targeted parameters bind the targeted service and events with filters and a set of fields to log ** `service` - protocol name @@ -24,7 +25,7 @@ things it allows tenants to get independent data logging configurations. extractor = { formatting = 'csv', - output = 'stdout', + connector = 'stdout', protocols = { @@ -88,10 +89,12 @@ inspection) would print some FTP logs to standard output in CSV format. FTP sessions with basic fields: + std_connector = { } + extractor = { formatting = csv', - output = 'stdout', + connector = 'stdout', protocols = { {service = 'ftp', on_events = 'eot', fields = 'ts, command, user'} @@ -110,10 +113,12 @@ Output: Or FTP requests with the same set of fields: + std_connector = { } + extractor = { formatting = 'csv', - output = 'stdout', + connector = 'stdout', protocols = { {service = 'ftp', on_events = 'request', fields = 'ts, command, user'} diff --git a/doc/user/features.txt b/doc/user/features.txt index 1db7db9ce..823903bd4 100644 --- a/doc/user/features.txt +++ b/doc/user/features.txt @@ -32,6 +32,10 @@ include::byte_math.txt[] include::byte_options.txt[] +==== Connectors + +include::connectors.txt[] + === Consolidated Config include::dump_config.txt[] @@ -53,10 +57,6 @@ messaging subsystems. include::high_availability.txt[] -==== Connector - -include::connectors.txt[] - ==== Side Channel include::side_channel.txt[] diff --git a/doc/user/tutorial.txt b/doc/user/tutorial.txt index 3d51a4095..82997ded8 100644 --- a/doc/user/tutorial.txt +++ b/doc/user/tutorial.txt @@ -7,7 +7,7 @@ out more advanced usage. Required: -* a compiler that supports the C++14 feature set +* a compiler that supports the C++17 feature set * cmake to build from source diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e0ffa4103..546b1e03e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -185,6 +185,7 @@ add_executable( snort $ $ $ + $ $ $ $ diff --git a/src/connectors/CMakeLists.txt b/src/connectors/CMakeLists.txt index dd6cc4f01..6453f1575 100644 --- a/src/connectors/CMakeLists.txt +++ b/src/connectors/CMakeLists.txt @@ -1,6 +1,7 @@ add_subdirectory(file_connector) add_subdirectory(tcp_connector) +add_subdirectory(std_connector) add_library( connectors OBJECT connectors.cc diff --git a/src/connectors/connectors.cc b/src/connectors/connectors.cc index f02c51b54..9a256528b 100644 --- a/src/connectors/connectors.cc +++ b/src/connectors/connectors.cc @@ -29,10 +29,12 @@ using namespace snort; extern const BaseApi* file_connector[]; extern const BaseApi* tcp_connector[]; +extern const BaseApi* std_connector[]; void load_connectors() { PluginManager::load_plugins(file_connector); PluginManager::load_plugins(tcp_connector); + PluginManager::load_plugins(std_connector); } diff --git a/src/connectors/dev_notes.txt b/src/connectors/dev_notes.txt index d53c02b40..1664d080d 100644 --- a/src/connectors/dev_notes.txt +++ b/src/connectors/dev_notes.txt @@ -1,9 +1,17 @@ Connectors house the collection of the plugin-type Connector. As defined .../framework/connector.h, the Connector object implements a simplex -communications channel that is in-turn used by SideChannel objects. +communications channel that is in-turn used by SideChannel and other objects. Connectors have the standard plugin api and instantiation/destruction protocols. The file_connector writes messages to a file and reads messages from a file. -Configuration entries map side channels to connector instances. +Configuration entries map user modules to connector instances. + +Connector also accepts metadata for the transmit_message function, which is +implemented using the `Connector::ID` object. The metadata can be presented +as a string or a number. The specific Connector implements the `Connector::get_id` +method, which should convert the input data into the form that it will work with +in the `transmit_message`. Therefore, the user module should only call the +`Connector::transmit_message` function with metadata obtained from the +corresponding `Connector::get_id call`. \ No newline at end of file diff --git a/src/connectors/file_connector/file_connector.cc b/src/connectors/file_connector/file_connector.cc index 7d448fc2a..4e312347b 100644 --- a/src/connectors/file_connector/file_connector.cc +++ b/src/connectors/file_connector/file_connector.cc @@ -36,23 +36,6 @@ using namespace snort; THREAD_LOCAL SimpleStats file_connector_stats; THREAD_LOCAL ProfileStats file_connector_perfstats; -FileConnectorCommon::FileConnectorCommon(FileConnectorConfig::FileConnectorConfigSet* conf) -{ - config_set = (ConnectorConfig::ConfigSet*)conf; -} - -FileConnectorCommon::~FileConnectorCommon() -{ - for ( auto conf : *config_set ) - { - FileConnectorConfig* fconf = (FileConnectorConfig*)conf; - delete fconf; - } - - config_set->clear(); - delete config_set; -} - bool FileConnector::internal_transmit_message(const ConnectorMsg& msg) { if ( !msg.get_data() or msg.get_length() == 0 ) @@ -74,10 +57,10 @@ bool FileConnector::internal_transmit_message(const ConnectorMsg& msg) return file.good(); } -bool FileConnector::transmit_message(const ConnectorMsg& msg) +bool FileConnector::transmit_message(const ConnectorMsg& msg, const ID&) { return internal_transmit_message(msg); } -bool FileConnector::transmit_message(const ConnectorMsg&& msg) +bool FileConnector::transmit_message(const ConnectorMsg&& msg, const ID&) { return internal_transmit_message(msg); } ConnectorMsg FileConnector::receive_message_binary() @@ -176,7 +159,7 @@ static Connector* file_connector_tinit_receive(std::string& filename, const File // Create a per-thread object static Connector* file_connector_tinit(const ConnectorConfig& config) { - const FileConnectorConfig& fconf = (const FileConnectorConfig&)config; + const FileConnectorConfig& fconf = static_cast(config); std::string filename = FILE_CONNECTOR_NAME; filename += "_"; @@ -202,16 +185,13 @@ static void file_connector_tterm(Connector* connector) static ConnectorCommon* file_connector_ctor(Module* m) { FileConnectorModule* mod = (FileConnectorModule*)m; - FileConnectorCommon* file_connector_common = new FileConnectorCommon( - mod->get_and_clear_config()); - return file_connector_common; + return new ConnectorCommon(mod->get_and_clear_config()); } static void file_connector_dtor(ConnectorCommon* c) { - FileConnectorCommon* fc = (FileConnectorCommon*)c; - delete fc; + delete c; } const ConnectorApi file_connector_api = @@ -220,7 +200,7 @@ const ConnectorApi file_connector_api = PT_CONNECTOR, sizeof(ConnectorApi), CONNECTOR_API_VERSION, - 1, + 2, API_RESERVED, API_OPTIONS, FILE_CONNECTOR_NAME, diff --git a/src/connectors/file_connector/file_connector.h b/src/connectors/file_connector/file_connector.h index f6cae75e1..062fd763b 100644 --- a/src/connectors/file_connector/file_connector.h +++ b/src/connectors/file_connector/file_connector.h @@ -43,20 +43,13 @@ public: uint32_t connector_msg_length; }; -class FileConnectorCommon : public snort::ConnectorCommon -{ -public: - FileConnectorCommon(FileConnectorConfig::FileConnectorConfigSet*); - ~FileConnectorCommon(); -}; - class FileConnector : public snort::Connector { public: FileConnector(const FileConnectorConfig& conf) : Connector(conf), cfg(conf) {} - bool transmit_message(const snort::ConnectorMsg&) override; - bool transmit_message(const snort::ConnectorMsg&&) override; + bool transmit_message(const snort::ConnectorMsg&, const ID& = null) override; + bool transmit_message(const snort::ConnectorMsg&&, const ID& = null) override; snort::ConnectorMsg receive_message(bool) override; diff --git a/src/connectors/file_connector/file_connector_config.h b/src/connectors/file_connector/file_connector_config.h index 25c0da164..eaa9eb81d 100644 --- a/src/connectors/file_connector/file_connector_config.h +++ b/src/connectors/file_connector/file_connector_config.h @@ -34,8 +34,6 @@ public: bool text_format; std::string name; - - typedef std::vector FileConnectorConfigSet; }; #endif diff --git a/src/connectors/file_connector/file_connector_module.cc b/src/connectors/file_connector/file_connector_module.cc index 490cfdb8b..789715361 100644 --- a/src/connectors/file_connector/file_connector_module.cc +++ b/src/connectors/file_connector/file_connector_module.cc @@ -58,19 +58,23 @@ extern THREAD_LOCAL ProfileStats file_connector_perfstats; FileConnectorModule::FileConnectorModule() : Module(FILE_CONNECTOR_NAME, FILE_CONNECTOR_HELP, file_connector_params, true) -{ - config_set = new FileConnectorConfig::FileConnectorConfigSet; -} - -FileConnectorModule::~FileConnectorModule() -{ - delete config; - delete config_set; -} +{ } ProfileStats* FileConnectorModule::get_profile() const { return &file_connector_perfstats; } +// clear my working config and hand-over the compiled list to the caller +ConnectorConfig::ConfigSet FileConnectorModule::get_and_clear_config() +{ return std::move(config_set); } + +bool FileConnectorModule::begin(const char*, int, SnortConfig*) +{ + if (!config) + config = std::make_unique(); + + return true; +} + bool FileConnectorModule::set(const char*, Value& v, SnortConfig*) { if ( v.is("connector") ) @@ -102,31 +106,10 @@ bool FileConnectorModule::set(const char*, Value& v, SnortConfig*) return true; } -// clear my working config and hand-over the compiled list to the caller -FileConnectorConfig::FileConnectorConfigSet* FileConnectorModule::get_and_clear_config() -{ - FileConnectorConfig::FileConnectorConfigSet* temp_config = config_set; - config = nullptr; - config_set = nullptr; - return temp_config; -} - -bool FileConnectorModule::begin(const char*, int, SnortConfig*) -{ - if ( !config ) - { - config = new FileConnectorConfig; - } - return true; -} - bool FileConnectorModule::end(const char*, int idx, SnortConfig*) { if (idx != 0) - { - config_set->emplace_back(config); - config = nullptr; - } + config_set.emplace_back(std::move(config)); return true; } diff --git a/src/connectors/file_connector/file_connector_module.h b/src/connectors/file_connector/file_connector_module.h index 76b17a13f..f68a0cc19 100644 --- a/src/connectors/file_connector/file_connector_module.h +++ b/src/connectors/file_connector/file_connector_module.h @@ -21,8 +21,11 @@ #ifndef FILE_CONNECTOR_MODULE_H #define FILE_CONNECTOR_MODULE_H +#include "framework/connector.h" #include "framework/module.h" +#include + #include "file_connector_config.h" #define FILE_CONNECTOR_NAME "file_connector" @@ -32,13 +35,12 @@ class FileConnectorModule : public snort::Module { public: FileConnectorModule(); - ~FileConnectorModule() override; bool set(const char*, snort::Value&, snort::SnortConfig*) override; bool begin(const char*, int, snort::SnortConfig*) override; bool end(const char*, int, snort::SnortConfig*) override; - FileConnectorConfig::FileConnectorConfigSet* get_and_clear_config(); + snort::ConnectorConfig::ConfigSet get_and_clear_config(); const PegInfo* get_pegs() const override; PegCount* get_counts() const override; @@ -49,8 +51,8 @@ public: { return GLOBAL; } private: - FileConnectorConfig::FileConnectorConfigSet* config_set; - FileConnectorConfig* config = nullptr; + snort::ConnectorConfig::ConfigSet config_set; + std::unique_ptr config; }; #endif diff --git a/src/connectors/file_connector/test/file_connector_module_test.cc b/src/connectors/file_connector/test/file_connector_module_test.cc index 1c26908c6..1719efe3e 100644 --- a/src/connectors/file_connector/test/file_connector_module_test.cc +++ b/src/connectors/file_connector/test/file_connector_module_test.cc @@ -83,23 +83,15 @@ TEST(file_connector_module, test) module.end("file_connector", 1, nullptr); module.end("file_connector", 0, nullptr); - FileConnectorConfig::FileConnectorConfigSet* config_set = module.get_and_clear_config(); + ConnectorConfig::ConfigSet config_set = module.get_and_clear_config(); - CHECK(nullptr != config_set); + CHECK(1 == config_set.size()); - CHECK(1 == config_set->size()); - - FileConnectorConfig config = *(config_set->front()); + const FileConnectorConfig& config = static_cast(*config_set.front()); CHECK("rx" == config.name); CHECK("rx" == config.connector_name); CHECK(Connector::CONN_RECEIVE == config.direction); CHECK(true == config.text_format); - - for ( auto conf : *config_set ) - delete conf; - - config_set->clear(); - delete config_set; } int main(int argc, char** argv) diff --git a/src/connectors/file_connector/test/file_connector_test.cc b/src/connectors/file_connector/test/file_connector_test.cc index aa320b4bc..b5afc7b98 100644 --- a/src/connectors/file_connector/test/file_connector_test.cc +++ b/src/connectors/file_connector/test/file_connector_test.cc @@ -64,12 +64,10 @@ unsigned ThreadConfig::get_instance_max() { return 1; } FileConnectorModule::FileConnectorModule() : Module("FC", "FC Help", nullptr) -{ config_set = nullptr; } +{ } -FileConnectorConfig::FileConnectorConfigSet* FileConnectorModule::get_and_clear_config() -{ return new FileConnectorConfig::FileConnectorConfigSet; } - -FileConnectorModule::~FileConnectorModule() = default; +ConnectorConfig::ConfigSet FileConnectorModule::get_and_clear_config() +{ return ConnectorConfig::ConfigSet(); } ProfileStats* FileConnectorModule::get_profile() const { return nullptr; } diff --git a/src/connectors/std_connector/CMakeLists.txt b/src/connectors/std_connector/CMakeLists.txt new file mode 100644 index 000000000..4c81ef8d1 --- /dev/null +++ b/src/connectors/std_connector/CMakeLists.txt @@ -0,0 +1,7 @@ + +add_library( std_connector OBJECT + std_connector.cc + std_connector.h +) + +add_subdirectory(test) \ No newline at end of file diff --git a/src/connectors/std_connector/dev_notes.txt b/src/connectors/std_connector/dev_notes.txt new file mode 100644 index 000000000..8f7b83722 --- /dev/null +++ b/src/connectors/std_connector/dev_notes.txt @@ -0,0 +1,21 @@ +Implement a connector plugin that is capable of reading and writing +messages from and to the standard input/output. + +This connector will print each message to the standard output, with +each message appearing on a new line. If Connector::ID is supplied +it will be prepended to the message in format _": \n"_. + +The std_connector can also read messages from the standard input, +using the newline character as the delimiter. + +The configuration of the std_connector Connector results in the creation +of a single ConnectorCommon object. This object is responsible for holding +a list of all the Connectors being configured. Within the ConnectorCommon +object, there is a vector<> that stores individual Connector config objects. +The ConnectorManager then uses this vector<> to instantiate the desired set +of Connectors. + +std_connector pre-configures 3 default connectors: +* stdout: default transmit connector +* stdin: default receive connector +* stdio: default duplex connector \ No newline at end of file diff --git a/src/connectors/std_connector/std_connector.cc b/src/connectors/std_connector/std_connector.cc new file mode 100644 index 000000000..969153b2a --- /dev/null +++ b/src/connectors/std_connector/std_connector.cc @@ -0,0 +1,244 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// std_connector.cc author Vitalii Horbatov +// based on work by Anna Norokh + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "std_connector.h" + +#include +#include + +#include "profiler/profiler_defs.h" +#include "log/text_log.h" + +using namespace snort; + +/* Globals ****************************************************************/ + +struct StdConnectorStats +{ + PegCount messages_received; + PegCount messages_transmitted; +}; + +THREAD_LOCAL StdConnectorStats std_connector_stats; +THREAD_LOCAL ProfileStats std_connector_perfstats; + +//------------------------------------------------------------------------- +// module stuff +//------------------------------------------------------------------------- + +static const Parameter std_connector_params[] = +{ + { "connector", Parameter::PT_STRING, nullptr, nullptr, + "connector name" }, + + { "direction", Parameter::PT_ENUM, "receive | transmit | duplex", nullptr, + "usage" }, + + { nullptr, Parameter::PT_MAX, nullptr, nullptr, nullptr } +}; + +static const PegInfo std_connector_pegs[] = +{ + { CountType::SUM, "messages_received", "total number of messages received" }, + { CountType::SUM, "messages_transmitted", "total number of messages transmitted" }, + { CountType::END, nullptr, nullptr } +}; + +StdConnectorModule::StdConnectorModule() : + Module(S_NAME, S_HELP, std_connector_params, true) +{ } + +ProfileStats* StdConnectorModule::get_profile() const +{ return &std_connector_perfstats; } + +bool StdConnectorModule::begin(const char* mod, int idx, SnortConfig*) +{ + if (idx == 0 and strcmp(mod, "std_connector") == 0) + { + auto out_conn = std::make_unique(); + out_conn->direction = Connector::CONN_TRANSMIT; + out_conn->connector_name = "stdout"; + config_set.push_back(std::move(out_conn)); + + auto in_conn = std::make_unique(); + in_conn->direction = Connector::CONN_RECEIVE; + in_conn->connector_name = "stdin"; + config_set.push_back(std::move(in_conn)); + + auto io_conn = std::make_unique(); + io_conn->direction = Connector::CONN_DUPLEX; + io_conn->connector_name = "stdio"; + config_set.push_back(std::move(io_conn)); + } + + if ( !config ) + config = std::make_unique(); + + return true; +} + +bool StdConnectorModule::set(const char*, Value& v, SnortConfig*) +{ + if ( v.is("connector") ) + config->connector_name = v.get_string(); + + else if ( v.is("direction") ) + { + switch ( v.get_uint8() ) + { + case 0: + config->direction = Connector::CONN_RECEIVE; + break; + case 1: + config->direction = Connector::CONN_TRANSMIT; + break; + case 2: + config->direction = Connector::CONN_DUPLEX; + break; + default: + return false; + } + } + return true; +} + +bool StdConnectorModule::end(const char*, int idx, SnortConfig*) +{ + if (idx != 0) + config_set.push_back(std::move(config)); + + return true; +} + +ConnectorConfig::ConfigSet StdConnectorModule::get_and_clear_config() +{ return std::move(config_set); } + +const PegInfo* StdConnectorModule::get_pegs() const +{ return std_connector_pegs; } + +PegCount* StdConnectorModule::get_counts() const +{ return (PegCount*)&std_connector_stats; } + +//------------------------------------------------------------------------- +// connector stuff +//------------------------------------------------------------------------- + +StdConnector::StdConnector(const snort::ConnectorConfig& conf) : + snort::Connector(conf), extr_std_log(TextLog_Init("stdout")) +{ } + +StdConnector::~StdConnector() +{ TextLog_Term(extr_std_log); } + +bool StdConnector::internal_transmit_message(const ConnectorMsg& msg) +{ + if ( !msg.get_data() or msg.get_length() == 0 ) + return false; + + return TextLog_Print(extr_std_log, "%.*s\n", msg.get_length(), msg.get_data()) && + ++(std_connector_stats.messages_transmitted); +} + +bool StdConnector::transmit_message(const ConnectorMsg& msg, const ID&) +{ return internal_transmit_message(msg); } + +bool StdConnector::transmit_message(const ConnectorMsg&& msg, const ID&) +{ return internal_transmit_message(msg); } + +bool StdConnector::flush() +{ return TextLog_Flush(extr_std_log); } + +ConnectorMsg StdConnector::receive_message(bool) +{ + std::string data; + std::getline(std::cin, data); + + uint8_t* data_buf = new uint8_t[data.size()]; + memcpy(data_buf, data.c_str(), data.size()); + + std_connector_stats.messages_received++; + + return ConnectorMsg(data_buf, data.size(), true); +} + +//------------------------------------------------------------------------- +// api stuff +//------------------------------------------------------------------------- + +static Module* mod_ctor() +{ return new StdConnectorModule; } + +static void mod_dtor(Module* m) +{ delete m; } + +static Connector* std_connector_tinit(const ConnectorConfig& config) +{ return new StdConnector(config); } + +static void std_connector_tterm(Connector* connector) +{ delete connector; } + +static ConnectorCommon* std_connector_ctor(Module* m) +{ + StdConnectorModule* mod = (StdConnectorModule*)m; + ConnectorCommon* std_connector_common = new ConnectorCommon(mod->get_and_clear_config()); + return std_connector_common; +} + +static void std_connector_dtor(ConnectorCommon* cc) +{ delete cc; } + +const ConnectorApi std_connector_api = +{ + { + PT_CONNECTOR, + sizeof(ConnectorApi), + CONNECTOR_API_VERSION, + 0, + API_RESERVED, + API_OPTIONS, + S_NAME, + S_HELP, + mod_ctor, + mod_dtor + }, + 0, + nullptr, + nullptr, + std_connector_tinit, + std_connector_tterm, + std_connector_ctor, + std_connector_dtor +}; + +#ifdef BUILDING_SO +SO_PUBLIC const BaseApi* snort_plugins[] = +#else +const BaseApi* std_connector[] = +#endif +{ + &std_connector_api.base, + nullptr +}; + diff --git a/src/connectors/std_connector/std_connector.h b/src/connectors/std_connector/std_connector.h new file mode 100644 index 000000000..3eefdb1ce --- /dev/null +++ b/src/connectors/std_connector/std_connector.h @@ -0,0 +1,82 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// std_connector.h author Vitalii Horbatov + +#ifndef STD_CONNECTOR_H +#define STD_CONNECTOR_H + +#include "framework/connector.h" +#include "framework/module.h" + +#define S_NAME "std_connector" +#define S_HELP "implement the stdout/stdin based connector" + +struct TextLog; + +//------------------------------------------------------------------------- +// module stuff +//------------------------------------------------------------------------- + +class StdConnectorModule : public snort::Module +{ +public: + StdConnectorModule(); + + bool begin(const char*, int, snort::SnortConfig*) override; + bool set(const char*, snort::Value&, snort::SnortConfig*) override; + bool end(const char*, int, snort::SnortConfig*) override; + + snort::ConnectorConfig::ConfigSet get_and_clear_config(); + + const PegInfo* get_pegs() const override; + PegCount* get_counts() const override; + + snort::ProfileStats* get_profile() const override; + + Usage get_usage() const override + { return GLOBAL; } + +private: + snort::ConnectorConfig::ConfigSet config_set; + std::unique_ptr config; +}; + +//------------------------------------------------------------------------- +// connector stuff +//------------------------------------------------------------------------- + +class StdConnector : public snort::Connector +{ +public: + StdConnector(const snort::ConnectorConfig& conf); + ~StdConnector() override; + + bool transmit_message(const snort::ConnectorMsg&, const ID& = null) override; + bool transmit_message(const snort::ConnectorMsg&&, const ID& = null) override; + bool flush() override; + + snort::ConnectorMsg receive_message(bool) override; + +private: + bool internal_transmit_message(const snort::ConnectorMsg&); + + TextLog* extr_std_log; +}; + +#endif diff --git a/src/connectors/std_connector/test/CMakeLists.txt b/src/connectors/std_connector/test/CMakeLists.txt new file mode 100644 index 000000000..52c6248d3 --- /dev/null +++ b/src/connectors/std_connector/test/CMakeLists.txt @@ -0,0 +1,5 @@ +add_cpputest( std_connector_test + SOURCES + ../std_connector.cc + ../../../framework/module.cc +) diff --git a/src/connectors/std_connector/test/std_connector_test.cc b/src/connectors/std_connector/test/std_connector_test.cc new file mode 100644 index 000000000..22da974f2 --- /dev/null +++ b/src/connectors/std_connector/test/std_connector_test.cc @@ -0,0 +1,371 @@ +//-------------------------------------------------------------------------- +// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved. +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License Version 2 as published +// by the Free Software Foundation. You may not use, modify or distribute +// this program under any other version of the GNU General Public License. +// +// This program is distributed in the hope that it will be useful, but +// WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +// General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +//-------------------------------------------------------------------------- + +// std_connector_test.cc author Vitalii Horbatov + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include +#include + +#include "connectors/std_connector/std_connector.h" +#include "main/thread_config.h" + +#include +#include + +using namespace snort; + + +extern const BaseApi* std_connector; +const ConnectorApi* stdc_api = nullptr; + +ConnectorConfig connector_transmit_config; +ConnectorConfig connector_receive_config; + +Module* mod; + +ConnectorCommon* connector_common; + +Connector* connector_transmit; +Connector* connector_receive; + +struct TextLog +{ + std::string accumulated_buffer; + std::string flushed_buffer; +}; + +TextLog output_buffer; + +namespace snort +{ +TextLog* TextLog_Init(const char*, unsigned int = 0, size_t = 0, bool = true) +{ + output_buffer.accumulated_buffer.clear(); + output_buffer.flushed_buffer.clear(); + return &output_buffer; +} + +bool TextLog_Print(TextLog* const txt, const char* fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + char data[100]; + vsnprintf(data, 100, fmt, ap); + txt->accumulated_buffer.append(data); + va_end(ap); + return true; +} + +bool TextLog_Flush(TextLog* const txt) +{ + txt->flushed_buffer.append(txt->accumulated_buffer); + txt->accumulated_buffer.clear(); + return true; +} + +void TextLog_Term(TextLog*) +{ + output_buffer.accumulated_buffer.clear(); + output_buffer.flushed_buffer.clear(); +} + +const char* get_instance_file(std::string& file, const char* name) { file += name; return nullptr; } +unsigned get_instance_id() { return 0; } +unsigned ThreadConfig::get_instance_max() { return 1; } +} + +void show_stats(PegCount*, const PegInfo*, unsigned, const char*) { } +void show_stats(PegCount*, const PegInfo*, const std::vector&, const char*, FILE*) { } + +//------------------------------------------------------------------------- +// connector test +//------------------------------------------------------------------------- + +TEST_GROUP(std_connector_construction) +{ + void setup() override + { + stdc_api = (const ConnectorApi*) std_connector; + connector_transmit_config.direction = Connector::CONN_TRANSMIT; + connector_transmit_config.connector_name = "std_tx"; + + connector_receive_config.direction = Connector::CONN_RECEIVE; + connector_receive_config.connector_name = "std_rx"; + } +}; + +TEST(std_connector_construction, mod_ctor_dtor) +{ + CHECK(std_connector != nullptr); + mod = std_connector->mod_ctor(); + CHECK(mod != nullptr); + std_connector->mod_dtor(mod); +} + +TEST(std_connector_construction, mod_instance_ctor_dtor) +{ + CHECK(std_connector != nullptr); + mod = std_connector->mod_ctor(); + CHECK(mod != nullptr); + connector_common = stdc_api->ctor(mod); + CHECK(connector_common != nullptr); + stdc_api->dtor(connector_common); + std_connector->mod_dtor(mod); +} + +TEST_GROUP(std_connector_tinit_tterm) +{ + void setup() override + { + stdc_api = (const ConnectorApi*)std_connector; + connector_transmit_config.direction = Connector::CONN_TRANSMIT; + connector_transmit_config.connector_name = "std_tx"; + + connector_receive_config.direction = Connector::CONN_RECEIVE; + connector_receive_config.connector_name = "std_rx"; + + CHECK(std_connector != nullptr); + mod = std_connector->mod_ctor(); + CHECK(mod != nullptr); + connector_common = stdc_api->ctor(mod); + CHECK(connector_common != nullptr); + + connector_transmit = stdc_api->tinit(connector_transmit_config); + connector_receive = stdc_api->tinit(connector_receive_config); + + CHECK(connector_transmit != nullptr); + CHECK(connector_receive != nullptr); + } + + void teardown() override + { + stdc_api->tterm(connector_transmit); + stdc_api->tterm(connector_receive); + stdc_api->dtor(connector_common); + std_connector->mod_dtor(mod); + } +}; + +TEST_GROUP(std_connector) +{ + void setup() override + { + stdc_api = (const ConnectorApi*)std_connector; + connector_transmit_config.direction = Connector::CONN_TRANSMIT; + connector_transmit_config.connector_name = "std_tx"; + connector_receive_config.direction = Connector::CONN_RECEIVE; + connector_receive_config.connector_name = "std_rx"; + } +}; + +TEST(std_connector, transmit_flush_empty) +{ + mod = std_connector->mod_ctor(); + CHECK(mod != nullptr); + connector_common = stdc_api->ctor(mod); + CHECK(connector_common != nullptr); + connector_transmit = stdc_api->tinit(connector_transmit_config); + CHECK(connector_transmit != nullptr); + StdConnector* stdc_tt = (StdConnector*)connector_transmit; + + ConnectorMsg null_msg(nullptr, 1, false); + CHECK(stdc_tt->transmit_message(null_msg) == false); + CHECK(output_buffer.accumulated_buffer.empty()); + + uint8_t data[] = ""; + ConnectorMsg empty_msg(data, 0, false); + CHECK(stdc_tt->transmit_message(empty_msg) == false); + CHECK(output_buffer.accumulated_buffer.empty()); + + CHECK(stdc_tt->flush()); + CHECK(output_buffer.accumulated_buffer.empty()); + CHECK(output_buffer.flushed_buffer.empty()); + + stdc_api->tterm(connector_transmit); + stdc_api->dtor(connector_common); + std_connector->mod_dtor(mod); +} + +TEST(std_connector, transmit_flush) +{ + uint32_t len = sizeof("foobar") - 1; + + mod = std_connector->mod_ctor(); + CHECK(mod != nullptr); + connector_common = stdc_api->ctor(mod); + CHECK(connector_common != nullptr); + connector_transmit = stdc_api->tinit(connector_transmit_config); + CHECK(connector_transmit != nullptr); + StdConnector* stdc_tt = (StdConnector*)connector_transmit; + + uint8_t* data = new uint8_t[len]; + memcpy(data, "foobar", len); + ConnectorMsg t_msg(data, len, true); + + CHECK(stdc_tt->transmit_message(t_msg)); + CHECK(output_buffer.accumulated_buffer == "foobar\n"); + + CHECK(stdc_tt->flush()); + CHECK(output_buffer.accumulated_buffer.empty()); + CHECK(output_buffer.flushed_buffer == "foobar\n"); + + stdc_api->tterm(connector_transmit); + stdc_api->dtor(connector_common); + std_connector->mod_dtor(mod); +} + + +TEST(std_connector, move_transmit_flush) +{ + uint32_t len = sizeof("foobar") - 1; + + mod = std_connector->mod_ctor(); + CHECK(mod != nullptr); + connector_common = stdc_api->ctor(mod); + CHECK(connector_common != nullptr); + connector_transmit = stdc_api->tinit(connector_transmit_config); + CHECK(connector_transmit != nullptr); + StdConnector* stdc_tt = (StdConnector*)connector_transmit; + + uint8_t* data = new uint8_t[len]; + memcpy(data, "foobar", len); + ConnectorMsg t_msg(data, len, true); + + CHECK(stdc_tt->transmit_message(std::move(t_msg))); + CHECK(output_buffer.accumulated_buffer == "foobar\n"); + + CHECK(stdc_tt->flush()); + CHECK(output_buffer.accumulated_buffer.empty()); + CHECK(output_buffer.flushed_buffer == "foobar\n"); + + stdc_api->tterm(connector_transmit); + stdc_api->dtor(connector_common); + std_connector->mod_dtor(mod); +} + +TEST(std_connector, receive) +{ + mod = std_connector->mod_ctor(); + CHECK(mod != nullptr); + connector_common = stdc_api->ctor(mod); + CHECK(connector_common != nullptr); + connector_receive = stdc_api->tinit(connector_receive_config); + CHECK(connector_receive != nullptr); + StdConnector* stdc_rt = (StdConnector*)connector_receive; + + std::istringstream ss("foo\nbar\n"); + std::cin.rdbuf(ss.rdbuf()); + + ConnectorMsg msg = stdc_rt->receive_message(false); + CHECK(std::string((char*)msg.get_data(), msg.get_length()) == "foo"); + + msg = stdc_rt->receive_message(false); + CHECK(std::string((char*)msg.get_data(), msg.get_length()) == "bar"); + + stdc_api->tterm(connector_receive); + stdc_api->dtor(connector_common); + std_connector->mod_dtor(mod); +} + +//------------------------------------------------------------------------- +// module test +//------------------------------------------------------------------------- + +TEST_GROUP(std_connector_module) +{ +}; + +TEST(std_connector_module, test) +{ + Value r_connector_val("rx"); + Value r_direction_val((uint64_t)0); + + Value t_connector_val("tx"); + Value t_direction_val((uint64_t)1); + + Value d_connector_val("duplex"); + Value d_direction_val((uint64_t)2); + + Parameter direction_param = + {"direction", Parameter::PT_ENUM, "receive | transmit | duplex", nullptr, "direction"}; + Parameter connector_param = + {"connector", Parameter::PT_STRING, nullptr, nullptr, "connector"}; + + StdConnectorModule module; + + r_direction_val.set(&direction_param); + r_connector_val.set(&connector_param); + + t_direction_val.set(&direction_param); + t_connector_val.set(&connector_param); + + d_direction_val.set(&direction_param); + d_connector_val.set(&connector_param); + + module.begin("std_connector", 0, nullptr); + module.begin("std_connector", 1, nullptr); + module.set("std_connector.direction", r_direction_val, nullptr); + module.set("std_connector.connector", r_connector_val, nullptr); + module.end("std_connector", 1, nullptr); + module.begin("std_connector", 1, nullptr); + module.set("std_connector.direction", t_direction_val, nullptr); + module.set("std_connector.connector", t_connector_val, nullptr); + module.end("std_connector", 1, nullptr); + module.begin("std_connector", 1, nullptr); + module.set("std_connector.direction", d_direction_val, nullptr); + module.set("std_connector.connector", d_connector_val, nullptr); + module.end("std_connector", 1, nullptr); + module.end("std_connector", 0, nullptr); + + ConnectorConfig::ConfigSet config_set = module.get_and_clear_config(); + + CHECK(6 == config_set.size()); + + ConnectorConfig config = *(config_set[0]); + CHECK("stdout" == config.connector_name); + CHECK(Connector::CONN_TRANSMIT == config.direction); + + config = *(config_set[1]); + CHECK("stdin" == config.connector_name); + CHECK(Connector::CONN_RECEIVE == config.direction); + + config = *(config_set[2]); + CHECK("stdio" == config.connector_name); + CHECK(Connector::CONN_DUPLEX == config.direction); + + config = *(config_set[3]); + CHECK("rx" == config.connector_name); + CHECK(Connector::CONN_RECEIVE == config.direction); + + config = *(config_set[4]); + CHECK("tx" == config.connector_name); + CHECK(Connector::CONN_TRANSMIT == config.direction); + + config = *(config_set[5]); + CHECK("duplex" == config.connector_name); + CHECK(Connector::CONN_DUPLEX == config.direction); +} + +int main(int argc, char** argv) +{ + return CommandLineTestRunner::RunAllTests(argc, argv); +} diff --git a/src/connectors/tcp_connector/tcp_connector.cc b/src/connectors/tcp_connector/tcp_connector.cc index f1f7a9089..6d9d20a67 100644 --- a/src/connectors/tcp_connector/tcp_connector.cc +++ b/src/connectors/tcp_connector/tcp_connector.cc @@ -41,20 +41,6 @@ using namespace snort; THREAD_LOCAL SimpleStats tcp_connector_stats; THREAD_LOCAL ProfileStats tcp_connector_perfstats; -TcpConnectorCommon::TcpConnectorCommon(TcpConnectorConfig::TcpConnectorConfigSet* conf) -{ - config_set = (ConnectorConfig::ConfigSet*)conf; -} - -TcpConnectorCommon::~TcpConnectorCommon() -{ - for ( auto conf : *config_set ) - delete conf; - - config_set->clear(); - delete config_set; -} - enum ReadDataOutcome { SUCCESS = 0, TRUNCATED, ERROR, CLOSED, PARTIAL, AGAIN }; static ReadDataOutcome read_data(int sockfd, uint8_t *data, uint16_t length, ssize_t& read_offset) @@ -243,10 +229,10 @@ bool TcpConnector::internal_transmit_message(const ConnectorMsg& msg) return true; } -bool TcpConnector::transmit_message(const ConnectorMsg& msg) +bool TcpConnector::transmit_message(const ConnectorMsg& msg, const ID&) { return internal_transmit_message(msg); } -bool TcpConnector::transmit_message(const ConnectorMsg&& msg) +bool TcpConnector::transmit_message(const ConnectorMsg&& msg, const ID&) { return internal_transmit_message(msg); } ConnectorMsg TcpConnector::receive_message(bool) @@ -400,7 +386,7 @@ static TcpConnector* tcp_connector_tinit_answer(const TcpConnectorConfig& cfg, c // Create a per-thread object static Connector* tcp_connector_tinit(const ConnectorConfig& config) { - const TcpConnectorConfig& cfg = (const TcpConnectorConfig&)config; + const TcpConnectorConfig& cfg = static_cast(config); const auto& ports = cfg.ports; auto idx = 0; @@ -431,16 +417,14 @@ static void tcp_connector_tterm(Connector* connector) static ConnectorCommon* tcp_connector_ctor(Module* m) { TcpConnectorModule* mod = (TcpConnectorModule*)m; - TcpConnectorCommon* tcp_connector_common = new TcpConnectorCommon( - mod->get_and_clear_config()); + ConnectorCommon* tcp_connector_common = new ConnectorCommon(mod->get_and_clear_config()); return tcp_connector_common; } static void tcp_connector_dtor(ConnectorCommon* c) { - TcpConnectorCommon* fc = (TcpConnectorCommon*)c; - delete fc; + delete c; } const ConnectorApi tcp_connector_api = @@ -449,7 +433,7 @@ const ConnectorApi tcp_connector_api = PT_CONNECTOR, sizeof(ConnectorApi), CONNECTOR_API_VERSION, - 1, + 2, API_RESERVED, API_OPTIONS, TCP_CONNECTOR_NAME, diff --git a/src/connectors/tcp_connector/tcp_connector.h b/src/connectors/tcp_connector/tcp_connector.h index 36051ce0b..9c7c50faa 100644 --- a/src/connectors/tcp_connector/tcp_connector.h +++ b/src/connectors/tcp_connector/tcp_connector.h @@ -47,21 +47,14 @@ public: uint16_t connector_msg_length; }; -class TcpConnectorCommon : public snort::ConnectorCommon -{ -public: - TcpConnectorCommon(TcpConnectorConfig::TcpConnectorConfigSet*); - ~TcpConnectorCommon(); -}; - class TcpConnector : public snort::Connector { public: TcpConnector(const TcpConnectorConfig&, int sock_fd); ~TcpConnector() override; - bool transmit_message(const snort::ConnectorMsg&) override; - bool transmit_message(const snort::ConnectorMsg&&) override; + bool transmit_message(const snort::ConnectorMsg&, const ID& = null) override; + bool transmit_message(const snort::ConnectorMsg&&, const ID& = null) override; snort::ConnectorMsg receive_message(bool) override; diff --git a/src/connectors/tcp_connector/tcp_connector_config.h b/src/connectors/tcp_connector/tcp_connector_config.h index 59381ee48..8313cb09b 100644 --- a/src/connectors/tcp_connector/tcp_connector_config.h +++ b/src/connectors/tcp_connector/tcp_connector_config.h @@ -38,8 +38,6 @@ public: std::string address; Setup setup = {}; bool async_receive; - - typedef std::vector TcpConnectorConfigSet; }; #endif diff --git a/src/connectors/tcp_connector/tcp_connector_module.cc b/src/connectors/tcp_connector/tcp_connector_module.cc index 3f34db8e3..9e5f84227 100644 --- a/src/connectors/tcp_connector/tcp_connector_module.cc +++ b/src/connectors/tcp_connector/tcp_connector_module.cc @@ -65,15 +65,7 @@ extern THREAD_LOCAL ProfileStats tcp_connector_perfstats; TcpConnectorModule::TcpConnectorModule() : Module(TCP_CONNECTOR_NAME, TCP_CONNECTOR_HELP, tcp_connector_params, true) -{ - config_set = new TcpConnectorConfig::TcpConnectorConfigSet; -} - -TcpConnectorModule::~TcpConnectorModule() -{ - delete config; - delete config_set; -} +{ } ProfileStats* TcpConnectorModule::get_profile() const { return &tcp_connector_perfstats; } @@ -116,20 +108,16 @@ bool TcpConnectorModule::set(const char*, Value& v, SnortConfig*) return true; } -// clear my working config and hand-over the compiled list to the caller -TcpConnectorConfig::TcpConnectorConfigSet* TcpConnectorModule::get_and_clear_config() +ConnectorConfig::ConfigSet TcpConnectorModule::get_and_clear_config() { - TcpConnectorConfig::TcpConnectorConfigSet* temp_config = config_set; - config = nullptr; - config_set = nullptr; - return temp_config; + return std::move(config_set); } bool TcpConnectorModule::begin(const char*, int, SnortConfig*) { if ( !config ) { - config = new TcpConnectorConfig; + config = std::make_unique(); config->direction = Connector::CONN_DUPLEX; } @@ -147,8 +135,7 @@ bool TcpConnectorModule::end(const char*, int idx, SnortConfig*) return false; } - config_set->emplace_back(config); - config = nullptr; + config_set.emplace_back(std::move(config)); } return true; diff --git a/src/connectors/tcp_connector/tcp_connector_module.h b/src/connectors/tcp_connector/tcp_connector_module.h index 63f089180..923491db4 100644 --- a/src/connectors/tcp_connector/tcp_connector_module.h +++ b/src/connectors/tcp_connector/tcp_connector_module.h @@ -21,6 +21,9 @@ #ifndef TCP_CONNECTOR_MODULE_H #define TCP_CONNECTOR_MODULE_H +#include + +#include "framework/connector.h" #include "framework/module.h" #include "tcp_connector_config.h" @@ -32,13 +35,12 @@ class TcpConnectorModule : public snort::Module { public: TcpConnectorModule(); - ~TcpConnectorModule() override; bool set(const char*, snort::Value&, snort::SnortConfig*) override; bool begin(const char*, int, snort::SnortConfig*) override; bool end(const char*, int, snort::SnortConfig*) override; - TcpConnectorConfig::TcpConnectorConfigSet* get_and_clear_config(); + snort::ConnectorConfig::ConfigSet get_and_clear_config(); const PegInfo* get_pegs() const override; PegCount* get_counts() const override; @@ -49,8 +51,8 @@ public: { return GLOBAL; } private: - TcpConnectorConfig::TcpConnectorConfigSet* config_set; - TcpConnectorConfig* config = nullptr; + snort::ConnectorConfig::ConfigSet config_set; + std::unique_ptr config; }; #endif diff --git a/src/connectors/tcp_connector/test/tcp_connector_module_test.cc b/src/connectors/tcp_connector/test/tcp_connector_module_test.cc index 50a0a7e39..a68b265ea 100644 --- a/src/connectors/tcp_connector/test/tcp_connector_module_test.cc +++ b/src/connectors/tcp_connector/test/tcp_connector_module_test.cc @@ -95,13 +95,11 @@ TEST(tcp_connector_module, test_call) CHECK(module.end("tcp_connector", 1, nullptr)); CHECK(module.end("tcp_connector", 0, nullptr)); - TcpConnectorConfig::TcpConnectorConfigSet* config_set = module.get_and_clear_config(); + ConnectorConfig::ConfigSet config_set = module.get_and_clear_config(); - CHECK(nullptr != config_set); + CHECK(1 == config_set.size()); - CHECK(1 == config_set->size()); - - TcpConnectorConfig config = *(config_set->front()); + const TcpConnectorConfig& config = static_cast(*config_set.front()); CHECK(10000 == std::stoi(config.ports[0])); CHECK(20000 == std::stoi(config.ports[1])); CHECK("127.0.0.1" == config.address); @@ -112,13 +110,6 @@ TEST(tcp_connector_module, test_call) CHECK(nullptr != module.get_pegs()); CHECK(nullptr != module.get_counts()); CHECK(nullptr != module.get_profile()); - - for ( auto conf : *config_set ) - delete conf; - - config_set->clear(); - delete config_set; - instance_max = 1; } TEST(tcp_connector_module, test_ports_count_failure) @@ -150,23 +141,15 @@ TEST(tcp_connector_module, test_ports_count_failure) CHECK(module.end("tcp_connector", 1, nullptr)); CHECK(module.end("tcp_connector", 0, nullptr)); - TcpConnectorConfig::TcpConnectorConfigSet* config_set = module.get_and_clear_config(); + ConnectorConfig::ConfigSet config_set = module.get_and_clear_config(); - CHECK(nullptr != config_set); + CHECK(1 == config_set.size()); - CHECK(1 == config_set->size()); - - TcpConnectorConfig config = *(config_set->front()); + const TcpConnectorConfig& config = static_cast(*config_set.front()); CHECK(20000 == stoi(config.ports[0])); CHECK(config.setup == TcpConnectorConfig::Setup::ANSWER); CHECK("tcp-a" == config.connector_name); CHECK(Connector::CONN_DUPLEX == config.direction); - - for ( auto conf : *config_set ) - delete conf; - - config_set->clear(); - delete config_set; } TEST(tcp_connector_module, test_answer) diff --git a/src/connectors/tcp_connector/test/tcp_connector_test.cc b/src/connectors/tcp_connector/test/tcp_connector_test.cc index 62b821320..7a5d3bf14 100644 --- a/src/connectors/tcp_connector/test/tcp_connector_test.cc +++ b/src/connectors/tcp_connector/test/tcp_connector_test.cc @@ -173,15 +173,13 @@ static void set_normal_status() TcpConnectorModule::TcpConnectorModule() : Module("TCPC", "TCPC Help", nullptr) -{ config_set = nullptr; } +{ } -TcpConnectorConfig::TcpConnectorConfigSet* TcpConnectorModule::get_and_clear_config() +ConnectorConfig::ConfigSet TcpConnectorModule::get_and_clear_config() { - return new TcpConnectorConfig::TcpConnectorConfigSet; + return ConnectorConfig::ConfigSet(); } -TcpConnectorModule::~TcpConnectorModule() = default; - ProfileStats* TcpConnectorModule::get_profile() const { return nullptr; } bool TcpConnectorModule::set(const char*, Value&, SnortConfig*) { return true; } diff --git a/src/framework/connector.h b/src/framework/connector.h index 118f2414b..9fdddef32 100644 --- a/src/framework/connector.h +++ b/src/framework/connector.h @@ -26,7 +26,9 @@ // the CONNECTOR_API_VERSION will change if anything in this file changes. // see also framework/base_api.h. +#include #include +#include #include #include "framework/base_api.h" @@ -35,7 +37,7 @@ namespace snort { // this is the current version of the api -#define CONNECTOR_API_VERSION ((BASE_API_VERSION << 16) | 2) +#define CONNECTOR_API_VERSION ((BASE_API_VERSION << 16) | 3) //------------------------------------------------------------------------- // api for class @@ -101,11 +103,16 @@ public: CONN_DUPLEX }; + using ID = std::variant; + Connector(const ConnectorConfig& config) : config(config) { } virtual ~Connector() = default; - virtual bool transmit_message(const ConnectorMsg&) = 0; - virtual bool transmit_message(const ConnectorMsg&&) = 0; + virtual const ID get_id(const char*) const + { return null; } + + virtual bool transmit_message(const ConnectorMsg&, const ID& = null) = 0; + virtual bool transmit_message(const ConnectorMsg&&, const ID& = null) = 0; virtual ConnectorMsg receive_message(bool block) = 0; @@ -120,12 +127,13 @@ public: protected: const ConnectorConfig& config; + static constexpr ID null {nullptr}; }; class ConnectorConfig { public: - typedef std::vector ConfigSet; + typedef std::vector> ConfigSet; Connector::Direction direction; std::string connector_name; @@ -141,7 +149,13 @@ const std::string& Connector::get_connector_name() const class SO_PUBLIC ConnectorCommon { public: - ConnectorConfig::ConfigSet* config_set; + ConnectorCommon(ConnectorConfig::ConfigSet&& c_s) : + config_set(std::move(c_s)) + { } + + virtual ~ConnectorCommon() = default; + + const ConnectorConfig::ConfigSet config_set; }; typedef ConnectorCommon* (* ConnectorNewFunc)(Module*); diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc index a5b99cee7..68048d0ba 100644 --- a/src/main/snort_config.cc +++ b/src/main/snort_config.cc @@ -51,6 +51,7 @@ #include "main/policy.h" #include "main/process.h" #include "managers/action_manager.h" +#include "managers/connector_manager.h" #include "managers/event_manager.h" #include "managers/inspector_manager.h" #include "managers/ips_manager.h" @@ -1059,6 +1060,7 @@ void SnortConfig::cleanup_fatal_error() EventManager::release_plugins(); IpsManager::release_plugins(); InspectorManager::release_plugins(); + ConnectorManager::release_plugins(); host_cache.term(); } #endif diff --git a/src/managers/connector_manager.cc b/src/managers/connector_manager.cc index 7774807ec..77f0059c9 100644 --- a/src/managers/connector_manager.cc +++ b/src/managers/connector_manager.cc @@ -28,7 +28,6 @@ #include #include -#include "framework/connector.h" #include "log/messages.h" #include "main/thread.h" #include "main/thread_config.h" @@ -41,10 +40,11 @@ using namespace snort; // One ConnectorElem for each Connector within the ConnectorCommon configuration struct ConnectorElem { - ConnectorElem() : config(nullptr), thread_connectors(ThreadConfig::get_instance_max(), nullptr) + ConnectorElem(const ConnectorConfig& config) : config(config), + thread_connectors(ThreadConfig::get_instance_max(), nullptr) { } - ConnectorConfig* config; + const ConnectorConfig& config; std::vector thread_connectors; }; @@ -53,7 +53,7 @@ struct ConnectorCommonElem { const ConnectorApi* api; ConnectorCommon* connector_common; - std::map connectors; + std::map connectors; ConnectorCommonElem(const ConnectorApi* p) { @@ -88,9 +88,6 @@ void ConnectorManager::release_plugins() if ( sc.api->dtor ) sc.api->dtor(sc.connector_common); - for ( const auto& conn : sc.connectors ) - delete conn.second; - sc.connectors.clear(); if ( sc.api->pterm ) @@ -102,14 +99,16 @@ void ConnectorManager::release_plugins() Connector* ConnectorManager::get_connector(const std::string& connector_name) { + unsigned instance = get_instance_id(); + for ( auto& sc : s_connector_commons ) { - unsigned instance = get_instance_id(); - if ( sc.connectors.count(connector_name) > 0 ) + auto connector_ptr = sc.connectors.find(connector_name); + + if ( connector_ptr != sc.connectors.end() ) { - ConnectorElem* map = sc.connectors[connector_name]; - if ( map->thread_connectors[instance] ) - return ( map->thread_connectors[instance] ); + if ( connector_ptr->second.thread_connectors[instance] ) + return ( connector_ptr->second.thread_connectors[instance] ); } } return ( nullptr ); @@ -123,13 +122,13 @@ void ConnectorManager::thread_init() { if ( sc.api->tinit ) { - for ( const auto& conn : sc.connectors ) + for ( auto& conn : sc.connectors ) { - assert(!conn.second->thread_connectors[instance]); + assert(!conn.second.thread_connectors[instance]); - Connector* connector = sc.api->tinit(*conn.second->config); + Connector* connector = sc.api->tinit(conn.second.config); assert(connector); - conn.second->thread_connectors[instance] = std::move(connector); + conn.second.thread_connectors[instance] = std::move(connector); } } } @@ -143,9 +142,9 @@ void ConnectorManager::thread_reinit() { for ( auto& conn : sc.connectors ) { - assert(conn.second->thread_connectors[instance]); + assert(conn.second.thread_connectors[instance]); - conn.second->thread_connectors[instance]->reinit(); + conn.second.thread_connectors[instance]->reinit(); } } } @@ -160,10 +159,10 @@ void ConnectorManager::thread_term() { for ( auto& conn : sc.connectors ) { - assert(conn.second->thread_connectors[instance]); + assert(conn.second.thread_connectors[instance]); - sc.api->tterm(conn.second->thread_connectors[instance]); - conn.second->thread_connectors[instance] = nullptr; + sc.api->tterm(conn.second.thread_connectors[instance]); + conn.second.thread_connectors[instance] = nullptr; } } } @@ -178,16 +177,32 @@ void ConnectorManager::instantiate(const ConnectorApi* api, Module* mod, SnortCo assert(connector_common); c.connector_common = connector_common; - ConnectorConfig::ConfigSet* config_set = connector_common->config_set; // iterate through the config_set and create the connector entries - for ( auto cfg : *config_set ) + for ( auto& cfg : connector_common->config_set ) { - ConnectorElem* connector_elem = new ConnectorElem; - connector_elem->config = &*cfg; + if ( is_instantiated(cfg->connector_name) != Connector::CONN_UNDEFINED ) + { + ParseError("redefinition of \"%s\" connector", cfg->connector_name.c_str()); + continue; + } + + ConnectorElem connector_elem(*cfg); c.connectors.emplace(cfg->connector_name, std::move(connector_elem)); } s_connector_commons.emplace_back(c); } +Connector::Direction ConnectorManager::is_instantiated(const std::string& name) +{ + for ( auto& conn : s_connector_commons ) + { + auto connector_ptr = conn.connectors.find(name); + + if ( connector_ptr != conn.connectors.end() ) + return connector_ptr->second.config.direction; + } + + return Connector::CONN_UNDEFINED; +} diff --git a/src/managers/connector_manager.h b/src/managers/connector_manager.h index adb805b2c..0db68d718 100644 --- a/src/managers/connector_manager.h +++ b/src/managers/connector_manager.h @@ -24,10 +24,10 @@ #include +#include "framework/connector.h" + namespace snort { -class Connector; -struct ConnectorApi; class Module; struct SnortConfig; } @@ -42,6 +42,7 @@ public: static void release_plugins(); static void instantiate(const snort::ConnectorApi*, snort::Module*, snort::SnortConfig*); + static snort::Connector::Direction is_instantiated(const std::string& name); static void thread_init(); static void thread_reinit(); diff --git a/src/network_inspectors/extractor/CMakeLists.txt b/src/network_inspectors/extractor/CMakeLists.txt index e8c0f6a1d..d67dd3565 100644 --- a/src/network_inspectors/extractor/CMakeLists.txt +++ b/src/network_inspectors/extractor/CMakeLists.txt @@ -14,8 +14,6 @@ set( FILE_LIST extractor_logger.h extractor_service.cc extractor_service.h - extractor_writer.cc - extractor_writer.h extractors.cc extractors.h ) diff --git a/src/network_inspectors/extractor/dev_notes.txt b/src/network_inspectors/extractor/dev_notes.txt index 5fa9099de..2472c97a4 100644 --- a/src/network_inspectors/extractor/dev_notes.txt +++ b/src/network_inspectors/extractor/dev_notes.txt @@ -44,15 +44,23 @@ A log unit is a log record. It is enclosed by `ExtractorLogger::open_record` and `ExtractorLogger::close_record` calls. A header (or a footer) can be added. They prepend (append) the set of log records with meta info. -`ExtractorWriter` interface defines the set of methods to printout formatted -data. - -Additionally, `StdExtractorWriter` (a writer to standard output) implements -synchronization between threads. But that is not required, and is done just for -user convenience. `StdExtractorWriter` is not a main writer, nor performant. -`ExtractorWriter` specialization may do things in asynchronous way. - -There are plans to convert `ExtractorWriter` to `Connector` type. +To printout formatted data the extractor utilizes `Connector` API, which allows +to transmit data using different pre-configured channels. Specific connector +is getting configured as a separate module and extractor accesses it by +name. + +Both Logger and its Connector are allocated per thread, so no syncronization +is required. If data channel poses multithreaded output restrictions, those +should be handled by the Connector. `Connector` specialization may do things +in asynchronous way and store the data for indefinite amount of time, but only +if it was moved to it. Therefore, each logger should choose the appropriate +method for transmitting the message. + +`ExtractorLogger` instance is global and shared among all `ExtractorEvent` +instances. Additionally, each `ExtractorService` has its own thread-local +service ID object obtained from the `ExtractorLogger`. The service ID object +is also utilized by the corresponding `ExtractorEvent` instances, although its +lifespan is controlled by the `ExtractorService`. ==== Logging Context @@ -133,7 +141,7 @@ functions, data extracting functions, formatting and writing functions. | 2. Data Event | (any type) | provides getter functions, resides in snort3/src/pub_sub/ | 3. Extractor Event | `char*` `uint64_t` `timeval` `SfIp` `bool` | converts any type to a fixed type | 4. Extractor Logger | `char*` `uint64_t` `timeval` `SfIp` `bool` | decorates a fixed type to fit formatting -| 5. Extractor Writer | `char*` `uint64_t` | accepts basic types: a number and a text string +| 5. Connector | `ConnectorMsg` | accepts an internal data type, which is an array of bytes of a given length |=============================================================================== _Inspector layer_ focuses on performance. It means we seek a minimal overhead @@ -179,7 +187,7 @@ Context... context)` ensure: Since _Extractor Logger_ interface accepts just a limited set of (basic) types, it should be able to decorate a data field and put it into a targeted format. -_Extractor Writer_ is the final layer before data leaves Snort. It may +_Connector_ is the final layer before data leaves Snort. It may implement output stream and external resource management (like, file rotation, socket operations), synchronization, buffering, queuing if needed. diff --git a/src/network_inspectors/extractor/extractor.cc b/src/network_inspectors/extractor/extractor.cc index b90471abf..ac752ec95 100644 --- a/src/network_inspectors/extractor/extractor.cc +++ b/src/network_inspectors/extractor/extractor.cc @@ -29,7 +29,9 @@ #include "framework/inspector.h" #include "framework/module.h" #include "log/messages.h" +#include "main/reload_tuner.h" #include "main/snort_config.h" +#include "managers/connector_manager.h" #include "protocols/packet.h" #include "extractors.h" @@ -40,6 +42,7 @@ using namespace snort; THREAD_LOCAL ExtractorStats extractor_stats; THREAD_LOCAL ProfileStats extractor_perf_stats; +THREAD_LOCAL ExtractorLogger* Extractor::logger = nullptr; //------------------------------------------------------------------------- // module stuff @@ -67,7 +70,7 @@ static const Parameter s_params[] = { "formatting", Parameter::PT_ENUM, "csv | json", "csv", "output format for extractor" }, - { "output", Parameter::PT_ENUM, "stdout", "stdout", + { "connector", Parameter::PT_STRING, nullptr, nullptr, "output destination for extractor" }, { "protocols", Parameter::PT_LIST, extractor_proto_params, nullptr, @@ -132,8 +135,8 @@ bool ExtractorModule::set(const char*, Value& v, SnortConfig*) if (v.is("formatting")) extractor_config.formatting = (FormatType)(v.get_uint8()); - else if (v.is("output")) - extractor_config.output = (OutputType)(v.get_uint8()); + else if (v.is("connector")) + extractor_config.output_conn = v.get_string(); else if (v.is("service")) service_config.service = (ServiceType)(v.get_uint8()); @@ -170,16 +173,37 @@ bool ExtractorModule::end(const char* fqn, int idx, SnortConfig*) // Inspector stuff //------------------------------------------------------------------------- +class ExtractorReloadSwapper : public ReloadSwapper +{ +public: + ExtractorReloadSwapper(Extractor& inspector) : inspector(inspector) + { } + + void tswap() override + { + inspector.logger->flush(); + + delete inspector.logger; + inspector.logger = ExtractorLogger::make_logger(inspector.format, inspector.output_conn); + + for (auto& s : inspector.services) + s->tinit(inspector.logger); + } + +private: + Extractor& inspector; +}; + Extractor::Extractor(ExtractorModule* m) { auto& cfg = m->get_config(); format = cfg.formatting; - output = cfg.output; + output_conn = cfg.output_conn; for (const auto& p : cfg.protocols) { - auto s = ExtractorService::make_service(*this, p, format, output); + auto s = ExtractorService::make_service(*this, p); if (s) services.push_back(s); @@ -192,10 +216,24 @@ Extractor::~Extractor() delete s; } +bool Extractor::configure(SnortConfig*) +{ + Connector::Direction mode = ConnectorManager::is_instantiated(output_conn); + + if (mode != Connector::CONN_TRANSMIT and mode != Connector::CONN_DUPLEX) + { + ParseError("can't initialize extractor, cannot find Connector \"%s\" in transmit mode.\n", + output_conn.c_str()); + return false; + } + + return true; +} + void Extractor::show(const SnortConfig*) const { ConfigLogger::log_value("formatting", format.c_str()); - ConfigLogger::log_value("output", output.c_str()); + ConfigLogger::log_value("connector", output_conn.c_str()); bool log_header = true; for (const auto& s : services) @@ -211,6 +249,31 @@ void Extractor::show(const SnortConfig*) const ConfigLogger::log_list("", str.c_str(), " "); } } + +void Extractor::tinit() +{ + logger = ExtractorLogger::make_logger(format, output_conn); + + for (auto& s : services) + s->tinit(logger); +} + +void Extractor::tterm() +{ + for (auto& s : services) + s->tterm(); + + logger->flush(); + + delete logger; + logger = nullptr; +} + +void Extractor::install_reload_handler(SnortConfig* sc) +{ + sc->register_reload_handler(new ExtractorReloadSwapper(*this)); +} + //------------------------------------------------------------------------- // api stuff //------------------------------------------------------------------------- diff --git a/src/network_inspectors/extractor/extractor.h b/src/network_inspectors/extractor/extractor.h index 17f14dbe9..8098b34b1 100644 --- a/src/network_inspectors/extractor/extractor.h +++ b/src/network_inspectors/extractor/extractor.h @@ -50,7 +50,7 @@ public: struct ExtractorConfig { FormatType formatting = FormatType::CSV; - OutputType output = OutputType::STD; + std::string output_conn; std::vector protocols; }; @@ -65,6 +65,8 @@ struct ExtractorStats PegCount total_event; }; +class ExtractorReloadSwapper; + extern THREAD_LOCAL ExtractorStats extractor_stats; extern THREAD_LOCAL snort::ProfileStats extractor_perf_stats; @@ -106,12 +108,21 @@ public: Extractor(ExtractorModule*); ~Extractor() override; + bool configure(snort::SnortConfig*) override; + void show(const snort::SnortConfig*) const override; + void tinit() override; + void tterm() override; + void install_reload_handler(snort::SnortConfig*) override; + private: std::vector services; FormatType format; - OutputType output; + std::string output_conn; + static THREAD_LOCAL ExtractorLogger* logger; + + friend class ExtractorReloadSwapper; }; #endif diff --git a/src/network_inspectors/extractor/extractor_csv_logger.cc b/src/network_inspectors/extractor/extractor_csv_logger.cc index adf6b8ed5..89dc9b006 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.cc +++ b/src/network_inspectors/extractor/extractor_csv_logger.cc @@ -24,15 +24,19 @@ #include "extractor_csv_logger.h" #include +#include #include #include "utils/util_cstring.h" +using namespace snort; +using namespace std; + static THREAD_LOCAL bool first_write; -void CsvExtractorLogger::add_header() +void CsvExtractorLogger::add_header(const vector& field_names, const Connector::ID& service_id) { - std::string header; + string header; char d = '#'; for (auto n : field_names) @@ -42,70 +46,65 @@ void CsvExtractorLogger::add_header() d = ','; } - header += "\n"; - - writer->write(header.c_str()); + ConnectorMsg cmsg((const uint8_t*)header.c_str(), header.size(), false); + output_conn->transmit_message(cmsg, service_id); } void CsvExtractorLogger::open_record() { first_write = true; - writer->lock(); } -void CsvExtractorLogger::close_record() +void CsvExtractorLogger::close_record(const Connector::ID& service_id) { - writer->write("\n"); - writer->unlock(); + ConnectorMsg cmsg((const uint8_t*)buffer.c_str(), buffer.size(), false); + output_conn->transmit_message(cmsg, service_id); + + buffer.clear(); } void CsvExtractorLogger::add_field(const char*, const char* v) { - first_write ? []() { first_write = false; } () : writer->write(","); - writer->write(v); + first_write ? []() { first_write = false; } () : buffer.push_back(','); + buffer.append(v); } void CsvExtractorLogger::add_field(const char*, const char* v, size_t len) { - first_write ? []() { first_write = false; } () : writer->write(","); - writer->write(v, len); + first_write ? []() { first_write = false; } () : buffer.push_back(','); + buffer.append(v, len); } void CsvExtractorLogger::add_field(const char*, uint64_t v) { - first_write ? []() { first_write = false; } () : writer->write(","); - writer->write(v); + first_write ? []() { first_write = false; } () : buffer.push_back(','); + buffer.append(to_string(v)); } void CsvExtractorLogger::add_field(const char*, struct timeval v) { - first_write ? []() { first_write = false; } () : writer->write(","); + first_write ? []() { first_write = false; } () : buffer.push_back(','); - char u_sec[8]; - snort::SnortSnprintf(u_sec, sizeof(u_sec), ".%06d", (unsigned)v.tv_usec); + char time_str[numeric_limits::digits10 + 8]; + snort::SnortSnprintf(time_str, sizeof(time_str), "%" PRIu64 ".%06d", (uint64_t)v.tv_sec, (unsigned)v.tv_usec); - writer->write(v.tv_sec); - writer->write(u_sec); + buffer.append(time_str); } void CsvExtractorLogger::add_field(const char*, const snort::SfIp& v) { - first_write ? []() { first_write = false; } () : writer->write(","); + first_write ? []() { first_write = false; } () : buffer.push_back(','); snort::SfIpString buf; v.ntop(buf); - writer->write(buf); + buffer.append(buf); } void CsvExtractorLogger::add_field(const char*, bool v) { - first_write ? []() { first_write = false; } () : writer->write(","); + first_write ? []() { first_write = false; } () : buffer.push_back(','); - writer->write(v ? "true" : "false"); + buffer.append(v ? "true" : "false"); } -CsvExtractorLogger::~CsvExtractorLogger() -{ - delete writer; -} diff --git a/src/network_inspectors/extractor/extractor_csv_logger.h b/src/network_inspectors/extractor/extractor_csv_logger.h index 5c44b8b45..183f59be6 100644 --- a/src/network_inspectors/extractor/extractor_csv_logger.h +++ b/src/network_inspectors/extractor/extractor_csv_logger.h @@ -23,20 +23,17 @@ #include "framework/value.h" #include "extractor_logger.h" -#include "extractor_writer.h" class CsvExtractorLogger : public ExtractorLogger { public: - CsvExtractorLogger(OutputType o_type) - : writer(ExtractorWriter::make_writer(o_type)) {} - - ~CsvExtractorLogger() override; + CsvExtractorLogger(snort::Connector* conn) : ExtractorLogger(conn) + { } virtual bool is_strict() const override { return true; } - void add_header() override; + void add_header(const std::vector& field_names, const snort::Connector::ID&) override; void add_field(const char*, const char*) override; void add_field(const char*, const char*, size_t) override; void add_field(const char*, uint64_t) override; @@ -44,10 +41,10 @@ public: void add_field(const char*, const snort::SfIp&) override; void add_field(const char*, bool) override; void open_record() override; - void close_record() override; + void close_record(const snort::Connector::ID&) override; private: - ExtractorWriter* const writer; + std::string buffer; }; #endif diff --git a/src/network_inspectors/extractor/extractor_enums.h b/src/network_inspectors/extractor/extractor_enums.h index 471be1d01..ea19e43e5 100644 --- a/src/network_inspectors/extractor/extractor_enums.h +++ b/src/network_inspectors/extractor/extractor_enums.h @@ -94,37 +94,4 @@ private: Value v = CSV; }; -class OutputType -{ -public: - enum Value : uint8_t - { - STD, - MAX - }; - - OutputType() = default; - constexpr OutputType(Value a) : v(a) {} - template constexpr OutputType(T a) : v((Value)a) {} - - constexpr operator Value() const { return v; } - explicit operator bool() const = delete; - - const char* c_str() const - { - switch (v) - { - case STD: - return "stdout"; - case MAX: // fallthrough - default: - return "(not set)"; - } - } - -private: - Value v = STD; -}; - - #endif diff --git a/src/network_inspectors/extractor/extractor_ftp.cc b/src/network_inspectors/extractor/extractor_ftp.cc index e3e75b20b..06b0f8587 100644 --- a/src/network_inspectors/extractor/extractor_ftp.cc +++ b/src/network_inspectors/extractor/extractor_ftp.cc @@ -75,8 +75,10 @@ static const map sub_str_getters = }; } -FtpRequestExtractor::FtpRequestExtractor(Extractor& i, ExtractorLogger& l, - uint32_t t, const vector& fields) : ExtractorEvent(i, l, t) +THREAD_LOCAL const snort::Connector::ID* FtpRequestExtractor::log_id = nullptr; + +FtpRequestExtractor::FtpRequestExtractor(Extractor& i, uint32_t t, const vector& fields) : + ExtractorEvent(i, t) { for (const auto& f : fields) { @@ -93,6 +95,9 @@ FtpRequestExtractor::FtpRequestExtractor(Extractor& i, ExtractorLogger& l, DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_REQUEST, new Req(*this, S_NAME)); } +void FtpRequestExtractor::internal_tinit(const snort::Connector::ID* service_id) +{ log_id = service_id; } + void FtpRequestExtractor::handle(DataEvent& event, Flow* flow) { // cppcheck-suppress unreadVariable @@ -111,12 +116,12 @@ void FtpRequestExtractor::handle(DataEvent& event, Flow* flow) Packet* packet = DetectionEngine::get_current_packet(); - logger.open_record(); + logger->open_record(); log(nts_fields, &event, packet, flow); log(sip_fields, &event, packet, flow); log(num_fields, &event, packet, flow); - log(str_fields, &event, packet, flow, logger.is_strict()); - logger.close_record(); + log(str_fields, &event, packet, flow, logger->is_strict()); + logger->close_record(*log_id); } static uint64_t parse_last_num(const char *str, uint16_t size) @@ -217,8 +222,10 @@ static const map sub_getters = }; } -FtpResponseExtractor::FtpResponseExtractor(Extractor& i, ExtractorLogger& l, - uint32_t t, const vector& fields) : ExtractorEvent(i, l, t) +THREAD_LOCAL const snort::Connector::ID* FtpResponseExtractor::log_id = nullptr; + +FtpResponseExtractor::FtpResponseExtractor(Extractor& i, uint32_t t, const vector& fields) : + ExtractorEvent(i, t) { for (const auto& f : fields) { @@ -241,6 +248,9 @@ FtpResponseExtractor::FtpResponseExtractor(Extractor& i, ExtractorLogger& l, DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_RESPONSE, new Resp(*this, S_NAME)); } +void FtpResponseExtractor::internal_tinit(const snort::Connector::ID* service_id) +{ log_id = service_id; } + template<> void ExtractorEvent::log, DataEvent*, Packet*, Flow*, bool>( const vector& fields, DataEvent* event, Packet* pkt, Flow* flow, bool strict) @@ -249,9 +259,9 @@ void ExtractorEvent::log, DataEvent*, Pac { const auto mode = f.get(event, pkt, flow); if (mode != FTPP_XFER_NOT_SET) - mode == FTPP_XFER_PASSIVE ? logger.add_field(f.name, true) : logger.add_field(f.name, false); + mode == FTPP_XFER_PASSIVE ? logger->add_field(f.name, true) : logger->add_field(f.name, false); else if (strict) - logger.add_field(f.name, ""); + logger->add_field(f.name, ""); } } @@ -273,13 +283,13 @@ void FtpResponseExtractor::handle(DataEvent& event, Flow* flow) Packet* packet = DetectionEngine::get_current_packet(); - logger.open_record(); + logger->open_record(); log(nts_fields, &event, packet, flow); log(sip_fields, &event, packet, flow); log(num_fields, &event, packet, flow); - log(str_fields, &event, packet, flow, logger.is_strict()); - log(sub_fields, &event, packet, flow, logger.is_strict()); - logger.close_record(); + log(str_fields, &event, packet, flow, logger->is_strict()); + log(sub_fields, &event, packet, flow, logger->is_strict()); + logger->close_record(*log_id); } vector FtpResponseExtractor::get_field_names() const @@ -406,8 +416,10 @@ static const map fd_sub_getters = }; } -FtpExtractor::FtpExtractor(Extractor& i, ExtractorLogger& l, - uint32_t t, const vector& fields) : ExtractorEvent(i, l, t) +THREAD_LOCAL const snort::Connector::ID* FtpExtractor::log_id = nullptr; + +FtpExtractor::FtpExtractor(Extractor& i, uint32_t t, const vector& fields) : + ExtractorEvent(i, t) { for (const auto& f : fields) { @@ -431,6 +443,9 @@ FtpExtractor::FtpExtractor(Extractor& i, ExtractorLogger& l, DataBus::subscribe(ftp_pub_key, FtpEventIds::FTP_RESPONSE, new Resp(*this, S_NAME)); } +void FtpExtractor::internal_tinit(const snort::Connector::ID* service_id) +{ log_id = service_id; } + vector FtpExtractor::get_field_names() const { vector res = ExtractorEvent::get_field_names(); @@ -473,7 +488,7 @@ void ExtractorEvent::log, const FtpExtractorFlo for (const auto& f : fields) { auto d = f.get(*fd); - logger.add_field(f.name, d); + logger->add_field(f.name, d); } } @@ -484,7 +499,7 @@ void ExtractorEvent::log, const FtpExtractorFlo for (const auto& f : fields) { auto d = f.get(*fd); - logger.add_field(f.name, d); + logger->add_field(f.name, d); } } @@ -495,7 +510,7 @@ void ExtractorEvent::log, const FtpExtractorFlo for (const auto& f : fields) { auto d = f.get(*fd); - logger.add_field(f.name, d); + logger->add_field(f.name, d); } } @@ -507,9 +522,9 @@ void ExtractorEvent::log, const FtpExtractorFlo { const auto mode = f.get(*fd); if (mode != FTPP_XFER_NOT_SET) - mode == FTPP_XFER_PASSIVE ? logger.add_field(f.name, true) : logger.add_field(f.name, false); + mode == FTPP_XFER_PASSIVE ? logger->add_field(f.name, true) : logger->add_field(f.name, false); else if (strict) - logger.add_field(f.name, ""); + logger->add_field(f.name, ""); } } @@ -539,15 +554,15 @@ void FtpExtractor::Req::handle(DataEvent& event, Flow* flow) else if (!fd->cmd.empty()) { // log existing flow data - owner.logger.open_record(); + owner.logger->open_record(); owner.log(owner.nts_fields, &event, p, flow); owner.log(owner.sip_fields, &event, p, flow); owner.log(owner.num_fields, &event, p, flow); owner.log(owner.fd_buf_fields, (const FtpExtractorFlowData*)fd); owner.log(owner.fd_sip_fields, (const FtpExtractorFlowData*)fd); owner.log(owner.fd_num_fields, (const FtpExtractorFlowData*)fd); - owner.log(owner.fd_sub_fields, (const FtpExtractorFlowData*)fd, owner.logger.is_strict()); - owner.logger.close_record(); + owner.log(owner.fd_sub_fields, (const FtpExtractorFlowData*)fd, owner.logger->is_strict()); + owner.logger->close_record(*log_id); fd->reset(); } @@ -629,21 +644,21 @@ void FtpExtractor::dump(const FtpExtractorFlowData& fd) // cppcheck-suppress unreadVariable Profile profile(extractor_perf_stats); - logger.open_record(); + logger->open_record(); for (const auto& f : nts_fields) - logger.add_field(f.name, fd.ts); + logger->add_field(f.name, fd.ts); for (const auto& f : sip_fields) - logger.add_field(f.name, ""); + logger->add_field(f.name, ""); for (const auto& f : num_fields) - logger.add_field(f.name, (uint64_t)0); + logger->add_field(f.name, (uint64_t)0); log(fd_buf_fields, &fd); log(fd_sip_fields, &fd); log(fd_num_fields, &fd); - log(fd_sub_fields, &fd, logger.is_strict()); + log(fd_sub_fields, &fd, logger->is_strict()); - logger.close_record(); + logger->close_record(*log_id); } diff --git a/src/network_inspectors/extractor/extractor_ftp.h b/src/network_inspectors/extractor/extractor_ftp.h index ae32b2551..71daa31ae 100644 --- a/src/network_inspectors/extractor/extractor_ftp.h +++ b/src/network_inspectors/extractor/extractor_ftp.h @@ -29,12 +29,16 @@ class FtpExtractorFlowData; class FtpRequestExtractor : public ExtractorEvent { public: - FtpRequestExtractor(Extractor&, ExtractorLogger&, uint32_t tenant, const std::vector& fields); + FtpRequestExtractor(Extractor&, uint32_t tenant, const std::vector& fields); void handle(DataEvent&, Flow*); private: using Req = Handler; + + void internal_tinit(const snort::Connector::ID*) override; + + static THREAD_LOCAL const snort::Connector::ID* log_id; }; class FtpResponseExtractor : public ExtractorEvent @@ -43,7 +47,7 @@ public: using SubGetFn = int8_t (*) (const DataEvent*, const Packet*, const Flow*); using SubField = DataField; - FtpResponseExtractor(Extractor&, ExtractorLogger&, uint32_t tenant, const std::vector& fields); + FtpResponseExtractor(Extractor&, uint32_t tenant, const std::vector& fields); std::vector get_field_names() const override; void handle(DataEvent&, Flow*); @@ -51,7 +55,10 @@ public: private: using Resp = Handler; + void internal_tinit(const snort::Connector::ID*) override; + std::vector sub_fields; + static THREAD_LOCAL const snort::Connector::ID* log_id; }; class FtpExtractor : public ExtractorEvent @@ -66,7 +73,7 @@ public: using FdSubGetFn = int8_t (*) (const FtpExtractorFlowData&); using FdSubField = DataField; - FtpExtractor(Extractor&, ExtractorLogger&, uint32_t tenant, const std::vector& fields); + FtpExtractor(Extractor&, uint32_t tenant, const std::vector& fields); std::vector get_field_names() const override; void dump(const FtpExtractorFlowData&); @@ -86,10 +93,13 @@ private: FtpExtractor& owner; }; + void internal_tinit(const snort::Connector::ID*) override; + std::vector fd_buf_fields; std::vector fd_sip_fields; std::vector fd_num_fields; std::vector fd_sub_fields; + static THREAD_LOCAL const snort::Connector::ID* log_id; }; #endif diff --git a/src/network_inspectors/extractor/extractor_http.cc b/src/network_inspectors/extractor/extractor_http.cc index 8633b79db..883c43d80 100644 --- a/src/network_inspectors/extractor/extractor_http.cc +++ b/src/network_inspectors/extractor/extractor_http.cc @@ -154,8 +154,10 @@ static const map sub_getters = {"info_msg", get_info_msg} }; -HttpExtractor::HttpExtractor(Extractor& i, ExtractorLogger& l, uint32_t t, const vector& fields) - : ExtractorEvent(i, l, t) +THREAD_LOCAL const snort::Connector::ID* HttpExtractor::log_id = nullptr; + +HttpExtractor::HttpExtractor(Extractor& i, uint32_t t, const vector& fields) + : ExtractorEvent(i, t) { for (const auto& f : fields) { @@ -176,6 +178,9 @@ HttpExtractor::HttpExtractor(Extractor& i, ExtractorLogger& l, uint32_t t, const DataBus::subscribe(http_pub_key, HttpEventIds::END_OF_TRANSACTION, new Eot(*this, S_NAME)); } +void HttpExtractor::internal_tinit(const snort::Connector::ID* service_id) +{ log_id = service_id; } + template<> void ExtractorEvent::log, DataEvent*, Packet*, Flow*, bool>( const vector& fields, DataEvent* event, Packet* pkt, Flow* flow, bool strict) @@ -184,9 +189,9 @@ void ExtractorEvent::log, DataEvent*, Packet*, F { const auto& field = f.get(event, pkt, flow); if (field.length() > 0) - logger.add_field(f.name, (const char*)field.start(), field.length()); + logger->add_field(f.name, (const char*)field.start(), field.length()); else if (strict) - logger.add_field(f.name, ""); + logger->add_field(f.name, ""); } } @@ -208,13 +213,13 @@ void HttpExtractor::handle(DataEvent& event, Flow* flow) Packet* packet = DetectionEngine::get_current_packet(); - logger.open_record(); + logger->open_record(); log(nts_fields, &event, packet, flow); log(sip_fields, &event, packet, flow); log(num_fields, &event, packet, flow); log(buf_fields, &event, packet, flow); - log(sub_fields, &event, packet, flow, logger.is_strict()); - logger.close_record(); + log(sub_fields, &event, packet, flow, logger->is_strict()); + logger->close_record(*log_id); } vector HttpExtractor::get_field_names() const diff --git a/src/network_inspectors/extractor/extractor_http.h b/src/network_inspectors/extractor/extractor_http.h index aa5b99999..2c33f579c 100644 --- a/src/network_inspectors/extractor/extractor_http.h +++ b/src/network_inspectors/extractor/extractor_http.h @@ -30,7 +30,7 @@ public: using SubGetFn = const Field& (*) (const DataEvent*, const Packet*, const Flow*); using SubField = DataField; - HttpExtractor(Extractor&, ExtractorLogger&, uint32_t tenant, const std::vector& fields); + HttpExtractor(Extractor&, uint32_t tenant, const std::vector& fields); std::vector get_field_names() const override; void handle(DataEvent&, Flow*); @@ -38,7 +38,10 @@ public: private: using Eot = Handler; + void internal_tinit(const snort::Connector::ID*) override; + std::vector sub_fields; + static THREAD_LOCAL const snort::Connector::ID* log_id; }; #endif diff --git a/src/network_inspectors/extractor/extractor_json_logger.cc b/src/network_inspectors/extractor/extractor_json_logger.cc index 7da0b5ecf..b0b4e3987 100644 --- a/src/network_inspectors/extractor/extractor_json_logger.cc +++ b/src/network_inspectors/extractor/extractor_json_logger.cc @@ -34,13 +34,16 @@ void JsonExtractorLogger::open_record() js.open(); } -void JsonExtractorLogger::close_record() +void JsonExtractorLogger::close_record(const snort::Connector::ID& service_id) { js.close(); - writer->lock(); - writer->write(oss.str().c_str()); - writer->unlock(); + // FIXIT-L: we're removing last character(\n) due to a limitation of + // Json Stream configuration + assert(oss.str()[oss.str().size() - 1] == '\n'); + + output_conn->transmit_message(snort::ConnectorMsg( + (const uint8_t*)oss.str().c_str(), oss.str().size() - 1, false), service_id); } void JsonExtractorLogger::add_field(const char* f, const char* v) diff --git a/src/network_inspectors/extractor/extractor_json_logger.h b/src/network_inspectors/extractor/extractor_json_logger.h index c009181c8..e24e7a76e 100644 --- a/src/network_inspectors/extractor/extractor_json_logger.h +++ b/src/network_inspectors/extractor/extractor_json_logger.h @@ -26,16 +26,12 @@ #include "helpers/json_stream.h" #include "extractor_logger.h" -#include "extractor_writer.h" class JsonExtractorLogger : public ExtractorLogger { public: - JsonExtractorLogger(OutputType o_type) - : writer(ExtractorWriter::make_writer(o_type)), oss(), js(oss) {} - - ~JsonExtractorLogger() override - { delete writer; } + JsonExtractorLogger(snort::Connector* conn) : ExtractorLogger(conn), oss(), js(oss) + { } void add_field(const char*, const char*) override; void add_field(const char*, const char*, size_t) override; @@ -44,10 +40,9 @@ public: void add_field(const char*, const snort::SfIp&) override; void add_field(const char*, bool) override; void open_record() override; - void close_record() override; + void close_record(const snort::Connector::ID&) override; private: - ExtractorWriter* const writer; std::ostringstream oss; snort::JsonStream js; diff --git a/src/network_inspectors/extractor/extractor_logger.cc b/src/network_inspectors/extractor/extractor_logger.cc index 44e708de5..3c65469cd 100644 --- a/src/network_inspectors/extractor/extractor_logger.cc +++ b/src/network_inspectors/extractor/extractor_logger.cc @@ -25,20 +25,54 @@ #include +#include "log/messages.h" +#include "managers/connector_manager.h" + #include "extractor_csv_logger.h" #include "extractor_json_logger.h" -ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, OutputType o_type) +using namespace snort; + +static Connector* get_connector(const std::string& conn_name) +{ + Connector* connector = ConnectorManager::get_connector(conn_name); + + if (connector == nullptr) + { + ErrorMessage("Can't initialize extractor, unable to find Connector \"%s\"\n", conn_name.c_str()); + abort(); + } + + switch (connector->get_connector_direction()) + { + case Connector::CONN_DUPLEX: + case Connector::CONN_TRANSMIT: + return connector; + + case Connector::CONN_RECEIVE: + case Connector::CONN_UNDEFINED: + default: + break; + } + + return nullptr; +} + +ExtractorLogger* ExtractorLogger::make_logger(FormatType f_type, const std::string& conn_name) { ExtractorLogger* logger = nullptr; + Connector* output_conn = get_connector(conn_name); + + assert(output_conn); + switch (f_type) { case FormatType::CSV: - logger = new CsvExtractorLogger(o_type); + logger = new CsvExtractorLogger(output_conn); break; case FormatType::JSON: - logger = new JsonExtractorLogger(o_type); + logger = new JsonExtractorLogger(output_conn); break; case FormatType::MAX: // fallthrough default: diff --git a/src/network_inspectors/extractor/extractor_logger.h b/src/network_inspectors/extractor/extractor_logger.h index 23b8a9ecc..05d9f09b4 100644 --- a/src/network_inspectors/extractor/extractor_logger.h +++ b/src/network_inspectors/extractor/extractor_logger.h @@ -23,17 +23,19 @@ #include #include +#include "framework/connector.h" #include "sfip/sf_ip.h" #include "extractor_enums.h" -#include "extractor_writer.h" class ExtractorLogger { public: - static ExtractorLogger* make_logger(FormatType, OutputType); + static ExtractorLogger* make_logger(FormatType, const std::string&); + + ExtractorLogger(snort::Connector* conn) : output_conn(conn) + { } - ExtractorLogger() = default; ExtractorLogger(const ExtractorLogger&) = delete; ExtractorLogger& operator=(const ExtractorLogger&) = delete; ExtractorLogger(ExtractorLogger&&) = delete; @@ -41,11 +43,9 @@ public: virtual bool is_strict() const { return false; } - virtual void set_fields(std::vector& names) - { field_names = names; } - virtual void add_header() {} - virtual void add_footer() {} + virtual void add_header(const std::vector&, const snort::Connector::ID&) {} + virtual void add_footer(const snort::Connector::ID&) {} virtual void add_field(const char*, const char*) {} virtual void add_field(const char*, const char*, size_t) {} @@ -54,11 +54,15 @@ public: virtual void add_field(const char*, const snort::SfIp&) {} virtual void add_field(const char*, bool) {} + const snort::Connector::ID get_id(const char* service_name) const + { return output_conn->get_id(service_name); } + virtual void open_record() {} - virtual void close_record() {} + virtual void close_record(const snort::Connector::ID&) {} + void flush() { output_conn->flush(); } protected: - std::vector field_names; + snort::Connector* const output_conn; }; #endif diff --git a/src/network_inspectors/extractor/extractor_service.cc b/src/network_inspectors/extractor/extractor_service.cc index 840c1eb98..afed145d4 100644 --- a/src/network_inspectors/extractor/extractor_service.cc +++ b/src/network_inspectors/extractor/extractor_service.cc @@ -46,23 +46,40 @@ std::vector ExtractorService::common_fields = "pkt_num" }; +THREAD_LOCAL ExtractorLogger* ExtractorService::logger = nullptr; + ExtractorService::ExtractorService(uint32_t tenant, const std::vector& srv_fields, const std::vector& srv_events, const ServiceBlueprint& srv_bp, ServiceType s_type, - FormatType f_type, OutputType o_type, Extractor& ins) : tenant_id(tenant), inspector(ins), sbp(srv_bp), type(s_type) + Extractor& ins) : tenant_id(tenant), inspector(ins), sbp(srv_bp), type(s_type) { add_fields(srv_fields); add_events(srv_events); - logger = ExtractorLogger::make_logger(f_type, o_type); } ExtractorService::~ExtractorService() { for (auto h : handlers) delete h; +} + +void ExtractorService::tinit(ExtractorLogger* new_logger) +{ + assert(new_logger); + + logger = new_logger; + + const Connector::ID& service_id = internal_tinit(); - delete logger; + for (auto handler : handlers) + { + handler->tinit(logger, &service_id); + logger->add_header(handler->get_field_names(), service_id); + } } +void ExtractorService::tterm() +{ logger->add_footer(get_log_id()); } + void ExtractorService::add_events(const std::vector& vals) { for (const auto& val : vals) @@ -85,8 +102,7 @@ void ExtractorService::add_fields(const std::vector& vals) } } -ExtractorService* ExtractorService::make_service(Extractor& ins, const ServiceConfig& cfg, - FormatType f_type, OutputType o_type) +ExtractorService* ExtractorService::make_service(Extractor& ins, const ServiceConfig& cfg) { if (cfg.on_events.empty()) { @@ -99,11 +115,11 @@ ExtractorService* ExtractorService::make_service(Extractor& ins, const ServiceCo switch (cfg.service) { case ServiceType::HTTP: - srv = new HttpExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, f_type, o_type, ins); + srv = new HttpExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, ins); break; case ServiceType::FTP: - srv = new FtpExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, f_type, o_type, ins); + srv = new FtpExtractorService(cfg.tenant_id, cfg.fields, cfg.on_events, cfg.service, ins); break; case ServiceType::UNDEFINED: // fallthrough @@ -180,30 +196,28 @@ ServiceBlueprint HttpExtractorService::blueprint = }, }; +THREAD_LOCAL Connector::ID HttpExtractorService::log_id; + HttpExtractorService::HttpExtractorService(uint32_t tenant, const std::vector& srv_fields, - const std::vector& srv_events, ServiceType s_type, FormatType f_type, OutputType o_type, Extractor& ins) - : ExtractorService(tenant, srv_fields, srv_events, blueprint, s_type, f_type, o_type, ins) + const std::vector& srv_events, ServiceType s_type, Extractor& ins) + : ExtractorService(tenant, srv_fields, srv_events, blueprint, s_type, ins) { - if (!logger) - return; - for (const auto& event : get_events()) { - ExtractorEvent* eh; - if (!strcmp("eot", event.c_str())) - eh = new HttpExtractor(ins, *logger, tenant_id, get_fields()); + handlers.push_back(new HttpExtractor(ins, tenant_id, get_fields())); else continue; - - auto names = eh->get_field_names(); - logger->set_fields(names); - logger->add_header(); - handlers.push_back(eh); } } +const snort::Connector::ID& HttpExtractorService::internal_tinit() +{ return log_id = logger->get_id(type.c_str()); } + +const snort::Connector::ID& HttpExtractorService::get_log_id() +{ return log_id; } + //------------------------------------------------------------------------- // FtpExtractorService //------------------------------------------------------------------------- @@ -231,36 +245,31 @@ ServiceBlueprint FtpExtractorService::blueprint = }, }; +THREAD_LOCAL Connector::ID FtpExtractorService::log_id; + FtpExtractorService::FtpExtractorService(uint32_t tenant, const std::vector& srv_fields, - const std::vector& srv_events, ServiceType s_type, FormatType f_type, OutputType o_type, Extractor& ins) - : ExtractorService(tenant, srv_fields, srv_events, blueprint, s_type, f_type, o_type, ins) + const std::vector& srv_events, ServiceType s_type, Extractor& ins) + : ExtractorService(tenant, srv_fields, srv_events, blueprint, s_type, ins) { - if (!logger) - return; - for (const auto& event : get_events()) { - ExtractorEvent* eh; - if (!strcmp("request", event.c_str())) - eh = new FtpRequestExtractor(ins, *logger, tenant_id, get_fields()); - + handlers.push_back(new FtpRequestExtractor(ins, tenant_id, get_fields())); else if (!strcmp("response", event.c_str())) - eh = new FtpResponseExtractor(ins, *logger, tenant_id, get_fields()); - + handlers.push_back(new FtpResponseExtractor(ins, tenant_id, get_fields())); else if (!strcmp("eot", event.c_str())) - eh = new FtpExtractor(ins, *logger, tenant_id, get_fields()); - + handlers.push_back(new FtpExtractor(ins, tenant_id, get_fields())); else continue; - - auto names = eh->get_field_names(); - logger->set_fields(names); - logger->add_header(); - handlers.push_back(eh); } } +const snort::Connector::ID& FtpExtractorService::internal_tinit() +{ return log_id = logger->get_id(type.c_str()); } + +const snort::Connector::ID& FtpExtractorService::get_log_id() +{ return log_id; } + //------------------------------------------------------------------------- // Unit Tests //------------------------------------------------------------------------- diff --git a/src/network_inspectors/extractor/extractor_service.h b/src/network_inspectors/extractor/extractor_service.h index 9ba24d339..95857b32b 100644 --- a/src/network_inspectors/extractor/extractor_service.h +++ b/src/network_inspectors/extractor/extractor_service.h @@ -24,6 +24,8 @@ #include #include +#include "framework/connector.h" + #include "extractor_enums.h" #include "extractor_logger.h" @@ -41,7 +43,7 @@ struct ServiceBlueprint class ExtractorService { public: - static ExtractorService* make_service(Extractor&, const ServiceConfig&, FormatType, OutputType); + static ExtractorService* make_service(Extractor&, const ServiceConfig&); ExtractorService() = delete; ExtractorService(const ExtractorService&) = delete; @@ -50,13 +52,20 @@ public: virtual ~ExtractorService(); void show(std::string&) const; + void tinit(ExtractorLogger* logger); + void tterm(); + uint32_t get_tenant() const { return tenant_id; } const std::vector& get_events() const { return events; } const std::vector& get_fields() const { return fields; } protected: ExtractorService(uint32_t tenant, const std::vector& fields, const std::vector& events, - const ServiceBlueprint& srv_bp, ServiceType, FormatType, OutputType, Extractor&); + const ServiceBlueprint& srv_bp, ServiceType, Extractor& ins); + + virtual const snort::Connector::ID& internal_tinit() = 0; + virtual const snort::Connector::ID& get_log_id() = 0; + void add_events(const std::vector& vals); void add_fields(const std::vector& vals); bool find_event(const std::string&) const; @@ -68,9 +77,9 @@ protected: std::vector fields; std::vector events; - ExtractorLogger* logger = nullptr; Extractor& inspector; std::vector handlers; + static THREAD_LOCAL ExtractorLogger* logger; const ServiceBlueprint& sbp; const ServiceType type; @@ -80,20 +89,28 @@ class HttpExtractorService : public ExtractorService { public: HttpExtractorService(uint32_t tenant, const std::vector& fields, - const std::vector& events, ServiceType, FormatType, OutputType, Extractor&); + const std::vector& events, ServiceType, Extractor&); private: + const snort::Connector::ID& internal_tinit() override; + const snort::Connector::ID& get_log_id() override; + static ServiceBlueprint blueprint; + static THREAD_LOCAL snort::Connector::ID log_id; }; class FtpExtractorService : public ExtractorService { public: FtpExtractorService(uint32_t tenant, const std::vector& fields, - const std::vector& events, ServiceType, FormatType, OutputType, Extractor&); + const std::vector& events, ServiceType, Extractor&); private: + const snort::Connector::ID& internal_tinit() override; + const snort::Connector::ID& get_log_id() override; + static ServiceBlueprint blueprint; + static THREAD_LOCAL snort::Connector::ID log_id; }; #endif diff --git a/src/network_inspectors/extractor/extractor_writer.cc b/src/network_inspectors/extractor/extractor_writer.cc deleted file mode 100644 index a5eee7d64..000000000 --- a/src/network_inspectors/extractor/extractor_writer.cc +++ /dev/null @@ -1,92 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- -// extractor_writer.cc author Anna Norokh - -#ifdef HAVE_CONFIG_H -#include "config.h" -#endif - -#include "extractor_writer.h" - -using namespace snort; - -ExtractorWriter* ExtractorWriter::make_writer(OutputType o_type) -{ - switch (o_type) - { - case OutputType::STD: - return new StdExtractorWriter(); - case OutputType::MAX: // fallthrough - default: - return nullptr; - } -} - -StdExtractorWriter::StdExtractorWriter() : ExtractorWriter(), extr_std_log(TextLog_Init("stdout")) -{} - -StdExtractorWriter::~StdExtractorWriter() -{ - TextLog_Term(extr_std_log); -} - -void StdExtractorWriter::write(const char* ss) -{ - TextLog_Print(extr_std_log, "%s", ss); -} - -void StdExtractorWriter::write(const char* ss, size_t len) -{ - TextLog_Print(extr_std_log, "%.*s", (int)len, ss); -} - -void StdExtractorWriter::write(uint64_t n) -{ - TextLog_Print(extr_std_log, STDu64, n); -} - -void StdExtractorWriter::lock() -{ - write_mutex.lock(); -} - -void StdExtractorWriter::unlock() -{ - TextLog_Flush(extr_std_log); // FIXIT-L: should be a part of API and have a well-defined point in the pipeline - write_mutex.unlock(); -} - -#ifdef UNIT_TEST - -#include "catch/snort_catch.h" - -#include - -TEST_CASE("Output Type", "[extractor]") -{ - SECTION("to string") - { - OutputType std = OutputType::STD; - OutputType max = OutputType::MAX; - - CHECK_FALSE(strcmp("stdout", std.c_str())); - CHECK_FALSE(strcmp("(not set)", max.c_str())); - } -} - -#endif diff --git a/src/network_inspectors/extractor/extractor_writer.h b/src/network_inspectors/extractor/extractor_writer.h deleted file mode 100644 index 4ac441e6b..000000000 --- a/src/network_inspectors/extractor/extractor_writer.h +++ /dev/null @@ -1,68 +0,0 @@ -//-------------------------------------------------------------------------- -// Copyright (C) 2024-2024 Cisco and/or its affiliates. All rights reserved. -// -// This program is free software; you can redistribute it and/or modify it -// under the terms of the GNU General Public License Version 2 as published -// by the Free Software Foundation. You may not use, modify or distribute -// this program under any other version of the GNU General Public License. -// -// This program is distributed in the hope that it will be useful, but -// WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -// General Public License for more details. -// -// You should have received a copy of the GNU General Public License along -// with this program; if not, write to the Free Software Foundation, Inc., -// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -//-------------------------------------------------------------------------- -// extractor_writer.h author Anna Norokh - -#ifndef EXTRACTOR_WRITER_H -#define EXTRACTOR_WRITER_H - -#include - -#include "log/text_log.h" -#include "main/snort_types.h" - -#include "extractor_enums.h" - -class ExtractorWriter -{ -public: - static ExtractorWriter* make_writer(OutputType); - - ExtractorWriter(const ExtractorWriter&) = delete; - ExtractorWriter& operator=(const ExtractorWriter&) = delete; - ExtractorWriter(ExtractorWriter&&) = delete; - - virtual ~ExtractorWriter() = default; - - virtual void write(const char*) = 0; - virtual void write(const char*, size_t) = 0; - virtual void write(uint64_t) = 0; - virtual void lock() { } - virtual void unlock() { } - -protected: - ExtractorWriter() = default; -}; - -class StdExtractorWriter : public ExtractorWriter -{ -public: - StdExtractorWriter(); - ~StdExtractorWriter() override; - - void write(const char* ss) override; - void write(const char* ss, size_t len) override; - void write(uint64_t n) override; - void lock() override; - void unlock() override; - -private: - std::mutex write_mutex; - TextLog* extr_std_log; -}; - -#endif diff --git a/src/network_inspectors/extractor/extractors.cc b/src/network_inspectors/extractor/extractors.cc index 1db66b8e2..6a778076a 100644 --- a/src/network_inspectors/extractor/extractors.cc +++ b/src/network_inspectors/extractor/extractors.cc @@ -24,6 +24,15 @@ #include "extractors.h" using namespace std; +using namespace snort; + +THREAD_LOCAL ExtractorLogger* ExtractorEvent::logger = nullptr; + +void ExtractorEvent::tinit(ExtractorLogger* l, const snort::Connector::ID* service_id) +{ + logger = l; + internal_tinit(service_id); +} vector ExtractorEvent::get_field_names() const { diff --git a/src/network_inspectors/extractor/extractors.h b/src/network_inspectors/extractor/extractors.h index 9f5dcdb08..402d0e3a1 100644 --- a/src/network_inspectors/extractor/extractors.h +++ b/src/network_inspectors/extractor/extractors.h @@ -26,6 +26,7 @@ #include "detection/detection_engine.h" #include "flow/flow_key.h" #include "framework/data_bus.h" +#include "framework/connector.h" #include "sfip/sf_ip.h" #include "extractor_logger.h" @@ -69,7 +70,10 @@ public: virtual ~ExtractorEvent() {} + void tinit(ExtractorLogger*, const snort::Connector::ID*); + Extractor& get_inspector() const { return inspector; } + virtual std::vector get_field_names() const; void handle(DataEvent&, Flow*) {} @@ -108,7 +112,7 @@ protected: void log(const T& fields, Context... context) { for (const auto& f : fields) - logger.add_field(f.name, f.get(context...)); + logger->add_field(f.name, f.get(context...)); } void log(const std::vector& fields, DataEvent* event, Packet* pkt, Flow* flow, bool strict) @@ -117,9 +121,9 @@ protected: { const auto& str = f.get(event, pkt, flow); if (str.second > 0) - logger.add_field(f.name, (const char*)str.first, str.second); + logger->add_field(f.name, (const char*)str.first, str.second); else if (strict) - logger.add_field(f.name, ""); + logger->add_field(f.name, ""); } } @@ -132,12 +136,14 @@ protected: return it != map.end(); } - ExtractorEvent(Extractor& i, ExtractorLogger& l, uint32_t tid) - : tenant_id(tid), logger(l), inspector(i) { } + ExtractorEvent(Extractor& i, uint32_t tid) : tenant_id(tid), inspector(i) + { } + + virtual void internal_tinit(const snort::Connector::ID*) = 0; uint32_t tenant_id; - ExtractorLogger& logger; Extractor& inspector; + static THREAD_LOCAL ExtractorLogger* logger; std::vector nts_fields; std::vector sip_fields; diff --git a/src/side_channel/test/side_channel_test.cc b/src/side_channel/test/side_channel_test.cc index 415e6db71..f5c5871e2 100644 --- a/src/side_channel/test/side_channel_test.cc +++ b/src/side_channel/test/side_channel_test.cc @@ -41,9 +41,9 @@ class DuplexConnector : public Connector public: DuplexConnector() : Connector(duplex_conf) { } - bool transmit_message(const ConnectorMsg&) override + bool transmit_message(const ConnectorMsg&, const Connector::ID&) override { return true; } - bool transmit_message(const ConnectorMsg&&) override + bool transmit_message(const ConnectorMsg&&, const Connector::ID&) override { return true; } ConnectorMsg receive_message(bool) override { @@ -61,9 +61,9 @@ class ReceiveConnector : public Connector public: ReceiveConnector() : Connector(receive_conf) { } - bool transmit_message(const ConnectorMsg&) override + bool transmit_message(const ConnectorMsg&, const Connector::ID&) override { return false; } - bool transmit_message(const ConnectorMsg&&) override + bool transmit_message(const ConnectorMsg&&, const Connector::ID&) override { return false; } ConnectorMsg receive_message(bool) override { @@ -81,9 +81,9 @@ class TransmitConnector : public Connector public: TransmitConnector() : Connector(transmit_conf) { } - bool transmit_message(const ConnectorMsg&) override + bool transmit_message(const ConnectorMsg&, const Connector::ID&) override { return true; } - bool transmit_message(const ConnectorMsg&&) override + bool transmit_message(const ConnectorMsg&&, const Connector::ID&) override { return true; } ConnectorMsg receive_message(bool) override { return ConnectorMsg(); }