Pull request #4575: dump_config: add --gen-dump-config option

Merge in SNORT/snort3 from ~YCHALOV/snort3:gen_config_dump_implementation to master

Squashed commit of the following:

commit 62e3fb3c6998ca0d71ff543bfb826fa83f68a22d
Author: Yurii Chalov <ychalov@cisco.com>
Date:   Mon Jan 13 13:17:29 2025 +0100

    dump_config: implement dump config generation in a file

diff --git a/doc/user/dump_config.txt b/doc/user/dump_config.txt
index 9fa72d9a9c302d962abc54158835a3a7e5659c2a..7cd4ca10d30c1e706bfa3d021426201b19912549 100644 (file)
--- a/doc/user/dump_config.txt
+++ b/doc/user/dump_config.txt
@@ -10,6 +10,10 @@ The dump mode is activated by the following options:
 --dump-config-text, --dump-config=all, --dump-config=top.
 They are described in detail below.
 
+The --gen-dump-config <file> option enables Snort to generate a dump
+configuration file with a timestamp and config generation ID
+during startup and reload.
+
 The simple configuration is used in examples.
 The output contains applied configurations (defaults and configured).
 To simplify the output we show a brief list of default options.
@@ -289,3 +293,15 @@ Example:
             "show_rebuilt_packets": true
         },
     }
+
+==== Configuration Dump Generation During Startup and Reload
+
+The --gen-dump-config <file> option dumps configuration in a file in
+JSON format, similar to the --dump-config=all option. It creates a file
+during startup and reload, with the specified name, timestamp and config
+generation ID appended.
+
+Example: snort -c snort.lua --gen-dump-config dump_output
+
+After execution, the "dump_output_<timestamp>_<config_gen_id>" file
+will be generated.

diff --git a/src/dump_config/config_output.cc b/src/dump_config/config_output.cc
index 0fa6dc2c74f6ca6fb1d4ed90af7ce20301679e23..94eda4082f96a7b356239397322eb3a56769ba3d 100644 (file)
--- a/src/dump_config/config_output.cc
+++ b/src/dump_config/config_output.cc
@@ -25,10 +25,11 @@
 
 #include "config_data.h"
 
-void ConfigOutput::dump_config(ConfigData& config_data)
+void ConfigOutput::dump_config(ConfigData& config_data, bool to_clear)
 {
     config_data.sort();
     dump(config_data);
-    config_data.clear();
+    if (to_clear)
+        config_data.clear();
 }
 

diff --git a/src/dump_config/config_output.h b/src/dump_config/config_output.h
index 90ab347879a5b4ed42492594e326472a3251cec0..85a1c060e7525be5299eca1346aeeab0f6fb8276 100644 (file)
--- a/src/dump_config/config_output.h
+++ b/src/dump_config/config_output.h
@@ -28,7 +28,7 @@ public:
     ConfigOutput() = default;
     virtual ~ConfigOutput() = default;
 
-    void dump_config(ConfigData&);
+    void dump_config(ConfigData&, bool to_clear = true);
 
 private:
     virtual void dump(const ConfigData&) = 0;

diff --git a/src/dump_config/json_config_output.cc b/src/dump_config/json_config_output.cc
index 60fab35497467a6fd649a7f0a221271ef9d2407b..06bf0a4688c6b89455472d352908582327e21f15 100644 (file)
--- a/src/dump_config/json_config_output.cc
+++ b/src/dump_config/json_config_output.cc
@@ -21,6 +21,11 @@
 #include "config.h"
 #endif
 
+#include <cstring>
+#include <cerrno>
+
+#include <log/messages.h>
+
 #include "json_config_output.h"
 
 #include "config_data.h"
@@ -90,24 +95,49 @@ static void dump_tree(JsonStream& json, const BaseConfigNode* node, bool list_no
         dump_value(json, node_name_cstr, node);
 }
 
-JsonAllConfigOutput::JsonAllConfigOutput() :
-    ConfigOutput(), json(std::cout)
-{ json.open_array(); }
+JsonAllConfigOutput::JsonAllConfigOutput(const char *file_name) :
+    ConfigOutput(), file(nullptr), json(nullptr)
+{
+    if (file_name)
+        file = new std::fstream(std::string(file_name), std::ios::out);
+
+    if (file and !file->is_open())
+    {
+        ErrorMessage("File %s is not opened: %s\n", file_name, std::strerror(errno));
+        delete file;
+        file = nullptr;
+        return;
+    }
+
+    json = file ? new snort::JsonStream(*file) : new snort::JsonStream(std::cout);
+
+    json->open_array();
+}
 
 JsonAllConfigOutput::~JsonAllConfigOutput()
-{ json.close_array(); }
+{
+    if (json)
+    {
+        json->close_array();
+        delete json;
+    }
+    if (file)
+        delete file;
+}
 
 void JsonAllConfigOutput::dump(const ConfigData& config_data)
 {
-    json.open();
-    json.put("filename", config_data.file_name);
-    json.open("config");
+    if (!json)
+        return;
+    json->open();
+    json->put("filename", config_data.file_name);
+    json->open("config");
 
     for ( const auto config_tree: config_data.config_trees )
-        dump_tree(json, config_tree);
+        dump_tree(*json, config_tree);
 
-    json.close();
-    json.close();
+    json->close();
+    json->close();
 }
 
 void JsonTopConfigOutput::dump(const ConfigData& config_data)

diff --git a/src/dump_config/json_config_output.h b/src/dump_config/json_config_output.h
index 422fb6da04170d0b8a8670c6ac902735b3b8d071..f1041092d1a9e56765603e6af69c5254a2275a95 100644 (file)
--- a/src/dump_config/json_config_output.h
+++ b/src/dump_config/json_config_output.h
@@ -20,6 +20,8 @@
 #ifndef JSON_CONFIG_OUTPUT_H
 #define JSON_CONFIG_OUTPUT_H
 
+#include <fstream>
+
 #include "config_output.h"
 #include "helpers/json_stream.h"
 
@@ -28,14 +30,15 @@ class BaseConfigNode;
 class JsonAllConfigOutput : public ConfigOutput
 {
 public:
-    JsonAllConfigOutput();
+    JsonAllConfigOutput(const char *file_name = nullptr);
     ~JsonAllConfigOutput() override;
 
 private:
     void dump(const ConfigData&) override;
 
 private:
-    snort::JsonStream json;
+    std::fstream *file;
+    snort::JsonStream *json;
 };
 
 class JsonTopConfigOutput : public ConfigOutput

diff --git a/src/main/shell.cc b/src/main/shell.cc
index bb7c69060aab86b0b758e845d7e91614d112458c..bb7b7134cee24ecd30fb34007d939760e30de7e3 100644 (file)
--- a/src/main/shell.cc
+++ b/src/main/shell.cc
@@ -259,7 +259,7 @@ void Shell::allowlist_append(const char* keyword, bool is_prefix)
 void Shell::config_open_table(bool is_root_node, bool is_list, int idx,
     const std::string& table_name, const Parameter* p)
 {
-    if ( !s_config_output )
+    if ( !dump_enabled() )
         return;
 
     Parameter::Type node_type = is_list ? Parameter::PT_LIST : Parameter::PT_TABLE;
@@ -291,7 +291,7 @@ void Shell::config_open_table(bool is_root_node, bool is_list, int idx,
 void Shell::add_config_child_node(const std::string& node_name, snort::Parameter::Type type,
     bool is_root_list_item)
 {
-    if ( !s_config_output || !s_current_node )
+    if ( !dump_enabled() or !s_current_node )
         return;
 
     // element of the top-level list is anonymous
@@ -305,7 +305,7 @@ void Shell::add_config_child_node(const std::string& node_name, snort::Parameter
 
 void Shell::add_config_root_node(const std::string& root_name, snort::Parameter::Type node_type)
 {
-    if ( !s_config_output )
+    if ( !dump_enabled() )
         return;
 
     Shell* sh = Shell::get_current_shell();
@@ -314,12 +314,12 @@ void Shell::add_config_root_node(const std::string& root_name, snort::Parameter:
         return;
 
     sh->s_current_node = new TreeConfigNode(nullptr, root_name, node_type);
-    sh->config_data.add_config_tree(sh->s_current_node);
+    sh->config_data->add_config_tree(sh->s_current_node);
 }
 
 void Shell::update_current_config_node(const std::string& node_name)
 {
-    if ( !s_config_output || !s_current_node )
+    if ( !dump_enabled() or !s_current_node )
         return;
 
     // node has been added during setting default options
@@ -335,7 +335,7 @@ void Shell::update_current_config_node(const std::string& node_name)
 
 void Shell::config_close_table()
 {
-    if ( !s_config_output )
+    if ( !dump_enabled() )
         return;
 
     if ( !s_close_table )
@@ -352,7 +352,7 @@ void Shell::config_close_table()
 
 void Shell::set_config_value(const std::string& fqn, const snort::Value& value)
 {
-    if ( !s_config_output || !s_current_node )
+    if ( !dump_enabled() or !s_current_node )
         return;
 
     // don't give names to list elements
@@ -486,6 +486,11 @@ bool Shell::load_string(const char* s, bool load_in_sandbox, const char* message
     return true;
 }
 
+bool Shell::dump_enabled()
+{
+    return s_config_output or snort::SnortConfig::get_conf()->gen_dump_config();
+}
+
 bool Shell::load_config(const char* file, bool load_in_sandbox)
 {
     if ( load_in_sandbox )
@@ -523,7 +528,7 @@ bool Shell::load_config(const char* file, bool load_in_sandbox)
 //-------------------------------------------------------------------------
 
 Shell::Shell(const char* s, bool load_defaults) :
-    config_data(s), load_defaults(load_defaults)
+    config_data(new ConfigData(s)), load_defaults(load_defaults)
 {
     // FIXIT-M should wrap in Lua::State
     lua = luaL_newstate();
@@ -556,12 +561,14 @@ Shell::Shell(const char* s, bool load_defaults) :
 Shell::~Shell()
 {
     lua_close(lua);
+    if (config_data)
+        delete config_data;
 }
 
 void Shell::set_file(const char* s)
 {
     file = s;
-    config_data.file_name = file;
+    config_data->file_name = file;
 }
 
 void Shell::set_overrides(const char* s)
@@ -574,7 +581,7 @@ void Shell::set_overrides(Shell* sh)
     overrides += sh->overrides;
 }
 
-bool Shell::configure(SnortConfig* sc, bool is_root)
+bool Shell::configure(SnortConfig* sc, bool is_root, std::list<ConfigData*> *config_data_to_dump)
 {
     assert(file.size());
     ModuleManager::set_config(sc);
@@ -648,7 +655,16 @@ bool Shell::configure(SnortConfig* sc, bool is_root)
 
     auto config_output = Shell::get_current_shell()->s_config_output;
     if ( config_output )
-        config_output->dump_config(config_data);
+    {
+        bool to_clear = sc->gen_dump_config() ? false : true;
+        config_output->dump_config(*config_data, to_clear);
+    }
+
+    if ( sc->gen_dump_config() )
+    {
+        config_data_to_dump->push_back(config_data);
+        config_data = nullptr;
+    }
 
     current_shells.pop();
 

diff --git a/src/main/shell.h b/src/main/shell.h
index a414d987df6e098e4a967c6cc8d71039bd75b47a..8d356e7331e02e9b82ef7e1f306dd25bf429f47c 100644 (file)
--- a/src/main/shell.h
+++ b/src/main/shell.h
@@ -54,7 +54,7 @@ public:
     void set_overrides(const char*);
     void set_overrides(Shell*);
 
-    bool configure(snort::SnortConfig*, bool is_root = false);
+    bool configure(snort::SnortConfig*, bool is_root = false, std::list<ConfigData*>* = nullptr);
     void install(const char*, const struct luaL_Reg*);
     void execute(const char*, std::string&);
 
@@ -131,6 +131,7 @@ private:
     bool set_sandbox_env();
     bool load_string(const char* s, bool load_in_sandbox, const char* message);
     bool load_config(const char* file, bool load_in_sandbox);
+    static bool dump_enabled();
 
 private:
     bool loaded;
@@ -142,7 +143,7 @@ private:
     Allowlist allowlist;
     Allowlist internal_allowlist;
     Allowlist allowlist_prefixes;
-    ConfigData config_data;
+    ConfigData* config_data;
     uint64_t network_user_policy_id = UNDEFINED_NETWORK_USER_POLICY_ID;
     bool load_defaults;
 };

diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc
index 68048d0baf32f656c391a530e8bbea7b1450a2e5..0e5ccfff1a1a50bee65c9180e11d1304f548863c 100644 (file)
--- a/src/main/snort_config.cc
+++ b/src/main/snort_config.cc
@@ -23,6 +23,7 @@
 
 #include "snort_config.h"
 
+#include <atomic>
 #include <grp.h>
 #include <mutex>
 #include <pwd.h>
@@ -34,6 +35,7 @@
 #include "detection/detection_engine.h"
 #include "detection/fp_config.h"
 #include "detection/fp_create.h"
+#include "dump_config/config_data.h"
 #include "dump_config/json_config_output.h"
 #include "dump_config/text_config_output.h"
 #include "events/event_queue.h"
@@ -207,6 +209,31 @@ void SnortConfig::init(const SnortConfig* const other_conf, ProtocolReference* p
     }
 }
 
+static const int threads_max = 16;
+static std::atomic<int> threads_cnt = 0;
+
+static void generate_config_dump(std::list<ConfigData*> *config_data, time_t timestamp,
+    unsigned int reload_id, std::string file_name)
+{
+    ++threads_cnt;
+
+    file_name += "_";
+    file_name += std::to_string(timestamp);
+    file_name += "_";
+    file_name += std::to_string(reload_id);
+
+    ConfigOutput* o = new JsonAllConfigOutput(file_name.c_str());
+    for (auto i : *config_data)
+    {
+        o->dump_config(*i);
+        delete i;
+    }
+    delete o;
+    delete config_data;
+
+    --threads_cnt;
+}
+
 //-------------------------------------------------------------------------
 // public methods
 //-------------------------------------------------------------------------
@@ -278,6 +305,12 @@ SnortConfig::~SnortConfig()
     InspectorManager::delete_config(this);
     ActionManager::delete_config(this);
 
+    if (config_dumper)
+    {
+        config_dumper->join();
+        delete config_dumper;
+    }
+
     delete[] state;
     delete thread_config;
     delete trace_config;
@@ -1046,6 +1079,19 @@ void SnortConfig::update_reload_id()
     reload_id = ++reload_id_tracker;
 }
 
+void SnortConfig::generate_dump(std::list<ConfigData*> *config_data_to_dump)
+{
+    if (threads_cnt < threads_max)
+    {
+        config_dumper = new std::thread(generate_config_dump, config_data_to_dump,
+            time(nullptr), SnortConfig::get_conf()->get_reload_id(), dump_config_file);
+    }
+    else
+    {
+        delete config_data_to_dump;
+    }
+}
+
 void SnortConfig::cleanup_fatal_error()
 {
     // FIXIT-L need a generic way to manage type other threads

diff --git a/src/main/snort_config.h b/src/main/snort_config.h
index 0d7f84da9ef27d23f1eb021fe2269c9e41346ff9..c4337e32055ef863893b766e66863c5b5039a386 100644 (file)
--- a/src/main/snort_config.h
+++ b/src/main/snort_config.h
@@ -27,6 +27,7 @@
 
 #include <list>
 #include <mutex>
+#include <thread>
 #include <unordered_map>
 #include <vector>
 
@@ -69,6 +70,7 @@ enum RunFlag
     RUN_FLAG__TRACK_ON_SYN        = 0x00100000,
     RUN_FLAG__IP_FRAGS_ONLY       = 0x00200000,
     RUN_FLAG__TEST_FEATURES       = 0x00400000,
+    RUN_FLAG__GEN_DUMP_CONFIG     = 0x00800000,
 
 #ifdef SHELL
     RUN_FLAG__SHELL               = 0x01000000,
@@ -131,6 +133,7 @@ class ControlConn;
 class FastPatternConfig;
 class RuleStateMap;
 class TraceConfig;
+class ConfigData;
 
 struct srmm_table_t;
 struct sopg_table_t;
@@ -413,6 +416,9 @@ public:
     SoRules* so_rules = nullptr;
 
     DumpConfigType dump_config_type = DUMP_CONFIG_NONE;
+
+    std::string dump_config_file;
+    std::thread* config_dumper = nullptr;
 private:
     std::list<ReloadResourceTuner*> reload_tuners;
     static std::mutex reload_id_mutex;
@@ -603,6 +609,9 @@ public:
     bool test_features() const
     { return run_flags & RUN_FLAG__TEST_FEATURES; }
 
+    bool gen_dump_config() const
+    { return run_flags & RUN_FLAG__GEN_DUMP_CONFIG; }
+
     // other stuff
     uint8_t min_ttl() const
     { return get_network_policy()->min_ttl; }
@@ -671,6 +680,8 @@ public:
     unsigned get_reload_id() const
     { return reload_id; }
 
+    void generate_dump(std::list<ConfigData*>*);
+
     bool get_default_rule_state() const;
 
     ConfigOutput* create_config_output() const;

diff --git a/src/main/snort_module.cc b/src/main/snort_module.cc
index dc4204345d977104262e6547e3dabea36e8467d4..9089ce551f2f992e8707c843457d7aab22ed0336 100644 (file)
--- a/src/main/snort_module.cc
+++ b/src/main/snort_module.cc
@@ -428,6 +428,9 @@ static const Parameter s_params[] =
     { "--enable-test-features", Parameter::PT_IMPLIED, nullptr, nullptr,
       "enable features used in testing" },
 
+    { "--gen-dump-config", Parameter::PT_STRING, nullptr, nullptr,
+      "<file> dump configuration to <file_timestamp> during startup and configuration reload" },
+
     { "--gen-msg-map", Parameter::PT_IMPLIED, nullptr, nullptr,
       "dump configured rules in gen-msg.map format for use by other tools" },
 
@@ -997,7 +1000,11 @@ bool SnortModule::set(const char*, Value& v, SnortConfig* sc)
         sc->run_flags |= RUN_FLAG__TEST_FEATURES;
         SfIp::test_features = true;
     }
-
+    else if ( is(v, "--gen-dump-config") )
+    {
+        sc->run_flags |= RUN_FLAG__GEN_DUMP_CONFIG;
+        sc->dump_config_file = v.get_string();
+    }
     else if ( is(v, "--gen-msg-map") )
     {
         sc->run_flags |= (RUN_FLAG__DUMP_MSG_MAP | RUN_FLAG__TEST);

diff --git a/src/managers/module_manager.cc b/src/managers/module_manager.cc
index 2a8366b505318c4d9ddaae85547e6a7517cc4607..4516effd2d4d7ca2d5e1302da71268ea58655dd4 100644 (file)
--- a/src/managers/module_manager.cc
+++ b/src/managers/module_manager.cc
@@ -833,7 +833,7 @@ SO_PUBLIC bool open_table(const char* s, int idx)
         s_current = unique_key;
     }
 
-    if ( s_config->dump_config_mode() )
+    if ( s_config->dump_config_mode() or s_config->gen_dump_config() )
     {
         std::string table_name = get_sub_table(s);
         bool is_top_level = false;

diff --git a/src/parser/parser.cc b/src/parser/parser.cc
index 04063257a5000286e97b2eb4a6b0dc6e05a94657..96cbdc53ab54b11e3d149d30fa29a035539c4ba9 100644 (file)
--- a/src/parser/parser.cc
+++ b/src/parser/parser.cc
@@ -289,14 +289,14 @@ static RuleListNode* addNodeToOrderedList
     return ordered_list;
 }
 
-static bool parse_file(SnortConfig* sc, Shell* sh, bool is_root)
+static bool parse_file(SnortConfig* sc, Shell* sh, bool is_root, std::list<ConfigData*> *config_data_to_dump)
 {
     const char* fname = sh->get_file();
 
     if ( !fname || !*fname )
         return false;
 
-    bool success = sh->configure(sc, is_root);
+    bool success = sh->configure(sc, is_root, config_data_to_dump);
 
     return success;
 }
@@ -331,6 +331,7 @@ SnortConfig* ParseSnortConf(const SnortConfig* cmd_line_conf, const char* fname)
     sc->output_flags = cmd_line_conf->output_flags;
     sc->tweaks = cmd_line_conf->tweaks;
     sc->dump_config_type = cmd_line_conf->dump_config_type;
+    sc->dump_config_file = cmd_line_conf->dump_config_file;
 
     if ( !fname )
         fname = get_snort_conf();
@@ -361,24 +362,39 @@ SnortConfig* ParseSnortConf(const SnortConfig* cmd_line_conf, const char* fname)
     bool parse_file_failed = false;
     auto output = current_conf->create_config_output();
     bool is_top = current_conf->dump_config_type == DUMP_CONFIG_JSON_TOP;
+    std::list<ConfigData*> *config_data_to_dump = new std::list<ConfigData*>;
+
     for ( unsigned i = 0; true; i++ )
     {
         sh = sc->policy_map->get_shell(i);
 
         if ( !sh )
+        {
+            if (!sc->gen_dump_config())
+                break;
+            sc->generate_dump(config_data_to_dump);
+            config_data_to_dump = nullptr;
             break;
+        }
 
         auto shell_output = ( i != 0 && is_top ) ? nullptr : output;
         Shell::set_config_output(shell_output);
         set_policies(sc, sh);
 
-        if (!parse_file(sc, sh, (i == 0)))
+        if (!parse_file(sc, sh, (i == 0), config_data_to_dump))
         {
             parse_file_failed = true;
             break;
         }
     }
 
+    if (config_data_to_dump)
+    {
+        for (auto i : *config_data_to_dump)
+            delete i;
+        delete config_data_to_dump;
+    }
+
     delete output;
     Shell::clear_config_output();