Pull request #4464: snort, search_engine: remove --dump-rule-databases

Merge in SNORT/snort3 from ~RUCOMBS/snort3:rule_db_dir to master

Squashed commit of the following:

commit 453e493024c93e33af63800afa5322e7b47c6013
Author: Russ Combs <rucombs@cisco.com>
Date:   Fri Oct 4 07:58:10 2024 -0400

    snort: bump minor version for MPSE API change

commit e2a836a2dcaafd000edebaf275244bd8f5e7424c
Author: Russ Combs <rucombs@cisco.com>
Date:   Thu Sep 12 12:28:39 2024 -0400

    snort, search_engine: remove --dump-rule-databases

    Rules are now automatically dumped to search_engine.rule_db_dir if not
    loaded. Combine with snort --mem-check to get equivalent functionality
    to --dump-rule-databases.

diff --git a/CMakeLists.txt b/CMakeLists.txt
index b351b09a5af850a16ef00ab65b5d0be099fff96c..e2e7f0704082db0d60b243fbd5c36549d0a1a431 100644 (file)
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2,8 +2,8 @@ cmake_minimum_required (VERSION 3.4.3)
 project (snort CXX C)
 
 set (VERSION_MAJOR 3)
-set (VERSION_MINOR 3)
-set (VERSION_PATCH 7)
+set (VERSION_MINOR 4)
+set (VERSION_PATCH 0)
 set (VERSION_SUBLEVEL 0)
 set (VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}.${VERSION_SUBLEVEL}")
 

diff --git a/src/detection/fp_config.h b/src/detection/fp_config.h
index 31b13fab08bdcfc1dbe937bf9e9cc2de199f54c9..fe6e8e36893277ddb92726f88817a0b3246cf8d1 100644 (file)
--- a/src/detection/fp_config.h
+++ b/src/detection/fp_config.h
@@ -112,11 +112,13 @@ public:
     void set_debug_print_rule_groups_uncompiled()
     { portlists_flags |= PL_DEBUG_PRINT_RULEGROUPS_UNCOMPILED; }
 
+#ifdef HAVE_HYPERSCAN
     void set_rule_db_dir(const char* s)
     { rule_db_dir = s; }
 
     const std::string& get_rule_db_dir() const
     { return rule_db_dir; }
+#endif
 
     bool set_search_method(const char*);
     const char* get_search_method() const;
@@ -160,7 +162,9 @@ private:
     int portlists_flags = 0;
     unsigned num_patterns_truncated = 0;  // due to max_pattern_len
 
+#ifdef HAVE_HYPERSCAN
     std::string rule_db_dir;
+#endif
 };
 
 #endif

diff --git a/src/detection/fp_create.cc b/src/detection/fp_create.cc
index 22c69bc156ace1e433f53d2343f46aec4bd499cc..bd1bb81ed0998e2c4ae84b4a9199d0086faf0458 100644 (file)
--- a/src/detection/fp_create.cc
+++ b/src/detection/fp_create.cc
@@ -1606,8 +1606,10 @@ int fpCreateFastPacketDetection(SnortConfig* sc)
 
     if ( !sc->test_mode() or sc->mem_check() )
     {
+#ifdef HAVE_HYPERSCAN
         if ( !fp->get_rule_db_dir().empty() )
             mpse_loaded = fp_deserialize(sc, fp->get_rule_db_dir());
+#endif
 
         unsigned c = compile_mpses(sc, can_build_mt(fp));
         unsigned expected = mpse_count + offload_mpse_count;
@@ -1619,8 +1621,10 @@ int fpCreateFastPacketDetection(SnortConfig* sc)
     bool label = fp_print_port_groups(port_tables);
     fp_print_service_groups(sc->spgmmTable, !label);
 
-    if ( !sc->rule_db_dir.empty() )
-        mpse_dumped = fp_serialize(sc, sc->rule_db_dir);
+#ifdef HAVE_HYPERSCAN
+    if ( !fp->get_rule_db_dir().empty() )
+        mpse_dumped = fp_serialize(sc, fp->get_rule_db_dir());
+#endif
 
     if ( mpse_count )
     {

diff --git a/src/detection/fp_utils.cc b/src/detection/fp_utils.cc
index 37cabefa6150d5c917f1ac3babe9790fb94562ab..b2e60a3e718f0b7899476da274e0d5be8d42c18b 100644 (file)
--- a/src/detection/fp_utils.cc
+++ b/src/detection/fp_utils.cc
@@ -323,7 +323,9 @@ static bool db_dump(const std::string& path, const char* proto, const char* dir,
             uint8_t* db = nullptr;
             size_t len = 0;
 
-            if ( it->group.normal_mpse->serialize(db, len) and db and len > 0 )
+            int result = it->group.normal_mpse->serialize(db, len);
+
+            if ( result == 1 and db and len > 0 )
             {
                 store(file, db, len);
                 free(db);
@@ -331,7 +333,8 @@ static bool db_dump(const std::string& path, const char* proto, const char* dir,
             }
             else
             {
-                ParseWarning(WARN_RULES, "Failed to serialize %s", file.c_str());
+                if ( result != 0 )
+                    ParseWarning(WARN_RULES, "Failed to serialize %s", file.c_str());
                 return false;
             }
         }

diff --git a/src/framework/mpse.h b/src/framework/mpse.h
index 1b3f629b253db71ff0f34b3024e78282bae0b025..d682f0a347a56bc8d58e79ec127d9495eae8f621 100644 (file)
--- a/src/framework/mpse.h
+++ b/src/framework/mpse.h
@@ -38,7 +38,7 @@
 namespace snort
 {
 // this is the current version of the api
-#define SEAPI_VERSION ((BASE_API_VERSION << 16) | 1)
+#define SEAPI_VERSION ((BASE_API_VERSION << 16) | 2)
 
 struct SnortConfig;
 struct MpseApi;
@@ -100,7 +100,7 @@ public:
     virtual int print_info() { return 0; }
     virtual int get_pattern_count() const { return 0; }
 
-    virtual bool serialize(uint8_t*&, size_t&) const { return false; }
+    virtual int serialize(uint8_t*&, size_t&) const { return -1; }
     virtual bool deserialize(const uint8_t*, size_t) { return false; }
     virtual void get_hash(std::string&) { }
 

diff --git a/src/main/modules.cc b/src/main/modules.cc
index 9c48acdcf4dfd9a851eaa6be2012f516223bfc92..a29a5df4932257e0fedf335805d56319e6bdb6d4 100644 (file)
--- a/src/main/modules.cc
+++ b/src/main/modules.cc
@@ -195,8 +195,10 @@ static const Parameter search_engine_params[] =
     { "offload_search_method", Parameter::PT_DYNAMIC, (void*)&get_search_methods, nullptr,
       "set fast pattern offload algorithm - choose available search engine" },
 
+#ifdef HAVE_HYPERSCAN
     { "rule_db_dir", Parameter::PT_STRING, nullptr, nullptr,
-      "deserialize rule databases from given directory" },
+      "directory for reading / writing rule group databases" },
+#endif
 
     { "split_any_any", Parameter::PT_BOOL, nullptr, "true",
       "evaluate any-any rules separately to save memory" },
@@ -294,8 +296,10 @@ bool SearchEngineModule::set(const char*, Value& v, SnortConfig* sc)
     else if ( v.is("detect_raw_tcp") )
         fp->set_stream_insert(v.get_bool());
 
+#ifdef HAVE_HYPERSCAN
     else if ( v.is("rule_db_dir") )
         fp->set_rule_db_dir(v.get_string());
+#endif
 
     else if ( v.is("search_method") )
     {

diff --git a/src/main/snort_config.cc b/src/main/snort_config.cc
index a672dee5ea3da96bab2bf49ef949ae7f51c90d16..09233b601a1f32ce2c72c9322c4936decb3e2df4 100644 (file)
--- a/src/main/snort_config.cc
+++ b/src/main/snort_config.cc
@@ -426,10 +426,6 @@ void SnortConfig::merge(const SnortConfig* cmd_line_conf)
     if (cmd_line_conf->dirty_pig)
         dirty_pig = cmd_line_conf->dirty_pig;
 
-    // --dump-rule-databases
-    if (!cmd_line_conf->rule_db_dir.empty())
-        rule_db_dir = cmd_line_conf->rule_db_dir;
-
     // --id-offset
     id_offset = cmd_line_conf->id_offset;
     // --id-subdir
@@ -679,12 +675,6 @@ void SnortConfig::set_obfuscation_mask(const char* mask)
     obfuscation_net.set(mask);
 }
 
-void SnortConfig::set_rule_db_dir(const char* directory)
-{
-    assert(directory);
-    rule_db_dir = directory;
-}
-
 void SnortConfig::set_gid(const char* args)
 {
     struct group* gr;

diff --git a/src/main/snort_config.h b/src/main/snort_config.h
index 39f89b5b6f04e99bcbf1b2d9e208ddc052d4ff6f..3cbaf2431fafff9f5e6c958ad2012b53637c0b7d 100644 (file)
--- a/src/main/snort_config.h
+++ b/src/main/snort_config.h
@@ -255,7 +255,6 @@ public:
     std::string chroot_dir;        /* -t or config chroot */
     std::string include_path;
     std::string plugin_path;
-    std::string rule_db_dir;
     std::vector<std::string> script_paths;
 
     mode_t file_mask = 0;
@@ -461,7 +460,6 @@ public:
     void set_overlay_trace_config(TraceConfig*);
     void set_include_path(const char*);
     void set_process_all_events(bool);
-    void set_rule_db_dir(const char*);
     void set_show_year(bool);
     void set_tunnel_verdicts(const char*);
     void set_tweaks(const char*);

diff --git a/src/main/snort_module.cc b/src/main/snort_module.cc
index 00607006a080979409375ac37703d1b0c57b9680..dc4204345d977104262e6547e3dabea36e8467d4 100644 (file)
--- a/src/main/snort_module.cc
+++ b/src/main/snort_module.cc
@@ -410,9 +410,6 @@ static const Parameter s_params[] =
     { "--dump-defaults", Parameter::PT_STRING, "(optional)", nullptr,
       "[<module prefix>] output module defaults in Lua format" },
 
-    { "--dump-rule-databases", Parameter::PT_STRING, nullptr, nullptr,
-      "dump rule databases to given directory (hyperscan only)" },
-
     { "--dump-rule-deps", Parameter::PT_IMPLIED, nullptr, nullptr,
       "dump rule dependencies in json format for use by other tools" },
 
@@ -973,11 +970,6 @@ bool SnortModule::set(const char*, Value& v, SnortConfig* sc)
     else if ( is(v, "--dump-defaults") )
         dump_defaults(sc, v.get_string());
 
-    else if ( is(v, "--dump-rule-databases") )
-    {
-        sc->set_rule_db_dir(v.get_string());
-        sc->run_flags |= (RUN_FLAG__TEST | RUN_FLAG__MEM_CHECK);
-    }
     else if ( is(v, "--dump-rule-deps") )
     {
         sc->run_flags |= (RUN_FLAG__DUMP_RULE_DEPS | RUN_FLAG__TEST);

diff --git a/src/search_engines/hyperscan.cc b/src/search_engines/hyperscan.cc
index da57d2225d769effa2b8a7e28734d6c322d26536..d2969f9aa867fcf543690b7bf107c45e6dd3073d 100644 (file)
--- a/src/search_engines/hyperscan.cc
+++ b/src/search_engines/hyperscan.cc
@@ -261,7 +261,7 @@ public:
         unsigned id, unsigned long long from, unsigned long long to,
         unsigned flags, void*);
 
-    bool serialize(uint8_t*&, size_t&) const override;
+    int serialize(uint8_t*&, size_t&) const override;
     bool deserialize(const uint8_t*, size_t) override;
 
     void get_hash(std::string&) override;
@@ -274,6 +274,7 @@ private:
     PatternVector pvector;
 
     hs_database_t* hs_db = nullptr;
+    bool compiled = false;
 
 public:
     static uint64_t instances;
@@ -283,8 +284,13 @@ public:
 uint64_t HyperscanMpse::instances = 0;
 uint64_t HyperscanMpse::patterns = 0;
 
-bool HyperscanMpse::serialize(uint8_t*& buf, size_t& sz) const
-{ return hs_db and (hs_serialize_database(hs_db, (char**)&buf, &sz) == HS_SUCCESS) and buf; }
+int HyperscanMpse::serialize(uint8_t*& buf, size_t& sz) const
+{
+    if ( !compiled )
+        return 0;
+
+    return (hs_db and (hs_serialize_database(hs_db, (char**)&buf, &sz) == HS_SUCCESS) and buf) ? 1 : -1;
+}
 
 bool HyperscanMpse::deserialize(const uint8_t* buf, size_t sz)
 {
@@ -392,6 +398,8 @@ int HyperscanMpse::prep_patterns(SnortConfig* sc)
         return -2;
     }
 
+    compiled = true;
+
     if ( agent )
         user_ctor(sc);
 
@@ -524,7 +532,7 @@ static const MpseApi hs_api =
         PT_SEARCH_ENGINE,
         sizeof(MpseApi),
         SEAPI_VERSION,
-        0,
+        1,
         API_RESERVED,
         API_OPTIONS,
         s_name,