]> git.ipfire.org Git - thirdparty/snort3.git/commitdiff
Pull request #4471: appid: Reading only required lua detectors for regtests
authorUmang Sharma (umasharm) <umasharm@cisco.com>
Thu, 10 Oct 2024 21:04:36 +0000 (21:04 +0000)
committerChris Sherwin (chsherwi) <chsherwi@cisco.com>
Thu, 10 Oct 2024 21:04:36 +0000 (21:04 +0000)
Merge in SNORT/snort3 from ~UMASHARM/snort3:appid_load_time to master

Squashed commit of the following:

commit 722f0b294738e25d3a62ffff1f71cc9673b4b925
Author: Umang Sharma <umasharm@cisco.com>
Date:   Tue Oct 1 04:17:46 2024 -0400

    appid: Reading and loading only required lua detectors for regtests

src/network_inspectors/appid/appid_config.cc
src/network_inspectors/appid/appid_config.h
src/network_inspectors/appid/appid_module.cc
src/network_inspectors/appid/lua_detector_module.cc
src/network_inspectors/appid/lua_detector_module.h

index 305d2e6bd5809b0cdcbf37a97bac4cb12ae0561f..a636516838046243acbab448fc28f6f2512cd1c6 100644 (file)
@@ -60,6 +60,9 @@ uint32_t OdpContext::next_version = 0;
 AppIdConfig::~AppIdConfig()
 {
     snort_free((void*)app_detector_dir);
+    #ifdef REG_TEST
+    snort_free((void*)required_lua_detectors);
+    #endif
 }
 
 void AppIdConfig::map_app_names_to_snort_ids(SnortConfig& sc)
index fefed2635024c0c9d9b9271d2cd6bce3ac12da05..80aa3f8f524be3772b9ca1adbfcf89db1c835b13 100644 (file)
@@ -100,6 +100,7 @@ public:
     bool log_eve_process_client_mappings = false;
     bool log_alpn_service_mappings = false;
     bool log_memory_and_pattern_count = false;
+    const char* required_lua_detectors = nullptr;
 #endif
     bool log_stats = false;
     uint32_t app_stats_period = 300;
index 371d0361bb1cdba4abe96fb025065e894eaa9074..8656484a1d8e03f9b09272e6ef1246ce467e911c 100644 (file)
@@ -78,6 +78,8 @@ static const Parameter s_params[] =
       "enable logging of alpn service mappings" },
     { "log_memory_and_pattern_count", Parameter::PT_BOOL, nullptr, "false",
       "enable logging of memory usage and pattern counts" },
+    { "required_lua_detectors", Parameter::PT_STRING, nullptr, nullptr,
+      "lists down the required lua detectors only" },
 #endif
     { "memcap", Parameter::PT_INT, "1024:maxSZ", "1048576",
       "max size of the service cache before we start pruning the cache" },
@@ -637,6 +639,8 @@ bool AppIdModule::set(const char*, Value& v, SnortConfig*)
         config->log_alpn_service_mappings = v.get_bool();
     else if (v.is("log_memory_and_pattern_count") )
         config->log_memory_and_pattern_count = v.get_bool();
+    else if (v.is("required_lua_detectors") )
+        config->required_lua_detectors = snort_strdup(v.get_string());
     else
 #endif
     if ( v.is("memcap") )
index bae975858df64e824e423922c22cbce9c1f1b019..f321d6308780336c9288d4ae05240047da5de50c 100644 (file)
@@ -516,6 +516,53 @@ void LuaDetectorManager::activate_lua_detectors(const SnortConfig* sc)
         ++lo;
     }
 }
+void ControlLuaDetectorManager::process_detector_file(char* detector_file_path, bool is_custom)
+{
+    ifstream file(detector_file_path, ios::ate);
+    int size = file.tellg();
+    //do not load empty lua files
+    if (size < MIN_LUA_DETECTOR_FILE_SIZE)
+    {
+        file.close();
+        return;
+    }
+    if (size > MAX_LUA_DETECTOR_FILE_SIZE)
+    {
+        appid_log(nullptr, TRACE_ERROR_LEVEL, "Error - appid: can not load Lua detector %s : \
+            size exceeded maximum limit\n", detector_file_path);
+        file.close();
+        return;
+    }
+    file.close();
+    // In the packet threads, we do not need to load Lua detectors that don't have validate
+    // function such as payload_group_*, ssl_group_*, etc. That's because the patterns they
+    // register are stored in global tables only in control thread. In packet threads, they
+    // do nothing. Skipping loading of these detectors in packet threads saves on the memory
+    // used by LuaJIT.
+
+    // Because the code flow for loading Lua detectors is different for initialization vs
+    // reload, the LuaJIT memory saving is achieved differently in these two cases.
+
+    // During initialization, load_lua_detectors() gets called for all the threads - first
+    // for the control thread and then for the packet threads. Control thread stores the
+    // detectors that have validate in lua_detectors_w_validate. Packet thread loads a
+    // detector in load_detector() only if it finds the detector in lua_detectors_w_validate.
+
+    // During reload, load_lua_detectors() gets called only for control thread. This
+    // function loads detectors for all the packet threads too during reload. It skips
+    // loading detectors that don't have validate for packet threads.
+
+    string buf;
+    bool has_validate = load_detector(detector_file_path, is_custom, buf);
+
+    for (auto& lua_detector_mgr : lua_detector_mgr_list)
+    {
+        if (has_validate)
+            lua_detector_mgr->load_detector(detector_file_path, is_custom, buf);
+    }
+    lua_settop(L, 0);
+}
+
 
 void ControlLuaDetectorManager::load_lua_detectors(const char* path, bool is_custom)
 {
@@ -523,6 +570,33 @@ void ControlLuaDetectorManager::load_lua_detectors(const char* path, bool is_cus
     snprintf(pattern, sizeof(pattern), "%s/*", path);
     glob_t globs;
 
+    #ifdef REG_TEST
+    if (!is_custom)
+    {
+        const char* required_lua_detectors = ctxt.config.required_lua_detectors;
+        if (required_lua_detectors)
+        {
+            std::ifstream file(required_lua_detectors);
+            if (!file)
+            {
+                appid_log(nullptr, TRACE_ERROR_LEVEL, "Error - appid: can not open required lua detectors list file %s\n", required_lua_detectors);
+            }
+            else
+            {
+                std::string line;
+                int lua_top = lua_gettop(L);
+                if (lua_top)
+                    appid_log(nullptr, TRACE_WARNING_LEVEL, "appid: leak of %d lua stack elements before detector load\n", lua_top);
+                while (std::getline(file, line))
+                {
+                    process_detector_file(const_cast<char*>(line.c_str()), is_custom);
+                }
+            }
+            return;
+        }
+    }
+    #endif
+
     memset(&globs, 0, sizeof(globs));
     int rval = glob(pattern, 0, nullptr, &globs);
     if (rval == 0 )
@@ -533,49 +607,7 @@ void ControlLuaDetectorManager::load_lua_detectors(const char* path, bool is_cus
 
         for (unsigned n = 0; n < globs.gl_pathc; n++)
         {
-            ifstream file(globs.gl_pathv[n], ios::ate);
-            int size = file.tellg();
-            //do not load empty lua files
-            if (size < MIN_LUA_DETECTOR_FILE_SIZE)
-            {
-                file.close();
-                continue;
-            }
-            if (size > MAX_LUA_DETECTOR_FILE_SIZE)
-            {
-                appid_log(nullptr, TRACE_ERROR_LEVEL, "Error - appid: can not load Lua detector %s : \
-                    size exceeded maximum limit\n", globs.gl_pathv[n]);
-                file.close();
-                continue;
-            }
-            file.close();
-
-            // In the packet threads, we do not need to load Lua detectors that don't have validate
-            // function such as payload_group_*, ssl_group_*, etc. That's because the patterns they
-            // register are stored in global tables only in control thread. In packet threads, they
-            // do nothing. Skipping loading of these detectors in packet threads saves on the memory
-            // used by LuaJIT.
-
-            // Because the code flow for loading Lua detectors is different for initialization vs
-            // reload, the LuaJIT memory saving is achieved differently in these two cases.
-
-            // During initialization, load_lua_detectors() gets called for all the threads - first
-            // for the control thread and then for the packet threads. Control thread stores the
-            // detectors that have validate in lua_detectors_w_validate. Packet thread loads a
-            // detector in load_detector() only if it finds the detector in lua_detectors_w_validate.
-
-            // During reload, load_lua_detectors() gets called only for control thread. This
-            // function loads detectors for all the packet threads too during reload. It skips
-            // loading detectors that don't have validate for packet threads.
-            string buf;
-            bool has_validate = load_detector(globs.gl_pathv[n], is_custom, buf);
-
-            for (auto& lua_detector_mgr : lua_detector_mgr_list)
-            {
-                if (has_validate)
-                    lua_detector_mgr->load_detector(globs.gl_pathv[n], is_custom, buf);
-            }
-            lua_settop(L, 0);
+            process_detector_file(globs.gl_pathv[n], is_custom);
         }
 
         globfree(&globs);
index 2ce5118cd5999cbde369cd57c2c3c2b05919aec7..27dc62bb3d198731c47ad0d9b3b95f52d4305f85 100644 (file)
@@ -121,6 +121,7 @@ private:
 
     void initialize_lua_detectors();
     void load_lua_detectors(const char* path, bool is_custom);
+    void process_detector_file(char* detector_file_path, bool is_custom);
     void list_lua_detectors() override;
 
     bool ignore_chp_cleanup = false;