From: Umang Sharma (umasharm) Date: Thu, 10 Oct 2024 21:04:36 +0000 (+0000) Subject: Pull request #4471: appid: Reading only required lua detectors for regtests X-Git-Tag: 3.4.0.0~6 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e269fc15c2a47ad9b02a99d91d699468bb66358f;p=thirdparty%2Fsnort3.git Pull request #4471: appid: Reading only required lua detectors for regtests Merge in SNORT/snort3 from ~UMASHARM/snort3:appid_load_time to master Squashed commit of the following: commit 722f0b294738e25d3a62ffff1f71cc9673b4b925 Author: Umang Sharma Date: Tue Oct 1 04:17:46 2024 -0400 appid: Reading and loading only required lua detectors for regtests --- diff --git a/src/network_inspectors/appid/appid_config.cc b/src/network_inspectors/appid/appid_config.cc index 305d2e6bd..a63651683 100644 --- a/src/network_inspectors/appid/appid_config.cc +++ b/src/network_inspectors/appid/appid_config.cc @@ -60,6 +60,9 @@ uint32_t OdpContext::next_version = 0; AppIdConfig::~AppIdConfig() { snort_free((void*)app_detector_dir); + #ifdef REG_TEST + snort_free((void*)required_lua_detectors); + #endif } void AppIdConfig::map_app_names_to_snort_ids(SnortConfig& sc) diff --git a/src/network_inspectors/appid/appid_config.h b/src/network_inspectors/appid/appid_config.h index fefed2635..80aa3f8f5 100644 --- a/src/network_inspectors/appid/appid_config.h +++ b/src/network_inspectors/appid/appid_config.h @@ -100,6 +100,7 @@ public: bool log_eve_process_client_mappings = false; bool log_alpn_service_mappings = false; bool log_memory_and_pattern_count = false; + const char* required_lua_detectors = nullptr; #endif bool log_stats = false; uint32_t app_stats_period = 300; diff --git a/src/network_inspectors/appid/appid_module.cc b/src/network_inspectors/appid/appid_module.cc index 371d0361b..8656484a1 100644 --- a/src/network_inspectors/appid/appid_module.cc +++ b/src/network_inspectors/appid/appid_module.cc @@ -78,6 +78,8 @@ static const Parameter s_params[] = "enable logging of alpn service mappings" }, { "log_memory_and_pattern_count", Parameter::PT_BOOL, nullptr, "false", "enable logging of memory usage and pattern counts" }, + { "required_lua_detectors", Parameter::PT_STRING, nullptr, nullptr, + "lists down the required lua detectors only" }, #endif { "memcap", Parameter::PT_INT, "1024:maxSZ", "1048576", "max size of the service cache before we start pruning the cache" }, @@ -637,6 +639,8 @@ bool AppIdModule::set(const char*, Value& v, SnortConfig*) config->log_alpn_service_mappings = v.get_bool(); else if (v.is("log_memory_and_pattern_count") ) config->log_memory_and_pattern_count = v.get_bool(); + else if (v.is("required_lua_detectors") ) + config->required_lua_detectors = snort_strdup(v.get_string()); else #endif if ( v.is("memcap") ) diff --git a/src/network_inspectors/appid/lua_detector_module.cc b/src/network_inspectors/appid/lua_detector_module.cc index bae975858..f321d6308 100644 --- a/src/network_inspectors/appid/lua_detector_module.cc +++ b/src/network_inspectors/appid/lua_detector_module.cc @@ -516,6 +516,53 @@ void LuaDetectorManager::activate_lua_detectors(const SnortConfig* sc) ++lo; } } +void ControlLuaDetectorManager::process_detector_file(char* detector_file_path, bool is_custom) +{ + ifstream file(detector_file_path, ios::ate); + int size = file.tellg(); + //do not load empty lua files + if (size < MIN_LUA_DETECTOR_FILE_SIZE) + { + file.close(); + return; + } + if (size > MAX_LUA_DETECTOR_FILE_SIZE) + { + appid_log(nullptr, TRACE_ERROR_LEVEL, "Error - appid: can not load Lua detector %s : \ + size exceeded maximum limit\n", detector_file_path); + file.close(); + return; + } + file.close(); + // In the packet threads, we do not need to load Lua detectors that don't have validate + // function such as payload_group_*, ssl_group_*, etc. That's because the patterns they + // register are stored in global tables only in control thread. In packet threads, they + // do nothing. Skipping loading of these detectors in packet threads saves on the memory + // used by LuaJIT. + + // Because the code flow for loading Lua detectors is different for initialization vs + // reload, the LuaJIT memory saving is achieved differently in these two cases. + + // During initialization, load_lua_detectors() gets called for all the threads - first + // for the control thread and then for the packet threads. Control thread stores the + // detectors that have validate in lua_detectors_w_validate. Packet thread loads a + // detector in load_detector() only if it finds the detector in lua_detectors_w_validate. + + // During reload, load_lua_detectors() gets called only for control thread. This + // function loads detectors for all the packet threads too during reload. It skips + // loading detectors that don't have validate for packet threads. + + string buf; + bool has_validate = load_detector(detector_file_path, is_custom, buf); + + for (auto& lua_detector_mgr : lua_detector_mgr_list) + { + if (has_validate) + lua_detector_mgr->load_detector(detector_file_path, is_custom, buf); + } + lua_settop(L, 0); +} + void ControlLuaDetectorManager::load_lua_detectors(const char* path, bool is_custom) { @@ -523,6 +570,33 @@ void ControlLuaDetectorManager::load_lua_detectors(const char* path, bool is_cus snprintf(pattern, sizeof(pattern), "%s/*", path); glob_t globs; + #ifdef REG_TEST + if (!is_custom) + { + const char* required_lua_detectors = ctxt.config.required_lua_detectors; + if (required_lua_detectors) + { + std::ifstream file(required_lua_detectors); + if (!file) + { + appid_log(nullptr, TRACE_ERROR_LEVEL, "Error - appid: can not open required lua detectors list file %s\n", required_lua_detectors); + } + else + { + std::string line; + int lua_top = lua_gettop(L); + if (lua_top) + appid_log(nullptr, TRACE_WARNING_LEVEL, "appid: leak of %d lua stack elements before detector load\n", lua_top); + while (std::getline(file, line)) + { + process_detector_file(const_cast(line.c_str()), is_custom); + } + } + return; + } + } + #endif + memset(&globs, 0, sizeof(globs)); int rval = glob(pattern, 0, nullptr, &globs); if (rval == 0 ) @@ -533,49 +607,7 @@ void ControlLuaDetectorManager::load_lua_detectors(const char* path, bool is_cus for (unsigned n = 0; n < globs.gl_pathc; n++) { - ifstream file(globs.gl_pathv[n], ios::ate); - int size = file.tellg(); - //do not load empty lua files - if (size < MIN_LUA_DETECTOR_FILE_SIZE) - { - file.close(); - continue; - } - if (size > MAX_LUA_DETECTOR_FILE_SIZE) - { - appid_log(nullptr, TRACE_ERROR_LEVEL, "Error - appid: can not load Lua detector %s : \ - size exceeded maximum limit\n", globs.gl_pathv[n]); - file.close(); - continue; - } - file.close(); - - // In the packet threads, we do not need to load Lua detectors that don't have validate - // function such as payload_group_*, ssl_group_*, etc. That's because the patterns they - // register are stored in global tables only in control thread. In packet threads, they - // do nothing. Skipping loading of these detectors in packet threads saves on the memory - // used by LuaJIT. - - // Because the code flow for loading Lua detectors is different for initialization vs - // reload, the LuaJIT memory saving is achieved differently in these two cases. - - // During initialization, load_lua_detectors() gets called for all the threads - first - // for the control thread and then for the packet threads. Control thread stores the - // detectors that have validate in lua_detectors_w_validate. Packet thread loads a - // detector in load_detector() only if it finds the detector in lua_detectors_w_validate. - - // During reload, load_lua_detectors() gets called only for control thread. This - // function loads detectors for all the packet threads too during reload. It skips - // loading detectors that don't have validate for packet threads. - string buf; - bool has_validate = load_detector(globs.gl_pathv[n], is_custom, buf); - - for (auto& lua_detector_mgr : lua_detector_mgr_list) - { - if (has_validate) - lua_detector_mgr->load_detector(globs.gl_pathv[n], is_custom, buf); - } - lua_settop(L, 0); + process_detector_file(globs.gl_pathv[n], is_custom); } globfree(&globs); diff --git a/src/network_inspectors/appid/lua_detector_module.h b/src/network_inspectors/appid/lua_detector_module.h index 2ce5118cd..27dc62bb3 100644 --- a/src/network_inspectors/appid/lua_detector_module.h +++ b/src/network_inspectors/appid/lua_detector_module.h @@ -121,6 +121,7 @@ private: void initialize_lua_detectors(); void load_lua_detectors(const char* path, bool is_custom); + void process_detector_file(char* detector_file_path, bool is_custom); void list_lua_detectors() override; bool ignore_chp_cleanup = false;