Merge pull request #3126 in SNORT/snort3 from ~SHRARANG/snort3:appid_lua_out_of_mem...

Squashed commit of the following:

commit 6ab78b0fdd275b475a568dc68e6ea4e03ef0383a
Author: Shravan Rangaraju <shrarang@cisco.com>
Date:   Fri Oct 22 16:08:04 2021 -0400

    appid: in packet threads, skip loading of detectors that don't have validate function on reload

diff --git a/src/network_inspectors/appid/lua_detector_api.cc b/src/network_inspectors/appid/lua_detector_api.cc
index b0946b529d47038c0cb376ec5bc8ed2bf408bf30..5d4caad85e570cd6bebf7d0137b715d5dca3f6f9 100644 (file)
--- a/src/network_inspectors/appid/lua_detector_api.cc
+++ b/src/network_inspectors/appid/lua_detector_api.cc
@@ -2880,7 +2880,7 @@ int LuaStateDescriptor::lua_validate(AppIdDiscoveryArgs& args)
     return rc;
 }
 
-static inline void init_lsd(LuaStateDescriptor* lsd, const std::string& detector_name,
+static bool init_lsd(LuaStateDescriptor* lsd, const std::string& detector_name,
     lua_State* L)
 {
     lsd->service_id = APP_ID_UNKNOWN;
@@ -2891,6 +2891,11 @@ static inline void init_lsd(LuaStateDescriptor* lsd, const std::string& detector
     lsd->package_info.name = detector_name;
     lua_pop(L, 1);    // pop client table
     lua_pop(L, 1);    // pop DetectorPackageInfo table
+
+    if (lsd->package_info.validateFunctionName.empty())
+        return false;
+
+    return true;
 }
 
 LuaServiceDetector::LuaServiceDetector(AppIdDiscovery* sdm, const std::string& detector_name,
@@ -2977,9 +2982,9 @@ LuaClientDetector::LuaClientDetector(AppIdDiscovery* cdm, const std::string& det
 
 LuaClientObject::LuaClientObject(const std::string& detector_name,
     const std::string& log_name, bool is_custom, IpProtocol protocol, lua_State* L,
-    OdpContext& odp_ctxt) : LuaObject(odp_ctxt)
+    OdpContext& odp_ctxt, bool& has_validate) : LuaObject(odp_ctxt)
 {
-    init_lsd(&lsd, detector_name, L);
+    has_validate = init_lsd(&lsd, detector_name, L);
 
     if (init(L))
     {

diff --git a/src/network_inspectors/appid/lua_detector_api.h b/src/network_inspectors/appid/lua_detector_api.h
index 3846366420f05e1640f1b4f8c0d88f16fa1e9dba..61d1db4f11c6eca7f92cbbf795d8f2a7d2762dc2 100644 (file)
--- a/src/network_inspectors/appid/lua_detector_api.h
+++ b/src/network_inspectors/appid/lua_detector_api.h
@@ -148,7 +148,7 @@ public:
     ClientDetector* cd;
     LuaClientObject(const std::string& detector_name,
         const std::string& log_name, bool is_custom, IpProtocol protocol, lua_State* L,
-        OdpContext& odp_ctxt);
+        OdpContext& odp_ctxt, bool& has_validate);
     ClientDetector* get_detector() override
     { return cd; }
 };

diff --git a/src/network_inspectors/appid/lua_detector_module.cc b/src/network_inspectors/appid/lua_detector_module.cc
index 15e022087109950d73ef10fe793e2a92301e7e5d..c8d21819f8c32d434355954a85ea11a15ff19d72 100644 (file)
--- a/src/network_inspectors/appid/lua_detector_module.cc
+++ b/src/network_inspectors/appid/lua_detector_module.cc
@@ -325,11 +325,13 @@ static inline uint32_t compute_lua_tracker_size(uint64_t rnaMemory, uint32_t num
 
 // Leaves 1 value (the Detector userdata) at the top of the stack when succeeds
 LuaObject* LuaDetectorManager::create_lua_detector(const char* detector_name,
-    bool is_custom, const char* detector_filename)
+    bool is_custom, const char* detector_filename, bool& has_validate)
 {
     std::string log_name;
     IpProtocol proto = IpProtocol::PROTO_NOT_SET;
 
+    has_validate = false;
+
     Lua::ManageStack mgr(L);
     lua_getfield(L, LUA_REGISTRYINDEX, detector_name);
 
@@ -375,7 +377,7 @@ LuaObject* LuaDetectorManager::create_lua_detector(const char* detector_name,
     lua_getfield(L, -1, "client");
     if ( lua_istable(L, -1) )
     {
-        return new LuaClientObject(detector_name, log_name, is_custom, proto, L, ctxt.get_odp_ctxt());
+        return new LuaClientObject(detector_name, log_name, is_custom, proto, L, ctxt.get_odp_ctxt(), has_validate);
     }
     else
     {
@@ -384,6 +386,7 @@ LuaObject* LuaDetectorManager::create_lua_detector(const char* detector_name,
         lua_getfield(L, -1, "server");
         if ( lua_istable(L, -1) )
         {
+            has_validate = true;
             return new LuaServiceObject(&ctxt.get_odp_ctxt().get_service_disco_mgr(),
                 detector_name, log_name, is_custom, proto, L, ctxt.get_odp_ctxt());
         }
@@ -406,7 +409,7 @@ static int dump(lua_State*, const void* buf,size_t size, void* data)
     return 0;
 }
 
-void LuaDetectorManager::load_detector(char* detector_filename, bool is_custom, bool reload, std::string& buf)
+bool LuaDetectorManager::load_detector(char* detector_filename, bool is_custom, bool reload, std::string& buf)
 {
     if (reload and !buf.empty())
     {
@@ -415,7 +418,7 @@ void LuaDetectorManager::load_detector(char* detector_filename, bool is_custom,
             if (init(L))
                 ErrorMessage("Error - appid: can not load Lua detector, %s\n", lua_tostring(L, -1));
             lua_pop(L, 1);
-            return;
+            return false;
         }
     }
     else
@@ -425,14 +428,14 @@ void LuaDetectorManager::load_detector(char* detector_filename, bool is_custom,
             if (init(L))
                 ErrorMessage("Error - appid: can not load Lua detector, %s\n", lua_tostring(L, -1));
             lua_pop(L, 1);
-            return;
+            return false;
         }
         if (reload and lua_dump(L, dump, &buf))
         {
             if (init(L))
                 ErrorMessage("Error - appid: can not compile Lua detector, %s\n", lua_tostring(L, -1));
             lua_pop(L, 1);
-            return;
+            return false;
         }
     }
 
@@ -462,12 +465,15 @@ void LuaDetectorManager::load_detector(char* detector_filename, bool is_custom,
         ErrorMessage("Error - appid: can not set env of Lua detector %s : %s\n",
             detector_filename, lua_tostring(L, -1));
         lua_pop(L, 1);
-        return;
+        return false;
     }
 
-    LuaObject* lua_object = create_lua_detector(detectorName, is_custom, detector_filename);
+    bool has_validate;
+    LuaObject* lua_object = create_lua_detector(detectorName, is_custom, detector_filename, has_validate);
     if (lua_object)
         allocated_objects.push_front(lua_object);
+
+    return has_validate;
 }
 
 void LuaDetectorManager::load_lua_detectors(const char* path, bool is_custom, bool reload)
@@ -504,12 +510,15 @@ void LuaDetectorManager::load_lua_detectors(const char* path, bool is_custom, bo
             }
             file.close();
 
-            load_detector(globs.gl_pathv[n], is_custom, reload, buf);
+            bool has_validate = load_detector(globs.gl_pathv[n], is_custom, reload, buf);
 
             if (reload)
             {
                 for (auto& lua_detector_mgr : lua_detector_mgr_list)
-                    lua_detector_mgr->load_detector(globs.gl_pathv[n], is_custom, reload, buf);
+                {
+                    if (has_validate)
+                        lua_detector_mgr->load_detector(globs.gl_pathv[n], is_custom, reload, buf);
+                }
                 buf.clear();
             }
             lua_settop(L, 0);

diff --git a/src/network_inspectors/appid/lua_detector_module.h b/src/network_inspectors/appid/lua_detector_module.h
index 25f89ceb1390105be1ca8aeb3502129929af7f7f..615edc633864eba490edfccc83848cedb80a0c12 100644 (file)
--- a/src/network_inspectors/appid/lua_detector_module.h
+++ b/src/network_inspectors/appid/lua_detector_module.h
@@ -72,10 +72,10 @@ private:
     void initialize_lua_detectors(bool reload = false);
     void activate_lua_detectors();
     void list_lua_detectors();
-    void load_detector(char* detector_name, bool is_custom, bool reload, std::string& buf);
+    bool load_detector(char* detector_name, bool is_custom, bool reload, std::string& buf);
     void load_lua_detectors(const char* path, bool is_custom, bool reload = false);
     LuaObject* create_lua_detector(const char* detector_name, bool is_custom,
-        const char* detector_filename);
+        const char* detector_filename, bool& has_validate);
 
     AppIdContext& ctxt;
     std::list<LuaObject*> allocated_objects;