MEDIUM: hlua: Dynamic list of frontend/backend in Lua

When HAproxy is loaded with a lot of frontends/backends (tested with 300k),
it is slow to start and it uses a lot of memory just for indexing backends
in the lua tables.

This patch uses the internal frontend/backend index of HAProxy in place of
lua table.

HAProxy startup is now quicker as each frontend/backend object is created
on demand and not at init.
This has to come with some cost: the execution of Lua will be a little bit
slower.

diff --git a/include/haproxy/hlua-t.h b/include/haproxy/hlua-t.h
index abcb29fadcca863356f63a62faee42d9d29dcd0c..af716d9c3d1daaf272b920994bc5cd41215e57d0 100644 (file)
--- a/include/haproxy/hlua-t.h
+++ b/include/haproxy/hlua-t.h
@@ -53,6 +53,7 @@
 #define CLASS_REGEX        "Regex"
 #define CLASS_STKTABLE     "StickTable"
 #define CLASS_CERTCACHE    "CertCache"
+#define CLASS_PROXY_LIST   "ProxyList"
 
 struct stream;
 
@@ -202,6 +203,15 @@ struct hlua_httpclient {
        struct mt_list by_hlua; /* linked in the current hlua task */
 };
 
+struct hlua_proxy_list {
+       char capabilities;
+};
+
+struct hlua_proxy_list_iterator_context {
+       struct proxy *next;
+       char capabilities;
+};
+
 #else /* USE_LUA */
 /************************ For use when Lua is disabled ********************/
 

diff --git a/include/haproxy/hlua_fcn.h b/include/haproxy/hlua_fcn.h
index dca345a2f22b88ca026e4437dbf750ce591ce97f..b54610df079aa49b77b163d7b808216c3db41ba2 100644 (file)
--- a/include/haproxy/hlua_fcn.h
+++ b/include/haproxy/hlua_fcn.h
@@ -32,7 +32,6 @@ void hlua_class_function(lua_State *L, const char *name, int (*function)(lua_Sta
 void *hlua_checkudata(lua_State *L, int ud, int class_ref);
 int hlua_register_metatable(struct lua_State *L, char *name);
 void hlua_fcn_reg_core_fcn(lua_State *L);
-int hlua_fcn_post_init(lua_State *L);
 int hlua_dump_object(lua_State *L);
 
 #endif /* _HAPROXY_HLUA_FCN_H */

diff --git a/src/hlua.c b/src/hlua.c
index 2ad52d14c2c3284648b1046a2ab1b1c6b874ae65..c669191fbac56af1e2dca6d16ee200a61b22ca3e 100644 (file)
--- a/src/hlua.c
+++ b/src/hlua.c
@@ -11810,8 +11810,6 @@ int hlua_post_init_state(lua_State *L)
                lua_atpanic(L, hlua_panic_ljmp);
        }
 
-       hlua_fcn_post_init(L);
-
        list_for_each_entry(init, &hlua_init_functions[hlua_state_id], l) {
                lua_rawgeti(L, LUA_REGISTRYINDEX, init->function_ref);
 

diff --git a/src/hlua_fcn.c b/src/hlua_fcn.c
index af5fdf4a89585c2bfc6b9c714cf8300d6e67d403..f67ac84fe52178a471d8860637b2559e5d98aa62 100644 (file)
--- a/src/hlua_fcn.c
+++ b/src/hlua_fcn.c
@@ -46,6 +46,7 @@ static int class_server_ref;
 static int class_listener_ref;
 static int class_regex_ref;
 static int class_stktable_ref;
+static int class_proxy_list_ref;
 
 #define STATS_LEN (MAX((int)ST_F_TOTAL_FIELDS, (int)INF_TOTAL_FIELDS))
 
@@ -1384,65 +1385,109 @@ int hlua_proxy_shut_bcksess(lua_State *L)
        return 0;
 }
 
-int hlua_fcn_post_init(lua_State *L)
+static struct hlua_proxy_list *hlua_check_proxy_list(lua_State *L, int ud)
 {
-       struct proxy *px;
+       return hlua_checkudata(L, ud, class_proxy_list_ref);
+}
 
-       /* get core array. */
-       if (lua_getglobal(L, "core") != LUA_TTABLE)
-               lua_error(L);
+/* does nothing and returns 0, only prevents insertions in the
+ * table which represent list of proxies
+ */
+int hlua_listable_proxies_newindex(lua_State *L) {
+       return 0;
+}
 
-       /* Create proxies entry. */
-       lua_pushstring(L, "proxies");
-       lua_newtable(L);
+/* first arg is the table (struct hlua_proxy_list * in metadata)
+ * second arg is the required index
+ */
+int hlua_listable_proxies_index(lua_State *L)
+{
+       struct hlua_proxy_list *hlua_px;
+       const char *name;
+       struct proxy *px;
 
-       /* List all proxies. */
-       for (px = proxies_list; px; px = px->next) {
-               if (px->cap & PR_CAP_INT)
-                       continue;
-               lua_pushstring(L, px->id);
-               hlua_fcn_new_proxy(L, px);
-               lua_settable(L, -3);
+       hlua_px = hlua_check_proxy_list(L, 1);
+       name = luaL_checkstring(L, 2);
+
+       px = NULL;
+       if (hlua_px->capabilities & PR_CAP_FE) {
+               px = proxy_find_by_name(name, PR_CAP_FE, 0);
+       }
+       if (!px && hlua_px->capabilities & PR_CAP_BE) {
+               px = proxy_find_by_name(name, PR_CAP_BE, 0);
+       }
+       if (px == NULL) {
+               lua_pushnil(L);
+               return 1;
        }
 
-       /* push "proxies" in "core" */
-       lua_settable(L, -3);
+       hlua_fcn_new_proxy(L, px);
+       return 1;
+}
 
-       /* Create proxies entry. */
-       lua_pushstring(L, "frontends");
-       lua_newtable(L);
+static inline int hlua_listable_proxies_match(struct proxy *px, char cap) {
+       return ((px->cap & cap) && !(px->cap & (PR_CAP_DEF | PR_CAP_INT)));
+}
 
-       /* List all proxies. */
-       for (px = proxies_list; px; px = px->next) {
-               if (!(px->cap & PR_CAP_FE) || (px->cap & PR_CAP_INT))
-                       continue;
-               lua_pushstring(L, px->id);
-               hlua_fcn_new_proxy(L, px);
-               lua_settable(L, -3);
+/* iterator must return key as string and value as proxy
+ * object, if we reach end of list, it returns nil
+ */
+int hlua_listable_proxies_pairs_iterator(lua_State *L)
+{
+       int context_index;
+       struct hlua_proxy_list_iterator_context *ctx;
+
+       context_index = lua_upvalueindex(1);
+       ctx = lua_touserdata(L, context_index);
+
+       if (ctx->next == NULL) {
+               lua_pushnil(L);
+               return 1;
        }
 
-       /* push "frontends" in "core" */
-       lua_settable(L, -3);
+       lua_pushstring(L, ctx->next->id);
+       hlua_fcn_new_proxy(L, ctx->next);
 
-       /* Create proxies entry. */
-       lua_pushstring(L, "backends");
-       lua_newtable(L);
+       for (ctx->next = ctx->next->next;
+            ctx->next && !hlua_listable_proxies_match(ctx->next, ctx->capabilities);
+            ctx->next = ctx->next->next);
 
-       /* List all proxies. */
-       for (px = proxies_list; px; px = px->next) {
-               if (!(px->cap & PR_CAP_BE) || (px->cap & PR_CAP_INT))
-                       continue;
-               lua_pushstring(L, px->id);
-               hlua_fcn_new_proxy(L, px);
-               lua_settable(L, -3);
-       }
+       return 2;
+}
 
-       /* push "backend" in "core" */
-       lua_settable(L, -3);
+/* init the iterator context, return iterator function
+ * with context as closure. The only argument is a
+ * proxy object.
+ */
+int hlua_listable_proxies_pairs(lua_State *L)
+{
+       struct hlua_proxy_list_iterator_context *ctx;
+       struct hlua_proxy_list *hlua_px;
+
+       hlua_px = hlua_check_proxy_list(L, 1);
 
+       ctx = lua_newuserdata(L, sizeof(*ctx));
+
+       ctx->capabilities = hlua_px->capabilities;
+       for (ctx->next = proxies_list;
+            ctx->next && !hlua_listable_proxies_match(ctx->next, ctx->capabilities);
+            ctx->next = ctx->next->next);
+       lua_pushcclosure(L, hlua_listable_proxies_pairs_iterator, 1);
        return 1;
 }
 
+void hlua_listable_proxies(lua_State *L, char capabilities)
+{
+       struct hlua_proxy_list *list;
+
+       lua_newtable(L);
+       list = lua_newuserdata(L, sizeof(*list));
+       list->capabilities = capabilities;
+       lua_rawseti(L, -2, 0);
+       lua_rawgeti(L, LUA_REGISTRYINDEX, class_proxy_list_ref);
+       lua_setmetatable(L, -2);
+}
+
 /* This Lua function take a string, a list of separators.
  * It tokenize the input string using the list of separators
  * as separator.
@@ -1773,4 +1818,34 @@ void hlua_fcn_reg_core_fcn(lua_State *L)
        lua_settable(L, -3); /* -> META["__index"] = TABLE */
        class_proxy_ref = hlua_register_metatable(L, CLASS_PROXY);
 
+       /* list of proxy objects. Instead of having a static array
+        * of proxies, we use special metamethods that rely on internal
+        * proxies list so that the array is resolved at runtime.
+        *
+        * To emulate the same behavior than Lua array, we implement some
+        * metatable functions:
+        *  - __newindex : prevent the insertion of a new item in the array
+        *  - __index : find a proxy in the list using "name" index
+        *  - __pairs : iterate through available proxies in the list
+        */
+       lua_newtable(L);
+       hlua_class_function(L, "__index", hlua_listable_proxies_index);
+       hlua_class_function(L, "__newindex", hlua_listable_proxies_newindex);
+       hlua_class_function(L, "__pairs", hlua_listable_proxies_pairs);
+       class_proxy_list_ref = hlua_register_metatable(L, CLASS_PROXY_LIST);
+
+       /* Create proxies entry. */
+       lua_pushstring(L, "proxies");
+       hlua_listable_proxies(L, PR_CAP_LISTEN);
+       lua_settable(L, -3);
+
+       /* Create frontends entry. */
+       lua_pushstring(L, "frontends");
+       hlua_listable_proxies(L, PR_CAP_FE);
+       lua_settable(L, -3);
+
+       /* Create backends entry. */
+       lua_pushstring(L, "backends");
+       hlua_listable_proxies(L, PR_CAP_BE);
+       lua_settable(L, -3);
 }