From: Josef 'Jeff' Sipek <jeff.sipek@open-xchange.com>
Date: Thu, 25 Feb 2021 15:22:28 +0000 (-0500)
Subject: lib-dict: Expose dict and dict transactions to lua scripts
X-Git-Tag: 2.3.15~170
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=1e67b93cc9f469bb144fc890a6cb9b9b9cd20811;p=thirdparty%2Fdovecot%2Fcore.git

lib-dict: Expose dict and dict transactions to lua scripts
---

diff --git a/src/lib-dict/Makefile.am b/src/lib-dict/Makefile.am
index 6d713e3691..d63522c0e5 100644
--- a/src/lib-dict/Makefile.am
+++ b/src/lib-dict/Makefile.am
@@ -1,4 +1,6 @@
-noinst_LTLIBRARIES = libdict.la
+noinst_LTLIBRARIES = \
+	libdict.la \
+	libdict_lua.la
 
 AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
@@ -24,6 +26,30 @@ headers = \
 	dict-private.h \
 	dict-transaction-memory.h
 
+if HAVE_LUA
+libdict_lua_la_SOURCES =
+
+# Internally, the dict methods yield via lua_yieldk() as implemented in Lua
+# 5.3 and newer.
+if DLUA_WITH_YIELDS
+libdict_lua_la_SOURCES += \
+	dict-lua.c \
+	dict-iter-lua.c \
+	dict-txn-lua.c
+libdict_lua_la_CPPFLAGS = \
+	$(AM_CPPFLAGS) \
+	$(LUA_CFLAGS) \
+	-I$(top_srcdir)/src/lib-lua
+libdict_lua_la_LIBADD =
+libdict_lua_la_DEPENDENCIES = \
+	libdict.la
+endif
+
+headers += \
+	dict-lua.h \
+	dict-lua-private.h
+endif
+
 pkginc_libdir=$(pkgincludedir)
 pkginc_lib_HEADERS = $(headers)
 
diff --git a/src/lib-dict/dict-iter-lua.c b/src/lib-dict/dict-iter-lua.c
new file mode 100644
index 0000000000..ee6b3a98bd
--- /dev/null
+++ b/src/lib-dict/dict-iter-lua.c
@@ -0,0 +1,182 @@
+/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "dict.h"
+#include "dlua-script-private.h"
+#include "dict-lua-private.h"
+#include "dlua-wrapper.h"
+
+struct lua_dict_iter {
+	pool_t pool;
+	struct dict_iterate_context *iter;
+	ARRAY(int) refs;
+	int error_ref;
+
+	lua_State *L;
+};
+
+static void lua_dict_iter_unref(struct lua_dict_iter *iter)
+{
+	const char *error;
+
+	/* deinit iteration if it hasn't been done yet */
+	if (dict_iterate_deinit(&iter->iter, &error) < 0) {
+		e_error(dlua_script_from_state(iter->L)->event,
+			"Dict iteration failed: %s", error);
+	}
+
+	pool_unref(&iter->pool);
+}
+
+DLUA_WRAP_C_DATA(dict_iter, struct lua_dict_iter, lua_dict_iter_unref, NULL);
+
+static int lua_dict_iterate_step(lua_State *L);
+
+/* resume after a yield */
+static int lua_dict_iterate_step_continue(lua_State *L,
+					  int status ATTR_UNUSED,
+					  lua_KContext ctx ATTR_UNUSED)
+{
+	return lua_dict_iterate_step(L);
+}
+
+static void lua_dict_iterate_more(struct lua_dict_iter *iter);
+
+/*
+ * Iteration step function
+ *
+ * Takes two args (a userdata state, and previous value) and returns the
+ * next value.
+ */
+static int lua_dict_iterate_step(lua_State *L)
+{
+	struct lua_dict_iter *iter;
+	const int *refs;
+	unsigned nrefs;
+
+	DLUA_REQUIRE_ARGS(L, 2);
+
+	iter = xlua_dict_iter_getptr(L, -2, NULL);
+
+	lua_dict_iterate_more(iter);
+
+	if (iter->iter != NULL) {
+		/* iteration didn't end yet - yield */
+		return lua_dict_iterate_step_continue(L,
+			      lua_yieldk(L, 0, 0, lua_dict_iterate_step_continue), 0);
+	}
+
+	/* dict iteration ended - return first key-value pair */
+	refs = array_get(&iter->refs, &nrefs);
+	i_assert(nrefs % 2 == 0);
+
+	if (nrefs == 0) {
+		/* iteration is over - clean up */
+		pool_unref(&iter->pool);
+
+		if (iter->error_ref != 0) {
+			/* dict iteration generated an error - raise it now */
+			lua_rawgeti(L, LUA_REGISTRYINDEX, iter->error_ref);
+			luaL_unref(L, LUA_REGISTRYINDEX, iter->error_ref);
+			return lua_error(L);
+		}
+
+		return 0; /* return nil */
+	}
+
+	/* get the key & value from the registry */
+	lua_rawgeti(L, LUA_REGISTRYINDEX, refs[0]);
+	lua_rawgeti(L, LUA_REGISTRYINDEX, refs[1]);
+	luaL_unref(L, LUA_REGISTRYINDEX, refs[0]);
+	luaL_unref(L, LUA_REGISTRYINDEX, refs[1]);
+
+	array_delete(&iter->refs, 0, 2);
+
+	return 2;
+}
+
+static void lua_dict_iterate_more(struct lua_dict_iter *iter)
+{
+	const char *key, *const *values;
+	lua_State *L = iter->L;
+	const char *error;
+
+	if (iter->iter == NULL)
+		return; /* done iterating the dict */
+
+	while (dict_iterate_values(iter->iter, &key, &values)) {
+		int ref;
+
+		/* stash key */
+		lua_pushstring(L, key);
+		ref = luaL_ref(L, LUA_REGISTRYINDEX);
+		array_push_back(&iter->refs, &ref);
+
+		/* stash values */
+		lua_newtable(L);
+		for (unsigned int i = 0; values[i] != NULL; i++) {
+			lua_pushstring(L, values[i]);
+			lua_seti(L, -2, i + 1);
+		}
+		ref = luaL_ref(L, LUA_REGISTRYINDEX);
+		array_push_back(&iter->refs, &ref);
+	}
+
+	if (dict_iterate_has_more(iter->iter))
+		return;
+
+	if (dict_iterate_deinit(&iter->iter, &error) < 0) {
+		lua_pushstring(L, error);
+		iter->error_ref = luaL_ref(L, LUA_REGISTRYINDEX);
+	}
+}
+
+/* dict iter callback */
+static void lua_dict_iterate_callback(struct lua_dict_iter *iter)
+{
+	dlua_pcall_yieldable_resume(iter->L, 1);
+}
+
+/*
+ * Iterate a dict at key [-3,+2,e]
+ *
+ * Args:
+ *   1) userdata: sturct dict *dict
+ *   2) string: key
+ *   3) integer: flags
+ *
+ * Returns:
+ *   Returns a iteration step function and dict iter userdata.
+ */
+int lua_dict_iterate(lua_State *L)
+{
+	enum dict_iterate_flags flags;
+	struct lua_dict_iter *iter;
+	struct dict *dict;
+	const char *path;
+	pool_t pool;
+
+	DLUA_REQUIRE_ARGS(L, 3);
+
+	dict = dlua_check_dict(L, -3);
+	path = luaL_checkstring(L, -2);
+	flags = luaL_checkinteger(L, -1);
+
+	/* set up iteration */
+	pool = pool_alloconly_create("lua dict iter", 128);
+	iter = p_new(pool, struct lua_dict_iter, 1);
+	iter->pool = pool;
+	iter->iter = dict_iterate_init(dict, path, flags |
+				      DICT_ITERATE_FLAG_ASYNC);
+	p_array_init(&iter->refs, iter->pool, 32);
+	iter->L = L;
+
+	dict_iterate_set_async_callback(iter->iter,
+					lua_dict_iterate_callback, iter);
+
+	/* push return values: func, state */
+	lua_pushcfunction(L, lua_dict_iterate_step);
+	xlua_pushdict_iter(L, iter, FALSE);
+	return 2;
+}
diff --git a/src/lib-dict/dict-lua-private.h b/src/lib-dict/dict-lua-private.h
new file mode 100644
index 0000000000..f9f9943981
--- /dev/null
+++ b/src/lib-dict/dict-lua-private.h
@@ -0,0 +1,9 @@
+#ifndef DICT_LUA_PRIVATE_H
+#define DICT_LUA_PRIVATE_H
+
+#include "dict-lua.h"
+
+int lua_dict_iterate(lua_State *l);
+int lua_dict_transaction_begin(lua_State *l);
+
+#endif
diff --git a/src/lib-dict/dict-lua.c b/src/lib-dict/dict-lua.c
new file mode 100644
index 0000000000..66baca7cc9
--- /dev/null
+++ b/src/lib-dict/dict-lua.c
@@ -0,0 +1,94 @@
+/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "dict.h"
+#include "dlua-script-private.h"
+#include "dict-lua-private.h"
+#include "dlua-wrapper.h"
+
+static int lua_dict_lookup(lua_State *);
+
+static luaL_Reg lua_dict_methods[] = {
+	{ "lookup", lua_dict_lookup },
+	{ "iterate", lua_dict_iterate },
+	{ "transaction_begin", lua_dict_transaction_begin },
+	{ NULL, NULL },
+};
+
+/* no actual ref counting */
+static void lua_dict_unref(struct dict *dict ATTR_UNUSED)
+{
+}
+
+DLUA_WRAP_C_DATA(dict, struct dict, lua_dict_unref, lua_dict_methods);
+
+static int lua_dict_async_continue(lua_State *L,
+				   int status ATTR_UNUSED,
+				   lua_KContext ctx ATTR_UNUSED)
+{
+	/*
+	 * lua_dict_*_callback() already pushed the result table or error
+	 * string.  We simply need to return/error out.
+	 */
+
+	if (lua_istable(L, -1))
+		return 1;
+	else
+		return lua_error(L);
+}
+
+static void lua_dict_lookup_callback(const struct dict_lookup_result *result,
+				     lua_State *L)
+{
+	if (result->ret < 0) {
+		lua_pushstring(L, result->error);
+	} else {
+		unsigned int i;
+
+		lua_newtable(L);
+
+		for (i = 0; i < str_array_length(result->values); i++) {
+			lua_pushstring(L, result->values[i]);
+			lua_seti(L, -2, i + 1);
+		}
+	}
+
+	dlua_pcall_yieldable_resume(L, 1);
+}
+
+/*
+ * Lookup a key in dict [-2,+1,e]
+ *
+ * Args:
+ *   1) userdata: struct dict *dict
+ *   2) string: key
+ *
+ * Returns:
+ *   If key is found, returns a table with values.  If key is not found,
+ *   returns nil.
+ */
+static int lua_dict_lookup(lua_State *L)
+{
+	struct dict *dict;
+	const char *key;
+
+	DLUA_REQUIRE_ARGS(L, 2);
+
+	dict = xlua_dict_getptr(L, -2, NULL);
+	key = luaL_checkstring(L, -1);
+
+	dict_lookup_async(dict, key, lua_dict_lookup_callback, L);
+
+	return lua_dict_async_continue(L,
+		lua_yieldk(L, 0, 0, lua_dict_async_continue), 0);
+}
+
+void dlua_push_dict(lua_State *L, struct dict *dict)
+{
+	xlua_pushdict(L, dict, FALSE);
+}
+
+struct dict *dlua_check_dict(lua_State *L, int idx)
+{
+	return xlua_dict_getptr(L, idx, NULL);
+}
diff --git a/src/lib-dict/dict-lua.h b/src/lib-dict/dict-lua.h
new file mode 100644
index 0000000000..186f4e9974
--- /dev/null
+++ b/src/lib-dict/dict-lua.h
@@ -0,0 +1,15 @@
+#ifndef DICT_LUA_H
+#define DICT_LUA_H
+
+#ifdef DLUA_WITH_YIELDS
+/*
+ * Internally, the dict methods yield via lua_yieldk() as implemented in Lua
+ * 5.3 and newer.
+ */
+
+void dlua_push_dict(lua_State *L, struct dict *dict);
+struct dict *dlua_check_dict(lua_State *L, int idx);
+
+#endif
+
+#endif
diff --git a/src/lib-dict/dict-txn-lua.c b/src/lib-dict/dict-txn-lua.c
new file mode 100644
index 0000000000..1e2b222a64
--- /dev/null
+++ b/src/lib-dict/dict-txn-lua.c
@@ -0,0 +1,195 @@
+/* Copyright (c) 2021 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "dict.h"
+#include "dlua-script-private.h"
+#include "dict-lua-private.h"
+#include "dlua-wrapper.h"
+
+struct lua_dict_txn {
+	pool_t pool;
+	struct dict_transaction_context *txn;
+	enum {
+		STATE_OPEN,
+		STATE_COMMITTED,
+		STATE_ABORTED,
+	} state;
+
+	lua_State *L;
+};
+
+static int lua_dict_transaction_rollback(lua_State *L);
+static int lua_dict_transaction_commit(lua_State *L);
+static int lua_dict_set(lua_State *L);
+
+static luaL_Reg lua_dict_txn_methods[] = {
+	{ "rollback", lua_dict_transaction_rollback },
+	{ "commit", lua_dict_transaction_commit },
+	{ "set", lua_dict_set },
+	{ NULL, NULL },
+};
+
+static void sanity_check_txn(lua_State *L, struct lua_dict_txn *txn)
+{
+	switch (txn->state) {
+	case STATE_OPEN:
+		return;
+	case STATE_COMMITTED:
+		luaL_error(L, "dict transaction already committed");
+		return;
+	case STATE_ABORTED:
+		luaL_error(L, "dict transaction already aborted");
+		return;
+	}
+
+	i_unreached();
+}
+
+/* no actual ref counting, but we use it for clean up */
+static void lua_dict_txn_unref(struct lua_dict_txn *txn)
+{
+	/* rollback any transactions that were forgotten about */
+	dict_transaction_rollback(&txn->txn);
+
+	pool_unref(&txn->pool);
+}
+
+DLUA_WRAP_C_DATA(dict_txn, struct lua_dict_txn, lua_dict_txn_unref,
+		 lua_dict_txn_methods);
+
+/*
+ * Abort a transaction [-1,+0,e]
+ *
+ * Args:
+ *   1) userdata: struct lua_dict_txn *
+ */
+static int lua_dict_transaction_rollback(lua_State *L)
+{
+	struct lua_dict_txn *txn;
+
+	DLUA_REQUIRE_ARGS(L, 1);
+
+	txn = xlua_dict_txn_getptr(L, -1, NULL);
+	sanity_check_txn(L, txn);
+
+	txn->state = STATE_ABORTED;
+	dict_transaction_rollback(&txn->txn);
+
+	return 0;
+}
+
+static int lua_dict_transaction_commit_continue(lua_State *L,
+						int status ATTR_UNUSED,
+						lua_KContext ctx ATTR_UNUSED)
+{
+	if (!lua_isnil(L, -1))
+		lua_error(L); /* commit failed */
+
+	lua_pop(L, 1); /* pop the nil indicating the lack of error */
+
+	return 0;
+}
+
+static void
+lua_dict_transaction_commit_callback(const struct dict_commit_result *result,
+				     struct lua_dict_txn *txn)
+{
+
+	switch (result->ret) {
+	case DICT_COMMIT_RET_OK:
+		/* push a nil to indicate everything is ok */
+		lua_pushnil(txn->L);
+		break;
+	case DICT_COMMIT_RET_NOTFOUND:
+		/* we don't expose dict_atomic_inc(), so this should never happen */
+		i_unreached();
+	case DICT_COMMIT_RET_FAILED:
+	case DICT_COMMIT_RET_WRITE_UNCERTAIN:
+		/* push the error we'll raise when we resume */
+		i_assert(result->error != NULL);
+		lua_pushfstring(txn->L, "dict transaction commit failed: %s",
+				result->error);
+		break;
+	}
+
+	dlua_pcall_yieldable_resume(txn->L, 1);
+}
+
+/*
+ * Commit a transaction [-1,+0,e]
+ *
+ * Args:
+ *   1) userdata: struct lua_dict_txn *
+ */
+static int lua_dict_transaction_commit(lua_State *L)
+{
+	struct lua_dict_txn *txn;
+
+	DLUA_REQUIRE_ARGS(L, 1);
+
+	txn = xlua_dict_txn_getptr(L, -1, NULL);
+	sanity_check_txn(L, txn);
+
+	txn->state = STATE_COMMITTED;
+	dict_transaction_commit_async(&txn->txn,
+		lua_dict_transaction_commit_callback, txn);
+
+	return lua_dict_transaction_commit_continue(L,
+		lua_yieldk(L, 0, 0, lua_dict_transaction_commit_continue), 0);
+}
+
+/*
+ * Set key to value [-3,+0,e]
+ *
+ * Args:
+ *   1) userdata: struct lua_dict_txn *
+ *   2) string: key
+ *   3) string: value
+ */
+static int lua_dict_set(lua_State *L)
+{
+	struct lua_dict_txn *txn;
+	const char *key, *value;
+
+	DLUA_REQUIRE_ARGS(L, 3);
+
+	txn = xlua_dict_txn_getptr(L, -3, NULL);
+	key = luaL_checkstring(L, -2);
+	value = luaL_checkstring(L, -1);
+
+	dict_set(txn->txn, key, value);
+
+	return 0;
+}
+
+/*
+ * Start a dict transaction [-1,+1,e]
+ *
+ * Args:
+ *   1) userdata: struct dict *
+ *
+ * Returns:
+ *   Returns a new transaction object.
+ */
+int lua_dict_transaction_begin(lua_State *L)
+{
+	struct lua_dict_txn *txn;
+	struct dict *dict;
+	pool_t pool;
+
+	DLUA_REQUIRE_ARGS(L, 1);
+
+	dict = dlua_check_dict(L, -1);
+
+	pool = pool_alloconly_create("lua dict txn", 128);
+	txn = p_new(pool, struct lua_dict_txn, 1);
+	txn->pool = pool;
+	txn->txn = dict_transaction_begin(dict);
+	txn->state = STATE_OPEN;
+	txn->L = L;
+
+	xlua_pushdict_txn(L, txn, FALSE);
+
+	return 1;
+}
diff --git a/src/lib-lua/Makefile.am b/src/lib-lua/Makefile.am
index 99a64636bc..a0cc19ba38 100644
--- a/src/lib-lua/Makefile.am
+++ b/src/lib-lua/Makefile.am
@@ -12,8 +12,13 @@ libdovecot_lua_la_SOURCES = \
 	dlua-table.c \
 	dlua-thread.c
 # Note: the only things this lib should depend on are libdovecot and lua.
-libdovecot_lua_la_DEPENDENCIES = ../lib-dovecot/libdovecot.la
-libdovecot_lua_la_LIBADD = ../lib-dovecot/libdovecot.la $(LUA_LIBS)
+libdovecot_lua_la_DEPENDENCIES = \
+	../lib-dovecot/libdovecot.la \
+	../lib-dict/libdict_lua.la
+libdovecot_lua_la_LIBADD = \
+	../lib-dovecot/libdovecot.la \
+	../lib-dict/libdict_lua.la \
+	$(LUA_LIBS)
 libdovecot_lua_la_LDFLAGS = -export-dynamic
 
 headers = \