From: Aki Tuomi <aki.tuomi@open-xchange.com>
Date: Wed, 28 Aug 2024 11:43:17 +0000 (+0300)
Subject: auth: Use new var_expand
X-Git-Tag: 2.4.1~599
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=d0b4a58cb934731e4b64934895a319b16e0d66e8;p=thirdparty%2Fdovecot%2Fcore.git

auth: Use new var_expand
---

diff --git a/src/auth/Makefile.am b/src/auth/Makefile.am
index e758591ff8..d6310d6b8e 100644
--- a/src/auth/Makefile.am
+++ b/src/auth/Makefile.am
@@ -222,7 +222,6 @@ test_auth_SOURCES = \
 	test-auth-request-var-expand.c \
 	test-auth-request-fields.c \
 	test-username-filter.c \
-	test-db-ldap.c \
 	test-lua.c \
 	test-mock.c \
 	test-main.c
diff --git a/src/auth/auth-cache.c b/src/auth/auth-cache.c
index d6cbe8bb52..d6790aab25 100644
--- a/src/auth/auth-cache.c
+++ b/src/auth/auth-cache.c
@@ -1,11 +1,12 @@
 /* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
 
 #include "auth-common.h"
+#include "array.h"
 #include "lib-signals.h"
 #include "hash.h"
 #include "str.h"
 #include "strescape.h"
-#include "var-expand.h"
+#include "var-expand-new.h"
 #include "auth-request.h"
 #include "auth-cache.h"
 
@@ -25,42 +26,29 @@ struct auth_cache {
 };
 
 static bool
-auth_request_var_expand_tab_find(const char *key, unsigned int size,
-				 unsigned int *idx_r)
+auth_request_var_expand_tab_find(const char *key, unsigned int *idx_r)
 {
 	const struct var_expand_table *tab = auth_request_var_expand_static_tab;
 	unsigned int i;
 
-	for (i = 0; tab[i].key != '\0' || tab[i].long_key != NULL; i++) {
-		if (size == 1) {
-			if (key[0] == tab[i].key) {
-				*idx_r = i;
-				return TRUE;
-			}
-		} else if (tab[i].long_key != NULL) {
-			if (strncmp(key, tab[i].long_key, size) == 0 &&
-			    tab[i].long_key[size] == '\0') {
-				*idx_r = i;
-				return TRUE;
-			}
+	for (i = 0; tab[i].key != NULL; i++) {
+		if (strcmp(key, tab[i].key) == 0) {
+			*idx_r = i;
+			return TRUE;
 		}
 	}
 	return FALSE;
 }
 
 static void
-auth_cache_key_add_var(string_t *str, const char *data, unsigned int len)
+auth_cache_key_add_var(string_t *str, const char *data)
 {
 	if (str_len(str) > 0)
 		str_append_c(str, '\t');
 	str_append_c(str, '%');
-	if (len == 1)
-		str_append_c(str, data[0]);
-	else {
-		str_append_c(str, '{');
-		str_append_data(str, data, len);
-		str_append_c(str, '}');
-	}
+	str_append_c(str, '{');
+	str_append(str, data);
+	str_append_c(str, '}');
 }
 
 static void auth_cache_key_add_tab_idx(string_t *str, unsigned int i)
@@ -71,13 +59,9 @@ static void auth_cache_key_add_tab_idx(string_t *str, unsigned int i)
 	if (str_len(str) > 0)
 		str_append_c(str, '\t');
 	str_append_c(str, '%');
-	if (tab->key != '\0')
-		str_append_c(str, tab->key);
-	else {
-		str_append_c(str, '{');
-		str_append(str, tab->long_key);
-		str_append_c(str, '}');
-	}
+	str_append_c(str, '{');
+	str_append(str, tab->key);
+	str_append_c(str, '}');
 }
 
 static char *auth_cache_parse_key_exclude(pool_t pool, const char *query,
@@ -85,40 +69,34 @@ static char *auth_cache_parse_key_exclude(pool_t pool, const char *query,
 {
 	string_t *str;
 	bool key_seen[AUTH_REQUEST_VAR_TAB_COUNT];
-	const char *extra_vars;
-	unsigned int i, idx, size, tab_idx, exclude_driver_len;
+	const char *extra_vars, *error ATTR_UNUSED;
+	unsigned int i, tab_idx;
 
 	memset(key_seen, 0, sizeof(key_seen));
-	exclude_driver_len = exclude_driver == NULL ? 0 :
-		strlen(exclude_driver);
 
+	struct var_expand_program *prog;
+	if (var_expand_program_create(query, &prog, &error) < 0) {
+		e_debug(auth_event, "auth-cache: var_expand_program_create('%s') failed: %s",
+			query, error);
+		return p_strdup(pool, "");
+	}
+
+	const char *const *vars = var_expand_program_variables(prog);
 	str = t_str_new(32);
-	for (; *query != '\0'; ) {
-		if (*query != '%') {
-			query++;
-			continue;
-		}
 
-		var_get_key_range(++query, &idx, &size);
-		if (size == 0) {
-			/* broken %variable ending too early */
-			break;
-		}
-		query += idx;
-
-		if (!auth_request_var_expand_tab_find(query, size, &tab_idx)) {
-			/* just add the key. it would be nice to prevent
-			   duplicates here as well, but that's just too
-			   much trouble and probably very rare. */
-			if (exclude_driver_len == 0 ||
-			    size < exclude_driver_len ||
-			    memcmp(query, exclude_driver, exclude_driver_len) != 0)
-				auth_cache_key_add_var(str, query, size);
+	for (; *vars != NULL; vars++) {
+		/* ignore any providers */
+		if (strchr(*vars, ':') != NULL &&
+		    !str_begins_with(*vars, "passdb:") &&
+		    !str_begins_with(*vars, "userdb:")) {
+			continue;
+		} else if (!auth_request_var_expand_tab_find(*vars, &tab_idx)) {
+			if (null_strcmp(*vars, exclude_driver) != 0)
+				auth_cache_key_add_var(str, *vars);
 		} else {
 			i_assert(tab_idx < N_ELEMENTS(key_seen));
 			key_seen[tab_idx] = TRUE;
 		}
-		query += size;
 	}
 
 	if (key_seen[AUTH_REQUEST_VAR_TAB_USERNAME_IDX] &&
@@ -147,6 +125,8 @@ static char *auth_cache_parse_key_exclude(pool_t pool, const char *query,
 		str_append(str, extra_vars);
 	}
 
+	var_expand_program_free(&prog);
+
 	return p_strdup(pool, str_c(str));
 }
 
@@ -376,8 +356,8 @@ auth_request_expand_cache_key(const struct auth_request *request,
 	const char *error;
 
 	/* Uniquely identify the request's passdb/userdb with the P/U prefix
-	   and by "%!", which expands to the passdb/userdb ID number. */
-	key = t_strconcat(request->userdb_lookup ? "U" : "P", "%!",
+	   and by "%{id}", which expands to the passdb/userdb ID number. */
+	key = t_strconcat(request->userdb_lookup ? "U" : "P", "%{id}",
 		request->fields.master_user == NULL ? "" : "+%{master_user}",
 		"\t", key, NULL);
 
@@ -392,7 +372,7 @@ auth_request_expand_cache_key(const struct auth_request *request,
 	unsigned int count = 0;
 	const struct var_expand_table *table =
 		auth_request_get_var_expand_table_full(request,
-			username, auth_cache_escape, &count);
+			username, &count);
 	if (auth_request_var_expand_with_table(value, key, request, table,
 					       auth_cache_escape, &error) < 0 &&
 	    !error_logged) {
diff --git a/src/auth/auth-common.h b/src/auth/auth-common.h
index f6b6aa9446..a895ecb3cd 100644
--- a/src/auth/auth-common.h
+++ b/src/auth/auth-common.h
@@ -2,6 +2,7 @@
 #define AUTH_COMMON_H
 
 #include "lib.h"
+#include "var-expand-new.h"
 #include "auth.h"
 #include "connection.h"
 
diff --git a/src/auth/auth-policy.c b/src/auth/auth-policy.c
index e922e3aea0..9cfc6c9027 100644
--- a/src/auth/auth-policy.c
+++ b/src/auth/auth-policy.c
@@ -484,24 +484,13 @@ policy_get_var_expand_table(struct auth_request *auth_request,
 
 	table = auth_request_get_var_expand_table_full(
 			auth_request, auth_request->fields.user,
-			auth_policy_escape_function, &count);
-	table[0].key = '\0';
-	table[0].long_key = "hashed_password";
+			&count);
+	table[0].key = "hashed_password";
 	table[0].value = hashed_password;
-	table[1].key = '\0';
-	table[1].long_key = "requested_username";
+	table[1].key = "requested_username";
 	table[1].value = requested_username;
-	table[2].key = '\0';
-	table[2].long_key = "fail_type";
+	table[2].key = "fail_type";
 	table[2].value = auth_policy_fail_type(auth_request);
-	if (table[0].value != NULL) {
-		table[0].value = auth_policy_escape_function(table[0].value,
-							     auth_request);
-	}
-	if (table[1].value != NULL) {
-		table[1].value = auth_policy_escape_function(table[1].value,
-							     auth_request);
-	}
 
 	return table;
 }
diff --git a/src/auth/auth-request-fields.c b/src/auth/auth-request-fields.c
index 45f1e7fb3b..73e30b386f 100644
--- a/src/auth/auth-request-fields.c
+++ b/src/auth/auth-request-fields.c
@@ -331,7 +331,7 @@ auth_request_fix_username(struct auth_request *request, const char **username,
 		unsigned int count = 0;
 		const struct var_expand_table *table =
 			auth_request_get_var_expand_table_full(request,
-				user, NULL, &count);
+				user, &count);
 		if (auth_request_var_expand_with_table(dest,
 				set->username_format, request,
 				table, NULL, &error) <= 0) {
diff --git a/src/auth/auth-request-var-expand.c b/src/auth/auth-request-var-expand.c
index 4542366964..5cd6b7e2d0 100644
--- a/src/auth/auth-request-var-expand.c
+++ b/src/auth/auth-request-var-expand.c
@@ -13,46 +13,45 @@ struct auth_request_var_expand_ctx {
 
 const struct var_expand_table
 auth_request_var_expand_static_tab[] = {
-	{ 'u', NULL, "user" },
-	{ 'n', NULL, "username" },
-	{ 'd', NULL, "domain" },
-	{ '\0', NULL, "protocol" },
-	{ 'h', NULL, "home" },
-	{ 'l', NULL, "local_ip" },
-	{ 'r', NULL, "remote_ip" },
-	{ 'p', NULL, "client_pid" },
-	{ 'w', NULL, "password" },
-	{ '!', NULL, NULL },
-	{ 'm', NULL, "mechanism" },
-	{ 'c', NULL, "secured" },
-	{ 'a', NULL, "local_port" },
-	{ 'b', NULL, "remote_port" },
-	{ 'k', NULL, "cert" },
-	{ '\0', NULL, "login_user" },
-	{ '\0', NULL, "login_username" },
-	{ '\0', NULL, "login_domain" },
-	{ '\0', NULL, "session" },
-	{ '\0', NULL, "real_local_ip" },
-	{ '\0', NULL, "real_remote_ip" },
-	{ '\0', NULL, "real_local_port" },
-	{ '\0', NULL, "real_remote_port" },
-	{ '\0', NULL, "domain_first" },
-	{ '\0', NULL, "domain_last" },
-	{ '\0', NULL, "master_user" },
-	{ '\0', NULL, "session_pid" },
-	{ '\0', NULL, "original_user" },
-	{ '\0', NULL, "original_username" },
-	{ '\0', NULL, "original_domain" },
-	{ '\0', NULL, "auth_user" },
-	{ '\0', NULL, "auth_username" },
-	{ '\0', NULL, "auth_domain" },
-	{ '\0', NULL, "local_name" },
-	{ '\0', NULL, "client_id" },
-	{ '\0', NULL, "ssl_ja3_hash" },
-	{ '\0', NULL, "owner_user" },
-
+	{ .key = "user", .value = NULL },
+	{ .key = "username", .value = NULL },
+	{ .key = "domain", .value = NULL },
+	{ .key = "protocol", .value = NULL },
+	{ .key = "home", .value = NULL },
+	{ .key = "local_ip", .value = NULL },
+	{ .key = "remote_ip", .value = NULL },
+	{ .key = "client_pid", .value = NULL },
+	{ .key = "password", .value = NULL },
+	{ .key = "id", .value = NULL },
+	{ .key = "mechanism", .value = NULL },
+	{ .key = "secured", .value = NULL },
+	{ .key = "local_port", .value = NULL },
+	{ .key = "remote_port", .value = NULL },
+	{ .key = "cert", .value = NULL },
+	{ .key = "login_user", .value = NULL },
+	{ .key = "login_username", .value = NULL },
+	{ .key = "login_domain", .value = NULL },
+	{ .key = "session", .value = NULL },
+	{ .key = "real_local_ip", .value = NULL },
+	{ .key = "real_remote_ip", .value = NULL },
+	{ .key = "real_local_port", .value = NULL },
+	{ .key = "real_remote_port", .value = NULL },
+	{ .key = "domain_first", .value = NULL },
+	{ .key = "domain_last", .value = NULL },
+	{ .key = "master_user", .value = NULL },
+	{ .key = "session_pid", .value = NULL },
+	{ .key = "original_user", .value = NULL },
+	{ .key = "original_username", .value = NULL },
+	{ .key = "original_domain", .value = NULL },
+	{ .key = "auth_user", .value = NULL },
+	{ .key = "auth_username", .value = NULL },
+	{ .key = "auth_domain", .value = NULL },
+	{ .key = "local_name", .value = NULL },
+	{ .key = "client_id", .value = NULL },
+	{ .key = "ssl_ja3_hash", .value = NULL },
+	{ .key = "owner_user", .value = NULL },
+	VAR_EXPAND_TABLE_END
 	/* be sure to update AUTH_REQUEST_VAR_TAB_COUNT */
-	{ '\0', NULL, NULL }
 };
 static_assert_array_size(auth_request_var_expand_static_tab,
 			 AUTH_REQUEST_VAR_TAB_COUNT+1);
@@ -74,7 +73,6 @@ auth_request_str_escape(const char *string,
 struct var_expand_table *
 auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 				       const char *username,
-				       auth_request_escape_func_t *escape_func,
 				       unsigned int *count)
 {
 	const struct auth_request_fields *fields = &auth_request->fields;
@@ -83,9 +81,6 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 	struct var_expand_table *tab, *ret_tab;
 	const char *orig_user, *auth_user;
 
-	if (escape_func == NULL)
-		escape_func = escape_none;
-
 	/* keep the extra fields at the beginning. the last static_tab field
 	   contains the ending NULL-fields. */
 	tab = ret_tab = t_new(struct var_expand_table,
@@ -98,158 +93,154 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 
 	if (username == NULL)
 		username = "";
-	tab[0].value = tab[36].value = escape_func(username, auth_request);
-	tab[1].value = escape_func(t_strcut(username, '@'),
-				   auth_request);
-	tab[2].value = i_strchr_to_next(username, '@');
-	if (tab[2].value != NULL)
-		tab[2].value = escape_func(tab[2].value, auth_request);
-	tab[3].value = escape_func(fields->protocol, auth_request);
-	/* tab[4] = we have no home dir */
-	if (fields->local_ip.family != 0)
-		tab[5].value = net_ip2addr(&fields->local_ip);
-	if (fields->remote_ip.family != 0)
-		tab[6].value = net_ip2addr(&fields->remote_ip);
-	tab[7].value = dec2str(auth_request->client_pid);
-	if (auth_request->mech_password != NULL) {
-		tab[8].value = escape_func(auth_request->mech_password,
-					   auth_request);
+
+	var_expand_table_set_value(tab, "user", username);
+	var_expand_table_set_value(tab, "username", t_strcut(username, '@'));
+	var_expand_table_set_value(tab, "domain", i_strchr_to_next(username, '@'));
+	var_expand_table_set_value(tab, "protocol", fields->protocol);
+	/* tab['home'] = we have no home dir */
+	if (fields->local_ip.family != 0) {
+		var_expand_table_set_value(tab, "local_ip",
+				net_ip2addr(&fields->local_ip));
+	}
+	if (fields->remote_ip.family != 0) {
+		var_expand_table_set_value(tab, "remote_ip",
+			net_ip2addr(&fields->remote_ip));
 	}
+	var_expand_table_set_value(tab, "client_pid",
+			dec2str(auth_request->client_pid));
+	var_expand_table_set_value(tab, "password", auth_request->mech_password);
 	if (auth_request->userdb_lookup) {
-		tab[9].value = auth_request->userdb == NULL ? "" :
-			dec2str(auth_request->userdb->userdb->id);
+		var_expand_table_set_value(tab, "id",
+				auth_request->userdb == NULL ? "" :
+				dec2str(auth_request->userdb->userdb->id));
 	} else {
-		tab[9].value = auth_request->passdb == NULL ? "" :
-			dec2str(auth_request->passdb->passdb->id);
+		var_expand_table_set_value(tab, "id",
+				auth_request->passdb == NULL ? "" :
+				dec2str(auth_request->passdb->passdb->id));
 	}
-	tab[10].value = fields->mech_name == NULL ? "" :
-		escape_func(fields->mech_name, auth_request);
+
+	var_expand_table_set_value(tab, "mechanism", fields->mech_name);
+
 	switch (fields->conn_secured) {
-	case AUTH_REQUEST_CONN_SECURED_NONE: tab[11].value = ""; break;
-	case AUTH_REQUEST_CONN_SECURED: tab[11].value = "secured"; break;
-	case AUTH_REQUEST_CONN_SECURED_TLS: tab[11].value = "TLS"; break;
-	default: tab[11].value = ""; break;
+	case AUTH_REQUEST_CONN_SECURED_NONE:
+		var_expand_table_set_value(tab, "secured", ""); break;
+	case AUTH_REQUEST_CONN_SECURED:
+		var_expand_table_set_value(tab, "secured", "secured"); break;
+	case AUTH_REQUEST_CONN_SECURED_TLS:
+		var_expand_table_set_value(tab, "secured", "TLS"); break;
+	default:
+		var_expand_table_set_value(tab, "secured", ""); break;
 	};
-	tab[12].value = dec2str(fields->local_port);
-	tab[13].value = dec2str(fields->remote_port);
-	tab[14].value = fields->valid_client_cert ? "valid" : "";
+
+	var_expand_table_set_value(tab, "local_port", dec2str(fields->local_port));
+	var_expand_table_set_value(tab, "remote_port", dec2str(fields->remote_port));
+	var_expand_table_set_value(tab, "cert",
+			fields->valid_client_cert ? "valid" : "");
 
 	if (fields->requested_login_user != NULL) {
 		const char *login_user = fields->requested_login_user;
 
-		tab[15].value = escape_func(login_user, auth_request);
-		tab[16].value = escape_func(t_strcut(login_user, '@'),
-					    auth_request);
-		tab[17].value = i_strchr_to_next(login_user, '@');
-		if (tab[17].value != NULL) {
-			tab[17].value = escape_func(tab[17].value,
-						    auth_request);
-		}
+		var_expand_table_set_value(tab, "login_user", login_user);
+		var_expand_table_set_value(tab, "login_username",
+					   t_strcut(login_user, '@'));
+		var_expand_table_set_value(tab, "login_domain",
+					   i_strchr_to_next(login_user, '@'));
 	}
-	tab[18].value = fields->session_id == NULL ? NULL :
-		escape_func(fields->session_id, auth_request);
-	if (fields->real_local_ip.family != 0)
-		tab[19].value = net_ip2addr(&fields->real_local_ip);
-	if (fields->real_remote_ip.family != 0)
-		tab[20].value = net_ip2addr(&fields->real_remote_ip);
-	tab[21].value = dec2str(fields->real_local_port);
-	tab[22].value = dec2str(fields->real_remote_port);
-	tab[23].value = i_strchr_to_next(username, '@');
-	if (tab[23].value != NULL) {
-		tab[23].value = escape_func(t_strcut(tab[23].value, '@'),
-					    auth_request);
+
+	var_expand_table_set_value(tab, "session", fields->session_id);
+	if (fields->real_local_ip.family != 0) {
+		var_expand_table_set_value(tab, "real_local_ip",
+			net_ip2addr(&fields->real_local_ip));
 	}
-	tab[24].value = strrchr(username, '@');
-	if (tab[24].value != NULL)
-		tab[24].value = escape_func(tab[24].value+1, auth_request);
-	tab[25].value = fields->master_user == NULL ? NULL :
-		escape_func(fields->master_user, auth_request);
-	tab[26].value = auth_request->session_pid == (pid_t)-1 ? NULL :
-		dec2str(auth_request->session_pid);
+	if (fields->real_remote_ip.family != 0) {
+		var_expand_table_set_value(tab, "real_remote_ip",
+			net_ip2addr(&fields->real_remote_ip));
+	}
+	var_expand_table_set_value(tab, "real_local_port",
+				   dec2str(fields->real_local_port));
+	var_expand_table_set_value(tab, "real_remote_port",
+				   dec2str(fields->real_remote_port));
+
+	const char *domain_first = i_strchr_to_next(username, '@');
+	if (domain_first != NULL)
+		domain_first = t_strcut(domain_first, '@');
+	var_expand_table_set_value(tab, "domain_first", domain_first);
+	const char *domain_last = strrchr(username, '@');
+	if (domain_last != NULL)
+		domain_last++;
+	var_expand_table_set_value(tab, "domain_last", domain_last);
+	var_expand_table_set_value(tab, "master_user", fields->master_user);
+
+	const char *session_pid = "";
+	if (auth_request->session_pid != (pid_t)-1)
+		session_pid = dec2str(auth_request->session_pid);
+	var_expand_table_set_value(tab, "session_pid", session_pid);
 
 	orig_user = fields->original_username != NULL ?
 		fields->original_username : username;
-	tab[27].value = escape_func(orig_user, auth_request);
-	tab[28].value = escape_func(t_strcut(orig_user, '@'), auth_request);
-	tab[29].value = i_strchr_to_next(orig_user, '@');
-	if (tab[29].value != NULL)
-		tab[29].value = escape_func(tab[29].value, auth_request);
-
-	if (fields->master_user != NULL)
-		auth_user = fields->master_user;
-	else
-		auth_user = orig_user;
-	tab[30].value = escape_func(auth_user, auth_request);
-	tab[31].value = escape_func(t_strcut(auth_user, '@'), auth_request);
-	tab[32].value = i_strchr_to_next(auth_user, '@');
-	if (tab[32].value != NULL)
-		tab[32].value = escape_func(tab[32].value, auth_request);
-	if (fields->local_name != NULL)
-		tab[33].value = escape_func(fields->local_name, auth_request);
-	if (fields->client_id != NULL)
-		tab[34].value = escape_func(fields->client_id, auth_request);
-	if (fields->ssl_ja3_hash != NULL)
-		tab[35].value = escape_func(fields->ssl_ja3_hash, auth_request);
+	var_expand_table_set_value(tab, "original_user", orig_user);
+	var_expand_table_set_value(tab, "original_username",
+				   t_strcut(orig_user, '@'));
+	var_expand_table_set_value(tab, "original_domain",
+				   i_strchr_to_next(orig_user, '@'));
+
+	auth_user = fields->master_user != NULL ?
+		fields->master_user : orig_user;
+	var_expand_table_set_value(tab, "auth_user", auth_user);
+	var_expand_table_set_value(tab, "auth_username",
+				   t_strcut(auth_user, '@'));
+	var_expand_table_set_value(tab, "auth_domain",
+				   i_strchr_to_next(auth_user, '@'));
+	var_expand_table_set_value(tab, "local_name", fields->local_name);
+	var_expand_table_set_value(tab, "client_id", fields->client_id);
+	var_expand_table_set_value(tab, "ssl_ja3_hash", fields->ssl_ja3_hash);
+	var_expand_table_set_value(tab, "owner_user", username);
 	return ret_tab;
 }
 
 const struct var_expand_table *
-auth_request_get_var_expand_table(const struct auth_request *auth_request,
-				  auth_request_escape_func_t *escape_func)
+auth_request_get_var_expand_table(const struct auth_request *auth_request)
 {
 	unsigned int count = 0;
 
 	return auth_request_get_var_expand_table_full(auth_request,
-		auth_request->fields.user, escape_func, &count);
-}
-
-static const char *field_get_default(const char *data)
-{
-	const char *p;
-
-	p = strchr(data, ':');
-	if (p == NULL)
-		return "";
-	else {
-		/* default value given */
-		return p+1;
-	}
+		auth_request->fields.user, &count);
 }
 
 static int
-auth_request_var_expand_func_passdb(const char *data, void *context,
-				    const char **value_r,
+auth_request_var_expand_func_passdb(const char *field_name, const char **value_r,
+				    void *context,
 				    const char **error_r ATTR_UNUSED)
 {
 	struct auth_request_var_expand_ctx *ctx = context;
-	const char *field_name = t_strcut(data, ':');
 	const char *value;
 
 	value = auth_fields_find(ctx->auth_request->fields.extra_fields, field_name);
-	*value_r = ctx->escape_func(value != NULL ? value : field_get_default(data),
-				    ctx->auth_request);
-	return 1;
+	if (value == NULL)
+		value = "";
+	*value_r = value;
+	return 0;
 }
 
 static int
-auth_request_var_expand_func_userdb(const char *data, void *context,
-				    const char **value_r,
-				    const char **error_r ATTR_UNUSED)
+auth_request_var_expand_func_userdb(const char *field_name, const char **value_r,
+				    void *context, const char **error_r ATTR_UNUSED)
 {
 	struct auth_request_var_expand_ctx *ctx = context;
-	const char *field_name = t_strcut(data, ':');
 	const char *value;
 
 	value = ctx->auth_request->fields.userdb_reply == NULL ? NULL :
 		auth_fields_find(ctx->auth_request->fields.userdb_reply, field_name);
-	*value_r = ctx->escape_func(value != NULL ? value : field_get_default(data),
-				    ctx->auth_request);
-	return 1;
+	if (value == NULL)
+		value = "";
+	*value_r = value;
+	return 0;
 }
 
-const struct var_expand_func_table auth_request_var_funcs_table[] = {
-	{ "passdb", auth_request_var_expand_func_passdb },
-	{ "userdb", auth_request_var_expand_func_userdb },
+const struct var_expand_provider auth_request_var_expand_providers[] = {
+	{ .key = "passdb", .func = auth_request_var_expand_func_passdb },
+	{ .key = "userdb", .func = auth_request_var_expand_func_userdb },
 	{ NULL, NULL }
 };
 
@@ -259,7 +250,7 @@ int auth_request_var_expand(string_t *dest, const char *str,
 			    const char **error_r)
 {
 	return auth_request_var_expand_with_table(dest, str, auth_request,
-		auth_request_get_var_expand_table(auth_request, escape_func),
+		auth_request_get_var_expand_table(auth_request),
 		escape_func, error_r);
 }
 
@@ -274,20 +265,28 @@ int auth_request_var_expand_with_table(string_t *dest, const char *str,
 	i_zero(&ctx);
 	ctx.auth_request = auth_request;
 	ctx.escape_func = escape_func == NULL ? escape_none : escape_func;
-	return var_expand_with_funcs(dest, str, table,
-				     auth_request_var_funcs_table, &ctx, error_r);
+	const struct var_expand_params params = {
+		.table = table,
+		.providers = auth_request_var_expand_providers,
+		.escape_func = (var_expand_escape_func_t *)ctx.escape_func,
+		.context = &ctx,
+		.escape_context = (void *)auth_request,
+		.event = auth_request->event,
+	};
+
+	return var_expand_new(dest, str, &params, error_r) < 0 ? -1 : 1;
 }
 
 int t_auth_request_var_expand(const char *str,
-			      const struct auth_request *auth_request ATTR_UNUSED,
-			      auth_request_escape_func_t *escape_func ATTR_UNUSED,
+			      const struct auth_request *auth_request,
+			      auth_request_escape_func_t *escape_func,
 			      const char **value_r, const char **error_r)
 {
 	string_t *dest = t_str_new(128);
 	int ret = auth_request_var_expand(dest, str, auth_request,
 					  escape_func, error_r);
 	*value_r = str_c(dest);
-	return ret;
+	return ret < 0 ? ret : 1;
 }
 
 static void
@@ -296,10 +295,9 @@ auth_request_event_var_expand_callback(void *context,
 {
 	struct auth_request_var_expand_ctx *ctx = context;
 
-	params_r->table = auth_request_get_var_expand_table(ctx->auth_request,
-							    ctx->escape_func);
-	params_r->func_table = auth_request_var_funcs_table;
-	params_r->func_context = ctx;
+	params_r->table = auth_request_get_var_expand_table(ctx->auth_request);
+	params_r->providers = auth_request_var_expand_providers;
+	params_r->context = ctx;
 }
 
 void auth_request_event_set_var_expand(struct auth_request *auth_request)
diff --git a/src/auth/auth-request-var-expand.h b/src/auth/auth-request-var-expand.h
index 55b89524ef..9708372fc5 100644
--- a/src/auth/auth-request-var-expand.h
+++ b/src/auth/auth-request-var-expand.h
@@ -12,17 +12,14 @@ auth_request_escape_func_t(const char *string,
 extern const struct var_expand_table
 auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1];
 
-extern const struct var_expand_func_table auth_request_var_funcs_table[];
+extern const struct var_expand_provider auth_request_var_expand_providers[];
 
 const struct var_expand_table *
-auth_request_get_var_expand_table(const struct auth_request *auth_request,
-				  auth_request_escape_func_t *escape_func)
-	ATTR_NULL(2);
+auth_request_get_var_expand_table(const struct auth_request *auth_request);
 struct var_expand_table *
 auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 				       const char *username,
-				       auth_request_escape_func_t *escape_func,
-				       unsigned int *count) ATTR_NULL(3);
+				       unsigned int *count);
 
 int auth_request_var_expand(string_t *dest, const char *str,
 			    const struct auth_request *auth_request,
diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 93cd31c3e1..cd18337147 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -803,29 +803,18 @@ static void auth_request_passdb_internal_failure(struct auth_request *request)
 }
 
 static int
-auth_request_fields_var_expand_lookup(const char *data, void *context,
-				      const char **value_r,
-				      const char **error_r ATTR_UNUSED)
+auth_request_fields_var_expand_lookup(const char *field_name, const char **value_r,
+				      void *context, const char **error_r)
 {
 	struct auth_fields *fields = context;
-	*value_r = NULL;
-
-	const char *default_value = strchr(data, ':');
-	if (default_value == NULL) {
-		if (fields != NULL)
-			*value_r = auth_fields_find(fields, data);
-		else
-			*value_r = "";
-		return 1;
+
+	if (fields != NULL) {
+		*value_r = auth_fields_find(fields, field_name);
+		return 0;
+	} else {
+		*error_r = t_strdup_printf("No such field '%s'", field_name);
+		return -1;
 	}
-	/* If the fields are not initialized do not try to find fields. */
-	if (fields != NULL)
-		*value_r = auth_fields_find(fields,
-				t_strdup_until(data, default_value));
-	default_value++;
-	if (*value_r == NULL)
-		*value_r = default_value;
-	return 1;
 }
 
 int auth_request_set_passdb_fields(struct auth_request *request,
@@ -833,8 +822,8 @@ int auth_request_set_passdb_fields(struct auth_request *request,
 {
 	const char *driver_name =
 		t_str_replace(request->passdb->passdb->iface.name, '-', '_');
-	const struct var_expand_func_table fn_table[] = {
-		{ driver_name, auth_request_fields_var_expand_lookup },
+	const struct var_expand_provider fn_table[] = {
+		{ .key = driver_name, .func = auth_request_fields_var_expand_lookup },
 		{ NULL, NULL }
 	};
 
@@ -844,15 +833,15 @@ int auth_request_set_passdb_fields(struct auth_request *request,
 int auth_request_set_passdb_fields_ex(struct auth_request *request,
 				      void *context,
 				      const char *default_password_scheme,
-				      const struct var_expand_func_table *fn_table)
+				      const struct var_expand_provider *fn_table)
 {
 	struct event *event = event_create(authdb_event(request));
 	const struct auth_passdb_post_settings *post_set;
 	const char *error;
 
 	struct var_expand_params params = {
-		.func_table = fn_table,
-		.func_context = context,
+		.providers = fn_table,
+		.context = context,
 	};
 	event_set_ptr(event, SETTINGS_EVENT_VAR_EXPAND_PARAMS, &params);
 
@@ -873,26 +862,26 @@ int auth_request_set_userdb_fields(struct auth_request *request,
 				   struct auth_fields *fields) {
 	const char *driver_name =
 		t_str_replace(request->userdb->userdb->iface->name, '-', '_');
-	const struct var_expand_func_table fn_table[] = {
-		{ driver_name, auth_request_fields_var_expand_lookup },
-		{ NULL, NULL }
+	const struct var_expand_provider fn_table[] = {
+		{ .key = driver_name, .func = auth_request_fields_var_expand_lookup },
+		VAR_EXPAND_TABLE_END
 	};
 
 	return auth_request_set_userdb_fields_ex(request, fields, fn_table);
 }
 
 int auth_request_set_userdb_fields_ex(struct auth_request *request, void *context,
-				      const struct var_expand_func_table *fn_table)
+				      const struct var_expand_provider *fn_table)
 {
 	struct event *event = event_create(authdb_event(request));
 	const struct auth_userdb_post_settings *post_set;
 	const char *error;
 
-	struct var_expand_params params = {
-		.func_table = fn_table,
-		.func_context = context,
+	const struct var_expand_params params = {
+		.providers = fn_table,
+		.context = context,
 	};
-	event_set_ptr(event, SETTINGS_EVENT_VAR_EXPAND_PARAMS, &params);
+	event_set_ptr(event, SETTINGS_EVENT_VAR_EXPAND_PARAMS, (void *)&params);
 
 	if (settings_get(event, &auth_userdb_post_setting_parser_info, 0,
 			 &post_set, &error) < 0) {
diff --git a/src/auth/auth-request.h b/src/auth/auth-request.h
index 5c8326eb69..32b2195ba7 100644
--- a/src/auth/auth-request.h
+++ b/src/auth/auth-request.h
@@ -7,7 +7,7 @@
 
 #include "array.h"
 #include "net.h"
-#include "var-expand.h"
+#include "var-expand-new.h"
 #include "mech.h"
 #include "userdb.h"
 #include "passdb.h"
@@ -328,12 +328,12 @@ int auth_request_set_passdb_fields(struct auth_request *request,
 				   struct auth_fields *fields);
 int auth_request_set_passdb_fields_ex(struct auth_request *request, void *context,
 				      const char *default_password_scheme,
-				      const struct var_expand_func_table *fn_table);
+				      const struct var_expand_provider *fn_table);
 
 int auth_request_set_userdb_fields(struct auth_request *request,
 				   struct auth_fields *fields);
 int auth_request_set_userdb_fields_ex(struct auth_request *request, void *context,
-				      const struct var_expand_func_table *fn_table);
+				      const struct var_expand_provider *fn_table);
 
 void auth_request_init_userdb_reply(struct auth_request *request);
 void auth_request_set_userdb_field(struct auth_request *request,
diff --git a/src/auth/auth-settings.c b/src/auth/auth-settings.c
index 36859c1519..6b22da36d1 100644
--- a/src/auth/auth-settings.c
+++ b/src/auth/auth-settings.c
@@ -364,7 +364,7 @@ static const struct auth_settings auth_default_settings = {
 	.cache_verify_password_with_worker = FALSE,
 	.username_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890.-_@",
 	.username_translation = "",
-	.username_format = "%Lu",
+	.username_format = "%{user | lower}",
 	.master_user_separator = "",
 	.anonymous_username = "anonymous",
 	.krb5_keytab = "",
diff --git a/src/auth/auth.c b/src/auth/auth.c
index 339d5d8938..c73dcb27c2 100644
--- a/src/auth/auth.c
+++ b/src/auth/auth.c
@@ -474,13 +474,11 @@ void auths_init(void)
 	struct auth *auth;
 
 	/* sanity checks */
-	i_assert(auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_USER_IDX].key == 'u');
-	i_assert(auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_USERNAME_IDX].key == 'n');
-	i_assert(auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_DOMAIN_IDX].key == 'd');
-	i_assert(auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT].key == '\0' &&
-		 auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT].long_key == NULL);
-	i_assert(auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT-1].key != '\0' ||
-		 auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT-1].long_key != NULL);
+	i_assert(*auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_USER_IDX].key == 'u');
+	i_assert(*auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_USERNAME_IDX].key == 'u');
+	i_assert(*auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_DOMAIN_IDX].key == 'd');
+	i_assert(auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT].key == NULL);
+	i_assert(auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT-1].key != NULL);
 
 	array_foreach_elem(&auths, auth)
 		auth_init(auth);
diff --git a/src/auth/db-ldap.c b/src/auth/db-ldap.c
index 9d1b2a9bcd..6c74243fe7 100644
--- a/src/auth/db-ldap.c
+++ b/src/auth/db-ldap.c
@@ -11,9 +11,9 @@
 #include "hash.h"
 #include "aqueue.h"
 #include "str.h"
+#include "strescape.h"
 #include "time-util.h"
 #include "env-util.h"
-#include "var-expand.h"
 #include "settings.h"
 #include "ssl-settings.h"
 #include "userdb.h"
@@ -1100,28 +1100,6 @@ static void db_ldap_conn_close(struct ldap_connection *conn)
 	}
 }
 
-struct ldap_field_find_context {
-	pool_t pool;
-	ARRAY_TYPE(const_string) attr_names;
-	ARRAY_TYPE(const_string) sensitive_attr_names;
-};
-
-static int
-db_ldap_field_find(const char *data, void *context,
-		   const char **value_r,
-		   const char **error_r ATTR_UNUSED)
-{
-	struct ldap_field_find_context *ctx = context;
-	const char *ldap_attr;
-
-	if (*data != '\0') {
-		ldap_attr = p_strdup(ctx->pool, t_strcut(data, ':'));
-		array_push_back(&ctx->attr_names, &ldap_attr);
-	}
-	*value_r = NULL;
-	return 1;
-}
-
 static bool
 db_ldap_is_sensitive_field(const char *name)
 {
@@ -1138,22 +1116,18 @@ void db_ldap_get_attribute_names(pool_t pool,
 				 const char *const **sensitive_r,
 				 const char *skip_attr)
 {
-	static const struct var_expand_func_table var_funcs_table[] = {
-		{ "ldap", db_ldap_field_find },
-		{ "ldap_multi", db_ldap_field_find },
-		{ NULL, NULL }
-	};
-
 	unsigned int count = array_is_empty(attrlist) ? 0 : array_count(attrlist);
 	i_assert(count % 2 == 0);
 
-	struct ldap_field_find_context ctx;
-	ctx.pool = pool;
-	p_array_init(&ctx.attr_names, pool, count / 2);
-	p_array_init(&ctx.sensitive_attr_names, pool, 2);
+	ARRAY_TYPE(const_string) attr_names;
+	ARRAY_TYPE(const_string) sensitive_attr_names;
+
+	p_array_init(&attr_names, pool, count / 2);
+	p_array_init(&sensitive_attr_names, pool, 2);
 	string_t *tmp_str = t_str_new(128);
 
 	for (unsigned int index = 0; index < count; ) {
+		struct var_expand_program *prog;
 		const char *name = array_idx_elem(attrlist, index++);
 		const char *value = array_idx_elem(attrlist, index++);
 
@@ -1166,8 +1140,25 @@ void db_ldap_get_attribute_names(pool_t pool,
 		/* Mark the current end of the array before adding the elements
 		   from the expansion of the field expression. This will be
 		   used later to see which elements have been added. */
-		unsigned int index = array_count(&ctx.attr_names);
-		(void)var_expand_with_funcs(tmp_str, value, NULL, var_funcs_table, &ctx, &error);
+		unsigned int index = array_count(&attr_names);
+
+		if (var_expand_program_create(value, &prog, &error) < 0) {
+			e_debug(auth_event, "db-ldap: var_expand_program_create('%s') failed: %s", value, error);
+			continue;
+		}
+
+		const char *const *vars = var_expand_program_variables(prog);
+		for (; *vars != NULL; vars++) {
+			const char *ldap_attr;
+			if (str_begins(*vars, "ldap:", &ldap_attr) ||
+			    str_begins(*vars, "ldap_multi:", &ldap_attr)) {
+				/* when we free program, this name
+				   would be invalid, so dup it here. */
+				ldap_attr = p_strdup(pool, ldap_attr);
+				array_push_back(&attr_names, &ldap_attr);
+			}
+		}
+		var_expand_program_free(&prog);
 
 		if (!db_ldap_is_sensitive_field(name))
 			continue;
@@ -1178,28 +1169,28 @@ void db_ldap_get_attribute_names(pool_t pool,
 		   allows for multiple attributes to be used. In this case, we
 		   mark them all. */
 
-		unsigned int count = array_count(&ctx.attr_names);
+		unsigned int count = array_count(&attr_names);
 		/* Now index points to the first attribute newly added to
 		   attr_names, and count points to the end of attr_names. */
 
 		for (; index < count; index++) {
-			const char *const *src = array_idx(&ctx.attr_names, index);
-			array_push_back(&ctx.sensitive_attr_names, src);
+			const char *const *src = array_idx(&attr_names, index);
+			array_push_back(&sensitive_attr_names, src);
 		}
 	}
-	array_append_zero(&ctx.attr_names);
-	array_append_zero(&ctx.sensitive_attr_names);
+	array_append_zero(&attr_names);
+	array_append_zero(&sensitive_attr_names);
 
-	*attr_names_r = array_front(&ctx.attr_names);
+	*attr_names_r = array_front(&attr_names);
 	if (sensitive_r != NULL)
-		*sensitive_r = array_front(&ctx.sensitive_attr_names);
+		*sensitive_r = array_front(&sensitive_attr_names);
 }
 
 #define IS_LDAP_ESCAPED_CHAR(c) \
 	((((unsigned char)(c)) & 0x80) != 0 || strchr(LDAP_ESCAPE_CHARS, (c)) != NULL)
 
 const char *ldap_escape(const char *str,
-			const struct auth_request *auth_request ATTR_UNUSED)
+			void *context ATTR_UNUSED)
 {
 	string_t *ret = NULL;
 
@@ -1324,131 +1315,64 @@ db_ldap_result_iterate_init(struct ldap_connection *conn,
 						skip_null_values);
 }
 
-void db_ldap_field_multi_expand_parse_data(
-	const char *data, const char **field_name_r,
-	const char **separator_r, const char **default_r)
-{
-	/* start with the defaults */
-	*separator_r = " ";
-	*default_r = "";
-
-	/* Normalize to lower case as fields names are case insensitive. */
-	*field_name_r = t_str_lcase(t_strcut(data, ':'));
-	const char *ptr = i_strchr_to_next(data, ':');
-
-	if (ptr == NULL || ptr[0] == '\0') {
-		/* Handling here the cases:
-		   attrName		-> *sep_r = (default), *default_r = (default)
-		   attrName:		-> *sep_r = (default), *default_r = (default)
-		*/
-		return;
-	}
-
-	if (ptr[0] == ':' && (ptr[1] == '\0' || ptr[1] == ':')) {
-		/* Handling here the cases (exceptions dealing with ':'):
-		   attrName::		-> *sep_r = ":", *default_r = (default)
-		   attrName:::		-> *sep_r = ":", *default_r = (default)
-		   attrName:::defl	-> *sep_r = ":", *default_r = "defl"
-		*/
-		*separator_r = ":";
-
-		/* The current ':' was not a field separator, but just datum.
-		   Advance paste it */
-		if (*++ptr == ':')
-			++ptr;
-	} else {
-		/* Handling here the cases (the normal ones):
-		   attrName::defl       -> *sep_r = (default), *default_r = "defl"
-		   attrName:sep         -> *sep_r = "sep", *default_r = (default)
-		   attrName:sep:defl    -> *sep_r = "sep", *default_r = "defl"
-		*/
-		const char *sep = t_strcut(ptr, ':');
-		ptr = i_strchr_to_next(ptr, ':');
-		if (*sep != '\0')
-			*separator_r = sep;
-	}
-
-	if (ptr == NULL || ptr[0] == '\0')
-		return;
-
-	*default_r = ptr;
-}
-
 const char *db_ldap_attribute_as_multi(const char *name)
 {
 	return t_strconcat(DB_LDAP_ATTR_MULTI_PREFIX, name, NULL);
 }
 
 static int
-db_ldap_field_multi_expand(const char *data, void *context,
-			   const char **value_r, const char **error_r ATTR_UNUSED)
+db_ldap_field_multi_expand(const char *data, const char **value_r,
+			   void *context, const char **error_r)
 {
 	struct db_ldap_field_expand_context *ctx = context;
 	struct auth_fields *fields = ctx->fields;
-
-	const char *field_name;
-	const char *field_separator;
-	const char *field_default;
-
-	db_ldap_field_multi_expand_parse_data(data, &field_name,
-					      &field_separator,
-					      &field_default);
-
-	if (strcasecmp(field_name, "dn") == 0) {
-		*value_r = auth_fields_find(fields, DB_LDAP_ATTR_DN);
-		i_assert(*value_r != NULL);
-		return 1;
-	}
+	const char *field_name = t_str_lcase(data);
 
 	const char *value = auth_fields_find(fields,
 					     db_ldap_attribute_as_multi(field_name));
 	if (value == NULL || *value == '\0')
 		value = auth_fields_find(fields, field_name);
 
-	if (value == NULL || *value == '\0')
-		value = field_default == NULL ? "" : field_default;
-	else {
-		const char **entries = t_strsplit(value, DB_LDAP_ATTR_SEPARATOR);
-		value = t_strarray_join(entries, field_separator);
+	if (value == NULL || *value == '\0') {
+		*error_r = t_strdup_printf("No such LDAP attribute '%s'", field_name);
+		return -1;
 	}
 	*value_r = value;
-	return 1;
+	return 0;
 }
 
 static int
-db_ldap_field_single_expand(const char *data ATTR_UNUSED, void *context,
-			    const char **value_r, const char **error_r ATTR_UNUSED)
+db_ldap_field_single_expand(const char *data, const char **value_r,
+			    void *context, const char **error_r)
 {
 	struct db_ldap_field_expand_context *ctx = context;
 	struct auth_fields *fields = ctx->fields;
-	const char *field_default = strchr(data, ':');
-	const char *field_name = field_default == NULL ? data : t_strdup_until(data, field_default);
+	const char *field_name = t_str_lcase(data);
 
-	if (strcasecmp(field_name, "dn") == 0) {
+	if (strcmp(field_name, "dn") == 0) {
+		/* DN must be always there */
 		*value_r = auth_fields_find(fields, DB_LDAP_ATTR_DN);
 		i_assert(*value_r != NULL);
-		return 1;
+		return 0;
 	}
 
-	/* Normalize to lower case as LDAP attributes are case insensitive. */
-	field_name = t_str_lcase(field_name);
-
 	*value_r = NULL;
 	if (fields != NULL)
 		*value_r = auth_fields_find(fields, field_name);
 
-	if (*value_r == NULL || **value_r == '\0')
-		*value_r = field_default == NULL ? "" : field_default + 1;
-	else if (auth_fields_find(fields,
+	if (*value_r == NULL || **value_r == '\0') {
+		*error_r = t_strdup_printf("No such LDAP attribute '%s'", field_name);
+		return -1;
+	} else if (auth_fields_find(fields,
 				  db_ldap_attribute_as_multi(field_name)) != NULL) {
 		e_warning(ctx->event, "Multiple values found for '%s': "
 			              "using value '%s'", field_name, *value_r);
 	}
 
-	return 1;
+	return 0;
 }
 
-const struct var_expand_func_table db_ldap_field_expand_fn_table[] = {
+const struct var_expand_provider db_ldap_field_expand_fn_table[] = {
 	{ "ldap",       db_ldap_field_single_expand },
 	{ "ldap_multi", db_ldap_field_multi_expand },
 	{ NULL, NULL }
@@ -1475,8 +1399,14 @@ ldap_query_get_fields(pool_t pool,
 		auth_fields_add(fields, name, values[0], 0);
 		if (values[0] != NULL && values[1] != NULL) {
 			const char *mname = db_ldap_attribute_as_multi(name);
-			const char *mvalue = t_strarray_join(values, DB_LDAP_ATTR_SEPARATOR);
-			auth_fields_add(fields, mname, mvalue, 0);
+			string_t *mvalue = t_str_new(32);
+			for (; *values != NULL; values++) {
+				str_append_tabescaped(mvalue, *values);
+				str_append_c(mvalue, '\t');
+			}
+			/* drop last \t */
+			str_truncate(mvalue, str_len(mvalue) - 1);
+			auth_fields_add(fields, mname, str_c(mvalue), 0);
 		}
 	}
 	db_ldap_result_iterate_deinit(&ldap_iter);
diff --git a/src/auth/db-ldap.h b/src/auth/db-ldap.h
index 1731e7429d..ca69740ffe 100644
--- a/src/auth/db-ldap.h
+++ b/src/auth/db-ldap.h
@@ -20,7 +20,7 @@
 #define DB_LDAP_IDLE_RECONNECT_SECS 60
 
 #include <ldap.h>
-#include "var-expand.h"
+#include "var-expand-new.h"
 #include "db-ldap-settings.h"
 
 #define DB_LDAP_ATTR_MULTI_PREFIX "+"
@@ -149,7 +149,7 @@ struct db_ldap_field_expand_context {
 	struct auth_fields *fields;
 };
 
-extern const struct var_expand_func_table db_ldap_field_expand_fn_table[];
+extern const struct var_expand_provider db_ldap_field_expand_fn_table[];
 
 /* Send/queue request */
 void db_ldap_request(struct ldap_connection *conn,
@@ -169,8 +169,7 @@ void db_ldap_connect_delayed(struct ldap_connection *conn);
 
 void db_ldap_enable_input(struct ldap_connection *conn, bool enable);
 
-const char *ldap_escape(const char *str,
-			const struct auth_request *auth_request);
+const char *ldap_escape(const char *str, void *context);
 const char *ldap_get_error(struct ldap_connection *conn);
 
 struct db_ldap_result_iterate_context *
diff --git a/src/auth/db-lua.c b/src/auth/db-lua.c
index b7e211b07f..ffdcf9cbd5 100644
--- a/src/auth/db-lua.c
+++ b/src/auth/db-lua.c
@@ -286,11 +286,11 @@ static int auth_request_lua_index(lua_State *L)
 	lua_pop(L, 1);
 
 	const struct var_expand_table *table =
-		auth_request_get_var_expand_table(req, NULL);
+		auth_request_get_var_expand_table(req);
 
 	/* check if it's variable */
 	for(unsigned int i = 0; i < AUTH_REQUEST_VAR_TAB_COUNT; i++) {
-		if (null_strcmp(table[i].long_key, key) == 0) {
+		if (null_strcmp(table[i].key, key) == 0) {
 			lua_pushstring(L, table[i].value);
 			return 1;
 		}
diff --git a/src/auth/db-oauth2.c b/src/auth/db-oauth2.c
index 565852b02f..c041db2b31 100644
--- a/src/auth/db-oauth2.c
+++ b/src/auth/db-oauth2.c
@@ -4,7 +4,6 @@
 #include "array.h"
 #include "str.h"
 #include "strescape.h"
-#include "var-expand.h"
 #include "env-util.h"
 #include "settings.h"
 #include "oauth2.h"
@@ -53,7 +52,7 @@ static const struct auth_oauth2_settings auth_oauth2_default_settings = {
 	.scope = ARRAY_INIT,
 	.force_introspection = FALSE,
 	.introspection_mode = ":auth:get:post:local",
-	.username_validation_format = "%u",
+	.username_validation_format = "%{user}",
 	.username_attribute = "email",
 	.active_attribute = "",
 	.active_value = "",
@@ -362,45 +361,38 @@ db_oauth2_have_all_fields(struct db_oauth2_request *req)
 	return TRUE;
 }
 
-static const char *field_get_default(const char *data)
-{
-	const char *p;
-
-	p = strchr(data, ':');
-	if (p == NULL)
-		return "";
-	else {
-		/* default value given */
-		return p+1;
-	}
-}
-
-static int db_oauth2_var_expand_func_oauth2(const char *data, void *context,
-					    const char **value_r,
-					    const char **error_r ATTR_UNUSED)
+static int db_oauth2_var_expand_func_oauth2(const char *field_name,
+					    const char **value_r, void *context,
+					    const char **error_r)
 {
 	struct db_oauth2_request *ctx = context;
-	const char *field_name = t_strcut(data, ':');
 	const char *value = NULL;
 
-	if (ctx->fields != NULL)
-		value = auth_fields_find(ctx->fields, field_name);
-	*value_r = value != NULL ? value : field_get_default(data);
-
-	return 1;
+	if (ctx->fields != NULL) {
+		*value_r = auth_fields_find(ctx->fields, field_name);
+		return 0;
+	} else {
+		*error_r = t_strdup_printf("OAuth2 field '%s' not found", field_name);
+		return -1;
+	}
+	*value_r = value;
 }
 
 static bool
 db_oauth2_add_extra_fields(struct db_oauth2_request *req, const char **error_r)
 {
-	struct var_expand_func_table func_table[] = {
+	const struct var_expand_provider func_table[] = {
 		{ "oauth2", db_oauth2_var_expand_func_oauth2 },
 		{ NULL, NULL }
 	};
+	const struct var_expand_provider *provider_arr[] = {
+		func_table,
+		NULL
+	};
 	struct var_expand_params params = {
-		.table = auth_request_get_var_expand_table(req->auth_request, NULL),
-		.func_table = func_table,
-		.func_context = req,
+		.table = auth_request_get_var_expand_table(req->auth_request),
+		.providers_arr = provider_arr,
+		.context = req,
 	};
 	struct auth_request *request = req->auth_request;
 	const struct auth_oauth2_post_settings *set;
@@ -476,10 +468,10 @@ db_oauth2_validate_username(struct db_oauth2_request *req,
 {
 	const char *error;
 	struct var_expand_table table[] = {
-		{ 'u', NULL, "user" },
-		{ 'n', NULL, "username" },
-		{ 'd', NULL, "domain" },
-		{ '\0', NULL, NULL }
+		{ .key = "user", .value = NULL },
+		{ .key = "username", .value = NULL },
+		{ .key = "domain", .value = NULL },
+		VAR_EXPAND_TABLE_END
 	};
 	const char *username_value =
 		auth_fields_find(req->fields, req->db->set->username_attribute);
@@ -496,8 +488,13 @@ db_oauth2_validate_username(struct db_oauth2_request *req,
 
 	string_t *username_val = t_str_new(strlen(username_value));
 
-	if (var_expand_with_table(username_val, req->db->set->username_validation_format, table,
-				  &error) <= 0) {
+	const struct var_expand_params params = {
+		.table = table,
+		.event = req->auth_request->event,
+	};
+
+	if (var_expand_new(username_val, req->db->set->username_validation_format,
+			   &params, &error) < 0) {
 		*error_r = t_strdup_printf("var_expand(%s) failed: %s",
 					req->db->set->username_validation_format, error);
 		*result_r = PASSDB_RESULT_INTERNAL_FAILURE;
diff --git a/src/auth/db-passwd-file.c b/src/auth/db-passwd-file.c
index 4e494d04aa..c900958159 100644
--- a/src/auth/db-passwd-file.c
+++ b/src/auth/db-passwd-file.c
@@ -56,6 +56,23 @@ const struct setting_parser_info passwd_file_setting_parser_info = {
 	.pool_offset1 = 1 + offsetof(struct passwd_file_settings, pool),
 };
 
+static int db_passwd_file_expand(const char *key, const char **value_r,
+				 void *context, const char **error_r)
+{
+	struct auth_fields *pwd_fields = context;
+	*value_r = auth_fields_find(pwd_fields, key);
+	if (*value_r == NULL) {
+		*error_r = t_strdup_printf("No such field '%s'", key);
+		return -1;
+	}
+	return 0;
+}
+
+const struct var_expand_provider db_passwd_file_var_expand_fn[] = {
+	{ .key = "passwd_file", .func = db_passwd_file_expand },
+	VAR_EXPAND_TABLE_END
+};
+
 static struct db_passwd_file *passwd_files;
 
 static void ATTR_NULL(3)
@@ -343,9 +360,8 @@ static void db_passwd_file_set_userdb(struct db_passwd_file *db)
 struct db_passwd_file *
 db_passwd_file_init(const char *path, bool userdb, bool debug)
 {
+	const char *error;
 	struct db_passwd_file *db;
-	const char *p;
-	bool percents = FALSE;
 
 	db = db_passwd_file_find(path);
 	if (db != NULL) {
@@ -362,30 +378,18 @@ db_passwd_file_init(const char *path, bool userdb, bool debug)
 	db->event = event_create(auth_event);
 	event_set_forced_debug(db->event, debug);
 
-	for (p = path; *p != '\0'; p++) {
-		if (*p == '%' && p[1] != '\0') {
-			if (var_get_key(++p) == '%')
-				percents = TRUE;
-			else
-				db->vars = TRUE;
-		}
-	}
-
-	if (percents && !db->vars) {
-		/* just extra escaped % chars. remove them. */
-		struct var_expand_table empty_table[1] = {
-			{ .key = '\0' },
-		};
-		string_t *dest;
-		const char *error;
-
-		dest = t_str_new(256);
-		if (var_expand_with_table(dest, path, empty_table, &error) <= 0)
-			i_unreached();
-		path = str_c(dest);
-	}
+	struct var_expand_program *prog;
+	if (var_expand_program_create(path, &prog, &error) < 0)
+		i_fatal("Invalid path '%s' for passwd-file", error);
 
+	const char *const *vars = var_expand_program_variables(prog);
 	db->path = i_strdup(path);
+
+	if (*vars != NULL) {
+		db->vars = TRUE;
+		db->prog = prog;
+	} else
+		var_expand_program_free(&prog);
 	if (db->vars) {
 		hash_table_create(&db->files, default_pool, 0,
 				  str_hash, strcmp);
@@ -428,6 +432,7 @@ void db_passwd_file_unref(struct db_passwd_file **_db)
 			break;
 		}
 	}
+	var_expand_program_free(&db->prog);
 
 	if (db->default_file != NULL)
 		passwd_file_free(db->default_file);
@@ -444,8 +449,7 @@ void db_passwd_file_unref(struct db_passwd_file **_db)
 }
 
 static const char *
-path_fix(const char *path,
-	 const struct auth_request *auth_request ATTR_UNUSED)
+path_fix(const char *path, void *context ATTR_UNUSED)
 {
 	const char *p;
 
@@ -471,9 +475,14 @@ int db_passwd_file_lookup(struct db_passwd_file *db,
 	if (!db->vars)
 		pw = db->default_file;
 	else {
+		const struct var_expand_params params = {
+			.table = auth_request_get_var_expand_table(request),
+			.providers = auth_request_var_expand_providers,
+			.context = request,
+			.escape_func = path_fix,
+		};
 		dest = t_str_new(256);
-		if (auth_request_var_expand(dest, db->path, request, path_fix,
-					    &error) <= 0) {
+		if (var_expand_program_execute(dest, db->prog, &params, &error) < 0) {
 			e_error(authdb_event(request),
 				"Failed to expand passwd-file path %s: %s",
 				db->path, error);
diff --git a/src/auth/db-passwd-file.h b/src/auth/db-passwd-file.h
index 7ef8b3b4dc..90a8d77270 100644
--- a/src/auth/db-passwd-file.h
+++ b/src/auth/db-passwd-file.h
@@ -34,6 +34,7 @@ struct db_passwd_file {
 	struct event *event;
 
 	char *path;
+	struct var_expand_program *prog;
 	HASH_TABLE(char *, struct passwd_file *) files;
         struct passwd_file *default_file;
 
@@ -49,6 +50,8 @@ struct passwd_file_settings {
 
 extern const struct setting_parser_info passwd_file_setting_parser_info;
 
+extern const struct var_expand_provider db_passwd_file_var_expand_fn[];
+
 int db_passwd_file_lookup(struct db_passwd_file *db,
 			  struct auth_request *request,
 			  const char *username_format,
diff --git a/src/auth/passdb-passwd-file.c b/src/auth/passdb-passwd-file.c
index 91c4fb07c1..897d7ea9f0 100644
--- a/src/auth/passdb-passwd-file.c
+++ b/src/auth/passdb-passwd-file.c
@@ -27,7 +27,7 @@ passwd_file_add_extra_fields(struct auth_request *request,
 	unsigned int i;
 	int ret = 0;
 
-	table = auth_request_get_var_expand_table(request, NULL);
+	table = auth_request_get_var_expand_table(request);
 
 	pool_t pool = pool_alloconly_create("passwd-file fields", 256);
 	struct auth_fields *pwd_fields = auth_fields_init(pool);
@@ -55,7 +55,8 @@ passwd_file_add_extra_fields(struct auth_request *request,
 			auth_fields_add(pwd_fields, key, value, 0);
 	}
 
-	if (ret == 0 && auth_request_set_passdb_fields(request, pwd_fields) < 0)
+	if (ret == 0 && auth_request_set_passdb_fields_ex(request, pwd_fields, "PLAIN",
+							  db_passwd_file_var_expand_fn) < 0)
 		ret = -1;
 	pool_unref(&pool);
 	return ret;
diff --git a/src/auth/passdb-template.c b/src/auth/passdb-template.c
index 3bb7f544b6..2f3da8a0dc 100644
--- a/src/auth/passdb-template.c
+++ b/src/auth/passdb-template.c
@@ -6,36 +6,53 @@
 #include "passdb.h"
 #include "passdb-template.h"
 
+struct passdb_template_arg {
+	const char *key;
+	struct var_expand_program *program;
+};
+
 struct passdb_template {
-	ARRAY(const char *) args;
+	ARRAY(struct passdb_template_arg) args;
+	ARRAY_TYPE(const_string) keys;
 };
 
 struct passdb_template *passdb_template_build(pool_t pool, const char *args)
 {
 	struct passdb_template *tmpl;
-	const char *const *tmp, *key, *value;
+	const char *const *tmp;
 
 	tmpl = p_new(pool, struct passdb_template, 1);
 
 	tmp = t_strsplit_spaces(args, " ");
-	p_array_init(&tmpl->args, pool, str_array_length(tmp));
+
+	p_array_init(&tmpl->args, pool, str_array_length(tmp) / 2);
+	p_array_init(&tmpl->keys, pool, str_array_length(tmp) / 2);
 
 	for (; *tmp != NULL; tmp++) {
-		value = strchr(*tmp, '=');
-		if (value == NULL)
-			key = *tmp;
+		const char *p = strchr(*tmp, '=');
+		const char *kp;
+		const char *error;
+
+		if (p == NULL)
+			kp = *tmp;
 		else
-			key = t_strdup_until(*tmp, value++);
+			kp = t_strdup_until(*tmp, p++);
 
-		if (*key == '\0')
+		if (*kp == '\0')
 			i_fatal("Invalid passdb template %s - key must not be empty",
 				args);
 
-		key = p_strdup(pool, key);
-		value = p_strdup(pool, value);
-		array_push_back(&tmpl->args, &key);
-		array_push_back(&tmpl->args, &value);
+		char *key = p_strdup(pool, kp);
+		struct var_expand_program *prog;
+		if (var_expand_program_create(p, &prog, &error) < 0)
+			i_fatal("Invalid passdb template value %s: %s", p, error);
+
+		struct passdb_template_arg *arg = array_append_space(&tmpl->args);
+		arg->key = key;
+		arg->program = prog;
+		array_push_back(&tmpl->keys, &arg->key);
 	}
+
 	return tmpl;
 }
 
@@ -43,60 +60,54 @@ int passdb_template_export(struct passdb_template *tmpl,
 			   struct auth_request *auth_request,
 			   const char **error_r)
 {
-        const struct var_expand_table *table;
 	string_t *str;
-	const char *const *args, *value;
-	unsigned int i, count;
+	const struct passdb_template_arg *arg;
+	int ret = 0;
 
 	if (passdb_template_is_empty(tmpl))
 		return 0;
 
+	const struct var_expand_params params = {
+		.table = auth_request_get_var_expand_table(auth_request),
+		.providers = auth_request_var_expand_providers,
+		.context = auth_request,
+	};
+
 	str = t_str_new(256);
-	table = auth_request_get_var_expand_table(auth_request, NULL);
-
-	args = array_get(&tmpl->args, &count);
-	i_assert((count % 2) == 0);
-	for (i = 0; i < count; i += 2) {
-		if (args[i+1] == NULL)
-			value = "";
-		else {
-			str_truncate(str, 0);
-			if (auth_request_var_expand_with_table(str, args[i+1],
-					auth_request, table, NULL, error_r) <= 0)
-				return -1;
-			value = str_c(str);
-		}
-		auth_request_set_field(auth_request, args[i], value,
+
+	array_foreach(&tmpl->args, arg) {
+		str_truncate(str, 0);
+		ret = var_expand_program_execute(str, arg->program, &params,
+						 error_r);
+		if (ret < 0)
+			break;
+		auth_request_set_field(auth_request, arg->key, str_c(str),
 				       STATIC_PASS_SCHEME);
 	}
-	return 0;
-}
 
-bool passdb_template_remove(struct passdb_template *tmpl,
-			    const char *key, const char **value_r)
-{
-	const char *const *args;
-	unsigned int i, count;
-
-	args = array_get(&tmpl->args, &count);
-	i_assert((count % 2) == 0);
-	for (i = 0; i < count; i += 2) {
-		if (strcmp(args[i], key) == 0) {
-			*value_r = args[i+1];
-			array_delete(&tmpl->args, i, 2);
-			return TRUE;
-		}
-	}
-	return FALSE;
+	return ret;
 }
 
 bool passdb_template_is_empty(struct passdb_template *tmpl)
 {
-	return array_count(&tmpl->args) == 0;
+	return array_is_empty(&tmpl->args);
 }
 
-const char *const *passdb_template_get_args(struct passdb_template *tmpl, unsigned int *count_r)
+const char *const *passdb_template_get_args(struct passdb_template *tmpl,
+					    unsigned int *count_r)
 {
-	return array_get(&tmpl->args, count_r);
+	return array_get(&tmpl->keys, count_r);
 }
 
+void passdb_template_free(struct passdb_template **_tmpl)
+{
+	struct passdb_template *tmpl = *_tmpl;
+	if (tmpl == NULL)
+		return;
+	*_tmpl = NULL;
+
+	struct passdb_template_arg *arg;
+
+	array_foreach_modifiable(&tmpl->args, arg)
+		var_expand_program_free(&arg->program);
+}
diff --git a/src/auth/passdb-template.h b/src/auth/passdb-template.h
index b681c80ede..1aa8af5af7 100644
--- a/src/auth/passdb-template.h
+++ b/src/auth/passdb-template.h
@@ -7,10 +7,10 @@ struct passdb_template *passdb_template_build(pool_t pool, const char *args);
 int passdb_template_export(struct passdb_template *tmpl,
 			   struct auth_request *auth_request,
 			   const char **error_r);
-bool passdb_template_remove(struct passdb_template *tmpl,
-			    const char *key, const char **value_r);
 bool passdb_template_is_empty(struct passdb_template *tmpl);
 
+void passdb_template_free(struct passdb_template **_tmpl);
+
 const char *const *passdb_template_get_args(struct passdb_template *tmpl, unsigned int *count_r);
 
 #endif
diff --git a/src/auth/test-auth-cache.c b/src/auth/test-auth-cache.c
index 6c862491f9..c1075bdb62 100644
--- a/src/auth/test-auth-cache.c
+++ b/src/auth/test-auth-cache.c
@@ -9,14 +9,14 @@
 const struct var_expand_table
 auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT + 1] = {
 	/* these 3 must be in this order */
-	{ 'u', NULL, "user" },
-	{ 'n', NULL, "username" },
-	{ 'd', NULL, "domain" },
+	{ .key = "user", .value = NULL },
+	{ .key = "username", .value = NULL },
+	{ .key = "domain", .value = NULL },
 
-	{ 'a', NULL, NULL },
-	{ '\0', NULL, "longb" },
-	{ 'c', NULL, "longc" },
-	{ '\0', NULL, NULL }
+	{ .key = "a", .value = NULL },
+	{ .key = "b", .value = NULL },
+	{ .key = "c", .value = NULL },
+	VAR_EXPAND_TABLE_END
 };
 
 struct event *auth_event;
@@ -24,21 +24,48 @@ struct event *auth_event;
 struct var_expand_table *
 auth_request_get_var_expand_table_full(const struct auth_request *auth_request ATTR_UNUSED,
 				       const char *username ATTR_UNUSED,
-				       auth_request_escape_func_t *escape_func ATTR_UNUSED,
 				       unsigned int *count ATTR_UNUSED)
 {
 	i_unreached();
 }
 
+static int mock_get_passdb(const char *key, const char **value_r,
+			   void *context ATTR_UNUSED, const char **error_r)
+{
+	if (strcmp(key, "pfield") == 0) {
+		*value_r = "pvalue";
+		return 0;
+	}
+	*error_r = "No such key";
+	return -1;
+}
+
+static int mock_get_userdb(const char *key, const char **value_r,
+			   void *context ATTR_UNUSED, const char **error_r)
+{
+	if (strcmp(key, "ufield") == 0) {
+		*value_r = "uvalue";
+		return 0;
+	}
+	*error_r = "No such key";
+	return -1;
+}
+
 int auth_request_var_expand_with_table(string_t *dest, const char *str,
 				       const struct auth_request *auth_request ATTR_UNUSED,
 				       const struct var_expand_table *table ATTR_UNUSED,
 				       auth_request_escape_func_t *escape_func ATTR_UNUSED,
 				       const char **error_r ATTR_UNUSED)
 {
-	return var_expand_with_table(dest, str,
-				     auth_request_var_expand_static_tab,
-				     error_r);
+	const struct var_expand_params params = {
+		.table = auth_request_var_expand_static_tab,
+		.providers = (const struct var_expand_provider[]) {
+			{ .key = "passdb", .func = mock_get_passdb },
+			{ .key = "userdb", .func = mock_get_userdb },
+			VAR_EXPAND_TABLE_END
+		},
+	};
+	return var_expand_new(dest, str, &params, error_r);
 }
 
 static void test_auth_cache_parse_key(void)
@@ -46,22 +73,24 @@ static void test_auth_cache_parse_key(void)
 	static const struct {
 		const char *in, *out;
 	} tests[] = {
-		{ "%n@%d", "%u" },
-		{ "%{username}@%{domain}", "%u" },
-		{ "%n%d%u", "%u" },
-		{ "%n", "%n" },
-		{ "%d", "%d" },
-		{ "%a%b%u", "%u\t%a\t%b" },
+		{ "%{username}@%{domain}", "%{user}" },
+		{ "%{username}@%{domain}", "%{user}" },
+		{ "%{username}%{domain}%{user}", "%{user}" },
+		{ "%{username}", "%{username}" },
+		{ "%{domain}", "%{domain}" },
+		{ "%{a}%{b}%{user}", "%{user}\t%{a}\t%{b}" },
 
-		{ "foo%5.5Mabar", "%a" },
-		{ "foo%5.5M{longb}bar", "%{longb}" },
-		{ "foo%5.5Mcbar", "%c" },
-		{ "foo%5.5M{longc}bar", "%c" },
-		{ "%a%b", "%a\t%b" },
-		{ "%a%{longb}%a", "%a\t%{longb}" },
-		{ "%{longc}%c", "%c" },
-		{ "%c%a%{longc}%c", "%a\t%c" },
-		{ "%a%{env:foo}%{env:foo}%a", "%a\t%{env:foo}\t%{env:foo}" }
+		{ "foo%{a | substr(5, 5) }bar", "%{a}" },
+		{ "foo%{b | substr(5, 5) }bar", "%{b}" },
+		{ "foo%{c | substr(5, 5) }bar", "%{c}" },
+		{ "%{a}%{b}", "%{a}\t%{b}" },
+		/* test that passdb/userdb works */
+		{
+			"%{a}%{passdb:pfield}%{userdb:ufield}",
+			"%{a}\t%{passdb:pfield}\t%{userdb:ufield}"
+		},
+		/* test that other providers are dropped */
+		{ "%{a}%{provider:user}", "%{a}" },
 	};
 	const char *cache_key;
 	unsigned int i;
@@ -71,7 +100,7 @@ static void test_auth_cache_parse_key(void)
 	for (i = 0; i < N_ELEMENTS(tests); i++) {
 		cache_key = auth_cache_parse_key(pool_datastack_create(),
 						 tests[i].in);
-		test_assert(strcmp(cache_key, tests[i].out) == 0);
+		test_assert_strcmp_idx(cache_key, tests[i].out, i);
 	}
 	test_end();
 }
diff --git a/src/auth/test-auth-request-var-expand.c b/src/auth/test-auth-request-var-expand.c
index f8e074f03c..9cf26397f3 100644
--- a/src/auth/test-auth-request-var-expand.c
+++ b/src/auth/test-auth-request-var-expand.c
@@ -6,6 +6,7 @@
 #include "passdb.h"
 #include "userdb.h"
 #include "auth-request.h"
+#include "auth-request-var-expand.h"
 
 static struct passdb_module test_passdb = {
 	.id = 40
@@ -71,21 +72,18 @@ test_escape(const char *string, const struct auth_request *request)
 
 static bool test_empty_request(string_t *str, const char *input)
 {
-	const struct var_expand_table *tab =
-		auth_request_get_var_expand_table(&empty_test_request, NULL);
+	const struct var_expand_params params = {
+		.table = auth_request_get_var_expand_table(&empty_test_request),
+	};
 	const char *error;
 
 	str_truncate(str, 0);
-	test_assert(var_expand_with_table(str, input, tab, &error) == 1);
+	test_assert(var_expand_new(str, input, &params, &error) == 0);
 	return strspn(str_c(str), "\n0") == str_len(str);
 }
 
-static void test_auth_request_var_expand_shortlong(void)
+static void test_auth_request_var_expand_keys(void)
 {
-	/* %{protocol} has no short option */
-	static const char *test_input_short =
-		"%u\n%n\n%d\n%{protocol}\n%h\n%l\n%r\n%p\n%w\n%m\n%c\n"
-		"%a\n%b\n%k\n";
 	static const char *test_input_long =
 		"%{user}\n%{username}\n%{domain}\n%{protocol}\n%{home}\n"
 		"%{local_ip}\n%{remote_ip}\n"
@@ -97,22 +95,21 @@ static void test_auth_request_var_expand_shortlong(void)
 		"7.91.205.21\n73.150.2.210\n"
 		"54321\n+password\n+mech\nsecured\n"
 		"21\n210\nvalid\n";
-	const struct var_expand_table *tab;
 	string_t *str = t_str_new(256);
 	const char *error;
 
-	test_begin("auth request var expand short and long");
+	test_begin("auth request var expand");
 
-	tab = auth_request_get_var_expand_table(&test_request, test_escape);
-	test_assert(var_expand_with_table(str, test_input_short, tab, &error) == 1);
-	test_assert(strcmp(str_c(str), test_output) == 0);
+	const struct var_expand_params params = {
+		.table = auth_request_get_var_expand_table(&test_request),
+		.escape_func = (var_expand_escape_func_t *)test_escape,
+		.escape_context = &test_request,
+	};
 
-	str_truncate(str, 0);
-	test_assert(var_expand_with_table(str, test_input_long, tab, &error) == 1);
-	test_assert(strcmp(str_c(str), test_output) == 0);
+	test_assert(var_expand_new(str, test_input_long, &params, &error) == 0);
+	test_assert_strcmp(str_c(str), test_output);
 
 	/* test with empty input that it won't crash */
-	test_assert(test_empty_request(str, test_input_short));
 	test_assert(test_empty_request(str, test_input_long));
 
 	test_end();
@@ -120,7 +117,7 @@ static void test_auth_request_var_expand_shortlong(void)
 
 static void test_auth_request_var_expand_flags(void)
 {
-	static const char *test_input = "%!\n%{secured}\n%{cert}\n";
+	static const char *test_input = "%{id}\n%{secured}\n%{cert}\n";
 	string_t *str = t_str_new(10);
 	const char *error;
 
@@ -129,20 +126,23 @@ static void test_auth_request_var_expand_flags(void)
 	test_request.userdb_lookup = FALSE;
 	test_request.fields.conn_secured = AUTH_REQUEST_CONN_SECURED_NONE;
 	test_request.fields.valid_client_cert = FALSE;
-	test_assert(var_expand_with_table(str, test_input,
-		auth_request_get_var_expand_table(&test_request, test_escape),
-		&error) == 1);
-	test_assert(strcmp(str_c(str), "40\n\n\n") == 0);
+
+	struct var_expand_params params = {
+		.table = auth_request_get_var_expand_table(&test_request),
+		.escape_func = (var_expand_escape_func_t *)test_escape,
+		.escape_context = &test_request
+	};
+	test_assert(var_expand_new(str, test_input, &params, &error) == 0);
+	test_assert_strcmp(str_c(str), "40\n\n\n");
 
 	test_request.userdb_lookup = TRUE;
 	test_request.fields.conn_secured = AUTH_REQUEST_CONN_SECURED;
 	test_request.fields.valid_client_cert = TRUE;
+	params.table = auth_request_get_var_expand_table(&test_request);
 
 	str_truncate(str, 0);
-	test_assert(var_expand_with_table(str, test_input,
-		auth_request_get_var_expand_table(&test_request, test_escape),
-		&error) == 1);
-	test_assert(strcmp(str_c(str), "41\nsecured\nvalid\n") == 0);
+	test_assert(var_expand_new(str, test_input, &params, &error) == 0);
+	test_assert_strcmp(str_c(str), "41\nsecured\nvalid\n");
 
 	test_assert(test_empty_request(str, test_input));
 	test_end();
@@ -167,10 +167,14 @@ static void test_auth_request_var_expand_long(void)
 
 	test_begin("auth request var expand long-only");
 
-	test_assert(var_expand_with_table(str, test_input,
-		auth_request_get_var_expand_table(&test_request, test_escape),
-		&error) == 1);
-	test_assert(strcmp(str_c(str), test_output) == 0);
+	const struct var_expand_params params = {
+		.table = auth_request_get_var_expand_table(&test_request),
+		.escape_func = (var_expand_escape_func_t *)test_escape,
+		.escape_context = &test_request,
+	};
+
+	test_assert(var_expand_new(str, test_input, &params, &error) == 0);
+	test_assert_strcmp(str_c(str), test_output);
 
 	test_assert(test_empty_request(str, test_input));
 	test_end();
@@ -194,10 +198,13 @@ static void test_auth_request_var_expand_usernames(void)
 	test_begin("auth request var expand usernames");
 	for (i = 0; i < N_ELEMENTS(tests); i++) {
 		test_request.fields.user = t_strdup_noconst(tests[i].username);
+		const struct var_expand_params params = {
+			.table = auth_request_get_var_expand_table(&test_request),
+			.escape_func = (var_expand_escape_func_t *)test_escape,
+			.escape_context = &test_request,
+		};
 		str_truncate(str, 0);
-		test_assert(var_expand_with_table(str, test_input,
-			auth_request_get_var_expand_table(&test_request, test_escape),
-			&error) == 1);
+		test_assert(var_expand_new(str, test_input, &params, &error) == 0);
 		test_assert_idx(strcmp(str_c(str), tests[i].output) == 0, i);
 	}
 	test_request.fields.user = default_test_request.fields.user;
@@ -222,21 +229,20 @@ static void test_auth_request_var_expand_funcs(void)
 	auth_fields_add(test_request.fields.userdb_reply, "ukey2", "", 0);
 
 	test_assert(t_auth_request_var_expand(
-			"%{passdb:pkey1}\n%{passdb:pkey1:default1}\n"
-			"%{passdb:pkey2}\n%{passdb:pkey2:default2}\n"
-			"%{passdb:pkey3}\n%{passdb:pkey3:default3}\n"
-			"%{passdb:ukey1}\n%{passdb:ukey1:default4}\n",
+			"%{passdb:pkey1}\n%{passdb:pkey1 | default('default1')}\n"
+			"%{passdb:pkey2}\n%{passdb:pkey2 | default('default2')}\n"
+			"%{passdb:pkey3|default}\n%{passdb:pkey3 | default('default3')}\n"
+			"%{passdb:ukey1|default}\n%{passdb:ukey1 | default('default4')}\n",
 			&test_request, test_escape, &value, &error) == 1);
-	test_assert(strcmp(value, "+pval1\n+pval1\n\n\n\ndefault3\n\ndefault4\n") == 0);
+	test_assert_strcmp(value, "+pval1\n+pval1\n\ndefault2\n\ndefault3\n\ndefault4\n");
 
 	test_assert(t_auth_request_var_expand(
-			"%{userdb:ukey1}\n%{userdb:ukey1:default1}\n"
-			"%{userdb:ukey2}\n%{userdb:ukey2:default2}\n"
-			"%{userdb:ukey3}\n%{userdb:ukey3:default3}\n"
-			"%{userdb:pkey1}\n%{userdb:pkey1:default4}\n",
+			"%{userdb:ukey1}\n%{userdb:ukey1 | default('default1')}\n"
+			"%{userdb:ukey2}\n%{userdb:ukey2 | default('default2')}\n"
+			"%{userdb:ukey3|default}\n%{userdb:ukey3 | default('default3')}\n"
+			"%{userdb:pkey1|default}\n%{userdb:pkey1 | default('default4')}\n",
 			&test_request, test_escape, &value, &error) == 1);
-	test_assert(strcmp(value, "+uval1\n+uval1\n\n\n\ndefault3\n\ndefault4\n") == 0);
-
+	test_assert_strcmp(value, "+uval1\n+uval1\n\ndefault2\n\ndefault3\n\ndefault4\n");
 	pool_unref(&pool);
 	test_end();
 }
@@ -250,7 +256,7 @@ void test_auth_request_var_expand(void)
 
 	test_request = default_test_request;
 
-	test_auth_request_var_expand_shortlong();
+	test_auth_request_var_expand_keys();
 	test_auth_request_var_expand_flags();
 	test_auth_request_var_expand_long();
 	test_auth_request_var_expand_usernames();
diff --git a/src/auth/test-auth.h b/src/auth/test-auth.h
index ff72073e3d..517347a7c7 100644
--- a/src/auth/test-auth.h
+++ b/src/auth/test-auth.h
@@ -19,8 +19,6 @@ struct auth_passdb *passdb_mock(void);
 void passdb_mock_mod_init(void);
 void passdb_mock_mod_deinit(void);
 
-void test_db_ldap_field_multi_expand_parse_data(void);
-
 void test_auth_init(void);
 void test_auth_deinit(void);
 
diff --git a/src/auth/test-db-ldap.c b/src/auth/test-db-ldap.c
deleted file mode 100644
index ffae3bb00b..0000000000
--- a/src/auth/test-db-ldap.c
+++ /dev/null
@@ -1,51 +0,0 @@
-/* Copyright (c) 2023 Dovecot authors, see the included COPYING file */
-
-#include "lib.h"
-#include "test-common.h"
-#include "test-auth.h"
-#if defined(BUILTIN_LDAP) || defined(PLUGIN_BUILD)
-
-#include "db-ldap.h"
-#include <stdio.h>
-
-void test_db_ldap_field_multi_expand_parse_data(void)
-{
-	struct vectors {
-		const char *inp;
-		const char *field;
-		const char *sep;
-		const char *defl;
-	} vectors[] = {
-		{.inp="",       .field="",  .sep=" ", .defl="" },
-		{.inp="f",      .field="f", .sep=" ", .defl="" },
-		{.inp="f:",     .field="f", .sep=" ", .defl="" },
-		{.inp="f::",    .field="f", .sep=":", .defl="" },
-		{.inp="f:::",   .field="f", .sep=":", .defl="" },
-		{.inp="f:s",    .field="f", .sep="s", .defl="" },
-		{.inp="f:s:",   .field="f", .sep="s", .defl="" },
-		{.inp="f:s::",  .field="f", .sep="s", .defl=":" },
-		{.inp="f::d",   .field="f", .sep=" ", .defl="d" },
-		{.inp="f:::d",  .field="f", .sep=":", .defl="d" },
-		{.inp="f::d:",  .field="f", .sep=" ", .defl="d:" },
-		{.inp="f:::d:", .field="f", .sep=":", .defl="d:" },
-		{}
-	};
-
-	test_begin("db ldap field multi expand parse data");
-	unsigned int index = 0;
-	for (struct vectors *vector = vectors; vector->inp != NULL; vector++, index++) {
-		const char *field = NULL;
-		const char *sep   = NULL;
-		const char *defl  = NULL;
-
-		db_ldap_field_multi_expand_parse_data(
-			vector->inp, &field, &sep, &defl);
-
-		test_assert_strcmp_idx(vector->field, field, index);
-		test_assert_strcmp_idx(vector->sep,   sep,   index);
-		test_assert_strcmp_idx(vector->defl,  defl,  index);
-	}
-	test_end();
-}
-
-#endif
diff --git a/src/auth/test-main.c b/src/auth/test-main.c
index 7c23ae173e..1312fedc5b 100644
--- a/src/auth/test-main.c
+++ b/src/auth/test-main.c
@@ -18,9 +18,6 @@ int main(int argc, char *argv[])
 		MASTER_SERVICE_FLAG_DONT_SEND_STATS;
 	int ret;
 	static const struct named_test test_functions[] = {
-#if defined(BUILTIN_LDAP) || defined(PLUGIN_BUILD)
-		TEST_NAMED(test_db_ldap_field_multi_expand_parse_data)
-#endif
 		TEST_NAMED(test_auth_request_var_expand)
 		TEST_NAMED(test_auth_request_fields)
 		TEST_NAMED(test_username_filter)
diff --git a/src/auth/userdb-ldap.c b/src/auth/userdb-ldap.c
index 8be21826f4..4b7c9e784c 100644
--- a/src/auth/userdb-ldap.c
+++ b/src/auth/userdb-ldap.c
@@ -185,8 +185,8 @@ static void userdb_ldap_iterate_callback(struct ldap_connection *conn,
 	};
 
 	struct var_expand_params params = {
-		.func_table = db_ldap_field_expand_fn_table,
-		.func_context = &fctx
+		.providers = db_ldap_field_expand_fn_table,
+		.context = &fctx
 	};
 
 	struct event *event = event_create(authdb_event(urequest->request.request.auth_request));
diff --git a/src/auth/userdb-passwd-file.c b/src/auth/userdb-passwd-file.c
index 4f6c9d8db8..f6024fa407 100644
--- a/src/auth/userdb-passwd-file.c
+++ b/src/auth/userdb-passwd-file.c
@@ -38,7 +38,7 @@ passwd_file_add_extra_fields(struct auth_request *request,
 	unsigned int i;
 	int ret = 0;
 
-	table = auth_request_get_var_expand_table(request, NULL);
+	table = auth_request_get_var_expand_table(request);
 
 	for (i = 0; fields[i] != NULL; i++) {
 		if (!str_begins(fields[i], "userdb_", &key))
@@ -64,7 +64,8 @@ passwd_file_add_extra_fields(struct auth_request *request,
 			auth_request_set_userdb_field(request, key, value);
 		auth_fields_add(pwd_fields, key, value, 0);
 	}
-	if (ret == 0 && auth_request_set_userdb_fields(request, pwd_fields) < 0)
+	if (ret == 0 && auth_request_set_userdb_fields_ex(request, pwd_fields,
+							  db_passwd_file_var_expand_fn) < 0)
 		ret = -1;
 	return ret;
 }
diff --git a/src/auth/userdb-prefetch.c b/src/auth/userdb-prefetch.c
index e2c6d4604e..735f559dce 100644
--- a/src/auth/userdb-prefetch.c
+++ b/src/auth/userdb-prefetch.c
@@ -6,7 +6,6 @@
 #ifdef USERDB_PREFETCH
 
 #include "str.h"
-#include "var-expand.h"
 
 
 static void prefetch_lookup(struct auth_request *auth_request,
diff --git a/src/auth/userdb-template.c b/src/auth/userdb-template.c
index a1e3011f9e..c95eaf8c40 100644
--- a/src/auth/userdb-template.c
+++ b/src/auth/userdb-template.c
@@ -6,60 +6,74 @@
 #include "userdb.h"
 #include "userdb-template.h"
 
+struct userdb_template_arg {
+	const char *key;
+	struct var_expand_program *program;
+};
+
 struct userdb_template {
-	ARRAY(const char *) args;
+	ARRAY(struct userdb_template_arg) args;
+	ARRAY_TYPE(const_string) keys;
 };
 
 struct userdb_template *
 userdb_template_build(pool_t pool, const char *userdb_name, const char *args)
 {
 	struct userdb_template *tmpl;
-	const char *const *tmp, *key, *value, *nonull_value;
+	const char *const *tmp;
 	uid_t uid;
 	gid_t gid;
 
 	tmpl = p_new(pool, struct userdb_template, 1);
 
 	tmp = t_strsplit_spaces(args, " ");
-	p_array_init(&tmpl->args, pool, str_array_length(tmp));
+	p_array_init(&tmpl->args, pool, str_array_length(tmp) / 2);
+	p_array_init(&tmpl->keys, pool, str_array_length(tmp) / 2);
 
 	for (; *tmp != NULL; tmp++) {
-		value = strchr(*tmp, '=');
-		if (value == NULL)
-			key = *tmp;
-		else
-			key = t_strdup_until(*tmp, value++);
+		const char *p = strchr(*tmp, '=');
+		const char *kp;
+		const char *error;
 
+		if (p == NULL)
+			kp = *tmp;
+		else
+			kp = t_strdup_until(*tmp, p++);
 
-		if (*key == '\0')
+		if (*kp == '\0')
 			i_fatal("Invalid userdb template %s - key must not be empty",
 				args);
 
-		nonull_value = value == NULL ? "" : value;
+		char *key = p_strdup(pool, kp);
+		const char *nonull_value = p == NULL ? "" : p;
 		if (strcasecmp(key, "uid") == 0) {
 			uid = userdb_parse_uid(NULL, nonull_value);
 			if (uid == (uid_t)-1) {
 				i_fatal("%s userdb: Invalid uid: %s",
 					userdb_name, nonull_value);
 			}
-			value = dec2str(uid);
+			p = dec2str(uid);
 		} else if (strcasecmp(key, "gid") == 0) {
 			gid = userdb_parse_gid(NULL, nonull_value);
 			if (gid == (gid_t)-1) {
 				i_fatal("%s userdb: Invalid gid: %s",
 					userdb_name, nonull_value);
 			}
-			value = dec2str(gid);
-		} else if (*key == '\0') {
+			p = dec2str(gid);
+		} else if (*kp == '\0') {
 			i_fatal("%s userdb: Empty key (=%s)",
 				userdb_name, nonull_value);
 		}
-		key = p_strdup(pool, key);
-		value = p_strdup(pool, value);
-
-		array_push_back(&tmpl->args, &key);
-		array_push_back(&tmpl->args, &value);
+		struct var_expand_program *prog;
+		if (var_expand_program_create(p, &prog, &error) < 0)
+			i_fatal("Invalid userdb template value %s: %s", p, error);
+
+		struct userdb_template_arg *arg = array_append_space(&tmpl->args);
+		arg->key = key;
+		arg->program = prog;
+		array_push_back(&tmpl->keys, &arg->key);
 	}
+
 	return tmpl;
 }
 
@@ -67,58 +81,53 @@ int userdb_template_export(struct userdb_template *tmpl,
 			   struct auth_request *auth_request,
 			   const char **error_r)
 {
-        const struct var_expand_table *table;
+	const struct userdb_template_arg *arg;
 	string_t *str;
-	const char *const *args, *value;
-	unsigned int i, count;
+	int ret = 0;
 
 	if (userdb_template_is_empty(tmpl))
 		return 0;
 
+	const struct var_expand_params params = {
+		.table = auth_request_get_var_expand_table(auth_request),
+		.providers = auth_request_var_expand_providers,
+		.context = auth_request,
+	};
+
 	str = t_str_new(256);
-	table = auth_request_get_var_expand_table(auth_request, NULL);
-
-	args = array_get(&tmpl->args, &count);
-	i_assert((count % 2) == 0);
-	for (i = 0; i < count; i += 2) {
-		if (args[i+1] == NULL)
-			value = "";
-		else {
-			str_truncate(str, 0);
-			if (auth_request_var_expand_with_table(str, args[i+1],
-					auth_request, table, NULL, error_r) <= 0)
-				return -1;
-			value = str_c(str);
-		}
-		auth_request_set_userdb_field(auth_request, args[i], value);
+
+	array_foreach(&tmpl->args, arg) {
+		str_truncate(str, 0);
+		ret = var_expand_program_execute(str, arg->program, &params,
+						 error_r);
+		if (ret < 0)
+			break;
+		auth_request_set_userdb_field(auth_request, arg->key, str_c(str));
 	}
-	return 0;
+
+	return ret;
 }
 
-bool userdb_template_remove(struct userdb_template *tmpl,
-			    const char *key, const char **value_r)
+bool userdb_template_is_empty(struct userdb_template *tmpl)
 {
-	const char *const *args;
-	unsigned int i, count;
-
-	args = array_get(&tmpl->args, &count);
-	i_assert((count % 2) == 0);
-	for (i = 0; i < count; i += 2) {
-		if (strcmp(args[i], key) == 0) {
-			*value_r = args[i+1];
-			array_delete(&tmpl->args, i, 2);
-			return TRUE;
-		}
-	}
-	return FALSE;
+	return array_is_empty(&tmpl->args);
 }
 
-bool userdb_template_is_empty(struct userdb_template *tmpl)
+const char *const *userdb_template_get_args(struct userdb_template *tmpl,
+					    unsigned int *count_r)
 {
-	return array_count(&tmpl->args) == 0;
+	return array_get(&tmpl->keys, count_r);
 }
 
-const char *const *userdb_template_get_args(struct userdb_template *tmpl, unsigned int *count_r)
+void userdb_template_free(struct userdb_template **_tmpl)
 {
-	return array_get(&tmpl->args, count_r);
+	struct userdb_template *tmpl = *_tmpl;
+	if (tmpl == NULL)
+		return;
+	*_tmpl = NULL;
+
+	struct userdb_template_arg *arg;
+
+	array_foreach_modifiable(&tmpl->args, arg)
+		var_expand_program_free(&arg->program);
 }
diff --git a/src/auth/userdb-template.h b/src/auth/userdb-template.h
index a146a87e96..26f3613ca6 100644
--- a/src/auth/userdb-template.h
+++ b/src/auth/userdb-template.h
@@ -11,5 +11,6 @@ bool userdb_template_remove(struct userdb_template *tmpl,
 bool userdb_template_is_empty(struct userdb_template *tmpl);
 const char *const *userdb_template_get_args(struct userdb_template *tmpl,
 					    unsigned int *count_r);
+void userdb_template_free(struct userdb_template **_tmpl);
 
 #endif