From: Gary Lockyer <gary@catalyst.net.nz>
Date: Mon, 1 Apr 2019 02:28:31 +0000 (+1300)
Subject: lib ldb key_value: set the cache size for re-indexing
X-Git-Tag: tdb-1.4.1~527
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0952f98e41558acdd2b9732a28f9469b2be4c757;p=thirdparty%2Fsamba.git

lib ldb key_value: set the cache size for re-indexing

Set the index cache size to the number of records in the databse when
reindexing.

This significantly improves reindex performance.  For a domain with
100,000 users the reindex times are reduced from 17 minutes to 45
seconds.

Signed-off-by: Gary Lockyer <gary@catalyst.net.nz>
Reviewed-by: Andrew Bartlett <abartlet@samba.org>
---

diff --git a/lib/ldb/ldb_key_value/ldb_kv_index.c b/lib/ldb/ldb_key_value/ldb_kv_index.c
index 003ed714df9..4101d4b986d 100644
--- a/lib/ldb/ldb_key_value/ldb_kv_index.c
+++ b/lib/ldb/ldb_key_value/ldb_kv_index.c
@@ -3092,6 +3092,7 @@ int ldb_kv_reindex(struct ldb_module *module)
 	    ldb_module_get_private(module), struct ldb_kv_private);
 	int ret;
 	struct ldb_kv_reindex_context ctx;
+	size_t index_cache_size = 0;
 
 	/*
 	 * Only triggered after a modification, but make clear we do
@@ -3112,7 +3113,16 @@ int ldb_kv_reindex(struct ldb_module *module)
 	 */
 	ldb_kv_index_transaction_cancel(module);
 
-	ret = ldb_kv_index_transaction_start(module, DEFAULT_INDEX_CACHE_SIZE);
+	/*
+	 * Calculate the size of the index cache that we'll need for
+	 * the re-index
+	 */
+	index_cache_size = ldb_kv->kv_ops->get_size(ldb_kv);
+	if (index_cache_size < DEFAULT_INDEX_CACHE_SIZE) {
+		index_cache_size = DEFAULT_INDEX_CACHE_SIZE;
+	}
+
+	ret = ldb_kv_index_transaction_start(module, index_cache_size);
 	if (ret != LDB_SUCCESS) {
 		return ret;
 	}
diff --git a/lib/ldb/tests/ldb_key_value_test.c b/lib/ldb/tests/ldb_key_value_test.c
index 213405f0734..c78b7fcb316 100644
--- a/lib/ldb/tests/ldb_key_value_test.c
+++ b/lib/ldb/tests/ldb_key_value_test.c
@@ -52,7 +52,6 @@
 #include <sys/wait.h>
 
 #include "ldb_key_value/ldb_kv.c"
-#include "ldb_key_value/ldb_kv_cache.c"
 #include "ldb_key_value/ldb_kv_index.c"
 #include "ldb_key_value/ldb_kv_search.c"
 
@@ -63,7 +62,18 @@
 #endif /* TEST_BE */
 
 #define NUM_RECS 1024
-
+int ldb_kv_cache_reload(struct ldb_module *module) {
+	return LDB_SUCCESS;
+}
+int ldb_kv_cache_load(struct ldb_module *module) {
+	return LDB_SUCCESS;
+}
+int ldb_kv_check_at_attributes_values(const struct ldb_val *value) {
+	return LDB_SUCCESS;
+}
+int ldb_kv_increase_sequence_number(struct ldb_module *module) {
+	return LDB_SUCCESS;
+}
 
 struct test_ctx {
 };
@@ -158,6 +168,65 @@ static void test_default_index_cache_size(void **state)
 	TALLOC_FREE(module);
 }
 
+static int db_size = 0;
+static size_t mock_get_size(struct ldb_kv_private *ldb_kv) {
+	return db_size;
+}
+
+static int mock_iterate(
+	struct ldb_kv_private *ldb_kv,
+	ldb_kv_traverse_fn fn,
+	void *ctx) {
+	return 1;
+}
+
+/*
+ * Test that the index cache is correctly sized by the re_index call
+ */
+static void test_reindex_cache_size(void **state)
+{
+	struct test_ctx *test_ctx = talloc_get_type_abort(
+		*state,
+		struct test_ctx);
+	struct ldb_module *module = NULL;
+	struct ldb_kv_private *ldb_kv = NULL;
+	int ret = LDB_SUCCESS;
+	const struct kv_db_ops ops = {
+		.iterate = mock_iterate,
+		.get_size = mock_get_size,
+	};
+
+	module = talloc_zero(test_ctx, struct ldb_module);
+	ldb_kv = talloc_zero(test_ctx, struct ldb_kv_private);
+	ldb_kv->kv_ops = &ops;
+	ldb_module_set_private(module, ldb_kv);
+
+	/*
+	 * Use a value less than the DEFAULT_INDEX_CACHE_SIZE
+	 * Should get the DEFAULT_INDEX_CACHE_SIZE
+	 */
+	db_size = DEFAULT_INDEX_CACHE_SIZE - 1;
+	ret = ldb_kv_reindex(module);
+	assert_int_equal(LDB_SUCCESS, ret);
+
+	assert_int_equal(
+		DEFAULT_INDEX_CACHE_SIZE,
+		tdb_hash_size(ldb_kv->idxptr->itdb));
+
+	/*
+	 * Use a value greater than the DEFAULT_INDEX_CACHE_SIZE
+	 * Should get the value specified.
+	 */
+	db_size = DEFAULT_INDEX_CACHE_SIZE + 1;
+	ret = ldb_kv_reindex(module);
+	assert_int_equal(LDB_SUCCESS, ret);
+
+	assert_int_equal(db_size, tdb_hash_size(ldb_kv->idxptr->itdb));
+
+	TALLOC_FREE(ldb_kv);
+	TALLOC_FREE(module);
+}
+
 int main(int argc, const char **argv)
 {
 	const struct CMUnitTest tests[] = {
@@ -169,6 +238,10 @@ int main(int argc, const char **argv)
 			test_default_index_cache_size,
 			setup,
 			teardown),
+		cmocka_unit_test_setup_teardown(
+			test_reindex_cache_size,
+			setup,
+			teardown),
 	};
 
 	return cmocka_run_group_tests(tests, NULL, NULL);