From: Vladimír Čunát Date: Mon, 17 Aug 2020 17:15:04 +0000 (+0200) Subject: utils/cache_gc: avoid too long RO transactions X-Git-Tag: v5.1.3~1^2~19 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c651fbf24017f26435b86e69e9ce73c7f5976b97;p=thirdparty%2Fknot-resolver.git utils/cache_gc: avoid too long RO transactions Until now the analyzing pass over full DB was taking place in a single RO transaction. For an unknown reason this caused kresd processes to get MDB_MAP_FULL from mdb_put(), even though clearly there were plenty free pages at that point. Basic experiments show that 1k steps are OK and 10k steps are not. --- diff --git a/utils/cache_gc/db.c b/utils/cache_gc/db.c index c70af7ac4..2da8136cf 100644 --- a/utils/cache_gc/db.c +++ b/utils/cache_gc/db.c @@ -158,7 +158,8 @@ static const struct entry_h *val2entry(const knot_db_val_t val, uint16_t ktype) return NULL; } -int kr_gc_cache_iter(knot_db_t * knot_db, kr_gc_iter_callback callback, void *ctx) +int kr_gc_cache_iter(knot_db_t * knot_db, const kr_cache_gc_cfg_t *cfg, + kr_gc_iter_callback callback, void *ctx) { #ifdef DEBUG unsigned int counter_iter = 0; @@ -185,6 +186,7 @@ int kr_gc_cache_iter(knot_db_t * knot_db, kr_gc_iter_callback callback, void *ct return KNOT_ERROR; } + int txn_steps = 0; while (it != NULL) { knot_db_val_t key = { 0 }, val = { 0 }; ret = api->iter_key(it, &key); @@ -199,10 +201,6 @@ int kr_gc_cache_iter(knot_db_t * knot_db, kr_gc_iter_callback callback, void *ct goto error; } -#ifdef DEBUG - counter_iter++; -#endif - info.entry_size = key.len + val.len; info.valid = false; const uint16_t *entry_type = @@ -226,6 +224,7 @@ int kr_gc_cache_iter(knot_db_t * knot_db, kr_gc_iter_callback callback, void *ct info.no_labels = entry_labels(&key, *entry_type); } #ifdef DEBUG + counter_iter++; counter_kr_consistent += info.valid; if (!entry_type || !entry) { // don't log fully consistent entries printf @@ -247,8 +246,34 @@ int kr_gc_cache_iter(knot_db_t * knot_db, kr_gc_iter_callback callback, void *ct return ret; } -skip: - it = api->iter_next(it); + skip: // Advance to the next GC item. + if (++txn_steps < cfg->ro_txn_items || !cfg->ro_txn_items/*unlimited*/) { + it = api->iter_next(it); + } else { + /* The transaction has been too long; let's reopen it. */ + txn_steps = 0; + uint8_t key_storage[key.len]; + memcpy(key_storage, key.data, key.len); + key.data = key_storage; + + api->iter_finish(it); + api->txn_abort(&txn); + + ret = api->txn_begin(knot_db, &txn, KNOT_DB_RDONLY); + if (ret != KNOT_EOK) { + printf("Error restarting DB transaction (%s).\n", + knot_strerror(ret)); + return ret; + } + it = api->iter_begin(&txn, KNOT_DB_NOOP); + if (it == NULL) { + printf("Error iterating database.\n"); + api->txn_abort(&txn); + return KNOT_ERROR; + } + it = api->iter_seek(it, &key, KNOT_DB_GEQ); + // NULL here means we'we reached the end + } } api->txn_abort(&txn); diff --git a/utils/cache_gc/db.h b/utils/cache_gc/db.h index 5167d75a5..74d876359 100644 --- a/utils/cache_gc/db.h +++ b/utils/cache_gc/db.h @@ -16,6 +16,7 @@ void kr_gc_cache_close(struct kr_cache *kres_db, knot_db_t * knot_db); typedef int (*kr_gc_iter_callback)(const knot_db_val_t * key, gc_record_info_t * info, void *ctx); -int kr_gc_cache_iter(knot_db_t * knot_db, kr_gc_iter_callback callback, void *ctx); +int kr_gc_cache_iter(knot_db_t * knot_db, const kr_cache_gc_cfg_t *cfg, + kr_gc_iter_callback callback, void *ctx); const uint16_t *kr_gc_key_consistent(knot_db_val_t key); diff --git a/utils/cache_gc/kr_cache_gc.c b/utils/cache_gc/kr_cache_gc.c index 787344d8c..71b96bbde 100644 --- a/utils/cache_gc/kr_cache_gc.c +++ b/utils/cache_gc/kr_cache_gc.c @@ -205,7 +205,7 @@ int kr_cache_gc(kr_cache_gc_cfg_t *cfg, kr_cache_gc_state_t **state) gc_timer_start(&timer_analyze); ctx_compute_categories_t cats = { { 0 } }; - ret = kr_gc_cache_iter(db, cb_compute_categories, &cats); + ret = kr_gc_cache_iter(db, cfg, cb_compute_categories, &cats); if (ret != KNOT_EOK) { kr_cache_gc_free_state(state); return ret; @@ -244,7 +244,7 @@ int kr_cache_gc(kr_cache_gc_cfg_t *cfg, kr_cache_gc_state_t **state) ctx_delete_categories_t to_del = { 0 }; to_del.cfg_temp_keys_space = cfg->temp_keys_space; to_del.limit_category = limit_category; - ret = kr_gc_cache_iter(db, cb_delete_categories, &to_del); + ret = kr_gc_cache_iter(db, cfg, cb_delete_categories, &to_del); if (ret != KNOT_EOK) { entry_dynarray_deep_free(&to_del.to_delete); kr_cache_gc_free_state(state); diff --git a/utils/cache_gc/kr_cache_gc.h b/utils/cache_gc/kr_cache_gc.h index ee360abbc..c64e99e99 100644 --- a/utils/cache_gc/kr_cache_gc.h +++ b/utils/cache_gc/kr_cache_gc.h @@ -21,6 +21,7 @@ typedef struct { size_t temp_keys_space; // maximum amount of temporary memory for copied keys in bytes (0 = unlimited) size_t rw_txn_items; // maximum number of deleted records per RW transaction (0 = unlimited) + size_t ro_txn_items; // maximum number of iterated records (RO transactions, 0 = unlimited) unsigned long rw_txn_duration; // maximum duration of RW transaction in usecs (0 = unlimited) unsigned long rw_txn_delay; // waiting time between two RW transactions in usecs diff --git a/utils/cache_gc/main.c b/utils/cache_gc/main.c index 31c9bf641..45a18765c 100644 --- a/utils/cache_gc/main.c +++ b/utils/cache_gc/main.c @@ -36,6 +36,7 @@ static void print_help() printf("Optional params:\n"); printf(" -d \n"); printf(" -l \n"); + printf(" -L \n"); printf(" -m \n"); printf(" -u \n"); printf(" -f \n"); @@ -71,13 +72,14 @@ int main(int argc, char *argv[]) signal(SIGINT, got_killed); kr_cache_gc_cfg_t cfg = { + .ro_txn_items = 200, .rw_txn_items = 100, .cache_max_usage = 80, .cache_to_be_freed = 10 }; int o; - while ((o = getopt(argc, argv, "hnc:d:l:m:u:f:w:t:")) != -1) { + while ((o = getopt(argc, argv, "hnc:d:l:L:m:u:f:w:t:")) != -1) { switch (o) { case 'c': cfg.cache_path = optarg; @@ -89,6 +91,9 @@ int main(int argc, char *argv[]) case 'l': cfg.rw_txn_items = get_nonneg_optarg(); break; + case 'L': + cfg.ro_txn_items = get_nonneg_optarg(); + break; case 'm': cfg.rw_txn_duration = get_nonneg_optarg(); break;