From: Marek Vavrusa Date: Sun, 15 May 2016 21:14:53 +0000 (-0700) Subject: lib: cache api v2, removed dep on libknot db.h X-Git-Tag: v1.0.0~14 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=e68c3a0a33d6ddd3f9112969a6271e2aca313481;p=thirdparty%2Fknot-resolver.git lib: cache api v2, removed dep on libknot db.h this change introduces new API for cache backends, that is a subset of knot_db_api_t from libknot with several cache-specific operations major changes are: * merged 'cachectl' module into 'cache' as it is 99% default-on and it simplifies things * not transaction oriented, transactions may be reused and cached for higher performance * scatter/gather API, this is important for latency and performance of non-local backends like Redis * faster and reliable cache clearing * cache-specific operations (prefix scan, ...) in the API not hacked in * simpler code for both backends and caller --- diff --git a/.travis.yml b/.travis.yml index cac31183c..0c669282c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ env: global: - PKG_CONFIG_PATH="${HOME}/.local/lib/pkgconfig" - PATH="${HOME}/.local/bin:/usr/local/bin:${PATH}" - - CFLAGS="-O2 -g -fno-omit-frame-pointer -DMP_FREELIST_SIZE=0" + - CFLAGS="-O2 -g -fno-omit-frame-pointer -DDEBUG" - LD_LIBRARY_PATH="${HOME}/.local/lib" - DYLD_LIBRARY_PATH="${HOME}/.local/lib" - MALLOC_CHECK_=3 diff --git a/config.mk b/config.mk index 103d4276a..2611c21dd 100644 --- a/config.mk +++ b/config.mk @@ -23,7 +23,7 @@ INSTALL := install # Flags BUILD_LDFLAGS += $(LDFLAGS) -BUILD_CFLAGS := $(CFLAGS) -std=c99 -D_GNU_SOURCE -Wno-unused -Wtype-limits -Wformat -Wformat-security -Wall -I$(abspath .) -I$(abspath lib/generic) -I$(abspath contrib) +BUILD_CFLAGS := $(CFLAGS) -std=c99 -D_GNU_SOURCE -Wno-unused -Wtype-limits -Wformat -Wformat-security -Wall -I$(abspath .) -I$(abspath lib/generic) -I$(abspath contrib) -I$(abspath contrib/lmdb) BUILD_CFLAGS += -DPACKAGE_VERSION="\"$(MAJOR).$(MINOR).$(PATCH)\"" -DPREFIX="\"$(PREFIX)\"" -DMODULEDIR="\"$(MODULEDIR)\"" -DETCDIR="\"$(ETCDIR)\"" ifeq (,$(findstring -O,$(CFLAGS))) BUILD_CFLAGS += -O2 diff --git a/contrib/contrib.mk b/contrib/contrib.mk index 5d924734a..b99c9ff88 100644 --- a/contrib/contrib.mk +++ b/contrib/contrib.mk @@ -8,4 +8,13 @@ contrib_SOURCES := \ contrib/base32hex.c contrib_CFLAGS := -fPIC contrib_TARGET := $(abspath contrib)/contrib$(AREXT) -$(eval $(call make_static,contrib,contrib)) + +# Use built-in LMDB if not found +ifneq ($(HAS_lmdb), yes) +contrib_SOURCES += contrib/lmdb/mdb.c \ + contrib/lmdb/midl.c +contrib_CFLAGS += -pthread +contrib_LIBS += -pthread +endif + +$(eval $(call make_static,contrib,contrib)) \ No newline at end of file diff --git a/daemon/README.rst b/daemon/README.rst index 5b9a812ad..888b03de9 100644 --- a/daemon/README.rst +++ b/daemon/README.rst @@ -169,7 +169,7 @@ Configuration example -- interfaces net = { '127.0.0.1', '::1' } -- load some modules - modules = { 'policy', 'cachectl' } + modules = { 'policy' } -- 10MB cache cache.size = 10*MB @@ -198,7 +198,6 @@ the modules use as the :ref:`input configuration `. .. code-block:: lua modules = { - cachectl = true, hints = '/etc/hosts' } @@ -274,7 +273,7 @@ Here's an example of an anonymous function with :func:`event.recurrent()`: -- every 5 minutes event.recurrent(5 * minute, function() - cachectl.prune() + cache.prune() end) Note that each scheduled event is identified by a number valid for the duration of the event, @@ -288,7 +287,7 @@ as a parameter, but it's not very useful as you don't have any *non-global* way local i = 0 -- pruning function return function(e) - cachectl.prune() + cache.prune() -- cancel event on 5th attempt i = i + 1 if i == 5 then @@ -609,7 +608,6 @@ The daemon provides an interface for dynamic loading of :ref:`daemon modules #include "lib/cache.h" +#include "lib/cdb.h" #include "daemon/bindings.h" #include "daemon/worker.h" @@ -349,13 +350,12 @@ int lib_net(lua_State *L) static int cache_backends(lua_State *L) { struct engine *engine = engine_luaget(L); - storage_registry_t *registry = &engine->storage_registry; lua_newtable(L); - for (unsigned i = 0; i < registry->len; ++i) { - struct storage_api *storage = ®istry->at[i]; - lua_pushboolean(L, storage->api() == engine->resolver.cache.api); - lua_setfield(L, -2, storage->prefix); + for (unsigned i = 0; i < engine->backends.len; ++i) { + const struct kr_cdb_api *api = engine->backends.at[i]; + lua_pushboolean(L, api == engine->resolver.cache.api); + lua_setfield(L, -2, api->name); } return 1; } @@ -364,20 +364,15 @@ static int cache_backends(lua_State *L) static int cache_count(lua_State *L) { struct engine *engine = engine_luaget(L); - const knot_db_api_t *storage = engine->resolver.cache.api; - - /* Fetch item count */ - struct kr_cache_txn txn; - int ret = kr_cache_txn_begin(&engine->resolver.cache, &txn, KNOT_DB_RDONLY); - if (ret != 0) { - format_error(L, kr_strerror(ret)); - lua_error(L); - } + const struct kr_cdb_api *api = engine->resolver.cache.api; /* First key is a version counter, omit it. */ - lua_pushinteger(L, storage->count(&txn.t) - 1); - kr_cache_txn_abort(&txn); - return 1; + struct kr_cache *cache = &engine->resolver.cache; + if (kr_cache_is_open(cache)) { + lua_pushinteger(L, api->count(cache->db) - 1); + return 1; + } + return 0; } /** Return cache statistics. */ @@ -394,27 +389,22 @@ static int cache_stats(lua_State *L) lua_setfield(L, -2, "insert"); lua_pushnumber(L, cache->stats.delete); lua_setfield(L, -2, "delete"); - lua_pushnumber(L, cache->stats.txn_read); - lua_setfield(L, -2, "txn_read"); - lua_pushnumber(L, cache->stats.txn_write); - lua_setfield(L, -2, "txn_write"); return 1; } -static struct storage_api *cache_select_storage(struct engine *engine, const char **conf) +static const struct kr_cdb_api *cache_select(struct engine *engine, const char **conf) { /* Return default backend */ - storage_registry_t *registry = &engine->storage_registry; - if (!*conf || !strstr(*conf, "://")) { - return ®istry->at[0]; + if (*conf == NULL || !strstr(*conf, "://")) { + return engine->backends.at[0]; } /* Find storage backend from config prefix */ - for (unsigned i = 0; i < registry->len; ++i) { - struct storage_api *storage = ®istry->at[i]; - if (strncmp(*conf, storage->prefix, strlen(storage->prefix)) == 0) { - *conf += strlen(storage->prefix); - return storage; + for (unsigned i = 0; i < engine->backends.len; ++i) { + const struct kr_cdb_api *api = engine->backends.at[i]; + if (strncmp(*conf, api->name, strlen(api->name)) == 0) { + *conf += strlen(api->name) + strlen("://"); + return api; } } @@ -436,8 +426,8 @@ static int cache_open(lua_State *L) unsigned cache_size = lua_tonumber(L, 1); const char *conf = n > 1 ? lua_tostring(L, 2) : NULL; const char *uri = conf; - struct storage_api *storage = cache_select_storage(engine, &conf); - if (!storage) { + const struct kr_cdb_api *api = cache_select(engine, &conf); + if (!api) { format_error(L, "unsupported cache backend"); lua_error(L); } @@ -446,9 +436,11 @@ static int cache_open(lua_State *L) kr_cache_close(&engine->resolver.cache); /* Reopen cache */ - void *storage_opts = storage->opts_create(conf, cache_size); - int ret = kr_cache_open(&engine->resolver.cache, storage->api(), storage_opts, engine->pool); - free(storage_opts); + struct kr_cdb_opts opts = { + (conf && strlen(conf)) ? conf : ".", + cache_size + }; + int ret = kr_cache_open(&engine->resolver.cache, api, &opts, engine->pool); if (ret != 0) { format_error(L, "can't open cache"); lua_error(L); @@ -481,6 +473,190 @@ static int cache_close(lua_State *L) return 1; } +/** @internal Prefix walk. */ +static int cache_prefixed(struct kr_cache *cache, const char *args, knot_db_val_t *results, int maxresults) +{ + /* Decode parameters */ + uint8_t namespace = 'R'; + char *extra = (char *)strchr(args, ' '); + if (extra != NULL) { + extra[0] = '\0'; + namespace = extra[1]; + } + + /* Convert to domain name */ + uint8_t buf[KNOT_DNAME_MAXLEN]; + if (!knot_dname_from_str(buf, args, sizeof(buf))) { + return kr_error(EINVAL); + } + + /* Start prefix search */ + return kr_cache_match(cache, namespace, buf, results, maxresults); +} + +/** @internal Delete iterated key. */ +static int cache_remove_prefix(struct kr_cache *cache, const char *args) +{ + /* Check if we can remove */ + if (!cache || !cache->api || !cache->api->remove) { + return kr_error(ENOSYS); + } + static knot_db_val_t result_set[1000]; + int ret = cache_prefixed(cache, args, result_set, 1000); + if (ret < 0) { + return ret; + } + /* Duplicate result set as we're going to remove it + * which will invalidate result set. */ + for (int i = 0; i < ret; ++i) { + void *dst = malloc(result_set[i].len); + if (!dst) { + return kr_error(ENOMEM); + } + memcpy(dst, result_set[i].data, result_set[i].len); + result_set[i].data = dst; + } + cache->api->remove(cache->db, result_set, ret); + /* Free keys */ + for (int i = 0; i < ret; ++i) { + free(result_set[i].data); + } + return ret; +} + +/** Prune expired/invalid records. */ +static int cache_prune(lua_State *L) +{ + /* Check parameters */ + int prune_max = UINT16_MAX; + int n = lua_gettop(L); + if (n >= 1 && lua_isnumber(L, 1)) { + prune_max = lua_tointeger(L, 1); + } + + struct engine *engine = engine_luaget(L); + struct kr_cache *cache = &engine->resolver.cache; + + /* Check if API supports pruning. */ + int ret = kr_error(ENOSYS); + if (cache->api->prune) { + ret = cache->api->prune(cache->db, prune_max); + } + /* Commit and format result. */ + if (ret < 0) { + format_error(L, kr_strerror(ret)); + lua_error(L); + } + lua_pushinteger(L, ret); + return 1; +} + +/** Clear all records. */ +static int cache_clear(lua_State *L) +{ + /* Check parameters */ + const char *args = NULL; + int n = lua_gettop(L); + if (n >= 1 && lua_isstring(L, 1)) { + args = lua_tostring(L, 1); + } + + /* Clear a sub-tree in cache. */ + struct engine *engine = engine_luaget(L); + struct kr_cache *cache = &engine->resolver.cache; + if (args && strlen(args) > 0) { + int ret = cache_remove_prefix(cache, args); + if (ret < 0) { + format_error(L, kr_strerror(ret)); + lua_error(L); + } + lua_pushinteger(L, ret); + return 1; + } + + /* Clear cache. */ + int ret = kr_cache_clear(cache); + if (ret < 0) { + format_error(L, kr_strerror(ret)); + lua_error(L); + } + + /* Clear reputation tables */ + lru_deinit(engine->resolver.cache_rtt); + lru_deinit(engine->resolver.cache_rep); + lru_init(engine->resolver.cache_rtt, LRU_RTT_SIZE); + lru_init(engine->resolver.cache_rep, LRU_REP_SIZE); + lua_pushboolean(L, true); + return 1; +} + +/** @internal Dump cache key into table on Lua stack. */ +static void cache_dump_key(lua_State *L, knot_db_val_t *key) +{ + char buf[KNOT_DNAME_MAXLEN]; + /* Extract type */ + uint16_t type = 0; + const char *endp = (const char *)key->data + key->len - sizeof(uint16_t); + memcpy(&type, endp, sizeof(uint16_t)); + endp -= 1; + /* Extract domain name */ + char *dst = buf; + const char *scan = endp - 1; + while (scan > (const char *)key->data) { + if (*scan == '\0') { + const size_t lblen = endp - scan - 1; + memcpy(dst, scan + 1, lblen); + dst += lblen; + *dst++ = '.'; + endp = scan; + } + --scan; + } + memcpy(dst, scan + 1, endp - scan); + /* If name typemap doesn't exist yet, create it */ + lua_getfield(L, -1, buf); + if (lua_isnil(L, -1)) { + lua_pop(L, 1); + lua_newtable(L); + } + /* Append to typemap */ + char type_buf[16] = { '\0' }; + knot_rrtype_to_string(type, type_buf, sizeof(type_buf)); + lua_pushboolean(L, true); + lua_setfield(L, -2, type_buf); + /* Set name typemap */ + lua_setfield(L, -2, buf); +} + +/** Query cached records. */ +static int cache_get(lua_State *L) +{ + /* Check parameters */ + int n = lua_gettop(L); + if (n < 1 || !lua_isstring(L, 1)) { + format_error(L, "expected 'cache.get(string key)'"); + lua_error(L); + } + + /* Clear a sub-tree in cache. */ + struct engine *engine = engine_luaget(L); + struct kr_cache *cache = &engine->resolver.cache; + const char *args = lua_tostring(L, 1); + /* Retrieve set of keys */ + static knot_db_val_t result_set[100]; + int ret = cache_prefixed(cache, args, result_set, 100); + if (ret < 0) { + format_error(L, kr_strerror(ret)); + lua_error(L); + } + /* Format output */ + lua_newtable(L); + for (int i = 0; i < ret; ++i) { + cache_dump_key(L, &result_set[i]); + } + return 1; +} + int lib_cache(lua_State *L) { static const luaL_Reg lib[] = { @@ -489,6 +665,9 @@ int lib_cache(lua_State *L) { "stats", cache_stats }, { "open", cache_open }, { "close", cache_close }, + { "prune", cache_prune }, + { "clear", cache_clear }, + { "get", cache_get }, { NULL, NULL } }; diff --git a/daemon/engine.c b/daemon/engine.c index 70fda1156..ac7d93710 100644 --- a/daemon/engine.c +++ b/daemon/engine.c @@ -21,8 +21,6 @@ #include #include #include -/* #include @todo Not supported (doesn't keep value copy) */ -#include #include #include "daemon/engine.h" @@ -31,6 +29,7 @@ #include "lib/nsrep.h" #include "lib/cache.h" #include "lib/defines.h" +#include "lib/cdb_lmdb.h" #include "lib/dnssec/ta.h" /** @internal Compatibility wrapper for Lua < 5.2 */ @@ -370,19 +369,6 @@ static int l_trampoline(lua_State *L) * Engine API. */ -/** @internal Make lmdb options. */ -void *knot_db_lmdb_mkopts(const char *conf, size_t maxsize) -{ - struct knot_db_lmdb_opts *opts = malloc(sizeof(*opts)); - if (opts) { - memset(opts, 0, sizeof(*opts)); - opts->path = (conf && strlen(conf)) ? conf : "."; - opts->mapsize = maxsize; - opts->flags.env = 0x80000 | 0x100000; /* MDB_WRITEMAP|MDB_MAPASYNC */ - } - return opts; -} - static int init_resolver(struct engine *engine) { /* Open resolution context */ @@ -415,12 +401,7 @@ static int init_resolver(struct engine *engine) engine_register(engine, "rrcache", NULL, NULL); engine_register(engine, "pktcache", NULL, NULL); - /* Initialize storage backends */ - struct storage_api lmdb = { - "lmdb://", knot_db_lmdb_api, knot_db_lmdb_mkopts - }; - - return array_push(engine->storage_registry, lmdb); + return array_push(engine->backends, kr_cdb_lmdb()); } static int init_state(struct engine *engine) @@ -538,7 +519,7 @@ void engine_deinit(struct engine *engine) /* Free data structures */ array_clear(engine->modules); - array_clear(engine->storage_registry); + array_clear(engine->backends); kr_ta_clear(&engine->resolver.trust_anchors); kr_ta_clear(&engine->resolver.negative_anchors); } diff --git a/daemon/engine.h b/daemon/engine.h index fde0c2aa9..136cd8f9f 100644 --- a/daemon/engine.h +++ b/daemon/engine.h @@ -45,22 +45,11 @@ struct lua_State; #include "lib/resolve.h" #include "daemon/network.h" -/** Cache storage backend. */ -struct storage_api { - const char *prefix; /**< Storage prefix, e.g. 'lmdb://' */ - const knot_db_api_t *(*api)(void); /**< Storage API implementation */ - void *(*opts_create)(const char *, size_t); /**< Storage options factory */ -}; - -/** @cond internal Array of cache backend options. */ -typedef array_t(struct storage_api) storage_registry_t; -/* @endcond */ - struct engine { struct kr_context resolver; struct network net; module_array_t modules; - storage_registry_t storage_registry; + array_t(const struct kr_cdb_api *) backends; knot_mm_t *pool; uv_timer_t *updater; struct lua_State *L; diff --git a/daemon/lua/sandbox.lua b/daemon/lua/sandbox.lua index d1ca36633..2398e8f55 100644 --- a/daemon/lua/sandbox.lua +++ b/daemon/lua/sandbox.lua @@ -72,7 +72,7 @@ setmetatable(modules, { modules.load(k) k = string.match(k, '%w+') local mod = _G[k] - local config = rawget(mod, 'config') + local config = mod and rawget(mod, 'config') if mod ~= nil and config ~= nil then if k ~= v then config(v) else config() @@ -83,8 +83,16 @@ setmetatable(modules, { }) -- Syntactic sugar for cache +-- `#cache -> cache.count()` +-- `cache[x] -> cache.get(x)` -- `cache.{size|storage} = value` setmetatable(cache, { + __len = function (t) + return t.count() + end, + __index = function (t, k) + return rawget(t, k) or (rawget(t, 'current_size') and t.get(k)) + end, __newindex = function (t,k,v) -- Defaults local storage = rawget(t, 'current_storage') diff --git a/doc/modules.rst b/doc/modules.rst index 0cad12aee..b552f61d2 100644 --- a/doc/modules.rst +++ b/doc/modules.rst @@ -17,7 +17,6 @@ Knot DNS Resolver modules .. include:: ../modules/kmemcached/README.rst .. include:: ../modules/redis/README.rst .. include:: ../modules/ketcd/README.rst -.. include:: ../modules/cachectl/README.rst .. include:: ../modules/tinyweb/README.rst .. include:: ../modules/dns64/README.rst .. include:: ../modules/renumber/README.rst diff --git a/etc/config.cluster b/etc/config.cluster index 5be5c923c..b8c9c2081 100644 --- a/etc/config.cluster +++ b/etc/config.cluster @@ -21,7 +21,6 @@ cache.size = 100 * GB -- Load Useful modules modules = { 'policy', -- Block queries to local zones/bad sites - 'cachectl', -- Cache control interface 'hints', -- Load /etc/hosts and allow custom root hints 'stats', -- Track internal statistics graphite = { -- Send statistics to local InfluxDB diff --git a/etc/config.isp b/etc/config.isp index 4d6e9910b..57b02d39d 100644 --- a/etc/config.isp +++ b/etc/config.isp @@ -19,7 +19,6 @@ cache.size = 4 * GB modules = { 'policy', -- Block queries to local zones/bad sites 'view', -- Views for certain clients - 'cachectl', -- Cache control interface 'hints', -- Load /etc/hosts and allow custom root hints 'stats', -- Track internal statistics graphite = { -- Send statistics to local InfluxDB diff --git a/etc/config.personal b/etc/config.personal index 12881b13d..782a8bdf3 100644 --- a/etc/config.personal +++ b/etc/config.personal @@ -15,7 +15,6 @@ trust_anchors.file = 'root.keys' -- Load Useful modules modules = { 'policy', -- Block queries to local zones/bad sites - 'cachectl', -- Cache control interface 'hints', -- Load /etc/hosts and allow custom root hints 'stats', -- Track internal statistics 'predict', -- Prefetch expiring/frequent records diff --git a/etc/config.splitview b/etc/config.splitview index ff26a09ad..f707b7d68 100644 --- a/etc/config.splitview +++ b/etc/config.splitview @@ -13,7 +13,6 @@ trust_anchors.file = 'root.keys' -- Load Useful modules modules = { 'policy', -- Block queries to local zones/bad sites - 'cachectl', -- Cache control interface 'hints', -- Load /etc/hosts and allow custom root hints 'stats', -- Track internal statistics graphite = { -- Send statistics to local InfluxDB diff --git a/lib/cache.c b/lib/cache.c index 194f24604..4a1327223 100644 --- a/lib/cache.c +++ b/lib/cache.c @@ -20,13 +20,14 @@ #include #include -#include #include #include #include #include +#include "contrib/cleanup.h" #include "lib/cache.h" +#include "lib/cdb_lmdb.h" #include "lib/defines.h" #include "lib/utils.h" @@ -35,32 +36,25 @@ /* Key size */ #define KEY_HSIZE (sizeof(uint8_t) + sizeof(uint16_t)) #define KEY_SIZE (KEY_HSIZE + KNOT_DNAME_MAXLEN) -#define txn_api(txn) ((txn)->owner->api) -#define txn_is_valid(txn) ((txn) && (txn)->owner && txn_api(txn)) +/* Shorthand for operations on cache backend */ +#define cache_isvalid(cache) ((cache) && (cache)->api && (cache)->db) +#define cache_op(cache, op, ...) (cache)->api->op((cache)->db, ## __VA_ARGS__) /** @internal Removes all records from cache. */ -static int cache_purge(struct kr_cache_txn *txn) +static inline int cache_purge(struct kr_cache *cache) { - int ret = kr_error(EINVAL); - if (txn_is_valid(txn)) { - txn->owner->stats.delete += 1; - ret = txn_api(txn)->clear(&txn->t); - } - return ret; + cache->stats.delete += 1; + return cache_op(cache, clear); } -/** @internal Check cache internal data version. Clear if it doesn't match. - * returns : EEXIST - cache data version matched. - * 0 - cache recreated, txn has to be committed. - * Otherwise - cache recreation fails. - */ -static int assert_right_version_txn(struct kr_cache_txn *txn) +/** @internal Open cache db transaction and check internal data version. */ +static int assert_right_version(struct kr_cache *cache) { /* Check cache ABI version */ knot_db_val_t key = { KEY_VERSION, 2 }; knot_db_val_t val = { NULL, 0 }; - int ret = txn_api(txn)->find(&txn->t, &key, &val, 0); + int ret = cache_op(cache, read, &key, &val, 1); if (ret == 0) { ret = kr_error(EEXIST); } else { @@ -68,101 +62,51 @@ static int assert_right_version_txn(struct kr_cache_txn *txn) * Version doesn't match. * Recreate cache and write version key. */ - ret = txn_api(txn)->count(&txn->t); + ret = cache_op(cache, count); if (ret != 0) { /* Non-empty cache, purge it. */ kr_log_info("[cache] purging cache\n"); - ret = cache_purge(txn); + ret = cache_purge(cache); } /* Either purged or empty. */ if (ret == 0) { - ret = txn_api(txn)->insert(&txn->t, &key, &val, 0); + ret = cache_op(cache, write, &key, &val, 1); } } return ret; } -/** @internal Open cache db transaction and check internal data version. */ -static void assert_right_version(struct kr_cache *cache) -{ - /* Check cache ABI version */ - struct kr_cache_txn txn; - int ret = kr_cache_txn_begin(cache, &txn, 0); - if (ret != 0) { - return; /* N/A, doesn't work. */ - } - ret = assert_right_version_txn(&txn); - if (ret == 0) { /* Cache recreated, commit. */ - kr_cache_txn_commit(&txn); - } else { - kr_cache_txn_abort(&txn); - } -} - -int kr_cache_open(struct kr_cache *cache, const knot_db_api_t *api, void *opts, knot_mm_t *mm) +int kr_cache_open(struct kr_cache *cache, const struct kr_cdb_api *api, struct kr_cdb_opts *opts, knot_mm_t *mm) { if (!cache) { return kr_error(EINVAL); } /* Open cache */ - cache->api = (api == NULL) ? knot_db_lmdb_api() : api; - int ret = cache->api->init(&cache->db, mm, opts); + if (!api) { + api = kr_cdb_lmdb(); + } + cache->api = api; + int ret = cache->api->open(&cache->db, opts, mm); if (ret != 0) { return ret; } memset(&cache->stats, 0, sizeof(cache->stats)); /* Check cache ABI version */ - assert_right_version(cache); - return kr_ok(); + (void) assert_right_version(cache); + return 0; } void kr_cache_close(struct kr_cache *cache) { - if (cache && cache->db) { - if (cache->api) { - cache->api->deinit(cache->db); - } + if (cache_isvalid(cache)) { + cache_op(cache, close); cache->db = NULL; } } -int kr_cache_txn_begin(struct kr_cache *cache, struct kr_cache_txn *txn, unsigned flags) +void kr_cache_sync(struct kr_cache *cache) { - if (!cache || !cache->db || !cache->api || !txn ) { - return kr_error(EINVAL); - } - /* Open new transaction */ - int ret = cache->api->txn_begin(cache->db, &txn->t, flags); - if (ret != 0) { - memset(txn, 0, sizeof(*txn)); - } else { - /* Count statistics */ - txn->owner = cache; - if (flags & KNOT_DB_RDONLY) { - cache->stats.txn_read += 1; - } else { - cache->stats.txn_write += 1; - } - } - return ret; -} - -int kr_cache_txn_commit(struct kr_cache_txn *txn) -{ - if (!txn_is_valid(txn)) { - return kr_error(EINVAL); - } - - int ret = txn_api(txn)->txn_commit(&txn->t); - if (ret != 0) { - kr_cache_txn_abort(txn); - } - return ret; -} - -void kr_cache_txn_abort(struct kr_cache_txn *txn) -{ - if (txn_is_valid(txn)) { - txn_api(txn)->txn_abort(&txn->t); + if (cache_isvalid(cache) && cache->api->sync) { + cache_op(cache, sync); } } @@ -185,19 +129,20 @@ static size_t cache_key(uint8_t *buf, uint8_t tag, const knot_dname_t *name, uin return name_len + KEY_HSIZE; } -static struct kr_cache_entry *lookup(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *name, uint16_t type) +static struct kr_cache_entry *lookup(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, uint16_t type) { - uint8_t keybuf[KEY_SIZE]; - size_t key_len = cache_key(keybuf, tag, name, type); - if (!txn_is_valid(txn) || !name) { + if (!name || !cache) { return NULL; } + uint8_t keybuf[KEY_SIZE]; + size_t key_len = cache_key(keybuf, tag, name, type); + /* Look up and return value */ knot_db_val_t key = { keybuf, key_len }; knot_db_val_t val = { NULL, 0 }; - int ret = txn_api(txn)->find(&txn->t, &key, &val, 0); - if (ret != KNOT_EOK) { + int ret = cache_op(cache, read, &key, &val, 1); + if (ret != 0) { return NULL; } @@ -224,16 +169,16 @@ static int check_lifetime(struct kr_cache_entry *found, uint32_t *timestamp) return kr_error(ESTALE); } -int kr_cache_peek(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *name, uint16_t type, +int kr_cache_peek(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, uint16_t type, struct kr_cache_entry **entry, uint32_t *timestamp) { - if (!txn_is_valid(txn) || !name || !entry) { + if (!cache_isvalid(cache) || !name || !entry) { return kr_error(EINVAL); } - struct kr_cache_entry *found = lookup(txn, tag, name, type); + struct kr_cache_entry *found = lookup(cache, tag, name, type); if (!found) { - txn->owner->stats.miss += 1; + cache->stats.miss += 1; return kr_error(ENOENT); } @@ -241,66 +186,61 @@ int kr_cache_peek(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *nam *entry = found; int ret = check_lifetime(found, timestamp); if (ret == 0) { - txn->owner->stats.hit += 1; + cache->stats.hit += 1; } else { - txn->owner->stats.miss += 1; + cache->stats.miss += 1; } return ret; } static void entry_write(struct kr_cache_entry *dst, struct kr_cache_entry *header, knot_db_val_t data) { - assert(dst && header); memcpy(dst, header, sizeof(*header)); if (data.data) memcpy(dst->data, data.data, data.len); } -int kr_cache_insert(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *name, uint16_t type, +int kr_cache_insert(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, uint16_t type, struct kr_cache_entry *header, knot_db_val_t data) { - if (!txn_is_valid(txn) || !name || !header) { + if (!cache_isvalid(cache) || !name || !header) { return kr_error(EINVAL); } - /* Insert key */ + /* Prepare key/value for insertion. */ uint8_t keybuf[KEY_SIZE]; size_t key_len = cache_key(keybuf, tag, name, type); if (key_len == 0) { return kr_error(EILSEQ); } + assert(data.len != 0); knot_db_val_t key = { keybuf, key_len }; knot_db_val_t entry = { NULL, sizeof(*header) + data.len }; - const knot_db_api_t *db_api = txn_api(txn); /* LMDB can do late write and avoid copy */ - txn->owner->stats.insert += 1; - if (db_api == knot_db_lmdb_api()) { - int ret = db_api->insert(&txn->t, &key, &entry, 0); + int ret = 0; + cache->stats.insert += 1; + if (cache->api == kr_cdb_lmdb()) { + ret = cache_op(cache, write, &key, &entry, 1); if (ret != 0) { return ret; } entry_write(entry.data, header, data); + ret = cache_op(cache, sync); /* Make sure the entry is comitted. */ } else { /* Other backends must prepare contiguous data first */ - entry.data = malloc(entry.len); - if (!entry.data) { - return kr_error(ENOMEM); - } + auto_free char *buffer = malloc(entry.len); + entry.data = buffer; entry_write(entry.data, header, data); - int ret = db_api->insert(&txn->t, &key, &entry, 0); - free(entry.data); - if (ret != 0) { - return ret; - } + ret = cache_op(cache, write, &key, &entry, 1); } - return kr_ok(); + return ret; } -int kr_cache_remove(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *name, uint16_t type) +int kr_cache_remove(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, uint16_t type) { - if (!txn_is_valid(txn) || !name ) { + if (!cache_isvalid(cache) || !name ) { return kr_error(EINVAL); } @@ -310,35 +250,51 @@ int kr_cache_remove(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *n return kr_error(EILSEQ); } knot_db_val_t key = { keybuf, key_len }; - txn->owner->stats.delete += 1; - return txn_api(txn)->del(&txn->t, &key); + cache->stats.delete += 1; + return cache_op(cache, remove, &key, 1); } -int kr_cache_clear(struct kr_cache_txn *txn) +int kr_cache_clear(struct kr_cache *cache) { - if (!txn_is_valid(txn)) { + if (!cache_isvalid(cache)) { return kr_error(EINVAL); } - int ret = cache_purge(txn); + int ret = cache_purge(cache); if (ret == 0) { - /* - * normally must return 0, never EEXIST - * (due to cache_purge()) - */ - ret = assert_right_version_txn(txn); + ret = assert_right_version(cache); } return ret; } -int kr_cache_peek_rr(struct kr_cache_txn *txn, knot_rrset_t *rr, uint8_t *rank, uint8_t *flags, uint32_t *timestamp) +int kr_cache_match(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, knot_db_val_t *val, int maxcount) +{ + if (!cache_isvalid(cache) || !name ) { + return kr_error(EINVAL); + } + if (!cache->api->match) { + return kr_error(ENOSYS); + } + + uint8_t keybuf[KEY_SIZE]; + size_t key_len = cache_key(keybuf, tag, name, 0); + if (key_len == 0) { + return kr_error(EILSEQ); + } + + /* Trim type from the search key */ + knot_db_val_t key = { keybuf, key_len - 2 }; + return cache_op(cache, match, &key, val, maxcount); +} + +int kr_cache_peek_rr(struct kr_cache *cache, knot_rrset_t *rr, uint8_t *rank, uint8_t *flags, uint32_t *timestamp) { - if (!txn_is_valid(txn) || !rr || !timestamp) { + if (!cache_isvalid(cache) || !rr || !timestamp) { return kr_error(EINVAL); } /* Check if the RRSet is in the cache. */ struct kr_cache_entry *entry = NULL; - int ret = kr_cache_peek(txn, KR_CACHE_RR, rr->owner, rr->type, &entry, timestamp); + int ret = kr_cache_peek(cache, KR_CACHE_RR, rr->owner, rr->type, &entry, timestamp); if (ret != 0) { return ret; } @@ -353,12 +309,12 @@ int kr_cache_peek_rr(struct kr_cache_txn *txn, knot_rrset_t *rr, uint8_t *rank, return kr_ok(); } -int kr_cache_peek_rank(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *name, uint16_t type, uint32_t timestamp) +int kr_cache_peek_rank(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, uint16_t type, uint32_t timestamp) { - if (!txn_is_valid(txn) || !name) { + if (!cache_isvalid(cache) || !name) { return kr_error(EINVAL); } - struct kr_cache_entry *found = lookup(txn, tag, name, type); + struct kr_cache_entry *found = lookup(cache, tag, name, type); if (!found) { return kr_error(ENOENT); } @@ -370,7 +326,7 @@ int kr_cache_peek_rank(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t int kr_cache_materialize(knot_rrset_t *dst, const knot_rrset_t *src, uint32_t drift, knot_mm_t *mm) { - if (!dst || !src) { + if (!dst || !src || dst == src) { return kr_error(EINVAL); } @@ -402,9 +358,9 @@ int kr_cache_materialize(knot_rrset_t *dst, const knot_rrset_t *src, uint32_t dr return kr_ok(); } -int kr_cache_insert_rr(struct kr_cache_txn *txn, const knot_rrset_t *rr, uint8_t rank, uint8_t flags, uint32_t timestamp) +int kr_cache_insert_rr(struct kr_cache *cache, const knot_rrset_t *rr, uint8_t rank, uint8_t flags, uint32_t timestamp) { - if (!txn_is_valid(txn) || !rr) { + if (!cache_isvalid(cache) || !rr) { return kr_error(EINVAL); } @@ -430,18 +386,18 @@ int kr_cache_insert_rr(struct kr_cache_txn *txn, const knot_rrset_t *rr, uint8_t } knot_db_val_t data = { rr->rrs.data, knot_rdataset_size(&rr->rrs) }; - return kr_cache_insert(txn, KR_CACHE_RR, rr->owner, rr->type, &header, data); + return kr_cache_insert(cache, KR_CACHE_RR, rr->owner, rr->type, &header, data); } -int kr_cache_peek_rrsig(struct kr_cache_txn *txn, knot_rrset_t *rr, uint8_t *rank, uint8_t *flags, uint32_t *timestamp) +int kr_cache_peek_rrsig(struct kr_cache *cache, knot_rrset_t *rr, uint8_t *rank, uint8_t *flags, uint32_t *timestamp) { - if (!txn_is_valid(txn) || !rr || !timestamp) { + if (!cache_isvalid(cache) || !rr || !timestamp) { return kr_error(EINVAL); } /* Check if the RRSet is in the cache. */ struct kr_cache_entry *entry = NULL; - int ret = kr_cache_peek(txn, KR_CACHE_SIG, rr->owner, rr->type, &entry, timestamp); + int ret = kr_cache_peek(cache, KR_CACHE_SIG, rr->owner, rr->type, &entry, timestamp); if (ret != 0) { return ret; } @@ -458,9 +414,9 @@ int kr_cache_peek_rrsig(struct kr_cache_txn *txn, knot_rrset_t *rr, uint8_t *ran return kr_ok(); } -int kr_cache_insert_rrsig(struct kr_cache_txn *txn, const knot_rrset_t *rr, uint8_t rank, uint8_t flags, uint32_t timestamp) +int kr_cache_insert_rrsig(struct kr_cache *cache, const knot_rrset_t *rr, uint8_t rank, uint8_t flags, uint32_t timestamp) { - if (!txn_is_valid(txn) || !rr) { + if (!cache_isvalid(cache) || !rr) { return kr_error(EINVAL); } @@ -486,5 +442,5 @@ int kr_cache_insert_rrsig(struct kr_cache_txn *txn, const knot_rrset_t *rr, uint uint16_t covered = knot_rrsig_type_covered(&rr->rrs, 0); knot_db_val_t data = { rr->rrs.data, knot_rdataset_size(&rr->rrs) }; - return kr_cache_insert(txn, KR_CACHE_SIG, rr->owner, covered, &header, data); + return kr_cache_insert(cache, KR_CACHE_SIG, rr->owner, covered, &header, data); } diff --git a/lib/cache.h b/lib/cache.h index 8f2da5d01..d7f80db3a 100644 --- a/lib/cache.h +++ b/lib/cache.h @@ -17,7 +17,7 @@ #pragma once #include -#include +#include "lib/cdb.h" #include "lib/defines.h" /** Cache entry tag */ @@ -72,23 +72,15 @@ struct kr_cache_entry struct kr_cache { knot_db_t *db; /**< Storage instance */ - const knot_db_api_t *api; /**< Storage engine */ + const struct kr_cdb_api *api; /**< Storage engine */ struct { uint32_t hit; /**< Number of cache hits */ uint32_t miss; /**< Number of cache misses */ uint32_t insert; /**< Number of insertions */ uint32_t delete; /**< Number of deletions */ - uint32_t txn_read; /**< Number of read transactions */ - uint32_t txn_write; /**< Number of write transactions */ } stats; }; -/** Cache transaction */ -struct kr_cache_txn { - knot_db_txn_t t; /**< Storage transaction */ - struct kr_cache *owner; /**< Transaction owner */ -}; - /** * Open/create cache with provided storage options. * @param cache cache structure to be initialized @@ -98,46 +90,35 @@ struct kr_cache_txn { * @return 0 or an error code */ KR_EXPORT -int kr_cache_open(struct kr_cache *cache, const knot_db_api_t *api, void *opts, knot_mm_t *mm); +int kr_cache_open(struct kr_cache *cache, const struct kr_cdb_api *api, struct kr_cdb_opts *opts, knot_mm_t *mm); /** * Close persistent cache. * @note This doesn't clear the data, just closes the connection to the database. - * @param cache database instance + * @param cache structure */ KR_EXPORT void kr_cache_close(struct kr_cache *cache); /** - * Begin cache transaction (read-only or write). - * - * @param cache database instance - * @param txn transaction instance to be initialized (output) - * @param flags transaction flags (see namedb.h in libknot) - * @return 0 or an errcode - */ -KR_EXPORT -int kr_cache_txn_begin(struct kr_cache *cache, struct kr_cache_txn *txn, unsigned flags); - -/** - * Commit existing transaction. - * @param txn transaction instance - * @return 0 or an errcode + * Synchronise cache with the backing store. + * @param cache structure */ KR_EXPORT -int kr_cache_txn_commit(struct kr_cache_txn *txn); +void kr_cache_sync(struct kr_cache *cache); /** - * Abort existing transaction instance. - * @param txn transaction instance + * Return true if cache is open and enabled. */ -KR_EXPORT -void kr_cache_txn_abort(struct kr_cache_txn *txn); +static inline bool kr_cache_is_open(struct kr_cache *cache) +{ + return cache->db != NULL; +} /** * Peek the cache for asset (name, type, tag) * @note The 'drift' is the time passed between the inception time and now (in seconds). - * @param txn transaction instance + * @param cache cache structure * @param tag asset tag * @param name asset name * @param type asset type @@ -146,14 +127,14 @@ void kr_cache_txn_abort(struct kr_cache_txn *txn); * @return 0 or an errcode */ KR_EXPORT -int kr_cache_peek(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *name, uint16_t type, +int kr_cache_peek(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, uint16_t type, struct kr_cache_entry **entry, uint32_t *timestamp); /** * Insert asset into cache, replacing any existing data. - * @param txn transaction instance + * @param cache cache structure * @param tag asset tag * @param name asset name * @param type asset type @@ -162,31 +143,43 @@ int kr_cache_peek(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *nam * @return 0 or an errcode */ KR_EXPORT -int kr_cache_insert(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *name, uint16_t type, +int kr_cache_insert(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, uint16_t type, struct kr_cache_entry *header, knot_db_val_t data); /** * Remove asset from cache. - * @param txn transaction instance + * @param cache cache structure * @param tag asset tag * @param name asset name * @param type record type * @return 0 or an errcode */ KR_EXPORT -int kr_cache_remove(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *name, uint16_t type); +int kr_cache_remove(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, uint16_t type); /** * Clear all items from the cache. - * @param txn transaction instance + * @param cache cache structure * @return 0 or an errcode */ KR_EXPORT -int kr_cache_clear(struct kr_cache_txn *txn); +int kr_cache_clear(struct kr_cache *cache); + +/** + * Prefix scan on cached items. + * @param cache cache structure + * @param tag asset tag + * @param name asset prefix key + * @param vals array of values to store the result + * @param valcnt maximum number of retrieved keys + * @return number of retrieved keys or an error + */ +KR_EXPORT +int kr_cache_match(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, knot_db_val_t *vals, int valcnt); /** * Peek the cache for given key and retrieve it's rank. - * @param txn transaction instance + * @param cache cache structure * @param tag asset tag * @param name asset name * @param type record type @@ -194,12 +187,12 @@ int kr_cache_clear(struct kr_cache_txn *txn); * @return rank (0 or positive), or an error (negative number) */ KR_EXPORT -int kr_cache_peek_rank(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t *name, uint16_t type, uint32_t timestamp); +int kr_cache_peek_rank(struct kr_cache *cache, uint8_t tag, const knot_dname_t *name, uint16_t type, uint32_t timestamp); /** * Peek the cache for given RRSet (name, type) * @note The 'drift' is the time passed between the cache time of the RRSet and now (in seconds). - * @param txn transaction instance + * @param cache cache structure * @param rr query RRSet (its rdataset may be changed depending on the result) * @param rank entry rank will be stored in this variable * @param flags entry flags @@ -207,7 +200,7 @@ int kr_cache_peek_rank(struct kr_cache_txn *txn, uint8_t tag, const knot_dname_t * @return 0 or an errcode */ KR_EXPORT -int kr_cache_peek_rr(struct kr_cache_txn *txn, knot_rrset_t *rr, uint8_t *rank, uint8_t *flags, uint32_t *timestamp); +int kr_cache_peek_rr(struct kr_cache *cache, knot_rrset_t *rr, uint8_t *rank, uint8_t *flags, uint32_t *timestamp); /** * Clone read-only RRSet and adjust TTLs. @@ -222,7 +215,7 @@ int kr_cache_materialize(knot_rrset_t *dst, const knot_rrset_t *src, uint32_t dr /** * Insert RRSet into cache, replacing any existing data. - * @param txn transaction instance + * @param cache cache structure * @param rr inserted RRSet * @param rank rank of the data * @param flags additional flags for the data @@ -230,12 +223,12 @@ int kr_cache_materialize(knot_rrset_t *dst, const knot_rrset_t *src, uint32_t dr * @return 0 or an errcode */ KR_EXPORT -int kr_cache_insert_rr(struct kr_cache_txn *txn, const knot_rrset_t *rr, uint8_t rank, uint8_t flags, uint32_t timestamp); +int kr_cache_insert_rr(struct kr_cache *cache, const knot_rrset_t *rr, uint8_t rank, uint8_t flags, uint32_t timestamp); /** * Peek the cache for the given RRset signature (name, type) * @note The RRset type must not be RRSIG but instead it must equal the type covered field of the sought RRSIG. - * @param txn transaction instance + * @param cache cache structure * @param rr query RRSET (its rdataset and type may be changed depending on the result) * @param rank entry rank will be stored in this variable * @param flags entry additional flags @@ -243,12 +236,12 @@ int kr_cache_insert_rr(struct kr_cache_txn *txn, const knot_rrset_t *rr, uint8_t * @return 0 or an errcode */ KR_EXPORT -int kr_cache_peek_rrsig(struct kr_cache_txn *txn, knot_rrset_t *rr, uint8_t *rank, uint8_t *flags, uint32_t *timestamp); +int kr_cache_peek_rrsig(struct kr_cache *cache, knot_rrset_t *rr, uint8_t *rank, uint8_t *flags, uint32_t *timestamp); /** * Insert the selected RRSIG RRSet of the selected type covered into cache, replacing any existing data. * @note The RRSet must contain RRSIGS with only the specified type covered. - * @param txn transaction instance + * @param cache cache structure * @param rr inserted RRSIG RRSet * @param rank rank of the data * @param flags additional flags for the data @@ -256,4 +249,4 @@ int kr_cache_peek_rrsig(struct kr_cache_txn *txn, knot_rrset_t *rr, uint8_t *ran * @return 0 or an errcode */ KR_EXPORT -int kr_cache_insert_rrsig(struct kr_cache_txn *txn, const knot_rrset_t *rr, uint8_t rank, uint8_t flags, uint32_t timestamp); +int kr_cache_insert_rrsig(struct kr_cache *cache, const knot_rrset_t *rr, uint8_t rank, uint8_t flags, uint32_t timestamp); diff --git a/lib/cdb.h b/lib/cdb.h new file mode 100644 index 000000000..e514b780e --- /dev/null +++ b/lib/cdb.h @@ -0,0 +1,52 @@ +/* Copyright (C) 2016 CZ.NIC, z.s.p.o. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#pragma once + +#include + +/* Cache options. */ +struct kr_cdb_opts { + const char *path; /*!< Cache URI path. */ + size_t maxsize; /*!< Suggested cache size in bytes. */ +}; + +/*! Cache database API. + * This is a simplified version of generic DB API from libknot, + * that is tailored to caching purposes. + */ +struct kr_cdb_api { + const char *name; + + /* Context operations */ + + int (*open)(knot_db_t **db, struct kr_cdb_opts *opts, knot_mm_t *mm); + void (*close)(knot_db_t *db); + int (*count)(knot_db_t *db); + int (*clear)(knot_db_t *db); + int (*sync)(knot_db_t *db); + + /* Data access */ + + int (*read)(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount); + int (*write)(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount); + int (*remove)(knot_db_t *db, knot_db_val_t *key, int maxcount); + + /* Specialised operations */ + + int (*match)(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount); + int (*prune)(knot_db_t *db, int maxcount); +}; \ No newline at end of file diff --git a/lib/cdb_lmdb.c b/lib/cdb_lmdb.c new file mode 100644 index 000000000..a6257085f --- /dev/null +++ b/lib/cdb_lmdb.c @@ -0,0 +1,543 @@ +/* Copyright (C) 2016 CZ.NIC, z.s.p.o. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include +#include +#include +#include +#include +#include +#include + +#include "contrib/cleanup.h" +#include "lib/cdb_lmdb.h" +#include "lib/cache.h" +#include "lib/utils.h" + + +/* Defines */ +#define LMDB_DIR_MODE 0770 +#define LMDB_FILE_MODE 0660 + +struct lmdb_env +{ + size_t mapsize; + MDB_dbi dbi; + MDB_env *env; + MDB_txn *rdtxn; + MDB_txn *wrtxn; +}; + +/** @brief Convert LMDB error code. */ +static int lmdb_error(int error) +{ + switch (error) { + case MDB_SUCCESS: return 0; + case MDB_NOTFOUND: return kr_error(ENOENT); + case MDB_MAP_FULL: /* Fallthrough */ + case MDB_TXN_FULL: /* Fallthrough */ + case ENOSPC: + return kr_error(ENOSPC); + default: + return -abs(error); + } +} + +/*! \brief Set the environment map size. + * \note This also sets the maximum database size, see \fn mdb_env_set_mapsize + */ +static int set_mapsize(MDB_env *env, size_t map_size) +{ + long page_size = sysconf(_SC_PAGESIZE); + if (page_size <= 0) { + return KNOT_ERROR; + } + + /* Round to page size. */ + map_size = (map_size / page_size) * page_size; + int ret = mdb_env_set_mapsize(env, map_size); + if (ret != MDB_SUCCESS) { + return lmdb_error(ret); + } + + return 0; +} + +static int txn_begin(struct lmdb_env *env, MDB_txn **txn, bool rdonly) +{ + /* Always barrier for write transaction. */ + assert(env && txn); + if (env->wrtxn) { + mdb_txn_abort(env->wrtxn); + env->wrtxn = NULL; + } + /* Renew pending read-only transaction + * or abort it to clear reader slot before writing. */ + if (env->rdtxn) { + if (rdonly) { + *txn = env->rdtxn; + env->rdtxn = NULL; + return 0; + } else { + mdb_txn_abort(env->rdtxn); + env->rdtxn = NULL; + } + } + unsigned flags = rdonly ? MDB_RDONLY : 0; + return lmdb_error(mdb_txn_begin(env->env, NULL, flags, txn)); +} + +static int txn_end(struct lmdb_env *env, MDB_txn *txn) +{ + assert(env && txn); + /* Cache read transactions */ + if (!env->rdtxn) { + env->rdtxn = txn; + } else { + mdb_txn_abort(txn); + } + return 0; +} + +static int cdb_sync(knot_db_t *db) +{ + struct lmdb_env *env = db; + int ret = 0; + if (env->wrtxn) { + ret = lmdb_error(mdb_txn_commit(env->wrtxn)); + env->wrtxn = NULL; /* In-flight transaction is committed. */ + } + if (env->rdtxn) { + mdb_txn_abort(env->rdtxn); + env->rdtxn = NULL; + } + return ret; +} + +/*! \brief Close the database. */ +static void cdb_close_env(struct lmdb_env *env) +{ + assert(env && env->env); + cdb_sync(env); + mdb_env_sync(env->env, 1); + mdb_dbi_close(env->env, env->dbi); + mdb_env_close(env->env); + memset(env, 0, sizeof(*env)); +} + +/*! \brief Open database environment. */ +static int cdb_open_env(struct lmdb_env *env, unsigned flags, const char *path, size_t mapsize) +{ + int ret = mkdir(path, LMDB_DIR_MODE); + if (ret == -1 && errno != EEXIST) { + return kr_error(errno); + } + + MDB_env *mdb_env = NULL; + ret = mdb_env_create(&mdb_env); + if (ret != MDB_SUCCESS) { + return lmdb_error(ret); + } + + ret = set_mapsize(mdb_env, mapsize); + if (ret != 0) { + mdb_env_close(mdb_env); + return ret; + } + + ret = mdb_env_open(mdb_env, path, flags, LMDB_FILE_MODE); + if (ret != MDB_SUCCESS) { + mdb_env_close(mdb_env); + return lmdb_error(ret); + } + + /* Keep the environment pointer. */ + env->env = mdb_env; + env->mapsize = mapsize; + return 0; +} + +static int cdb_open(struct lmdb_env *env, const char *path, size_t mapsize) +{ + /* Cache doesn't require durability, we can be + * loose with the requirements as a tradeoff for speed. */ + const unsigned flags = MDB_WRITEMAP | MDB_MAPASYNC | MDB_NOTLS; + int ret = cdb_open_env(env, flags, path, mapsize); + if (ret != 0) { + return ret; + } + + /* Open the database. */ + MDB_txn *txn = NULL; + ret = mdb_txn_begin(env->env, NULL, 0, &txn); + if (ret != MDB_SUCCESS) { + mdb_env_close(env->env); + return lmdb_error(ret); + } + + ret = mdb_dbi_open(txn, NULL, 0, &env->dbi); + if (ret != MDB_SUCCESS) { + mdb_txn_abort(txn); + mdb_env_close(env->env); + return lmdb_error(ret); + } + + ret = mdb_txn_commit(txn); + if (ret != MDB_SUCCESS) { + mdb_env_close(env->env); + return lmdb_error(ret); + } + + return 0; +} + +static int cdb_init(knot_db_t **db, struct kr_cdb_opts *opts, knot_mm_t *pool) +{ + if (!db || !opts) { + return kr_error(EINVAL); + } + + struct lmdb_env *env = malloc(sizeof(*env)); + if (!env) { + return kr_error(ENOMEM); + } + memset(env, 0, sizeof(struct lmdb_env)); + + /* Clear stale lockfiles. */ + auto_free char *lockfile = kr_strcatdup(2, opts->path, "/.cachelock"); + if (lockfile && access(lockfile, R_OK) == 0) { + kr_log_info("[system] cache: clearing stale lockfile '%s'\n", lockfile); + unlink(lockfile); + } + + /* Open the database. */ + int ret = cdb_open(env, opts->path, opts->maxsize); + if (ret != 0) { + free(env); + return ret; + } + + *db = env; + return 0; +} + +static void cdb_deinit(knot_db_t *db) +{ + struct lmdb_env *env = db; + cdb_close_env(env); + free(env); +} + +static int cdb_count(knot_db_t *db) +{ + struct lmdb_env *env = db; + MDB_txn *txn = NULL; + int ret = txn_begin(env, &txn, true); + if (ret != 0) { + return ret; + } + + MDB_stat stat; + ret = mdb_stat(txn, env->dbi, &stat); + + /* Always abort, serves as a checkpoint for in-flight transaction. */ + mdb_txn_abort(txn); + return (ret == MDB_SUCCESS) ? stat.ms_entries : lmdb_error(ret); +} + +static int cdb_clear(knot_db_t *db) +{ + struct lmdb_env *env = db; + + /* Since there is no guarantee that there will be free + * pages to hold whole dirtied db for transaction-safe clear, + * we simply remove the database files and reopen. + * We can afford this since other readers will continue to read + * from removed file, but will reopen when encountering next + * error. */ + mdb_filehandle_t fd = -1; + int ret = mdb_env_get_fd(env->env, &fd); + if (ret != MDB_SUCCESS) { + return lmdb_error(ret); + } + const char *path = NULL; + ret = mdb_env_get_path(env->env, &path); + if (ret != MDB_SUCCESS) { + return lmdb_error(ret); + } + + /* Check if the fd is pointing to the same file. + * man open(2): + * > Portable programs that want to perform atomic file locking using a lockfile, + * > and need to avoid reliance on NFS support for O_EXCL, can create a unique + * > file on the same file system (e.g., incorporating hostname and PID), + * > and use link(2) to make a link to the lockfile. + * > If link(2) returns 0, the lock is successful. */ + auto_free char *mdb_datafile = kr_strcatdup(2, path, "/data.mdb"); + auto_free char *mdb_lockfile = kr_strcatdup(2, path, "/lock.mdb"); + auto_free char *lockfile = kr_strcatdup(2, path, "/.cachelock"); + if (!mdb_datafile || !mdb_lockfile || !lockfile) { + return kr_error(ENOMEM); + } + ret = link(mdb_lockfile, lockfile); + if (ret != 0) { + return kr_error(errno); + } + struct stat old_stat, new_stat; + ret = fstat(fd, &new_stat); + if (ret != 0) { + unlink(lockfile); + return kr_error(errno); + } + ret = stat(mdb_datafile, &old_stat); + if (ret != 0) { + unlink(lockfile); + return kr_error(errno); + } + /* Remove underlying files only if current open environment + * points to file on the disk. Otherwise just reopen as someone + * else has already removed the files. + */ + if (old_stat.st_dev == new_stat.st_dev && old_stat.st_ino == new_stat.st_ino) { + unlink(mdb_datafile); + unlink(mdb_lockfile); + } + /* Keep copy as it points to current handle internals. */ + auto_free char *path_copy = strdup(path); + size_t mapsize = env->mapsize; + cdb_close_env(env); + ret = cdb_open(env, path_copy, mapsize); + /* Environment updated, release lockfile. */ + unlink(lockfile); + return ret; +} + +static int cdb_readv(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount) +{ + struct lmdb_env *env = db; + MDB_txn *txn = NULL; + int ret = txn_begin(env, &txn, true); + if (ret != 0) { + return ret; + } + + for (int i = 0; i < maxcount; ++i) { + /* Convert key structs */ + MDB_val _key = { .mv_size = key[i].len, .mv_data = key[i].data }; + MDB_val _val = { .mv_size = val[i].len, .mv_data = val[i].data }; + ret = mdb_get(txn, env->dbi, &_key, &_val); + /* Update the result. */ + val[i].data = _val.mv_data; + val[i].len = _val.mv_size; + } + + txn_end(env, txn); + return lmdb_error(ret); +} + +static int cdb_write(struct lmdb_env *env, MDB_txn *txn, knot_db_val_t *key, knot_db_val_t *val, unsigned flags) +{ + /* Convert key structs and write */ + MDB_val _key = { key->len, key->data }; + MDB_val _val = { val->len, val->data }; + int ret = mdb_put(txn, env->dbi, &_key, &_val, flags); + if (ret != MDB_SUCCESS) { + return lmdb_error(ret); + } + /* Update the result. */ + val->data = _val.mv_data; + val->len = _val.mv_size; + return 0; +} + +static int cdb_writev(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount) +{ + struct lmdb_env *env = db; + MDB_txn *txn = NULL; + int ret = txn_begin(env, &txn, false); + if (ret != 0) { + return ret; + } + + bool reserved = false; + for (int i = 0; i < maxcount; ++i) { + /* This is LMDB specific optimisation, + * if caller specifies value with NULL data and non-zero length, + * LMDB will preallocate the entry for caller and leave write + * transaction open, caller is responsible for syncing thus comitting transaction. + */ + unsigned mdb_flags = 0; + if (val[i].len > 0 && val[i].data == NULL) { + mdb_flags |= MDB_RESERVE; + reserved = true; + } + ret = cdb_write(env, txn, &key[i], &val[i], mdb_flags); + if (ret != 0) { + mdb_txn_abort(txn); + return ret; + } + } + + /* Leave transaction open if reserved. */ + if (reserved) { + assert(env->wrtxn == NULL); + env->wrtxn = txn; + } else { + ret = lmdb_error(mdb_txn_commit(txn)); + } + return ret; +} + +static int cdb_remove(knot_db_t *db, knot_db_val_t *key, int maxcount) +{ + struct lmdb_env *env = db; + MDB_txn *txn = NULL; + int ret = txn_begin(env, &txn, false); + if (ret != 0) { + return ret; + } + + for (int i = 0; i < maxcount; ++i) { + MDB_val _key = { key[i].len, key[i].data }; + MDB_val val = { 0, NULL }; + ret = mdb_del(txn, env->dbi, &_key, &val); + if (ret != 0) { + mdb_txn_abort(txn); + return lmdb_error(ret); + } + } + + return lmdb_error(mdb_txn_commit(txn)); +} + +static int cdb_match(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount) +{ + struct lmdb_env *env = db; + MDB_txn *txn = NULL; + int ret = txn_begin(env, &txn, true); + if (ret != 0) { + return ret; + } + + /* Turn wildcard into prefix scan. */ + const uint8_t *endp = (const uint8_t *)key->data + (key->len - 2); + if (key->len > 2 && endp[0] == '*' && endp[1] == '\0') { + key->len -= 2; /* Skip '*' label */ + } + + MDB_cursor *cur = NULL; + ret = mdb_cursor_open(txn, env->dbi, &cur); + if (ret != 0) { + mdb_txn_abort(txn); + return lmdb_error(ret); + } + + MDB_val cur_key = { key->len, key->data }, cur_val = { 0, NULL }; + ret = mdb_cursor_get(cur, &cur_key, &cur_val, MDB_SET_RANGE); + if (ret != 0) { + mdb_cursor_close(cur); + mdb_txn_abort(txn); + return lmdb_error(ret); + } + + int results = 0; + while (ret == 0) { + /* Retrieve current key and compare with prefix */ + if (cur_key.mv_size < key->len || memcmp(cur_key.mv_data, key->data, key->len) != 0) { + break; + } + /* Add to result set */ + if (results < maxcount) { + val[results].len = cur_key.mv_size; + val[results].data = cur_key.mv_data; + ++results; + } else { + break; + } + ret = mdb_cursor_get(cur, &cur_key, &cur_val, MDB_NEXT); + } + + mdb_cursor_close(cur); + txn_end(env, txn); + return results; +} + + +static int cdb_prune(knot_db_t *db, int limit) +{ + /* Sync in-flight transactions */ + cdb_sync(db); + + /* Prune old records */ + struct lmdb_env *env = db; + MDB_txn *txn = NULL; + int ret = txn_begin(env, &txn, false); + if (ret != 0) { + return ret; + } + + MDB_cursor *cur = NULL; + ret = mdb_cursor_open(txn, env->dbi, &cur); + if (ret != 0) { + mdb_txn_abort(txn); + return lmdb_error(ret); + } + + MDB_val cur_key, cur_val; + ret = mdb_cursor_get(cur, &cur_key, &cur_val, MDB_FIRST); + if (ret != 0) { + mdb_cursor_close(cur); + mdb_txn_abort(txn); + return lmdb_error(ret); + } + + int results = 0; + struct timeval now; + gettimeofday(&now, NULL); + while (ret == 0 && results < limit) { + /* Ignore special namespaces. */ + if (cur_key.mv_size < 2 || ((const char *)cur_key.mv_data)[0] == 'V') { + ret = mdb_cursor_get(cur, &cur_key, &cur_val, MDB_NEXT); + continue; + } + /* Check entry age. */ + struct kr_cache_entry *entry = cur_val.mv_data; + if (entry->timestamp > now.tv_sec || + (now.tv_sec - entry->timestamp) < entry->ttl) { + ret = mdb_cursor_get(cur, &cur_key, &cur_val, MDB_NEXT); + continue; + } + /* Remove entry */ + mdb_cursor_del(cur, 0); + ++results; + ret = mdb_cursor_get(cur, &cur_key, &cur_val, MDB_NEXT); + } + mdb_cursor_close(cur); + ret = lmdb_error(mdb_txn_commit(txn)); + return ret < 0 ? ret : results; +} + +const struct kr_cdb_api *kr_cdb_lmdb(void) +{ + static const struct kr_cdb_api api = { + "lmdb", + cdb_init, cdb_deinit, cdb_count, cdb_clear, cdb_sync, + cdb_readv, cdb_writev, cdb_remove, + cdb_match, cdb_prune + }; + + return &api; +} diff --git a/lib/cdb_lmdb.h b/lib/cdb_lmdb.h new file mode 100644 index 000000000..543192089 --- /dev/null +++ b/lib/cdb_lmdb.h @@ -0,0 +1,23 @@ +/* Copyright (C) 2016 CZ.NIC, z.s.p.o. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#pragma once + +#include "lib/cdb.h" +#include "lib/defines.h" + +KR_EXPORT KR_CONST +const struct kr_cdb_api *kr_cdb_lmdb(void); diff --git a/lib/layer/pktcache.c b/lib/layer/pktcache.c index 25cb92175..b16b8f0df 100644 --- a/lib/layer/pktcache.c +++ b/lib/layer/pktcache.c @@ -44,11 +44,11 @@ static void adjust_ttl(knot_rrset_t *rr, uint32_t drift) } } -static int loot_cache_pkt(struct kr_cache_txn *txn, knot_pkt_t *pkt, const knot_dname_t *qname, +static int loot_cache_pkt(struct kr_cache *cache, knot_pkt_t *pkt, const knot_dname_t *qname, uint16_t rrtype, bool want_secure, uint32_t timestamp, uint8_t *flags) { struct kr_cache_entry *entry = NULL; - int ret = kr_cache_peek(txn, KR_CACHE_PKT, qname, rrtype, &entry, ×tamp); + int ret = kr_cache_peek(cache, KR_CACHE_PKT, qname, rrtype, &entry, ×tamp); if (ret != 0) { /* Not in the cache */ return ret; } @@ -89,13 +89,13 @@ static int loot_cache_pkt(struct kr_cache_txn *txn, knot_pkt_t *pkt, const knot_ } /** @internal Try to find a shortcut directly to searched packet. */ -static int loot_pktcache(struct kr_cache_txn *txn, knot_pkt_t *pkt, struct kr_query *qry, uint8_t *flags) +static int loot_pktcache(struct kr_cache *cache, knot_pkt_t *pkt, struct kr_query *qry, uint8_t *flags) { uint32_t timestamp = qry->timestamp.tv_sec; const knot_dname_t *qname = qry->sname; uint16_t rrtype = qry->stype; const bool want_secure = (qry->flags & QUERY_DNSSEC_WANT); - return loot_cache_pkt(txn, pkt, qname, rrtype, want_secure, timestamp, flags); + return loot_cache_pkt(cache, pkt, qname, rrtype, want_secure, timestamp, flags); } static int pktcache_peek(knot_layer_t *ctx, knot_pkt_t *pkt) @@ -112,17 +112,10 @@ static int pktcache_peek(knot_layer_t *ctx, knot_pkt_t *pkt) return ctx->state; /* Only IN class */ } - /* Prepare read transaction */ - struct kr_cache_txn txn; - struct kr_cache *cache = &req->ctx->cache; - if (kr_cache_txn_begin(cache, &txn, KNOT_DB_RDONLY) != 0) { - return ctx->state; - } - /* Fetch either answer to original or minimized query */ uint8_t flags = 0; - int ret = loot_pktcache(&txn, pkt, qry, &flags); - kr_cache_txn_abort(&txn); + struct kr_cache *cache = &req->ctx->cache; + int ret = loot_pktcache(cache, pkt, qry, &flags); if (ret == 0) { DEBUG_MSG(qry, "=> satisfied from cache\n"); qry->flags |= QUERY_CACHED|QUERY_NO_MINIMIZE; @@ -197,11 +190,6 @@ static int pktcache_stash(knot_layer_t *ctx, knot_pkt_t *pkt) if (!qname) { return ctx->state; } - /* Open write transaction and prepare answer */ - struct kr_cache_txn txn; - if (kr_cache_txn_begin(&req->ctx->cache, &txn, 0) != 0) { - return ctx->state; /* Couldn't acquire cache, ignore. */ - } knot_db_val_t data = { pkt->wire, pkt->size }; struct kr_cache_entry header = { .timestamp = qry->timestamp.tv_sec, @@ -224,22 +212,20 @@ static int pktcache_stash(knot_layer_t *ctx, knot_pkt_t *pkt) } /* Check if we can replace (allow current or better rank, SECURE is always accepted). */ + struct kr_cache *cache = &req->ctx->cache; if (header.rank < KR_RANK_SECURE) { - int cached_rank = kr_cache_peek_rank(&txn, KR_CACHE_PKT, qname, qtype, header.timestamp); + int cached_rank = kr_cache_peek_rank(cache, KR_CACHE_PKT, qname, qtype, header.timestamp); if (cached_rank > header.rank) { - kr_cache_txn_abort(&txn); return ctx->state; } } /* Stash answer in the cache */ - int ret = kr_cache_insert(&txn, KR_CACHE_PKT, qname, qtype, &header, data); - if (ret != 0) { - kr_cache_txn_abort(&txn); - } else { + int ret = kr_cache_insert(cache, KR_CACHE_PKT, qname, qtype, &header, data); + if (ret == 0) { DEBUG_MSG(qry, "=> answer cached for TTL=%u\n", ttl); - kr_cache_txn_commit(&txn); } + kr_cache_sync(cache); return ctx->state; } diff --git a/lib/layer/rrcache.c b/lib/layer/rrcache.c index 1e8d72233..bce801e01 100644 --- a/lib/layer/rrcache.c +++ b/lib/layer/rrcache.c @@ -38,9 +38,9 @@ static inline bool is_expiring(const knot_rrset_t *rr, uint32_t drift) return 100 * (drift + 5) > 99 * knot_rrset_ttl(rr); } -static int loot_rr(struct kr_cache_txn *txn, knot_pkt_t *pkt, const knot_dname_t *name, +static int loot_rr(struct kr_cache *cache, knot_pkt_t *pkt, const knot_dname_t *name, uint16_t rrclass, uint16_t rrtype, struct kr_query *qry, - uint8_t *rank, uint8_t *flags, bool fetch_rrsig) + uint8_t *rank, uint8_t *flags, bool fetch_rrsig) { /* Check if record exists in cache */ int ret = 0; @@ -48,9 +48,9 @@ static int loot_rr(struct kr_cache_txn *txn, knot_pkt_t *pkt, const knot_dname_t knot_rrset_t cache_rr; knot_rrset_init(&cache_rr, (knot_dname_t *)name, rrtype, rrclass); if (fetch_rrsig) { - ret = kr_cache_peek_rrsig(txn, &cache_rr, rank, flags, &drift); + ret = kr_cache_peek_rrsig(cache, &cache_rr, rank, flags, &drift); } else { - ret = kr_cache_peek_rr(txn, &cache_rr, rank, flags, &drift); + ret = kr_cache_peek_rr(cache, &cache_rr, rank, flags, &drift); } if (ret != 0) { return ret; @@ -90,18 +90,13 @@ static int loot_rr(struct kr_cache_txn *txn, knot_pkt_t *pkt, const knot_dname_t /** @internal Try to find a shortcut directly to searched record. */ static int loot_rrcache(struct kr_cache *cache, knot_pkt_t *pkt, struct kr_query *qry, uint16_t rrtype, bool dobit) { - struct kr_cache_txn txn; - int ret = kr_cache_txn_begin(cache, &txn, KNOT_DB_RDONLY); - if (ret != 0) { - return ret; - } /* Lookup direct match first */ uint8_t rank = 0; uint8_t flags = 0; - ret = loot_rr(&txn, pkt, qry->sname, qry->sclass, rrtype, qry, &rank, &flags, 0); + int ret = loot_rr(cache, pkt, qry->sname, qry->sclass, rrtype, qry, &rank, &flags, 0); if (ret != 0 && rrtype != KNOT_RRTYPE_CNAME) { /* Chase CNAME if no direct hit */ rrtype = KNOT_RRTYPE_CNAME; - ret = loot_rr(&txn, pkt, qry->sname, qry->sclass, rrtype, qry, &rank, &flags, 0); + ret = loot_rr(cache, pkt, qry->sname, qry->sclass, rrtype, qry, &rank, &flags, 0); } /* Record is flagged as INSECURE => doesn't have RRSIG. */ if (ret == 0 && (rank & KR_RANK_INSECURE)) { @@ -109,9 +104,8 @@ static int loot_rrcache(struct kr_cache *cache, knot_pkt_t *pkt, struct kr_query qry->flags &= ~QUERY_DNSSEC_WANT; /* Record may have RRSIG, try to find it. */ } else if (ret == 0 && dobit) { - ret = loot_rr(&txn, pkt, qry->sname, qry->sclass, rrtype, qry, &rank, &flags, true); + ret = loot_rr(cache, pkt, qry->sname, qry->sclass, rrtype, qry, &rank, &flags, true); } - kr_cache_txn_abort(&txn); return ret; } @@ -160,7 +154,7 @@ struct rrcache_baton { struct kr_request *req; struct kr_query *qry; - struct kr_cache_txn *txn; + struct kr_cache *cache; unsigned timestamp; uint32_t min_ttl; }; @@ -172,7 +166,7 @@ static int commit_rrsig(struct rrcache_baton *baton, uint8_t rank, uint8_t flags return kr_ok(); } /* Commit covering RRSIG to a separate cache namespace. */ - return kr_cache_insert_rrsig(baton->txn, rr, rank, flags, baton->timestamp); + return kr_cache_insert_rrsig(baton->cache, rr, rank, flags, baton->timestamp); } static int commit_rr(const char *key, void *val, void *data) @@ -205,7 +199,7 @@ static int commit_rr(const char *key, void *val, void *data) } /* Accept only better rank (if not overriding) */ if (!(rank & KR_RANK_SECURE) && !(baton->qry->flags & QUERY_NO_CACHE)) { - int cached_rank = kr_cache_peek_rank(baton->txn, KR_CACHE_RR, rr->owner, rr->type, baton->timestamp); + int cached_rank = kr_cache_peek_rank(baton->cache, KR_CACHE_RR, rr->owner, rr->type, baton->timestamp); if (cached_rank >= rank) { return kr_ok(); } @@ -217,15 +211,15 @@ static int commit_rr(const char *key, void *val, void *data) if ((rank & KR_RANK_AUTH) && (baton->qry->flags & QUERY_DNSSEC_WEXPAND)) { flags |= KR_CACHE_FLAG_WCARD_PROOF; } - return kr_cache_insert_rr(baton->txn, rr, rank, flags, baton->timestamp); + return kr_cache_insert_rr(baton->cache, rr, rank, flags, baton->timestamp); } -static int stash_commit(map_t *stash, struct kr_query *qry, struct kr_cache_txn *txn, struct kr_request *req) +static int stash_commit(map_t *stash, struct kr_query *qry, struct kr_cache *cache, struct kr_request *req) { struct rrcache_baton baton = { .req = req, .qry = qry, - .txn = txn, + .cache = cache, .timestamp = qry->timestamp.tv_sec, .min_ttl = DEFAULT_MINTTL }; @@ -352,26 +346,15 @@ static int rrcache_stash(knot_layer_t *ctx, knot_pkt_t *pkt) if (ret == 0 && stash.root != NULL) { /* Open write transaction */ struct kr_cache *cache = &req->ctx->cache; - struct kr_cache_txn txn; - if (kr_cache_txn_begin(cache, &txn, 0) == 0) { - ret = stash_commit(&stash, qry, &txn, req); - if (ret == 0) { - kr_cache_txn_commit(&txn); - } else { - kr_cache_txn_abort(&txn); - } - } + ret = stash_commit(&stash, qry, cache, req); /* Clear if full */ - if (ret == KNOT_ESPACE) { - if (kr_cache_txn_begin(cache, &txn, 0) == 0) { - ret = kr_cache_clear(&txn); - if (ret == 0) { - kr_cache_txn_commit(&txn); - } else { - kr_cache_txn_abort(&txn); - } + if (ret == kr_error(ENOSPC)) { + ret = kr_cache_clear(cache); + if (ret != 0) { + kr_log_error("[cache] failed to clear cache: %s\n", kr_strerror(ret)); } } + kr_cache_sync(cache); } return ctx->state; } diff --git a/lib/lib.mk b/lib/lib.mk index db7260dd3..384e4232c 100644 --- a/lib/lib.mk +++ b/lib/lib.mk @@ -15,7 +15,8 @@ libkres_SOURCES := \ lib/resolve.c \ lib/zonecut.c \ lib/rplan.c \ - lib/cache.c + lib/cache.c \ + lib/cdb_lmdb.c libkres_HEADERS := \ lib/generic/array.h \ @@ -33,15 +34,9 @@ libkres_HEADERS := \ lib/resolve.h \ lib/zonecut.h \ lib/rplan.h \ - lib/cache.h -# Use built-in LMDB if not found -ifneq ($(HAS_lmdb), yes) -libkres_SOURCES := $(libkres_SOURCES) \ - contrib/lmdb/mdb.c \ - contrib/lmdb/midl.c -lmdb_CFLAGS := -Icontrib/lmdb -lmdb_LIBS := -endif + lib/cache.h \ + lib/cdb.h \ + lib/cdb_lmdb.h # Dependencies libkres_DEPEND := $(contrib) diff --git a/lib/resolve.c b/lib/resolve.c index 3782fcd84..262351dc9 100644 --- a/lib/resolve.c +++ b/lib/resolve.c @@ -55,6 +55,12 @@ static int reset_yield(knot_layer_t *ctx) { return kr_ok(); } static int finish_yield(knot_layer_t *ctx) { return kr_ok(); } static int produce_yield(knot_layer_t *ctx, knot_pkt_t *pkt) { return kr_ok(); } +/** Enforce cache flushing in debug mode. */ +static void flush_caches(struct kr_request *req) { +#ifdef DEBUG + kr_cache_sync(&req->ctx->cache); +#endif +} /** @internal Macro for iterating module layers. */ #define RESUME_LAYERS(from, req, qry, func, ...) \ (req)->current_query = (qry); \ @@ -63,6 +69,7 @@ static int produce_yield(knot_layer_t *ctx, knot_pkt_t *pkt) { return kr_ok(); } if (mod->layer) { \ struct knot_layer layer = {.state = (req)->state, .api = mod->layer(mod), .data = (req)}; \ if (layer.api && layer.api->func) { \ + flush_caches(req); \ (req)->state = layer.api->func(&layer, ##__VA_ARGS__); \ if ((req)->state == KNOT_STATE_YIELD) { \ func ## _yield(&layer, ##__VA_ARGS__); \ @@ -125,7 +132,7 @@ static int invalidate_ns(struct kr_rplan *rplan, struct kr_query *qry) /** This turns of QNAME minimisation if there is a non-terminal between current zone cut, and name target. * It save several minimization steps, as the zone cut is likely final one. */ -static void check_empty_nonterms(struct kr_query *qry, knot_pkt_t *pkt, struct kr_cache_txn *txn, uint32_t timestamp) +static void check_empty_nonterms(struct kr_query *qry, knot_pkt_t *pkt, struct kr_cache *cache, uint32_t timestamp) { if (qry->flags & QUERY_NO_MINIMIZE) { return; @@ -145,7 +152,7 @@ static void check_empty_nonterms(struct kr_query *qry, knot_pkt_t *pkt, struct k --labels; } for (int i = 0; i < labels; ++i) { - int ret = kr_cache_peek(txn, KR_CACHE_PKT, target, KNOT_RRTYPE_NS, &entry, ×tamp); + int ret = kr_cache_peek(cache, KR_CACHE_PKT, target, KNOT_RRTYPE_NS, &entry, ×tamp); if (ret == 0) { /* Either NXDOMAIN or NODATA, start here. */ /* @todo We could stop resolution here for NXDOMAIN, but we can't because of broken CDNs */ qry->flags |= QUERY_NO_MINIMIZE; @@ -162,21 +169,21 @@ static int ns_fetch_cut(struct kr_query *qry, struct kr_request *req, knot_pkt_t int ret = 0; /* Find closest zone cut from cache */ - struct kr_cache_txn txn; - if (kr_cache_txn_begin(&req->ctx->cache, &txn, KNOT_DB_RDONLY) == 0) { + struct kr_cache *cache = &req->ctx->cache; + if (kr_cache_is_open(cache)) { /* If at/subdomain of parent zone cut, start from its encloser. * This is for case when we get to a dead end (and need glue from parent), or DS refetch. */ struct kr_query *parent = qry->parent; bool secured = (qry->flags & QUERY_DNSSEC_WANT); if (parent && parent->zone_cut.name[0] != '\0' && knot_dname_in(parent->zone_cut.name, qry->sname)) { const knot_dname_t *encloser = knot_wire_next_label(parent->zone_cut.name, NULL); - ret = kr_zonecut_find_cached(req->ctx, &qry->zone_cut, encloser, &txn, qry->timestamp.tv_sec, &secured); + ret = kr_zonecut_find_cached(req->ctx, &qry->zone_cut, encloser, qry->timestamp.tv_sec, &secured); } else { - ret = kr_zonecut_find_cached(req->ctx, &qry->zone_cut, qry->sname, &txn, qry->timestamp.tv_sec, &secured); + ret = kr_zonecut_find_cached(req->ctx, &qry->zone_cut, qry->sname, qry->timestamp.tv_sec, &secured); } /* Check if there's a non-terminal between target and current cut. */ if (ret == 0) { - check_empty_nonterms(qry, pkt, &txn, qry->timestamp.tv_sec); + check_empty_nonterms(qry, pkt, cache, qry->timestamp.tv_sec); /* Go insecure if the zone cut is provably insecure */ if ((qry->flags & QUERY_DNSSEC_WANT) && !secured) { DEBUG_MSG(qry, "=> NS is provably without DS, going insecure\n"); @@ -184,7 +191,6 @@ static int ns_fetch_cut(struct kr_query *qry, struct kr_request *req, knot_pkt_t qry->flags |= QUERY_DNSSEC_INSECURE; } } - kr_cache_txn_abort(&txn); } else { ret = kr_error(ENOENT); } diff --git a/lib/zonecut.c b/lib/zonecut.c index b3de6e7ed..2293a4733 100644 --- a/lib/zonecut.c +++ b/lib/zonecut.c @@ -298,12 +298,12 @@ int kr_zonecut_set_sbelt(struct kr_context *ctx, struct kr_zonecut *cut) } /** Fetch address for zone cut. */ -static void fetch_addr(struct kr_zonecut *cut, const knot_dname_t *ns, uint16_t rrtype, struct kr_cache_txn *txn, uint32_t timestamp) +static void fetch_addr(struct kr_zonecut *cut, struct kr_cache *cache, const knot_dname_t *ns, uint16_t rrtype, uint32_t timestamp) { uint8_t rank = 0; knot_rrset_t cached_rr; knot_rrset_init(&cached_rr, (knot_dname_t *)ns, rrtype, KNOT_CLASS_IN); - if (kr_cache_peek_rr(txn, &cached_rr, &rank, NULL, ×tamp) != 0) { + if (kr_cache_peek_rr(cache, &cached_rr, &rank, NULL, ×tamp) != 0) { return; } @@ -317,40 +317,48 @@ static void fetch_addr(struct kr_zonecut *cut, const knot_dname_t *ns, uint16_t } /** Fetch best NS for zone cut. */ -static int fetch_ns(struct kr_context *ctx, struct kr_zonecut *cut, const knot_dname_t *name, struct kr_cache_txn *txn, uint32_t timestamp, uint8_t * restrict rank) +static int fetch_ns(struct kr_context *ctx, struct kr_zonecut *cut, const knot_dname_t *name, uint32_t timestamp, uint8_t * restrict rank) { uint32_t drift = timestamp; knot_rrset_t cached_rr; knot_rrset_init(&cached_rr, (knot_dname_t *)name, KNOT_RRTYPE_NS, KNOT_CLASS_IN); - int ret = kr_cache_peek_rr(txn, &cached_rr, rank, NULL, &drift); + int ret = kr_cache_peek_rr(&ctx->cache, &cached_rr, rank, NULL, &drift); + if (ret != 0) { + return ret; + } + + /* Materialize as we'll going to do more cache lookups. */ + knot_rrset_t rr_copy; + ret = kr_cache_materialize(&rr_copy, &cached_rr, drift, cut->pool); if (ret != 0) { return ret; } /* Insert name servers for this zone cut, addresses will be looked up * on-demand (either from cache or iteratively) */ - for (unsigned i = 0; i < cached_rr.rrs.rr_count; ++i) { - const knot_dname_t *ns_name = knot_ns_name(&cached_rr.rrs, i); + for (unsigned i = 0; i < rr_copy.rrs.rr_count; ++i) { + const knot_dname_t *ns_name = knot_ns_name(&rr_copy.rrs, i); kr_zonecut_add(cut, ns_name, NULL); /* Fetch NS reputation and decide whether to prefetch A/AAAA records. */ unsigned *cached = lru_get(ctx->cache_rep, (const char *)ns_name, knot_dname_size(ns_name)); unsigned reputation = (cached) ? *cached : 0; if (!(reputation & KR_NS_NOIP4) && !(ctx->options & QUERY_NO_IPV4)) { - fetch_addr(cut, ns_name, KNOT_RRTYPE_A, txn, timestamp); + fetch_addr(cut, &ctx->cache, ns_name, KNOT_RRTYPE_A, timestamp); } if (!(reputation & KR_NS_NOIP6) && !(ctx->options & QUERY_NO_IPV6)) { - fetch_addr(cut, ns_name, KNOT_RRTYPE_AAAA, txn, timestamp); + fetch_addr(cut, &ctx->cache, ns_name, KNOT_RRTYPE_AAAA, timestamp); } } + knot_rrset_clear(&rr_copy, cut->pool); return kr_ok(); } /** * Fetch RRSet of given type. */ -static int fetch_rrset(knot_rrset_t **rr, const knot_dname_t *owner, uint16_t type, - struct kr_cache_txn *txn, knot_mm_t *pool, uint32_t timestamp) +static int fetch_rrset(knot_rrset_t **rr, struct kr_cache *cache, + const knot_dname_t *owner, uint16_t type, knot_mm_t *pool, uint32_t timestamp) { if (!rr) { return kr_error(ENOENT); @@ -360,7 +368,7 @@ static int fetch_rrset(knot_rrset_t **rr, const knot_dname_t *owner, uint16_t ty uint32_t drift = timestamp; knot_rrset_t cached_rr; knot_rrset_init(&cached_rr, (knot_dname_t *)owner, type, KNOT_CLASS_IN); - int ret = kr_cache_peek_rr(txn, &cached_rr, &rank, NULL, &drift); + int ret = kr_cache_peek_rr(cache, &cached_rr, &rank, NULL, &drift); if (ret != 0) { return ret; } @@ -384,19 +392,19 @@ static int fetch_rrset(knot_rrset_t **rr, const knot_dname_t *owner, uint16_t ty * Fetch trust anchors for zone cut. * @note The trust anchor can theoretically be a DNSKEY but for now lets use only DS. */ -static int fetch_ta(struct kr_zonecut *cut, const knot_dname_t *name, struct kr_cache_txn *txn, uint32_t timestamp) +static int fetch_ta(struct kr_zonecut *cut, struct kr_cache *cache, const knot_dname_t *name, uint32_t timestamp) { - return fetch_rrset(&cut->trust_anchor, name, KNOT_RRTYPE_DS, txn, cut->pool, timestamp); + return fetch_rrset(&cut->trust_anchor, cache, name, KNOT_RRTYPE_DS, cut->pool, timestamp); } /** Fetch DNSKEY for zone cut. */ -static int fetch_dnskey(struct kr_zonecut *cut, const knot_dname_t *name, struct kr_cache_txn *txn, uint32_t timestamp) +static int fetch_dnskey(struct kr_zonecut *cut, struct kr_cache *cache, const knot_dname_t *name, uint32_t timestamp) { - return fetch_rrset(&cut->key, name, KNOT_RRTYPE_DNSKEY, txn, cut->pool, timestamp); + return fetch_rrset(&cut->key, cache, name, KNOT_RRTYPE_DNSKEY, cut->pool, timestamp); } int kr_zonecut_find_cached(struct kr_context *ctx, struct kr_zonecut *cut, const knot_dname_t *name, - struct kr_cache_txn *txn, uint32_t timestamp, bool * restrict secured) + uint32_t timestamp, bool * restrict secured) { if (!ctx || !cut || !name) { return kr_error(EINVAL); @@ -408,18 +416,18 @@ int kr_zonecut_find_cached(struct kr_context *ctx, struct kr_zonecut *cut, const return kr_error(ENOMEM); } /* Start at QNAME parent. */ - while (txn) { + while (true) { /* Fetch NS first and see if it's insecure. */ uint8_t rank = 0; const bool is_root = (label[0] == '\0'); - if (fetch_ns(ctx, cut, label, txn, timestamp, &rank) == 0) { + if (fetch_ns(ctx, cut, label, timestamp, &rank) == 0) { /* Flag as insecure if cached as this */ if (rank & KR_RANK_INSECURE) *secured = false; /* Fetch DS if caller wants secure zone cut */ if (*secured || is_root) { - fetch_ta(cut, label, txn, timestamp); - fetch_dnskey(cut, label, txn, timestamp); + fetch_ta(cut, &ctx->cache, label, timestamp); + fetch_dnskey(cut, &ctx->cache, label, timestamp); } update_cut_name(cut, label); mm_free(cut->pool, qname); diff --git a/lib/zonecut.h b/lib/zonecut.h index 823a258ae..29b90da56 100644 --- a/lib/zonecut.h +++ b/lib/zonecut.h @@ -133,11 +133,10 @@ int kr_zonecut_set_sbelt(struct kr_context *ctx, struct kr_zonecut *cut); * @param ctx resolution context (to fetch data from LRU caches) * @param cut zone cut to be populated * @param name QNAME to start finding zone cut for - * @param txn cache transaction (read) * @param timestamp transaction timestamp * @param secured set to true if want secured zone cut, will return false if it is provably insecure * @return 0 or error code (ENOENT if it doesn't find anything) */ KR_EXPORT int kr_zonecut_find_cached(struct kr_context *ctx, struct kr_zonecut *cut, const knot_dname_t *name, - struct kr_cache_txn *txn, uint32_t timestamp, bool * restrict secured); + uint32_t timestamp, bool * restrict secured); diff --git a/modules/cachectl/README.rst b/modules/cachectl/README.rst deleted file mode 100644 index 605a9cfd3..000000000 --- a/modules/cachectl/README.rst +++ /dev/null @@ -1,53 +0,0 @@ -Cache control -------------- - -Module providing an interface to cache database, for inspection, manipulation and purging. - -Example -^^^^^^^ - -.. code-block:: lua - - -- Query cache for 'domain.cz' - cachectl['domain.cz'] - -- Query cache for all records at/below 'insecure.net' - cachectl['*.insecure.net'] - -- Clear 'bad.cz' records - cachectl.clear('bad.cz') - -- Clear records at/below 'bad.cz' - cachectl.clear('*.bad.cz') - -- Clear packet cache - cachectl.clear('*. P') - -- Clear whole cache - cachectl.clear() - -Properties -^^^^^^^^^^ - -.. function:: cachectl.prune([max_count]) - - :param number max_count: maximum number of items to be pruned at once (default: 65536) - :return: ``{ pruned: int }`` - - Prune expired/invalid records. - -.. function:: cachectl.get([domain]) - - :return: list of matching records in cache - - Fetches matching records from cache. The **domain** can either be: - - - a domain name (e.g. ``"domain.cz"``) - - a wildcard (e.g. ``"*.domain.cz"``) - - The domain name fetches all records matching this name, while the wildcard matches all records at or below that name. - - You can also use a special namespace ``"P"`` to purge NODATA/NXDOMAIN matching this name (e.g. ``"domain.cz P"``). - - .. note:: This is equivalent to ``cachectl['domain']`` getter. - -.. function:: cachectl.clear([domain]) - - :return: ``bool`` - - Purge cache records. If the domain isn't provided, whole cache is purged. See *cachectl.get()* documentation for subtree matching policy. diff --git a/modules/cachectl/cachectl.c b/modules/cachectl/cachectl.c deleted file mode 100644 index abc93d22a..000000000 --- a/modules/cachectl/cachectl.c +++ /dev/null @@ -1,323 +0,0 @@ -/* Copyright (C) 2014 CZ.NIC, z.s.p.o. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - */ - -/** - * Partial sweep is not feasible as we don't have a list of items sorted - * by age nor any sort of LRU/MRU, completely random replace is not possible - * as well. - * - Idea - make poor man's LRU with two databases doing following: - * - Fill up 1, mark that it's unwritable - * - Fill up 2, mark that it's unwritable - * - Clear 1, all writes will now go in there - * - This gives us LR(written) with resolution 2 - */ - -#include -#include -#include -#include - -#include "daemon/engine.h" -#include "lib/module.h" -#include "lib/cache.h" - -/* Max number of records pruned at one go. */ -#define PRUNE_GRANULARITY UINT16_MAX - -/* - * Properties. - */ - -typedef int (*cache_cb_t)(struct kr_cache_txn *txn, knot_db_iter_t *it, knot_db_val_t *key, void *baton); - -/** @internal Prefix walk. */ -static int cache_prefixed(struct engine *engine, const char *args, unsigned txn_flags, cache_cb_t cb, void *baton) -{ - /* Decode parameters */ - uint8_t namespace = 'R'; - char *extra = (char *)strchr(args, ' '); - if (extra != NULL) { - extra[0] = '\0'; - namespace = extra[1]; - } - - /* Convert to domain name */ - uint8_t buf[KNOT_DNAME_MAXLEN]; - if (!knot_dname_from_str(buf, args, sizeof(buf))) { - return kr_error(EINVAL); - } - /* '*' starts subtree search */ - const uint8_t *dname = buf; - bool subtree_match = false; - if (dname[0] == '\1' && dname[1] == '*') { - subtree_match = true; - dname = knot_wire_next_label(dname, NULL); - } - /* Convert to search key prefix */ - uint8_t prefix[sizeof(uint8_t) + KNOT_DNAME_MAXLEN]; - int ret = knot_dname_lf(prefix, dname, NULL); - if (ret != 0) { - return kr_error(EINVAL); - } - size_t prefix_len = prefix[0] + sizeof(uint8_t); - prefix[0] = namespace; - - /* Start search transaction */ - struct kr_cache *cache = &engine->resolver.cache; - const knot_db_api_t *api = cache->api; - struct kr_cache_txn txn; - ret = kr_cache_txn_begin(cache, &txn, txn_flags); - if (ret != 0) { - return kr_error(EIO); - } - - /* Walk through cache records matching given prefix. - * Note that since the backend of the cache is opaque, there's no exactly efficient - * way to do prefix search (i.e. Redis uses hashtable but offers SCAN, LMDB can do lexical closest match, ...). */ - knot_db_val_t key = { prefix, prefix_len }; - knot_db_iter_t *it = api->iter_begin(&txn.t, 0); - if (it) { /* Seek first key matching the prefix. */ - it = api->iter_seek(it, &key, KNOT_DB_GEQ); - } - while (it != NULL) { - if (api->iter_key(it, &key) != 0) { - break; - } - /* If not subtree match, allow only keys with the same length. */ - if (!subtree_match && key.len != prefix_len + sizeof(uint16_t)) { - break; - } - /* Allow equal or longer keys with the same prefix. */ - if (key.len < prefix_len || memcmp(key.data, prefix, prefix_len) != 0) { - break; - } - /* Callback */ - ret = cb(&txn, it, &key, baton); - if (ret != 0) { - break; - } - /* Next key */ - it = api->iter_next(it); - } - api->iter_finish(it); - kr_cache_txn_commit(&txn); - return ret; -} - -/** Return boolean true if a record is expired. */ -static bool is_expired(struct kr_cache_entry *entry, uint32_t drift) -{ - return drift > entry->ttl; -} - -/** @internal Delete iterated key. */ -static int cache_delete_cb(struct kr_cache_txn *txn, knot_db_iter_t *it, knot_db_val_t *key, void *baton) -{ - struct kr_cache *cache = txn->owner; - return cache->api->del(&txn->t, key); -} - -/** - * Prune expired/invalid records. - * - * Input: N/A - * Output: { pruned: int } - * - */ -static char* prune(void *env, struct kr_module *module, const char *args) -{ - struct engine *engine = env; - struct kr_cache *cache = &engine->resolver.cache; - const knot_db_api_t *storage = cache->api; - - struct kr_cache_txn txn; - int ret = kr_cache_txn_begin(cache, &txn, 0); - if (ret != 0) { - return NULL; - } - - /* Iterate cache and find expired records. */ - int pruned = 0; - int prune_max = 0; - if (args) { - prune_max = atoi(args); - } - /* Default pruning granularity */ - if (prune_max == 0) { - prune_max = PRUNE_GRANULARITY; - } - /* Fetch current time and start iterating */ - struct timeval now; - gettimeofday(&now, NULL); - knot_db_iter_t *it = storage->iter_begin(&txn.t, 0); - while (it && pruned < prune_max) { - /* Fetch RR from cache */ - knot_db_val_t key, val; - if (storage->iter_key(it, &key) != 0 || - storage->iter_val(it, &val) != 0) { - break; - } - /* Ignore special namespaces. */ - if (key.len < 2 || ((const char *)key.data)[0] == 'V') { - it = storage->iter_next(it); - continue; - } - /* Prune expired records. */ - struct kr_cache_entry *entry = val.data; - if (entry->timestamp > now.tv_sec) { - continue; - } - if (is_expired(entry, now.tv_sec - entry->timestamp)) { - storage->del(&txn.t, &key); - cache->stats.delete += 1; - pruned += 1; - } - it = storage->iter_next(it); - } - - /* Commit and format result. */ - char *result = NULL; - ret = kr_cache_txn_commit(&txn); - if (ret != 0) { - if (-1 == asprintf(&result, "{ \"pruned\": %d, \"error\": \"%s\" }", pruned, knot_strerror(ret))) - result = NULL; - } else { - if (-1 == asprintf(&result, "{ \"pruned\": %d }", pruned)) - result = NULL; - } - - return result; -} - -/** - * Clear all records. - * - * Input: N/A - * Output: { result: bool } - * - */ -static char* clear(void *env, struct kr_module *module, const char *args) -{ - struct engine *engine = env; - - /* Partial clear (potentially slow/unsupported). */ - if (args && strlen(args) > 0) { - int ret = cache_prefixed(env, args, 0, &cache_delete_cb, NULL); - if (ret != 0) { - return strdup(kr_strerror(ret)); - } - return strdup("true"); - } - - struct kr_cache_txn txn; - int ret = kr_cache_txn_begin(&engine->resolver.cache, &txn, 0); - if (ret != 0) { - return NULL; - } - - /* Clear cache and commit. */ - ret = kr_cache_clear(&txn); - if (ret == 0) { - ret = kr_cache_txn_commit(&txn); - } else { - kr_cache_txn_abort(&txn); - } - - /* Clear reputation tables */ - lru_deinit(engine->resolver.cache_rtt); - lru_deinit(engine->resolver.cache_rep); - lru_init(engine->resolver.cache_rtt, LRU_RTT_SIZE); - lru_init(engine->resolver.cache_rep, LRU_REP_SIZE); - return strdup(ret == 0 ? "true" : kr_strerror(ret)); -} - -/** @internal Serialize cached record name into JSON. */ -static int cache_dump_cb(struct kr_cache_txn *txn, knot_db_iter_t *it, knot_db_val_t *key, void *baton) -{ - JsonNode* json_records = baton; - char buf[KNOT_DNAME_MAXLEN]; - /* Extract type */ - uint16_t type = 0; - const char *endp = (const char *)key->data + key->len - sizeof(uint16_t); - memcpy(&type, endp, sizeof(uint16_t)); - endp -= 1; - /* Extract domain name */ - char *dst = buf; - const char *scan = endp - 1; - while (scan > (const char *)key->data) { - if (*scan == '\0') { - const size_t lblen = endp - scan - 1; - memcpy(dst, scan + 1, lblen); - dst += lblen; - *dst++ = '.'; - endp = scan; - } - --scan; - } - memcpy(dst, scan + 1, endp - scan); - JsonNode *json_item = json_find_member(json_records, buf); - if (!json_item) { - json_item = json_mkarray(); - json_append_member(json_records, buf, json_item); - } - knot_rrtype_to_string(type, buf, sizeof(buf)); - json_append_element(json_item, json_mkstring(buf)); - return kr_ok(); -} - -/** - * Query cached records. - * - * Input: [string] domain name - * Output: { result: bool } - * - */ -static char* get(void *env, struct kr_module *module, const char *args) -{ - if (!args) { - return NULL; - } - /* Dump all keys matching prefix */ - char *result = NULL; - JsonNode *json_records = json_mkobject(); - if (json_records) { - int ret = cache_prefixed(env, args, KNOT_DB_RDONLY, &cache_dump_cb, json_records); - if (ret == 0) { - result = json_encode(json_records); - } - json_delete(json_records); - } - - return result; -} - -/* - * Module implementation. - */ - -KR_EXPORT -struct kr_prop *cachectl_props(void) -{ - static struct kr_prop prop_list[] = { - { &prune, "prune", "Prune expired/invalid records." }, - { &clear, "clear", "Clear cache records." }, - { &get, "get", "Get a list of cached record(s)." }, - { NULL, NULL, NULL } - }; - return prop_list; -} - -KR_MODULE_EXPORT(cachectl); diff --git a/modules/cachectl/cachectl.mk b/modules/cachectl/cachectl.mk deleted file mode 100644 index f39cfa60a..000000000 --- a/modules/cachectl/cachectl.mk +++ /dev/null @@ -1,5 +0,0 @@ -cachectl_CFLAGS := -fvisibility=hidden -fPIC -cachectl_SOURCES := modules/cachectl/cachectl.c -cachectl_DEPEND := $(libkres) -cachectl_LIBS := $(contrib_TARGET) $(libkres_TARGET) $(libkres_LIBS) -$(call make_c_module,cachectl) \ No newline at end of file diff --git a/modules/ketcd/README.rst b/modules/ketcd/README.rst index 566222388..113ddfc0c 100644 --- a/modules/ketcd/README.rst +++ b/modules/ketcd/README.rst @@ -12,7 +12,6 @@ The subtree structure corresponds to the configuration variables in the declarat .. code-block:: bash $ etcdctl set /kresd/net/127.0.0.1 53 - $ etcdctl set /kresd/modules/cachectl true $ etcdctl set /kresd/cache/size 10000000 Configures all listening nodes to following configuration: @@ -20,7 +19,6 @@ Configures all listening nodes to following configuration: .. code-block:: lua net = { '127.0.0.1' } - modules = { 'cachectl' } cache.size = 10000000 Example configuration diff --git a/modules/kmemcached/cdb_memcached.c b/modules/kmemcached/cdb_memcached.c new file mode 100644 index 000000000..e77386b60 --- /dev/null +++ b/modules/kmemcached/cdb_memcached.c @@ -0,0 +1,202 @@ +/* Copyright (C) 2014 CZ.NIC, z.s.p.o. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +/** @file cdb_memcached.c + * @brief Implemented all the things that the resolver cache needs, + * it's not a general-purpose namedb implementation, and it can't + * be since it's *cache* by principle and it doesn't guarantee persistence anyway. + */ + +#include +#include +#include +#include +#include "contrib/cleanup.h" + +#include "lib/generic/array.h" +#include "lib/cdb.h" +#include "lib/cache.h" +#include "lib/utils.h" + +/* memcached client */ +struct memcached_cli { + memcached_st *handle; + memcached_result_st res; +}; + +static int cdb_init(knot_db_t **db, struct kr_cdb_opts *opts, knot_mm_t *pool) +{ + if (!db || !opts) { + return kr_error(EINVAL); + } + + struct memcached_cli *cli = malloc(sizeof(*cli)); + if (!cli) { + return kr_error(ENOMEM); + } + memset(cli, 0, sizeof(*cli)); + + /* Make sure we're running on binary protocol, as the + * textual protocol is broken for binary keys. */ + auto_free char *config_str = kr_strcatdup(2, opts->path, " --BINARY-PROTOCOL"); + cli->handle = memcached(config_str, strlen(config_str)); + if (!cli->handle) { + free(cli); + return kr_error(EIO); + } + + /* Create result set */ + memcached_result_st *res = memcached_result_create(cli->handle, &cli->res); + if (!res) { + memcached_free(cli->handle); + free(cli); + return kr_error(ENOMEM); + } + + *db = cli; + return 0; +} + +static void cdb_deinit(knot_db_t *db) +{ + struct memcached_cli *cli = db; + memcached_result_free(&cli->res); + memcached_free(cli->handle); + free(cli); +} + +static int cdb_sync(knot_db_t *db) +{ + return 0; +} + +static int cdb_count(knot_db_t *db) +{ + struct memcached_cli *cli = db; + memcached_return_t error = 0; + memcached_stat_st *stats = memcached_stat(cli->handle, NULL, &error); + if (error != 0) { + return kr_error(EIO); + } + size_t ret = stats->curr_items; + free(stats); + return (ret > INT_MAX) ? INT_MAX : ret; +} + +static int cdb_clear(knot_db_t *db) +{ + struct memcached_cli *cli = db; + memcached_return_t ret = memcached_flush(cli->handle, 0); + if (ret != 0) { + return kr_error(EIO); + } + return 0; +} + +static int cdb_readv(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount) +{ + if (!db || !key || !val) { + return kr_error(EINVAL); + } + + struct memcached_cli *cli = db; + + /* Convert to libmemcached query format */ + assert(maxcount < 1000); /* Sane upper bound */ + const char *keys [maxcount]; + size_t lengths [maxcount]; + for (int i = 0; i < maxcount; ++i) { + keys[i] = key[i].data; + lengths[i] = key[i].len; + } + + /* Execute multiple get and retrieve results */ + memcached_return_t status = memcached_mget(cli->handle, keys, lengths, maxcount); + memcached_result_free(&cli->res); + memcached_result_create(cli->handle, &cli->res); + for (int i = 0; i < maxcount; ++i) { + memcached_result_st *res = memcached_fetch_result(cli->handle, &cli->res, &status); + if (!res) { /* Less results than expected */ + return kr_error(ENOENT); + } + val[i].len = memcached_result_length(res); + val[i].data = (void *)memcached_result_value(res); + } + return 0; +} + +static int cdb_writev(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount) +{ + if (!db || !key || !val) { + return kr_error(EINVAL); + } + + struct memcached_cli *cli = db; + /* @warning This expects usage only for recursor cache, if anyone + * desires to port this somewhere else, TTL shouldn't be interpreted. + */ + memcached_return_t ret = 0; + for (int i = 0; i < maxcount; ++i) { + if (val->len < 2) { + /* @note Special values/namespaces, not a RR entry with TTL. */ + ret = memcached_set(cli->handle, key[i].data, key[i].len, val[i].data, val[i].len, 0, 0); + } else { + struct kr_cache_entry *entry = val[i].data; + ret = memcached_set(cli->handle, key[i].data, key[i].len, val[i].data, val[i].len, entry->ttl, 0); + } + if (ret != 0) { + break; + } + } + return ret; +} + +static int cdb_remove(knot_db_t *db, knot_db_val_t *key, int maxcount) +{ + if (!db || !key) { + return kr_error(EINVAL); + } + + struct memcached_cli *cli = db; + memcached_return_t ret = 0; + for (int i = 0; i < maxcount; ++i) { + memcached_return_t ret = memcached_delete(cli->handle, key[i].data, key[i].len, 0); + if (ret != 0) { + break; + } + } + return ret; +} + +static int cdb_match(knot_db_t *cache, knot_db_val_t *key, knot_db_val_t *val, int maxcount) +{ + if (!cache || !key || !val) { + return kr_error(EINVAL); + } + return kr_error(ENOSYS); +} + +const struct kr_cdb_api *cdb_memcached(void) +{ + static const struct kr_cdb_api api = { + "memcached", + cdb_init, cdb_deinit, cdb_count, cdb_clear, cdb_sync, + cdb_readv, cdb_writev, cdb_remove, + cdb_match, NULL /* prune */ + }; + + return &api; +} diff --git a/modules/kmemcached/kmemcached.c b/modules/kmemcached/kmemcached.c index d1979818a..af3fe5703 100644 --- a/modules/kmemcached/kmemcached.c +++ b/modules/kmemcached/kmemcached.c @@ -14,32 +14,22 @@ along with this program. If not, see . */ -#include #include #include "daemon/engine.h" +#include "lib/cdb.h" #include "lib/module.h" #include "lib/cache.h" -/** @internal Memcached API */ -extern const knot_db_api_t *namedb_memcached_api(void); - -/** @internal Make memcached options. */ -void *namedb_memcached_mkopts(const char *conf, size_t maxsize) -{ - return strdup(conf); -} +/** @internal Redis API */ +const struct kr_cdb_api *cdb_memcached(void); KR_EXPORT int kmemcached_init(struct kr_module *module) { struct engine *engine = module->data; - /* Register new storage option */ - static struct storage_api memcached = { - "memcached://", namedb_memcached_api, namedb_memcached_mkopts - }; - array_push(engine->storage_registry, memcached); - return kr_ok(); + array_push(engine->backends, cdb_memcached()); + return 0; } KR_EXPORT @@ -47,18 +37,18 @@ int kmemcached_deinit(struct kr_module *module) { struct engine *engine = module->data; /* It was currently loaded, close cache */ - if (engine->resolver.cache.api == namedb_memcached_api()) { + if (engine->resolver.cache.api == cdb_memcached()) { kr_cache_close(&engine->resolver.cache); } /* Prevent from loading it again */ - for (unsigned i = 0; i < engine->storage_registry.len; ++i) { - struct storage_api *storage = &engine->storage_registry.at[i]; - if (strcmp(storage->prefix, "memcached://") == 0) { - array_del(engine->storage_registry, i); + for (unsigned i = 0; i < engine->backends.len; ++i) { + const struct kr_cdb_api *api = engine->backends.at[i]; + if (strcmp(api->name, "memcached") == 0) { + array_del(engine->backends, i); break; } } - return kr_ok(); + return 0; } KR_MODULE_EXPORT(kmemcached); diff --git a/modules/kmemcached/kmemcached.mk b/modules/kmemcached/kmemcached.mk index 9ac8d893c..24c0ac604 100644 --- a/modules/kmemcached/kmemcached.mk +++ b/modules/kmemcached/kmemcached.mk @@ -1,5 +1,5 @@ kmemcached_CFLAGS := -fvisibility=hidden -fPIC -kmemcached_SOURCES := modules/kmemcached/kmemcached.c modules/kmemcached/namedb_memcached.c +kmemcached_SOURCES := modules/kmemcached/kmemcached.c modules/kmemcached/cdb_memcached.c kmemcached_DEPEND := $(libkres) kmemcached_LIBS := $(libkres_TARGET) $(libkres_LIBS) $(libmemcached_LIBS) $(call make_c_module,kmemcached) diff --git a/modules/kmemcached/namedb_memcached.c b/modules/kmemcached/namedb_memcached.c deleted file mode 100644 index 5d764565e..000000000 --- a/modules/kmemcached/namedb_memcached.c +++ /dev/null @@ -1,207 +0,0 @@ -/* Copyright (C) 2014 CZ.NIC, z.s.p.o. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ - -/** @file namedb_memcached.c - * @brief Implemented all the things that the resolver cache needs, - * it's not a general-purpose namedb implementation, and it can't - * be since it's *cache* by principle and it doesn't guarantee persistence anyway. - * @note The implemented functions are not thread-safe, see http://docs.libmemcached.org/libmemcachedutil.html - * @note Write transactions can't be aborted. - * @note No iteration support. - */ - -#include -#include -#include -#include -#include -#include - -#include "lib/generic/array.h" -#include "lib/cache.h" -#include "lib/utils.h" - -/* Oh, the irony... */ -typedef array_t(char *) freelist_t; - -static int init(knot_db_t **db, knot_mm_t *mm, void *arg) -{ - if (!db || !arg) { - return KNOT_EINVAL; - } - - /* Make sure we're running on binary protocol, as the - * textual protocol is broken for binary keys. */ - auto_free char *config_str = kr_strcatdup(2, arg, " --BINARY-PROTOCOL"); - memcached_st *handle = memcached(config_str, strlen(config_str)); - if (!handle) { - return KNOT_ERROR; - } - - *db = handle; - return KNOT_EOK; -} - -static void deinit(knot_db_t *db) -{ - memcached_free((memcached_st *)db); -} - -static int txn_begin(knot_db_t *db, knot_db_txn_t *txn, unsigned flags) -{ - freelist_t *freelist = malloc(sizeof(*freelist)); - if (!freelist) { - return KNOT_ENOMEM; - } - txn->txn = freelist; - txn->db = db; - array_init(*freelist); - return KNOT_EOK; -} - -static int txn_commit(knot_db_txn_t *txn) -{ - freelist_t *freelist = txn->txn; - if (freelist) { - for (unsigned i = 0; i < freelist->len; ++i) { - free(freelist->at[i]); - } - array_clear(*freelist); - free(freelist); - txn->txn = NULL; - } - return KNOT_EOK; -} - -static void txn_abort(knot_db_txn_t *txn) -{ - /** @warning No real transactions here, - * all the reads/writes are done synchronously. - * If it is needed, we would need to buffer writes in - * the freelist first and put on commit. - */ - txn_commit(txn); -} - -static int count(knot_db_txn_t *txn) -{ - memcached_return_t error = 0; - memcached_stat_st *stats = memcached_stat(txn->db, NULL, &error); - if (error != 0) { - return KNOT_ERROR; - } - size_t ret = stats->curr_items; - free(stats); - return ret; -} - -static int clear(knot_db_txn_t *txn) -{ - memcached_return_t ret = memcached_flush(txn->db, 0); - if (ret != 0) { - return KNOT_ERROR; - } - return KNOT_EOK; -} - -static int find(knot_db_txn_t *txn, knot_db_val_t *key, knot_db_val_t *val, unsigned flags) -{ - uint32_t mc_flags = 0; - memcached_return_t error = 0; - char *ret = memcached_get(txn->db, key->data, key->len, &val->len, &mc_flags, &error); - if (error != 0) { - return KNOT_ENOENT; - } - freelist_t *freelist = txn->txn; - if (array_push(*freelist, ret) < 0) { - free(ret); /* Can't track this, must free */ - return KNOT_ENOMEM; - } - val->data = ret; - return KNOT_EOK; -} - -static int insert(knot_db_txn_t *txn, knot_db_val_t *key, knot_db_val_t *val, unsigned flags) -{ - if (!txn || !key || !val) { - return KNOT_EINVAL; - } - /* @warning This expects usage only for recursor cache, if anyone - * desires to port this somewhere else, TTL shouldn't be interpreted. - */ - struct kr_cache_entry *entry = val->data; - memcached_return_t ret = memcached_set(txn->db, key->data, key->len, val->data, val->len, entry->ttl, 0); - if (ret != 0) { - return KNOT_ERROR; - } - return KNOT_EOK; -} - -static int del(knot_db_txn_t *txn, knot_db_val_t *key) -{ - memcached_return_t ret = memcached_delete(txn->db, key->data, key->len, 0); - if (ret != 0) { - return KNOT_ERROR; - } - return KNOT_EOK; -} - -static knot_db_iter_t *iter_begin(knot_db_txn_t *txn, unsigned flags) -{ - /* Iteration is not supported, pruning should be - * left on the memcached server */ - return NULL; -} - -static knot_db_iter_t *iter_seek(knot_db_iter_t *iter, knot_db_val_t *key, unsigned flags) -{ - assert(0); - return NULL; /* ENOTSUP */ -} - -static knot_db_iter_t *iter_next(knot_db_iter_t *iter) -{ - assert(0); - return NULL; -} - -static int iter_key(knot_db_iter_t *iter, knot_db_val_t *val) -{ - return KNOT_ENOTSUP; -} - -static int iter_val(knot_db_iter_t *iter, knot_db_val_t *val) -{ - return KNOT_ENOTSUP; -} - -static void iter_finish(knot_db_iter_t *iter) -{ - assert(0); -} - -const knot_db_api_t *namedb_memcached_api(void) -{ - static const knot_db_api_t api = { - "memcached", - init, deinit, - txn_begin, txn_commit, txn_abort, - count, clear, find, insert, del, - iter_begin, iter_seek, iter_next, iter_key, iter_val, iter_finish - }; - - return &api; -} diff --git a/modules/modules.mk b/modules/modules.mk index 2c94dae98..81bf80cac 100644 --- a/modules/modules.mk +++ b/modules/modules.mk @@ -1,7 +1,6 @@ # List of built-in modules modules_TARGETS := hints \ - stats \ - cachectl + stats # Memcached ifeq ($(HAS_libmemcached),yes) diff --git a/modules/redis/cdb_redis.c b/modules/redis/cdb_redis.c new file mode 100644 index 000000000..d571aafb5 --- /dev/null +++ b/modules/redis/cdb_redis.c @@ -0,0 +1,369 @@ +/* Copyright (C) 2015 CZ.NIC, z.s.p.o. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +/** @file cdb_redis.c + * @brief Implemented all the things that the resolver cache needs (get, set, expiration). + */ + +#include +#include +#include + +#include "modules/redis/redis.h" +#include "contrib/ccan/asprintf/asprintf.h" +#include "contrib/cleanup.h" +#include "contrib/ucw/lib.h" + + +#include "lib/cdb.h" +#include "lib/cache.h" +#include "lib/utils.h" +#include "lib/defines.h" + +#define REDIS_BATCHSIZE 100 + +static int cli_connect(struct redis_cli *cli) +{ + /* Connect to either UNIX socket or TCP */ + if (cli->port == 0) { + cli->handle = redisConnectUnix(cli->addr); + } else { + cli->handle = redisConnect(cli->addr, cli->port); + } + /* Catch errors */ + if (!cli->handle) { + return kr_error(ENOMEM); + } else if (cli->handle->err) { + redisFree(cli->handle); + cli->handle = NULL; + return kr_error(ECONNREFUSED); + } + /* Set max bufsize */ + cli->handle->reader->maxbuf = REDIS_BUFSIZE; + /* Select database */ + redisReply *reply = redisCommand(cli->handle, "SELECT %d", cli->database); + if (!reply) { + redisFree(cli->handle); + cli->handle = NULL; + return kr_error(ENOTDIR); + } + freeReplyObject(reply); + return kr_ok(); +} + +static void cli_decommit(struct redis_cli *cli) +{ + redis_freelist_t *freelist = &cli->freelist; + for (unsigned i = 0; i < freelist->len; ++i) { + freeReplyObject(freelist->at[i]); + } + freelist->len = 0; +} + +static void cli_free(struct redis_cli *cli) +{ + if (cli->handle) { + redisFree(cli->handle); + } + cli_decommit(cli); + array_clear(cli->freelist); + free(cli->addr); + free(cli); +} + +/** @internal Make redis options. */ +static struct redis_cli *cli_make(const char *conf_) +{ + auto_free char *conf = strdup(conf_); + struct redis_cli *cli = malloc(sizeof(*cli)); + if (!cli || !conf) { + free(cli); + return NULL; + } + /* Parse database */ + memset(cli, 0, sizeof(*cli)); + char *bp = conf; + char *p = strchr(bp, '@'); + if (p) { + *p = '\0'; + cli->database = atoi(conf); + bp = (p + 1); + } + /* Parse host / ip / sock */ + if (access(bp, W_OK) == 0) { /* UNIX */ + cli->addr = strdup(bp); + return cli; + } + struct sockaddr_in6 ip6; + p = strchr(bp, ':'); + if (!p) { /* IPv4 */ + cli->addr = strdup(bp); + cli->port = REDIS_PORT; + return cli; + } + if (!strchr(p + 1, ':')) { /* IPv4 + port */ + *p = '\0'; + cli->addr = strdup(bp); + cli->port = atoi(p + 1); + } else { /* IPv6 */ + if (uv_ip6_addr(bp, 0, &ip6) == 0) { + cli->addr = strdup(bp); + cli->port = REDIS_PORT; + } else { /* IPv6 + port */ + p = strrchr(bp, ':'); + *p = '\0'; + cli->addr = strdup(bp); + cli->port = atoi(p + 1); + } + } + return cli; +} + +static int cdb_init(knot_db_t **cache, struct kr_cdb_opts *opts, knot_mm_t *pool) +{ + if (!cache || !opts) { + return kr_error(EINVAL); + } + /* Clone redis cli and connect */ + struct redis_cli *cli = cli_make(opts->path); + if (!cli) { + return kr_error(ENOMEM); + } + int ret = cli_connect(cli); + if (ret != 0) { + cli_free(cli); + return ret; + } + *cache = cli; + return ret; +} + +static void cdb_deinit(knot_db_t *cache) +{ + struct redis_cli *cli = cache; + cli_free(cli); +} + +static int cdb_sync(knot_db_t *cache) +{ + if (!cache) { + return kr_error(EINVAL); + } + struct redis_cli *cli = cache; + cli_decommit(cli); + return 0; +} + +/* Disconnect client */ +#define CLI_DISCONNECT(cli) \ + if ((cli)->handle->err != REDIS_ERR_OTHER) { \ + redisFree((cli)->handle); \ + (cli)->handle = NULL; \ + } +/* Attempt to reconnect */ +#define CLI_KEEPALIVE(cli_) \ + if ((cli_)->freelist.len > REDIS_MAXFREELIST) { \ + cli_decommit(cli_); \ + } \ + if (!(cli_)->handle) { \ + int ret = cli_connect((cli_)); \ + if (ret != 0) { \ + return ret; \ + } \ + } + +static int cdb_count(knot_db_t *cache) +{ + if (!cache) { + return kr_error(EINVAL); + } + int ret = 0; + struct redis_cli *cli = cache; + CLI_KEEPALIVE(cli); + redisReply *reply = redisCommand(cli->handle, "DBSIZE"); + if (!reply) { + CLI_DISCONNECT(cli); + return kr_error(EIO); + } + if (reply->type == REDIS_REPLY_INTEGER) { + ret = reply->integer; + } + freeReplyObject(reply); + return ret; +} + +static int cdb_clear(knot_db_t *cache) +{ + if (!cache) { + return kr_error(EINVAL); + } + struct redis_cli *cli = cache; + CLI_KEEPALIVE(cli); + redisReply *reply = redisCommand(cli->handle, "FLUSHDB"); + if (!reply) { + CLI_DISCONNECT(cli); + return kr_error(EIO); + } + freeReplyObject(reply); + return kr_ok(); +} + +static int cdb_readv(knot_db_t *cache, knot_db_val_t *key, knot_db_val_t *val, int maxcount) +{ + if (!cache || !key || !val) { + return kr_error(EINVAL); + } + struct redis_cli *cli = cache; + CLI_KEEPALIVE(cli); + + /* Build command pipeline */ + for (int i = 0; i < maxcount; ++i) { + redisAppendCommand(cli->handle, "GET %b", key[i].data, key[i].len); + } + /* Gather replies */ + for (int i = 0; i < maxcount; ++i) { + redisReply *reply = NULL; + redisGetReply(cli->handle, (void **)&reply); + if (!reply) { + CLI_DISCONNECT(cli); + return kr_error(EIO); + } + /* Track reply in a freelist for this transaction */ + if (array_push(cli->freelist, reply) < 0) { + freeReplyObject(reply); /* Can't track this, must free */ + return kr_error(ENOMEM); + } + /* Return value */ + if (reply->type != REDIS_REPLY_STRING) { + return kr_error(EPROTO); + } + val[i].data = reply->str; + val[i].len = reply->len; + } + return kr_ok(); +} + +static int cdb_writev(knot_db_t *cache, knot_db_val_t *key, knot_db_val_t *val, int maxcount) +{ + if (!cache || !key || !val) { + return kr_error(EINVAL); + } + + struct redis_cli *cli = cache; + CLI_KEEPALIVE(cli); + + /* Build command pipeline */ + for (int i = 0; i < maxcount; ++i) { + if (val->len < 2) { + /* @note Special values/namespaces, not a RR entry with TTL. */ + redisAppendCommand(cli->handle, "SET %b %b", key[i].data, key[i].len, val[i].data, val[i].len); + } else { + /* @warning This expects usage only for recursor cache, if anyone + * desires to port this somewhere else, TTL shouldn't be interpreted. */ + struct kr_cache_entry *entry = val[i].data; + redisAppendCommand(cli->handle, "SETEX %b %d %b", key[i].data, key[i].len, entry->ttl, val[i].data, val[i].len); + } + } + /* Gather replies */ + for (int i = 0; i < maxcount; ++i) { + redisReply *reply = NULL; + redisGetReply(cli->handle, (void **)&reply); + if (!reply) { + CLI_DISCONNECT(cli); + return kr_error(EIO); + } + freeReplyObject(reply); + } + return kr_ok(); +} + +static int cdb_remove(knot_db_t *cache, knot_db_val_t *key, int maxcount) +{ + if (!cache || !key) { + return kr_error(EINVAL); + } + + struct redis_cli *cli = cache; + CLI_KEEPALIVE(cli); + + /* Build command pipeline */ + for (int i = 0; i < maxcount; ++i) { + redisAppendCommand(cli->handle, "DEL %b", key[i].data, key[i].len); + } + /* Gather replies */ + for (int i = 0; i < maxcount; ++i) { + redisReply *reply = NULL; + redisGetReply(cli->handle, (void **)&reply); + if (!reply) { + CLI_DISCONNECT(cli); + return kr_error(EIO); + } + freeReplyObject(reply); + } + return kr_ok(); +} + +static int cdb_match(knot_db_t *cache, knot_db_val_t *key, knot_db_val_t *val, int maxcount) +{ + if (!cache || !key || !val) { + return kr_error(EINVAL); + } + + /* Turn wildcard into prefix scan. */ + const uint8_t *endp = (const uint8_t *)key->data + (key->len - 2); + if (key->len > 2 && endp[0] == '*' && endp[1] == '\0') { + --key->len; /* Trim terminal byte for right-side wildcard search */ + } + + struct redis_cli *cli = cache; + CLI_KEEPALIVE(cli); + redisReply *reply = redisCommand(cli->handle, "SCAN 0 MATCH %b COUNT 100", key->data, key->len); + if (!reply) { + CLI_DISCONNECT(cli); + return kr_error(EIO); + } + /* Track reply in a freelist for this transaction */ + if (array_push(cli->freelist, reply) < 0) { + freeReplyObject(reply); /* Can't track this, must free */ + return kr_error(ENOMEM); + } + /* SCAN returns array of 2 elements, first is iterator 'next' and second an array of results. */ + int results = 0; + if (reply->type == REDIS_REPLY_ARRAY && reply->elements == 2) { + redisReply *data = reply->element[1]; + results = MIN(data->elements, maxcount); + assert(data->type == REDIS_REPLY_ARRAY); + for (size_t i = 0; i < results; ++i) { + redisReply *elem = data->element[i]; + assert(elem->type == REDIS_REPLY_STRING); + val[i].data = elem->str; + val[i].len = elem->len; + } + } + return results; +} + +const struct kr_cdb_api *cdb_redis(void) +{ + static const struct kr_cdb_api api = { + "redis", + cdb_init, cdb_deinit, cdb_count, cdb_clear, cdb_sync, + cdb_readv, cdb_writev, cdb_remove, + cdb_match, NULL /* prune */ + }; + + return &api; +} diff --git a/modules/redis/namedb_redis.c b/modules/redis/namedb_redis.c deleted file mode 100644 index db10c5d51..000000000 --- a/modules/redis/namedb_redis.c +++ /dev/null @@ -1,281 +0,0 @@ -/* Copyright (C) 2015 CZ.NIC, z.s.p.o. - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ - -/** @file namedb_redis.c - * @brief Implemented all the things that the resolver cache needs (get, set, expiration). - * @note No real transactions. - * @note No iteration support. - */ - -#include -#include -#include - -#include "modules/redis/redis.h" - -#include "lib/cache.h" -#include "lib/utils.h" -#include "lib/defines.h" - -static int cli_connect(struct redis_cli *cli) -{ - /* Connect to either UNIX socket or TCP */ - if (cli->port == 0) { - cli->handle = redisConnectUnix(cli->addr); - } else { - cli->handle = redisConnect(cli->addr, cli->port); - } - /* Catch errors */ - if (!cli->handle) { - return kr_error(ENOMEM); - } else if (cli->handle->err) { - redisFree(cli->handle); - cli->handle = NULL; - return kr_error(ECONNREFUSED); - } - /* Set max bufsize */ - cli->handle->reader->maxbuf = REDIS_BUFSIZE; - /* Select database */ - redisReply *reply = redisCommand(cli->handle, "SELECT %d", cli->database); - if (!reply) { - redisFree(cli->handle); - cli->handle = NULL; - return kr_error(ENOTDIR); - } - freeReplyObject(reply); - return kr_ok(); -} - -static void cli_decommit(struct redis_cli *cli) -{ - redis_freelist_t *freelist = &cli->freelist; - for (unsigned i = 0; i < freelist->len; ++i) { - freeReplyObject(freelist->at[i]); - } - freelist->len = 0; -} - -static void cli_free(struct redis_cli *cli) -{ - if (cli->handle) { - redisFree(cli->handle); - } - cli_decommit(cli); - array_clear(cli->freelist); - free(cli->addr); - free(cli); -} - -static int init(knot_db_t **db, knot_mm_t *mm, void *arg) -{ - if (!db || !arg) { - return kr_error(EINVAL); - } - /* Clone redis cli and connect */ - struct redis_cli *cli = malloc(sizeof(*cli)); - if (!cli) { - return kr_error(ENOMEM); - } - memcpy(cli, arg, sizeof(*cli)); - int ret = cli_connect(cli); - if (ret != 0) { - cli_free(cli); - return ret; - } - *db = cli; - return ret; -} - -static void deinit(knot_db_t *db) -{ - struct redis_cli *cli = db; - cli_free(cli); -} - -static int txn_begin(knot_db_t *db, knot_db_txn_t *txn, unsigned flags) -{ - if (!db || !txn) { - return kr_error(EINVAL); - } - txn->db = db; - return kr_ok(); -} - -static int txn_commit(knot_db_txn_t *txn) -{ - if (!txn || !txn->db) { - return kr_error(EINVAL); - } - cli_decommit(txn->db); - txn->db = NULL; - return kr_ok(); -} - -static void txn_abort(knot_db_txn_t *txn) -{ - /** @warning No real transactions here. */ - txn_commit(txn); -} - -/* Disconnect client */ -#define CLI_DISCONNECT(cli) \ - if ((cli)->handle->err != REDIS_ERR_OTHER) { \ - redisFree((cli)->handle); \ - (cli)->handle = NULL; \ - } -/* Attempt to reconnect */ -#define CLI_KEEPALIVE(cli_) \ - if (!(cli_)->handle) { \ - int ret = cli_connect((cli_)); \ - if (ret != 0) { \ - return ret; \ - } \ - } - -static int count(knot_db_txn_t *txn) -{ - if (!txn || !txn->db) { - return kr_error(EINVAL); - } - int ret = 0; - struct redis_cli *cli = txn->db; - CLI_KEEPALIVE(cli); - redisReply *reply = redisCommand(cli->handle, "DBSIZE"); - if (!reply) { - CLI_DISCONNECT(cli); - return kr_error(EIO); - } - if (reply->type == REDIS_REPLY_INTEGER) { - ret = reply->integer; - } - freeReplyObject(reply); - return ret; -} - -static int clear(knot_db_txn_t *txn) -{ - if (!txn || !txn->db) { - return kr_error(EINVAL); - } - struct redis_cli *cli = txn->db; - CLI_KEEPALIVE(cli); - redisReply *reply = redisCommand(cli->handle, "FLUSHDB"); - if (!reply) { - CLI_DISCONNECT(cli); - return kr_error(EIO); - } - freeReplyObject(reply); - return kr_ok(); -} - -static int find(knot_db_txn_t *txn, knot_db_val_t *key, knot_db_val_t *val, unsigned flags) -{ - if (!txn || !key || !val) { - return kr_error(EINVAL); - } - struct redis_cli *cli = txn->db; - CLI_KEEPALIVE(cli); - redisReply *reply = redisCommand(cli->handle, "GET %b", key->data, key->len); - if (!reply) { - CLI_DISCONNECT(cli); - return kr_error(EIO); - } - /* Track reply in a freelist for this transaction */ - if (array_push(cli->freelist, reply) < 0) { - freeReplyObject(reply); /* Can't track this, must free */ - return kr_error(ENOMEM); - } - /* Return value */ - if (reply->type != REDIS_REPLY_STRING) { - return kr_error(EPROTO); - } - val->data = reply->str; - val->len = reply->len; - return kr_ok(); -} - -static int insert(knot_db_txn_t *txn, knot_db_val_t *key, knot_db_val_t *val, unsigned flags) -{ - if (!txn || !key || !val) { - return kr_error(EINVAL); - } - /* @warning This expects usage only for recursor cache, if anyone - * desires to port this somewhere else, TTL shouldn't be interpreted. - */ - struct redis_cli *cli = txn->db; - CLI_KEEPALIVE(cli); - struct kr_cache_entry *entry = val->data; - redisReply *reply = redisCommand(cli->handle, "SETEX %b %d %b", - key->data, key->len, entry->ttl, val->data, val->len); - if (!reply) { - CLI_DISCONNECT(cli); - return kr_error(EIO); - } - freeReplyObject(reply); - return kr_ok(); -} - -static int del(knot_db_txn_t *txn, knot_db_val_t *key) -{ - return kr_error(ENOSYS); -} - -static knot_db_iter_t *iter_begin(knot_db_txn_t *txn, unsigned flags) -{ - /* Iteration is not supported, pruning should be - * left on the Redis server setting */ - return NULL; -} - -static knot_db_iter_t *iter_seek(knot_db_iter_t *iter, knot_db_val_t *key, unsigned flags) -{ - assert(0); - return NULL; /* ENOSYS */ -} - -static knot_db_iter_t *iter_next(knot_db_iter_t *iter) -{ - assert(0); - return NULL; -} - -static int iter_key(knot_db_iter_t *iter, knot_db_val_t *val) -{ - return kr_error(ENOSYS); -} - -static int iter_val(knot_db_iter_t *iter, knot_db_val_t *val) -{ - return kr_error(ENOSYS); -} - -static void iter_finish(knot_db_iter_t *iter) -{ - assert(0); -} - -const knot_db_api_t *namedb_redis_api(void) -{ - static const knot_db_api_t api = { - "redis", - init, deinit, - txn_begin, txn_commit, txn_abort, - count, clear, find, insert, del, - iter_begin, iter_seek, iter_next, iter_key, iter_val, iter_finish - }; - - return &api; -} diff --git a/modules/redis/redis.c b/modules/redis/redis.c index 79f7c66cb..61be304e6 100644 --- a/modules/redis/redis.c +++ b/modules/redis/redis.c @@ -24,65 +24,13 @@ #include "lib/cache.h" /** @internal Redis API */ -extern const knot_db_api_t *namedb_redis_api(void); - -/** @internal Make redis options. */ -void *namedb_redis_mkopts(const char *conf_, size_t maxsize) -{ - auto_free char *conf = strdup(conf_); - struct redis_cli *cli = malloc(sizeof(*cli)); - if (!cli || !conf) { - free(cli); - return NULL; - } - /* Parse database */ - memset(cli, 0, sizeof(*cli)); - char *bp = conf; - char *p = strchr(bp, '@'); - if (p) { - *p = '\0'; - cli->database = atoi(conf); - bp = (p + 1); - } - /* Parse host / ip / sock */ - if (access(bp, W_OK) == 0) { /* UNIX */ - cli->addr = strdup(bp); - return cli; - } - struct sockaddr_in6 ip6; - p = strchr(bp, ':'); - if (!p) { /* IPv4 */ - cli->addr = strdup(bp); - cli->port = REDIS_PORT; - return cli; - } - if (!strchr(p + 1, ':')) { /* IPv4 + port */ - *p = '\0'; - cli->addr = strdup(bp); - cli->port = atoi(p + 1); - } else { /* IPv6 */ - if (uv_ip6_addr(bp, 0, &ip6) == 0) { - cli->addr = strdup(bp); - cli->port = REDIS_PORT; - } else { /* IPv6 + port */ - p = strrchr(bp, ':'); - *p = '\0'; - cli->addr = strdup(bp); - cli->port = atoi(p + 1); - } - } - return cli; -} +const struct kr_cdb_api *cdb_redis(void); KR_EXPORT int redis_init(struct kr_module *module) { struct engine *engine = module->data; - /* Register new storage option */ - static struct storage_api redis = { - "redis://", namedb_redis_api, namedb_redis_mkopts - }; - array_push(engine->storage_registry, redis); + array_push(engine->backends, cdb_redis()); return kr_ok(); } @@ -91,14 +39,14 @@ int redis_deinit(struct kr_module *module) { struct engine *engine = module->data; /* It was currently loaded, close cache */ - if (engine->resolver.cache.api == namedb_redis_api()) { + if (engine->resolver.cache.api == cdb_redis()) { kr_cache_close(&engine->resolver.cache); } /* Prevent from loading it again */ - for (unsigned i = 0; i < engine->storage_registry.len; ++i) { - struct storage_api *storage = &engine->storage_registry.at[i]; - if (strcmp(storage->prefix, "redis://") == 0) { - array_del(engine->storage_registry, i); + for (unsigned i = 0; i < engine->backends.len; ++i) { + const struct kr_cdb_api *api = engine->backends.at[i]; + if (strcmp(api->name, "redis") == 0) { + array_del(engine->backends, i); break; } } diff --git a/modules/redis/redis.h b/modules/redis/redis.h index 9570bc74a..38dcb1db4 100644 --- a/modules/redis/redis.h +++ b/modules/redis/redis.h @@ -20,7 +20,8 @@ #include "lib/generic/array.h" /** Redis buffer size */ -#define REDIS_BUFSIZE (512 * 1024) +#define REDIS_MAXFREELIST 1024 +#define REDIS_BUFSIZE (1024 * 1024) #define REDIS_PORT 6379 typedef array_t(redisReply *) redis_freelist_t; diff --git a/modules/redis/redis.mk b/modules/redis/redis.mk index b33b03ef5..250605e34 100644 --- a/modules/redis/redis.mk +++ b/modules/redis/redis.mk @@ -1,5 +1,5 @@ redis_CFLAGS := -fvisibility=hidden -fPIC -redis_SOURCES := modules/redis/redis.c modules/redis/namedb_redis.c +redis_SOURCES := modules/redis/redis.c modules/redis/cdb_redis.c redis_DEPEND := $(libkres) redis_LIBS := $(libkres_TARGET) $(libkres_LIBS) $(hiredis_LIBS) $(libuv_LIBS) $(call make_c_module,redis) diff --git a/tests/test_cache.c b/tests/test_cache.c index 2e5fc3f17..97f8fa97f 100644 --- a/tests/test_cache.c +++ b/tests/test_cache.c @@ -14,18 +14,18 @@ along with this program. If not, see . */ -#include +#include +#include +#include #include #include "tests/test.h" #include "lib/cache.h" +#include "lib/cdb_lmdb.h" + -#include -#include -#include knot_mm_t global_mm; -struct kr_cache_txn global_txn; knot_rrset_t global_rr; const char *global_env; struct kr_cache_entry global_fake_ce; @@ -45,97 +45,72 @@ int knot_rdataset_add(knot_rdataset_t *rrs, const knot_rdata_t *rr, knot_mm_t *m { int err, err_mock; err_mock = (int)mock(); - if (original_knot_rdataset_add == NULL) - { + if (original_knot_rdataset_add == NULL) { original_knot_rdataset_add = dlsym(RTLD_NEXT,"knot_rdataset_add"); assert_non_null (original_knot_rdataset_add); } err = original_knot_rdataset_add(rrs, rr, mm); - if (err_mock != KNOT_EOK) + if (err_mock != 0) err = err_mock; return err; } /* Simulate init failure */ -static int fake_test_init(knot_db_t **db_ptr, knot_mm_t *mm, void *arg) +static int fake_test_init(knot_db_t **db, struct kr_cdb_opts *opts, knot_mm_t *pool) { - static char db[1024]; - *db_ptr = db; + static char static_buffer[1024]; + *db = static_buffer; return mock(); } static void fake_test_deinit(knot_db_t *db) { - return; -} - -/* Simulate commit failure */ -static int fake_test_commit(knot_db_txn_t *txn) -{ - return KNOT_ESPACE; -} - -/* Dummy abort */ -static void fake_test_abort(knot_db_txn_t *txn) -{ - return; } /* Stub for find */ -static int fake_test_find(knot_db_txn_t *txn, knot_db_val_t *key, knot_db_val_t *val, unsigned flags) +static int fake_test_find(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount) { val->data = &global_fake_ce; - return KNOT_EOK; + return 0; } /* Stub for insert */ -static int fake_test_ins(knot_db_txn_t *txn, knot_db_val_t *key, knot_db_val_t *val, unsigned flags) +static int fake_test_ins(knot_db_t *db, knot_db_val_t *key, knot_db_val_t *val, int maxcount) { struct kr_cache_entry *header = val->data; - int res_cmp, err = (int)mock(); - if (val->len == sizeof (*header) + NAMEDB_DATA_SIZE) - { - header = val->data; - res_cmp = memcmp(header->data,namedb_data,NAMEDB_DATA_SIZE); - if (header->timestamp != global_fake_ce.timestamp || - header->ttl != global_fake_ce.ttl || - header->ttl != global_fake_ce.ttl || - res_cmp != 0) - { - err = KNOT_EINVAL; - } + int ret, err = (int)mock(); + if (val->len == sizeof(*header) + NAMEDB_DATA_SIZE) { + header = val->data; + ret = memcmp(header->data,namedb_data,NAMEDB_DATA_SIZE); + if (header->timestamp != global_fake_ce.timestamp || header->ttl != global_fake_ce.ttl || ret != 0) { + err = KNOT_EINVAL; + } } return err; } -static int fake_test_txn_begin(knot_db_t *db, knot_db_txn_t *txn, unsigned flags) -{ - return KNOT_EOK; -} - /* Fake api */ -static knot_db_api_t *fake_knot_db_lmdb_api(void) +static const struct kr_cdb_api *fake_knot_db_lmdb_api(void) { - static knot_db_api_t fake_api = { + static const struct kr_cdb_api api = { "lmdb_fake_api", - fake_test_init, fake_test_deinit, - fake_test_txn_begin, fake_test_commit, fake_test_abort, - NULL, NULL, fake_test_find, fake_test_ins, NULL, - NULL, NULL, NULL, NULL, NULL, NULL + fake_test_init, fake_test_deinit, NULL, NULL, NULL, + fake_test_find, fake_test_ins, NULL, + NULL, NULL }; - return &fake_api; + return &api; } /* Test cache open */ -static int test_open(void **state, knot_db_api_t *api) +static int test_open(void **state, const struct kr_cdb_api *api) { static struct kr_cache cache; - struct knot_db_lmdb_opts opts; + struct kr_cdb_opts opts = { + global_env, + CACHE_SIZE, + }; memset(&cache, 0, sizeof(cache)); - memset(&opts, 0, sizeof(opts)); - opts.path = global_env; - opts.mapsize = CACHE_SIZE; *state = &cache; return kr_cache_open(&cache, api, &opts, &global_mm); } @@ -143,20 +118,20 @@ static int test_open(void **state, knot_db_api_t *api) /* fake api test open */ static void test_open_fake_api(void **state) { - bool res; - will_return(fake_test_init,KNOT_EINVAL); - assert_int_equal(test_open(state, fake_knot_db_lmdb_api()),KNOT_EINVAL); - will_return(fake_test_init,KNOT_EOK); - assert_int_equal(test_open(state, fake_knot_db_lmdb_api()),KNOT_EOK); + bool res = false; + will_return(fake_test_init, KNOT_EINVAL); + assert_int_equal(test_open(state, fake_knot_db_lmdb_api()), KNOT_EINVAL); + will_return(fake_test_init, 0); + assert_int_equal(test_open(state, fake_knot_db_lmdb_api()), 0); res = (((struct kr_cache *)(*state))->api == fake_knot_db_lmdb_api()); assert_true(res); } static void test_open_conventional_api(void **state) { - bool res; - assert_int_equal(test_open(state, NULL),KNOT_EOK); - res = (((struct kr_cache *)(*state))->api == knot_db_lmdb_api()); + bool res = false; + assert_int_equal(test_open(state, NULL),0); + res = (((struct kr_cache *)(*state))->api == kr_cdb_lmdb()); assert_true(res); } @@ -168,40 +143,21 @@ static void test_close(void **state) *state = NULL; } -/* Open transaction */ -static struct kr_cache_txn *test_txn_write(void **state) -{ - assert_non_null(*state); - assert_int_equal(kr_cache_txn_begin(*state, &global_txn, 0), KNOT_EOK); - return &global_txn; -} - -/* Open transaction */ -static struct kr_cache_txn *test_txn_rdonly(void **state) -{ - assert_non_null(*state); - assert_int_equal(kr_cache_txn_begin(*state, &global_txn, KNOT_DB_RDONLY), 0); - return &global_txn; -} - /* test invalid parameters and some api failures */ static void test_fake_invalid (void **state) { - struct kr_cache_txn *txn = NULL; - const knot_db_api_t *api_saved = NULL; + const struct kr_cdb_api *api_saved = NULL; knot_dname_t dname[] = ""; + struct kr_cache *cache = *state; struct kr_cache_entry *entry = NULL; int ret = 0; - assert_int_not_equal(kr_cache_txn_commit(txn), 0); - txn = test_txn_write(state); - assert_int_not_equal(kr_cache_txn_commit(txn), 0); - ret = kr_cache_peek(txn, KR_CACHE_USER, dname, KNOT_RRTYPE_TSIG, &entry, 0); + ret = kr_cache_peek(cache, KR_CACHE_USER, dname, KNOT_RRTYPE_TSIG, &entry, 0); assert_int_equal(ret, 0); - api_saved = txn->owner->api; - txn->owner->api = NULL; - ret = kr_cache_peek(txn, KR_CACHE_USER, dname, KNOT_RRTYPE_TSIG, &entry, 0); - txn->owner->api = api_saved; + api_saved = cache->api; + cache->api = NULL; + ret = kr_cache_peek(cache, KR_CACHE_USER, dname, KNOT_RRTYPE_TSIG, &entry, 0); + cache->api = api_saved; assert_int_not_equal(ret, 0); } @@ -209,17 +165,17 @@ static void test_fake_insert(void **state) { int ret_cache_ins_ok, ret_cache_ins_inval; knot_dname_t dname[] = ""; - struct kr_cache_txn *txn = test_txn_write(state); - test_randstr((char *)&global_fake_ce,sizeof(global_fake_ce)); - test_randstr((char *)namedb_data,NAMEDB_DATA_SIZE); + struct kr_cache *cache = (*state); + test_randstr((char *)&global_fake_ce, sizeof(global_fake_ce)); + test_randstr((char *)namedb_data, NAMEDB_DATA_SIZE); - will_return(fake_test_ins,KNOT_EOK); - ret_cache_ins_ok = kr_cache_insert(txn, KR_CACHE_USER, dname, + will_return(fake_test_ins, 0); + ret_cache_ins_ok = kr_cache_insert(cache, KR_CACHE_USER, dname, KNOT_RRTYPE_TSIG, &global_fake_ce, global_namedb_data); will_return(fake_test_ins,KNOT_EINVAL); - ret_cache_ins_inval = kr_cache_insert(txn, KR_CACHE_USER, dname, + ret_cache_ins_inval = kr_cache_insert(cache, KR_CACHE_USER, dname, KNOT_RRTYPE_TSIG, &global_fake_ce, global_namedb_data); - assert_int_equal(ret_cache_ins_ok, KNOT_EOK); + assert_int_equal(ret_cache_ins_ok, 0); assert_int_equal(ret_cache_ins_inval, KNOT_EINVAL); } @@ -228,33 +184,30 @@ static void test_invalid(void **state) { knot_dname_t dname[] = ""; uint32_t timestamp = CACHE_TIME; - struct knot_db_lmdb_opts opts; struct kr_cache_entry *entry = NULL; - - memset(&opts, 0, sizeof(opts)); - opts.path = global_env; - opts.mapsize = CACHE_SIZE; + struct kr_cache *cache = (*state); + struct kr_cdb_opts opts = { + global_env, + CACHE_SIZE, + }; knot_rrset_init_empty(&global_rr); assert_int_equal(kr_cache_open(NULL, NULL, &opts, &global_mm),KNOT_EINVAL); - assert_int_not_equal(kr_cache_txn_begin(NULL, &global_txn, 0), 0); - assert_int_not_equal(kr_cache_txn_begin(*state, NULL, 0), 0); - assert_int_not_equal(kr_cache_txn_commit(NULL), 0); assert_int_not_equal(kr_cache_peek(NULL, KR_CACHE_USER, dname, KNOT_RRTYPE_TSIG, NULL, ×tamp), 0); - assert_int_not_equal(kr_cache_peek(&global_txn, KR_CACHE_USER, NULL, KNOT_RRTYPE_TSIG, &entry, ×tamp), 0); + assert_int_not_equal(kr_cache_peek(cache, KR_CACHE_USER, NULL, KNOT_RRTYPE_TSIG, &entry, ×tamp), 0); assert_int_not_equal(kr_cache_peek_rr(NULL, NULL, NULL, NULL, NULL), 0); - assert_int_not_equal(kr_cache_peek_rr(&global_txn, NULL, NULL, NULL, NULL), 0); - assert_int_not_equal(kr_cache_insert_rr(&global_txn, NULL, 0, 0, 0), 0); + assert_int_not_equal(kr_cache_peek_rr(cache, NULL, NULL, NULL, NULL), 0); + assert_int_not_equal(kr_cache_insert_rr(cache, NULL, 0, 0, 0), 0); assert_int_not_equal(kr_cache_insert_rr(NULL, NULL, 0, 0, 0), 0); assert_int_not_equal(kr_cache_insert(NULL, KR_CACHE_USER, dname, KNOT_RRTYPE_TSIG, &global_fake_ce, global_namedb_data), 0); - assert_int_not_equal(kr_cache_insert(&global_txn, KR_CACHE_USER, NULL, + assert_int_not_equal(kr_cache_insert(cache, KR_CACHE_USER, NULL, KNOT_RRTYPE_TSIG, &global_fake_ce, global_namedb_data), 0); - assert_int_not_equal(kr_cache_insert(&global_txn, KR_CACHE_USER, dname, + assert_int_not_equal(kr_cache_insert(cache, KR_CACHE_USER, dname, KNOT_RRTYPE_TSIG, NULL, global_namedb_data), 0); - assert_int_not_equal(kr_cache_remove(&global_txn, 0, NULL, 0), 0); - assert_int_not_equal(kr_cache_remove(&global_txn, KR_CACHE_RR, NULL, 0), 0); + assert_int_not_equal(kr_cache_remove(cache, 0, NULL, 0), 0); + assert_int_not_equal(kr_cache_remove(cache, KR_CACHE_RR, NULL, 0), 0); assert_int_not_equal(kr_cache_remove(NULL, 0, NULL, 0), 0); assert_int_not_equal(kr_cache_clear(NULL), 0); } @@ -263,14 +216,9 @@ static void test_invalid(void **state) static void test_insert_rr(void **state) { test_random_rr(&global_rr, CACHE_TTL); - struct kr_cache_txn *txn = test_txn_write(state); - int ret = kr_cache_insert_rr(txn, &global_rr, 0, 0, CACHE_TIME); - if (ret == KNOT_EOK) { - ret = kr_cache_txn_commit(txn); - } else { - kr_cache_txn_abort(txn); - } - assert_int_equal(ret, KNOT_EOK); + struct kr_cache *cache = (*state); + int ret = kr_cache_insert_rr(cache, &global_rr, 0, 0, CACHE_TIME); + assert_int_equal(ret, 0); } static void test_materialize(void **state) @@ -285,46 +233,42 @@ static void test_materialize(void **state) kr_cache_materialize(&output_rr, &global_rr, 0, &global_mm); res_cmp_ok_empty = knot_rrset_equal(&global_rr, &output_rr, KNOT_RRSET_COMPARE_HEADER); res_cmp_fail_empty = knot_rrset_equal(&global_rr, &output_rr, KNOT_RRSET_COMPARE_WHOLE); - knot_rrset_clear(&output_rr,&global_mm); + knot_rrset_clear(&output_rr, &global_mm); global_rr.owner = owner_saved; assert_true(res_cmp_ok_empty); assert_false(res_cmp_fail_empty); knot_rrset_init(&output_rr, NULL, 0, 0); - will_return (knot_rdataset_add,KNOT_EOK); + will_return (knot_rdataset_add, 0); kr_cache_materialize(&output_rr, &global_rr, 0, &global_mm); res_cmp_ok = knot_rrset_equal(&global_rr, &output_rr, KNOT_RRSET_COMPARE_WHOLE); - knot_rrset_clear(&output_rr,&global_mm); + knot_rrset_clear(&output_rr, &global_mm); assert_true(res_cmp_ok); knot_rrset_init(&output_rr, NULL, 0, 0); - will_return (knot_rdataset_add,KNOT_EINVAL); + will_return (knot_rdataset_add, KNOT_EINVAL); kr_cache_materialize(&output_rr, &global_rr, 0, &global_mm); res_cmp_fail = knot_rrset_equal(&global_rr, &output_rr, KNOT_RRSET_COMPARE_WHOLE); - knot_rrset_clear(&output_rr,&global_mm); + knot_rrset_clear(&output_rr, &global_mm); assert_false(res_cmp_fail); } /* Test cache read */ static void test_query(void **state) { - + struct kr_cache *cache = (*state); knot_rrset_t cache_rr; knot_rrset_init(&cache_rr, global_rr.owner, global_rr.type, global_rr.rclass); - struct kr_cache_txn *txn = test_txn_rdonly(state); - for (uint32_t timestamp = CACHE_TIME; timestamp < CACHE_TIME + CACHE_TTL; ++timestamp) { uint8_t rank = 0; uint8_t flags = 0; uint32_t drift = timestamp; - int query_ret = kr_cache_peek_rr(txn, &cache_rr, &rank, &flags, &drift); + int query_ret = kr_cache_peek_rr(cache, &cache_rr, &rank, &flags, &drift); bool rr_equal = knot_rrset_equal(&global_rr, &cache_rr, KNOT_RRSET_COMPARE_WHOLE); - assert_int_equal(query_ret, KNOT_EOK); + assert_int_equal(query_ret, 0); assert_true(rr_equal); } - - kr_cache_txn_abort(txn); } /* Test cache read (simulate aged entry) */ @@ -336,10 +280,9 @@ static void test_query_aged(void **state) knot_rrset_t cache_rr; knot_rrset_init(&cache_rr, global_rr.owner, global_rr.type, global_rr.rclass); - struct kr_cache_txn *txn = test_txn_rdonly(state); - int ret = kr_cache_peek_rr(txn, &cache_rr, &rank, &flags, ×tamp); + struct kr_cache *cache = (*state); + int ret = kr_cache_peek_rr(cache, &cache_rr, &rank, &flags, ×tamp); assert_int_equal(ret, kr_error(ESTALE)); - kr_cache_txn_abort(txn); } /* Test cache removal */ @@ -351,56 +294,41 @@ static void test_remove(void **state) knot_rrset_t cache_rr; knot_rrset_init(&cache_rr, global_rr.owner, global_rr.type, global_rr.rclass); - struct kr_cache_txn *txn = test_txn_write(state); - int ret = kr_cache_remove(txn, KR_CACHE_RR, cache_rr.owner, cache_rr.type); - assert_int_equal(ret, KNOT_EOK); - ret = kr_cache_peek_rr(txn, &cache_rr, &rank, &flags, ×tamp); + struct kr_cache *cache = (*state); + int ret = kr_cache_remove(cache, KR_CACHE_RR, cache_rr.owner, cache_rr.type); + assert_int_equal(ret, 0); + ret = kr_cache_peek_rr(cache, &cache_rr, &rank, &flags, ×tamp); assert_int_equal(ret, KNOT_ENOENT); - kr_cache_txn_commit(txn); } /* Test cache fill */ static void test_fill(void **state) { - struct kr_cache_txn *txn = test_txn_write(state); + struct kr_cache *cache = (*state); /* Fill with random values. */ - int ret = KNOT_EOK; + int ret = 0; for (unsigned i = 0; i < CACHE_SIZE; ++i) { knot_rrset_t rr; test_random_rr(&rr, CACHE_TTL); - ret = kr_cache_insert_rr(txn, &rr, 0, 0, CACHE_TTL - 1); - if (ret != KNOT_EOK) { + ret = kr_cache_insert_rr(cache, &rr, 0, 0, CACHE_TTL - 1); + if (ret != 0) { break; } - /* Intermediate commit */ - if (i % 10 == 0) { - ret = kr_cache_txn_commit(txn); - if (ret != KNOT_EOK) { - txn = NULL; - break; - } - txn = test_txn_write(state); - } } - /* Abort last transaction (if valid) */ - kr_cache_txn_abort(txn); - /* Expect we run out of space */ - assert_int_equal(ret, KNOT_ESPACE); + assert_int_equal(ret, kr_error(ENOSPC)); } /* Test cache clear */ static void test_clear(void **state) { - struct kr_cache_txn *txn = test_txn_write(state); - int preempt_ret = kr_cache_clear(txn); - int count_ret = txn->owner->api->count(&txn->t); - int commit_ret = kr_cache_txn_commit(txn); + struct kr_cache *cache = (*state); + int preempt_ret = kr_cache_clear(cache); + int count_ret = cache->api->count(cache->db); - assert_int_equal(preempt_ret, KNOT_EOK); - assert_int_equal(commit_ret, KNOT_EOK); + assert_int_equal(preempt_ret, 0); assert_int_equal(count_ret, 1); /* Version record */ } diff --git a/tests/test_zonecut.c b/tests/test_zonecut.c index e090b6c44..ff4791eff 100644 --- a/tests/test_zonecut.c +++ b/tests/test_zonecut.c @@ -32,7 +32,7 @@ static void test_zonecut_params(void **state) assert_null((void *)kr_zonecut_find(NULL, NULL)); assert_null((void *)kr_zonecut_find(&cut, NULL)); assert_int_not_equal(kr_zonecut_set_sbelt(NULL, NULL), 0); - assert_int_not_equal(kr_zonecut_find_cached(NULL, NULL, NULL, NULL, 0, 0), 0); + assert_int_not_equal(kr_zonecut_find_cached(NULL, NULL, NULL, 0, 0), 0); } static void test_zonecut_copy(void **state) diff --git a/tests/unit.mk b/tests/unit.mk index 9e1689095..6ab0e6703 100644 --- a/tests/unit.mk +++ b/tests/unit.mk @@ -20,7 +20,7 @@ $(eval $(call make_lib,mock_cmodule,tests)) # Dependencies tests_DEPEND := $(libkres) $(mock_cmodule) $(mock_gomodule) -tests_LIBS := $(libkres_TARGET) $(libkres_LIBS) $(cmocka_LIBS) +tests_LIBS := $(libkres_TARGET) $(libkres_LIBS) $(cmocka_LIBS) $(lmdb_LIBS) # Platform-specific library injection ifeq ($(PLATFORM),Darwin)