From: Libor Peltan Date: Fri, 13 Apr 2018 13:50:32 +0000 (+0200) Subject: kr_cache_gc: initial commit X-Git-Tag: v4.1.0^2~45 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bfb9686bea67b7856d24ccb6f3cdfdbd51e88d20;p=thirdparty%2Fknot-resolver.git kr_cache_gc: initial commit --- diff --git a/contrib/dynarray.h b/contrib/dynarray.h new file mode 100644 index 000000000..88559c0d6 --- /dev/null +++ b/contrib/dynarray.h @@ -0,0 +1,124 @@ +/* Copyright (C) 2017 CZ.NIC, z.s.p.o. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + */ + +/*! + * \brief Simple write-once allocation-optimal dynamic array. + * + * Include it into your .c file + * + * prefix - identifier prefix, e.g. ptr -> struct ptr_dynarray, ptr_dynarray_add(), ... + * ntype - data type to be stored. Let it be a number, pointer or small struct + * initial_capacity - how many data items will be allocated on stac and copied with assignment + * + * prefix_dynarray_add() - add a data item + * prefix_dynarray_fix() - call EVERYTIME the array is copied from some already invalid stack + * prefix_dynarray_free() - call EVERYTIME you dismiss all copies of the array + * + */ + +#include +#include + +#pragma once + +#define DYNARRAY_VISIBILITY_STATIC static +#define DYNARRAY_VISIBILITY_PUBLIC +#define DYNARRAY_VISIBILITY_LIBRARY __public__ + +#define dynarray_declare(prefix, ntype, visibility, initial_capacity) \ + typedef struct prefix ## _dynarray { \ + ssize_t capacity; \ + ssize_t size; \ + ntype *(*arr)(struct prefix ## _dynarray *dynarray); \ + ntype init[initial_capacity]; \ + ntype *_arr; \ + } prefix ## _dynarray_t; \ + \ + visibility ntype *prefix ## _dynarray_arr(prefix ## _dynarray_t *dynarray); \ + visibility void prefix ## _dynarray_add(prefix ## _dynarray_t *dynarray, \ + ntype const *to_add); \ + visibility void prefix ## _dynarray_free(prefix ## _dynarray_t *dynarray); + +#define dynarray_foreach(prefix, ntype, ptr, array) \ + for (ntype *ptr = prefix ## _dynarray_arr(&(array)); \ + ptr < prefix ## _dynarray_arr(&(array)) + (array).size; ptr++) + +#define dynarray_define(prefix, ntype, visibility) \ + \ + static void prefix ## _dynarray_free__(struct prefix ## _dynarray *dynarray) \ + { \ + if (dynarray->capacity > sizeof(dynarray->init) / sizeof(*dynarray->init)) { \ + free(dynarray->_arr); \ + } \ + } \ + \ + __attribute__((unused)) \ + visibility ntype *prefix ## _dynarray_arr(struct prefix ## _dynarray *dynarray) \ + { \ + assert(dynarray->size <= dynarray->capacity); \ + return (dynarray->capacity <= sizeof(dynarray->init) / sizeof(*dynarray->init) ? \ + dynarray->init : dynarray->_arr); \ + } \ + \ + static ntype *prefix ## _dynarray_arr_init__(struct prefix ## _dynarray *dynarray) \ + { \ + assert(dynarray->capacity == sizeof(dynarray->init) / sizeof(*dynarray->init)); \ + return dynarray->init; \ + } \ + \ + static ntype *prefix ## _dynarray_arr_arr__(struct prefix ## _dynarray *dynarray) \ + { \ + assert(dynarray->capacity > sizeof(dynarray->init) / sizeof(*dynarray->init)); \ + return dynarray->_arr; \ + } \ + \ + __attribute__((unused)) \ + visibility void prefix ## _dynarray_add(struct prefix ## _dynarray *dynarray, \ + ntype const *to_add) \ + { \ + if (dynarray->capacity < 0) { \ + return; \ + } \ + if (dynarray->capacity == 0) { \ + dynarray->capacity = sizeof(dynarray->init) / sizeof(*dynarray->init); \ + dynarray->arr = prefix ## _dynarray_arr_init__; \ + } \ + if (dynarray->size >= dynarray->capacity) { \ + ssize_t new_capacity = dynarray->capacity * 2 + 1; \ + ntype *new_arr = calloc(new_capacity, sizeof(ntype)); \ + if (new_arr == NULL) { \ + prefix ## _dynarray_free__(dynarray); \ + dynarray->capacity = dynarray->size = -1; \ + return; \ + } \ + if (dynarray->capacity > 0) { \ + memcpy(new_arr, prefix ## _dynarray_arr(dynarray), \ + dynarray->capacity * sizeof(ntype)); \ + } \ + prefix ## _dynarray_free__(dynarray); \ + dynarray->_arr = new_arr; \ + dynarray->capacity = new_capacity; \ + dynarray->arr = prefix ## _dynarray_arr_arr__; \ + } \ + prefix ## _dynarray_arr(dynarray)[dynarray->size++] = *to_add; \ + } \ + \ + __attribute__((unused)) \ + visibility void prefix ## _dynarray_free(struct prefix ## _dynarray *dynarray) \ + { \ + prefix ## _dynarray_free__(dynarray); \ + memset(dynarray, 0, sizeof(*dynarray)); \ + } diff --git a/lib/cache/impl.h b/lib/cache/impl.h index 6431a6a56..83a82879f 100644 --- a/lib/cache/impl.h +++ b/lib/cache/impl.h @@ -63,6 +63,7 @@ struct entry_apex; /** Check basic consistency of entry_h for 'E' entries, not looking into ->data. * (for is_packet the length of data is checked) */ +KR_EXPORT struct entry_h * entry_h_consistent(knot_db_val_t data, uint16_t type); struct entry_apex * entry_apex_consistent(knot_db_val_t val); diff --git a/utils/kr_cache_gc/.gitignore b/utils/kr_cache_gc/.gitignore new file mode 100644 index 000000000..c27063b46 --- /dev/null +++ b/utils/kr_cache_gc/.gitignore @@ -0,0 +1,2 @@ +kr_cache_gc + diff --git a/utils/kr_cache_gc/Makefile b/utils/kr_cache_gc/Makefile new file mode 100644 index 000000000..e38925cd6 --- /dev/null +++ b/utils/kr_cache_gc/Makefile @@ -0,0 +1,7 @@ + +all: kr_cache_gc + +kr_cache_gc: kr_cache_gc.c ../../contrib/dynarray.h ../../lib/defines.h ../../lib/cache/api.h ../../lib/cache/impl.h + gcc -std=gnu99 -o $@ $< -Wl,-Bdynamic -L../../lib -lknot -lkres -I../.. -I../../contrib -I/usr/include/luajit-2.0 + + diff --git a/utils/kr_cache_gc/kr_cache_gc.c b/utils/kr_cache_gc/kr_cache_gc.c new file mode 100644 index 000000000..4f3555518 --- /dev/null +++ b/utils/kr_cache_gc/kr_cache_gc.c @@ -0,0 +1,176 @@ +// standard includes +#include +#include +#include + +// libknot includes +#include + +// resolver includes +#include +#include +#include +#include + +int64_t now = 1523701784; + +static const uint16_t *key_consistent(knot_db_val_t key) +{ + const static uint16_t NSEC1 = KNOT_RRTYPE_NSEC; + uint8_t *p = key.data; + while(*p != 0) { + while(*p++ != 0) { + if (p - (uint8_t *)key.data >= key.len) { + return NULL; + } + } + } + if (p - (uint8_t *)key.data >= key.len) { + return NULL; + } + switch (*++p) { + case 'E': + return (p + 2 - (uint8_t *)key.data >= key.len ? NULL : (uint16_t *)(p + 1)); + case '1': + return &NSEC1; + default: + return NULL; + } +} + +struct libknot_lmdb_env +{ + bool shared; + unsigned dbi; + void *env; + knot_mm_t *pool; +}; + +struct kres_lmdb_env +{ + size_t mapsize; + unsigned dbi; + void *env; + // sub-struct txn ommited +}; + +static knot_db_t *knot_db_t_kres2libknot(const knot_db_t *db) +{ + const struct kres_lmdb_env *kres_db = db; // this is struct lmdb_env as in resolver/cdb_lmdb.c + struct libknot_lmdb_env *libknot_db = malloc(sizeof(*libknot_db)); + if (libknot_db != NULL) { + libknot_db->shared = false; + libknot_db->pool = NULL; + libknot_db->env = kres_db->env; + libknot_db->dbi = kres_db->dbi; + } + return libknot_db; +} + +dynarray_declare(entry, knot_db_val_t, static, 256); +dynarray_define(entry, knot_db_val_t, static); + +int main(int argc, char *argv[]) +{ + if (argc < 2 || argv[1][0] == '-') { + printf("Usage: %s \n", argv[0]); + return 0; + } + + const char *cache = argv[1]; + char cache_data[strlen(cache) + 10]; + snprintf(cache_data, sizeof(cache_data), "%s/data.mdb", cache); + + struct stat st = { 0 }; + if (stat(cache, &st) || !(st.st_mode & S_IFDIR) || stat(cache_data, &st)) { + printf("Error: %s does not exist or is not a LMDB.\n", cache); + return 1; + } + + size_t cache_size = st.st_size; + + struct kr_cdb_opts opts = { cache, cache_size }; + struct kr_cache krc = { 0 }; + + int ret = kr_cache_open(&krc, NULL, &opts, NULL); + if (ret || krc.db == NULL) { + printf("Error opening Resolver cache (%s).\n", kr_strerror(ret)); + return 2; + } + + const knot_db_api_t *api = knot_db_lmdb_api(); + knot_db_txn_t txn = { 0 }; + knot_db_t *db = knot_db_t_kres2libknot(krc.db); + if (db == NULL) { + printf("Out of memory.\n"); + ret = KNOT_ENOMEM; + goto fail; + } + + ret = api->txn_begin(db, &txn, 0); + if (ret != KNOT_EOK) { + printf("Error starting DB transaction (%s).\n", knot_strerror(ret)); + goto fail; + } + + knot_db_iter_t *it = NULL; + it = api->iter_begin(&txn, KNOT_DB_FIRST); + if (it == NULL) { + printf("Error iterating DB.\n"); + ret = KNOT_ERROR; + goto fail; + } + + entry_dynarray_t to_del = { 0 }; + + while (it != NULL) { + knot_db_val_t key = { 0 }, val = { 0 }; + if ((ret = api->iter_key(it, &key)) != KNOT_EOK || + (ret = api->iter_val(it, &val)) != KNOT_EOK) { + printf("Warning: skipping a key due to error (%s).\n", knot_strerror(ret)); + } + const uint16_t *entry_type = ret == KNOT_EOK ? key_consistent(key) : NULL; + if (entry_type == NULL) { + printf("Inconsistent entry.\n"); + } else { + char type_s[32] = { 0 }; + knot_rrtype_to_string(*entry_type, type_s, sizeof(type_s)); + struct entry_h *entry = entry_h_consistent(val, *entry_type); + int64_t over = entry->time + entry->ttl; + over -= now; + printf("%s time=%u ttl=%u rel=%ld\n", type_s, entry->time, entry->ttl, over); + if (over < 0) { + entry_dynarray_add(&to_del, &key); + } + } + + it = api->iter_next(it); + } + + dynarray_foreach(entry, knot_db_val_t, i, to_del) { + ret = api->del(&txn, i); + if (ret != KNOT_EOK) { + printf("Warning: skipping deleting because of error (%s)\n", knot_strerror(ret)); + } + } + + entry_dynarray_free(&to_del); + + //api->iter_finish(it); + //it = NULL; + ret = api->txn_commit(&txn); + txn.txn = NULL; + +fail: + api->iter_finish(it); + if (txn.txn) { + api->txn_abort(&txn); + } + + free(db); + kr_cache_close(&krc); + + return (ret ? 10 : 0); +} + + diff --git a/utils/utils.mk b/utils/utils.mk new file mode 100644 index 000000000..4524f4613 --- /dev/null +++ b/utils/utils.mk @@ -0,0 +1,8 @@ + +utils: kr_cache_gc + +kr_cache_gc: + make -C utils/kr_cache_gc all + +.PHONY: utils kr_cache_gc +