From: Libor Peltan Date: Mon, 28 Jul 2025 15:03:37 +0000 (+0200) Subject: redis: load from DB within zone loading X-Git-Tag: v3.5.0~11^2~4 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e1bdc3eda3a96e09197838a930b1c0055fdb5ade;p=thirdparty%2Fknot-dns.git redis: load from DB within zone loading --- diff --git a/src/knot/events/handlers/load.c b/src/knot/events/handlers/load.c index 3073a7af22..3fe68262bd 100644 --- a/src/knot/events/handlers/load.c +++ b/src/knot/events/handlers/load.c @@ -14,6 +14,7 @@ #include "knot/events/handlers.h" #include "knot/events/replan.h" #include "knot/zone/digest.h" +#include "knot/zone/redis.h" #include "knot/zone/reverse.h" #include "knot/zone/serial.h" #include "knot/zone/zone-diff.h" @@ -44,11 +45,18 @@ static bool allowed_xfr(conf_t *conf, const zone_t *zone) return false; } +static int upd_add_rem(const knot_rrset_t *rr, bool add, void *ctx) +{ + return add ? zone_update_add(ctx, rr) : zone_update_remove(ctx, rr); +} + int event_load(conf_t *conf, zone_t *zone) { zone_update_t up = { 0 }; zone_contents_t *journal_conts = NULL, *zf_conts = NULL; bool old_contents_exist = (zone->contents != NULL), zone_in_journal_exists = false; + const char *zone_src = "zone file"; + struct redisContext *db_ctx = NULL; conf_val_t val = conf_zone_get(conf, C_JOURNAL_CONTENT, zone->name); unsigned load_from = conf_opt(&val); @@ -98,8 +106,76 @@ int event_load(conf_t *conf, zone_t *zone) unsigned digest_alg = conf_opt(&val); bool update_zonemd = (digest_alg != ZONE_DIGEST_NONE); + uint8_t db_instance = 0; + bool db_enabled = conf_zone_rdb_enabled(conf, zone->name, true, &db_instance); + if (db_enabled) { + zone_src = "database"; + db_ctx = zone_redis_connect(conf); + } + + // Attempt to load changes from database. If fails, load full zone from there later. + if (db_enabled && (old_contents_exist || journal_conts != NULL) && + zone->cat_members == NULL && EMPTY_LIST(zone->include_from) && + zf_from != ZONEFILE_LOAD_DIFSE) { + zone_redis_err_t err; + uint32_t db_serial = 0; + ret = zone_redis_serial(db_ctx, db_instance, zone->name, &db_serial, err); + if (ret == KNOT_EOK && old_contents_exist && db_serial == zone_contents_serial(zone->contents)) { + log_zone_info(zone->name, "database is up-to-date, serial %u", db_serial); + goto cleanup; + } else if (ret == KNOT_EOK && journal_conts != NULL && db_serial == zone_contents_serial(journal_conts)) { + log_zone_info(zone->name, "database is up-to-date with zone-in-journal, serial %u", db_serial); + assert(!old_contents_exist); + db_enabled = false; // skip both zone_redis_load_upd() and zone_redis_load(), just load from journal. Also skip zone_update_semcheck() later as we do not in fact load from DB. + } else if (ret != KNOT_EOK) { + log_zone_error(zone->name, "failed to get database status (%s)", + ret == KNOT_ERDB ? err : knot_strerror(ret)); + goto cleanup; // NOTE this includes the case of KNOT_ENOENT, where DB load is configured but not available + } + + if (old_contents_exist) { + ret = zone_update_init(&up, zone, UPDATE_INCREMENTAL); + } else { + ret = zone_update_from_contents(&up, zone, journal_conts, UPDATE_HYBRID); + } + if (ret != KNOT_EOK) { + log_zone_error(zone->name, "failed to initialize update (%s)", knot_strerror(ret)); + goto cleanup; + } + if (db_enabled) { + uint32_t serial_current = zone_contents_serial(up.new_cont); + ret = zone_redis_load_upd(db_ctx, db_instance, zone->name, serial_current, + upd_add_rem, &up, err); + if (ret == KNOT_EOK) { + log_zone_info(zone->name, "database updates loaded, instance %u, serial %u -> %u", + db_instance, serial_current, zone_contents_serial(up.new_cont)); + } + } + if (ret == KNOT_EOK) { + goto load_end; // all OK, skip zone_redis_load() and proceed with incremental zone_update + } else if (ret == KNOT_ERDB) { + log_zone_error(zone->name, "failed to load updates from database (%s)", err); + goto cleanup; // Redis error, surrender + } else { + zone_update_clear(&up); + ret = KNOT_EOK; // just unable to apply DB changesets atop running zone version, go ahead with full zone load from DB + } + } + // If configured, attempt to load zonefile. - if (zf_from != ZONEFILE_LOAD_NONE && zone->cat_members == NULL) { + if ((zf_from != ZONEFILE_LOAD_NONE || db_enabled) && zone->cat_members == NULL) { + if (db_enabled) { + zone_redis_err_t err; + ret = zone_redis_load(db_ctx, db_instance, zone->name, &zf_conts, err); + if (ret != KNOT_EOK) { + log_zone_error(zone->name, "failed to load from database (%s)", + ret == KNOT_ERDB ? err : knot_strerror(ret)); + goto cleanup; + } + zone->zonefile.serial = zone_contents_serial(zf_conts); // for logging + goto zonefile_loaded; + } + struct timespec mtime; char *filename = conf_zonefile(conf, zone->name); ret = zonefile_exists(filename, &mtime); @@ -136,6 +212,7 @@ int event_load(conf_t *conf, zone_t *zone) zone->zonefile.exists = (zf_conts != NULL); zone->zonefile.mtime = mtime; +zonefile_loaded: ; // If configured, add reverse records to zone contents const knot_dname_t *fail_fwd = NULL; ret = zones_reverse(&zone->include_from, zf_conts, &fail_fwd); @@ -155,12 +232,14 @@ int event_load(conf_t *conf, zone_t *zone) uint32_t serial = zone_contents_serial(relevant); uint32_t set = serial_next(serial, conf, zone->name, SERIAL_POLICY_AUTO, 1); zone_contents_set_soa_serial(zf_conts, set); - log_zone_info(zone->name, "zone file parsed, serial updated %u -> %u", - zone->zonefile.serial, set); + log_zone_info(zone->name, "%s loaded%s%.0u, serial updated %u -> %u", + zone_src, (db_enabled ? ", instance " : ""), + db_instance, zone->zonefile.serial, set); zone->zonefile.serial = set; } else { - log_zone_info(zone->name, "zone file parsed, serial %u", - zone->zonefile.serial); + log_zone_info(zone->name, "%s loaded%s%.0u, serial %u", + zone_src, (db_enabled ? ", instance " : ""), + db_instance, zone->zonefile.serial); } // If configured and appliable to zonefile, load journal changes. @@ -277,13 +356,13 @@ load_end: } break; case KNOT_ESEMCHECK: - log_zone_warning(zone->name, "zone file changed without SOA serial update"); + log_zone_warning(zone->name, "%s changed without SOA serial update", zone_src); break; case KNOT_ERANGE: if (serial_compare(zone->zonefile.serial, zone_contents_serial(zone->contents)) == SERIAL_INCOMPARABLE) { - log_zone_warning(zone->name, "zone file changed with incomparable SOA serial"); + log_zone_warning(zone->name, "%s changed with incomparable SOA serial", zone_src); } else { - log_zone_warning(zone->name, "zone file changed with decreased SOA serial"); + log_zone_warning(zone->name, "%s changed with decreased SOA serial", zone_src); } break; } @@ -296,6 +375,13 @@ load_end: zf_conts = NULL; journal_conts = NULL; + if (db_enabled) { + ret = zone_update_semcheck(conf, &up); + if (ret != KNOT_EOK) { + goto cleanup; + } + } + ret = zone_update_verify_digest(conf, &up); if (ret != KNOT_EOK) { goto cleanup; @@ -322,7 +408,7 @@ load_end: log_zone_warning(zone->name, "with automatic DNSSEC signing and outgoing transfers enabled, " "'zonefile-load: difference' should be set to avoid malformed " - "IXFR after manual zone file update"); + "IXFR after manual %s update", zone_src); } } else if (update_zonemd) { /* Don't update ZONEMD if no change and ZONEMD is up-to-date. @@ -426,6 +512,7 @@ load_end: if (!zone_timers_serial_notified(&zone->timers, new_serial)) { zone_schedule_notify(conf, zone, 0); } + zone_redis_disconnect(db_ctx); zone_skip_free(&skip); zone->started = true; @@ -438,6 +525,7 @@ cleanup: zone_update_clear(&up); zone_contents_deep_free(zf_conts); zone_contents_deep_free(journal_conts); + zone_redis_disconnect(db_ctx); zone_skip_free(&skip); zone->started = true; diff --git a/src/knot/zone/redis.c b/src/knot/zone/redis.c index e98ddd794f..7de92b3d1f 100644 --- a/src/knot/zone/redis.c +++ b/src/knot/zone/redis.c @@ -3,12 +3,13 @@ * For more information, see */ -#include "knot/zone/redis.h" - #include -#ifdef ENABLE_REDIS +#include "knot/zone/redis.h" +#include "knot/zone/contents.h" +#ifdef ENABLE_REDIS +#include "contrib/openbsd/strlcpy.h" #include "knot/common/hiredis.h" struct redisContext *zone_redis_connect(conf_t *conf) @@ -169,6 +170,192 @@ int zone_redis_txn_abort(zone_redis_txn_t *txn) return KNOT_EOK; } +int zone_redis_serial(struct redisContext *rdb, uint8_t instance, + const knot_dname_t *zone, uint32_t *serial, + zone_redis_err_t err) +{ + if (rdb == NULL) { + return KNOT_NET_ECONNECT; + } else if (zone == NULL || serial == NULL || err == NULL) { + return KNOT_EINVAL; + } + + redisReply *reply = redisCommand(rdb, RDB_CMD_ZONE_EXISTS " %b %b", + zone, knot_dname_size(zone), + &instance, sizeof(instance)); + int ret = check_reply(rdb, reply, REDIS_REPLY_INTEGER, err); + if (ret != KNOT_EOK) { + freeReplyObject(reply); + return ret; + } + + *serial = reply->integer; + freeReplyObject(reply); + + return KNOT_EOK; +} + +static int process_rdb_rr(zone_contents_t *contents, redisReply *data) +{ + if (data->type != REDIS_REPLY_ARRAY || data->elements != 5) { + return KNOT_EMALF; + } + + knot_dname_t *r_owner = (knot_dname_t *)data->element[0]->str; + uint16_t r_type = data->element[1]->integer; + uint32_t r_ttl = data->element[2]->integer; + knot_rdataset_t r_data = { + .count = data->element[3]->integer, + .size = data->element[4]->len, + .rdata = (knot_rdata_t *)data->element[4]->str + }; + + knot_dname_t *owner = knot_dname_copy(r_owner, NULL); + if (owner == NULL) { + return KNOT_ENOMEM; + } + + knot_rrset_t rrs; + knot_rrset_init(&rrs, owner, r_type, KNOT_CLASS_IN, r_ttl); + + int ret = knot_rdataset_copy(&rrs.rrs, &r_data, NULL); + if (ret == KNOT_EOK) { + zone_node_t *n = NULL; + ret = zone_contents_add_rr(contents, &rrs, &n); + } + + knot_rrset_clear(&rrs, NULL); + + return ret; +} + +int zone_redis_load(struct redisContext *rdb, uint8_t instance, + const knot_dname_t *zone_name, struct zone_contents **out, + zone_redis_err_t err) +{ + if (rdb == NULL) { + return KNOT_NET_ECONNECT; + } else if (zone_name == NULL || out == NULL || err == NULL) { + return KNOT_EINVAL; + } + + redisReply *reply = redisCommand(rdb, RDB_CMD_ZONE_LOAD " %b %b", + zone_name, knot_dname_size(zone_name), + &instance, sizeof(instance)); + int ret = check_reply(rdb, reply, REDIS_REPLY_ARRAY, err); + if (ret != KNOT_EOK) { + freeReplyObject(reply); + return ret; + } + + zone_contents_t *cont = zone_contents_new(zone_name, true); + if (cont == NULL) { + freeReplyObject(reply); + return KNOT_ENOMEM; + } + + for (size_t i = 0; i < reply->elements; i++) { + redisReply *data = reply->element[i]; + ret = process_rdb_rr(cont, data); + if (ret != KNOT_EOK) { + break; + } + } + + if (ret == KNOT_EOK) { + *out = cont; + } else { + zone_contents_deep_free(cont); + } + + freeReplyObject(reply); + + return ret; +} + +static int process_rdb_upd(zone_redis_load_upd_cb_t cb, void *ctx, redisReply *data) +{ + if (data->type != REDIS_REPLY_ARRAY || data->elements != 8) { + return KNOT_EMALF; + } + + knot_dname_t *r_owner = (knot_dname_t *)data->element[0]->str; + uint16_t r_type = data->element[1]->integer; + uint32_t r_ttl_rem = data->element[2]->integer; + uint32_t r_ttl_add = data->element[3]->integer; + knot_rdataset_t r_data_rem = { + .count = data->element[4]->integer, + .size = data->element[5]->len, + .rdata = (knot_rdata_t *)data->element[5]->str + }; + knot_rdataset_t r_data_add = { + .count = data->element[6]->integer, + .size = data->element[7]->len, + .rdata = (knot_rdata_t *)data->element[7]->str + }; + + knot_dname_t *owner = knot_dname_copy(r_owner, NULL); + if (owner == NULL) { + return KNOT_ENOMEM; + } + + knot_rrset_t rrs; + knot_rrset_init(&rrs, owner, r_type, KNOT_CLASS_IN, r_ttl_rem); + + int ret = knot_rdataset_copy(&rrs.rrs, &r_data_rem, NULL); + if (ret == KNOT_EOK) { + ret = cb(&rrs, false, ctx); + } + if (ret == KNOT_EOK) { + knot_rdataset_clear(&rrs.rrs, NULL); + ret = knot_rdataset_copy(&rrs.rrs, &r_data_add, NULL); + rrs.ttl = r_ttl_add; + } + if (ret == KNOT_EOK) { + ret = cb(&rrs, true, ctx); + } + + knot_rrset_clear(&rrs, NULL); + + return ret; +} + +int zone_redis_load_upd(struct redisContext *rdb, uint8_t instance, + const knot_dname_t *zone_name, uint32_t soa_from, + zone_redis_load_upd_cb_t cb, void *ctx, + zone_redis_err_t err) +{ + if (rdb == NULL) { + return KNOT_NET_ECONNECT; + } else if (zone_name == NULL || cb == NULL || err == NULL) { + return KNOT_EINVAL; + } + + redisReply *reply = redisCommand(rdb, RDB_CMD_UPD_LOAD " %b %b %d", + zone_name, knot_dname_size(zone_name), + &instance, sizeof(instance), soa_from); + int ret = check_reply(rdb, reply, REDIS_REPLY_ARRAY, err); + if (ret != KNOT_EOK) { + freeReplyObject(reply); + return ret; + } + + for (size_t i = 0; i < reply->elements && ret == KNOT_EOK; i++) { + redisReply *changeset = reply->element[i]; + for (size_t j = 0; j < changeset->elements && ret == KNOT_EOK; j++) { + redisReply *data = changeset->element[j]; + ret = process_rdb_upd(cb, ctx, data); + if (ret != KNOT_EOK) { + break; + } + } + } + + freeReplyObject(reply); + + return ret; +} + #else // ENABLE_REDIS struct redisContext *zone_redis_connect(conf_t *conf) @@ -208,4 +395,26 @@ int zone_redis_txn_abort(zone_redis_txn_t *txn) return KNOT_ENOTSUP; } +int zone_redis_serial(struct redisContext *rdb, uint8_t instance, + const knot_dname_t *zone, uint32_t *serial, + zone_redis_err_t err) +{ + return KNOT_ENOTSUP; +} + +int zone_redis_load(struct redisContext *rdb, uint8_t instance, + const knot_dname_t *zone_name, struct zone_contents **out, + zone_redis_err_t err) +{ + return KNOT_ENOTSUP; +} + +int zone_redis_load_upd(struct redisContext *rdb, uint8_t instance, + const knot_dname_t *zone_name, uint32_t soa_from, + zone_redis_load_upd_cb_t cb, void *ctx, + zone_redis_err_t err) +{ + return KNOT_ENOTSUP; +} + #endif // ENABLE_REDIS diff --git a/src/knot/zone/redis.h b/src/knot/zone/redis.h index 203c06cbb8..9b347cd72c 100644 --- a/src/knot/zone/redis.h +++ b/src/knot/zone/redis.h @@ -14,6 +14,7 @@ #else // ENABLE_REDIS struct redisContext; #endif // ENABLE_REDIS +struct zone_contents; typedef char zone_redis_err_t[128]; @@ -80,3 +81,69 @@ int zone_redis_txn_commit(zone_redis_txn_t *txn); * \note You might want to ignore the return code. */ int zone_redis_txn_abort(zone_redis_txn_t *txn); + +/*! + * \brief Check if the zone exists in the database+instance and read out SOA serial. + * + * \param rdb Redis context (just pass zone_redis_connect()). + * \param instance Zone instance number (from configuration). + * \param zone Zone name. + * \param serial Output: SOA serial of stored zone. + * \param err Output: error message in case of Redis error. + * + * \retval KNOT_ERDB Redis-related error with err set. + * \return KNOT_E* + */ +int zone_redis_serial(struct redisContext *rdb, uint8_t instance, + const knot_dname_t *zone, uint32_t *serial, + zone_redis_err_t err); + +/*! + * \brief Load whole zone contents from Redis. + * + * \param rdb Redis context (just pass zone_redis_connect()). + * \param instance Zone instance number (from configuration). + * \param zone_name Zone name. + * \param out Output: zone contents. + * \param err Output: error message in case of Redis error. + * + * \retval KNOT_ERDB Redis-related error with err set. + * \return KNOT_E* + */ +int zone_redis_load(struct redisContext *rdb, uint8_t instance, + const knot_dname_t *zone_name, struct zone_contents **out, + zone_redis_err_t err); + +/*! + * \brief Callback type for handling data read by zone_redis_load_upd(). + * + * \param rr Loaded RRset. + * \param add The RRset is an addition in the changeset (removal otherwise). + * \param ctx Transparent context passed to zone_redis_load_upd(). + * + * \return KNOT_E* + */ +typedef int (*zone_redis_load_upd_cb_t)(const knot_rrset_t *rr, bool add, void *ctx); + +/*! + * \brief Load one or more changesets from Redis. + * + * \param rdb Redis context (just pass zone_redis_connect()). + * \param instance Zone instance number (from configuration). + * \param zone_name Zone name. + * \param soa_from SOA serial to start at. + * \param cb Callback to be called for each removed/added RRset. + * \param ctx Transparent context for the callback. + * \param err Output: error message in case of Redis error. + * + * \note In case of error, the callback might have been called several times, + * so that the real target structure (zone_update or whatever) might + * contain partial invalid data. + * + * \retval KNOT_ERDB Redis-related error with err set. + * \return KNOT_E* + */ +int zone_redis_load_upd(struct redisContext *rdb, uint8_t instance, + const knot_dname_t *zone_name, uint32_t soa_from, + zone_redis_load_upd_cb_t cb, void *ctx, + zone_redis_err_t err);