From: Aki Tuomi Date: Fri, 10 Sep 2021 06:52:28 +0000 (+0300) Subject: lib-dns: Add caching to dns client X-Git-Tag: 2.4.0~4740 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=47f44374201e83ade6204bb237b163fbd066134a;p=thirdparty%2Fdovecot%2Fcore.git lib-dns: Add caching to dns client Caching can be used with the client API. Setting cache_ttl_secs to a suitable value will cache the entries for given seconds. Entries are refreshed if looked up after half the ttl has elapsed. --- diff --git a/src/lib-dns/dns-lookup.c b/src/lib-dns/dns-lookup.c index ac769c4902..176f9dfa53 100644 --- a/src/lib-dns/dns-lookup.c +++ b/src/lib-dns/dns-lookup.c @@ -4,6 +4,8 @@ #include "ioloop.h" #include "str.h" #include "array.h" +#include "hash.h" +#include "priorityq.h" #include "ostream.h" #include "connection.h" #include "lib-event.h" @@ -22,11 +24,17 @@ static struct event_category event_category_dns = { .name = "dns" }; +struct dns_cache_lookup { + struct dns_client *client; + char *key; +}; + struct dns_lookup { struct dns_lookup *prev, *next; struct dns_client *client; pool_t pool; bool ptr_lookup; + bool cached; struct timeout *to; @@ -35,26 +43,231 @@ struct dns_lookup { struct dns_lookup_result result; struct event *event; + const char *cache_key; /* cache lookup key */ dns_lookup_callback_t *callback; void *context; }; +struct dns_client_cache_entry { + struct priorityq_item item; + time_t expires; + unsigned int ips_count; + bool refresh:1; + bool refreshing:1; + + char *cache_key; + char *name; + struct ip_addr *ips; +}; + struct dns_client { struct connection conn; struct connection_list *clist; struct dns_lookup *head, *tail; struct timeout *to_idle; struct ioloop *ioloop; + struct timeout *to_cache_clean; char *path; + HASH_TABLE(char *, struct dns_client_cache_entry *) cache_table; + struct priorityq *cache_queue; unsigned int timeout_msecs; unsigned int idle_timeout_msecs; + unsigned int cache_ttl_secs; bool connected:1; bool deinit_client_at_free:1; }; +/* cache code */ +static int dns_client_cache_entry_cmp(const void *p1, const void *p2) +{ + const struct dns_client_cache_entry *entry1 = p1; + const struct dns_client_cache_entry *entry2 = p2; + return entry1->expires - entry2->expires; +} + +static void dns_client_cache_entry_free(struct dns_client_cache_entry **_entry) +{ + struct dns_client_cache_entry *entry = *_entry; + *_entry = NULL; + i_free(entry->ips); + i_free(entry->name); + i_free(entry->cache_key); + i_free(entry); +} + +static void dns_client_cache_entry(struct dns_client *client, + const struct dns_lookup *lookup) +{ + if (client->cache_ttl_secs == 0) + return; + struct dns_client_cache_entry *entry = + hash_table_lookup(client->cache_table, lookup->cache_key); + if (lookup->result.ret < 0) { + if (entry != NULL) + entry->refreshing = FALSE; + return; + } + if (entry != NULL) { + /* remove entry */ + priorityq_remove(client->cache_queue, &entry->item); + hash_table_remove(client->cache_table, entry->cache_key); + dns_client_cache_entry_free(&entry); + } + entry = i_new(struct dns_client_cache_entry, 1); + entry->expires = ioloop_time + client->cache_ttl_secs; + entry->cache_key = i_strdup(lookup->cache_key); + entry->name = i_strdup(lookup->result.name); + entry->ips_count = lookup->result.ips_count; + if (lookup->result.ips_count > 0) { + entry->ips = i_memdup(lookup->result.ips, + sizeof(struct ip_addr) * lookup->result.ips_count); + } + priorityq_add(client->cache_queue, &entry->item); + hash_table_insert(client->cache_table, entry->cache_key, entry); +} + +static void dns_cache_lookup_free(struct dns_cache_lookup **_ctx) +{ + struct dns_cache_lookup *ctx = *_ctx; + *_ctx = NULL; + + i_free(ctx->key); + i_free(ctx); +} + +static void dns_client_cache_callback(const struct dns_lookup_result *result, + struct dns_cache_lookup *ctx) +{ + if (result->ret < 0) + e_debug(ctx->client->conn.event, "Background entry refresh failed for %s '%s': %s", + *ctx->key == 'I' ? "IP" : "name", + ctx->key + 1, result->error); + dns_cache_lookup_free(&ctx); +} + +static void dns_client_cache_entry_refresh(struct dns_client *client, + struct dns_client_cache_entry *entry) +{ + struct dns_lookup *lookup; + struct dns_cache_lookup *ctx; + /* about to expire, next lookup should go to client */ + entry->refresh = TRUE; + if (*entry->cache_key == 'I') { + struct ip_addr ip; + if (net_addr2ip(entry->cache_key + 1, &ip) < 0) + i_unreached(); + ctx = i_new(struct dns_cache_lookup, 1); + ctx->key = i_strdup(entry->cache_key); + if (dns_client_lookup_ptr(client, &ip, + dns_client_cache_callback, + ctx, &lookup) < 0) { + e_debug(client->conn.event, + "Cannot refresh IP '%s' (trying again later)", + entry->cache_key + 1); + dns_cache_lookup_free(&ctx); + } else { + /* ensure we don't trigger this again. this gets + changed in dns_client_cache_entry(). */ + entry->refreshing = TRUE; + } + } else if (*entry->cache_key == 'N') { + ctx = i_new(struct dns_cache_lookup, 1); + ctx->key = i_strdup(entry->cache_key); + if (dns_client_lookup(client, entry->cache_key + 1, + dns_client_cache_callback, + ctx, &lookup) < 0) { + e_debug(client->conn.event, + "Cannot refresh name '%s' (trying again later)", + entry->cache_key + 1); + dns_cache_lookup_free(&ctx); + } else { + entry->refreshing = TRUE; + } + } else { + i_unreached(); + } + /* reset back to false to allow further lookups to use cache while + the entry is being refreshed. */ + entry->refresh = FALSE; +} + +static bool dns_client_cache_lookup(struct dns_client *client, + struct dns_lookup *lookup) +{ + if (client->cache_ttl_secs == 0) + return FALSE; + struct dns_client_cache_entry *entry = + hash_table_lookup(client->cache_table, lookup->cache_key); + if (entry == NULL) + return FALSE; + if (entry->expires <= ioloop_time) { + priorityq_remove(client->cache_queue, &entry->item); + hash_table_remove(client->cache_table, entry->cache_key); + dns_client_cache_entry_free(&entry); + return FALSE; + } + if (entry->refresh) + return FALSE; + lookup->result.ret = 0; + lookup->result.name = p_strdup(lookup->pool, entry->name); + lookup->result.ips_count = entry->ips_count; + if (entry->ips_count > 0) { + lookup->result.ips = + p_memdup(lookup->pool, entry->ips, + sizeof(struct ip_addr) * entry->ips_count); + } + lookup->cached = TRUE; + if (!entry->refreshing && + entry->expires <= ioloop_time + client->cache_ttl_secs / 2) + dns_client_cache_entry_refresh(client, entry); + return TRUE; +} + +static void dns_client_cache_clean(struct dns_client *client) +{ + while (priorityq_count(client->cache_queue) > 0) { + struct priorityq_item *item = priorityq_peek(client->cache_queue); + struct dns_client_cache_entry *entry = + container_of(item, struct dns_client_cache_entry, item); + if (entry->expires <= ioloop_time) { + /* drop item */ + (void)priorityq_pop(client->cache_queue); + hash_table_remove(client->cache_table, entry->cache_key); + dns_client_cache_entry_free(&entry); + } else { + /* no more entries that need attention */ + break; + } + } +} + +static void dns_client_cache_init(struct dns_client *client, unsigned int ttl_secs) +{ + client->cache_ttl_secs = ttl_secs; + hash_table_create(&client->cache_table, default_pool, 0, strfastcase_hash, + strcmp); + client->cache_queue = priorityq_init(dns_client_cache_entry_cmp, 0); + client->to_cache_clean = timeout_add((client->cache_ttl_secs/2)*1000, + dns_client_cache_clean, client); +} + +static void dns_client_cache_deinit(struct dns_client *client) +{ + while (priorityq_count(client->cache_queue) > 0) { + struct priorityq_item *item = priorityq_pop(client->cache_queue); + struct dns_client_cache_entry *entry = + container_of(item, struct dns_client_cache_entry, item); + hash_table_remove(client->cache_table, entry->cache_key); + dns_client_cache_entry_free(&entry); + } + timeout_remove(&client->to_cache_clean); + hash_table_destroy(&client->cache_table); + priorityq_deinit(&client->cache_queue); +} + #undef dns_lookup #undef dns_lookup_ptr #undef dns_client_lookup @@ -78,12 +291,20 @@ static void dns_lookup_callback(struct dns_lookup *lookup) e_debug(e->event(), "Lookup failed after %u msecs: %s", lookup->result.msecs, lookup->result.error); } else { + e->add_str("cached", lookup->cached ? "yes" : "no"); e_debug(e->event(), "Lookup successful after %u msecs", lookup->result.msecs); } lookup->callback(&lookup->result, lookup->context); } +static void dns_lookup_callback_cached(struct dns_lookup *lookup) +{ + timeout_remove(&lookup->to); + dns_lookup_callback(lookup); + dns_lookup_free(&lookup); +} + static void dns_client_disconnect(struct dns_client *client, const char *error) { struct dns_lookup *lookup, *next; @@ -179,6 +400,7 @@ static int dns_client_input_args(struct connection *conn, const char *const *arg return -1; } else if (ret > 0) { dns_lookup_callback(lookup); + dns_client_cache_entry(client, lookup); retry = !lookup->client->deinit_client_at_free; dns_lookup_free(&lookup); } @@ -200,6 +422,7 @@ int dns_lookup(const char *host, const struct dns_lookup_settings *set, { struct dns_client *client; + i_assert(set->cache_ttl_secs == 0); client = dns_client_init(set); event_add_category(client->conn.event, &event_category_dns); client->deinit_client_at_free = TRUE; @@ -213,6 +436,7 @@ int dns_lookup_ptr(const struct ip_addr *ip, { struct dns_client *client; + i_assert(set->cache_ttl_secs == 0); client = dns_client_init(set); event_add_category(client->conn.event, &event_category_dns); client->deinit_client_at_free = TRUE; @@ -303,6 +527,8 @@ struct dns_client *dns_client_init(const struct dns_lookup_settings *set) client->ioloop = set->ioloop == NULL ? current_ioloop : set->ioloop; client->path = i_strdup(set->dns_client_socket_path); client->conn.event_parent=set->event_parent; + if (set->cache_ttl_secs > 0) + dns_client_cache_init(client, set->cache_ttl_secs); connection_init_client_unix(client->clist, &client->conn, client->path); return client; } @@ -318,6 +544,10 @@ void dns_client_deinit(struct dns_client **_client) /* dns_client_disconnect() is supposed to clear out all queries */ i_assert(client->head == NULL); connection_list_deinit(&clist); + + if (client->cache_ttl_secs > 0) + dns_client_cache_deinit(client); + i_free(client->path); i_free(client); } @@ -380,12 +610,20 @@ dns_client_lookup_common(struct dns_client *client, lookup->ptr_lookup = ptr_lookup; lookup->result.ret = EAI_FAIL; lookup->event = event_create(client->conn.event); + lookup->cache_key = p_strdup_printf(lookup->pool, "%c%s", + ptr_lookup ? 'I' : 'N', param); event_set_append_log_prefix(lookup->event, t_strconcat("dns(", param, "): ", NULL)); struct event_passthrough *e = event_create_passthrough(lookup->event)-> set_name("dns_request_started"); e_debug(e->event(), "Lookup started"); + if (dns_client_cache_lookup(client, lookup)) { + lookup->to = timeout_add_short(0, dns_lookup_callback_cached, + lookup); + return 0; + } + if ((ret = dns_client_send_request(client, cmd, &lookup->result.error)) <= 0) { if (ret == 0) { /* retry once */ diff --git a/src/lib-dns/dns-lookup.h b/src/lib-dns/dns-lookup.h index 8256d739a8..615528baa9 100644 --- a/src/lib-dns/dns-lookup.h +++ b/src/lib-dns/dns-lookup.h @@ -11,6 +11,9 @@ struct dns_lookup_settings { /* the idle_timeout_msecs works only with the dns_client_* API. 0 = disconnect immediately */ unsigned int idle_timeout_msecs; + /* Non-zero enables caching for the client, is not supported with + dns_lookup() or dns_lookup_ptr(). Note that DNS TTL is ignored. */ + unsigned int cache_ttl_secs; /* ioloop to run the lookup on (defaults to current_ioloop) */ struct ioloop *ioloop; diff --git a/src/lib-dns/test-dns-lookup.c b/src/lib-dns/test-dns-lookup.c index 9db7fdb931..a2a982419d 100644 --- a/src/lib-dns/test-dns-lookup.c +++ b/src/lib-dns/test-dns-lookup.c @@ -252,12 +252,88 @@ static void test_dns_lookup_abort(void) test_end(); } +static void test_dns_lookup_cached(void) +{ + struct test_expect_result ctx; + struct dns_lookup *lookup; + struct timeout *to; + + test_begin("dns lookup (cached)"); + create_dns_server(&test_server); + const struct dns_lookup_settings set = { + .dns_client_socket_path = TEST_SOCKET_NAME, + .ioloop = test_server.loop, + .timeout_msecs = 1000, + .cache_ttl_secs = 4, + }; + + + struct dns_client *client = dns_client_init(&set); + + /* lookup localhost */ + ctx.result = "127.0.0.1\t::1"; + ctx.ret = 0; + + /* should cause only one lookup */ + test_assert(dns_client_lookup(client, "localhost", test_callback_ips, + &ctx, &lookup) == 0); + io_loop_run(current_ioloop); + test_assert(dns_client_lookup(client, "localhost", test_callback_ips, + &ctx, &lookup) == 0); + io_loop_run(current_ioloop); + test_assert_cmp(test_server.lookup_counter, ==, 1); + + to = timeout_add(3*1000, io_loop_stop, test_server.loop); + io_loop_run(current_ioloop); + timeout_remove(&to); + + /* entry should get refreshed */ + test_assert(dns_client_lookup(client, "localhost", test_callback_ips, + &ctx, &lookup) == 0); + io_loop_run(current_ioloop); + while (dns_client_has_pending_queries(client)) { + io_loop_handler_run(current_ioloop); + io_loop_set_running(current_ioloop); + } + test_assert_cmp(test_server.lookup_counter, ==, 2); + + /* should get looked up again */ + to = timeout_add(5*1000, io_loop_stop, test_server.loop); + io_loop_run(current_ioloop); + timeout_remove(&to); + + test_assert(dns_client_lookup(client, "localhost", test_callback_ips, + &ctx, &lookup) == 0); + io_loop_run(current_ioloop); + + test_assert_cmp(test_server.lookup_counter, ==, 3); + + /* Ensure failures do not get cached */ + ctx.result = NULL; + ctx.ret = -1; + test_assert(dns_client_lookup(client, "failhost", test_callback_ips, + &ctx, &lookup) == 0); + io_loop_run(current_ioloop); + test_assert_cmp(test_server.lookup_counter, ==, 4); + + test_assert(dns_client_lookup(client, "failhost", test_callback_ips, + &ctx, &lookup) == 0); + io_loop_run(current_ioloop); + test_assert_cmp(test_server.lookup_counter, ==, 5); + + dns_client_deinit(&client); + destroy_dns_server(&test_server); + + test_end(); +} + int main(void) { static void (*const test_functions[])(void) = { test_dns_lookup, test_dns_lookup_timeout, test_dns_lookup_abort, + test_dns_lookup_cached, NULL };