lib-dns: Add caching to dns client

Caching can be used with the client API. Setting cache_ttl_secs
to a suitable value will cache the entries for given seconds.
Entries are refreshed if looked up after half the ttl has elapsed.

diff --git a/src/lib-dns/dns-lookup.c b/src/lib-dns/dns-lookup.c
index ac769c4902b8c94ab48b5f7f540fbc6ef0f145db..176f9dfa5369c0f15e6e2fb576804ef5cfba7b8c 100644 (file)
--- a/src/lib-dns/dns-lookup.c
+++ b/src/lib-dns/dns-lookup.c
@@ -4,6 +4,8 @@
 #include "ioloop.h"
 #include "str.h"
 #include "array.h"
+#include "hash.h"
+#include "priorityq.h"
 #include "ostream.h"
 #include "connection.h"
 #include "lib-event.h"
@@ -22,11 +24,17 @@ static struct event_category event_category_dns = {
        .name = "dns"
 };
 
+struct dns_cache_lookup {
+       struct dns_client *client;
+       char *key;
+};
+
 struct dns_lookup {
        struct dns_lookup *prev, *next;
        struct dns_client *client;
        pool_t pool;
        bool ptr_lookup;
+       bool cached;
 
        struct timeout *to;
 
@@ -35,26 +43,231 @@ struct dns_lookup {
 
        struct dns_lookup_result result;
        struct event *event;
+       const char *cache_key; /* cache lookup key */
 
        dns_lookup_callback_t *callback;
        void *context;
 };
 
+struct dns_client_cache_entry {
+       struct priorityq_item item;
+       time_t expires;
+       unsigned int ips_count;
+       bool refresh:1;
+       bool refreshing:1;
+
+       char *cache_key;
+       char *name;
+       struct ip_addr *ips;
+};
+
 struct dns_client {
        struct connection conn;
        struct connection_list *clist;
        struct dns_lookup *head, *tail;
        struct timeout *to_idle;
        struct ioloop *ioloop;
+       struct timeout *to_cache_clean;
        char *path;
+       HASH_TABLE(char *, struct dns_client_cache_entry *) cache_table;
+       struct priorityq *cache_queue;
 
        unsigned int timeout_msecs;
        unsigned int idle_timeout_msecs;
+       unsigned int cache_ttl_secs;
 
        bool connected:1;
        bool deinit_client_at_free:1;
 };
 
+/* cache code */
+static int dns_client_cache_entry_cmp(const void *p1, const void *p2)
+{
+       const struct dns_client_cache_entry *entry1 = p1;
+       const struct dns_client_cache_entry *entry2 = p2;
+       return entry1->expires - entry2->expires;
+}
+
+static void dns_client_cache_entry_free(struct dns_client_cache_entry **_entry)
+{
+       struct dns_client_cache_entry *entry = *_entry;
+       *_entry = NULL;
+       i_free(entry->ips);
+       i_free(entry->name);
+       i_free(entry->cache_key);
+       i_free(entry);
+}
+
+static void dns_client_cache_entry(struct dns_client *client,
+                                  const struct dns_lookup *lookup)
+{
+       if (client->cache_ttl_secs == 0)
+               return;
+       struct dns_client_cache_entry *entry =
+               hash_table_lookup(client->cache_table, lookup->cache_key);
+       if (lookup->result.ret < 0) {
+               if (entry != NULL)
+                       entry->refreshing = FALSE;
+               return;
+       }
+       if (entry != NULL) {
+               /* remove entry */
+               priorityq_remove(client->cache_queue, &entry->item);
+               hash_table_remove(client->cache_table, entry->cache_key);
+               dns_client_cache_entry_free(&entry);
+       }
+       entry = i_new(struct dns_client_cache_entry, 1);
+       entry->expires = ioloop_time + client->cache_ttl_secs;
+       entry->cache_key = i_strdup(lookup->cache_key);
+       entry->name = i_strdup(lookup->result.name);
+       entry->ips_count = lookup->result.ips_count;
+       if (lookup->result.ips_count > 0) {
+               entry->ips = i_memdup(lookup->result.ips,
+                                     sizeof(struct ip_addr) * lookup->result.ips_count);
+       }
+       priorityq_add(client->cache_queue, &entry->item);
+       hash_table_insert(client->cache_table, entry->cache_key, entry);
+}
+
+static void dns_cache_lookup_free(struct dns_cache_lookup **_ctx)
+{
+       struct dns_cache_lookup *ctx = *_ctx;
+       *_ctx = NULL;
+
+       i_free(ctx->key);
+       i_free(ctx);
+}
+
+static void dns_client_cache_callback(const struct dns_lookup_result *result,
+                                     struct dns_cache_lookup *ctx)
+{
+       if (result->ret < 0)
+               e_debug(ctx->client->conn.event, "Background entry refresh failed for %s '%s': %s",
+                       *ctx->key == 'I' ? "IP" : "name",
+                       ctx->key + 1, result->error);
+       dns_cache_lookup_free(&ctx);
+}
+
+static void dns_client_cache_entry_refresh(struct dns_client *client,
+                                          struct dns_client_cache_entry *entry)
+{
+       struct dns_lookup *lookup;
+       struct dns_cache_lookup *ctx;
+       /* about to expire, next lookup should go to client */
+       entry->refresh = TRUE;
+       if (*entry->cache_key == 'I') {
+               struct ip_addr ip;
+               if (net_addr2ip(entry->cache_key + 1, &ip) < 0)
+                       i_unreached();
+               ctx = i_new(struct dns_cache_lookup, 1);
+               ctx->key = i_strdup(entry->cache_key);
+               if (dns_client_lookup_ptr(client, &ip,
+                                         dns_client_cache_callback,
+                                         ctx, &lookup) < 0) {
+                       e_debug(client->conn.event,
+                               "Cannot refresh IP '%s' (trying again later)",
+                               entry->cache_key + 1);
+                       dns_cache_lookup_free(&ctx);
+               } else {
+                       /* ensure we don't trigger this again. this gets
+                          changed in dns_client_cache_entry(). */
+                       entry->refreshing = TRUE;
+               }
+       } else if (*entry->cache_key == 'N') {
+               ctx = i_new(struct dns_cache_lookup, 1);
+               ctx->key = i_strdup(entry->cache_key);
+               if (dns_client_lookup(client, entry->cache_key + 1,
+                                     dns_client_cache_callback,
+                                     ctx, &lookup) < 0) {
+                       e_debug(client->conn.event,
+                               "Cannot refresh name '%s' (trying again later)",
+                               entry->cache_key + 1);
+                       dns_cache_lookup_free(&ctx);
+               } else {
+                       entry->refreshing = TRUE;
+               }
+       } else {
+               i_unreached();
+       }
+       /* reset back to false to allow further lookups to use cache while
+          the entry is being refreshed. */
+       entry->refresh = FALSE;
+}
+
+static bool dns_client_cache_lookup(struct dns_client *client,
+                                   struct dns_lookup *lookup)
+{
+       if (client->cache_ttl_secs == 0)
+               return FALSE;
+       struct dns_client_cache_entry *entry =
+               hash_table_lookup(client->cache_table, lookup->cache_key);
+       if (entry == NULL)
+               return FALSE;
+       if (entry->expires <= ioloop_time) {
+               priorityq_remove(client->cache_queue, &entry->item);
+               hash_table_remove(client->cache_table, entry->cache_key);
+               dns_client_cache_entry_free(&entry);
+               return FALSE;
+       }
+       if (entry->refresh)
+               return FALSE;
+       lookup->result.ret = 0;
+       lookup->result.name = p_strdup(lookup->pool, entry->name);
+       lookup->result.ips_count = entry->ips_count;
+       if (entry->ips_count > 0) {
+               lookup->result.ips =
+                       p_memdup(lookup->pool, entry->ips,
+                                sizeof(struct ip_addr) * entry->ips_count);
+       }
+       lookup->cached = TRUE;
+       if (!entry->refreshing &&
+           entry->expires <= ioloop_time + client->cache_ttl_secs / 2)
+               dns_client_cache_entry_refresh(client, entry);
+       return TRUE;
+}
+
+static void dns_client_cache_clean(struct dns_client *client)
+{
+       while (priorityq_count(client->cache_queue) > 0) {
+               struct priorityq_item *item = priorityq_peek(client->cache_queue);
+               struct dns_client_cache_entry *entry =
+                       container_of(item, struct dns_client_cache_entry, item);
+               if (entry->expires <= ioloop_time) {
+                       /* drop item */
+                       (void)priorityq_pop(client->cache_queue);
+                       hash_table_remove(client->cache_table, entry->cache_key);
+                       dns_client_cache_entry_free(&entry);
+               } else {
+                       /* no more entries that need attention */
+                       break;
+               }
+       }
+}
+
+static void dns_client_cache_init(struct dns_client *client, unsigned int ttl_secs)
+{
+       client->cache_ttl_secs = ttl_secs;
+       hash_table_create(&client->cache_table, default_pool, 0, strfastcase_hash,
+                         strcmp);
+       client->cache_queue = priorityq_init(dns_client_cache_entry_cmp, 0);
+       client->to_cache_clean = timeout_add((client->cache_ttl_secs/2)*1000,
+                                              dns_client_cache_clean, client);
+}
+
+static void dns_client_cache_deinit(struct dns_client *client)
+{
+       while (priorityq_count(client->cache_queue) > 0) {
+               struct priorityq_item *item = priorityq_pop(client->cache_queue);
+               struct dns_client_cache_entry *entry =
+                       container_of(item, struct dns_client_cache_entry, item);
+               hash_table_remove(client->cache_table, entry->cache_key);
+               dns_client_cache_entry_free(&entry);
+       }
+       timeout_remove(&client->to_cache_clean);
+       hash_table_destroy(&client->cache_table);
+       priorityq_deinit(&client->cache_queue);
+}
+
 #undef dns_lookup
 #undef dns_lookup_ptr
 #undef dns_client_lookup
@@ -78,12 +291,20 @@ static void dns_lookup_callback(struct dns_lookup *lookup)
                e_debug(e->event(), "Lookup failed after %u msecs: %s",
                        lookup->result.msecs, lookup->result.error);
        } else {
+               e->add_str("cached", lookup->cached ? "yes" : "no");
                e_debug(e->event(), "Lookup successful after %u msecs",
                        lookup->result.msecs);
        }
        lookup->callback(&lookup->result, lookup->context);
 }
 
+static void dns_lookup_callback_cached(struct dns_lookup *lookup)
+{
+       timeout_remove(&lookup->to);
+       dns_lookup_callback(lookup);
+       dns_lookup_free(&lookup);
+}
+
 static void dns_client_disconnect(struct dns_client *client, const char *error)
 {
        struct dns_lookup *lookup, *next;
@@ -179,6 +400,7 @@ static int dns_client_input_args(struct connection *conn, const char *const *arg
                return -1;
        } else if (ret > 0) {
                dns_lookup_callback(lookup);
+               dns_client_cache_entry(client, lookup);
                retry = !lookup->client->deinit_client_at_free;
                dns_lookup_free(&lookup);
        }
@@ -200,6 +422,7 @@ int dns_lookup(const char *host, const struct dns_lookup_settings *set,
 {
        struct dns_client *client;
 
+       i_assert(set->cache_ttl_secs == 0);
        client = dns_client_init(set);
        event_add_category(client->conn.event, &event_category_dns);
        client->deinit_client_at_free = TRUE;
@@ -213,6 +436,7 @@ int dns_lookup_ptr(const struct ip_addr *ip,
 {
        struct dns_client *client;
 
+       i_assert(set->cache_ttl_secs == 0);
        client = dns_client_init(set);
        event_add_category(client->conn.event, &event_category_dns);
        client->deinit_client_at_free = TRUE;
@@ -303,6 +527,8 @@ struct dns_client *dns_client_init(const struct dns_lookup_settings *set)
        client->ioloop = set->ioloop == NULL ? current_ioloop : set->ioloop;
        client->path = i_strdup(set->dns_client_socket_path);
        client->conn.event_parent=set->event_parent;
+       if (set->cache_ttl_secs > 0)
+               dns_client_cache_init(client, set->cache_ttl_secs);
        connection_init_client_unix(client->clist, &client->conn, client->path);
        return client;
 }
@@ -318,6 +544,10 @@ void dns_client_deinit(struct dns_client **_client)
        /* dns_client_disconnect() is supposed to clear out all queries */
        i_assert(client->head == NULL);
        connection_list_deinit(&clist);
+
+       if (client->cache_ttl_secs > 0)
+               dns_client_cache_deinit(client);
+
        i_free(client->path);
        i_free(client);
 }
@@ -380,12 +610,20 @@ dns_client_lookup_common(struct dns_client *client,
        lookup->ptr_lookup = ptr_lookup;
        lookup->result.ret = EAI_FAIL;
        lookup->event = event_create(client->conn.event);
+       lookup->cache_key = p_strdup_printf(lookup->pool, "%c%s",
+                                     ptr_lookup ? 'I' : 'N', param);
        event_set_append_log_prefix(lookup->event, t_strconcat("dns(", param, "): ", NULL));
        struct event_passthrough *e =
                event_create_passthrough(lookup->event)->
                set_name("dns_request_started");
        e_debug(e->event(), "Lookup started");
 
+       if (dns_client_cache_lookup(client, lookup)) {
+               lookup->to = timeout_add_short(0, dns_lookup_callback_cached,
+                                              lookup);
+               return 0;
+       }
+
        if ((ret = dns_client_send_request(client, cmd, &lookup->result.error)) <= 0) {
                if (ret == 0) {
                        /* retry once */

diff --git a/src/lib-dns/dns-lookup.h b/src/lib-dns/dns-lookup.h
index 8256d739a8f47a0ca1aa93c25f95f39010d200b8..615528baa92b3186b7bc30ff229470f0041b9529 100644 (file)
--- a/src/lib-dns/dns-lookup.h
+++ b/src/lib-dns/dns-lookup.h
@@ -11,6 +11,9 @@ struct dns_lookup_settings {
        /* the idle_timeout_msecs works only with the dns_client_* API.
           0 = disconnect immediately */
        unsigned int idle_timeout_msecs;
+       /* Non-zero enables caching for the client, is not supported with
+          dns_lookup() or dns_lookup_ptr(). Note that DNS TTL is ignored. */
+       unsigned int cache_ttl_secs;
 
        /* ioloop to run the lookup on (defaults to current_ioloop) */
        struct ioloop *ioloop;

diff --git a/src/lib-dns/test-dns-lookup.c b/src/lib-dns/test-dns-lookup.c
index 9db7fdb931e649f61753e06f92529f7fed7dc63a..a2a982419dd2a665ad530b4184dba8abb8cc8233 100644 (file)
--- a/src/lib-dns/test-dns-lookup.c
+++ b/src/lib-dns/test-dns-lookup.c
@@ -252,12 +252,88 @@ static void test_dns_lookup_abort(void)
        test_end();
 }
 
+static void test_dns_lookup_cached(void)
+{
+       struct test_expect_result ctx;
+       struct dns_lookup *lookup;
+       struct timeout *to;
+
+       test_begin("dns lookup (cached)");
+       create_dns_server(&test_server);
+       const struct dns_lookup_settings set = {
+               .dns_client_socket_path = TEST_SOCKET_NAME,
+               .ioloop = test_server.loop,
+               .timeout_msecs = 1000,
+               .cache_ttl_secs = 4,
+       };
+
+
+       struct dns_client *client = dns_client_init(&set);
+
+       /* lookup localhost */
+       ctx.result = "127.0.0.1\t::1";
+       ctx.ret = 0;
+
+       /* should cause only one lookup */
+       test_assert(dns_client_lookup(client, "localhost", test_callback_ips,
+                                     &ctx, &lookup) == 0);
+       io_loop_run(current_ioloop);
+       test_assert(dns_client_lookup(client, "localhost", test_callback_ips,
+                                     &ctx, &lookup) == 0);
+       io_loop_run(current_ioloop);
+       test_assert_cmp(test_server.lookup_counter, ==, 1);
+
+       to = timeout_add(3*1000, io_loop_stop, test_server.loop);
+       io_loop_run(current_ioloop);
+       timeout_remove(&to);
+
+       /* entry should get refreshed */
+       test_assert(dns_client_lookup(client, "localhost", test_callback_ips,
+                                     &ctx, &lookup) == 0);
+       io_loop_run(current_ioloop);
+       while (dns_client_has_pending_queries(client)) {
+               io_loop_handler_run(current_ioloop);
+               io_loop_set_running(current_ioloop);
+       }
+       test_assert_cmp(test_server.lookup_counter, ==, 2);
+
+       /* should get looked up again */
+       to = timeout_add(5*1000, io_loop_stop, test_server.loop);
+       io_loop_run(current_ioloop);
+       timeout_remove(&to);
+
+       test_assert(dns_client_lookup(client, "localhost", test_callback_ips,
+                                     &ctx, &lookup) == 0);
+       io_loop_run(current_ioloop);
+
+       test_assert_cmp(test_server.lookup_counter, ==, 3);
+
+       /* Ensure failures do not get cached */
+       ctx.result = NULL;
+       ctx.ret = -1;
+       test_assert(dns_client_lookup(client, "failhost", test_callback_ips,
+                                     &ctx, &lookup) == 0);
+       io_loop_run(current_ioloop);
+       test_assert_cmp(test_server.lookup_counter, ==, 4);
+
+       test_assert(dns_client_lookup(client, "failhost", test_callback_ips,
+                                     &ctx, &lookup) == 0);
+       io_loop_run(current_ioloop);
+       test_assert_cmp(test_server.lookup_counter, ==, 5);
+
+       dns_client_deinit(&client);
+       destroy_dns_server(&test_server);
+
+       test_end();
+}
+
 int main(void)
 {
        static void (*const test_functions[])(void) = {
                test_dns_lookup,
                test_dns_lookup_timeout,
                test_dns_lookup_abort,
+               test_dns_lookup_cached,
                NULL
        };