From: Marek Vavruša <marek.vavrusa@nic.cz>
Date: Sun, 5 Jul 2015 20:14:10 +0000 (+0200)
Subject: modules/prefetch: basic prefetching of frequently used records
X-Git-Tag: v1.0.0-beta1~88^2~3
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cc05831d245e7976afe59e2fb2418a22d943a31d;p=thirdparty%2Fknot-resolver.git

modules/prefetch: basic prefetching of frequently used records
---

diff --git a/daemon/worker.c b/daemon/worker.c
index 728616cf5..63f4dad13 100644
--- a/daemon/worker.c
+++ b/daemon/worker.c
@@ -266,7 +266,6 @@ static int qr_task_step(struct qr_task *task, knot_pkt_t *packet)
 		}
 		connect->data = task;
 	} else {
-		printf("sending: %s %u\n", knot_dname_to_str_alloc(knot_pkt_qname(next_query)), knot_pkt_qtype(next_query));
 		if (qr_task_send(task, task->next_handle, addr, next_query) != 0) {
 			return qr_task_step(task, NULL);
 		}
diff --git a/doc/modules.rst b/doc/modules.rst
index 2a1e4cfd8..5f6f2bb76 100644
--- a/doc/modules.rst
+++ b/doc/modules.rst
@@ -12,6 +12,7 @@ Implemented modules
 .. include:: ../modules/hints/README.rst
 .. include:: ../modules/block/README.rst
 .. include:: ../modules/stats/README.rst
+.. include:: ../modules/prefetch/README.rst
 .. include:: ../modules/cachectl/README.rst
 .. include:: ../modules/graphite/README.rst
 .. include:: ../modules/ketcd/README.rst
diff --git a/modules/prefetch/README.rst b/modules/prefetch/README.rst
index fefb476d9..8d5619028 100644
--- a/modules/prefetch/README.rst
+++ b/modules/prefetch/README.rst
@@ -1,4 +1,9 @@
 .. _mod-prefetch:
 
-Prefetching
------------
+Prefetching records
+-------------------
+
+The module tracks expiring records (having less than 5% of original TTL) and batches them for prefetch.
+This improves latency for frequently used records, as they are fetched in advance.
+
+.. todo:: Learn usage patterns from browser history, track usage pattern over time.
\ No newline at end of file
diff --git a/modules/prefetch/prefetch.lua b/modules/prefetch/prefetch.lua
index 4035cad4a..8246ce7a8 100644
--- a/modules/prefetch/prefetch.lua
+++ b/modules/prefetch/prefetch.lua
@@ -1,41 +1,53 @@
+-- Batch soon-expiring records in a queue and fetch them periodically.
+-- This helps to reduce a latency for records that are often accessed.
+-- @module prefetch
+-- @field queue table of scheduled records
+-- @field queue_max maximum length of the queue
+-- @field queue_len current length of the queue
+-- @field window length of the coalescing window
 local prefetch = {
 	queue = {},
-	frequency = 2
-}
-
--- @function Block layer implementation
-prefetch.layer = {
-	produce = function(state, req, pkt)
+	queue_max = 100,
+	queue_len = 0,
+	window = 60,
+	layer = {
 		-- Schedule cached entries that are expiring soon
-		local qry = kres.query_current(req)
-		if not kres.query_has_flag(qry, kres.query.CACHED) then
-			return state
-		end
-		local rr = pkt:get(kres.ANSWER, 0)
-		if rr and rr.ttl > 0 and rr.ttl < prefetch.frequency then
-			local key = rr.owner..rr.type
+		finish = function(state, req, answer)
+			local qry = kres.query_resolved(req)
+			if not kres.query.has_flag(qry, kres.query.EXPIRING) then
+				return state
+			end
+			-- Refresh entries that probably expire in this time window
+			local qlen = prefetch.queue_len
+			if qlen > prefetch.queue_max then
+				return state
+			end
+			-- Key: {qtype [1], qname [1-255]}
+			local key = string.char(answer:qtype())..answer:qname()
 			local val = prefetch.queue[key]
 			if not val then
 				prefetch.queue[key] = 1
+				prefetch.queue_len = qlen + 1
 			else
 				prefetch.queue[key] = val + 1
 			end
+			return state
 		end
-		return state
-	end
+	}
 }
 
+-- Resolve queued records and flush the queue
 function prefetch.batch(module)
 	for key, val in pairs(prefetch.queue) do
-		print('prefetching',key,val)
+		worker.resolve(string.sub(key, 2), string.byte(key))
+		prefetch.queue[key] = nil
 	end
-	prefetch.queue = {}
-	-- @TODO: next batch interval
-	event.after(prefetch.frequency * sec, prefetch.batch)
+	prefetch.queue_len = 0
+	return 0
 end
 
 function prefetch.init(module)
-	event.after(prefetch.frequency * sec, prefetch.batch)
+	event.recurrent(prefetch.window * sec, prefetch.batch)
 end
 
 function prefetch.deinit(module)