From: Wouter Wijngaards <wouter@nlnetlabs.nl>
Date: Wed, 26 Aug 2009 13:23:49 +0000 (+0000)
Subject: autotrust probing and testbound with fake timer support.
X-Git-Tag: release-1.4.0rc1~116
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=902323da2f947c914337894dbb761509d1254725;p=thirdparty%2Funbound.git

autotrust probing and testbound with fake timer support.


git-svn-id: file:///svn/unbound/trunk@1787 be551aaa-1e26-0410-a405-d3ace91eadb9
---

diff --git a/daemon/worker.c b/daemon/worker.c
index a65867e99..63436244d 100644
--- a/daemon/worker.c
+++ b/daemon/worker.c
@@ -66,6 +66,7 @@
 #include "util/fptr_wlist.h"
 #include "util/tube.h"
 #include "iterator/iter_fwd.h"
+#include "validator/autotrust.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #  include <sys/types.h>
@@ -940,6 +941,15 @@ void worker_stat_timer_cb(void* arg)
 	worker_restart_timer(worker);
 }
 
+void worker_probe_timer_cb(void* arg)
+{
+	struct worker* worker = (struct worker*)arg;
+	struct timeval tv;
+	tv.tv_sec = autr_probe_timer(&worker->env);
+	tv.tv_usec = 0;
+	comm_timer_set(worker->probe_timer, &tv);
+}
+
 struct worker* 
 worker_create(struct daemon* daemon, int id, int* ports, int n)
 {
@@ -1054,6 +1064,23 @@ worker_init(struct worker* worker, struct config_file *cfg,
 	if(!worker->stat_timer) {
 		log_err("could not create statistics timer");
 	}
+	/* one probe timer per process -- if we have 5011 anchors */
+	if(autr_get_num_anchors(worker->daemon->env->anchors) > 0
+#ifndef THREADS_DISABLED
+		&& worker->thread_num == 0
+#endif
+		) {
+		struct timeval tv;
+		tv.tv_sec = 0;
+		tv.tv_usec = 0;
+		worker->probe_timer = comm_timer_create(worker->base,
+			worker_probe_timer_cb, worker);
+		if(!worker->probe_timer) {
+			log_err("could not create 5011-probe timer");
+		}
+		/* let timer fire, then it can reset itself */
+		comm_timer_set(worker->probe_timer, &tv);
+	}
 
 	/* we use the msg_buffer_size as a good estimate for what the 
 	 * user wants for memory usage sizes */
@@ -1130,6 +1157,7 @@ worker_delete(struct worker* worker)
 	comm_signal_delete(worker->comsig);
 	tube_delete(worker->cmd);
 	comm_timer_delete(worker->stat_timer);
+	comm_timer_delete(worker->probe_timer);
 	free(worker->ports);
 	if(worker->thread_num == 0) {
 		log_set_time(NULL);
diff --git a/daemon/worker.h b/daemon/worker.h
index 874a64ddb..993fa5e2b 100644
--- a/daemon/worker.h
+++ b/daemon/worker.h
@@ -101,6 +101,8 @@ struct worker {
 	struct comm_point* cmd_com;
 	/** timer for statistics */
 	struct comm_timer* stat_timer;
+	/** timer for autotrust probes */
+	struct comm_timer* probe_timer;
 
 	/** number of requests that can be handled by this worker */
 	size_t request_size;
@@ -236,4 +238,7 @@ void worker_stats_clear(struct worker* worker);
 /** statistics timer callback handler */
 void worker_stat_timer_cb(void* arg);
 
+/** probe timer callback handler */
+void worker_probe_timer_cb(void* arg);
+
 #endif /* DAEMON_WORKER_H */
diff --git a/doc/Changelog b/doc/Changelog
index b03b5d12d..f34982664 100644
--- a/doc/Changelog
+++ b/doc/Changelog
@@ -1,3 +1,6 @@
+26 August 2009: Wouter
+	- autotrust probing.
+
 25 August 2009: Wouter
 	- fixup memleak in trust anchor unsupported algorithm check.
 	- iana portlist updated.
diff --git a/libunbound/libworker.c b/libunbound/libworker.c
index 9f42df2c9..44d21419d 100644
--- a/libunbound/libworker.c
+++ b/libunbound/libworker.c
@@ -833,6 +833,11 @@ void worker_stat_timer_cb(void* ATTR_UNUSED(arg))
 	log_assert(0);
 }
 
+void worker_probe_timer_cb(void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
 int order_lock_cmp(const void* ATTR_UNUSED(e1), const void* ATTR_UNUSED(e2))
 {
 	log_assert(0);
diff --git a/smallapp/worker_cb.c b/smallapp/worker_cb.c
index f756f9cd4..8e7dc5b57 100644
--- a/smallapp/worker_cb.c
+++ b/smallapp/worker_cb.c
@@ -204,6 +204,11 @@ void worker_stat_timer_cb(void* ATTR_UNUSED(arg))
 	log_assert(0);
 }
 
+void worker_probe_timer_cb(void* ATTR_UNUSED(arg))
+{
+	log_assert(0);
+}
+
 /** keep track of lock id in lock-verify application */
 struct order_id {
         /** the thread id that created it */
diff --git a/testcode/fake_event.c b/testcode/fake_event.c
index f465efe3c..c3407bfde 100644
--- a/testcode/fake_event.c
+++ b/testcode/fake_event.c
@@ -57,6 +57,7 @@
 #include "testcode/replay.h"
 #include "testcode/ldns-testpkts.h"
 #include "util/log.h"
+#include "util/fptr_wlist.h"
 #include <signal.h>
 struct worker;
 
@@ -77,6 +78,22 @@ timeval_add(struct timeval* d, const struct timeval* add)
 #endif
 }
 
+/** compare of time values */
+static int
+timeval_smaller(const struct timeval* x, const struct timeval* y)
+{
+#ifndef S_SPLINT_S
+	if(x->tv_sec < y->tv_sec)
+		return 1;
+	else if(x->tv_sec == y->tv_sec) {
+		if(x->tv_usec <= y->tv_usec)
+			return 1;
+		else	return 0;
+	}
+	else	return 0;
+#endif
+}
+
 void 
 fake_temp_file(const char* adj, const char* id, char* buf, size_t len)
 {
@@ -438,10 +455,29 @@ fake_pending_callback(struct replay_runtime* runtime,
 	ldns_buffer_free(c.buffer);
 }
 
+/** get oldest enabled fake timer */
+static struct fake_timer*
+get_oldest_timer(struct replay_runtime* runtime)
+{
+	struct fake_timer* p, *res = NULL;
+	for(p=runtime->timer_list; p; p=p->next) {
+		if(!p->enabled)
+			continue;
+		if(timeval_smaller(&p->tv, &runtime->now_tv)) {
+			if(!res)
+				res = p;
+			else if(timeval_smaller(&p->tv, &res->tv))
+				res = p;
+		}
+	}
+	return res;
+}
+
 /** pass time */
 static void
 time_passes(struct replay_runtime* runtime, struct replay_moment* mom)
 {
+	struct fake_timer *t;
 	timeval_add(&runtime->now_tv, &mom->elapse);
 	runtime->now_secs = (uint32_t)runtime->now_tv.tv_sec;
 #ifndef S_SPLINT_S
@@ -449,6 +485,13 @@ time_passes(struct replay_runtime* runtime, struct replay_moment* mom)
 		(int)mom->elapse.tv_sec, (int)mom->elapse.tv_usec,
 		(int)runtime->now_tv.tv_sec, (int)runtime->now_tv.tv_usec);
 #endif
+	/* see if any timers have fired; and run them */
+	while( (t=get_oldest_timer(runtime)) ) {
+		t->enabled = 0;
+		log_info("fake_timer callback");
+		fptr_ok(fptr_whitelist_comm_timer(t->cb));
+		(*t->cb)(t->cb_arg);
+	}
 }
 
 /** check autotrust file contents */
@@ -674,6 +717,7 @@ comm_base_delete(struct comm_base* b)
 	struct replay_runtime* runtime = (struct replay_runtime*)b;
 	struct fake_pending* p, *np;
 	struct replay_answer* a, *na;
+	struct fake_timer* t, *nt;
 	if(!runtime)
 		return;
 	runtime->scenario= NULL;
@@ -689,6 +733,12 @@ comm_base_delete(struct comm_base* b)
 		delete_replay_answer(a);
 		a = na;
 	}
+	t = runtime->timer_list;
+	while(t) {
+		nt = t->next;
+		free(t);
+		t = nt;
+	}
 	free(runtime);
 }
 
@@ -1201,25 +1251,57 @@ int serviced_cmp(const void* ATTR_UNUSED(a), const void* ATTR_UNUSED(b))
 	return 0;
 }
 
-/* no statistics timers in testbound */
-struct comm_timer* comm_timer_create(struct comm_base* ATTR_UNUSED(base), 
-	void (*cb)(void*), void* ATTR_UNUSED(cb_arg))
+/* timers in testbound for autotrust. statistics tested in tpkg. */
+struct comm_timer* comm_timer_create(struct comm_base* base, 
+	void (*cb)(void*), void* cb_arg)
 {
-	(void)cb;
-	return malloc(1);
+	struct replay_runtime* runtime = (struct replay_runtime*)base;
+	struct fake_timer* t = (struct fake_timer*)calloc(1, sizeof(*t));
+	t->cb = cb;
+	t->cb_arg = cb_arg;
+	fptr_ok(fptr_whitelist_comm_timer(t->cb)); /* check in advance */
+	t->runtime = runtime;
+	t->next = runtime->timer_list;
+	runtime->timer_list = t;
+	return (struct comm_timer*)t;
 }
 
-void comm_timer_disable(struct comm_timer* ATTR_UNUSED(timer))
+void comm_timer_disable(struct comm_timer* timer)
 {
+	struct fake_timer* t = (struct fake_timer*)timer;
+	log_info("fake timer disabled");
+	t->enabled = 0;
 }
 
-void comm_timer_set(struct comm_timer* ATTR_UNUSED(timer), 
-	struct timeval* ATTR_UNUSED(tv))
+void comm_timer_set(struct comm_timer* timer, struct timeval* tv)
 {
+	struct fake_timer* t = (struct fake_timer*)timer;
+	t->enabled = 1;
+	t->tv = *tv;
+	log_info("fake timer set %d.%6.6d", 
+		(int)t->tv.tv_sec, (int)t->tv.tv_usec);
+	timeval_add(&t->tv, &t->runtime->now_tv);
 }
 
 void comm_timer_delete(struct comm_timer* timer)
 {
+	struct fake_timer* t = (struct fake_timer*)timer;
+	struct fake_timer** pp, *p;
+	if(!t) return;
+
+	/* remove from linked list */
+	pp = &t->runtime->timer_list;
+	p = t->runtime->timer_list;
+	while(p) {
+		if(p == t) {
+			/* snip from list */
+			*pp = p->next;
+			break;
+		}
+		pp = &p->next;
+		p = p->next;
+	}
+
 	free(timer);
 }
 
diff --git a/testcode/replay.h b/testcode/replay.h
index 09e42b0cc..1428428d3 100644
--- a/testcode/replay.h
+++ b/testcode/replay.h
@@ -112,6 +112,7 @@ struct replay_answer;
 struct replay_moment;
 struct replay_range;
 struct fake_pending;
+struct fake_timer;
 
 /**
  * A replay scenario.
@@ -244,6 +245,9 @@ struct replay_runtime {
 	/** last element in answer list. */
 	struct replay_answer* answer_last;
 
+	/** list of fake timer callbacks that are pending */
+	struct fake_timer* timer_list;
+
 	/** callback to call for incoming queries */
 	comm_point_callback_t* callback_query;
 	/** user argument for incoming query callback */
@@ -306,6 +310,24 @@ struct replay_answer {
 	ldns_pkt* pkt;
 };
 
+/**
+ * Timers with callbacks, fake replay version.
+ */
+struct fake_timer {
+	/** next in list */
+	struct fake_timer* next;
+	/** the runtime structure this is part of */
+	struct replay_runtime* runtime;
+	/** the callback to call */
+	void (*cb)(void*);
+	/** the callback user argument */
+	void* cb_arg;
+	/** if timer is enabled */
+	int enabled;
+	/** when the timer expires */
+	struct timeval tv;
+};
+
 /**
  * Read a replay scenario from the file.
  * @param in: file to read from.
diff --git a/testdata/autotrust_init.rpl b/testdata/autotrust_init.rpl
index 53de376e7..3663a9b65 100644
--- a/testdata/autotrust_init.rpl
+++ b/testdata/autotrust_init.rpl
@@ -99,6 +99,21 @@ RANGE_END
 ; set date/time to Aug 24 09:46:40  (2009).
 STEP 5 TIME_PASSES ELAPSE 1251100000
 
+; the auto probing should have been done now.
+STEP 7 CHECK_AUTOTRUST example.com
+FILE_BEGIN
+; autotrust trust anchor file
+;;id: example.com. 1
+;;last_queried: 1251100000 ;;Mon Aug 24 09:46:40 2009
+;;last_success: 1251100000 ;;Mon Aug 24 09:46:40 2009
+;;next_probe_time: 1251105400 ;;Mon Aug 24 11:16:40 2009
+;;query_failed: 0
+;;query_interval: 5400
+;;retry_time: 3600
+example.com.	10800	IN	DNSKEY	257 3 5 AwEAAc3Z5DQDJpH4oPdNtC4BUQHk50XMD+dHr4r8psHmivIa83hxR5CRgCtd9sENCW9Ae8OIO19xw9t/RPaEAqQa+OE= ;{id = 55582 (ksk), size = 512b} ;;state=2 [  VALID  ] ;;count=0 ;;lastchange=1251100000 ;;Mon Aug 24 09:46:40 2009
+FILE_END
+
+
 STEP 10 QUERY
 ENTRY_BEGIN
 REPLY RD DO
@@ -131,7 +146,7 @@ FILE_BEGIN
 ;;id: example.com. 1
 ;;last_queried: 1251100000 ;;Mon Aug 24 09:46:40 2009
 ;;last_success: 1251100000 ;;Mon Aug 24 09:46:40 2009
-;;next_probe_time: 0 ;;Thu Jan  1 01:00:00 1970
+;;next_probe_time: 1251105400 ;;Mon Aug 24 11:16:40 2009
 ;;query_failed: 0
 ;;query_interval: 5400
 ;;retry_time: 3600
diff --git a/util/fptr_wlist.c b/util/fptr_wlist.c
index 71e723e7e..5b395a671 100644
--- a/util/fptr_wlist.c
+++ b/util/fptr_wlist.c
@@ -104,6 +104,7 @@ fptr_whitelist_comm_timer(void (*fptr)(void*))
 	if(fptr == &pending_udp_timer_cb) return 1;
 	else if(fptr == &outnet_tcptimer) return 1;
 	else if(fptr == &worker_stat_timer_cb) return 1;
+	else if(fptr == &worker_probe_timer_cb) return 1;
 #ifdef UB_ON_WINDOWS
 	else if(fptr == &wsvc_cron_cb) return 1;
 #endif
@@ -383,5 +384,6 @@ int fptr_whitelist_mesh_cb(mesh_cb_func_t fptr)
 {
 	if(fptr == &libworker_fg_done_cb) return 1;
 	else if(fptr == &libworker_bg_done_cb) return 1;
+	else if(fptr == &probe_answer_cb) return 1;
 	return 0;
 }
diff --git a/validator/autotrust.c b/validator/autotrust.c
index 61cea54c4..4086fb24e 100644
--- a/validator/autotrust.c
+++ b/validator/autotrust.c
@@ -51,6 +51,9 @@
 #include "util/net_help.h"
 #include "util/config_file.h"
 #include "util/regional.h"
+#include "util/random.h"
+#include "util/data/msgparse.h"
+#include "services/mesh.h"
 
 /** number of times a key must be seen before it can become valid */
 #define MIN_PENDINGCOUNT 2
@@ -61,7 +64,7 @@ struct autr_global_data* autr_global_create(void)
 	global = (struct autr_global_data*)malloc(sizeof(*global));
 	if(!global) 
 		return NULL;
-	rbtree_init(&global->probetree, &probetree_cmp);
+	rbtree_init(&global->probe, &probetree_cmp);
 	return global;
 }
 
@@ -78,11 +81,24 @@ int probetree_cmp(const void* x, const void* y)
 {
 	struct trust_anchor* a = (struct trust_anchor*)x;
 	struct trust_anchor* b = (struct trust_anchor*)y;
+	log_assert(a->autr && b->autr);
 	if(a->autr->next_probe_time < b->autr->next_probe_time)
 		return -1;
 	if(a->autr->next_probe_time > b->autr->next_probe_time)
 		return 1;
-	return 0;
+	/* time is equal, sort on trust point identity */
+	return anchor_cmp(x, y);
+}
+
+size_t 
+autr_get_num_anchors(struct val_anchors* anchors)
+{
+	size_t res = 0;
+	lock_basic_lock(&anchors->lock);
+	if(anchors->autr)
+		res = anchors->autr->probe.count;
+	lock_basic_unlock(&anchors->lock);
+	return res;
 }
 
 /** Position in string */
@@ -328,6 +344,15 @@ autr_tp_create(struct val_anchors* anchors, ldns_rdf* own, uint16_t dc)
 		free(tp);
 		return NULL;
 	}
+	if(!rbtree_insert(&anchors->autr->probe, &tp->autr->pnode)) {
+		(void)rbtree_delete(anchors->tree, tp);
+		lock_basic_unlock(&anchors->lock);
+		log_err("trust anchor in probetree twice");
+		free(tp->name);
+		free(tp->autr);
+		free(tp);
+		return NULL;
+	}
 	lock_basic_unlock(&anchors->lock);
 	lock_basic_init(&tp->lock);
 	lock_protect(&tp->lock, tp, sizeof(*tp));
@@ -458,7 +483,6 @@ load_trustanchor(struct val_anchors* anchors, char* str, const char* fname)
 		return NULL;
 	}
 	if(!tp->autr->file) {
-		/* TODO insert tp into probe tree */
 		tp->autr->file = strdup(fname);
 		if(!tp->autr->file) {
 			lock_basic_unlock(&tp->lock);
@@ -630,10 +654,13 @@ parse_var_line(char* line, struct val_anchors* anchors,
 		lock_basic_unlock(&tp->lock);
 	} else if(strncmp(line, ";;next_probe_time: ", 19) == 0) {
 		if(!tp) return -1;
+		lock_basic_lock(&anchors->lock);
 		lock_basic_lock(&tp->lock);
+		(void)rbtree_delete(&anchors->autr->probe, tp);
 		tp->autr->next_probe_time = (time_t)parse_int(line+19, &r);
-		/* TODO manage probetree */
+		(void)rbtree_insert(&anchors->autr->probe, &tp->autr->pnode);
 		lock_basic_unlock(&tp->lock);
+		lock_basic_unlock(&anchors->lock);
 	} else if(strncmp(line, ";;query_failed: ", 16) == 0) {
 		if(!tp) return -1;
 		lock_basic_lock(&tp->lock);
@@ -1415,14 +1442,66 @@ autr_cleanup_keys(struct trust_anchor* tp)
 	}
 }
 
+/** calculate next probe time */
+static time_t
+calc_next_probe(struct module_env* env, uint32_t wait)
+{
+	/* TODO (how test?) make it random, 90-100% */
+	if(wait < 3600)
+		wait = 3600;
+	return (time_t)(*env->now + wait);
+}
+
+/** set next probe for trust anchor */
+static int
+set_next_probe(struct module_env* env, struct trust_anchor* tp,
+	struct ub_packed_rrset_key* dnskey_rrset)
+{
+	struct trust_anchor key, *tp2;
+	/* use memory allocated in rrset for temporary name storage */
+	key.node.key = &key;
+	key.name = dnskey_rrset->rk.dname;
+	key.namelen = dnskey_rrset->rk.dname_len;
+	key.namelabs = dname_count_labels(key.name);
+	key.dclass = tp->dclass;
+	lock_basic_unlock(&tp->lock);
+
+	/* fetch tp again and lock anchors */
+	lock_basic_lock(&env->anchors->lock);
+	tp2 = (struct trust_anchor*)rbtree_search(env->anchors->tree, &key);
+	if(!tp2) {
+		verbose(VERB_ALGO, "trustpoint was deleted in set_next_probe");
+		lock_basic_unlock(&env->anchors->lock);
+		return 0;
+	}
+	log_assert(tp == tp2);
+	lock_basic_lock(&tp->lock);
+
+	/* schedule */
+	(void)rbtree_delete(&env->anchors->autr->probe, tp);
+	tp->autr->next_probe_time = calc_next_probe(env, 
+		tp->autr->query_interval);
+	(void)rbtree_insert(&env->anchors->autr->probe, &tp->autr->pnode);
+
+	lock_basic_unlock(&env->anchors->lock);
+	verbose(VERB_ALGO, "next probe set in %d seconds", 
+		(int)tp->autr->next_probe_time - (int)*env->now);
+	return 1;
+}
+
 /** Delete trust point that was revoked */
 static void
 autr_tp_remove(struct module_env* env, struct trust_anchor* tp)
 {
 	struct trust_anchor key;
+	struct autr_point_data pd;
 	/* save name */
 	memset(&key, 0, sizeof(key));
+	memset(&pd, 0, sizeof(pd));
+	key.autr = &pd;
 	key.node.key = &key;
+	pd.pnode.key = &key;
+	pd.next_probe_time = tp->autr->next_probe_time;
 	key.name = regional_alloc_init(env->scratch, tp->name, tp->namelen);
 	if(!key.name) {
 		log_err("out of scratch memory in trust point delete");
@@ -1439,6 +1518,7 @@ autr_tp_remove(struct module_env* env, struct trust_anchor* tp)
 	/* take from tree. It could be deleted by someone else. */
 	lock_basic_lock(&env->anchors->lock);
 	(void)rbtree_delete(env->anchors->tree, &key);
+	(void)rbtree_delete(&env->anchors->autr->probe, &key);
 	anchors_init_parents_locked(env->anchors);
 	lock_basic_unlock(&env->anchors->lock);
 	/* delete */
@@ -1505,8 +1585,10 @@ int autr_process_prime(struct module_env* env, struct val_env* ve,
 	}
 
 	if(changed) {
-		verbose(VERB_ALGO, "autotrust: point changed, write to disk");
 		autr_cleanup_keys(tp);
+		if(!set_next_probe(env, tp, dnskey_rrset))
+			return 0; /* trust point does not exist */
+		verbose(VERB_ALGO, "autotrust: point changed, write to disk");
 		autr_write_file(env, tp);
 		if(!autr_assemble(tp)) {
 			log_err("malloc failure assembling autotrust keys");
@@ -1552,7 +1634,7 @@ autr_debug_print_tp(struct trust_anchor* tp)
 	log_info("trust point %s : %d", buf, (int)tp->dclass);
 	log_info("assembled %d DS and %d DNSKEYs", 
 		(int)tp->numDS, (int)tp->numDNSKEY);
-	if(1) { /* DEBUG */
+	if(0) {
 		ldns_buffer* buf = ldns_buffer_new(70000);
 		ldns_rr_list* list;
 		if(tp->ds_rrset) {
@@ -1601,3 +1683,94 @@ autr_debug_print(struct val_anchors* anchors)
 	}
 	lock_basic_unlock(&anchors->lock);
 }
+
+void probe_answer_cb(void* arg, int rcode, ldns_buffer* buf, 
+	enum sec_status sec)
+{
+	struct module_env* env = (struct module_env*)arg;
+	verbose(VERB_ALGO, "autotrust probe answer cb");
+}
+
+/** probe a trust anchor DNSKEY and unlocks tp */
+static void
+probe_anchor(struct module_env* env, struct trust_anchor* tp)
+{
+	struct query_info qinfo;
+	uint16_t qflags = BIT_RD;
+	struct edns_data edns;
+	ldns_buffer* buf = env->scratch_buffer;
+	qinfo.qname = regional_alloc_init(env->scratch, tp->name, tp->namelen);
+	if(!qinfo.qname) {
+		log_err("out of memory making 5011 probe");
+		return;
+	}
+	qinfo.qname_len = tp->namelen;
+	qinfo.qtype = LDNS_RR_TYPE_DNSKEY;
+	qinfo.qclass = tp->dclass;
+	log_query_info(VERB_ALGO, "autotrust probe", &qinfo);
+	verbose(VERB_ALGO, "retry probe set in %d seconds", 
+		(int)tp->autr->next_probe_time - (int)*env->now);
+	edns.edns_present = 1;
+	edns.ext_rcode = 0;
+	edns.edns_version = 0;
+	edns.bits = EDNS_DO;
+	if(ldns_buffer_capacity(buf) < 65535)
+		edns.udp_size = (uint16_t)ldns_buffer_capacity(buf);
+	else	edns.udp_size = 65535;
+
+	/* can't hold the lock while mesh_run is processing */
+	lock_basic_unlock(&tp->lock);
+	if(!mesh_new_callback(env->mesh, &qinfo, qflags, &edns, buf, 0, 
+		&probe_answer_cb, env)) {
+		log_err("out of memory making 5011 probe");
+	}
+}
+
+/** fetch first to-probe trust-anchor and lock it and set retrytime */
+static struct trust_anchor*
+todo_probe(struct module_env* env, uint32_t* next)
+{
+	struct trust_anchor* tp;
+	rbnode_t* el;
+	/* get first one */
+	lock_basic_lock(&env->anchors->lock);
+	if( !(el=rbtree_first(&env->anchors->autr->probe)) ) {
+		/* in case of revoked anchors */
+		lock_basic_unlock(&env->anchors->lock);
+		return NULL;
+	}
+	tp = (struct trust_anchor*)el->key;
+	lock_basic_lock(&tp->lock);
+
+	/* is it eligible? */
+	if((uint32_t)tp->autr->next_probe_time > *env->now) {
+		/* no more to probe */
+		*next = (uint32_t)tp->autr->next_probe_time - *env->now;
+		lock_basic_unlock(&tp->lock);
+		lock_basic_unlock(&env->anchors->lock);
+		return NULL;
+	}
+
+	/* reset its next probe time */
+	(void)rbtree_delete(&env->anchors->autr->probe, tp);
+	tp->autr->next_probe_time = calc_next_probe(env, tp->autr->retry_time);
+	(void)rbtree_insert(&env->anchors->autr->probe, &tp->autr->pnode);
+	lock_basic_unlock(&env->anchors->lock);
+
+	return tp;
+}
+
+uint32_t 
+autr_probe_timer(struct module_env* env)
+{
+	struct trust_anchor* tp;
+	uint32_t next_probe = 3600;
+	verbose(VERB_ALGO, "autotrust probe timer callback");
+	/* while there are still anchors to probe */
+	while( (tp = todo_probe(env, &next_probe)) ) {
+		/* make a probe for this anchor */
+		probe_anchor(env, tp);
+	}
+	verbose(VERB_ALGO, "autotrust probe timer callback done");
+	return next_probe;
+}
diff --git a/validator/autotrust.h b/validator/autotrust.h
index ff336ce63..211c58c6c 100644
--- a/validator/autotrust.h
+++ b/validator/autotrust.h
@@ -42,6 +42,7 @@
 #ifndef VALIDATOR_AUTOTRUST_H
 #define VALIDATOR_AUTOTRUST_H
 #include "util/rbtree.h"
+#include "util/data/packed_rrset.h"
 struct val_anchors;
 struct trust_anchor;
 struct ub_packed_rrset_key;
@@ -113,8 +114,9 @@ struct autr_point_data {
  * Autotrust global metadata.
  */
 struct autr_global_data {
-	/** rbtree of autotrust anchors sorted by next probe time */
-	rbtree_t probetree;
+	/** rbtree of autotrust anchors sorted by next probe time.
+	 * When time is equal, sorted by anchor class, name. */
+	rbtree_t probe;
 };
 
 /**
@@ -129,6 +131,19 @@ struct autr_global_data* autr_global_create(void);
  */
 void autr_global_delete(struct autr_global_data* global);
 
+/**
+ * See if autotrust anchors are configured and how many.
+ * @param anchors: the trust anchors structure.
+ */
+size_t autr_get_num_anchors(struct val_anchors* anchors);
+
+/**
+ * Process probe timer.  Add new probes if needed.
+ * @param env: module environment with time, with anchors and with the mesh.
+ * @return time of next probe (in seconds from now).
+ */
+uint32_t autr_probe_timer(struct module_env* env);
+
 /** probe tree compare function */
 int probetree_cmp(const void* x, const void* y);
 
@@ -173,4 +188,8 @@ int autr_process_prime(struct module_env* env, struct val_env* ve,
  */
 void autr_debug_print(struct val_anchors* anchors);
 
+/** callback for query answer to 5011 probe */
+void probe_answer_cb(void* arg, int rcode, ldns_buffer* buf, 
+	enum sec_status sec);
+
 #endif /* VALIDATOR_AUTOTRUST_H */
diff --git a/validator/val_anchor.c b/validator/val_anchor.c
index 3ce86ffd2..cfc2c0dbb 100644
--- a/validator/val_anchor.c
+++ b/validator/val_anchor.c
@@ -91,6 +91,7 @@ anchors_create()
 	}
 	lock_basic_init(&a->lock);
 	lock_protect(&a->lock, a, sizeof(*a));
+	lock_protect(&a->lock, a->autr, sizeof(*a->autr));
 	return a;
 }
 
@@ -111,6 +112,7 @@ anchors_delete(struct val_anchors* anchors)
 {
 	if(!anchors)
 		return;
+	lock_unprotect(&anchors->lock, anchors->autr);
 	lock_unprotect(&anchors->lock, anchors);
 	lock_basic_destroy(&anchors->lock);
 	traverse_postorder(anchors->tree, anchors_delfunc, NULL);
@@ -1089,7 +1091,7 @@ anchors_apply_cfg(struct val_anchors* anchors, struct config_file* cfg)
 	anchors_assemble_rrsets(anchors);
 	init_parents(anchors);
 	ldns_buffer_free(parsebuf);
-	/*autr_debug_print(anchors); */ /* DEBUG */
+	if(verbosity >= VERB_ALGO) autr_debug_print(anchors);
 	return 1;
 }
 
diff --git a/validator/val_anchor.h b/validator/val_anchor.h
index bc39efb07..f6cab1116 100644
--- a/validator/val_anchor.h
+++ b/validator/val_anchor.h
@@ -56,7 +56,10 @@ struct autr_global_data;
 struct val_anchors {
 	/** lock on trees */
 	lock_basic_t lock;
-	/** region where trust anchors are allocated */
+	/** 
+	 * region where trust anchors are allocated.
+	 * Autotrust anchors are malloced so they can be updated. 
+	 */
 	struct regional* region;
 	/**
 	 * Anchors are store in this tree. Sort order is chosen, so that