From 448a9f414f94e5a9a92a9706003aa0a38ea1801c Mon Sep 17 00:00:00 2001
From: Wouter Wijngaards <wouter@nlnetlabs.nl>
Date: Tue, 14 Aug 2007 13:15:36 +0000
Subject: [PATCH] fixups for memory.

git-svn-id: file:///svn/unbound/trunk@512 be551aaa-1e26-0410-a405-d3ace91eadb9
---
 configure.ac               | 16 ++++++++++
 daemon/worker.c            | 64 ++++++++++++++++++++++++++++----------
 doc/Changelog              |  8 +++++
 services/cache/infra.c     | 49 ++++++++++++++++++++++++++---
 services/cache/infra.h     |  7 +++++
 services/outbound_list.c   |  4 +--
 services/outside_network.c | 26 +++++++++++++++-
 services/outside_network.h |  9 ++++++
 testcode/checklocks.c      | 15 +++++++++
 testcode/checklocks.h      |  7 +++++
 testcode/fake_event.c      |  5 +++
 util/alloc.c               | 62 ++++++++++++++++++++++++++++++++++++
 util/config_file.c         |  2 +-
 util/data/msgreply.c       |  4 +--
 util/data/packed_rrset.c   |  2 +-
 util/locks.h               |  1 +
 util/region-allocator.c    | 19 +++++++++--
 util/storage/lruhash.c     | 16 ++++++++++
 util/storage/lruhash.h     |  7 +++++
 util/storage/slabhash.c    |  6 ++--
 validator/val_kentry.c     |  2 +-
 21 files changed, 297 insertions(+), 34 deletions(-)

diff --git a/configure.ac b/configure.ac
index f13c9017b..52fac5494 100644
--- a/configure.ac
+++ b/configure.ac
@@ -646,14 +646,30 @@ struct sockaddr_storage;
 #  ifdef free
 #    undef free
 #  endif
+
+#if 0
+#  define malloc(s) unbound_stat_malloc_log(s, __FILE__, __LINE__, __func__)
+#  define calloc(n,s) unbound_stat_calloc_log(n, s, __FILE__, __LINE__, __func__)
+#  define free(p) unbound_stat_free_log(p, __FILE__, __LINE__, __func__)
+#  define realloc(p,s) unbound_stat_realloc_log(p, s, __FILE__, __LINE__, __func__)
+#else
 #  define malloc unbound_stat_malloc
 #  define calloc unbound_stat_calloc
 #  define free unbound_stat_free
 #  define realloc unbound_stat_realloc
+#endif
 void *unbound_stat_malloc(size_t size);
 void *unbound_stat_calloc(size_t nmemb, size_t size);
 void unbound_stat_free(void *ptr);
 void *unbound_stat_realloc(void *ptr, size_t size);
+void *unbound_stat_malloc_log(size_t size, const char* file, int line,
+	const char* func);
+void *unbound_stat_calloc_log(size_t nmemb, size_t size, const char* file,
+	int line, const char* func);
+void unbound_stat_free_log(void *ptr, const char* file, int line,
+	const char* func);
+void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
+	int line, const char* func);
 #endif /* UNBOUND_ALLOC_STATS */
 
 /** default port for DNS traffic. */
diff --git a/daemon/worker.c b/daemon/worker.c
index 1773a3756..209311c82 100644
--- a/daemon/worker.c
+++ b/daemon/worker.c
@@ -69,9 +69,33 @@
 /** Size of an UDP datagram */
 #define NORMAL_UDP_SIZE	512 /* bytes */
 
+/** measure memory leakage */
+static void
+debug_memleak(size_t accounted, size_t heap, 
+	size_t total_alloc, size_t total_free)
+{
+	static int init = 0;
+	static size_t base_heap, base_accounted, base_alloc, base_free;
+	size_t base_af, cur_af, grow_af, grow_acc;
+	if(!init) {
+		init = 1;
+		base_heap = heap;
+		base_accounted = accounted;
+		base_alloc = total_alloc;
+		base_free = total_free;
+	}
+	base_af = base_alloc - base_free;
+	cur_af = total_alloc - total_free;
+	grow_af = cur_af - base_af;
+	grow_acc = accounted - base_accounted;
+	log_info("Leakage: %u leaked. growth: %u use, %u acc, %u heap",
+		(unsigned)(grow_af - grow_acc), (unsigned)grow_af, 
+		(unsigned)grow_acc, (unsigned)(heap - base_heap));
+}
+
 /** give debug heap size indication */
 static void
-debug_total_mem()
+debug_total_mem(size_t calctotal)
 {
 	extern void* unbound_start_brk;
 	extern size_t unbound_mem_alloc, unbound_mem_freed;
@@ -80,11 +104,13 @@ debug_total_mem()
 	log_info("Total heap memory estimate: %u  total-alloc: %u  "
 		"total-free: %u", (unsigned)total, 
 		(unsigned)unbound_mem_alloc, (unsigned)unbound_mem_freed);
+	debug_memleak(calctotal, (size_t)total, 
+		unbound_mem_alloc, unbound_mem_freed);
 }
 
 /** Report on memory usage by this thread and global */
-static void
-worker_mem_report(struct worker* worker)
+void
+worker_mem_report(struct worker* worker, struct serviced_query* cur_serv)
 {
 	size_t total, front, back, mesh, msg, rrset, infra, ac, superac;
 	size_t me;
@@ -94,13 +120,17 @@ worker_mem_report(struct worker* worker)
 	back = outnet_get_mem(worker->back);
 	msg = slabhash_get_mem(worker->env.msg_cache);
 	rrset = slabhash_get_mem(&worker->env.rrset_cache->table);
-	infra = slabhash_get_mem(worker->env.infra_cache->hosts);
+	infra = infra_get_mem(worker->env.infra_cache);
 	mesh = mesh_get_mem(worker->env.mesh);
 	ac = alloc_get_mem(&worker->alloc);
 	superac = alloc_get_mem(&worker->daemon->superalloc);
 	me = sizeof(*worker) + sizeof(*worker->base) + sizeof(*worker->comsig)
 		+ comm_point_get_mem(worker->cmd_com) + 
-		sizeof(worker->rndstate) + region_get_mem(worker->scratchpad);
+		sizeof(worker->rndstate) + region_get_mem(worker->scratchpad)+
+		sizeof(*worker->env.scratch_buffer) + 
+		ldns_buffer_capacity(worker->env.scratch_buffer);
+	if(cur_serv)
+		me += serviced_get_mem(cur_serv);
 	total = front+back+mesh+msg+rrset+infra+ac+superac+me;
 	log_info("Memory conditions: %u front=%u back=%u mesh=%u msg=%u "
 		"rrset=%u infra=%u alloccache=%u globalalloccache=%u me=%u",
@@ -108,7 +138,7 @@ worker_mem_report(struct worker* worker)
 		(unsigned)mesh, (unsigned)msg, (unsigned)rrset, 
 		(unsigned)infra, (unsigned)ac, (unsigned)superac, 
 		(unsigned)me);
-	debug_total_mem();
+	debug_total_mem(total);
 }
 
 void 
@@ -138,7 +168,7 @@ worker_handle_reply(struct comm_point* c, void* arg, int error,
 
 	if(error != 0) {
 		mesh_report_reply(worker->env.mesh, &e, 0, reply_info);
-		worker_mem_report(worker);
+		worker_mem_report(worker, NULL);
 		return 0;
 	}
 	/* sanity check. */
@@ -149,11 +179,11 @@ worker_handle_reply(struct comm_point* c, void* arg, int error,
 		/* error becomes timeout for the module as if this reply
 		 * never arrived. */
 		mesh_report_reply(worker->env.mesh, &e, 0, reply_info);
-		worker_mem_report(worker);
+		worker_mem_report(worker, NULL);
 		return 0;
 	}
 	mesh_report_reply(worker->env.mesh, &e, 1, reply_info);
-	worker_mem_report(worker);
+	worker_mem_report(worker, NULL);
 	return 0;
 }
 
@@ -164,11 +194,12 @@ worker_handle_service_reply(struct comm_point* c, void* arg, int error,
 {
 	struct outbound_entry* e = (struct outbound_entry*)arg;
 	struct worker* worker = e->qstate->env->worker;
+	struct serviced_query *sq = e->qsent;
 
 	verbose(VERB_ALGO, "worker svcd callback for qstate %p", e->qstate);
 	if(error != 0) {
 		mesh_report_reply(worker->env.mesh, e, 0, reply_info);
-		worker_mem_report(worker);
+		worker_mem_report(worker, sq);
 		return 0;
 	}
 	/* sanity check. */
@@ -180,11 +211,11 @@ worker_handle_service_reply(struct comm_point* c, void* arg, int error,
 		 * never arrived. */
 		verbose(VERB_ALGO, "worker: bad reply handled as timeout");
 		mesh_report_reply(worker->env.mesh, e, 0, reply_info);
-		worker_mem_report(worker);
+		worker_mem_report(worker, sq);
 		return 0;
 	}
 	mesh_report_reply(worker->env.mesh, e, 1, reply_info);
-	worker_mem_report(worker);
+	worker_mem_report(worker, sq);
 	return 0;
 }
 
@@ -556,7 +587,7 @@ worker_handle_request(struct comm_point* c, void* arg, int error,
 		/* the max request number has been reached, stop accepting */
 		listen_pushback(worker->front);
 	}
-	worker_mem_report(worker);
+	worker_mem_report(worker, NULL);
 	return 0;
 }
 
@@ -731,7 +762,7 @@ worker_init(struct worker* worker, struct config_file *cfg,
 		worker_delete(worker);
 		return 0;
 	}
-	worker_mem_report(worker);
+	worker_mem_report(worker, NULL);
 	return 1;
 }
 
@@ -748,7 +779,7 @@ worker_delete(struct worker* worker)
 		return;
 	mesh_stats(worker->env.mesh, "mesh has");
 	server_stats_log(&worker->stats, worker->thread_num);
-	worker_mem_report(worker);
+	worker_mem_report(worker, NULL);
 	mesh_delete(worker->env.mesh);
 	ldns_buffer_free(worker->env.scratch_buffer);
 	listen_delete(worker->front);
@@ -804,7 +835,8 @@ worker_send_query(uint8_t* qname, size_t qnamelen, uint16_t qtype,
 	struct module_qstate* q)
 {
 	struct worker* worker = q->env->worker;
-	struct outbound_entry* e = (struct outbound_entry*)malloc(sizeof(*e));
+	struct outbound_entry* e = (struct outbound_entry*)region_alloc(
+		q->region, sizeof(*e));
 	if(!e) 
 		return NULL;
 	e->qstate = q;
diff --git a/doc/Changelog b/doc/Changelog
index 3db371724..7a48d1e0d 100644
--- a/doc/Changelog
+++ b/doc/Changelog
@@ -1,3 +1,11 @@
+14 August 2007: Wouter
+	- default outgoing ports changed to avoid port 2049 by default.
+	  This port is widely blocked by firewalls.
+	- count infra lameness cache in memory size.
+	- accounting of memory improved
+	- outbound entries are allocated in the query region they are for.
+	- extensive debugging for memory allocations.
+
 13 August 2007: Wouter
 	- fixup makefile, if lexer is missing give nice error and do not
 	  mess up the dependencies.
diff --git a/services/cache/infra.c b/services/cache/infra.c
index 043e72f9d..8cdaebfbc 100644
--- a/services/cache/infra.c
+++ b/services/cache/infra.c
@@ -50,9 +50,11 @@
 /** calculate size for the hashtable, does not count size of lameness,
  * so the hashtable is a fixed number of items */
 static size_t 
-infra_host_sizefunc(void* ATTR_UNUSED(k), void* ATTR_UNUSED(d))
+infra_host_sizefunc(void* k, void* ATTR_UNUSED(d))
 {
-	return sizeof(struct infra_host_key) + sizeof(struct infra_host_data);
+	struct infra_host_key* key = (struct infra_host_key*)k;
+	return sizeof(struct infra_host_key) + sizeof(struct infra_host_data) 
+		+ lock_get_mem(&key->entry.lock);
 }
 
 /** compare two addresses, returns -1, 0, or +1 */
@@ -291,9 +293,11 @@ infra_lookup_lame(struct infra_host_data* host,
 /** calculate size, which is fixed, zonename does not count so that
  * a fixed number of items is stored */
 static size_t 
-infra_lame_sizefunc(void* ATTR_UNUSED(k), void* ATTR_UNUSED(d))
+infra_lame_sizefunc(void* k, void* ATTR_UNUSED(d))
 {
-	return sizeof(struct infra_lame_key)+sizeof(struct infra_lame_data);
+	struct infra_lame_key* key = (struct infra_lame_key*)k;
+	return sizeof(struct infra_lame_key)+sizeof(struct infra_lame_data)
+		+ lock_get_mem(&key->entry.lock);
 }
 
 /** compare zone names, returns -1, 0, +1 */
@@ -486,3 +490,40 @@ infra_get_lame_rtt(struct infra_cache* infra,
 	lock_rw_unlock(&e->lock);
 	return 1;
 }
+
+/** helper memory count for a host lame cache */
+static size_t
+count_host_lame(struct lruhash_entry* e)
+{
+	struct infra_host_data* host_data = (struct infra_host_data*)e->data;
+	if(!host_data->lameness)
+		return 0;
+	return lruhash_get_mem(host_data->lameness);
+}
+
+size_t 
+infra_get_mem(struct infra_cache* infra)
+{
+	size_t i, bin;
+	size_t s = sizeof(*infra) +
+		slabhash_get_mem(infra->hosts);
+	struct lruhash_entry* e;
+	for(i=0; i<infra->hosts->size; i++) {
+		lock_quick_lock(&infra->hosts->array[i]->lock);
+		for(bin=0; bin<infra->hosts->array[i]->size; bin++) {
+			lock_quick_lock(&infra->hosts->array[i]->
+				array[bin].lock);
+			/* count data size in bin items. */
+			for(e = infra->hosts->array[i]->array[bin].
+				overflow_list; e; e = e->overflow_next) {
+				lock_rw_rdlock(&e->lock);
+				s += count_host_lame(e);
+				lock_rw_unlock(&e->lock);
+			}
+			lock_quick_unlock(&infra->hosts->array[i]->
+				array[bin].lock);
+		}
+		lock_quick_unlock(&infra->hosts->array[i]->lock);
+	}
+	return s;
+}
diff --git a/services/cache/infra.h b/services/cache/infra.h
index e9c1b6c37..4a0b36d42 100644
--- a/services/cache/infra.h
+++ b/services/cache/infra.h
@@ -235,4 +235,11 @@ int infra_get_lame_rtt(struct infra_cache* infra,
         struct sockaddr_storage* addr, socklen_t addrlen, 
 	uint8_t* name, size_t namelen, int* lame, int* rtt, time_t timenow);
 
+/**
+ * Get memory used by the infra cache.
+ * @param infra: infrastructure cache.
+ * @return memory in use in bytes.
+ */
+size_t infra_get_mem(struct infra_cache* infra);
+
 #endif /* SERVICES_CACHE_INFRA_H */
diff --git a/services/outbound_list.c b/services/outbound_list.c
index 5fce67f7b..80928dec4 100644
--- a/services/outbound_list.c
+++ b/services/outbound_list.c
@@ -57,7 +57,7 @@ outbound_list_clear(struct outbound_list* list)
 	while(p) {
 		np = p->next;
 		outnet_serviced_query_stop(p->qsent, p);
-		free(p);
+		/* in region, no free needed */
 		p = np;
 	}
 	outbound_list_init(list);
@@ -84,5 +84,5 @@ outbound_list_remove(struct outbound_list* list, struct outbound_entry* e)
 	if(e->prev)
 		e->prev->next = e->next;
 	else	list->first = e->next;
-	free(e);
+	/* in region, no free needed */
 }
diff --git a/services/outside_network.c b/services/outside_network.c
index 3cc81b9b0..f38874d78 100644
--- a/services/outside_network.c
+++ b/services/outside_network.c
@@ -897,6 +897,7 @@ serviced_delete(struct serviced_query* sq)
 					(struct pending_tcp*)p->next_waiting);
 			} else {
 				waiting_list_remove(sq->outnet, p);
+				waiting_tcp_delete(p);
 			}
 		}
 	}
@@ -1241,7 +1242,8 @@ size_t outnet_get_mem(struct outside_network* outnet)
 	for(w=outnet->tcp_wait_first; w; w = w->next_waiting)
 		s += waiting_tcp_get_mem(w);
 	s += sizeof(*outnet->pending);
-	s += sizeof(struct pending) * outnet->pending->count;
+	s += (sizeof(struct pending) + comm_timer_get_mem(NULL)) * 
+		outnet->pending->count;
 	s += sizeof(*outnet->serviced);
 	RBTREE_FOR(sq, struct serviced_query*, outnet->serviced) {
 		s += sizeof(*sq) + sq->qbuflen;
@@ -1250,3 +1252,25 @@ size_t outnet_get_mem(struct outside_network* outnet)
 	}
 	return s;
 }
+
+size_t 
+serviced_get_mem(struct serviced_query* sq)
+{
+	struct service_callback* sb;
+	size_t s;
+	s = sizeof(*sq) + sq->qbuflen;
+	for(sb = sq->cblist; sb; sb = sb->next)
+		s += sizeof(*sb);
+	/* always sq->pending existed, but is null to delete after callback */
+	if(sq->status == serviced_query_UDP_EDNS ||
+		sq->status == serviced_query_UDP) {
+		s += sizeof(struct pending);
+		s += comm_timer_get_mem(NULL);
+	} else {
+		/* does not have size of the pkt pointer */
+		s += sizeof(struct waiting_tcp);
+		/* always has a timer expect on malloc failures */
+		s += comm_timer_get_mem(NULL);
+	}
+	return s;
+}
diff --git a/services/outside_network.h b/services/outside_network.h
index 06555f863..60cf8b8cb 100644
--- a/services/outside_network.h
+++ b/services/outside_network.h
@@ -373,4 +373,13 @@ void outnet_serviced_query_stop(struct serviced_query* sq, void* cb_arg);
  */
 size_t outnet_get_mem(struct outside_network* outnet);
 
+/**
+ * Get memory size in use by serviced query while it is servicing callbacks.
+ * This takes into account the pre-deleted status of it; it will be deleted
+ * when the callbacks are done.
+ * @param sq: serviced query. 
+ * @return size in bytes.
+ */
+size_t serviced_get_mem(struct serviced_query* sq);
+
 #endif /* OUTSIDE_NETWORK_H */
diff --git a/testcode/checklocks.c b/testcode/checklocks.c
index febc518bc..d65dc2600 100644
--- a/testcode/checklocks.c
+++ b/testcode/checklocks.c
@@ -205,6 +205,21 @@ prot_store(struct checked_lock* lock)
 	}
 }
 
+size_t lock_get_mem(void* pp)
+{
+	size_t s;
+	struct checked_lock* lock = *(struct checked_lock**)pp;
+	struct protected_area* p;
+	s = sizeof(struct checked_lock);
+	acquire_locklock(lock, __func__, __FILE__, __LINE__);
+	for(p = lock->prot; p; p = p->next) {
+		s += sizeof(struct protected_area);
+		s += p->size;
+	}
+	LOCKRET(pthread_mutex_unlock(&lock->lock));
+	return s;
+}
+
 /** write lock trace info to file, while you hold those locks */
 static void
 ordercheck_locklock(struct thr_check* thr, struct checked_lock* lock)
diff --git a/testcode/checklocks.h b/testcode/checklocks.h
index 500bb9fc3..54c2ef02e 100644
--- a/testcode/checklocks.h
+++ b/testcode/checklocks.h
@@ -196,6 +196,13 @@ void lock_protect(void* lock, void* area, size_t size);
  */
 void lock_unprotect(void* lock, void* area);
 
+/**
+ * Get memory associated with a checked lock
+ * @param lock: the checked lock, any type. (struct checked_lock**).
+ * @return: in bytes, including protected areas.
+ */
+size_t lock_get_mem(void* lock);
+
 /**
  * Initialise checklock. Sets up internal debug structures.
  */
diff --git a/testcode/fake_event.c b/testcode/fake_event.c
index 341328d2d..00086e5af 100644
--- a/testcode/fake_event.c
+++ b/testcode/fake_event.c
@@ -952,4 +952,9 @@ size_t comm_point_get_mem(struct comm_point* ATTR_UNUSED(c))
 	return 0;
 }
 
+size_t serviced_get_mem(struct serviced_query* ATTR_UNUSED(c))
+{
+	return 0;
+}
+
 /*********** End of Dummy routines ***********/
diff --git a/util/alloc.c b/util/alloc.c
index 97cd1eebc..1350faf9d 100644
--- a/util/alloc.c
+++ b/util/alloc.c
@@ -242,11 +242,15 @@ alloc_stats(struct alloc_cache* alloc)
 
 size_t alloc_get_mem(struct alloc_cache* alloc)
 {
+	alloc_special_t* p;
 	size_t s = sizeof(*alloc);
 	if(!alloc->super) { 
 		lock_quick_lock(&alloc->lock); /* superalloc needs locking */
 	}
 	s += sizeof(alloc_special_t) * alloc->num_quar;
+	for(p = alloc->quar; p; p = alloc_special_next(p)) {
+		s += lock_get_mem(&p->entry.lock);
+	}
 	if(!alloc->super) {
 		lock_quick_unlock(&alloc->lock);
 	}
@@ -271,6 +275,7 @@ void *unbound_stat_malloc(size_t size)
 	res = malloc(size+16);
 	if(!res) return NULL;
 	unbound_mem_alloc += size;
+	log_info("stat %p=malloc(%u)", res+16, (unsigned)size);
 	memcpy(res, &size, sizeof(size));
 	memcpy(res+8, &mem_special, sizeof(mem_special));
 	return res+16;
@@ -284,6 +289,7 @@ void *unbound_stat_calloc(size_t nmemb, size_t size)
 	size_t s = (nmemb*size==0)?(size_t)1:nmemb*size;
 	void* res = calloc(1, s+16);
 	if(!res) return NULL;
+	log_info("stat %p=calloc(%u, %u)", res+16, (unsigned)nmemb, (unsigned)size);
 	unbound_mem_alloc += s;
 	memcpy(res, &s, sizeof(s));
 	memcpy(res+8, &mem_special, sizeof(mem_special));
@@ -303,6 +309,7 @@ void unbound_stat_free(void *ptr)
 	}
 	ptr-=16;
 	memcpy(&s, ptr, sizeof(s));
+	log_info("stat free(%p) size %u", ptr+16, (unsigned)s);
 	memset(ptr+8, 0, 8);
 	unbound_mem_freed += s;
 	free(ptr);
@@ -333,6 +340,7 @@ void *unbound_stat_realloc(void *ptr, size_t size)
 	if(!res) return NULL;
 	unbound_mem_alloc += size;
 	unbound_mem_freed += cursz;
+	log_info("stat realloc(%p, %u) from %u", ptr+16, (unsigned)size, (unsigned)cursz);
 	if(cursz > size) {
 		memcpy(res+16, ptr+16, size);
 	} else if(size > cursz) {
@@ -344,4 +352,58 @@ void *unbound_stat_realloc(void *ptr, size_t size)
 	memcpy(res+8, &mem_special, sizeof(mem_special));
 	return res+16;
 }
+
+void *unbound_stat_malloc_log(size_t size, const char* file, int line,
+        const char* func)
+{
+	log_info("%s:%d %s malloc(%u)", file, line, func, (unsigned)size);
+	return unbound_stat_malloc(size);
+}
+
+void *unbound_stat_calloc_log(size_t nmemb, size_t size, const char* file,
+        int line, const char* func)
+{
+	log_info("%s:%d %s calloc(%u, %u)", file, line, func, 
+		(unsigned) nmemb, (unsigned)size);
+	return unbound_stat_calloc(nmemb, size);
+}
+
+void unbound_stat_free_log(void *ptr, const char* file, int line,
+        const char* func)
+{
+	if(ptr && memcmp(ptr-8, &mem_special, sizeof(mem_special)) == 0) {
+		size_t s;
+		memcpy(&s, ptr-16, sizeof(s));
+		log_info("%s:%d %s free(%p) size %u", 
+			file, line, func, ptr, (unsigned)s);
+	} else
+		log_info("%s:%d %s unmatched free(%p)", file, line, func, ptr);
+	unbound_stat_free(ptr);
+}
+
+void *unbound_stat_realloc_log(void *ptr, size_t size, const char* file,
+        int line, const char* func)
+{
+	log_info("%s:%d %s realloc(%p, %u)", file, line, func, 
+		ptr, (unsigned)size);
+	return unbound_stat_realloc(ptr, size);
+}
+
+void *unbound_stat_malloc_region(size_t size)
+{
+	log_info("region malloc(%u)", (unsigned)size);
+	return unbound_stat_malloc(size);
+}
+
+void unbound_stat_free_region(void *ptr)
+{
+	if(ptr && memcmp(ptr-8, &mem_special, sizeof(mem_special)) == 0) {
+		size_t s;
+		memcpy(&s, ptr-16, sizeof(s));
+		log_info("region free(%p) size %u", ptr, (unsigned)s);
+	} else
+		log_info("region unmatched free(%p)", ptr);
+	unbound_stat_free(ptr);
+}
+
 #endif /* UNBOUND_ALLOC_STATS */
diff --git a/util/config_file.c b/util/config_file.c
index 5d831a4b0..ac1ed9e5f 100644
--- a/util/config_file.c
+++ b/util/config_file.c
@@ -75,7 +75,7 @@ config_create()
 	cfg->do_ip6 = 1;
 	cfg->do_udp = 1;
 	cfg->do_tcp = 1;
-	cfg->outgoing_base_port = cfg->port + 1000;
+	cfg->outgoing_base_port = cfg->port + 2000;
 	cfg->outgoing_num_ports = 16;
 	cfg->outgoing_num_tcp = 10;
 	cfg->msg_cache_size = 4 * 1024 * 1024;
diff --git a/util/data/msgreply.c b/util/data/msgreply.c
index d890fa9f1..8acdbb6e3 100644
--- a/util/data/msgreply.c
+++ b/util/data/msgreply.c
@@ -526,10 +526,10 @@ query_info_clear(struct query_info* m)
 size_t 
 msgreply_sizefunc(void* k, void* d)
 {
-	struct query_info* q = (struct query_info*)k;
+	struct msgreply_entry* q = (struct msgreply_entry*)k;
 	struct reply_info* r = (struct reply_info*)d;
 	size_t s = sizeof(struct msgreply_entry) + sizeof(struct reply_info)
-		+ q->qname_len;
+		+ q->key.qname_len + lock_get_mem(&q->entry.lock);
 	s += (r->rrset_count-1) * sizeof(struct rrset_ref);
 	s += r->rrset_count * sizeof(struct ub_packed_rrset_key*);
 	return s;
diff --git a/util/data/packed_rrset.c b/util/data/packed_rrset.c
index 286beb18e..a9d1abfdc 100644
--- a/util/data/packed_rrset.c
+++ b/util/data/packed_rrset.c
@@ -68,7 +68,7 @@ ub_rrset_sizefunc(void* key, void* data)
 	struct ub_packed_rrset_key* k = (struct ub_packed_rrset_key*)key;
 	struct packed_rrset_data* d = (struct packed_rrset_data*)data;
 	size_t s = sizeof(struct ub_packed_rrset_key) + k->rk.dname_len;
-	s += packed_rrset_sizeof(d);
+	s += packed_rrset_sizeof(d) + lock_get_mem(&k->entry.lock);
 	return s;
 }
 
diff --git a/util/locks.h b/util/locks.h
index 34f4f7607..706574ac2 100644
--- a/util/locks.h
+++ b/util/locks.h
@@ -82,6 +82,7 @@
 #else /* USE_THREAD_DEBUG */
 #define lock_protect(lock, area, size) /* nop */
 #define lock_unprotect(lock, area) /* nop */
+#define lock_get_mem(lock) (0) /* nothing */
 #define checklock_start() /* nop */
 #define checklock_stop() /* nop */
 
diff --git a/util/region-allocator.c b/util/region-allocator.c
index b3946edf4..69b0b0e7a 100644
--- a/util/region-allocator.c
+++ b/util/region-allocator.c
@@ -161,7 +161,14 @@ alloc_region_base(void *(*allocator)(size_t size),
 region_type *
 region_create(void *(*allocator)(size_t), void (*deallocator)(void *))
 {
-	region_type* result = alloc_region_base(allocator, deallocator, 
+	region_type* result;
+#ifdef UNBOUND_ALLOC_STATS
+	void *unbound_stat_malloc_region(size_t size);
+	void unbound_stat_free_region(void *ptr);
+	allocator = &unbound_stat_malloc_region;
+	deallocator = &unbound_stat_free_region;
+#endif
+	result = alloc_region_base(allocator, deallocator, 
 		DEFAULT_INITIAL_CLEANUP_SIZE);
 	if(!result)
 		return NULL;
@@ -184,7 +191,14 @@ region_type *region_create_custom(void *(*allocator)(size_t),
 				  size_t initial_cleanup_size,
 				  int recycle)
 {
-	region_type* result = alloc_region_base(allocator, deallocator, 
+	region_type* result;
+#ifdef UNBOUND_ALLOC_STATS
+	void *unbound_stat_malloc_region(size_t size);
+	void unbound_stat_free_region(void *ptr);
+	allocator = &unbound_stat_malloc_region;
+	deallocator = &unbound_stat_free_region;
+#endif
+	result = alloc_region_base(allocator, deallocator, 
 		initial_cleanup_size);
 	if(!result)
 		return NULL;
@@ -517,5 +531,6 @@ region_get_mem(region_type* region)
 	s += region->maximum_cleanup_count * sizeof(cleanup_type);
 	if(region->recycle_bin)
 		s += sizeof(struct recycle_elem*)*region->large_object_size;
+	log_assert(s >= region->chunk_size * region->chunk_count);
 	return s;
 }
diff --git a/util/storage/lruhash.c b/util/storage/lruhash.c
index 8068c36b4..f5c81c853 100644
--- a/util/storage/lruhash.c
+++ b/util/storage/lruhash.c
@@ -421,3 +421,19 @@ lruhash_status(struct lruhash* table, const char* id, int extended)
 	}
 	lock_quick_unlock(&table->lock);
 }
+
+size_t
+lruhash_get_mem(struct lruhash* table)
+{
+	size_t s;
+	size_t i;
+	lock_quick_lock(&table->lock);
+	s = sizeof(struct lruhash) + table->space_used;
+	for(i=0; i<table->size; i++) {
+		s +=  sizeof(struct lruhash_bin) + 
+			lock_get_mem(&table->array[i].lock);
+	}
+	lock_quick_unlock(&table->lock);
+	s += lock_get_mem(&table->lock);
+	return s;
+}
diff --git a/util/storage/lruhash.h b/util/storage/lruhash.h
index 81c45de01..035f4a49a 100644
--- a/util/storage/lruhash.h
+++ b/util/storage/lruhash.h
@@ -379,4 +379,11 @@ void lru_remove(struct lruhash* table, struct lruhash_entry* entry);
  */
 void lruhash_status(struct lruhash* table, const char* id, int extended);
 
+/**
+ * Get memory in use now by the lruhash table.
+ * @param table: hash table. Will be locked before use. And unlocked after.
+ * @return size in bytes.
+ */
+size_t lruhash_get_mem(struct lruhash* table);
+
 #endif /* UTIL_STORAGE_LRUHASH_H */
diff --git a/util/storage/slabhash.c b/util/storage/slabhash.c
index 2ce58f7cc..0483e8b53 100644
--- a/util/storage/slabhash.c
+++ b/util/storage/slabhash.c
@@ -147,11 +147,9 @@ size_t slabhash_get_size(struct slabhash* sl)
 size_t slabhash_get_mem(struct slabhash* sl)
 {	
 	size_t i, total = sizeof(*sl);
+	total += sizeof(struct lruhash*)*sl->size;
 	for(i=0; i<sl->size; i++) {
-		lock_quick_lock(&sl->array[i]->lock);
-		total += sizeof(struct lruhash) + sl->array[i]->space_used +
-			sizeof(struct lruhash_bin)*sl->array[i]->size;
-		lock_quick_unlock(&sl->array[i]->lock);
+		total += lruhash_get_mem(sl->array[i]);
 	}
 	return total;
 }
diff --git a/validator/val_kentry.c b/validator/val_kentry.c
index 350221e63..5ef050b7c 100644
--- a/validator/val_kentry.c
+++ b/validator/val_kentry.c
@@ -52,7 +52,7 @@ key_entry_sizefunc(void* key, void* data)
 	struct key_entry_key* kk = (struct key_entry_key*)key;
 	struct key_entry_data* kd = (struct key_entry_data*)data;
 	size_t s = sizeof(*kk) + kk->namelen;
-	s += sizeof(*kd);
+	s += sizeof(*kd) + lock_get_mem(&kk->entry.lock);
 	if(kd->rrset_data)
 		s += packed_rrset_sizeof(kd->rrset_data);
 	return s;
-- 
2.47.2