From: Marek Vavruša <mvavrusa@cloudflare.com>
Date: Sun, 26 Nov 2017 00:23:46 +0000 (-0800)
Subject: tests/config: added a TAP-based test environment for modules/configs
X-Git-Tag: v1.5.1~12^2~7
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f41676d299309b775401994079788647d6798e8b;p=thirdparty%2Fknot-resolver.git

tests/config: added a TAP-based test environment for modules/configs

I moved the test files to module directories because it allows
vendoring of whole modules including tests etc.

The test environment provides convenience functions and produces
test output in TAP format. Ideally all tests should use a common
format, so that CI can parse it provide better test output on PRs.
It seems like Gitlab CI doesn't support anything yet, but there
are two sort-of standards supported in CI tools - TAP and JUnit.
I chose TAP because it's easier to read for humans, cmocka supports it,
and it should be easier to adapt Deckard. There are also tools to
convert TAP into JUnit XML file.

Also added more tests for global functions and variables, and the
test tool now also tracks coverage (if `luacov` is installed).
---

diff --git a/.gitignore b/.gitignore
index ae255f2c7..b499e0894 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,6 +18,8 @@
 *.gcno
 *.gcda
 *.gcov
+*.info
+luacov.*.out
 .dirstamp
 .libs
 .deps
diff --git a/.gitmodules b/.gitmodules
index 6a6c94579..2c60e9d04 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -4,3 +4,6 @@
 [submodule "modules/policy/lua-aho-corasick"]
 	path = modules/policy/lua-aho-corasick
 	url = git://github.com/cloudflare/lua-aho-corasick.git
+[submodule "tests/config/tapered"]
+	path = tests/config/tapered
+	url = https://github.com/telemachus/tapered.git
diff --git a/.luacheckrc b/.luacheckrc
index c514ccb57..d17076c7a 100644
--- a/.luacheckrc
+++ b/.luacheckrc
@@ -61,6 +61,7 @@ end
 -- Ignore test files
 exclude_files = {
 	'modules/policy/lua-aho-corasick', -- Vendored
+	'tests/config/tapered',
 }
 
 -- Ignore some pedantic checks
@@ -75,4 +76,4 @@ files['daemon/lua/kres-gen.lua'].ignore = {'631'} -- Allow overly long lines
 -- Tests and scripts can use global variables
 files['scripts'].ignore = {'111', '112', '113'}
 files['tests'].ignore = {'111', '112', '113'}
-files['tests/config/test_utils.lua'].ignore = {'121'}
\ No newline at end of file
+files['modules/*/*_test.lua'].ignore = {'111', '112', '113'}
\ No newline at end of file
diff --git a/modules/hints/hints_test.lua b/modules/hints/hints_test.lua
new file mode 100644
index 000000000..5e24ff33b
--- /dev/null
+++ b/modules/hints/hints_test.lua
@@ -0,0 +1,34 @@
+local utils = require('test_utils')
+
+-- setup resolver
+modules = { 'hints' }
+
+-- test for default configuration
+local function test_default()
+	-- get loaded root hints and change names to lowercase
+	hints_data = utils.table_keys_to_lower(hints.root())
+
+	-- root hints loaded from default location 
+	-- check correct ip address of a.root-server.net
+	utils.contains(hints_data['a.root-servers.net.'], '198.41.0.4', 'has IP address for a.root-servers.net.')
+end
+
+-- test loading from config file
+local function test_custom()
+	-- load custom root hints file with fake ip address for a.root-server.net
+	err_msg = hints.root_file(TEST_DIR .. 'hints_test.zone')
+	same(err_msg, '', 'load root hints from file')
+
+	-- get loaded root hints and change names to lowercase
+	hints_data = utils.table_keys_to_lower(hints.root())
+	isnt(hints_data['a.root-servers.net.'], nil, 'can retrieve root hints')
+
+	-- check loaded ip address of a.root-server.net
+	utils.not_contains(hints_data['a.root-servers.net.'], '198.41.0.4', 'real IP address for a.root-servers.net. is replaced')
+	utils.contains(hints_data['a.root-servers.net.'], '10.0.0.1', 'real IP address for a.root-servers.net. is correct')
+end
+
+return {
+	test_default,
+	test_custom
+}
\ No newline at end of file
diff --git a/tests/config/hints/hints.zone b/modules/hints/hints_test.zone
similarity index 100%
rename from tests/config/hints/hints.zone
rename to modules/hints/hints_test.zone
diff --git a/tests/config/predict/test.cfg b/modules/predict/predict_test.lua
similarity index 62%
rename from tests/config/predict/test.cfg
rename to modules/predict/predict_test.lua
index 3344f97bd..1f6d241ad 100644
--- a/tests/config/predict/test.cfg
+++ b/modules/predict/predict_test.lua
@@ -1,60 +1,59 @@
-dofile('./test_utils.lua') -- load test utilities
-
 -- setup resolver
 modules = { 'predict' }
 
 -- mock global functions
 local resolve_count = 0
+local current_epoch = 0
+
 worker.resolve = function ()
 	resolve_count = resolve_count + 1
 end
+
 stats.frequent = function ()
 	return {
 		{name = 'example.com', type = 'TYPE65535'},
 		{name = 'example.com', type = 'SOA'},
 	}
 end
-local current_epoch = 0
+
 predict.epoch = function ()
 	return current_epoch % predict.period + 1
 end
 
 -- test if draining of prefetch queue works
-function test_predict_drain()
+local function test_predict_drain()
 	predict.queue_len = 2
 	predict.queue['TYPE65535 example.com'] = 1
 	predict.queue['SOA example.com'] = 1
 	predict.drain()
 	-- test that it attempted to prefetch
-	assert.same(2, resolve_count)
-	assert.same(0, predict.queue_len)
+	same(resolve_count, 2, 'attempted to prefetch on drain')
+	same(predict.queue_len, 0, 'prefetch queue empty after drain')
 end
 
 -- test if prediction process works
-function test_predict_process()
+local function test_predict_process()
 	-- start new epoch
 	predict.process()
-	assert.same(0, predict.queue_len)
+	same(predict.queue_len, 0, 'first epoch, empty prefetch queue')
 	-- next epoch, still no period for frequent queries
 	current_epoch = current_epoch + 1
 	predict.process()
-	assert.same(0, predict.queue_len)
+	same(predict.queue_len, 0, 'second epoch, empty prefetch queue')
 	-- next epoch, found period
 	current_epoch = current_epoch + 1
 	predict.process()
-	assert.same(2, predict.queue_len)
+	same(predict.queue_len, 2, 'third epoch, prefetching')
 	-- drain works with scheduled prefetches (two batches)
 	resolve_count = 0
 	predict.drain()
 	predict.drain()
-	assert.same(2, resolve_count)
-	assert.same(0, predict.queue_len)
+	same(resolve_count, 2, 'attempted to resolve queries in queue')
+	same(predict.queue_len, 0, 'prefetch queue is empty')
 end
 
--- run test after processed config file
--- default config will be used and we can test it.
-event.after(0, function (ev) 
-	test(test_predict_drain)
-	test(test_predict_process)
-	quit()
-end)
+-- return test set
+return {
+	test_predict_drain,
+	test_predict_process
+}
\ No newline at end of file
diff --git a/tests/config/basic/test.cfg b/tests/config/basic/test.cfg
deleted file mode 100644
index 19f21d13a..000000000
--- a/tests/config/basic/test.cfg
+++ /dev/null
@@ -1,26 +0,0 @@
-dofile('./test_utils.lua') -- load test utilities
-
--- test if constants work properly
-function test_constants()
-	assert.same(1, kres.class.IN)
-	assert.same(1, kres.class['IN'])
-	assert.same(2, kres.type.NS)
-	assert.same(2, kres.type.TYPE2)
-	assert.same(nil, kres.type.BADTYPE)
-	assert.same(2, kres.rcode.SERVFAIL)
-end
-
--- test if rrsets interfaces work
-function test_rrsets()
-	local rr = {owner = '\3com', ttl = 1, type = kres.type.TXT, rdata = '\5hello'}
-	local rr_text = tostring(kres.rr2str(rr))
-	assert.same('com. 1 TXT "hello"', rr_text:gsub('%s+', ' '))
-end
-
--- run test after processed config file
--- default config will be used and we can test it.
-event.after(0, function (ev) 
-	test(test_constants)
-	test(test_rrsets)
-	quit()
-end)
diff --git a/tests/config/basic_test.lua b/tests/config/basic_test.lua
new file mode 100644
index 000000000..4dae147d0
--- /dev/null
+++ b/tests/config/basic_test.lua
@@ -0,0 +1,34 @@
+-- test if constants work properly
+local function test_constants()
+	same(kres.class.IN, 1, 'class constants work')
+	same(kres.type.NS, 2, 'record type constants work')
+	same(kres.type.TYPE2, 2, 'unnamed record type constants work')
+	same(kres.type.BADTYPE, nil, 'non-existent type constants are checked')
+	same(kres.rcode.SERVFAIL, 2, 'rcode constants work')
+end
+
+-- test globals
+local function test_globals()
+	ok(mode('strict'), 'changing strictness mode')
+	boom(mode, {'badmode'}, 'changing to non-existent strictness mode')
+	same(reorder_RR(true), true, 'answer section reordering')
+	same(option('REORDER_RR', false), false, 'generic option call')
+	boom(option, {'REORDER_RR', 'potato'}, 'generic option call argument check')
+	boom(option, {'MARS_VACATION', false}, 'generic option check name')
+	same(table_print('crabdiary'), 'crabdiary\n', 'table print works')
+	same(table_print({fakepizza=1}), '[fakepizza] => 1\n', 'table print works on tables')
+end
+
+-- test if dns library functions work
+local function test_kres_functions()
+	local rr = {owner = '\3com', ttl = 1, type = kres.type.TXT, rdata = '\5hello'}
+	local rr_text = tostring(kres.rr2str(rr))
+	same(rr_text:gsub('%s+', ' '), 'com. 1 TXT "hello"', 'rrset to text works')
+	same(kres.dname2str(todname('com.')), 'com.', 'domain name conversion works')
+end
+
+return {
+	test_constants,
+	test_globals,
+	test_kres_functions,
+}
\ No newline at end of file
diff --git a/tests/config/hints/test.cfg b/tests/config/hints/test.cfg
deleted file mode 100644
index dc1c1e99f..000000000
--- a/tests/config/hints/test.cfg
+++ /dev/null
@@ -1,44 +0,0 @@
-dofile('./test_utils.lua') -- load test utilities
-
--- setup resolver
-modules = { 'hints' }
-
--- test for default configuration
-function test_default()
-	-- get loaded root hints and change names to lowercase
-	hints_data = table_keys_to_lower(hints.root())
-
-	-- root hints loaded from default location 
-	-- check correct ip address of a.root-server.net
-	if not contains(hints_data['a.root-servers.net.'], '198.41.0.4') then
-		fail("Real IP address for a.root-servers.net. not found.")
-	end
-end
-
--- test loading from config file
-function test_custom()
-        -- load custom root hints file with fake ip address for a.root-server.net
-        err_msg = hints.root_file('hints.zone')
-        if err_msg ~= '' then
-                fail("hints.root_file error: %s", err_msg)
-        end
-
-        -- get loaded root hints and change names to lowercase
-	hints_data = table_keys_to_lower(hints.root())
-
-        -- check loaded ip address of a.root-server.net
-        if contains(hints_data['a.root-servers.net.'], '198.41.0.4') then
-                fail("Real IP address for a.root-servers.net. not removed")
-        end
-        if not contains(hints_data['a.root-servers.net.'], '10.0.0.1') then
-                fail("Fake IP address for a.root-servers.net. not found.")
-        end
-end
-
--- run test after processed config file
--- default config will be used and we can test it.
-ev = event.after(0, function (ev) 
-        test_default()
-	test_custom()
-        quit()
-end)
diff --git a/tests/config/runtest.sh b/tests/config/runtest.sh
index 25b9874d4..48dad22e2 100755
--- a/tests/config/runtest.sh
+++ b/tests/config/runtest.sh
@@ -1,11 +1,12 @@
 #!/bin/bash -e
+export SOURCE_PATH=$(cd "$(dirname "$0")" && pwd -P)
+export TEST_FILE=${2}
 export TMP_RUNDIR="$(mktemp -d)"
+export KRESD_NO_LISTEN=1
 function finish {
-  rm -rf "${TMP_RUNDIR}"
+	rm -rf "${TMP_RUNDIR}"
 }
 trap finish EXIT
 
-echo "config-test: ${2}"
-cp "tests/config/${2}/"* "${TMP_RUNDIR}/"
-cp tests/config/test_utils.lua "${TMP_RUNDIR}/"
-KRESD_NO_LISTEN=1 ${DEBUGGER} ${1} -f 1 -c test.cfg "${TMP_RUNDIR}"
+echo "# $(basename ${TEST_FILE})"
+${DEBUGGER} ${1} -f 1 -c ${SOURCE_PATH}/test.cfg "${TMP_RUNDIR}"
\ No newline at end of file
diff --git a/tests/config/tapered b/tests/config/tapered
new file mode 160000
index 000000000..be84b64d1
--- /dev/null
+++ b/tests/config/tapered
@@ -0,0 +1 @@
+Subproject commit be84b64d18293a29ca0acdf0431b0345084afe33
diff --git a/tests/config/test.cfg b/tests/config/test.cfg
new file mode 100644
index 000000000..a578aaca5
--- /dev/null
+++ b/tests/config/test.cfg
@@ -0,0 +1,32 @@
+package.path = package.path .. ';' .. env.SOURCE_PATH .. '/?.lua'
+TEST_DIR = env.TEST_FILE:match('(.*/)')
+
+-- optional code coverage
+local ok, runner = pcall(require, 'luacov.runner')
+if ok then
+	runner.init({
+		savestepsize = 2,
+		statsfile = TEST_DIR .. '/luacov.stats.out',
+		exclude = {'test', 'tapered'},
+	})
+	jit.off()
+end
+
+-- export testing module in globals
+local tapered = require('tapered.src.tapered')
+for k, v in pairs(tapered) do
+	_G[k] = v
+end
+
+-- load test
+local tests = dofile(env.TEST_FILE) or {}
+
+-- run test after processed config file
+-- default config will be used and we can test it.
+local runtest = require('test_utils').test
+event.after(0, function ()
+	for _, t in ipairs(tests) do
+		runtest(t)
+	end
+	done()
+end)
diff --git a/tests/config/test_config.mk b/tests/config/test_config.mk
index 25ea28d98..65f9d3a7c 100644
--- a/tests/config/test_config.mk
+++ b/tests/config/test_config.mk
@@ -6,17 +6,18 @@
 # Check return code of kresd. Passed test have to call quit().
 
 tests_config := \
-	basic \
-	hints \
-	predict
+	$(wildcard modules/*/*_test.lua) \
+	$(wildcard tests/config/*_test.lua)
 
 define make_config_test
-test-config-$(1): tests/config/$(1)/test.cfg check-install-precond
-	@$(preload_syms) ./tests/config/runtest.sh $(abspath $(SBINDIR)/kresd) $(1)
-.PHONY: test-$(1)
+$(1): check-install-precond
+	@$(preload_syms) ./tests/config/runtest.sh $(abspath $(SBINDIR)/kresd) $(abspath $(1))
+$(1)-clean:
+	@$(RM) $(dir $(1))/luacov.stats.out
+.PHONY: $(1)
 endef
 
 $(foreach test,$(tests_config),$(eval $(call make_config_test,$(test))))
-check-config: $(foreach test,$(tests_config),test-config-$(test))
-
+check-config: $(tests_config)
+check-config-clean: $(foreach test,$(tests_config),$(test)-clean)
 .PHONY: check-config
diff --git a/tests/config/test_utils.lua b/tests/config/test_utils.lua
index 33e81edbd..3c2c63271 100644
--- a/tests/config/test_utils.lua
+++ b/tests/config/test_utils.lua
@@ -1,18 +1,16 @@
-function fail(fmt, ...)
-	io.stderr:write(string.format(fmt..'\n', ...))
-	os.exit(2)
-end
+local M = {}
 
-function test(f, ...)
+function M.test(f, ...)
 	local res, exception = pcall(f, ...)
 	if not res then
 		local trace = debug.getinfo(2)
-		fail('%s:%d %s', trace.source, trace.currentline, exception)
+		io.stderr:write(string.format('%s:%d %s\n', trace.source, trace.currentline, exception))
+		os.exit(2)
 	end
 	return res
 end
 
-function table_keys_to_lower(table)
+function M.table_keys_to_lower(table)
 	local res = {}
 	for k, v in pairs(table) do
 		res[k:lower()] = v
@@ -20,32 +18,24 @@ function table_keys_to_lower(table)
 	return res
 end
 
-function contains(table, value)
+local function contains(pass, fail, table, value, message)
+	message = message or string.format('table contains "%s"', value)
 	for _, v in pairs(table) do
 		if v == value then
-			return true
+			pass(message)
+			return
 		end
 	end
-	return false
+	fail(message)
+	return
+end
+
+function M.contains(table, value, message)
+	return contains(pass, fail, table, value, message)
+end
+
+function M.not_contains(table, value, message)
+	return contains(fail, pass, table, value, message)
 end
 
--- Emulate busted testing interface
-local assert_builtin = assert
-assert = setmetatable({}, {
-	__call = function (_, ...)
-		return assert_builtin(...)
-	end,
-	__index = {
-		truthy = function (expr)
-			assert_builtin(expr)
-		end,
-		falsy = function (expr)
-			assert_builtin(not expr)
-		end,
-		same = function (a, b)
-			if a ~= b then
-				assert_builtin(false, string.format('expected: %s got: %s', a, b))
-			end
-		end,
-	}
-})
\ No newline at end of file
+return M
\ No newline at end of file
diff --git a/tests/tests.mk b/tests/tests.mk
index ff079f040..e8aa1ff7e 100644
--- a/tests/tests.mk
+++ b/tests/tests.mk
@@ -48,6 +48,6 @@ tests: check-unit
 # installcheck requires kresd to be installed in its final destination
 # (DESTDIR is not supported right now because module path gets hardcoded)
 installcheck: check-config
-tests-clean: $(foreach test,$(tests_BIN),$(test)-clean) mock_cmodule-clean $(CLEAN_DNSTAP)
+tests-clean: $(foreach test,$(tests_BIN),$(test)-clean) mock_cmodule-clean $(CLEAN_DNSTAP) check-config-clean
 
 .PHONY: check-integration deckard installcheck tests tests-clean