From: Petr Špaček <petr.spacek@nic.cz>
Date: Fri, 22 Dec 2017 12:21:24 +0000 (+0100)
Subject: CI coverage: merge test coverage data from parallel runs
X-Git-Tag: v2.0.0~49^2~2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=b2497b13e7d13d6430c53dc537183f47e0d6ded7;p=thirdparty%2Fknot-resolver.git

CI coverage: merge test coverage data from parallel runs

We run tests in paralell so have to make sure that coverage tools
do not overwrite results from each run.

This is hacky because lcov tool insists on having gcno and gcda files
in the same place as original source, so we have to copy files
to workaround this.
---

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c60c3a6b6..d66b1c2cc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -127,6 +127,7 @@ test:linux:amd64:valgrind:
 respdiff:iter:udp:linux:amd64:
   stage: test
   script:
+    - source <(./scripts/coverage_env.sh "$(pwd)" "$(pwd)/coverage.stats/respdiff" "iter/udp" --export)
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh udp
     - cat results/respdiff.txt
@@ -150,6 +151,7 @@ respdiff:iter:udp:linux:amd64:
 respdiff:iter:tcp:linux:amd64:
   stage: test
   script:
+    - source <(./scripts/coverage_env.sh "$(pwd)" "$(pwd)/coverage.stats/respdiff" "iter/tcp" --export)
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh tcp
     - cat results/respdiff.txt
@@ -173,6 +175,7 @@ respdiff:iter:tcp:linux:amd64:
 respdiff:iter:tls:linux:amd64:
   stage: test
   script:
+    - source <(./scripts/coverage_env.sh "$(pwd)" "$(pwd)/coverage.stats/respdiff" "iter/tls" --export)
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh tls
     - cat results/respdiff.txt
diff --git a/Makefile b/Makefile
index 897a17407..7812b21d2 100644
--- a/Makefile
+++ b/Makefile
@@ -6,33 +6,12 @@ all: info lib daemon client modules etc
 install: lib-install daemon-install client-install modules-install etc-install
 check: all tests
 clean: contrib-clean lib-clean daemon-clean client-clean modules-clean \
-	tests-clean doc-clean bench-clean
+	tests-clean doc-clean bench-clean coverage-clean
 doc: doc-html
 lint: $(patsubst %.lua.in,%.lua,$(wildcard */*/*.lua.in))
 	luacheck --codes --formatter TAP .
-coverage-c:
-	@echo "# C coverage in $(COVERAGE_STAGE).c.info"
-	@$(LCOV) --no-external --capture -d lib -d daemon -d modules -o $(COVERAGE_STAGE).c.info > /dev/null
-coverage-lua: $(shell find -type f -name 'luacov.stats.out')
-	@# map install paths to source paths
-	@$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua --sed > .luacov_path_map
-	@find -type f -name 'luacov.stats.out' | xargs sed -i -f .luacov_path_map
-	@rm .luacov_path_map
-	@# add missing Lua files
-	@$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua | cut -f 2 | grep '\.lua$$' | scripts/luacov_gen_empty.sh > luacov.empty_stats.out
-	@echo "# Lua coverage in $(COVERAGE_STAGE).lua.info"
-	@scripts/luacov_to_info.lua luacov.empty_stats.out $^ > $(COVERAGE_STAGE).lua.info
-	@rm luacov.empty_stats.out
-coverage:
-	@$(LCOV) $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STAGE)*.info)) --output-file coverage.info
-	@$(GENHTML) --no-function-coverage --no-branch-coverage -q -o coverage -p $(realpath $(CURDIR)) -t "Knot DNS Resolver $(VERSION)-$(PLATFORM) coverage report" --legend coverage.info
-
-.PHONY: all install check clean doc info coverage
-
-# Options
-ifdef COVERAGE
-BUILD_CFLAGS += --coverage
-endif
+
+.PHONY: all install check clean doc info lint
 
 # Dependencies
 KNOT_MINVER := 2.4.0
@@ -194,6 +173,7 @@ $(DESTDIR)$(ETCDIR):
 
 # Sub-targets
 include contrib/contrib.mk
+include coverage.mk
 include lib/lib.mk
 include client/client.mk
 include daemon/daemon.mk
diff --git a/ci/respdiff/kresd.config b/ci/respdiff/kresd.config
index b804aad2a..e7827b00c 100644
--- a/ci/respdiff/kresd.config
+++ b/ci/respdiff/kresd.config
@@ -1,6 +1,3 @@
--- measure code coverage: kresd must run from $GIT_DIR
-require('tests.config.coverage')
-
 -- Refer to manual: https://knot-resolver.readthedocs.io/en/latest/daemon.html#configuration
 -- Listen on localhost and external interface
 net.listen('127.0.0.1', 5353)
diff --git a/config.mk b/config.mk
index 3bdb65f18..fcb62a37b 100644
--- a/config.mk
+++ b/config.mk
@@ -21,6 +21,8 @@ MODULEDIR ?= $(LIBDIR)/kdns_modules
 ETCDIR ?= $(PREFIX)/etc/kresd
 ROOTHINTS ?= $(ETCDIR)/root.hints
 COVERAGE_STAGE ?= gcov
+COVERAGE_STATSDIR ?= $(CURDIR)/coverage.stats
+TOPSRCDIR := $(CURDIR)
 
 # Tools
 CC      ?= cc
diff --git a/coverage.mk b/coverage.mk
new file mode 100644
index 000000000..786c49c41
--- /dev/null
+++ b/coverage.mk
@@ -0,0 +1,44 @@
+# Measure code coverage using luacov and gcov
+# C and Lua code is measured separately and resutls are combined together
+# Define COVERAGE=1 during build *and* test runs to enable measurement.
+#
+# Beware: Tests are typically run in parallel and neither luacov not gcov
+# support that, so we have to store results from each run separatelly
+# and combine them.
+
+coverage-c-combine-gcda:
+	@# combine trees of gcda files into one info file per tree
+	@mkdir -p "$(COVERAGE_STATSDIR)/tmp.c"
+	@LCOV=$(LCOV) ./scripts/coverage_c_combine.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)" "$(COVERAGE_STATSDIR)/tmp.c"
+
+coverage-c: coverage-c-combine-gcda
+	@# combine info files for each tree into resulting c.info file
+	@$(LCOV) -q $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STATSDIR)/tmp.c/*.info)) --output-file "$(COVERAGE_STAGE).c.info"
+	@$(RM) -r "$(COVERAGE_STATSDIR)/tmp.c"
+
+coverage-lua-fix-paths: $(shell find -type f -name 'luacov.stats.out')
+	@# map Lua install paths to source paths
+	@$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua --sed > .luacov_path_map
+	@sed -i -f .luacov_path_map $^
+	@rm .luacov_path_map
+
+coverage-lua: coverage-lua-fix-paths
+	@# add missing Lua files to stats
+	@$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua | cut -f 2 | grep '\.lua$$' | scripts/luacov_gen_empty.sh > luacov.empty_stats.out
+	@echo "# Lua coverage in $(COVERAGE_STAGE).lua.info"
+	@scripts/luacov_to_info.lua luacov.empty_stats.out $(shell find -type f -name 'luacov.stats.out') > $(COVERAGE_STAGE).lua.info
+	@rm luacov.empty_stats.out
+
+coverage:
+	@$(LCOV) $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STAGE)*.info)) --output-file coverage.info
+	@$(GENHTML) --no-function-coverage --no-branch-coverage -q -o coverage -p $(realpath $(CURDIR)) -t "Knot DNS Resolver $(VERSION)-$(PLATFORM) coverage report" --legend coverage.info
+
+coverage-clean:
+	@rm -rf "$(COVERAGE_STATSDIR)"
+
+.PHONY: coverage-c-combine-gcda coverage-c coverage-lua-fix-paths coverage-lua coverage coverage-clean
+
+# Options
+ifdef COVERAGE
+BUILD_CFLAGS += --coverage
+endif
diff --git a/scripts/coverage_c_combine.sh b/scripts/coverage_c_combine.sh
new file mode 100755
index 000000000..23006c188
--- /dev/null
+++ b/scripts/coverage_c_combine.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+# $1 = top source directory
+# $2 = coverage data directory path
+# $3 = output directory for *.info files
+
+set -o errexit -o nounset
+shopt -s nullglob
+IFS=$'\n'
+
+TOPSRCDIR="$1"
+DATAROOT="$2"
+OUTDIR="$3"
+
+cd "${TOPSRCDIR}"
+for COVNAME in $(find "${DATAROOT}" -name .topdir_kresd_coverage)
+do
+	find "${DATAROOT}" -name '*.gcda' -not -path "${DATAROOT}/*" -delete
+	COVDIR="$(dirname "${COVNAME}")"
+	COVDATA_FILENAMES=("${COVDIR}"/*)  # filenames in BASH array
+	(( ${#COVDATA_FILENAMES[*]} )) || continue  # skip empty dirs
+
+	cp -r -t ${TOPSRCDIR} "${COVDIR}"/*
+	${LCOV} -q --no-external --capture -d lib -d daemon -d modules -o "$(mktemp -p "${OUTDIR}" -t XXXXXXXX.c.info)" > /dev/null
+done
diff --git a/scripts/coverage_env.sh b/scripts/coverage_env.sh
new file mode 100755
index 000000000..bb856496d
--- /dev/null
+++ b/scripts/coverage_env.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# generate variables for coverage testing
+# $1 = top source directory
+# $2 = coverage data directory path
+# $3 = name of test/new subdirectory name
+# $4 = [optional] --export to generate export commands
+
+set -o errexit -o nounset
+shopt -s nullglob
+
+test -z "${COVERAGE:-}" && exit 0  # not enabled, do nothing
+test ! -z "${V:-}" && set -o xtrace  # verbose mode
+
+EXPORT=""
+test "${4:-}" == "--export" && EXPORT="export "
+TOPSRCDIR="$1"
+DATAROOT="$2"
+OUTPATH="$2/$3"
+
+# check that output directory is empty
+# beware: Makefile will always call coverage_env.sh for all targets
+# so directories get created but not populated
+# i.e. test -d is not sufficient check
+OUTPATH_FILENAMES=("${OUTPATH}"/*)  # filenames in BASH array
+(( ${#OUTPATH_FILENAMES[*]} )) && echo "false" && >&2 echo "fatal: output directory ${OUTPATH} must be empty (or non-existent)" && exit 1
+
+mkdir -p "${OUTPATH}"
+# convert paths to absolute
+pushd "${OUTPATH}" &> /dev/null
+touch .topdir_kresd_coverage
+OUTPATH="$(pwd -P)"
+popd &> /dev/null
+
+# determine GCOV_PREFIX_STRIP value for current source directory
+TOPSRCDIR_SLASHES="${TOPSRCDIR//[^\/]/}" # remove everything except /
+GCOV_PREFIX_STRIP="${#TOPSRCDIR_SLASHES}" # numer of / == number of components
+
+KRESD_COVERAGE_STATS="${OUTPATH}/luacov.stats.out"
+GCOV_PREFIX="${OUTPATH}"
+echo "${EXPORT}KRESD_COVERAGE_STATS=\"${KRESD_COVERAGE_STATS}\" ${EXPORT}GCOV_PREFIX=\"${GCOV_PREFIX}\" ${EXPORT}GCOV_PREFIX_STRIP=\"${GCOV_PREFIX_STRIP}\""
diff --git a/tests/config/coverage.lua b/tests/config/coverage.lua
deleted file mode 100644
index 5af55d23a..000000000
--- a/tests/config/coverage.lua
+++ /dev/null
@@ -1,12 +0,0 @@
--- optional code coverage
--- include this file into config if you want to generate coverage data
-
-local ok, runner = pcall(require, 'luacov.runner')
-if ok then
-	runner.init({
-		savestepsize = 2, -- TODO
-		statsfile = 'luacov.stats.out',
-		exclude = {'test', 'tapered'},
-	})
-	jit.off()
-end
diff --git a/tests/config/test.cfg b/tests/config/test.cfg
index cc8b0ff9b..743ab6bae 100644
--- a/tests/config/test.cfg
+++ b/tests/config/test.cfg
@@ -1,8 +1,6 @@
 package.path = package.path .. ';' .. env.SOURCE_PATH .. '/?.lua'
 TEST_DIR = env.TEST_FILE:match('(.*/)')
 
-require('coverage')
-
 -- export testing module in globals
 local tapered = require('tapered.src.tapered')
 for k, v in pairs(tapered) do
diff --git a/tests/config/test_config.mk b/tests/config/test_config.mk
index 65f9d3a7c..28a34bfb4 100644
--- a/tests/config/test_config.mk
+++ b/tests/config/test_config.mk
@@ -11,13 +11,10 @@ tests_config := \
 
 define make_config_test
 $(1): check-install-precond
-	@$(preload_syms) ./tests/config/runtest.sh $(abspath $(SBINDIR)/kresd) $(abspath $(1))
-$(1)-clean:
-	@$(RM) $(dir $(1))/luacov.stats.out
+	@$(shell ./scripts/coverage_env.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)/tests_config" "$(1)") $(preload_syms) ./tests/config/runtest.sh $(abspath $(SBINDIR)/kresd) $(abspath $(1))
 .PHONY: $(1)
 endef
 
 $(foreach test,$(tests_config),$(eval $(call make_config_test,$(test))))
 check-config: $(tests_config)
-check-config-clean: $(foreach test,$(tests_config),$(test)-clean)
 .PHONY: check-config
diff --git a/tests/deckard b/tests/deckard
index 8e533979f..387e8845a 160000
--- a/tests/deckard
+++ b/tests/deckard
@@ -1 +1 @@
-Subproject commit 8e533979fe667fdc0bf6975d643d9e98c37f3b41
+Subproject commit 387e8845a45a7bab6ab1c8e5be55a8d88d0315ae
diff --git a/tests/tests.mk b/tests/tests.mk
index e8aa1ff7e..6c88d2da1 100644
--- a/tests/tests.mk
+++ b/tests/tests.mk
@@ -39,7 +39,7 @@ check-install-precond:
 check-integration: check-install-precond $(deckard_DIR)/Makefile
 	$(if $(SUBMODULES_DIRTY), $(warning Warning: Git submodules are not up-to-date),)
 	@mkdir -p $(deckard_DIR)/contrib/libswrap/obj
-	+TESTS=$(TESTS) DAEMON=$(abspath $(SBINDIR)/kresd) TEMPLATE=$(TEMPLATE) $(preload_syms) $(deckard_DIR)/kresd_run.sh
+	+TESTS=$(TESTS) DAEMON=$(abspath $(SBINDIR)/kresd) TEMPLATE=$(TEMPLATE) COVERAGE_ENV_SCRIPT=$(TOPSRCDIR)/scripts/coverage_env.sh DAEMONSRCDIR=$(TOPSRCDIR) COVERAGE_STATSDIR=$(COVERAGE_STATSDIR)/deckard $(preload_syms) $(deckard_DIR)/kresd_run.sh
 
 deckard: check-integration
 
diff --git a/tests/unit.mk b/tests/unit.mk
index 9913e4614..84ff621c4 100644
--- a/tests/unit.mk
+++ b/tests/unit.mk
@@ -30,7 +30,7 @@ $(1)_LIBS := $(tests_LIBS)
 $(1)_DEPEND := $(tests_DEPEND)
 $(call make_bin,$(1),tests)
 $(1): $$($(1))
-	@$(preload_syms) $(DEBUGGER) $$<
+	@$(shell ./scripts/coverage_env.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)/tests_unit" "$(1)") $(preload_syms) $(DEBUGGER) $$<
 .PHONY: $(1)
 endef