CI coverage: merge test coverage data from parallel runs

We run tests in paralell so have to make sure that coverage tools
do not overwrite results from each run.

This is hacky because lcov tool insists on having gcno and gcda files
in the same place as original source, so we have to copy files
to workaround this.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c60c3a6b612aa1daec32572858c2d5c7a1e8e8d7..d66b1c2cc7f4e523f7f6b78667fdef33fd99f233 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -127,6 +127,7 @@ test:linux:amd64:valgrind:
 respdiff:iter:udp:linux:amd64:
   stage: test
   script:
+    - source <(./scripts/coverage_env.sh "$(pwd)" "$(pwd)/coverage.stats/respdiff" "iter/udp" --export)
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh udp
     - cat results/respdiff.txt
@@ -150,6 +151,7 @@ respdiff:iter:udp:linux:amd64:
 respdiff:iter:tcp:linux:amd64:
   stage: test
   script:
+    - source <(./scripts/coverage_env.sh "$(pwd)" "$(pwd)/coverage.stats/respdiff" "iter/tcp" --export)
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh tcp
     - cat results/respdiff.txt
@@ -173,6 +175,7 @@ respdiff:iter:tcp:linux:amd64:
 respdiff:iter:tls:linux:amd64:
   stage: test
   script:
+    - source <(./scripts/coverage_env.sh "$(pwd)" "$(pwd)/coverage.stats/respdiff" "iter/tls" --export)
     - PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
     - ./ci/respdiff/run-respdiff-tests.sh tls
     - cat results/respdiff.txt

diff --git a/Makefile b/Makefile
index 897a1740734cf24afd1af7251a913eca0a4b77b0..7812b21d217888437ee377764e9ad9f3cdf4d4dc 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -6,33 +6,12 @@ all: info lib daemon client modules etc
 install: lib-install daemon-install client-install modules-install etc-install
 check: all tests
 clean: contrib-clean lib-clean daemon-clean client-clean modules-clean \
-       tests-clean doc-clean bench-clean
+       tests-clean doc-clean bench-clean coverage-clean
 doc: doc-html
 lint: $(patsubst %.lua.in,%.lua,$(wildcard */*/*.lua.in))
        luacheck --codes --formatter TAP .
-coverage-c:
-       @echo "# C coverage in $(COVERAGE_STAGE).c.info"
-       @$(LCOV) --no-external --capture -d lib -d daemon -d modules -o $(COVERAGE_STAGE).c.info > /dev/null
-coverage-lua: $(shell find -type f -name 'luacov.stats.out')
-       @# map install paths to source paths
-       @$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua --sed > .luacov_path_map
-       @find -type f -name 'luacov.stats.out' | xargs sed -i -f .luacov_path_map
-       @rm .luacov_path_map
-       @# add missing Lua files
-       @$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua | cut -f 2 | grep '\.lua$$' | scripts/luacov_gen_empty.sh > luacov.empty_stats.out
-       @echo "# Lua coverage in $(COVERAGE_STAGE).lua.info"
-       @scripts/luacov_to_info.lua luacov.empty_stats.out $^ > $(COVERAGE_STAGE).lua.info
-       @rm luacov.empty_stats.out
-coverage:
-       @$(LCOV) $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STAGE)*.info)) --output-file coverage.info
-       @$(GENHTML) --no-function-coverage --no-branch-coverage -q -o coverage -p $(realpath $(CURDIR)) -t "Knot DNS Resolver $(VERSION)-$(PLATFORM) coverage report" --legend coverage.info
-
-.PHONY: all install check clean doc info coverage
-
-# Options
-ifdef COVERAGE
-BUILD_CFLAGS += --coverage
-endif
+
+.PHONY: all install check clean doc info lint
 
 # Dependencies
 KNOT_MINVER := 2.4.0
@@ -194,6 +173,7 @@ $(DESTDIR)$(ETCDIR):
 
 # Sub-targets
 include contrib/contrib.mk
+include coverage.mk
 include lib/lib.mk
 include client/client.mk
 include daemon/daemon.mk

diff --git a/ci/respdiff/kresd.config b/ci/respdiff/kresd.config
index b804aad2a91de13a5ea64925ad168cc306a1af4e..e7827b00c240ffe93ceeebb7acd2d8793ddc3e3a 100644 (file)
--- a/ci/respdiff/kresd.config
+++ b/ci/respdiff/kresd.config
@@ -1,6 +1,3 @@
--- measure code coverage: kresd must run from $GIT_DIR
-require('tests.config.coverage')
-
 -- Refer to manual: https://knot-resolver.readthedocs.io/en/latest/daemon.html#configuration
 -- Listen on localhost and external interface
 net.listen('127.0.0.1', 5353)

diff --git a/config.mk b/config.mk
index 3bdb65f182393dbf684d3374c8dc8cc6bcae77a6..fcb62a37b8dbd2ad80e0116a0578efd5c458b1ae 100644 (file)
--- a/config.mk
+++ b/config.mk
@@ -21,6 +21,8 @@ MODULEDIR ?= $(LIBDIR)/kdns_modules
 ETCDIR ?= $(PREFIX)/etc/kresd
 ROOTHINTS ?= $(ETCDIR)/root.hints
 COVERAGE_STAGE ?= gcov
+COVERAGE_STATSDIR ?= $(CURDIR)/coverage.stats
+TOPSRCDIR := $(CURDIR)
 
 # Tools
 CC      ?= cc

diff --git a/coverage.mk b/coverage.mk
new file mode 100644 (file)
index 0000000..786c49c
--- /dev/null
+++ b/coverage.mk
@@ -0,0 +1,44 @@
+# Measure code coverage using luacov and gcov
+# C and Lua code is measured separately and resutls are combined together
+# Define COVERAGE=1 during build *and* test runs to enable measurement.
+#
+# Beware: Tests are typically run in parallel and neither luacov not gcov
+# support that, so we have to store results from each run separatelly
+# and combine them.
+
+coverage-c-combine-gcda:
+       @# combine trees of gcda files into one info file per tree
+       @mkdir -p "$(COVERAGE_STATSDIR)/tmp.c"
+       @LCOV=$(LCOV) ./scripts/coverage_c_combine.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)" "$(COVERAGE_STATSDIR)/tmp.c"
+
+coverage-c: coverage-c-combine-gcda
+       @# combine info files for each tree into resulting c.info file
+       @$(LCOV) -q $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STATSDIR)/tmp.c/*.info)) --output-file "$(COVERAGE_STAGE).c.info"
+       @$(RM) -r "$(COVERAGE_STATSDIR)/tmp.c"
+
+coverage-lua-fix-paths: $(shell find -type f -name 'luacov.stats.out')
+       @# map Lua install paths to source paths
+       @$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua --sed > .luacov_path_map
+       @sed -i -f .luacov_path_map $^
+       @rm .luacov_path_map
+
+coverage-lua: coverage-lua-fix-paths
+       @# add missing Lua files to stats
+       @$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua | cut -f 2 | grep '\.lua$$' | scripts/luacov_gen_empty.sh > luacov.empty_stats.out
+       @echo "# Lua coverage in $(COVERAGE_STAGE).lua.info"
+       @scripts/luacov_to_info.lua luacov.empty_stats.out $(shell find -type f -name 'luacov.stats.out') > $(COVERAGE_STAGE).lua.info
+       @rm luacov.empty_stats.out
+
+coverage:
+       @$(LCOV) $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STAGE)*.info)) --output-file coverage.info
+       @$(GENHTML) --no-function-coverage --no-branch-coverage -q -o coverage -p $(realpath $(CURDIR)) -t "Knot DNS Resolver $(VERSION)-$(PLATFORM) coverage report" --legend coverage.info
+
+coverage-clean:
+       @rm -rf "$(COVERAGE_STATSDIR)"
+
+.PHONY: coverage-c-combine-gcda coverage-c coverage-lua-fix-paths coverage-lua coverage coverage-clean
+
+# Options
+ifdef COVERAGE
+BUILD_CFLAGS += --coverage
+endif

diff --git a/scripts/coverage_c_combine.sh b/scripts/coverage_c_combine.sh
new file mode 100755 (executable)
index 0000000..23006c1
--- /dev/null
+++ b/scripts/coverage_c_combine.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+# $1 = top source directory
+# $2 = coverage data directory path
+# $3 = output directory for *.info files
+
+set -o errexit -o nounset
+shopt -s nullglob
+IFS=$'\n'
+
+TOPSRCDIR="$1"
+DATAROOT="$2"
+OUTDIR="$3"
+
+cd "${TOPSRCDIR}"
+for COVNAME in $(find "${DATAROOT}" -name .topdir_kresd_coverage)
+do
+       find "${DATAROOT}" -name '*.gcda' -not -path "${DATAROOT}/*" -delete
+       COVDIR="$(dirname "${COVNAME}")"
+       COVDATA_FILENAMES=("${COVDIR}"/*)  # filenames in BASH array
+       (( ${#COVDATA_FILENAMES[*]} )) || continue  # skip empty dirs
+
+       cp -r -t ${TOPSRCDIR} "${COVDIR}"/*
+       ${LCOV} -q --no-external --capture -d lib -d daemon -d modules -o "$(mktemp -p "${OUTDIR}" -t XXXXXXXX.c.info)" > /dev/null
+done

diff --git a/scripts/coverage_env.sh b/scripts/coverage_env.sh
new file mode 100755 (executable)
index 0000000..bb85649
--- /dev/null
+++ b/scripts/coverage_env.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+# generate variables for coverage testing
+# $1 = top source directory
+# $2 = coverage data directory path
+# $3 = name of test/new subdirectory name
+# $4 = [optional] --export to generate export commands
+
+set -o errexit -o nounset
+shopt -s nullglob
+
+test -z "${COVERAGE:-}" && exit 0  # not enabled, do nothing
+test ! -z "${V:-}" && set -o xtrace  # verbose mode
+
+EXPORT=""
+test "${4:-}" == "--export" && EXPORT="export "
+TOPSRCDIR="$1"
+DATAROOT="$2"
+OUTPATH="$2/$3"
+
+# check that output directory is empty
+# beware: Makefile will always call coverage_env.sh for all targets
+# so directories get created but not populated
+# i.e. test -d is not sufficient check
+OUTPATH_FILENAMES=("${OUTPATH}"/*)  # filenames in BASH array
+(( ${#OUTPATH_FILENAMES[*]} )) && echo "false" && >&2 echo "fatal: output directory ${OUTPATH} must be empty (or non-existent)" && exit 1
+
+mkdir -p "${OUTPATH}"
+# convert paths to absolute
+pushd "${OUTPATH}" &> /dev/null
+touch .topdir_kresd_coverage
+OUTPATH="$(pwd -P)"
+popd &> /dev/null
+
+# determine GCOV_PREFIX_STRIP value for current source directory
+TOPSRCDIR_SLASHES="${TOPSRCDIR//[^\/]/}" # remove everything except /
+GCOV_PREFIX_STRIP="${#TOPSRCDIR_SLASHES}" # numer of / == number of components
+
+KRESD_COVERAGE_STATS="${OUTPATH}/luacov.stats.out"
+GCOV_PREFIX="${OUTPATH}"
+echo "${EXPORT}KRESD_COVERAGE_STATS=\"${KRESD_COVERAGE_STATS}\" ${EXPORT}GCOV_PREFIX=\"${GCOV_PREFIX}\" ${EXPORT}GCOV_PREFIX_STRIP=\"${GCOV_PREFIX_STRIP}\""

diff --git a/tests/config/coverage.lua b/tests/config/coverage.lua
deleted file mode 100644 (file)
index 5af55d2..0000000
--- a/tests/config/coverage.lua
+++ /dev/null
@@ -1,12 +0,0 @@
--- optional code coverage
--- include this file into config if you want to generate coverage data
-
-local ok, runner = pcall(require, 'luacov.runner')
-if ok then
-       runner.init({
-               savestepsize = 2, -- TODO
-               statsfile = 'luacov.stats.out',
-               exclude = {'test', 'tapered'},
-       })
-       jit.off()
-end

diff --git a/tests/config/test.cfg b/tests/config/test.cfg
index cc8b0ff9b510a63b7c0e18250da69ae6fa5c8690..743ab6baed141a3ef91272f3fa71c7513ec8bfff 100644 (file)
--- a/tests/config/test.cfg
+++ b/tests/config/test.cfg
@@ -1,8 +1,6 @@
 package.path = package.path .. ';' .. env.SOURCE_PATH .. '/?.lua'
 TEST_DIR = env.TEST_FILE:match('(.*/)')
 
-require('coverage')
-
 -- export testing module in globals
 local tapered = require('tapered.src.tapered')
 for k, v in pairs(tapered) do

diff --git a/tests/config/test_config.mk b/tests/config/test_config.mk
index 65f9d3a7c7191eca7d12776d6ace3ffef29d0452..28a34bfb481d8dc07da58fedc86dd842b8c4f4b1 100644 (file)
--- a/tests/config/test_config.mk
+++ b/tests/config/test_config.mk
@@ -11,13 +11,10 @@ tests_config := \
 
 define make_config_test
 $(1): check-install-precond
-       @$(preload_syms) ./tests/config/runtest.sh $(abspath $(SBINDIR)/kresd) $(abspath $(1))
-$(1)-clean:
-       @$(RM) $(dir $(1))/luacov.stats.out
+       @$(shell ./scripts/coverage_env.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)/tests_config" "$(1)") $(preload_syms) ./tests/config/runtest.sh $(abspath $(SBINDIR)/kresd) $(abspath $(1))
 .PHONY: $(1)
 endef
 
 $(foreach test,$(tests_config),$(eval $(call make_config_test,$(test))))
 check-config: $(tests_config)
-check-config-clean: $(foreach test,$(tests_config),$(test)-clean)
 .PHONY: check-config

diff --git a/tests/deckard b/tests/deckard
index 8e533979fe667fdc0bf6975d643d9e98c37f3b41..387e8845a45a7bab6ab1c8e5be55a8d88d0315ae 160000 (submodule)
--- a/tests/deckard
+++ b/tests/deckard
@@ -1 +1 @@
-Subproject commit 8e533979fe667fdc0bf6975d643d9e98c37f3b41
+Subproject commit 387e8845a45a7bab6ab1c8e5be55a8d88d0315ae

diff --git a/tests/tests.mk b/tests/tests.mk
index e8aa1ff7e3c5d6a9d8f62485974ee186405d3cb1..6c88d2da13c80be848307c8d27ce3030f3473955 100644 (file)
--- a/tests/tests.mk
+++ b/tests/tests.mk
@@ -39,7 +39,7 @@ check-install-precond:
 check-integration: check-install-precond $(deckard_DIR)/Makefile
        $(if $(SUBMODULES_DIRTY), $(warning Warning: Git submodules are not up-to-date),)
        @mkdir -p $(deckard_DIR)/contrib/libswrap/obj
-       +TESTS=$(TESTS) DAEMON=$(abspath $(SBINDIR)/kresd) TEMPLATE=$(TEMPLATE) $(preload_syms) $(deckard_DIR)/kresd_run.sh
+       +TESTS=$(TESTS) DAEMON=$(abspath $(SBINDIR)/kresd) TEMPLATE=$(TEMPLATE) COVERAGE_ENV_SCRIPT=$(TOPSRCDIR)/scripts/coverage_env.sh DAEMONSRCDIR=$(TOPSRCDIR) COVERAGE_STATSDIR=$(COVERAGE_STATSDIR)/deckard $(preload_syms) $(deckard_DIR)/kresd_run.sh
 
 deckard: check-integration
 

diff --git a/tests/unit.mk b/tests/unit.mk
index 9913e4614800c718746f12ba820150988ab9d7fe..84ff621c4a44c465981f5ec7b4f347834a6ee55b 100644 (file)
--- a/tests/unit.mk
+++ b/tests/unit.mk
@@ -30,7 +30,7 @@ $(1)_LIBS := $(tests_LIBS)
 $(1)_DEPEND := $(tests_DEPEND)
 $(call make_bin,$(1),tests)
 $(1): $$($(1))
-       @$(preload_syms) $(DEBUGGER) $$<
+       @$(shell ./scripts/coverage_env.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)/tests_unit" "$(1)") $(preload_syms) $(DEBUGGER) $$<
 .PHONY: $(1)
 endef