CI coverage: parallelize Lua code coverage processing

Execution of scripts/luacov_to_info.lua can take quite a long time so it
is now done in paralell.

Beware: This needs proper depedencies on coverage-lua-fix-paths target
and so on to avoid race conditions.

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d66b1c2cc7f4e523f7f6b78667fdef33fd99f233..7ba99d20f65965e716830603b4d8188deb1ea802 100644 (file)
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -30,7 +30,7 @@ test:linux:amd64:
   stage: test
   script:
     - PREFIX=$(pwd)/.local make -k check
-    - PREFIX=$(pwd)/.local make coverage-c COVERAGE_STAGE=gcov-check
+    - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c COVERAGE_STAGE=gcov-check
   dependencies:
     - build:linux:amd64
   artifacts:
@@ -45,8 +45,8 @@ test:linux:amd64:
 installcheck:linux:amd64:
   stage: test
   script:
-    - PREFIX=$(pwd)/.local make -k installcheck
-    - PREFIX=$(pwd)/.local make coverage-c coverage-lua COVERAGE_STAGE=gcov-installcheck
+    - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc) --keep-going" make -k installcheck
+    - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-installcheck
   dependencies:
     - build:linux:amd64
   artifacts:
@@ -62,7 +62,7 @@ deckard:linux:amd64:
   stage: test
   script:
     - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc) --keep-going" make check-integration
-    - PREFIX=$(pwd)/.local make coverage-c coverage-lua COVERAGE_STAGE=gcov-deckard
+    - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-deckard
   dependencies:
     - build:linux:amd64
   artifacts:
@@ -134,7 +134,7 @@ respdiff:iter:udp:linux:amd64:
     - echo 'test if mismatch rate >= 1 %'
     - grep -q '^target diagrees.*0\.[0-9][0-9] %' results/respdiff.txt
     - killall --wait kresd
-    - PREFIX=$(pwd)/.local make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-udp
+    - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-udp
   dependencies:
     - build:linux:amd64
   artifacts:
@@ -158,7 +158,7 @@ respdiff:iter:tcp:linux:amd64:
     - echo 'test if mismatch rate >= 1 %'
     - grep -q '^target diagrees.*0\.[0-9][0-9] %' results/respdiff.txt
     - killall --wait kresd
-    - PREFIX=$(pwd)/.local make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tcp
+    - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tcp
   dependencies:
     - build:linux:amd64
   artifacts:
@@ -182,7 +182,7 @@ respdiff:iter:tls:linux:amd64:
     - echo 'test if mismatch rate >= 1 %'
     - grep -q '^target diagrees.*0\.[0-9][0-9] %' results/respdiff.txt
     - killall --wait kresd
-    - PREFIX=$(pwd)/.local make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tls
+    - PREFIX=$(pwd)/.local MAKEFLAGS="--jobs $(nproc)" make coverage-c coverage-lua COVERAGE_STAGE=gcov-respdiff-iter-tls
   dependencies:
     - build:linux:amd64
   artifacts:

diff --git a/coverage.mk b/coverage.mk
index 786c49c4107204993c2bf9a9cfe2913e580d54aa..f00f446084c00f197ca39e176d5e97f65781d929 100644 (file)
--- a/coverage.mk
+++ b/coverage.mk
@@ -8,33 +8,44 @@
 
 coverage-c-combine-gcda:
        @# combine trees of gcda files into one info file per tree
-       @mkdir -p "$(COVERAGE_STATSDIR)/tmp.c"
-       @LCOV=$(LCOV) ./scripts/coverage_c_combine.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)" "$(COVERAGE_STATSDIR)/tmp.c"
+       @mkdir -p '$(COVERAGE_STATSDIR)/tmp.c'
+       @LCOV=$(LCOV) ./scripts/coverage_c_combine.sh '$(TOPSRCDIR)' '$(COVERAGE_STATSDIR)' '$(COVERAGE_STATSDIR)/tmp.c'
 
 coverage-c: coverage-c-combine-gcda
        @# combine info files for each tree into resulting c.info file
-       @$(LCOV) -q $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STATSDIR)/tmp.c/*.info)) --output-file "$(COVERAGE_STAGE).c.info"
-       @$(RM) -r "$(COVERAGE_STATSDIR)/tmp.c"
+       @$(LCOV) -q $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STATSDIR)/tmp.c/*.info)) --output-file '$(COVERAGE_STAGE).c.info'
+       @$(RM) -r '$(COVERAGE_STATSDIR)/tmp.c'
 
-coverage-lua-fix-paths: $(shell find -type f -name 'luacov.stats.out')
+LUA_STATS_OUT := $(shell find '$(COVERAGE_STATSDIR)' -type f -name 'luacov.stats.out')
+LUA_INFOS_OUT := $(patsubst %.stats.out,%.lua.info,$(LUA_STATS_OUT))
+
+coverage-lua-fix-paths: $(LUA_STATS_OUT)
        @# map Lua install paths to source paths
        @$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua --sed > .luacov_path_map
        @sed -i -f .luacov_path_map $^
-       @rm .luacov_path_map
+       @$(RM) .luacov_path_map
 
-coverage-lua: coverage-lua-fix-paths
-       @# add missing Lua files to stats
+luacov.empty_stats.out:
+       @# generate list of all Lua files to fill holes in luacov stats
        @$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua | cut -f 2 | grep '\.lua$$' | scripts/luacov_gen_empty.sh > luacov.empty_stats.out
-       @echo "# Lua coverage in $(COVERAGE_STAGE).lua.info"
-       @scripts/luacov_to_info.lua luacov.empty_stats.out $(shell find -type f -name 'luacov.stats.out') > $(COVERAGE_STAGE).lua.info
-       @rm luacov.empty_stats.out
+
+%.lua.info: %.stats.out coverage-lua-fix-paths
+       @scripts/luacov_to_info.lua $*.stats.out > $@
+
+coverage-lua: $(LUA_INFOS_OUT) luacov.empty_stats.out
+       @echo '# Lua coverage in $(COVERAGE_STAGE).lua.info'
+       @# add missing files to luacov stats
+       @scripts/luacov_to_info.lua luacov.empty_stats.out > luacov.empty_stats.lua.info
+       @# combine info files for each tree into resulting lua.info file
+       @$(LCOV) -q $(addprefix --add-tracefile ,$(LUA_INFOS_OUT)) --add-tracefile luacov.empty_stats.lua.info --output-file '$(COVERAGE_STAGE).lua.info'
+       @$(RM) luacov.empty_stats.out luacov.empty_stats.lua.info
 
 coverage:
        @$(LCOV) $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STAGE)*.info)) --output-file coverage.info
-       @$(GENHTML) --no-function-coverage --no-branch-coverage -q -o coverage -p $(realpath $(CURDIR)) -t "Knot DNS Resolver $(VERSION)-$(PLATFORM) coverage report" --legend coverage.info
+       @$(GENHTML) --no-function-coverage --no-branch-coverage -q -o coverage -p '$(realpath $(CURDIR))' -t 'Knot DNS Resolver $(VERSION)-$(PLATFORM) coverage report' --legend coverage.info
 
 coverage-clean:
-       @rm -rf "$(COVERAGE_STATSDIR)"
+       @$(RM) -rf '$(COVERAGE_STATSDIR)'
 
 .PHONY: coverage-c-combine-gcda coverage-c coverage-lua-fix-paths coverage-lua coverage coverage-clean