debuginfod test: fix groom/stale race condition

Additional tracing, and use of "% make check VERBOSE=1" in a .spec
file allowed tracking down of this intermittent problem.  The race was
between a SIGUSR1 or two to a debuginfod server (triggering two
traverse/scan phases), followed shortly by a SIGUSR2 (triggering a
groom).  If those signals were received too close together, the groom
phase could be stopped early, and the rm'd files not noticed.

New testsuite code adds metric polls after SIGUSR1 & SIGUSR2 to ensure
the respective processing phases are complete.  It also turns on "set -x"
tracing, so as to avoid pulling out quite as much hair next time.
"make check VERBOSE=1" is also important for spec files.

Signed-off-by: Frank Ch. Eigler <fche@redhat.com>

diff --git a/tests/ChangeLog b/tests/ChangeLog
index c02fb050aab9ea4e32bd5bb633a7769cac8dcf36..d8fa97fa892c5e3ea9a2064eaef626e661bec57e 100644 (file)
--- a/tests/ChangeLog
+++ b/tests/ChangeLog
@@ -1,3 +1,8 @@
+2021-06-16  Frank Ch. Eigler <fche@redhat.com>
+
+       * run-debuginfod-find.sh: Fix intermittent groom/stale failure,
+       due to SIGUSR1/SIGUSR2 races.  Trace more.
+
 2021-06-15  Frank Ch. Eigler <fche@redhat.com>
 
        * run-debuginfod-find.sh (err): Elaborate.  Use as the reliable

diff --git a/tests/run-debuginfod-find.sh b/tests/run-debuginfod-find.sh
index c9ee11b1f24aac86666de784e0308d18ace4207e..456dc2f812e2f94181d96468e2f2464819e2692f 100755 (executable)
--- a/tests/run-debuginfod-find.sh
+++ b/tests/run-debuginfod-find.sh
@@ -26,7 +26,7 @@ bsdtar --version | grep -q zstd && zstd=true || zstd=false
 echo "zstd=$zstd bsdtar=`bsdtar --version`"
 
 # for test case debugging, uncomment:
-#set -x
+set -x
 VERBOSE=-vvv
 
 DB=${PWD}/.debuginfod_tmp.sqlite
@@ -74,7 +74,6 @@ errfiles() {
 }
 
 
-
 # find an unused port number
 while true; do
     PORT1=`expr '(' $RANDOM % 1000 ')' + 9000`
@@ -317,6 +316,11 @@ fi
 
 cp -rvp ${abs_srcdir}/debuginfod-tars Z
 kill -USR1 $PID1
+# Wait till both files are in the index and scan/index fully finished
+wait_ready $PORT1 'thread_work_total{role="traverse"}' 4
+wait_ready $PORT1 'thread_work_pending{role="scan"}' 0
+wait_ready $PORT1 'thread_busy{role="scan"}' 0
+
 # All rpms need to be in the index, except the dummy permission-000 one
 rpms=$(find R -name \*rpm | grep -v nothing | wc -l)
 wait_ready $PORT1 'scanned_files_total{source=".rpm archive"}' $rpms
@@ -326,6 +330,11 @@ tb2=$(find Z -name \*tar.bz2 | wc -l)
 wait_ready $PORT1 'scanned_files_total{source=".tar.bz2 archive"}' $tb2
 
 kill -USR1 $PID1  # two hits of SIGUSR1 may be needed to resolve .debug->dwz->srefs
+# Wait till both files are in the index and scan/index fully finished
+wait_ready $PORT1 'thread_work_total{role="traverse"}' 5
+wait_ready $PORT1 'thread_work_pending{role="scan"}' 0
+wait_ready $PORT1 'thread_busy{role="scan"}' 0
+
 # Expect all source files found in the rpms (they are all called hello.c :)
 # We will need to extract all rpms (in their own directory) and could all
 # sources referenced in the .debug files.
@@ -495,6 +504,10 @@ if type bsdtar 2>/dev/null; then
     # copy in the deb files
     cp -rvp ${abs_srcdir}/debuginfod-debs/*deb D
     kill -USR1 $PID2
+    wait_ready $PORT2 'thread_work_total{role="traverse"}' 2
+    wait_ready $PORT2 'thread_work_pending{role="scan"}' 0
+    wait_ready $PORT2 'thread_busy{role="scan"}' 0
+
     # All debs need to be in the index
     debs=$(find D -name \*.deb | wc -l)
     wait_ready $PORT2 'scanned_files_total{source=".deb archive"}' `expr $debs`