tests: Collect code coverage separately from each component in vm

Use a more robust design for collecting the gcov logs from the case
where test cases are run within a virtual machine. This generates a
writable-from-vm build tree for each component separately so that the
lcov and gcov can easily find the matching source code and data files.
In addition, prepare the reports automatically at the end of the
vm-run.sh --codecov execution.

Signed-hostap: Jouni Malinen <j@w1.fi>

diff --git a/tests/hwsim/example-wpa_supplicant.config b/tests/hwsim/example-wpa_supplicant.config
index 67c9768d428f2ffba1e72f0dc4d56177b9f89fc8..2923e835db1fe1eb753f7a548dbabb5f89b0e594 100644 (file)
--- a/tests/hwsim/example-wpa_supplicant.config
+++ b/tests/hwsim/example-wpa_supplicant.config
@@ -111,7 +111,3 @@ CFLAGS += -DALL_DH_GROUPS
 CONFIG_WNM=y
 
 CONFIG_TESTING_OPTIONS=y
-
-# gcov code coverage from the virtual machine
-#CONFIG_CODE_COVERAGE=y
-#CFLAGS += -fprofile-dir=/tmp/logs/gcov

diff --git a/tests/hwsim/start.sh b/tests/hwsim/start.sh
index f2722ac87c213fe50fe82795a395ae3d7947d669..346ea2c08b23ac875bf84245b172c75042a74cc4 100755 (executable)
--- a/tests/hwsim/start.sh
+++ b/tests/hwsim/start.sh
@@ -4,6 +4,7 @@ DIR="$( cd "$( dirname "$0" )" && pwd )"
 WPAS=$DIR/../../wpa_supplicant/wpa_supplicant
 WPACLI=$DIR/../../wpa_supplicant/wpa_cli
 HAPD=$DIR/../../hostapd/hostapd
+HAPD_AS=$DIR/../../hostapd/hostapd
 WLANTEST=$DIR/../../wlantest/wlantest
 HLR_AUC_GW=$DIR/../../hostapd/hlr_auc_gw
 
@@ -13,6 +14,17 @@ if [ -z "$LOGDIR" ] ; then
     mkdir -p $LOGDIR
     rm -rf $DIR/logs/current
     ln -sf $DATE $DIR/logs/current
+else
+    if [ -e $LOGDIR/alt-wpa_supplicant/wpa_supplicant/wpa_supplicant ]; then
+       WPAS=$LOGDIR/alt-wpa_supplicant/wpa_supplicant/wpa_supplicant
+    fi
+    if [ -e $LOGDIR/alt-hostapd/hostapd/hostapd ]; then
+       HAPD=$LOGDIR/alt-hostapd/hostapd/hostapd
+    fi
+    if [ -e $LOGDIR/alt-hostapd-as/hostapd/hostapd ]; then
+       HAPD_AS=$LOGDIR/alt-hostapd-as/hostapd/hostapd
+       HLR_AUC_GW=$LOGDIR/alt-hostapd-as/hostapd/hlr_auc_gw
+    fi
 fi
 
 if groups | tr ' ' "\n" | grep -q ^admin$; then
@@ -65,7 +77,7 @@ if [ -x $HLR_AUC_GW ]; then
     $HLR_AUC_GW -m $DIR/auth_serv/hlr_auc_gw.milenage_db > $LOGDIR/hlr_auc_gw &
 fi
 
-$HAPD -ddKt $DIR/auth_serv/as.conf > $LOGDIR/auth_serv &
+$HAPD_AS -ddKt $DIR/auth_serv/as.conf > $LOGDIR/auth_serv &
 
 # wait for programs to be fully initialized
 for i in 0 1 2; do

diff --git a/tests/hwsim/vm/README b/tests/hwsim/vm/README
index ba124ef581fb76f65f7ed2bedc2509d65ac7119d..88cfc9f23b3752ca176119c0c41d51e1b06ee889 100644 (file)
--- a/tests/hwsim/vm/README
+++ b/tests/hwsim/vm/README
@@ -21,7 +21,20 @@ give it are passed through to run-all.sh, see there.
 
 --------------------------------------------------------------------------------
 
-Code Coverage Analysis
+Code Coverage Analysis for user space code
+
+Code coverage for wpa_supplicant and hostapd can be generated from the
+test run with following command line:
+
+./vm-run.sh --codecov [other arguments..]
+
+This builds a separate copies of wpa_supplicant and hostapd into a
+directory that is writable from the virtual machine to collect the gcov
+data. lcov is then used to prepare the reports at the end of the test
+run.
+
+
+Code Coverage Analysis for kernel code
 
 In order to do code coverage analysis, reconfigure the kernel to include
 

diff --git a/tests/hwsim/vm/vm-run.sh b/tests/hwsim/vm/vm-run.sh
index bf6933b2700ff7e0506b18c1b54ea8721338e30c..0f556decbdbf17007fb3aba4cbba37837f9eac47 100755 (executable)
--- a/tests/hwsim/vm/vm-run.sh
+++ b/tests/hwsim/vm/vm-run.sh
@@ -6,7 +6,7 @@ if [ -z "$TESTDIR" ] ; then
        TESTDIR=$(pwd)/../
 fi
 
-LOGS=/tmp/hwsim-test-logs/
+LOGS=/tmp/hwsim-test-logs
 
 # increase the memory size if you want to run with valgrind, 512 MB works
 MEMORY=128
@@ -40,9 +40,63 @@ fi
 
 
 CMD=$TESTDIR/vm/inside.sh
-LOGDIR=$LOGS/$(date +%s)
+DATE=$(date +%s)
+LOGDIR=$LOGS/$DATE
 mkdir -p $LOGDIR
 
+if [ "$1" = "--codecov" ]; then
+    shift
+    CODECOV=yes
+    DIR=$PWD
+    if [ -e /tmp/logs ]; then
+       echo "/tmp/logs exists - cannot prepare build trees"
+       exit 1
+    fi
+    mkdir /tmp/logs
+    echo "Preparing separate build trees for hostapd/wpa_supplicant"
+    cd ../../..
+    git archive --format=tar --prefix=hostap/ HEAD > /tmp/logs/hostap.tar
+    cd $DIR
+    cat ../../../wpa_supplicant/.config > /tmp/logs/wpa_supplicant.config
+    echo "CONFIG_CODE_COVERAGE=y" >> /tmp/logs/wpa_supplicant.config
+    cat ../../../hostapd/.config > /tmp/logs/hostapd.config
+    echo "CONFIG_CODE_COVERAGE=y" >> /tmp/logs/hostapd.config
+
+    cd /tmp/logs
+    tar xf hostap.tar
+    mv hostap alt-wpa_supplicant
+    mv wpa_supplicant.config alt-wpa_supplicant/wpa_supplicant/.config
+    tar xf hostap.tar
+    mv hostap alt-hostapd
+    cp hostapd.config alt-hostapd/hostapd/.config
+    tar xf hostap.tar
+    mv hostap alt-hostapd-as
+    mv hostapd.config alt-hostapd-as/hostapd/.config
+    rm hostap.tar
+
+    cd /tmp/logs/alt-wpa_supplicant/wpa_supplicant
+    echo "Building wpa_supplicant"
+    make -j8 > /dev/null
+
+    cd /tmp/logs/alt-hostapd/hostapd
+    echo "Building hostapd"
+    make -j8 hostapd > /dev/null
+
+    cd /tmp/logs/alt-hostapd-as/hostapd
+    echo "Building hostapd/hlr_auc_gw (AS)"
+    make -j8 hostapd hlr_auc_gw > /dev/null
+
+    cd $DIR
+
+    mv /tmp/logs/alt-wpa_supplicant $LOGDIR
+    mv /tmp/logs/alt-hostapd $LOGDIR
+    mv /tmp/logs/alt-hostapd-as $LOGDIR
+else
+    CODECOV=no
+fi
+
+echo "Starting test run in a virtual machine"
+
 kvm \
        -kernel $KERNEL -smp 4 \
        $KVMARGS -m $MEMORY -nographic \
@@ -53,12 +107,42 @@ kvm \
        -monitor null -serial stdio -serial file:$LOGDIR/console \
        -append "mac80211_hwsim.channels=$CHANNELS mac80211_hwsim.radios=5 init=$CMD testdir=$TESTDIR console=$KVMOUT root=/dev/root rootflags=trans=virtio,version=9p2000.u ro rootfstype=9p EPATH=$EPATH ARGS=$*"
 
-echo LOGDIR=$LOGDIR
+if [ $CODECOV = "yes" ]; then
+    mv $LOGDIR/alt-wpa_supplicant /tmp/logs
+    mv $LOGDIR/alt-hostapd /tmp/logs
+    mv $LOGDIR/alt-hostapd-as /tmp/logs
+
+    echo "Generating code coverage report for wpa_supplicant"
+    cd /tmp/logs/alt-wpa_supplicant/wpa_supplicant
+    lcov -c -d .. > lcov.info 2> lcov.log
+    genhtml -t "wpa_supplicant hwsim test run $DATE" lcov.info --output-directory $LOGDIR/lcov-wpa_supplicant >> lcov.log 2>&1
+    mv lcov.info lcov.log $LOGDIR/lcov-wpa_supplicant
+
+    echo "Generating code coverage report for hostapd"
+    cd /tmp/logs/alt-hostapd/hostapd
+    lcov -c -d .. > lcov.info 2> lcov.log
+    genhtml -t "hostapd hwsim test run $DATE" lcov.info --output-directory $LOGDIR/lcov-hostapd >> lcov.log 2>&1
+    mv lcov.info lcov.log $LOGDIR/lcov-hostapd
+
+    echo "Generating code coverage report for hostapd/hlr_auc_gw (AS)"
+    cd /tmp/logs/alt-hostapd-as/hostapd
+    lcov -c -d .. > lcov.info 2> lcov.log
+    genhtml -t "hostapd/hlr_auc_gw (AS) hwsim test run $DATE" lcov.info --output-directory $LOGDIR/lcov-hostapd-as >> lcov.log 2>&1
+    mv lcov.info lcov.log $LOGDIR/lcov-hostapd-as
+
+    cd $DIR
+    rm -r /tmp/logs/alt-wpa_supplicant
+    rm -r /tmp/logs/alt-hostapd
+    rm -r /tmp/logs/alt-hostapd-as
+    rmdir /tmp/logs
+fi
 
-if [ -d $LOGDIR/gcov ]; then
-    echo "Move gcov data files from vm logdir to build directories"
-    for i in $LOGDIR/gcov/*.gcda; do
-       file=`basename $i | sed "s/.gcda$//"`
-       find ../../.. -name $file.gcno | sed s/.gcno/.gcda/ | xargs mv $i
-    done
+echo
+echo "Test run completed"
+echo "Logfiles are at $LOGDIR"
+if [ $CODECOV = "yes" ]; then
+    echo "Code coverage reports:"
+    echo "wpa_supplicant: file://$LOGDIR/lcov-wpa_supplicant/index.html"
+    echo "hostapd: file://$LOGDIR/lcov-hostapd/index.html"
+    echo "hostapd/hlr_auc_gw (AS): file://$LOGDIR/lcov-hostapd-as/index.html"
 fi