REGTEST: replace LEVEL option by a more human readable one.

This patch replaces LEVEL variable by REGTESTS_TYPES variable which is more
mnemonic and human readable. It is uses as a filter to run the reg tests scripts
where a commented #REGTEST_TYPE may be defined to designate their types.
Running the following command:

    $ REGTESTS_TYPES=slow,default

will start all the reg tests where REGTEST_TYPE is defines as 'slow' or 'default'.
Note that 'default' is also the default value of REGTEST_TYPE when not specified
dedicated to run all the current h*.vtc files. When REGTESTS_TYPES is not specified
there is no filter at all. All the tests are run.

This patches also defines REGTEST_TYPE with 'slow' value for all the s*.vtc files,
'bug' value for al the b*.vtc files, 'broken' value for all the k*.vtc files.

diff --git a/Makefile b/Makefile
index dd5274fa939bad01116f5a5610b5cda178001020..f598e26c8076d9e4b447f61d0a6f9e5f5b50669a 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -166,6 +166,7 @@ ERR =
 
 #### May be used to force running a specific set of reg-tests
 REG_TEST_FILES =
+REG_TEST_SCRIPT=./scripts/run-regtests.sh
 
 #### Compiler-specific flags that may be used to disable some negative over-
 # optimization or to silence some warnings. -fno-strict-aliasing is needed with
@@ -967,17 +968,19 @@ endif
 
 # Target to run the regression testing script files.
 reg-tests:
-       $(Q)./scripts/run-regtests.sh --LEVEL "$(LEVEL)" $(REGTEST_ARGS) $(REG_TEST_FILES)
+       $(Q)$(REG_TEST_SCRIPT) --type "$(REGTESTS_TYPES)" $(REGTEST_ARGS) $(REG_TEST_FILES)
 .PHONY: $(REGTEST_ARGS)
 
 reg-tests-help:
        @echo
-       @echo "To launch the reg tests for haproxy, first export to your environment VTEST_PROGRAM variable to point to your vtest program:"
+       @echo "To launch the reg tests for haproxy, first export to your environment "
+       @echo "VTEST_PROGRAM variable to point to your vtest program:"
        @echo "    $$ export VTEST_PROGRAM=/opt/local/bin/vtest"
        @echo "or"
        @echo "    $$ setenv VTEST_PROGRAM /opt/local/bin/vtest"
        @echo
-       @echo "The same thing may be done to set your haproxy program with HAPROXY_PROGRAM but with ./haproxy as default value."
+       @echo "The same thing may be done to set your haproxy program with HAPROXY_PROGRAM "
+       @echo "but with ./haproxy as default value."
        @echo
        @echo "To run all the tests:"
        @echo "    $$ make reg-tests"
@@ -985,16 +988,23 @@ reg-tests-help:
        @echo "You can also set the programs to be used on the command line:"
        @echo "    $$ VTEST_PROGRAM=<...> HAPROXY_PROGRAM=<...> make reg-tests"
        @echo
-       @echo "To run tests with specific levels:"
-       @echo "    $$ LEVEL=1,3,4   make reg-tests  #list of levels"
-       @echo "    $$ LEVEL=1-3,5-6 make reg-tests  #list of range of levels"
+       @echo "To run tests with specific types:"
+       @echo "    $$ REGTESTS_TYPES=slow,default make reg-tests"
        @echo
-       @echo "About the levels:"
-       @echo "    LEVEL 1 scripts are dedicated to pure haproxy compliance tests (prefixed with 'h' letter)."
-       @echo "    LEVEL 2 scripts are slow scripts (prefixed with 's' letter)."
-       @echo "    LEVEL 3 scripts are low interest scripts (prefixed with 'l' letter)."
-       @echo "    LEVEL 4 scripts are in relation with bugs they help to reproduce (prefixed with 'b' letter)."
-       @echo "    LEVEL 5 scripts are scripts triggering known broken behaviors for which there is still no fix (prefixed with 'k' letter)."
-       @echo "    LEVEL 6 scripts are experimental, typically used to develop new scripts (prefixed with 'e' lettre)."
+       @echo "with 'any' as default value for REGTESTS_TYPES variable."
+       @echo
+       @echo "About the reg test types:"
+       @echo "    any         : all the tests without distinction (this is the default"
+       @echo "                  value of REGTESTS_TYPES."
+       @echo "    default     : dedicated to pure haproxy compliance tests."
+       @echo "    slow        : scripts which take non negligible time to run."
+       @echo "    bug         : scripts in relation with bugs they help to reproduce."
+       @echo "    broken      : scripts triggering known broken behaviors for which"
+       @echo "                  there is still no fix."
+       @echo "    experimental: for scripts which are experimental, typically used to"
+       @echo "                  develop new scripts."
+       @echo
+       @echo "Note that 'reg-tests' target run '"$(REG_TEST_SCRIPT)"' script"
+       @echo "(see --help option of this script for more information)."
 
 .PHONY: reg-tests reg-tests-help

diff --git a/reg-tests/checks/s00000.vtc b/reg-tests/checks/s00000.vtc
index 93575d347a1a7e516218b00822a79ef9d75f24c1..a9b3a81d35c348ebc1ca5d5ed209cd2fc08ab217 100644 (file)
--- a/reg-tests/checks/s00000.vtc
+++ b/reg-tests/checks/s00000.vtc
@@ -3,6 +3,7 @@ feature ignore_unknown_macro
 
 #REQUIRE_VERSION=1.8
 #EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=slow
 
 # This script test health-checks for four backends with one server by backend.
 # A syslog server is attached to each backend to check the syslog messages

diff --git a/reg-tests/checks/s00001.vtc b/reg-tests/checks/s00001.vtc
index 2e38867c8de9674f8f60bfd3c8824dbc6ed1af23..b37e86836b61bf89d9ebdac998c05ab0aba3ab7f 100644 (file)
--- a/reg-tests/checks/s00001.vtc
+++ b/reg-tests/checks/s00001.vtc
@@ -8,6 +8,7 @@ feature ignore_unknown_macro
 
 #REQUIRE_VERSION=1.8
 #EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=slow
 
 # Note that the first syslog message received is: "Proxy <name> started."
 syslog S -repeat 21 -level notice {

diff --git a/reg-tests/checks/s00002.vtc b/reg-tests/checks/s00002.vtc
index 2449ba752c53e82875c495cbfc9429954dd3a502..84fd6b1700fa360b12b900aaf057aedb49939c90 100644 (file)
--- a/reg-tests/checks/s00002.vtc
+++ b/reg-tests/checks/s00002.vtc
@@ -3,6 +3,7 @@ feature ignore_unknown_macro
 
 #REQUIRE_VERSION=1.8
 #EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=slow
 
 # This script start 40 servers named s0 up to s39.
 # For 0 <= i <= 19:

diff --git a/reg-tests/checks/s00003.vtc b/reg-tests/checks/s00003.vtc
index 6ce10b0e2e3697adeb4074ce7f777aaf3a6a8da5..ffd31235e64db8b4b9c30b5734f831de8c148602 100644 (file)
--- a/reg-tests/checks/s00003.vtc
+++ b/reg-tests/checks/s00003.vtc
@@ -1,6 +1,8 @@
 varnishtest "Check: smptchk option"
 feature ignore_unknown_macro
 
+#REGTEST_TYPE=slow
+
 barrier b cond 3
 
 syslog S1 -level notice {

diff --git a/reg-tests/compression/s00000.vtc b/reg-tests/compression/s00000.vtc
index 6baddeae120f114fce3552d31c796e29867d8315..c3be36e6679aa5c1360d3776b798daaadc41ba0e 100644 (file)
--- a/reg-tests/compression/s00000.vtc
+++ b/reg-tests/compression/s00000.vtc
@@ -3,6 +3,7 @@
 varnishtest "Compression validation"
 #REQUIRE_VERSION=1.6
 #REQUIRE_OPTIONS=ZLIB|SLZ,LUA
+#REGTEST_TYPE=slow
 
 feature ignore_unknown_macro
 

diff --git a/reg-tests/connection/b00000.vtc b/reg-tests/connection/b00000.vtc
index b5b06297bbf75109a0db396120c57645f2fa6874..ad4f010e23bfcc889a67614e5e70f5ad6a9fcd44 100644 (file)
--- a/reg-tests/connection/b00000.vtc
+++ b/reg-tests/connection/b00000.vtc
@@ -13,6 +13,8 @@ varnishtest "PROXY protocol random failures"
 
 feature ignore_unknown_macro
 
+#REGTEST_TYPE=bug
+
 syslog Slog_1 -repeat 8 -level info {
     recv
     expect ~ "Connect from .* to ${h1_ssl_addr}:${h1_ssl_port}"

diff --git a/reg-tests/http-rules/b00000.vtc b/reg-tests/http-rules/b00000.vtc
index 897c3b48b2addf58ce907c6eab027c59b71447bb..8fd39f766261d2c5302f676cbe0c97a32bcef10a 100644 (file)
--- a/reg-tests/http-rules/b00000.vtc
+++ b/reg-tests/http-rules/b00000.vtc
@@ -11,6 +11,8 @@ varnishtest "map_regm get_trash_chunk test"
 feature ignore_unknown_macro
 
 #REQUIRE_VERSION=1.6
+#REGTEST_TYPE=bug
+
 syslog S1 -level notice {
     recv
     expect ~ "[^:\\[ ]\\[${h1_pid}\\]: Proxy (fe|be)1 started."

diff --git a/reg-tests/log/b00000.vtc b/reg-tests/log/b00000.vtc
index bd4ba0fd3554a6aff7a40663cd5f464fcc53c4ca..228c3fdc20c566d867e1203a78e22fae072c0c27 100644 (file)
--- a/reg-tests/log/b00000.vtc
+++ b/reg-tests/log/b00000.vtc
@@ -18,6 +18,8 @@
 # Note that the problem doesn't happen when using %[src] or %[dst] since
 # all sample expressions set LW_XPRT.
 
+#REGTEST_TYPE=bug
+
 varnishtest "Wrong ip/port logging"
 feature ignore_unknown_macro
 

diff --git a/reg-tests/lua/b00000.vtc b/reg-tests/lua/b00000.vtc
index 577329eb3070b520135ca9369045ca555afa8028..65d4cbbb8b0990d52444391c54ef88e406e49124 100644 (file)
--- a/reg-tests/lua/b00000.vtc
+++ b/reg-tests/lua/b00000.vtc
@@ -32,6 +32,7 @@
 
 varnishtest "Basic LUA test h00000"
 #REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
 
 feature ignore_unknown_macro
 

diff --git a/reg-tests/lua/b00001.vtc b/reg-tests/lua/b00001.vtc
index 588a5e30ebfd495ee822747eba2a562573024652..427613362e24b937cfc1974db3aa544627711e4b 100644 (file)
--- a/reg-tests/lua/b00001.vtc
+++ b/reg-tests/lua/b00001.vtc
@@ -21,6 +21,7 @@
 
 varnishtest "LUA bug"
 #REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
 
 feature ignore_unknown_macro
 

diff --git a/reg-tests/lua/b00002.vtc b/reg-tests/lua/b00002.vtc
index e1fe7f33558b00465efbdf2fd751544c04e33c6d..ecb9955c0bf1daebe7fda3e743f144485170a435 100644 (file)
--- a/reg-tests/lua/b00002.vtc
+++ b/reg-tests/lua/b00002.vtc
@@ -1,5 +1,6 @@
 varnishtest "Lua: txn:get_priv() scope"
 #REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
 
 feature ignore_unknown_macro
 

diff --git a/reg-tests/lua/b00003.vtc b/reg-tests/lua/b00003.vtc
index 3fd70ebc646a2eb72c52a59dbf08e04cb59e411a..26e71ad441b9fbb3b58b090771108964ba78ea08 100644 (file)
--- a/reg-tests/lua/b00003.vtc
+++ b/reg-tests/lua/b00003.vtc
@@ -15,6 +15,7 @@
 
 varnishtest "possible CLOSE-WAIT with '\n' headers"
 #REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
 
 feature ignore_unknown_macro
 

diff --git a/reg-tests/mailers/k_healthcheckmail.vtc b/reg-tests/mailers/k_healthcheckmail.vtc
index d3af3589924f61cfa04d48e02126039225c7cf8f..ed8c2f15e5e44f2dcb9a83777bfff48d427b6128 100644 (file)
--- a/reg-tests/mailers/k_healthcheckmail.vtc
+++ b/reg-tests/mailers/k_healthcheckmail.vtc
@@ -1,6 +1,7 @@
 varnishtest "Lua: txn:get_priv() scope"
 #REQUIRE_OPTIONS=LUA
 #REQUIRE_VERSION=1.6
+#REGTEST_TYPE=broken
 
 feature ignore_unknown_macro
 

diff --git a/reg-tests/peers/s_basic_sync.vtc b/reg-tests/peers/s_basic_sync.vtc
index ecf8fd059551ab844a7530683e9351b0592bebaf..3dddafa20b226f00039aca1eb9f61a51bc7e4e69 100644 (file)
--- a/reg-tests/peers/s_basic_sync.vtc
+++ b/reg-tests/peers/s_basic_sync.vtc
@@ -2,6 +2,7 @@ vtest "Basic test for peers protocol"
 feature ignore_unknown_macro
 
 #REQUIRE_VERSION=2.0
+#REGTEST_TYPE=slow
 
 haproxy h1 -arg "-L A" -conf {
     defaults

diff --git a/reg-tests/peers/s_tls_basic_sync.vtc b/reg-tests/peers/s_tls_basic_sync.vtc
index 41e874b4d24b41c6c8834fe360b9d5c8549d2341..d69b01d0aaab269642b4d302c1dba53aa5da9a50 100644 (file)
--- a/reg-tests/peers/s_tls_basic_sync.vtc
+++ b/reg-tests/peers/s_tls_basic_sync.vtc
@@ -3,6 +3,8 @@ vtest "Basic test for peers protocol over SSL/TLS"
 #REQUIRE_VERSION=2.0
 feature ignore_unknown_macro
 
+#REGTEST_TYPE=slow
+
 haproxy h1 -arg "-L A" -conf {
     defaults
         timeout client  1s

diff --git a/reg-tests/seamless-reload/b00000.vtc b/reg-tests/seamless-reload/b00000.vtc
index f533860fa099281a0efa5bbb6fe01858e711643a..e29312ca99fcf062eae85cad4a174c16e5de3cf6 100644 (file)
--- a/reg-tests/seamless-reload/b00000.vtc
+++ b/reg-tests/seamless-reload/b00000.vtc
@@ -15,6 +15,7 @@ feature ignore_unknown_macro
 #REQUIRE_VERSION=1.8
 # abns@ sockets are not available on freebsd
 #EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=bug
 
 haproxy h1 -W -conf {
   global

diff --git a/reg-tests/server/b00000.vtc b/reg-tests/server/b00000.vtc
index 831a63d26d4816200c9636895863cf7f0af994b0..cf6b809f1a4ae9df36bad9acef234fb97b60c38c 100644 (file)
--- a/reg-tests/server/b00000.vtc
+++ b/reg-tests/server/b00000.vtc
@@ -2,6 +2,8 @@ varnishtest "Set server FQDN via CLI crash"
 
 feature ignore_unknown_macro
 
+#REGTEST_TYPE=bug
+
 # Do nothing. Is there only to create s1_* macros
 server s1 {
 } -start

diff --git a/reg-tests/spoe/b00000.vtc b/reg-tests/spoe/b00000.vtc
index 76b41d60bbf2f8c37ae87b2c9211c570328bbd4e..ce4e16fa67c65a4c2de9f42c22923b155aceb815 100644 (file)
--- a/reg-tests/spoe/b00000.vtc
+++ b/reg-tests/spoe/b00000.vtc
@@ -5,6 +5,8 @@
 # segmentaion fault when first errors are catched, when the "filter spoe" line is
 # parsed.
 
+#REGTEST_TYPE=bug
+
 varnishtest "SPOE bug: missing configuration file"
 
 feature ignore_unknown_macro

diff --git a/reg-tests/ssl/b00000.vtc b/reg-tests/ssl/b00000.vtc
index 9171c4554782a1c7767453a30155b59c98152649..9f0bdd78941547317ec9a74a6a2e78814ff1109b 100644 (file)
--- a/reg-tests/ssl/b00000.vtc
+++ b/reg-tests/ssl/b00000.vtc
@@ -15,6 +15,7 @@
 #    #define SSL_set_app_data(s,arg)     (SSL_set_ex_data(s,0,(char *)arg))
 #    #define SSL_get_app_data(s)      (SSL_get_ex_data(s,0))
 
+#REGTEST_TYPE=bug
 
 varnishtest "OpenSSL bug: Random crashes"
 #REQUIRE_OPTIONS=OPENSSL

diff --git a/reg-tests/stick-table/b00000.vtc b/reg-tests/stick-table/b00000.vtc
index 58851939d17c4b129ffa9d86a602ab7e00d12df2..ca84c0bef963c4457e3655fdb8ade2532e09bee1 100644 (file)
--- a/reg-tests/stick-table/b00000.vtc
+++ b/reg-tests/stick-table/b00000.vtc
@@ -1,5 +1,7 @@
 # Shipped with the commit fixing the bug.
 
+#REGTEST_TYPE=bug
+
 varnishtest "Stick Table: Crash when accessing unknown key."
 feature ignore_unknown_macro
 

diff --git a/reg-tests/stick-table/b00001.vtc b/reg-tests/stick-table/b00001.vtc
index 3435712ef275eb6bab6f1baaffa700eb63b939f2..d4fb8620d41240851bc99415f01f85af2f851530 100644 (file)
--- a/reg-tests/stick-table/b00001.vtc
+++ b/reg-tests/stick-table/b00001.vtc
@@ -14,6 +14,8 @@
 #
 # This should be backported to 1.8
 
+#REGTEST_TYPE=bug
+
 varnishtest "stick-tables: Test expirations when used with table_*"
 
 # As some macros for haproxy are used in this file, this line is mandatory.

diff --git a/scripts/run-regtests.sh b/scripts/run-regtests.sh
index 84a6c1c8ffb7efcae18c344246b12a304c1985b5..27f738056f6f77deb5a18fac48e79ad9782590a5 100755 (executable)
--- a/scripts/run-regtests.sh
+++ b/scripts/run-regtests.sh
@@ -28,6 +28,13 @@ _help()
     --vtestparams <ARGS>, passes custom ARGS to vtest
       run-regtests.sh --vtestparams "-n 10"
 
+    --type <reg tests types> filter the types of the tests to be run, depending on
+      the commented REGTESTS_TYPE variable value in each VTC file.
+      The value of REGTESTS_TYPE supported are: default, slow, bug, broken and
+      experimental. When not specified, it is set to 'default' as default value.
+
+      run-regtest.sh --type slow,default
+
     --clean to cleanup previous reg-tests log directories and exit
       run-regtests.sh --clean
 
@@ -105,39 +112,6 @@ add_range_to_test_list()
     echo $list
 }
 
-
-build_test_list()
-{
-    # Remove any spacing character
-    LEVEL="$(echo $LEVEL | tr -d ' ')"
-    # Replave any comma character by a space character
-    LEVEL="$(echo $LEVEL | tr ',' ' ')"
-    list=
-    for range in $LEVEL ; do
-        if [ -z "$list" ] ; then
-            list=$(add_range_to_test_list $range)
-        else
-            list="$list $(add_range_to_test_list $range)"
-        fi
-    done
-
-    echo $list
-}
-
-build_find_expr()
-{
-    expr=
-    for i in $@; do
-        if [ -z "$expr" ] ; then
-            expr="-name \"$i\""
-        else
-            expr="$expr -o -name \"$i\""
-        fi
-    done
-
-    echo $expr
-}
-
 _startswith() {
   _str="$1"
   _sub="$2"
@@ -146,20 +120,25 @@ _startswith() {
 
 _findtests() {
   set -f
-  LEVEL=${LEVEL:-0};
-  list=$(build_test_list "$LEVEL")
-  if [ -z "$list" ] ; then
-      echo "Invalid level specification '"$LEVEL"' or no file was found."
-      exit 1
-  fi
-  EXPR=$(build_find_expr $list)
 
-  for i in $( find "$1" $(eval echo $EXPR) ); do
+  REGTESTS_TYPES="${REGTESTS_TYPES:-any}"
+  any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
+  for i in $( find "$1" -name *.vtc ); do
     skiptest=
     require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")"
     require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")"
     require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed  -e 's/,/ /g')"
     exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed  -e 's/,/ /g')"
+    if [ $any_test -ne 1 ] ; then
+        regtest_type="$(sed -ne 's/^#REGTEST_TYPE=//p' "$i")"
+        if [ -z $regtest_type ] ; then
+            regtest_type=default
+        fi
+        if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
+            echo "  Skip $i because its type '"$regtest_type"' is excluded"
+            skiptest=1
+        fi
+    fi
 
     requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed  -e 's/,.*//')"
     if [ -n "$requiredoption" ]; then
@@ -264,10 +243,10 @@ _process() {
         --keep-logs)
           keep_logs="-L"
           ;;
-        --LEVEL)
-          LEVEL="$2"
-          shift
-          ;;
+        --type)
+             REGTESTS_TYPES="$2"
+             shift
+             ;;
         --use-htx)
           no_htx=""
           ;;