REGTEST: replace LEVEL option by a more human readable one.
This patch replaces LEVEL variable by REGTESTS_TYPES variable which is more
mnemonic and human readable. It is uses as a filter to run the reg tests scripts
where a commented #REGTEST_TYPE may be defined to designate their types.
Running the following command:
$ REGTESTS_TYPES=slow,default
will start all the reg tests where REGTEST_TYPE is defines as 'slow' or 'default'.
Note that 'default' is also the default value of REGTEST_TYPE when not specified
dedicated to run all the current h*.vtc files. When REGTESTS_TYPES is not specified
there is no filter at all. All the tests are run.
This patches also defines REGTEST_TYPE with 'slow' value for all the s*.vtc files,
'bug' value for al the b*.vtc files, 'broken' value for all the k*.vtc files.
diff --git a/Makefile b/Makefile
index dd5274f..f598e26 100644
--- a/Makefile
+++ b/Makefile
@@ -166,6 +166,7 @@
#### May be used to force running a specific set of reg-tests
REG_TEST_FILES =
+REG_TEST_SCRIPT=./scripts/run-regtests.sh
#### Compiler-specific flags that may be used to disable some negative over-
# optimization or to silence some warnings. -fno-strict-aliasing is needed with
@@ -967,17 +968,19 @@
# Target to run the regression testing script files.
reg-tests:
- $(Q)./scripts/run-regtests.sh --LEVEL "$(LEVEL)" $(REGTEST_ARGS) $(REG_TEST_FILES)
+ $(Q)$(REG_TEST_SCRIPT) --type "$(REGTESTS_TYPES)" $(REGTEST_ARGS) $(REG_TEST_FILES)
.PHONY: $(REGTEST_ARGS)
reg-tests-help:
@echo
- @echo "To launch the reg tests for haproxy, first export to your environment VTEST_PROGRAM variable to point to your vtest program:"
+ @echo "To launch the reg tests for haproxy, first export to your environment "
+ @echo "VTEST_PROGRAM variable to point to your vtest program:"
@echo " $$ export VTEST_PROGRAM=/opt/local/bin/vtest"
@echo "or"
@echo " $$ setenv VTEST_PROGRAM /opt/local/bin/vtest"
@echo
- @echo "The same thing may be done to set your haproxy program with HAPROXY_PROGRAM but with ./haproxy as default value."
+ @echo "The same thing may be done to set your haproxy program with HAPROXY_PROGRAM "
+ @echo "but with ./haproxy as default value."
@echo
@echo "To run all the tests:"
@echo " $$ make reg-tests"
@@ -985,16 +988,23 @@
@echo "You can also set the programs to be used on the command line:"
@echo " $$ VTEST_PROGRAM=<...> HAPROXY_PROGRAM=<...> make reg-tests"
@echo
- @echo "To run tests with specific levels:"
- @echo " $$ LEVEL=1,3,4 make reg-tests #list of levels"
- @echo " $$ LEVEL=1-3,5-6 make reg-tests #list of range of levels"
+ @echo "To run tests with specific types:"
+ @echo " $$ REGTESTS_TYPES=slow,default make reg-tests"
+ @echo
+ @echo "with 'any' as default value for REGTESTS_TYPES variable."
+ @echo
+ @echo "About the reg test types:"
+ @echo " any : all the tests without distinction (this is the default"
+ @echo " value of REGTESTS_TYPES."
+ @echo " default : dedicated to pure haproxy compliance tests."
+ @echo " slow : scripts which take non negligible time to run."
+ @echo " bug : scripts in relation with bugs they help to reproduce."
+ @echo " broken : scripts triggering known broken behaviors for which"
+ @echo " there is still no fix."
+ @echo " experimental: for scripts which are experimental, typically used to"
+ @echo " develop new scripts."
@echo
- @echo "About the levels:"
- @echo " LEVEL 1 scripts are dedicated to pure haproxy compliance tests (prefixed with 'h' letter)."
- @echo " LEVEL 2 scripts are slow scripts (prefixed with 's' letter)."
- @echo " LEVEL 3 scripts are low interest scripts (prefixed with 'l' letter)."
- @echo " LEVEL 4 scripts are in relation with bugs they help to reproduce (prefixed with 'b' letter)."
- @echo " LEVEL 5 scripts are scripts triggering known broken behaviors for which there is still no fix (prefixed with 'k' letter)."
- @echo " LEVEL 6 scripts are experimental, typically used to develop new scripts (prefixed with 'e' lettre)."
+ @echo "Note that 'reg-tests' target run '"$(REG_TEST_SCRIPT)"' script"
+ @echo "(see --help option of this script for more information)."
.PHONY: reg-tests reg-tests-help
diff --git a/reg-tests/checks/s00000.vtc b/reg-tests/checks/s00000.vtc
index 93575d3..a9b3a81 100644
--- a/reg-tests/checks/s00000.vtc
+++ b/reg-tests/checks/s00000.vtc
@@ -3,6 +3,7 @@
#REQUIRE_VERSION=1.8
#EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=slow
# This script test health-checks for four backends with one server by backend.
# A syslog server is attached to each backend to check the syslog messages
diff --git a/reg-tests/checks/s00001.vtc b/reg-tests/checks/s00001.vtc
index 2e38867..b37e868 100644
--- a/reg-tests/checks/s00001.vtc
+++ b/reg-tests/checks/s00001.vtc
@@ -8,6 +8,7 @@
#REQUIRE_VERSION=1.8
#EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=slow
# Note that the first syslog message received is: "Proxy <name> started."
syslog S -repeat 21 -level notice {
diff --git a/reg-tests/checks/s00002.vtc b/reg-tests/checks/s00002.vtc
index 2449ba7..84fd6b1 100644
--- a/reg-tests/checks/s00002.vtc
+++ b/reg-tests/checks/s00002.vtc
@@ -3,6 +3,7 @@
#REQUIRE_VERSION=1.8
#EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=slow
# This script start 40 servers named s0 up to s39.
# For 0 <= i <= 19:
diff --git a/reg-tests/checks/s00003.vtc b/reg-tests/checks/s00003.vtc
index 6ce10b0..ffd3123 100644
--- a/reg-tests/checks/s00003.vtc
+++ b/reg-tests/checks/s00003.vtc
@@ -1,6 +1,8 @@
varnishtest "Check: smptchk option"
feature ignore_unknown_macro
+#REGTEST_TYPE=slow
+
barrier b cond 3
syslog S1 -level notice {
diff --git a/reg-tests/compression/s00000.vtc b/reg-tests/compression/s00000.vtc
index 6baddea..c3be36e 100644
--- a/reg-tests/compression/s00000.vtc
+++ b/reg-tests/compression/s00000.vtc
@@ -3,6 +3,7 @@
varnishtest "Compression validation"
#REQUIRE_VERSION=1.6
#REQUIRE_OPTIONS=ZLIB|SLZ,LUA
+#REGTEST_TYPE=slow
feature ignore_unknown_macro
diff --git a/reg-tests/connection/b00000.vtc b/reg-tests/connection/b00000.vtc
index b5b0629..ad4f010 100644
--- a/reg-tests/connection/b00000.vtc
+++ b/reg-tests/connection/b00000.vtc
@@ -13,6 +13,8 @@
feature ignore_unknown_macro
+#REGTEST_TYPE=bug
+
syslog Slog_1 -repeat 8 -level info {
recv
expect ~ "Connect from .* to ${h1_ssl_addr}:${h1_ssl_port}"
diff --git a/reg-tests/http-rules/b00000.vtc b/reg-tests/http-rules/b00000.vtc
index 897c3b4..8fd39f7 100644
--- a/reg-tests/http-rules/b00000.vtc
+++ b/reg-tests/http-rules/b00000.vtc
@@ -11,6 +11,8 @@
feature ignore_unknown_macro
#REQUIRE_VERSION=1.6
+#REGTEST_TYPE=bug
+
syslog S1 -level notice {
recv
expect ~ "[^:\\[ ]\\[${h1_pid}\\]: Proxy (fe|be)1 started."
diff --git a/reg-tests/log/b00000.vtc b/reg-tests/log/b00000.vtc
index bd4ba0f..228c3fd 100644
--- a/reg-tests/log/b00000.vtc
+++ b/reg-tests/log/b00000.vtc
@@ -18,6 +18,8 @@
# Note that the problem doesn't happen when using %[src] or %[dst] since
# all sample expressions set LW_XPRT.
+#REGTEST_TYPE=bug
+
varnishtest "Wrong ip/port logging"
feature ignore_unknown_macro
diff --git a/reg-tests/lua/b00000.vtc b/reg-tests/lua/b00000.vtc
index 577329e..65d4cbb 100644
--- a/reg-tests/lua/b00000.vtc
+++ b/reg-tests/lua/b00000.vtc
@@ -32,6 +32,7 @@
varnishtest "Basic LUA test h00000"
#REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
feature ignore_unknown_macro
diff --git a/reg-tests/lua/b00001.vtc b/reg-tests/lua/b00001.vtc
index 588a5e3..4276133 100644
--- a/reg-tests/lua/b00001.vtc
+++ b/reg-tests/lua/b00001.vtc
@@ -21,6 +21,7 @@
varnishtest "LUA bug"
#REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
feature ignore_unknown_macro
diff --git a/reg-tests/lua/b00002.vtc b/reg-tests/lua/b00002.vtc
index e1fe7f3..ecb9955 100644
--- a/reg-tests/lua/b00002.vtc
+++ b/reg-tests/lua/b00002.vtc
@@ -1,5 +1,6 @@
varnishtest "Lua: txn:get_priv() scope"
#REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
feature ignore_unknown_macro
diff --git a/reg-tests/lua/b00003.vtc b/reg-tests/lua/b00003.vtc
index 3fd70eb..26e71ad 100644
--- a/reg-tests/lua/b00003.vtc
+++ b/reg-tests/lua/b00003.vtc
@@ -15,6 +15,7 @@
varnishtest "possible CLOSE-WAIT with '\n' headers"
#REQUIRE_OPTIONS=LUA
+#REGTEST_TYPE=bug
feature ignore_unknown_macro
diff --git a/reg-tests/mailers/k_healthcheckmail.vtc b/reg-tests/mailers/k_healthcheckmail.vtc
index d3af358..ed8c2f1 100644
--- a/reg-tests/mailers/k_healthcheckmail.vtc
+++ b/reg-tests/mailers/k_healthcheckmail.vtc
@@ -1,6 +1,7 @@
varnishtest "Lua: txn:get_priv() scope"
#REQUIRE_OPTIONS=LUA
#REQUIRE_VERSION=1.6
+#REGTEST_TYPE=broken
feature ignore_unknown_macro
diff --git a/reg-tests/peers/s_basic_sync.vtc b/reg-tests/peers/s_basic_sync.vtc
index ecf8fd0..3dddafa 100644
--- a/reg-tests/peers/s_basic_sync.vtc
+++ b/reg-tests/peers/s_basic_sync.vtc
@@ -2,6 +2,7 @@
feature ignore_unknown_macro
#REQUIRE_VERSION=2.0
+#REGTEST_TYPE=slow
haproxy h1 -arg "-L A" -conf {
defaults
diff --git a/reg-tests/peers/s_tls_basic_sync.vtc b/reg-tests/peers/s_tls_basic_sync.vtc
index 41e874b..d69b01d 100644
--- a/reg-tests/peers/s_tls_basic_sync.vtc
+++ b/reg-tests/peers/s_tls_basic_sync.vtc
@@ -3,6 +3,8 @@
#REQUIRE_VERSION=2.0
feature ignore_unknown_macro
+#REGTEST_TYPE=slow
+
haproxy h1 -arg "-L A" -conf {
defaults
timeout client 1s
diff --git a/reg-tests/seamless-reload/b00000.vtc b/reg-tests/seamless-reload/b00000.vtc
index f533860..e29312c 100644
--- a/reg-tests/seamless-reload/b00000.vtc
+++ b/reg-tests/seamless-reload/b00000.vtc
@@ -15,6 +15,7 @@
#REQUIRE_VERSION=1.8
# abns@ sockets are not available on freebsd
#EXCLUDE_TARGETS=freebsd
+#REGTEST_TYPE=bug
haproxy h1 -W -conf {
global
diff --git a/reg-tests/server/b00000.vtc b/reg-tests/server/b00000.vtc
index 831a63d..cf6b809 100644
--- a/reg-tests/server/b00000.vtc
+++ b/reg-tests/server/b00000.vtc
@@ -2,6 +2,8 @@
feature ignore_unknown_macro
+#REGTEST_TYPE=bug
+
# Do nothing. Is there only to create s1_* macros
server s1 {
} -start
diff --git a/reg-tests/spoe/b00000.vtc b/reg-tests/spoe/b00000.vtc
index 76b41d6..ce4e16f 100644
--- a/reg-tests/spoe/b00000.vtc
+++ b/reg-tests/spoe/b00000.vtc
@@ -5,6 +5,8 @@
# segmentaion fault when first errors are catched, when the "filter spoe" line is
# parsed.
+#REGTEST_TYPE=bug
+
varnishtest "SPOE bug: missing configuration file"
feature ignore_unknown_macro
diff --git a/reg-tests/ssl/b00000.vtc b/reg-tests/ssl/b00000.vtc
index 9171c45..9f0bdd7 100644
--- a/reg-tests/ssl/b00000.vtc
+++ b/reg-tests/ssl/b00000.vtc
@@ -15,6 +15,7 @@
# #define SSL_set_app_data(s,arg) (SSL_set_ex_data(s,0,(char *)arg))
# #define SSL_get_app_data(s) (SSL_get_ex_data(s,0))
+#REGTEST_TYPE=bug
varnishtest "OpenSSL bug: Random crashes"
#REQUIRE_OPTIONS=OPENSSL
diff --git a/reg-tests/stick-table/b00000.vtc b/reg-tests/stick-table/b00000.vtc
index 5885193..ca84c0b 100644
--- a/reg-tests/stick-table/b00000.vtc
+++ b/reg-tests/stick-table/b00000.vtc
@@ -1,5 +1,7 @@
# Shipped with the commit fixing the bug.
+#REGTEST_TYPE=bug
+
varnishtest "Stick Table: Crash when accessing unknown key."
feature ignore_unknown_macro
diff --git a/reg-tests/stick-table/b00001.vtc b/reg-tests/stick-table/b00001.vtc
index 3435712..d4fb862 100644
--- a/reg-tests/stick-table/b00001.vtc
+++ b/reg-tests/stick-table/b00001.vtc
@@ -14,6 +14,8 @@
#
# This should be backported to 1.8
+#REGTEST_TYPE=bug
+
varnishtest "stick-tables: Test expirations when used with table_*"
# As some macros for haproxy are used in this file, this line is mandatory.
diff --git a/scripts/run-regtests.sh b/scripts/run-regtests.sh
index 84a6c1c..27f7380 100755
--- a/scripts/run-regtests.sh
+++ b/scripts/run-regtests.sh
@@ -28,6 +28,13 @@
--vtestparams <ARGS>, passes custom ARGS to vtest
run-regtests.sh --vtestparams "-n 10"
+ --type <reg tests types> filter the types of the tests to be run, depending on
+ the commented REGTESTS_TYPE variable value in each VTC file.
+ The value of REGTESTS_TYPE supported are: default, slow, bug, broken and
+ experimental. When not specified, it is set to 'default' as default value.
+
+ run-regtest.sh --type slow,default
+
--clean to cleanup previous reg-tests log directories and exit
run-regtests.sh --clean
@@ -105,39 +112,6 @@
echo $list
}
-
-build_test_list()
-{
- # Remove any spacing character
- LEVEL="$(echo $LEVEL | tr -d ' ')"
- # Replave any comma character by a space character
- LEVEL="$(echo $LEVEL | tr ',' ' ')"
- list=
- for range in $LEVEL ; do
- if [ -z "$list" ] ; then
- list=$(add_range_to_test_list $range)
- else
- list="$list $(add_range_to_test_list $range)"
- fi
- done
-
- echo $list
-}
-
-build_find_expr()
-{
- expr=
- for i in $@; do
- if [ -z "$expr" ] ; then
- expr="-name \"$i\""
- else
- expr="$expr -o -name \"$i\""
- fi
- done
-
- echo $expr
-}
-
_startswith() {
_str="$1"
_sub="$2"
@@ -146,20 +120,25 @@
_findtests() {
set -f
- LEVEL=${LEVEL:-0};
- list=$(build_test_list "$LEVEL")
- if [ -z "$list" ] ; then
- echo "Invalid level specification '"$LEVEL"' or no file was found."
- exit 1
- fi
- EXPR=$(build_find_expr $list)
- for i in $( find "$1" $(eval echo $EXPR) ); do
+ REGTESTS_TYPES="${REGTESTS_TYPES:-any}"
+ any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
+ for i in $( find "$1" -name *.vtc ); do
skiptest=
require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")"
require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")"
require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed -e 's/,/ /g')"
exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed -e 's/,/ /g')"
+ if [ $any_test -ne 1 ] ; then
+ regtest_type="$(sed -ne 's/^#REGTEST_TYPE=//p' "$i")"
+ if [ -z $regtest_type ] ; then
+ regtest_type=default
+ fi
+ if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
+ echo " Skip $i because its type '"$regtest_type"' is excluded"
+ skiptest=1
+ fi
+ fi
requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed -e 's/,.*//')"
if [ -n "$requiredoption" ]; then
@@ -264,10 +243,10 @@
--keep-logs)
keep_logs="-L"
;;
- --LEVEL)
- LEVEL="$2"
- shift
- ;;
+ --type)
+ REGTESTS_TYPES="$2"
+ shift
+ ;;
--use-htx)
no_htx=""
;;