blob: 27bb13cbf75b83b87988d6e60bf147ae6e3e337e [file] [log] [blame]
Frédéric Lécaille51e01b52018-11-29 21:41:42 +01001#!/bin/sh
PiBa-NL72504042018-11-27 22:26:38 +01002
Christopher Faulet8d67cf82018-12-18 22:41:20 +01003_help()
4{
PiBa-NL72504042018-11-27 22:26:38 +01005 cat << EOF
6### run-regtests.sh ###
7 Running run-regtests.sh --help shows this information about how to use it
8
9 Run without parameters to run all tests in the current folder (including subfolders)
10 run-regtests.sh
11
12 Provide paths to run tests from (including subfolders):
13 run-regtests.sh ./tests1 ./tests2
14
15 Parameters:
Frédéric Lécaille43361232019-01-11 10:10:21 +010016 --j <NUM>, To run vtest with multiple jobs / threads for a faster overall result
PiBa-NL72504042018-11-27 22:26:38 +010017 run-regtests.sh ./fasttest --j 16
18
19 --v, to run verbose
Frédéric Lécaille43361232019-01-11 10:10:21 +010020 run-regtests.sh --v, disables the default vtest 'quiet' parameter
PiBa-NL72504042018-11-27 22:26:38 +010021
Ilya Shipitsin856aabc2020-04-16 23:51:34 +050022 --debug to show test logs on standard output (implies --v)
Christopher Faulet2a7cf922018-12-19 10:22:01 +010023 run-regtests.sh --debug
24
Christopher Faulet9c6df5e2018-12-19 10:25:07 +010025 --keep-logs to keep all log directories (by default kept if test fails)
26 run-regtests.sh --keep-logs
27
Frédéric Lécaille43361232019-01-11 10:10:21 +010028 --vtestparams <ARGS>, passes custom ARGS to vtest
29 run-regtests.sh --vtestparams "-n 10"
PiBa-NL72504042018-11-27 22:26:38 +010030
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +010031 --type <reg tests types> filter the types of the tests to be run, depending on
32 the commented REGTESTS_TYPE variable value in each VTC file.
Christopher Faulet1eee6ca2019-12-11 16:26:11 +010033 The value of REGTESTS_TYPE supported are: default, slow, bug, broken, devel
34 and experimental. When not specified, it is set to 'default' as default value.
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +010035
36 run-regtest.sh --type slow,default
37
Christopher Faulet8d0fdf52018-12-19 10:18:40 +010038 --clean to cleanup previous reg-tests log directories and exit
39 run-regtests.sh --clean
40
Christopher Faulet78f31bf2019-04-12 16:53:41 +020041 --use-htx to use the HTX in tests (deprecated, the default mode now)
42
43 --no-htx to use the legacy HTTP in tests
44 run-regtests.sh --no-htx, sets the macro \${no-htx}
Christopher Fauletfa6798f2018-12-19 11:22:04 +010045 In .vtc files, in HAProxy configuration, you should use the following line
46 to "templatize" your tests:
47
48 \${no-htx} option http-use-htx
49
PiBa-NL72504042018-11-27 22:26:38 +010050 Including text below into a .vtc file will check for its requirements
51 related to haproxy's target and compilation options
Bertrand Jacquin7e30b502019-05-16 22:15:31 +010052 # Below targets are not capable of completing this test successfully
PiBa-NL72504042018-11-27 22:26:38 +010053 #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd
54
55 #EXCLUDE_TARGETS=dos,freebsd,windows
56
Bertrand Jacquin7e30b502019-05-16 22:15:31 +010057 # Below option is required to complete this test successfully
PiBa-NL72504042018-11-27 22:26:38 +010058 #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in.
Christopher Faulet7e245df2018-12-21 15:18:02 +010059 #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA
PiBa-NL72504042018-11-27 22:26:38 +010060
William Dauchyaabde712021-01-09 17:26:20 +010061 #REQUIRE_SERVICE=prometheus-exporter
62 #REQUIRE_SERVICES=prometheus-exporter,foo
63
PiBa-NL72504042018-11-27 22:26:38 +010064 # To define a range of versions that a test can run with:
65 #REQUIRE_VERSION=0.0
66 #REQUIRE_VERSION_BELOW=99.9
67
William Lallemandd5b464b2019-12-19 14:30:00 +010068 # To define required binaries for a test:
69 #REQUIRE_BINARIES=socat,curl
70
Frédéric Lécaille43361232019-01-11 10:10:21 +010071 Configure environment variables to set the haproxy and vtest binaries to use
PiBa-NL72504042018-11-27 22:26:38 +010072 setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy
Frédéric Lécaille43361232019-01-11 10:10:21 +010073 setenv VTEST_PROGRAM /usr/local/bin/vtest
Willy Tarreaua4009cd2020-12-16 10:39:20 +010074 setenv HAPROXY_ARGS "-dM -de -m 50"
Frederic Lecailled4f36e32018-12-13 22:15:05 +010075 or
76 export HAPROXY_PROGRAM=/usr/local/sbin/haproxy
Frédéric Lécaille43361232019-01-11 10:10:21 +010077 export VTEST_PROGRAM=/usr/local/bin/vtest
Willy Tarreaua4009cd2020-12-16 10:39:20 +010078 export HAPROXY_ARGS="-dM -de -m 50"
PiBa-NL72504042018-11-27 22:26:38 +010079EOF
Christopher Faulet8d67cf82018-12-18 22:41:20 +010080 exit 0
81}
PiBa-NL72504042018-11-27 22:26:38 +010082
Frederic Lecailled4f36e32018-12-13 22:15:05 +010083add_range_to_test_list()
84{
85 level0="*.vtc"
86 level1="h*.vtc"
87 level2="s*.vtc"
88 level3="l*.vtc"
89 level4="b*.vtc"
90 level5="k*.vtc"
91 level6="e*.vtc"
92
93 new_range=$(echo $1 | tr '-' ' ')
94 non_digit=$(echo $new_range | grep '[^0-9 ]')
95 if [ -n "$non_digit" ] ; then
96 return
97 fi
98 if [ "$new_range" = "$1" ] ; then
99 if [ $1 -gt 6 ] ; then
100 return
101 fi
102 eval echo '$'level$1
103 return
104 fi
105 if [ -z "$new_range" ] ; then
106 return
107 fi
108 list=
109 for l in $(seq $new_range) ; do
110 if [ -n "l" ] ; then
111 if [ -z "$list" ] ; then
112 list="$(eval echo '$'level${l})"
113 else
114 list="$list $(eval echo '$'level${l})"
115 fi
116 fi
117 done
118
119 echo $list
120}
121
PiBa-NL72504042018-11-27 22:26:38 +0100122_startswith() {
123 _str="$1"
124 _sub="$2"
125 echo "$_str" | grep "^$_sub" >/dev/null 2>&1
126}
127
128_findtests() {
129 set -f
PiBa-NL72504042018-11-27 22:26:38 +0100130
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100131 REGTESTS_TYPES="${REGTESTS_TYPES:-any}"
132 any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
133 for i in $( find "$1" -name *.vtc ); do
PiBa-NL72504042018-11-27 22:26:38 +0100134 skiptest=
Willy Tarreau939193a2018-12-06 15:49:27 +0100135 require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")"
136 require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")"
Christopher Faulet7e245df2018-12-21 15:18:02 +0100137 require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed -e 's/,/ /g')"
William Dauchyaabde712021-01-09 17:26:20 +0100138 require_services="$(sed -ne 's/^#REQUIRE_SERVICES=//p' "$i" | sed -e 's/,/ /g')"
Christopher Faulet7e245df2018-12-21 15:18:02 +0100139 exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed -e 's/,/ /g')"
William Lallemandd5b464b2019-12-19 14:30:00 +0100140 require_binaries="$(sed -ne 's/^#REQUIRE_BINARIES=//p' "$i" | sed -e 's/,/ /g')"
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100141 if [ $any_test -ne 1 ] ; then
142 regtest_type="$(sed -ne 's/^#REGTEST_TYPE=//p' "$i")"
143 if [ -z $regtest_type ] ; then
144 regtest_type=default
145 fi
146 if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
147 echo " Skip $i because its type '"$regtest_type"' is excluded"
148 skiptest=1
149 fi
150 fi
Christopher Faulet7e245df2018-12-21 15:18:02 +0100151
152 requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed -e 's/,.*//')"
153 if [ -n "$requiredoption" ]; then
154 require_options="$require_options $requiredoption"
155 fi
156
William Dauchyaabde712021-01-09 17:26:20 +0100157 requiredservice="$(sed -ne 's/^#REQUIRE_SERVICE=//p' "$i" | sed -e 's/,.*//')"
158 if [ -n "$requiredservice" ]; then
159 require_services="$require_services $requiredservice"
160 fi
161
Christopher Faulet7e245df2018-12-21 15:18:02 +0100162 excludedtarget="$(sed -ne 's/^#EXCLUDE_TARGET=//p' "$i" | sed -e 's/,.*//')"
163 if [ -n "$excludedtarget" ]; then
164 exclude_targets="$exclude_targets $excludedtarget"
165 fi
PiBa-NL72504042018-11-27 22:26:38 +0100166
167 if [ -n "$require_version" ]; then
168 if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then
169 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
170 echo " REASON: this test requires at least version: $require_version"
171 skiptest=1
172 fi
173 fi
174 if [ -n "$require_version_below" ]; then
175 if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then
176 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
177 echo " REASON: this test requires a version below: $require_version_below"
178 skiptest=1
179 fi
180 fi
181
Christopher Faulet7e245df2018-12-21 15:18:02 +0100182 for excludedtarget in $exclude_targets; do
183 if [ "$excludedtarget" = "$TARGET" ]; then
184 echo " Skip $i because haproxy is compiled for the excluded target $TARGET"
PiBa-NL72504042018-11-27 22:26:38 +0100185 skiptest=1
186 fi
187 done
Christopher Faulet7e245df2018-12-21 15:18:02 +0100188
189 for requiredoption in $require_options; do
190 alternatives=$(echo "$requiredoption" | sed -e 's/|/ /g')
191 found=
192 for alt in $alternatives; do
Willy Tarreau87586e12019-03-27 13:52:39 +0100193 if echo "$FEATURES" | grep -qw "\+$alt"; then
Christopher Faulet7e245df2018-12-21 15:18:02 +0100194 found=1;
195 fi
196 done
197 if [ -z $found ]; then
198 echo " Skip $i because haproxy is not compiled with the required option $requiredoption"
PiBa-NL72504042018-11-27 22:26:38 +0100199 skiptest=1
200 fi
201 done
PiBa-NL72504042018-11-27 22:26:38 +0100202
William Dauchyaabde712021-01-09 17:26:20 +0100203 for requiredservice in $require_services; do
204 alternatives=$(echo "$requiredservice" | sed -e 's/|/ /g')
205 found=
206 for alt in $alternatives; do
William Dauchy44884342021-01-10 21:13:05 +0100207 if echo "$SERVICES" | grep -qw "$alt"; then
William Dauchyaabde712021-01-09 17:26:20 +0100208 found=1;
209 fi
210 done
211 if [ -z $found ]; then
212 echo " Skip $i because haproxy is not compiled with the required service $requiredservice"
213 skiptest=1
214 fi
215 done
216
William Lallemandd5b464b2019-12-19 14:30:00 +0100217 for requiredbin in $require_binaries; do
Ilya Shipitsin7faeea92020-09-26 11:54:27 +0500218 if ! command -v $requiredbin >/dev/null 2>&1
219 then
William Lallemandd5b464b2019-12-19 14:30:00 +0100220 echo " Skip $i because '"$requiredbin"' is not installed"
221 skiptest=1
222 fi
223 done
224
PiBa-NL72504042018-11-27 22:26:38 +0100225 if [ -z $skiptest ]; then
226 echo " Add test: $i"
227 testlist="$testlist $i"
228 fi
229 done
230}
231
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100232_cleanup()
233{
234 DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null)
235 if [ -z "${DIRS}" ]; then
236 echo "No reg-tests log directory found"
237 else
238 echo "Cleanup following reg-tests log directories:"
239 for d in ${DIRS}; do
240 echo " o ${TESTDIR}/$d"
241 done
242 read -p "Continue (y/n)?" reply
243 case "$reply" in
244 y|Y)
245 for d in ${DIRS}; do
246 rm -r "${TESTDIR}/$d"
247 done
248 echo "done"
249 exit 0
250 ;;
251 *)
252 echo "aborted"
253 exit 1
254 ;;
255 esac
256 fi
257}
258
259
PiBa-NL72504042018-11-27 22:26:38 +0100260_process() {
PiBa-NL72504042018-11-27 22:26:38 +0100261 while [ ${#} -gt 0 ]; do
262 if _startswith "$1" "-"; then
263 case "${1}" in
264 --j)
265 jobcount="$2"
266 shift
267 ;;
Frédéric Lécaille43361232019-01-11 10:10:21 +0100268 --vtestparams)
269 vtestparams="$2"
PiBa-NL72504042018-11-27 22:26:38 +0100270 shift
271 ;;
272 --v)
273 verbose=""
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100274 ;;
275 --debug)
276 verbose=""
277 debug="-v"
PiBa-NL72504042018-11-27 22:26:38 +0100278 ;;
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100279 --keep-logs)
280 keep_logs="-L"
281 ;;
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100282 --type)
283 REGTESTS_TYPES="$2"
284 shift
285 ;;
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100286 --use-htx)
287 no_htx=""
288 ;;
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200289 --no-htx)
290 no_htx="no "
291 ;;
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100292 --clean)
293 _cleanup
294 exit 0
295 ;;
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100296 --help)
297 _help
298 ;;
PiBa-NL72504042018-11-27 22:26:38 +0100299 *)
300 echo "Unknown parameter : $1"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100301 exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100302 ;;
303 esac
304 else
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100305 REGTESTS="${REGTESTS} $1"
PiBa-NL72504042018-11-27 22:26:38 +0100306 fi
307 shift 1
308 done
PiBa-NL72504042018-11-27 22:26:38 +0100309}
310
311_version() {
312 echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\012", $1,$2,$3,$4); }';
313}
314
PiBa-NL72504042018-11-27 22:26:38 +0100315
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100316HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}"
Willy Tarreaua4009cd2020-12-16 10:39:20 +0100317HAPROXY_ARGS="${HAPROXY_ARGS--dM}"
Frédéric Lécaille43361232019-01-11 10:10:21 +0100318VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}"
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100319TESTDIR="${TMPDIR:-/tmp}"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100320REGTESTS=""
321
322jobcount=""
323verbose="-q"
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100324debug=""
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100325keep_logs="-l"
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200326no_htx=""
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100327testlist=""
328
329_process "$@";
330
331echo ""
332echo "########################## Preparing to run tests ##########################"
PiBa-NL72504042018-11-27 22:26:38 +0100333
334preparefailed=
335if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then
336 echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable"
337 preparefailed=1
338fi
Frédéric Lécaille43361232019-01-11 10:10:21 +0100339if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then
340 echo "vtest not found in path, please specify VTEST_PROGRAM environment variable"
PiBa-NL72504042018-11-27 22:26:38 +0100341 preparefailed=1
342fi
343if [ $preparefailed ]; then
344 exit 1
345fi
346
William Dauchyaabde712021-01-09 17:26:20 +0100347{ read HAPROXY_VERSION; read TARGET; read FEATURES; read SERVICES; } << EOF
348$($HAPROXY_PROGRAM $HAPROXY_ARGS -vv | grep 'HA-Proxy version\|TARGET.*=\|^Feature\|^Available services' | sed 's/.* [:=] //')
PiBa-NL72504042018-11-27 22:26:38 +0100349EOF
350
351HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3)
352echo "Testing with haproxy version: $HAPROXY_VERSION"
353
354TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')"
355
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100356mkdir -p "$TESTDIR" || exit 1
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100357TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100358
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100359export TMPDIR="$TESTDIR"
360export HAPROXY_PROGRAM="$HAPROXY_PROGRAM"
Willy Tarreaua4009cd2020-12-16 10:39:20 +0100361if [ -n "$HAPROXY_ARGS" ]; then
362 export HAPROXY_ARGS
363fi
PiBa-NL72504042018-11-27 22:26:38 +0100364
PiBa-NL72504042018-11-27 22:26:38 +0100365echo "Target : $TARGET"
Willy Tarreau87586e12019-03-27 13:52:39 +0100366echo "Options : $FEATURES"
William Dauchyaabde712021-01-09 17:26:20 +0100367echo "Services : $SERVICES"
PiBa-NL72504042018-11-27 22:26:38 +0100368
369echo "########################## Gathering tests to run ##########################"
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200370# if htx is enable, but HAProxy version is lower to 1.9, disable it
371if [ $(_version "$HAPROXY_VERSION") -lt $(_version "1.9") ]; then
372 no_htx="#"
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100373fi
374
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100375if [ -z "$REGTESTS" ]; then
Willy Tarreauca8df4c2019-04-23 16:09:50 +0200376 _findtests reg-tests/
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100377else
378 for t in $REGTESTS; do
379 _findtests $t
380 done
381fi
PiBa-NL72504042018-11-27 22:26:38 +0100382
Frédéric Lécaille43361232019-01-11 10:10:21 +0100383echo "########################## Starting vtest ##########################"
PiBa-NL72504042018-11-27 22:26:38 +0100384echo "Testing with haproxy version: $HAPROXY_VERSION"
385_vtresult=0
386if [ -n "$testlist" ]; then
387 if [ -n "$jobcount" ]; then
388 jobcount="-j $jobcount"
389 fi
Frédéric Lécaille85a7ea02019-04-25 20:14:43 +0200390 cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t 10 -Dno-htx=${no_htx} $keep_logs $verbose $debug $jobcount $vtestparams $testlist"
Christopher Faulet1ecf0ea2018-12-18 22:47:23 +0100391 eval $cmd
PiBa-NL72504042018-11-27 22:26:38 +0100392 _vtresult=$?
393else
394 echo "No tests found that meet the required criteria"
395fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100396
397
398if [ $_vtresult -eq 0 ]; then
Bertrand Jacquin7e30b502019-05-16 22:15:31 +0100399 # all tests were successful, removing tempdir (the last part.)
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100400 # ignore errors is the directory is not empty or if it does not exist
401 rmdir "$TESTDIR" 2>/dev/null
402fi
403
404if [ -d "${TESTDIR}" ]; then
405 echo "########################## Gathering results ##########################"
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100406 export TESTDIR
407 find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
408 if [ ! -e "$i/LOG" ] ; then continue; fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100409
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100410 cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
411$(echo "###### $(cat "$i/INFO") ######")
412$(echo "## test results in: \"$i\"")
Frédéric Lécaillef9a48ef2019-01-08 11:30:28 +0100413$(grep -E -- "^(----|\* diag)" "$i/LOG")
PiBa-NL72504042018-11-27 22:26:38 +0100414EOF
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100415 done' sh {} +
PiBa-NL72504042018-11-27 22:26:38 +0100416fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100417
418exit $_vtresult