blob: da20ae3dd634aaac096438c8f4b8ab1e1fb1141d [file] [log] [blame]
Frédéric Lécaille51e01b52018-11-29 21:41:42 +01001#!/bin/sh
PiBa-NL72504042018-11-27 22:26:38 +01002
Christopher Faulet8d67cf82018-12-18 22:41:20 +01003_help()
4{
PiBa-NL72504042018-11-27 22:26:38 +01005 cat << EOF
6### run-regtests.sh ###
7 Running run-regtests.sh --help shows this information about how to use it
8
9 Run without parameters to run all tests in the current folder (including subfolders)
10 run-regtests.sh
11
12 Provide paths to run tests from (including subfolders):
13 run-regtests.sh ./tests1 ./tests2
14
15 Parameters:
Frédéric Lécaille43361232019-01-11 10:10:21 +010016 --j <NUM>, To run vtest with multiple jobs / threads for a faster overall result
PiBa-NL72504042018-11-27 22:26:38 +010017 run-regtests.sh ./fasttest --j 16
18
19 --v, to run verbose
Frédéric Lécaille43361232019-01-11 10:10:21 +010020 run-regtests.sh --v, disables the default vtest 'quiet' parameter
PiBa-NL72504042018-11-27 22:26:38 +010021
Ilya Shipitsin856aabc2020-04-16 23:51:34 +050022 --debug to show test logs on standard output (implies --v)
Christopher Faulet2a7cf922018-12-19 10:22:01 +010023 run-regtests.sh --debug
24
Christopher Faulet9c6df5e2018-12-19 10:25:07 +010025 --keep-logs to keep all log directories (by default kept if test fails)
26 run-regtests.sh --keep-logs
27
Frédéric Lécaille43361232019-01-11 10:10:21 +010028 --vtestparams <ARGS>, passes custom ARGS to vtest
29 run-regtests.sh --vtestparams "-n 10"
PiBa-NL72504042018-11-27 22:26:38 +010030
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +010031 --type <reg tests types> filter the types of the tests to be run, depending on
32 the commented REGTESTS_TYPE variable value in each VTC file.
Christopher Faulet1eee6ca2019-12-11 16:26:11 +010033 The value of REGTESTS_TYPE supported are: default, slow, bug, broken, devel
34 and experimental. When not specified, it is set to 'default' as default value.
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +010035
36 run-regtest.sh --type slow,default
37
Christopher Faulet8d0fdf52018-12-19 10:18:40 +010038 --clean to cleanup previous reg-tests log directories and exit
39 run-regtests.sh --clean
40
Christopher Faulet78f31bf2019-04-12 16:53:41 +020041 --use-htx to use the HTX in tests (deprecated, the default mode now)
42
43 --no-htx to use the legacy HTTP in tests
44 run-regtests.sh --no-htx, sets the macro \${no-htx}
Christopher Fauletfa6798f2018-12-19 11:22:04 +010045 In .vtc files, in HAProxy configuration, you should use the following line
46 to "templatize" your tests:
47
48 \${no-htx} option http-use-htx
49
PiBa-NL72504042018-11-27 22:26:38 +010050 Including text below into a .vtc file will check for its requirements
51 related to haproxy's target and compilation options
Bertrand Jacquin7e30b502019-05-16 22:15:31 +010052 # Below targets are not capable of completing this test successfully
PiBa-NL72504042018-11-27 22:26:38 +010053 #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd
54
55 #EXCLUDE_TARGETS=dos,freebsd,windows
56
Bertrand Jacquin7e30b502019-05-16 22:15:31 +010057 # Below option is required to complete this test successfully
PiBa-NL72504042018-11-27 22:26:38 +010058 #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in.
59
Christopher Faulet7e245df2018-12-21 15:18:02 +010060 #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA
PiBa-NL72504042018-11-27 22:26:38 +010061
62 # To define a range of versions that a test can run with:
63 #REQUIRE_VERSION=0.0
64 #REQUIRE_VERSION_BELOW=99.9
65
William Lallemandd5b464b2019-12-19 14:30:00 +010066 # To define required binaries for a test:
67 #REQUIRE_BINARIES=socat,curl
68
Frédéric Lécaille43361232019-01-11 10:10:21 +010069 Configure environment variables to set the haproxy and vtest binaries to use
PiBa-NL72504042018-11-27 22:26:38 +010070 setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy
Frédéric Lécaille43361232019-01-11 10:10:21 +010071 setenv VTEST_PROGRAM /usr/local/bin/vtest
Willy Tarreauf13c7db2020-12-16 10:39:20 +010072 setenv HAPROXY_ARGS "-dM -de -m 50"
Frederic Lecailled4f36e32018-12-13 22:15:05 +010073 or
74 export HAPROXY_PROGRAM=/usr/local/sbin/haproxy
Frédéric Lécaille43361232019-01-11 10:10:21 +010075 export VTEST_PROGRAM=/usr/local/bin/vtest
Willy Tarreauf13c7db2020-12-16 10:39:20 +010076 export HAPROXY_ARGS="-dM -de -m 50"
PiBa-NL72504042018-11-27 22:26:38 +010077EOF
Christopher Faulet8d67cf82018-12-18 22:41:20 +010078 exit 0
79}
PiBa-NL72504042018-11-27 22:26:38 +010080
Frederic Lecailled4f36e32018-12-13 22:15:05 +010081add_range_to_test_list()
82{
83 level0="*.vtc"
84 level1="h*.vtc"
85 level2="s*.vtc"
86 level3="l*.vtc"
87 level4="b*.vtc"
88 level5="k*.vtc"
89 level6="e*.vtc"
90
91 new_range=$(echo $1 | tr '-' ' ')
92 non_digit=$(echo $new_range | grep '[^0-9 ]')
93 if [ -n "$non_digit" ] ; then
94 return
95 fi
96 if [ "$new_range" = "$1" ] ; then
97 if [ $1 -gt 6 ] ; then
98 return
99 fi
100 eval echo '$'level$1
101 return
102 fi
103 if [ -z "$new_range" ] ; then
104 return
105 fi
106 list=
107 for l in $(seq $new_range) ; do
108 if [ -n "l" ] ; then
109 if [ -z "$list" ] ; then
110 list="$(eval echo '$'level${l})"
111 else
112 list="$list $(eval echo '$'level${l})"
113 fi
114 fi
115 done
116
117 echo $list
118}
119
PiBa-NL72504042018-11-27 22:26:38 +0100120_startswith() {
121 _str="$1"
122 _sub="$2"
123 echo "$_str" | grep "^$_sub" >/dev/null 2>&1
124}
125
126_findtests() {
127 set -f
PiBa-NL72504042018-11-27 22:26:38 +0100128
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100129 REGTESTS_TYPES="${REGTESTS_TYPES:-any}"
130 any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
131 for i in $( find "$1" -name *.vtc ); do
PiBa-NL72504042018-11-27 22:26:38 +0100132 skiptest=
Willy Tarreau939193a2018-12-06 15:49:27 +0100133 require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")"
134 require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")"
Christopher Faulet7e245df2018-12-21 15:18:02 +0100135 require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed -e 's/,/ /g')"
136 exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed -e 's/,/ /g')"
William Lallemandd5b464b2019-12-19 14:30:00 +0100137 require_binaries="$(sed -ne 's/^#REQUIRE_BINARIES=//p' "$i" | sed -e 's/,/ /g')"
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100138 if [ $any_test -ne 1 ] ; then
139 regtest_type="$(sed -ne 's/^#REGTEST_TYPE=//p' "$i")"
140 if [ -z $regtest_type ] ; then
141 regtest_type=default
142 fi
143 if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
144 echo " Skip $i because its type '"$regtest_type"' is excluded"
145 skiptest=1
146 fi
147 fi
Christopher Faulet7e245df2018-12-21 15:18:02 +0100148
149 requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed -e 's/,.*//')"
150 if [ -n "$requiredoption" ]; then
151 require_options="$require_options $requiredoption"
152 fi
153
154 excludedtarget="$(sed -ne 's/^#EXCLUDE_TARGET=//p' "$i" | sed -e 's/,.*//')"
155 if [ -n "$excludedtarget" ]; then
156 exclude_targets="$exclude_targets $excludedtarget"
157 fi
PiBa-NL72504042018-11-27 22:26:38 +0100158
159 if [ -n "$require_version" ]; then
160 if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then
161 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
162 echo " REASON: this test requires at least version: $require_version"
163 skiptest=1
164 fi
165 fi
166 if [ -n "$require_version_below" ]; then
167 if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then
168 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
169 echo " REASON: this test requires a version below: $require_version_below"
170 skiptest=1
171 fi
172 fi
173
Christopher Faulet7e245df2018-12-21 15:18:02 +0100174 for excludedtarget in $exclude_targets; do
175 if [ "$excludedtarget" = "$TARGET" ]; then
176 echo " Skip $i because haproxy is compiled for the excluded target $TARGET"
PiBa-NL72504042018-11-27 22:26:38 +0100177 skiptest=1
178 fi
179 done
Christopher Faulet7e245df2018-12-21 15:18:02 +0100180
181 for requiredoption in $require_options; do
182 alternatives=$(echo "$requiredoption" | sed -e 's/|/ /g')
183 found=
184 for alt in $alternatives; do
Willy Tarreau87586e12019-03-27 13:52:39 +0100185 if echo "$FEATURES" | grep -qw "\+$alt"; then
Christopher Faulet7e245df2018-12-21 15:18:02 +0100186 found=1;
187 fi
188 done
189 if [ -z $found ]; then
190 echo " Skip $i because haproxy is not compiled with the required option $requiredoption"
PiBa-NL72504042018-11-27 22:26:38 +0100191 skiptest=1
192 fi
193 done
PiBa-NL72504042018-11-27 22:26:38 +0100194
William Lallemandd5b464b2019-12-19 14:30:00 +0100195 for requiredbin in $require_binaries; do
Ilya Shipitsin7faeea92020-09-26 11:54:27 +0500196 if ! command -v $requiredbin >/dev/null 2>&1
197 then
William Lallemandd5b464b2019-12-19 14:30:00 +0100198 echo " Skip $i because '"$requiredbin"' is not installed"
199 skiptest=1
200 fi
201 done
202
PiBa-NL72504042018-11-27 22:26:38 +0100203 if [ -z $skiptest ]; then
204 echo " Add test: $i"
205 testlist="$testlist $i"
206 fi
207 done
208}
209
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100210_cleanup()
211{
212 DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null)
213 if [ -z "${DIRS}" ]; then
214 echo "No reg-tests log directory found"
215 else
216 echo "Cleanup following reg-tests log directories:"
217 for d in ${DIRS}; do
218 echo " o ${TESTDIR}/$d"
219 done
220 read -p "Continue (y/n)?" reply
221 case "$reply" in
222 y|Y)
223 for d in ${DIRS}; do
224 rm -r "${TESTDIR}/$d"
225 done
226 echo "done"
227 exit 0
228 ;;
229 *)
230 echo "aborted"
231 exit 1
232 ;;
233 esac
234 fi
235}
236
237
PiBa-NL72504042018-11-27 22:26:38 +0100238_process() {
PiBa-NL72504042018-11-27 22:26:38 +0100239 while [ ${#} -gt 0 ]; do
240 if _startswith "$1" "-"; then
241 case "${1}" in
242 --j)
243 jobcount="$2"
244 shift
245 ;;
Frédéric Lécaille43361232019-01-11 10:10:21 +0100246 --vtestparams)
247 vtestparams="$2"
PiBa-NL72504042018-11-27 22:26:38 +0100248 shift
249 ;;
250 --v)
251 verbose=""
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100252 ;;
253 --debug)
254 verbose=""
255 debug="-v"
PiBa-NL72504042018-11-27 22:26:38 +0100256 ;;
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100257 --keep-logs)
258 keep_logs="-L"
259 ;;
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100260 --type)
261 REGTESTS_TYPES="$2"
262 shift
263 ;;
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100264 --use-htx)
265 no_htx=""
266 ;;
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200267 --no-htx)
268 no_htx="no "
269 ;;
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100270 --clean)
271 _cleanup
272 exit 0
273 ;;
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100274 --help)
275 _help
276 ;;
PiBa-NL72504042018-11-27 22:26:38 +0100277 *)
278 echo "Unknown parameter : $1"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100279 exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100280 ;;
281 esac
282 else
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100283 REGTESTS="${REGTESTS} $1"
PiBa-NL72504042018-11-27 22:26:38 +0100284 fi
285 shift 1
286 done
PiBa-NL72504042018-11-27 22:26:38 +0100287}
288
289_version() {
290 echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\012", $1,$2,$3,$4); }';
291}
292
PiBa-NL72504042018-11-27 22:26:38 +0100293
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100294HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}"
Willy Tarreauf13c7db2020-12-16 10:39:20 +0100295HAPROXY_ARGS="${HAPROXY_ARGS--dM}"
Frédéric Lécaille43361232019-01-11 10:10:21 +0100296VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}"
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100297TESTDIR="${TMPDIR:-/tmp}"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100298REGTESTS=""
299
300jobcount=""
301verbose="-q"
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100302debug=""
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100303keep_logs="-l"
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200304no_htx=""
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100305testlist=""
306
307_process "$@";
308
309echo ""
310echo "########################## Preparing to run tests ##########################"
PiBa-NL72504042018-11-27 22:26:38 +0100311
312preparefailed=
313if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then
314 echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable"
315 preparefailed=1
316fi
Frédéric Lécaille43361232019-01-11 10:10:21 +0100317if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then
318 echo "vtest not found in path, please specify VTEST_PROGRAM environment variable"
PiBa-NL72504042018-11-27 22:26:38 +0100319 preparefailed=1
320fi
321if [ $preparefailed ]; then
322 exit 1
323fi
324
Willy Tarreauad25c262019-03-27 14:00:10 +0100325{ read HAPROXY_VERSION; read TARGET; read FEATURES; } << EOF
Willy Tarreauf13c7db2020-12-16 10:39:20 +0100326$($HAPROXY_PROGRAM $HAPROXY_ARGS -vv |grep 'HA-Proxy version\|TARGET.*=\|^Feature' | sed 's/.* [:=] //')
PiBa-NL72504042018-11-27 22:26:38 +0100327EOF
328
329HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3)
330echo "Testing with haproxy version: $HAPROXY_VERSION"
331
332TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')"
333
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100334mkdir -p "$TESTDIR" || exit 1
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100335TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100336
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100337export TMPDIR="$TESTDIR"
338export HAPROXY_PROGRAM="$HAPROXY_PROGRAM"
Willy Tarreauf13c7db2020-12-16 10:39:20 +0100339if [ -n "$HAPROXY_ARGS" ]; then
340 export HAPROXY_ARGS
341fi
PiBa-NL72504042018-11-27 22:26:38 +0100342
PiBa-NL72504042018-11-27 22:26:38 +0100343echo "Target : $TARGET"
Willy Tarreau87586e12019-03-27 13:52:39 +0100344echo "Options : $FEATURES"
PiBa-NL72504042018-11-27 22:26:38 +0100345
346echo "########################## Gathering tests to run ##########################"
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200347# if htx is enable, but HAProxy version is lower to 1.9, disable it
348if [ $(_version "$HAPROXY_VERSION") -lt $(_version "1.9") ]; then
349 no_htx="#"
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100350fi
351
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100352if [ -z "$REGTESTS" ]; then
Willy Tarreauca8df4c2019-04-23 16:09:50 +0200353 _findtests reg-tests/
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100354else
355 for t in $REGTESTS; do
356 _findtests $t
357 done
358fi
PiBa-NL72504042018-11-27 22:26:38 +0100359
Frédéric Lécaille43361232019-01-11 10:10:21 +0100360echo "########################## Starting vtest ##########################"
PiBa-NL72504042018-11-27 22:26:38 +0100361echo "Testing with haproxy version: $HAPROXY_VERSION"
362_vtresult=0
363if [ -n "$testlist" ]; then
364 if [ -n "$jobcount" ]; then
365 jobcount="-j $jobcount"
366 fi
Frédéric Lécaille85a7ea02019-04-25 20:14:43 +0200367 cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t 10 -Dno-htx=${no_htx} $keep_logs $verbose $debug $jobcount $vtestparams $testlist"
Christopher Faulet1ecf0ea2018-12-18 22:47:23 +0100368 eval $cmd
PiBa-NL72504042018-11-27 22:26:38 +0100369 _vtresult=$?
370else
371 echo "No tests found that meet the required criteria"
372fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100373
374
375if [ $_vtresult -eq 0 ]; then
Bertrand Jacquin7e30b502019-05-16 22:15:31 +0100376 # all tests were successful, removing tempdir (the last part.)
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100377 # ignore errors is the directory is not empty or if it does not exist
378 rmdir "$TESTDIR" 2>/dev/null
379fi
380
381if [ -d "${TESTDIR}" ]; then
382 echo "########################## Gathering results ##########################"
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100383 export TESTDIR
384 find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
385 if [ ! -e "$i/LOG" ] ; then continue; fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100386
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100387 cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
388$(echo "###### $(cat "$i/INFO") ######")
389$(echo "## test results in: \"$i\"")
Frédéric Lécaillef9a48ef2019-01-08 11:30:28 +0100390$(grep -E -- "^(----|\* diag)" "$i/LOG")
PiBa-NL72504042018-11-27 22:26:38 +0100391EOF
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100392 done' sh {} +
PiBa-NL72504042018-11-27 22:26:38 +0100393fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100394
395exit $_vtresult