blob: 7bcc394cfc605fef94b2c19d56bd3974f7991df2 [file] [log] [blame]
Frédéric Lécaille51e01b52018-11-29 21:41:42 +01001#!/bin/sh
PiBa-NL72504042018-11-27 22:26:38 +01002
Christopher Faulet8d67cf82018-12-18 22:41:20 +01003_help()
4{
PiBa-NL72504042018-11-27 22:26:38 +01005 cat << EOF
6### run-regtests.sh ###
7 Running run-regtests.sh --help shows this information about how to use it
8
9 Run without parameters to run all tests in the current folder (including subfolders)
10 run-regtests.sh
11
12 Provide paths to run tests from (including subfolders):
13 run-regtests.sh ./tests1 ./tests2
14
15 Parameters:
Frédéric Lécaille43361232019-01-11 10:10:21 +010016 --j <NUM>, To run vtest with multiple jobs / threads for a faster overall result
PiBa-NL72504042018-11-27 22:26:38 +010017 run-regtests.sh ./fasttest --j 16
18
19 --v, to run verbose
Frédéric Lécaille43361232019-01-11 10:10:21 +010020 run-regtests.sh --v, disables the default vtest 'quiet' parameter
PiBa-NL72504042018-11-27 22:26:38 +010021
Ilya Shipitsin856aabc2020-04-16 23:51:34 +050022 --debug to show test logs on standard output (implies --v)
Christopher Faulet2a7cf922018-12-19 10:22:01 +010023 run-regtests.sh --debug
24
Christopher Faulet9c6df5e2018-12-19 10:25:07 +010025 --keep-logs to keep all log directories (by default kept if test fails)
26 run-regtests.sh --keep-logs
27
Frédéric Lécaille43361232019-01-11 10:10:21 +010028 --vtestparams <ARGS>, passes custom ARGS to vtest
29 run-regtests.sh --vtestparams "-n 10"
PiBa-NL72504042018-11-27 22:26:38 +010030
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +010031 --type <reg tests types> filter the types of the tests to be run, depending on
32 the commented REGTESTS_TYPE variable value in each VTC file.
Christopher Faulet1eee6ca2019-12-11 16:26:11 +010033 The value of REGTESTS_TYPE supported are: default, slow, bug, broken, devel
34 and experimental. When not specified, it is set to 'default' as default value.
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +010035
36 run-regtest.sh --type slow,default
37
Christopher Faulet8d0fdf52018-12-19 10:18:40 +010038 --clean to cleanup previous reg-tests log directories and exit
39 run-regtests.sh --clean
40
Christopher Faulet78f31bf2019-04-12 16:53:41 +020041 --use-htx to use the HTX in tests (deprecated, the default mode now)
42
43 --no-htx to use the legacy HTTP in tests
44 run-regtests.sh --no-htx, sets the macro \${no-htx}
Christopher Fauletfa6798f2018-12-19 11:22:04 +010045 In .vtc files, in HAProxy configuration, you should use the following line
46 to "templatize" your tests:
47
48 \${no-htx} option http-use-htx
49
PiBa-NL72504042018-11-27 22:26:38 +010050 Including text below into a .vtc file will check for its requirements
51 related to haproxy's target and compilation options
Bertrand Jacquin7e30b502019-05-16 22:15:31 +010052 # Below targets are not capable of completing this test successfully
PiBa-NL72504042018-11-27 22:26:38 +010053 #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd
54
55 #EXCLUDE_TARGETS=dos,freebsd,windows
56
Bertrand Jacquin7e30b502019-05-16 22:15:31 +010057 # Below option is required to complete this test successfully
PiBa-NL72504042018-11-27 22:26:38 +010058 #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in.
59
Christopher Faulet7e245df2018-12-21 15:18:02 +010060 #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA
PiBa-NL72504042018-11-27 22:26:38 +010061
62 # To define a range of versions that a test can run with:
63 #REQUIRE_VERSION=0.0
64 #REQUIRE_VERSION_BELOW=99.9
65
William Lallemandd5b464b2019-12-19 14:30:00 +010066 # To define required binaries for a test:
67 #REQUIRE_BINARIES=socat,curl
68
Frédéric Lécaille43361232019-01-11 10:10:21 +010069 Configure environment variables to set the haproxy and vtest binaries to use
PiBa-NL72504042018-11-27 22:26:38 +010070 setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy
Frédéric Lécaille43361232019-01-11 10:10:21 +010071 setenv VTEST_PROGRAM /usr/local/bin/vtest
Frederic Lecailled4f36e32018-12-13 22:15:05 +010072 or
73 export HAPROXY_PROGRAM=/usr/local/sbin/haproxy
Frédéric Lécaille43361232019-01-11 10:10:21 +010074 export VTEST_PROGRAM=/usr/local/bin/vtest
PiBa-NL72504042018-11-27 22:26:38 +010075EOF
Christopher Faulet8d67cf82018-12-18 22:41:20 +010076 exit 0
77}
PiBa-NL72504042018-11-27 22:26:38 +010078
Frederic Lecailled4f36e32018-12-13 22:15:05 +010079add_range_to_test_list()
80{
81 level0="*.vtc"
82 level1="h*.vtc"
83 level2="s*.vtc"
84 level3="l*.vtc"
85 level4="b*.vtc"
86 level5="k*.vtc"
87 level6="e*.vtc"
88
89 new_range=$(echo $1 | tr '-' ' ')
90 non_digit=$(echo $new_range | grep '[^0-9 ]')
91 if [ -n "$non_digit" ] ; then
92 return
93 fi
94 if [ "$new_range" = "$1" ] ; then
95 if [ $1 -gt 6 ] ; then
96 return
97 fi
98 eval echo '$'level$1
99 return
100 fi
101 if [ -z "$new_range" ] ; then
102 return
103 fi
104 list=
105 for l in $(seq $new_range) ; do
106 if [ -n "l" ] ; then
107 if [ -z "$list" ] ; then
108 list="$(eval echo '$'level${l})"
109 else
110 list="$list $(eval echo '$'level${l})"
111 fi
112 fi
113 done
114
115 echo $list
116}
117
PiBa-NL72504042018-11-27 22:26:38 +0100118_startswith() {
119 _str="$1"
120 _sub="$2"
121 echo "$_str" | grep "^$_sub" >/dev/null 2>&1
122}
123
124_findtests() {
125 set -f
PiBa-NL72504042018-11-27 22:26:38 +0100126
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100127 REGTESTS_TYPES="${REGTESTS_TYPES:-any}"
128 any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
129 for i in $( find "$1" -name *.vtc ); do
PiBa-NL72504042018-11-27 22:26:38 +0100130 skiptest=
Willy Tarreau939193a2018-12-06 15:49:27 +0100131 require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")"
132 require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")"
Christopher Faulet7e245df2018-12-21 15:18:02 +0100133 require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed -e 's/,/ /g')"
134 exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed -e 's/,/ /g')"
William Lallemandd5b464b2019-12-19 14:30:00 +0100135 require_binaries="$(sed -ne 's/^#REQUIRE_BINARIES=//p' "$i" | sed -e 's/,/ /g')"
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100136 if [ $any_test -ne 1 ] ; then
137 regtest_type="$(sed -ne 's/^#REGTEST_TYPE=//p' "$i")"
138 if [ -z $regtest_type ] ; then
139 regtest_type=default
140 fi
141 if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
142 echo " Skip $i because its type '"$regtest_type"' is excluded"
143 skiptest=1
144 fi
145 fi
Christopher Faulet7e245df2018-12-21 15:18:02 +0100146
147 requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed -e 's/,.*//')"
148 if [ -n "$requiredoption" ]; then
149 require_options="$require_options $requiredoption"
150 fi
151
152 excludedtarget="$(sed -ne 's/^#EXCLUDE_TARGET=//p' "$i" | sed -e 's/,.*//')"
153 if [ -n "$excludedtarget" ]; then
154 exclude_targets="$exclude_targets $excludedtarget"
155 fi
PiBa-NL72504042018-11-27 22:26:38 +0100156
157 if [ -n "$require_version" ]; then
158 if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then
159 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
160 echo " REASON: this test requires at least version: $require_version"
161 skiptest=1
162 fi
163 fi
164 if [ -n "$require_version_below" ]; then
165 if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then
166 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
167 echo " REASON: this test requires a version below: $require_version_below"
168 skiptest=1
169 fi
170 fi
171
Christopher Faulet7e245df2018-12-21 15:18:02 +0100172 for excludedtarget in $exclude_targets; do
173 if [ "$excludedtarget" = "$TARGET" ]; then
174 echo " Skip $i because haproxy is compiled for the excluded target $TARGET"
PiBa-NL72504042018-11-27 22:26:38 +0100175 skiptest=1
176 fi
177 done
Christopher Faulet7e245df2018-12-21 15:18:02 +0100178
179 for requiredoption in $require_options; do
180 alternatives=$(echo "$requiredoption" | sed -e 's/|/ /g')
181 found=
182 for alt in $alternatives; do
Willy Tarreau87586e12019-03-27 13:52:39 +0100183 if echo "$FEATURES" | grep -qw "\+$alt"; then
Christopher Faulet7e245df2018-12-21 15:18:02 +0100184 found=1;
185 fi
186 done
187 if [ -z $found ]; then
188 echo " Skip $i because haproxy is not compiled with the required option $requiredoption"
PiBa-NL72504042018-11-27 22:26:38 +0100189 skiptest=1
190 fi
191 done
PiBa-NL72504042018-11-27 22:26:38 +0100192
William Lallemandd5b464b2019-12-19 14:30:00 +0100193 for requiredbin in $require_binaries; do
Ilya Shipitsin7faeea92020-09-26 11:54:27 +0500194 if ! command -v $requiredbin >/dev/null 2>&1
195 then
William Lallemandd5b464b2019-12-19 14:30:00 +0100196 echo " Skip $i because '"$requiredbin"' is not installed"
197 skiptest=1
198 fi
199 done
200
PiBa-NL72504042018-11-27 22:26:38 +0100201 if [ -z $skiptest ]; then
202 echo " Add test: $i"
203 testlist="$testlist $i"
204 fi
205 done
206}
207
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100208_cleanup()
209{
210 DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null)
211 if [ -z "${DIRS}" ]; then
212 echo "No reg-tests log directory found"
213 else
214 echo "Cleanup following reg-tests log directories:"
215 for d in ${DIRS}; do
216 echo " o ${TESTDIR}/$d"
217 done
218 read -p "Continue (y/n)?" reply
219 case "$reply" in
220 y|Y)
221 for d in ${DIRS}; do
222 rm -r "${TESTDIR}/$d"
223 done
224 echo "done"
225 exit 0
226 ;;
227 *)
228 echo "aborted"
229 exit 1
230 ;;
231 esac
232 fi
233}
234
235
PiBa-NL72504042018-11-27 22:26:38 +0100236_process() {
PiBa-NL72504042018-11-27 22:26:38 +0100237 while [ ${#} -gt 0 ]; do
238 if _startswith "$1" "-"; then
239 case "${1}" in
240 --j)
241 jobcount="$2"
242 shift
243 ;;
Frédéric Lécaille43361232019-01-11 10:10:21 +0100244 --vtestparams)
245 vtestparams="$2"
PiBa-NL72504042018-11-27 22:26:38 +0100246 shift
247 ;;
248 --v)
249 verbose=""
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100250 ;;
251 --debug)
252 verbose=""
253 debug="-v"
PiBa-NL72504042018-11-27 22:26:38 +0100254 ;;
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100255 --keep-logs)
256 keep_logs="-L"
257 ;;
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100258 --type)
259 REGTESTS_TYPES="$2"
260 shift
261 ;;
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100262 --use-htx)
263 no_htx=""
264 ;;
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200265 --no-htx)
266 no_htx="no "
267 ;;
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100268 --clean)
269 _cleanup
270 exit 0
271 ;;
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100272 --help)
273 _help
274 ;;
PiBa-NL72504042018-11-27 22:26:38 +0100275 *)
276 echo "Unknown parameter : $1"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100277 exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100278 ;;
279 esac
280 else
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100281 REGTESTS="${REGTESTS} $1"
PiBa-NL72504042018-11-27 22:26:38 +0100282 fi
283 shift 1
284 done
PiBa-NL72504042018-11-27 22:26:38 +0100285}
286
287_version() {
288 echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\012", $1,$2,$3,$4); }';
289}
290
PiBa-NL72504042018-11-27 22:26:38 +0100291
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100292HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}"
Frédéric Lécaille43361232019-01-11 10:10:21 +0100293VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}"
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100294TESTDIR="${TMPDIR:-/tmp}"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100295REGTESTS=""
296
297jobcount=""
298verbose="-q"
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100299debug=""
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100300keep_logs="-l"
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200301no_htx=""
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100302testlist=""
303
304_process "$@";
305
306echo ""
307echo "########################## Preparing to run tests ##########################"
PiBa-NL72504042018-11-27 22:26:38 +0100308
309preparefailed=
310if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then
311 echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable"
312 preparefailed=1
313fi
Frédéric Lécaille43361232019-01-11 10:10:21 +0100314if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then
315 echo "vtest not found in path, please specify VTEST_PROGRAM environment variable"
PiBa-NL72504042018-11-27 22:26:38 +0100316 preparefailed=1
317fi
318if [ $preparefailed ]; then
319 exit 1
320fi
321
Willy Tarreauad25c262019-03-27 14:00:10 +0100322{ read HAPROXY_VERSION; read TARGET; read FEATURES; } << EOF
323$($HAPROXY_PROGRAM -vv |grep 'HA-Proxy version\|TARGET.*=\|^Feature' | sed 's/.* [:=] //')
PiBa-NL72504042018-11-27 22:26:38 +0100324EOF
325
326HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3)
327echo "Testing with haproxy version: $HAPROXY_VERSION"
328
329TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')"
330
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100331mkdir -p "$TESTDIR" || exit 1
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100332TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100333
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100334export TMPDIR="$TESTDIR"
335export HAPROXY_PROGRAM="$HAPROXY_PROGRAM"
PiBa-NL72504042018-11-27 22:26:38 +0100336
PiBa-NL72504042018-11-27 22:26:38 +0100337echo "Target : $TARGET"
Willy Tarreau87586e12019-03-27 13:52:39 +0100338echo "Options : $FEATURES"
PiBa-NL72504042018-11-27 22:26:38 +0100339
340echo "########################## Gathering tests to run ##########################"
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200341# if htx is enable, but HAProxy version is lower to 1.9, disable it
342if [ $(_version "$HAPROXY_VERSION") -lt $(_version "1.9") ]; then
343 no_htx="#"
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100344fi
345
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100346if [ -z "$REGTESTS" ]; then
Willy Tarreauca8df4c2019-04-23 16:09:50 +0200347 _findtests reg-tests/
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100348else
349 for t in $REGTESTS; do
350 _findtests $t
351 done
352fi
PiBa-NL72504042018-11-27 22:26:38 +0100353
Frédéric Lécaille43361232019-01-11 10:10:21 +0100354echo "########################## Starting vtest ##########################"
PiBa-NL72504042018-11-27 22:26:38 +0100355echo "Testing with haproxy version: $HAPROXY_VERSION"
356_vtresult=0
357if [ -n "$testlist" ]; then
358 if [ -n "$jobcount" ]; then
359 jobcount="-j $jobcount"
360 fi
Frédéric Lécaille85a7ea02019-04-25 20:14:43 +0200361 cmd="$VTEST_PROGRAM -b $((2<<20)) -k -t 10 -Dno-htx=${no_htx} $keep_logs $verbose $debug $jobcount $vtestparams $testlist"
Christopher Faulet1ecf0ea2018-12-18 22:47:23 +0100362 eval $cmd
PiBa-NL72504042018-11-27 22:26:38 +0100363 _vtresult=$?
364else
365 echo "No tests found that meet the required criteria"
366fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100367
368
369if [ $_vtresult -eq 0 ]; then
Bertrand Jacquin7e30b502019-05-16 22:15:31 +0100370 # all tests were successful, removing tempdir (the last part.)
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100371 # ignore errors is the directory is not empty or if it does not exist
372 rmdir "$TESTDIR" 2>/dev/null
373fi
374
375if [ -d "${TESTDIR}" ]; then
376 echo "########################## Gathering results ##########################"
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100377 export TESTDIR
378 find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
379 if [ ! -e "$i/LOG" ] ; then continue; fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100380
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100381 cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
382$(echo "###### $(cat "$i/INFO") ######")
383$(echo "## test results in: \"$i\"")
Frédéric Lécaillef9a48ef2019-01-08 11:30:28 +0100384$(grep -E -- "^(----|\* diag)" "$i/LOG")
PiBa-NL72504042018-11-27 22:26:38 +0100385EOF
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100386 done' sh {} +
PiBa-NL72504042018-11-27 22:26:38 +0100387fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100388
389exit $_vtresult