blob: 10f29e408bdc1db2fadcbee7f17ec192b2ef76f2 [file] [log] [blame]
Frédéric Lécaille51e01b52018-11-29 21:41:42 +01001#!/bin/sh
PiBa-NL72504042018-11-27 22:26:38 +01002
Christopher Faulet8d67cf82018-12-18 22:41:20 +01003_help()
4{
PiBa-NL72504042018-11-27 22:26:38 +01005 cat << EOF
6### run-regtests.sh ###
7 Running run-regtests.sh --help shows this information about how to use it
8
9 Run without parameters to run all tests in the current folder (including subfolders)
10 run-regtests.sh
11
12 Provide paths to run tests from (including subfolders):
13 run-regtests.sh ./tests1 ./tests2
14
15 Parameters:
Frédéric Lécaille43361232019-01-11 10:10:21 +010016 --j <NUM>, To run vtest with multiple jobs / threads for a faster overall result
PiBa-NL72504042018-11-27 22:26:38 +010017 run-regtests.sh ./fasttest --j 16
18
19 --v, to run verbose
Frédéric Lécaille43361232019-01-11 10:10:21 +010020 run-regtests.sh --v, disables the default vtest 'quiet' parameter
PiBa-NL72504042018-11-27 22:26:38 +010021
Christopher Faulet2a7cf922018-12-19 10:22:01 +010022 --debug to show test logs on standard ouput (implies --v)
23 run-regtests.sh --debug
24
Christopher Faulet9c6df5e2018-12-19 10:25:07 +010025 --keep-logs to keep all log directories (by default kept if test fails)
26 run-regtests.sh --keep-logs
27
Frédéric Lécaille43361232019-01-11 10:10:21 +010028 --vtestparams <ARGS>, passes custom ARGS to vtest
29 run-regtests.sh --vtestparams "-n 10"
PiBa-NL72504042018-11-27 22:26:38 +010030
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +010031 --type <reg tests types> filter the types of the tests to be run, depending on
32 the commented REGTESTS_TYPE variable value in each VTC file.
33 The value of REGTESTS_TYPE supported are: default, slow, bug, broken and
34 experimental. When not specified, it is set to 'default' as default value.
35
36 run-regtest.sh --type slow,default
37
Christopher Faulet8d0fdf52018-12-19 10:18:40 +010038 --clean to cleanup previous reg-tests log directories and exit
39 run-regtests.sh --clean
40
Christopher Faulet78f31bf2019-04-12 16:53:41 +020041 --use-htx to use the HTX in tests (deprecated, the default mode now)
42
43 --no-htx to use the legacy HTTP in tests
44 run-regtests.sh --no-htx, sets the macro \${no-htx}
Christopher Fauletfa6798f2018-12-19 11:22:04 +010045 In .vtc files, in HAProxy configuration, you should use the following line
46 to "templatize" your tests:
47
48 \${no-htx} option http-use-htx
49
PiBa-NL72504042018-11-27 22:26:38 +010050 Including text below into a .vtc file will check for its requirements
51 related to haproxy's target and compilation options
52 # Below targets are not capable of completing this test succesfully
53 #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd
54
55 #EXCLUDE_TARGETS=dos,freebsd,windows
56
57 # Below option is required to complete this test succesfully
58 #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in.
59
Christopher Faulet7e245df2018-12-21 15:18:02 +010060 #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA
PiBa-NL72504042018-11-27 22:26:38 +010061
62 # To define a range of versions that a test can run with:
63 #REQUIRE_VERSION=0.0
64 #REQUIRE_VERSION_BELOW=99.9
65
Frédéric Lécaille43361232019-01-11 10:10:21 +010066 Configure environment variables to set the haproxy and vtest binaries to use
PiBa-NL72504042018-11-27 22:26:38 +010067 setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy
Frédéric Lécaille43361232019-01-11 10:10:21 +010068 setenv VTEST_PROGRAM /usr/local/bin/vtest
Frederic Lecailled4f36e32018-12-13 22:15:05 +010069 or
70 export HAPROXY_PROGRAM=/usr/local/sbin/haproxy
Frédéric Lécaille43361232019-01-11 10:10:21 +010071 export VTEST_PROGRAM=/usr/local/bin/vtest
PiBa-NL72504042018-11-27 22:26:38 +010072EOF
Christopher Faulet8d67cf82018-12-18 22:41:20 +010073 exit 0
74}
PiBa-NL72504042018-11-27 22:26:38 +010075
Frederic Lecailled4f36e32018-12-13 22:15:05 +010076add_range_to_test_list()
77{
78 level0="*.vtc"
79 level1="h*.vtc"
80 level2="s*.vtc"
81 level3="l*.vtc"
82 level4="b*.vtc"
83 level5="k*.vtc"
84 level6="e*.vtc"
85
86 new_range=$(echo $1 | tr '-' ' ')
87 non_digit=$(echo $new_range | grep '[^0-9 ]')
88 if [ -n "$non_digit" ] ; then
89 return
90 fi
91 if [ "$new_range" = "$1" ] ; then
92 if [ $1 -gt 6 ] ; then
93 return
94 fi
95 eval echo '$'level$1
96 return
97 fi
98 if [ -z "$new_range" ] ; then
99 return
100 fi
101 list=
102 for l in $(seq $new_range) ; do
103 if [ -n "l" ] ; then
104 if [ -z "$list" ] ; then
105 list="$(eval echo '$'level${l})"
106 else
107 list="$list $(eval echo '$'level${l})"
108 fi
109 fi
110 done
111
112 echo $list
113}
114
PiBa-NL72504042018-11-27 22:26:38 +0100115_startswith() {
116 _str="$1"
117 _sub="$2"
118 echo "$_str" | grep "^$_sub" >/dev/null 2>&1
119}
120
121_findtests() {
122 set -f
PiBa-NL72504042018-11-27 22:26:38 +0100123
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100124 REGTESTS_TYPES="${REGTESTS_TYPES:-any}"
125 any_test=$(echo $REGTESTS_TYPES | grep -cw "any")
126 for i in $( find "$1" -name *.vtc ); do
PiBa-NL72504042018-11-27 22:26:38 +0100127 skiptest=
Willy Tarreau939193a2018-12-06 15:49:27 +0100128 require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")"
129 require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")"
Christopher Faulet7e245df2018-12-21 15:18:02 +0100130 require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed -e 's/,/ /g')"
131 exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed -e 's/,/ /g')"
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100132 if [ $any_test -ne 1 ] ; then
133 regtest_type="$(sed -ne 's/^#REGTEST_TYPE=//p' "$i")"
134 if [ -z $regtest_type ] ; then
135 regtest_type=default
136 fi
137 if ! $(echo $REGTESTS_TYPES | grep -wq $regtest_type) ; then
138 echo " Skip $i because its type '"$regtest_type"' is excluded"
139 skiptest=1
140 fi
141 fi
Christopher Faulet7e245df2018-12-21 15:18:02 +0100142
143 requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed -e 's/,.*//')"
144 if [ -n "$requiredoption" ]; then
145 require_options="$require_options $requiredoption"
146 fi
147
148 excludedtarget="$(sed -ne 's/^#EXCLUDE_TARGET=//p' "$i" | sed -e 's/,.*//')"
149 if [ -n "$excludedtarget" ]; then
150 exclude_targets="$exclude_targets $excludedtarget"
151 fi
PiBa-NL72504042018-11-27 22:26:38 +0100152
153 if [ -n "$require_version" ]; then
154 if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then
155 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
156 echo " REASON: this test requires at least version: $require_version"
157 skiptest=1
158 fi
159 fi
160 if [ -n "$require_version_below" ]; then
161 if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then
162 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
163 echo " REASON: this test requires a version below: $require_version_below"
164 skiptest=1
165 fi
166 fi
167
Christopher Faulet7e245df2018-12-21 15:18:02 +0100168 for excludedtarget in $exclude_targets; do
169 if [ "$excludedtarget" = "$TARGET" ]; then
170 echo " Skip $i because haproxy is compiled for the excluded target $TARGET"
PiBa-NL72504042018-11-27 22:26:38 +0100171 skiptest=1
172 fi
173 done
Christopher Faulet7e245df2018-12-21 15:18:02 +0100174
175 for requiredoption in $require_options; do
176 alternatives=$(echo "$requiredoption" | sed -e 's/|/ /g')
177 found=
178 for alt in $alternatives; do
Willy Tarreau87586e12019-03-27 13:52:39 +0100179 if echo "$FEATURES" | grep -qw "\+$alt"; then
Christopher Faulet7e245df2018-12-21 15:18:02 +0100180 found=1;
181 fi
182 done
183 if [ -z $found ]; then
184 echo " Skip $i because haproxy is not compiled with the required option $requiredoption"
PiBa-NL72504042018-11-27 22:26:38 +0100185 skiptest=1
186 fi
187 done
PiBa-NL72504042018-11-27 22:26:38 +0100188
189 if [ -z $skiptest ]; then
190 echo " Add test: $i"
191 testlist="$testlist $i"
192 fi
193 done
194}
195
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100196_cleanup()
197{
198 DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null)
199 if [ -z "${DIRS}" ]; then
200 echo "No reg-tests log directory found"
201 else
202 echo "Cleanup following reg-tests log directories:"
203 for d in ${DIRS}; do
204 echo " o ${TESTDIR}/$d"
205 done
206 read -p "Continue (y/n)?" reply
207 case "$reply" in
208 y|Y)
209 for d in ${DIRS}; do
210 rm -r "${TESTDIR}/$d"
211 done
212 echo "done"
213 exit 0
214 ;;
215 *)
216 echo "aborted"
217 exit 1
218 ;;
219 esac
220 fi
221}
222
223
PiBa-NL72504042018-11-27 22:26:38 +0100224_process() {
PiBa-NL72504042018-11-27 22:26:38 +0100225 while [ ${#} -gt 0 ]; do
226 if _startswith "$1" "-"; then
227 case "${1}" in
228 --j)
229 jobcount="$2"
230 shift
231 ;;
Frédéric Lécaille43361232019-01-11 10:10:21 +0100232 --vtestparams)
233 vtestparams="$2"
PiBa-NL72504042018-11-27 22:26:38 +0100234 shift
235 ;;
236 --v)
237 verbose=""
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100238 ;;
239 --debug)
240 verbose=""
241 debug="-v"
PiBa-NL72504042018-11-27 22:26:38 +0100242 ;;
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100243 --keep-logs)
244 keep_logs="-L"
245 ;;
Frédéric Lécailledc1a3bd2019-03-29 15:07:24 +0100246 --type)
247 REGTESTS_TYPES="$2"
248 shift
249 ;;
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100250 --use-htx)
251 no_htx=""
252 ;;
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200253 --no-htx)
254 no_htx="no "
255 ;;
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100256 --clean)
257 _cleanup
258 exit 0
259 ;;
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100260 --help)
261 _help
262 ;;
PiBa-NL72504042018-11-27 22:26:38 +0100263 *)
264 echo "Unknown parameter : $1"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100265 exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100266 ;;
267 esac
268 else
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100269 REGTESTS="${REGTESTS} $1"
PiBa-NL72504042018-11-27 22:26:38 +0100270 fi
271 shift 1
272 done
PiBa-NL72504042018-11-27 22:26:38 +0100273}
274
275_version() {
276 echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\012", $1,$2,$3,$4); }';
277}
278
PiBa-NL72504042018-11-27 22:26:38 +0100279
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100280HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}"
Frédéric Lécaille43361232019-01-11 10:10:21 +0100281VTEST_PROGRAM="${VTEST_PROGRAM:-vtest}"
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100282TESTDIR="${TMPDIR:-/tmp}"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100283REGTESTS=""
284
285jobcount=""
286verbose="-q"
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100287debug=""
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100288keep_logs="-l"
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200289no_htx=""
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100290testlist=""
291
292_process "$@";
293
294echo ""
295echo "########################## Preparing to run tests ##########################"
PiBa-NL72504042018-11-27 22:26:38 +0100296
297preparefailed=
298if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then
299 echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable"
300 preparefailed=1
301fi
Frédéric Lécaille43361232019-01-11 10:10:21 +0100302if ! [ -x "$(command -v $VTEST_PROGRAM)" ]; then
303 echo "vtest not found in path, please specify VTEST_PROGRAM environment variable"
PiBa-NL72504042018-11-27 22:26:38 +0100304 preparefailed=1
305fi
306if [ $preparefailed ]; then
307 exit 1
308fi
309
Willy Tarreauad25c262019-03-27 14:00:10 +0100310{ read HAPROXY_VERSION; read TARGET; read FEATURES; } << EOF
311$($HAPROXY_PROGRAM -vv |grep 'HA-Proxy version\|TARGET.*=\|^Feature' | sed 's/.* [:=] //')
PiBa-NL72504042018-11-27 22:26:38 +0100312EOF
313
314HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3)
315echo "Testing with haproxy version: $HAPROXY_VERSION"
316
317TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')"
318
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100319mkdir -p "$TESTDIR" || exit 1
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100320TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100321
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100322export TMPDIR="$TESTDIR"
323export HAPROXY_PROGRAM="$HAPROXY_PROGRAM"
PiBa-NL72504042018-11-27 22:26:38 +0100324
PiBa-NL72504042018-11-27 22:26:38 +0100325echo "Target : $TARGET"
Willy Tarreau87586e12019-03-27 13:52:39 +0100326echo "Options : $FEATURES"
PiBa-NL72504042018-11-27 22:26:38 +0100327
328echo "########################## Gathering tests to run ##########################"
Christopher Faulet78f31bf2019-04-12 16:53:41 +0200329# if htx is enable, but HAProxy version is lower to 1.9, disable it
330if [ $(_version "$HAPROXY_VERSION") -lt $(_version "1.9") ]; then
331 no_htx="#"
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100332fi
333
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100334if [ -z "$REGTESTS" ]; then
Willy Tarreauca8df4c2019-04-23 16:09:50 +0200335 _findtests reg-tests/
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100336else
337 for t in $REGTESTS; do
338 _findtests $t
339 done
340fi
PiBa-NL72504042018-11-27 22:26:38 +0100341
Frédéric Lécaille43361232019-01-11 10:10:21 +0100342echo "########################## Starting vtest ##########################"
PiBa-NL72504042018-11-27 22:26:38 +0100343echo "Testing with haproxy version: $HAPROXY_VERSION"
344_vtresult=0
345if [ -n "$testlist" ]; then
346 if [ -n "$jobcount" ]; then
347 jobcount="-j $jobcount"
348 fi
Frédéric Lécaille43361232019-01-11 10:10:21 +0100349 cmd="$VTEST_PROGRAM -k -t 10 -Dno-htx=${no_htx} $keep_logs $verbose $debug $jobcount $vtestparams $testlist"
Christopher Faulet1ecf0ea2018-12-18 22:47:23 +0100350 eval $cmd
PiBa-NL72504042018-11-27 22:26:38 +0100351 _vtresult=$?
352else
353 echo "No tests found that meet the required criteria"
354fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100355
356
357if [ $_vtresult -eq 0 ]; then
358 # all tests were succesfull, removing tempdir (the last part.)
359 # ignore errors is the directory is not empty or if it does not exist
360 rmdir "$TESTDIR" 2>/dev/null
361fi
362
363if [ -d "${TESTDIR}" ]; then
364 echo "########################## Gathering results ##########################"
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100365 export TESTDIR
366 find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
367 if [ ! -e "$i/LOG" ] ; then continue; fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100368
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100369 cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
370$(echo "###### $(cat "$i/INFO") ######")
371$(echo "## test results in: \"$i\"")
Frédéric Lécaillef9a48ef2019-01-08 11:30:28 +0100372$(grep -E -- "^(----|\* diag)" "$i/LOG")
PiBa-NL72504042018-11-27 22:26:38 +0100373EOF
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100374 done' sh {} +
PiBa-NL72504042018-11-27 22:26:38 +0100375fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100376
377exit $_vtresult