blob: 80353de904f3ed4d9a2bef227bf85c94a62e7947 [file] [log] [blame]
Frédéric Lécaille51e01b52018-11-29 21:41:42 +01001#!/bin/sh
PiBa-NL72504042018-11-27 22:26:38 +01002
Christopher Faulet8d67cf82018-12-18 22:41:20 +01003_help()
4{
PiBa-NL72504042018-11-27 22:26:38 +01005 cat << EOF
6### run-regtests.sh ###
7 Running run-regtests.sh --help shows this information about how to use it
8
9 Run without parameters to run all tests in the current folder (including subfolders)
10 run-regtests.sh
11
12 Provide paths to run tests from (including subfolders):
13 run-regtests.sh ./tests1 ./tests2
14
15 Parameters:
16 --j <NUM>, To run varnishtest with multiple jobs / threads for a faster overall result
17 run-regtests.sh ./fasttest --j 16
18
19 --v, to run verbose
20 run-regtests.sh --v, disables the default varnishtest 'quiet' parameter
21
Christopher Faulet2a7cf922018-12-19 10:22:01 +010022 --debug to show test logs on standard ouput (implies --v)
23 run-regtests.sh --debug
24
Christopher Faulet9c6df5e2018-12-19 10:25:07 +010025 --keep-logs to keep all log directories (by default kept if test fails)
26 run-regtests.sh --keep-logs
27
PiBa-NL72504042018-11-27 22:26:38 +010028 --varnishtestparams <ARGS>, passes custom ARGS to varnishtest
29 run-regtests.sh --varnishtestparams "-n 10"
30
Christopher Faulet8d0fdf52018-12-19 10:18:40 +010031 --clean to cleanup previous reg-tests log directories and exit
32 run-regtests.sh --clean
33
Christopher Fauletfa6798f2018-12-19 11:22:04 +010034 --use-htx to use the HTX in tests
35 run-regtests.sh --use-htx, unsets the macro \${no-htx}
36 In .vtc files, in HAProxy configuration, you should use the following line
37 to "templatize" your tests:
38
39 \${no-htx} option http-use-htx
40
PiBa-NL72504042018-11-27 22:26:38 +010041 Including text below into a .vtc file will check for its requirements
42 related to haproxy's target and compilation options
43 # Below targets are not capable of completing this test succesfully
44 #EXCLUDE_TARGET=freebsd, abns sockets are not available on freebsd
45
46 #EXCLUDE_TARGETS=dos,freebsd,windows
47
48 # Below option is required to complete this test succesfully
49 #REQUIRE_OPTION=OPENSSL, this test needs OPENSSL compiled in.
50
Christopher Faulet7e245df2018-12-21 15:18:02 +010051 #REQUIRE_OPTIONS=ZLIB|SLZ,OPENSSL,LUA
PiBa-NL72504042018-11-27 22:26:38 +010052
53 # To define a range of versions that a test can run with:
54 #REQUIRE_VERSION=0.0
55 #REQUIRE_VERSION_BELOW=99.9
56
57 Configure environment variables to set the haproxy and varnishtest binaries to use
58 setenv HAPROXY_PROGRAM /usr/local/sbin/haproxy
59 setenv VARNISHTEST_PROGRAM /usr/local/bin/varnishtest
Frederic Lecailled4f36e32018-12-13 22:15:05 +010060 or
61 export HAPROXY_PROGRAM=/usr/local/sbin/haproxy
62 export VARNISHTEST_PROGRAM=/usr/local/bin/varnishtest
PiBa-NL72504042018-11-27 22:26:38 +010063EOF
Christopher Faulet8d67cf82018-12-18 22:41:20 +010064 exit 0
65}
PiBa-NL72504042018-11-27 22:26:38 +010066
Frederic Lecailled4f36e32018-12-13 22:15:05 +010067add_range_to_test_list()
68{
69 level0="*.vtc"
70 level1="h*.vtc"
71 level2="s*.vtc"
72 level3="l*.vtc"
73 level4="b*.vtc"
74 level5="k*.vtc"
75 level6="e*.vtc"
76
77 new_range=$(echo $1 | tr '-' ' ')
78 non_digit=$(echo $new_range | grep '[^0-9 ]')
79 if [ -n "$non_digit" ] ; then
80 return
81 fi
82 if [ "$new_range" = "$1" ] ; then
83 if [ $1 -gt 6 ] ; then
84 return
85 fi
86 eval echo '$'level$1
87 return
88 fi
89 if [ -z "$new_range" ] ; then
90 return
91 fi
92 list=
93 for l in $(seq $new_range) ; do
94 if [ -n "l" ] ; then
95 if [ -z "$list" ] ; then
96 list="$(eval echo '$'level${l})"
97 else
98 list="$list $(eval echo '$'level${l})"
99 fi
100 fi
101 done
102
103 echo $list
104}
105
106
107build_test_list()
108{
109 # Remove any spacing character
110 LEVEL="$(echo $LEVEL | tr -d ' ')"
111 # Replave any comma character by a space character
112 LEVEL="$(echo $LEVEL | tr ',' ' ')"
113 list=
114 for range in $LEVEL ; do
115 if [ -z "$list" ] ; then
116 list=$(add_range_to_test_list $range)
117 else
118 list="$list $(add_range_to_test_list $range)"
119 fi
120 done
121
122 echo $list
123}
124
125build_find_expr()
126{
127 expr=
128 for i in $@; do
129 if [ -z "$expr" ] ; then
130 expr="-name \"$i\""
131 else
132 expr="$expr -o -name \"$i\""
133 fi
134 done
135
136 echo $expr
137}
138
PiBa-NL72504042018-11-27 22:26:38 +0100139_startswith() {
140 _str="$1"
141 _sub="$2"
142 echo "$_str" | grep "^$_sub" >/dev/null 2>&1
143}
144
145_findtests() {
146 set -f
147 LEVEL=${LEVEL:-0};
Frederic Lecailled4f36e32018-12-13 22:15:05 +0100148 list=$(build_test_list "$LEVEL")
149 if [ -z "$list" ] ; then
150 echo "Invalid level specification '"$LEVEL"' or no file was found."
151 exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100152 fi
Frederic Lecailled4f36e32018-12-13 22:15:05 +0100153 EXPR=$(build_find_expr $list)
PiBa-NL72504042018-11-27 22:26:38 +0100154
Frederic Lecailled4f36e32018-12-13 22:15:05 +0100155 for i in $( find "$1" $(eval echo $EXPR) ); do
PiBa-NL72504042018-11-27 22:26:38 +0100156 skiptest=
Willy Tarreau939193a2018-12-06 15:49:27 +0100157 require_version="$(sed -ne 's/^#REQUIRE_VERSION=//p' "$i")"
158 require_version_below="$(sed -ne 's/^#REQUIRE_VERSION_BELOW=//p' "$i")"
Christopher Faulet7e245df2018-12-21 15:18:02 +0100159 require_options="$(sed -ne 's/^#REQUIRE_OPTIONS=//p' "$i" | sed -e 's/,/ /g')"
160 exclude_targets="$(sed -ne 's/^#EXCLUDE_TARGETS=//p' "$i" | sed -e 's/,/ /g')"
161
162 requiredoption="$(sed -ne 's/^#REQUIRE_OPTION=//p' "$i" | sed -e 's/,.*//')"
163 if [ -n "$requiredoption" ]; then
164 require_options="$require_options $requiredoption"
165 fi
166
167 excludedtarget="$(sed -ne 's/^#EXCLUDE_TARGET=//p' "$i" | sed -e 's/,.*//')"
168 if [ -n "$excludedtarget" ]; then
169 exclude_targets="$exclude_targets $excludedtarget"
170 fi
PiBa-NL72504042018-11-27 22:26:38 +0100171
172 if [ -n "$require_version" ]; then
173 if [ $(_version "$HAPROXY_VERSION") -lt $(_version "$require_version") ]; then
174 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
175 echo " REASON: this test requires at least version: $require_version"
176 skiptest=1
177 fi
178 fi
179 if [ -n "$require_version_below" ]; then
180 if [ $(_version "$HAPROXY_VERSION") -ge $(_version "$require_version_below") ]; then
181 echo " Skip $i because option haproxy is version: $HAPROXY_VERSION"
182 echo " REASON: this test requires a version below: $require_version_below"
183 skiptest=1
184 fi
185 fi
186
Christopher Faulet7e245df2018-12-21 15:18:02 +0100187 for excludedtarget in $exclude_targets; do
188 if [ "$excludedtarget" = "$TARGET" ]; then
189 echo " Skip $i because haproxy is compiled for the excluded target $TARGET"
PiBa-NL72504042018-11-27 22:26:38 +0100190 skiptest=1
191 fi
192 done
Christopher Faulet7e245df2018-12-21 15:18:02 +0100193
194 for requiredoption in $require_options; do
195 alternatives=$(echo "$requiredoption" | sed -e 's/|/ /g')
196 found=
197 for alt in $alternatives; do
198 if [ -n "$( echo "$OPTIONS" | grep "USE_$alt=1" )" ]; then
199 found=1;
200 fi
201 done
202 if [ -z $found ]; then
203 echo " Skip $i because haproxy is not compiled with the required option $requiredoption"
PiBa-NL72504042018-11-27 22:26:38 +0100204 skiptest=1
205 fi
206 done
PiBa-NL72504042018-11-27 22:26:38 +0100207
208 if [ -z $skiptest ]; then
209 echo " Add test: $i"
210 testlist="$testlist $i"
211 fi
212 done
213}
214
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100215_cleanup()
216{
217 DIRS=$(find "${TESTDIR}" -maxdepth 1 -type d -name "haregtests-*" -exec basename {} \; 2>/dev/null)
218 if [ -z "${DIRS}" ]; then
219 echo "No reg-tests log directory found"
220 else
221 echo "Cleanup following reg-tests log directories:"
222 for d in ${DIRS}; do
223 echo " o ${TESTDIR}/$d"
224 done
225 read -p "Continue (y/n)?" reply
226 case "$reply" in
227 y|Y)
228 for d in ${DIRS}; do
229 rm -r "${TESTDIR}/$d"
230 done
231 echo "done"
232 exit 0
233 ;;
234 *)
235 echo "aborted"
236 exit 1
237 ;;
238 esac
239 fi
240}
241
242
PiBa-NL72504042018-11-27 22:26:38 +0100243_process() {
PiBa-NL72504042018-11-27 22:26:38 +0100244 while [ ${#} -gt 0 ]; do
245 if _startswith "$1" "-"; then
246 case "${1}" in
247 --j)
248 jobcount="$2"
249 shift
250 ;;
251 --varnishtestparams)
252 varnishtestparams="$2"
253 shift
254 ;;
255 --v)
256 verbose=""
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100257 ;;
258 --debug)
259 verbose=""
260 debug="-v"
PiBa-NL72504042018-11-27 22:26:38 +0100261 ;;
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100262 --keep-logs)
263 keep_logs="-L"
264 ;;
PiBa-NL72504042018-11-27 22:26:38 +0100265 --LEVEL)
266 LEVEL="$2"
267 shift
268 ;;
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100269 --use-htx)
270 no_htx=""
271 ;;
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100272 --clean)
273 _cleanup
274 exit 0
275 ;;
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100276 --help)
277 _help
278 ;;
PiBa-NL72504042018-11-27 22:26:38 +0100279 *)
280 echo "Unknown parameter : $1"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100281 exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100282 ;;
283 esac
284 else
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100285 REGTESTS="${REGTESTS} $1"
PiBa-NL72504042018-11-27 22:26:38 +0100286 fi
287 shift 1
288 done
PiBa-NL72504042018-11-27 22:26:38 +0100289}
290
291_version() {
292 echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\012", $1,$2,$3,$4); }';
293}
294
PiBa-NL72504042018-11-27 22:26:38 +0100295
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100296HAPROXY_PROGRAM="${HAPROXY_PROGRAM:-${PWD}/haproxy}"
297VARNISHTEST_PROGRAM="${VARNISHTEST_PROGRAM:-varnishtest}"
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100298TESTDIR="${TMPDIR:-/tmp}"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100299REGTESTS=""
300
301jobcount=""
302verbose="-q"
Christopher Faulet2a7cf922018-12-19 10:22:01 +0100303debug=""
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100304keep_logs="-l"
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100305no_htx="#"
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100306testlist=""
307
308_process "$@";
309
310echo ""
311echo "########################## Preparing to run tests ##########################"
PiBa-NL72504042018-11-27 22:26:38 +0100312
313preparefailed=
314if ! [ -x "$(command -v $HAPROXY_PROGRAM)" ]; then
315 echo "haproxy not found in path, please specify HAPROXY_PROGRAM environment variable"
316 preparefailed=1
317fi
318if ! [ -x "$(command -v $VARNISHTEST_PROGRAM)" ]; then
319 echo "varnishtest not found in path, please specify VARNISHTEST_PROGRAM environment variable"
320 preparefailed=1
321fi
322if [ $preparefailed ]; then
323 exit 1
324fi
325
326{ read HAPROXY_VERSION; read TARGET; read OPTIONS; } << EOF
327$($HAPROXY_PROGRAM -vv |grep 'HA-Proxy version\|TARGET\|OPTIONS' | sed 's/.* = //')
328EOF
329
330HAPROXY_VERSION=$(echo $HAPROXY_VERSION | cut -d " " -f 3)
331echo "Testing with haproxy version: $HAPROXY_VERSION"
332
333TESTRUNDATETIME="$(date '+%Y-%m-%d_%H-%M-%S')"
334
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100335mkdir -p "$TESTDIR" || exit 1
Christopher Faulet8d0fdf52018-12-19 10:18:40 +0100336TESTDIR=$(mktemp -d "$TESTDIR/haregtests-$TESTRUNDATETIME.XXXXXX") || exit 1
PiBa-NL72504042018-11-27 22:26:38 +0100337
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100338export TMPDIR="$TESTDIR"
339export HAPROXY_PROGRAM="$HAPROXY_PROGRAM"
PiBa-NL72504042018-11-27 22:26:38 +0100340
341# Mimic implicit build options from haproxy MakeFile that are present for each target:
342
343if [ $TARGET = generic ] ; then
344 #generic system target has nothing specific
345 OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1"
346fi
347if [ $TARGET = haiku ] ; then
348 #For Haiku
349 OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1"
350fi
351if [ $TARGET = linux22 ] ; then
352 #This is for Linux 2.2
353 OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1 USE_LIBCRYPT=1 USE_DL=1 USE_RT=1"
354fi
355if [ $TARGET = linux24 ] ; then
356 #This is for standard Linux 2.4 with netfilter but without epoll()
357 OPTIONS="$OPTIONS USE_NETFILTER=1 USE_POLL=1 USE_TPROXY=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 USE_DL=1 USE_RT=1"
358fi
359if [ $TARGET = linux24e ] ; then
360 #This is for enhanced Linux 2.4 with netfilter and epoll() patch>0.21
361 OPTIONS="$OPTIONS USE_NETFILTER=1 USE_POLL=1 USE_EPOLL=1 USE_MY_EPOLL=1 USE_TPROXY=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 USE_DL=1 USE_RT=1"
362fi
363if [ $TARGET = linux26 ] ; then
364 #This is for standard Linux 2.6 with netfilter and standard epoll()
365 OPTIONS="$OPTIONS USE_NETFILTER=1 USE_POLL=1 USE_EPOLL=1 USE_TPROXY=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 USE_FUTEX=1 USE_DL=1 USE_RT=1"
366fi
367if [ $TARGET = linux2628 ] ; then
368 #This is for standard Linux >= 2.6.28 with netfilter, epoll, tproxy and splice
369 OPTIONS="$OPTIONS USE_NETFILTER=1 USE_POLL=1 USE_EPOLL=1 USE_TPROXY=1 USE_CRYPT_H=1 USE_LIBCRYPT=1 USE_LINUX_SPLICE=1 USE_LINUX_TPROXY=1 USE_ACCEPT4=1 USE_FUTEX=1 USE_CPU_AFFINITY=1 ASSUME_SPLICE_WORKS=1 USE_DL=1 USE_RT=1 USE_THREAD=1"
370fi
371if [ $TARGET = solaris ] ; then
372 #This is for Solaris8
373 OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1 USE_LIBCRYPT=1 USE_CRYPT_H=1 USE_GETADDRINFO=1 USE_THREAD=1"
374fi
375if [ $TARGET = freebsd ] ; then
376 #This is for FreeBSD
377 OPTIONS="$OPTIONS USE_POLL=1 USE_KQUEUE=1 USE_TPROXY=1 USE_LIBCRYPT=1 USE_THREAD=1 USE_CPU_AFFINITY=1"
378fi
379if [ $TARGET = osx ] ; then
380 #This is for MacOS/X
381 OPTIONS="$OPTIONS USE_POLL=1 USE_KQUEUE=1 USE_TPROXY=1"
382fi
383if [ $TARGET = openbsd ] ; then
384 #This is for OpenBSD >= 5.7
385 OPTIONS="$OPTIONS USE_POLL=1 USE_KQUEUE=1 USE_TPROXY=1 USE_ACCEPT4=1 USE_THREAD=1"
386fi
387if [ $TARGET = netbsd ] ; then
388 #This is for NetBSD
389 OPTIONS="$OPTIONS USE_POLL=1 USE_KQUEUE=1 USE_TPROXY=1"
390fi
391if [ $TARGET = aix51 ] ; then
392 #This is for AIX 5.1
393 OPTIONS="$OPTIONS USE_POLL=1 USE_LIBCRYPT=1"
394fi
395if [ $TARGET = aix52 ] ; then
396 #This is for AIX 5.2 and later
397 OPTIONS="$OPTIONS USE_POLL=1 USE_LIBCRYPT=1"
398fi
399if [ $TARGET = cygwin ] ; then
400 #This is for Cygwin
401 OPTIONS="$OPTIONS USE_POLL=1 USE_TPROXY=1"
402fi
403
404echo "Target : $TARGET"
405echo "Options : $OPTIONS"
406
407echo "########################## Gathering tests to run ##########################"
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100408# if 'use-htx' option is set, but HAProxy version is lower to 1.9, disable it
409if [ -z "$no_htx" ]; then
410 if [ $(_version "$HAPROXY_VERSION") -lt $(_version "1.9") ]; then
411 echo ""
412 echo "WARNING : Unset HTX for haproxy (version: $HAPROXY_VERSION)"
413 echo " REASON: this test requires at least version: 1.9"
414 echo ""
415 no_htx="#"
416 fi
417fi
418
Christopher Faulet8d67cf82018-12-18 22:41:20 +0100419if [ -z "$REGTESTS" ]; then
420 _findtests ./
421else
422 for t in $REGTESTS; do
423 _findtests $t
424 done
425fi
PiBa-NL72504042018-11-27 22:26:38 +0100426
427echo "########################## Starting varnishtest ##########################"
428echo "Testing with haproxy version: $HAPROXY_VERSION"
429_vtresult=0
430if [ -n "$testlist" ]; then
431 if [ -n "$jobcount" ]; then
432 jobcount="-j $jobcount"
433 fi
Christopher Fauletfa6798f2018-12-19 11:22:04 +0100434 cmd="$VARNISHTEST_PROGRAM -k -t 10 -Dno-htx=${no_htx} $keep_logs $verbose $debug $jobcount $varnishtestparams $testlist"
Christopher Faulet1ecf0ea2018-12-18 22:47:23 +0100435 eval $cmd
PiBa-NL72504042018-11-27 22:26:38 +0100436 _vtresult=$?
437else
438 echo "No tests found that meet the required criteria"
439fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100440
441
442if [ $_vtresult -eq 0 ]; then
443 # all tests were succesfull, removing tempdir (the last part.)
444 # ignore errors is the directory is not empty or if it does not exist
445 rmdir "$TESTDIR" 2>/dev/null
446fi
447
448if [ -d "${TESTDIR}" ]; then
449 echo "########################## Gathering results ##########################"
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100450 export TESTDIR
451 find "$TESTDIR" -type d -name "vtc.*" -exec sh -c 'for i; do
452 if [ ! -e "$i/LOG" ] ; then continue; fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100453
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100454 cat <<- EOF | tee -a "$TESTDIR/failedtests.log"
455$(echo "###### $(cat "$i/INFO") ######")
456$(echo "## test results in: \"$i\"")
Christopher Faulet6bd82cd2018-12-19 11:28:49 +0100457$(grep -- ^---- "$i/LOG")
PiBa-NL72504042018-11-27 22:26:38 +0100458EOF
Frédéric Lécaille51e01b52018-11-29 21:41:42 +0100459 done' sh {} +
PiBa-NL72504042018-11-27 22:26:38 +0100460fi
Christopher Faulet9c6df5e2018-12-19 10:25:07 +0100461
462exit $_vtresult