run.sh.in 7.99 KB
Newer Older
Andreas Gustafsson's avatar
Andreas Gustafsson committed
1
#!/bin/sh
Michael Sawyer's avatar
Michael Sawyer committed
2
#
3
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
Mark Andrews's avatar
Mark Andrews committed
4
#
5 6 7
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 9 10
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
Mark Andrews's avatar
Mark Andrews committed
11

Andreas Gustafsson's avatar
Andreas Gustafsson committed
12 13 14
#
# Run a system test.
#
15

16 17 18
top_builddir=@top_builddir@
builddir=@abs_builddir@
srcdir=@abs_srcdir@
Andreas Gustafsson's avatar
Andreas Gustafsson committed
19

20
# shellcheck source=conf.sh
21 22 23
. ${builddir}/conf.sh

SYSTEMTESTTOP="$(cd -P -- "${builddir}" && pwd -P)"
24
export SYSTEMTESTTOP
25 26 27 28 29 30
export builddir
export srcdir

date_with_args() (
    date -R
)
31

32
stopservers=true
33 34
# baseport == 0 means random
baseport=0
35

36
if [ "${SYSTEMTEST_NO_CLEAN:-0}" -eq 1 ]; then
37 38 39 40 41
	clean=false
else
	clean=true
fi

42 43
do_run=false
log_flags="-r"
44
while getopts "sknp:r-:" OPT; do
45 46 47 48 49 50 51
	log_flags="$log_flags -$OPT$OPTARG"
	if [ "$OPT" = "-" ] && [ -n "$OPTARG" ]; then
		OPT="${OPTARG%%=*}"
		OPTARG="${OPTARG#$OPT}"
		OPTARG="${OPTARG#=}"
	fi

52
	# shellcheck disable=SC2214
53 54 55 56 57 58 59 60 61
	case "$OPT" in
		k | keep) stopservers=false ;;
		n | noclean) clean=false ;;
		p | port) baseport=$OPTARG ;;
		r | run) do_run=true ;;
		s | skip) exit 77 ;;
		-) break ;;
		*) echo "invalid option" >&2; exit 1 ;;
	esac
62
done
63

64
shift $((OPTIND-1))
65

66
if ! $do_run; then
67 68 69 70 71
    if [ "$baseport" -eq 0 ]; then
	log_flags="$log_flags -p 5300"
    fi
    env - TESTS="$*" TEST_SUITE_LOG=run.log LOG_DRIVER_FLAGS="--verbose yes --color-tests yes" LOG_FLAGS="$log_flags" make -e check
    exit $?
72 73
fi

74
if [ $# -eq 0 ]; then
75 76
	echofail "Usage: $0 [-k] [-n] [-p <PORT>] test-directory [test-options]" >&2;
	exit 1
77
fi
Andreas Gustafsson's avatar
Andreas Gustafsson committed
78

79
systest=$(basename "${1%%/}")
Andreas Gustafsson's avatar
Andreas Gustafsson committed
80 81
shift

82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
if [ ! -d "${srcdir}/$systest" ]; then
    echofail "$0: $systest: no such test" >&2
    exit 1
fi

if [ "${srcdir}" != "${builddir}" ]; then
    if [ ! -d common ] || [ ! -r common/.prepared ]; then
	cp -a "${srcdir}/common" "${builddir}"
    fi
    if [ ! -d "$systest" ] || [ ! -r "$systest/.prepared" ]; then
	mkdir -p "${builddir}/$systest"
	cp -a "${srcdir}/$systest" "${builddir}/"
	touch "$systest/.prepared"
    fi
fi

if [ ! -d "${systest}" ]; then
99
    echofail "$0: $systest: no such test" >&2
100 101
    exit 1
fi
Andreas Gustafsson's avatar
Andreas Gustafsson committed
102

103

104
# Get the first 10 ports in the set (it is assumed that each test has access
105 106 107 108
# to ten or more ports): the query port, the control port and eight extra
# ports.  Since the lowest numbered port (specified in the command line)
# will usually be a multiple of 10, the names are chosen so that if this is
# true, the last digit of EXTRAPORTn is "n".
109
eval "$(${srcdir}/get_ports.sh -p "$baseport")"
110

Ondřej Surý's avatar
Ondřej Surý committed
111 112 113 114 115 116 117 118 119 120 121 122
restart=false

start_servers_failed() {
    echoinfo "I:$systest:starting servers failed"
    echofail "R:$systest:FAIL"
    echoend  "E:$systest:$(date_with_args)"
    exit 1
}

start_servers() {
    echoinfo "I:$systest:starting servers"
    if $restart; then
123
        $PERL start.pl --restart --port "$PORT" "$systest" || start_servers_failed
Ondřej Surý's avatar
Ondřej Surý committed
124 125
    else
        restart=true
126
        $PERL start.pl --port "$PORT" "$systest" || start_servers_failed
Ondřej Surý's avatar
Ondřej Surý committed
127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
    fi
}

stop_servers_failed() {
    echoinfo "I:$systest:stopping servers failed"
    echofail "R:$systest:FAIL"
    echoend  "E:$systest:$(date_with_args)"
    exit 1
}

stop_servers() {
    if $stopservers; then
        echoinfo "I:$systest:stopping servers"
        $PERL stop.pl "$systest" || stop_servers_failed
    fi
}

144
echostart "S:$systest:$(date_with_args)"
145 146
echoinfo  "T:$systest:1:A"
echoinfo  "A:$systest:System test $systest"
147
echoinfo  "I:$systest:PORTS:${PORT},${EXTRAPORT1},${EXTRAPORT2},${EXTRAPORT3},${EXTRAPORT4},${EXTRAPORT5},${EXTRAPORT6},${EXTRAPORT7},${EXTRAPORT8},${CONTROLPORT}"
148

149
$PERL ${srcdir}/testsock.pl -p "$PORT"  || {
150
    echowarn "I:$systest:Network interface aliases not set up.  Skipping test."
151 152 153
    echowarn "R:$systest:FAIL"
    echoend  "E:$systest:$(date_with_args)"
    exit 1;
154 155
}

Andreas Gustafsson's avatar
typo  
Andreas Gustafsson committed
156
# Check for test-specific prerequisites.
157
test ! -f "$systest/prereq.sh" || ( cd "${systest}" && $SHELL prereq.sh "$@" )
158 159 160
result=$?

if [ $result -eq 0 ]; then
161 162
    : prereqs ok
else
163
    echowarn "I:$systest:Prerequisites missing, skipping test."
164 165 166 167 168 169
    if [ $result -eq 255 ]; then
	echowarn "R:$systest:SKIPPED";
    else
	echowarn "R:$systest:UNTESTED"
    fi
    echoend "E:$systest:$(date_with_args)"
170 171 172
    exit 0
fi

173 174
# Check for PKCS#11 support
if
175
    test ! -f "$systest/usepkcs11" || $SHELL cleanpkcs11.sh
176 177 178
then
    : pkcs11 ok
else
179 180
    echowarn "I:$systest:Need PKCS#11, skipping test."
    echowarn "R:$systest:PKCS11ONLY"
181
    echoend  "E:$systest:$(date_with_args)"
182 183 184
    exit 0
fi

185
# Clean up files left from any potential previous runs
186
if test -f "$systest/clean.sh"
187
then
188 189 190 191 192 193
    ( cd "${systest}" && $SHELL clean.sh "$@" )
    ret=$?
    if [ $ret -ne 0 ]; then
	echowarn "I:$systest:clean.sh script failed with $ret"
    fi

194 195
fi

Andreas Gustafsson's avatar
Andreas Gustafsson committed
196
# Set up any dynamically generated test data
197
if test -f "$systest/setup.sh"
Andreas Gustafsson's avatar
Andreas Gustafsson committed
198
then
199 200 201 202 203
    ( cd "${systest}" && $SHELL setup.sh "$@" )
    ret=$?
    if [ $ret -ne 0 ]; then
	echowarn "I:$systest:clean.sh script failed with $ret"
    fi
Andreas Gustafsson's avatar
Andreas Gustafsson committed
204 205
fi

Ondřej Surý's avatar
Ondřej Surý committed
206 207
status=0
run=0
208
# Run the tests
Ondřej Surý's avatar
Ondřej Surý committed
209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238
if [ -r "$systest/tests.sh" ]; then
    start_servers
    ( cd "$systest" && $SHELL tests.sh "$@" )
    status=$?
    run=$((run+1))
    stop_servers
fi

if [ -n "$PYTEST" ]; then
    run=$((run+1))
    for test in $(cd "${systest}" && find . -name "tests*.py"); do
	start_servers
	rm -f "$systest/$test.status"
	test_status=0
	(cd "$systest" && "$PYTEST" -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
	if [ -f "$systest/$test.status" ]; then
	    echo_i "FAILED"
	    test_status=$(cat "$systest/$test.status")
	fi
	status=$((status+test_status))
	stop_servers
    done
else
    echoinfo "I:$systest:pytest not installed, skipping python tests"
fi

if [ "$run" -eq "0" ]; then
    echoinfo "I:$systest:No tests were found and run"
    status=255
fi
239

240
if $stopservers
241
then
242 243
    :
else
244 245 246
    exit $status
fi

247
if [ $status != 0 ]; then
248
    echofail "R:$systest:FAIL"
249
    # Do not clean up - we need the evidence.
250
else
251 252 253
    core_dumps="$(find "$systest/" -name 'core*' -or -name '*.core' | sort | tr '\n' ' ')"
    assertion_failures=$(find "$systest/" -name named.run -print0 | xargs -0 grep "assertion failure" | wc -l)
    sanitizer_summaries=$(find "$systest/" -name 'tsan.*' | wc -l)
254 255 256 257
    if [ -n "$core_dumps" ]; then
        echoinfo "I:$systest:Test claims success despite crashes: $core_dumps"
        echofail "R:$systest:FAIL"
        # Do not clean up - we need the evidence.
Ondřej Surý's avatar
Ondřej Surý committed
258
	find "$systest/" -name 'core*' -or -name '*.core' | while read -r coredump; do
259
		export SYSTESTDIR="$systest"
260 261
		echoinfo "D:$systest:backtrace from $coredump start"
		binary=$(gdb --batch --core="$coredump" | sed -ne "s/Core was generated by \`//;s/ .*'.$//p;")
262 263 264 265 266 267
		"${top_builddir}/libtool" --mode=execute gdb \
					  --batch \
					  --command=run.gdb \
					  --core="$coredump" \
					  -- \
					  "$binary"
268 269
		echoinfo "D:$systest:backtrace from $coredump end"
	done
270
    elif [ "$assertion_failures" -ne 0 ]; then
271
	SYSTESTDIR="$systest"
272
        echoinfo "I:$systest:Test claims success despite $assertion_failures assertion failure(s)"
273
	find "$systest/" -name 'tsan.*' -print0 | xargs -0 grep "SUMMARY: " | sort -u | cat_d
274 275
        echofail "R:$systest:FAIL"
        # Do not clean up - we need the evidence.
276
    elif [ "$sanitizer_summaries" -ne 0 ]; then
277 278
        echoinfo "I:$systest:Test claims success despite $sanitizer_summaries sanitizer reports(s)"
        echofail "R:$systest:FAIL"
279 280 281 282
    else
        echopass "R:$systest:PASS"
        if $clean
        then
283 284 285 286 287
            ( cd "${systest}" && $SHELL clean.sh "$@" )
	    if [ "${srcdir}" != "${builddir}" ]; then
		rm -rf "./${systest}" ## FIXME (this also removes compiled binaries)
	    fi
            if test -d ${srcdir}/../../../.git
288
            then
289
                git status -su --ignored "${systest}" 2>/dev/null | \
290 291 292 293 294
                sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \
                -e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \
                -e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p'
            fi
        fi
295
    fi
296
fi
297

298
echoend "E:$systest:$(date_with_args)"
299

300
exit $status