Commit d179d126 authored by Ondřej Surý's avatar Ondřej Surý

Cleanup the shell scripts used to run tests on Windows

parent 54a049d9
......@@ -51,27 +51,99 @@ SEQUENTIAL_COMMON=""
# rpzrecurse are scheduled first, in order to get more benefit from
# parallelism.
#
PARALLEL_COMMON="dnssec rpzrecurse serve-stale \
acl additional addzone allow-query auth autosign \
builtin cacheclean case catz cds chain \
checkconf checknames checkzone \
cookie database digdelv dlz dlzexternal \
dns64 dscp dsdigest dyndb \
ecdsa eddsa ednscompliance emptyzones \
fetchlimit filter-aaaa formerr forward \
geoip2 glue idna include-multiplecfg \
inline integrity ixfr \
kasp keepalive legacy limits \
masterfile masterformat metadata mirror mkeys \
names notify nslookup nsupdate nzd2nzf \
padding pending pipelined qmin \
reclimit redirect resolver rndc rootkeysentinel rpz \
rrchecker rrl rrsetorder rsabigexponent runtime \
sfcache smartsign sortlist \
spf staticstub statistics statschannel stub synthfromdnssec \
tcp tkey tools tsig tsiggss ttl \
unknown upforwd verify views wildcard \
xfer xferquota zero zonechecks"
PARALLEL_COMMON="dnssec rpzrecurse serve-stale
acl
additional
addzone
allow-query
auth
autosign
builtin
cacheclean
case
catz
cds
chain
checkconf
checknames
checkzone
cookie
database
digdelv
dlz
dlzexternal
dns64
dscp
dsdigest
dyndb
ecdsa
eddsa
ednscompliance
emptyzones
fetchlimit
filter-aaaa
formerr
forward
geoip2
glue
idna
include-multiplecfg
inline
integrity
ixfr
kasp
keepalive
legacy
limits
masterfile
masterformat
metadata
mirror
mkeys
names
notify
nslookup
nsupdate
nzd2nzf
padding
pending
pipelined
qmin
reclimit
redirect
resolver
rndc
rootkeysentinel
rpz
rrchecker
rrl
rrsetorder
rsabigexponent
runtime
sfcache
smartsign
sortlist
spf
staticstub
statistics
statschannel
stub
synthfromdnssec
tcp
tkey
tools
tsig
tsiggss
ttl
unknown
upforwd
verify
views
wildcard
xfer
xferquota
zero
zonechecks"
#
# Set up color-coded test output
......
......@@ -15,66 +15,64 @@
#
# Find the top of the BIND9 tree.
TOP=${SYSTEMTESTTOP:=.}/../../..
# Make it absolute so that it continues to work after we cd.
TOP=`cd $TOP && pwd`
TOP_BUILDDIR=@abs_top_builddir@
TOP_SRCDIR=@abs_top_srcdir@
# This is the windows build. This disables certain tests cases
# and changes some specific behaviors where necessary.
CYGWIN=1
# Load common values shared between windows and unix/linux.
. $TOP/bin/tests/system/conf.sh.common
. $TOP_BUILDDIR/bin/tests/system/conf.sh.common
# Visual Studio build configurations are Release and Debug
VSCONF=${VSCONF:-Debug}
ARPANAME=$TOP/Build/$VSCONF/arpaname@EXEEXT@
CDS=$TOP/Build/$VSCONF/dnssec-cds@EXEEXT@
CHECKCONF=$TOP/Build/$VSCONF/named-checkconf@EXEEXT@
CHECKZONE=$TOP/Build/$VSCONF/named-checkzone@EXEEXT@
DDNSCONFGEN=$TOP/Build/$VSCONF/ddns-confgen@EXEEXT@
DELV=$TOP/Build/$VSCONF/delv@EXEEXT@
DIG=$TOP/Build/$VSCONF/dig@EXEEXT@
DNSTAPREAD=$TOP/Build/$VSCONF/dnstap-read@EXEEXT@
DSFROMKEY=$TOP/Build/$VSCONF/dnssec-dsfromkey@EXEEXT@
FEATURETEST=$TOP/Build/$VSCONF/feature-test@EXEEXT@
ARPANAME=$TOP_BUILDDIR/Build/$VSCONF/arpaname@EXEEXT@
CDS=$TOP_BUILDDIR/Build/$VSCONF/dnssec-cds@EXEEXT@
CHECKCONF=$TOP_BUILDDIR/Build/$VSCONF/named-checkconf@EXEEXT@
CHECKZONE=$TOP_BUILDDIR/Build/$VSCONF/named-checkzone@EXEEXT@
DDNSCONFGEN=$TOP_BUILDDIR/Build/$VSCONF/ddns-confgen@EXEEXT@
DELV=$TOP_BUILDDIR/Build/$VSCONF/delv@EXEEXT@
DIG=$TOP_BUILDDIR/Build/$VSCONF/dig@EXEEXT@
DNSTAPREAD=$TOP_BUILDDIR/Build/$VSCONF/dnstap-read@EXEEXT@
DSFROMKEY=$TOP_BUILDDIR/Build/$VSCONF/dnssec-dsfromkey@EXEEXT@
FEATURETEST=$TOP_BUILDDIR/Build/$VSCONF/feature-test@EXEEXT@
FSTRM_CAPTURE=@FSTRM_CAPTURE@
IMPORTKEY=$TOP/Build/$VSCONF/dnssec-importkey@EXEEXT@
JOURNALPRINT=$TOP/Build/$VSCONF/named-journalprint@EXEEXT@
KEYFRLAB=$TOP/Build/$VSCONF/dnssec-keyfromlabel@EXEEXT@
KEYGEN=$TOP/Build/$VSCONF/dnssec-keygen@EXEEXT@
MDIG=$TOP/Build/$VSCONF/mdig@EXEEXT@
NAMED=$TOP/Build/$VSCONF/named@EXEEXT@
NSEC3HASH=$TOP/Build/$VSCONF/nsec3hash@EXEEXT@
NSLOOKUP=$TOP/Build/$VSCONF/nslookup@EXEEXT@
NSUPDATE=$TOP/Build/$VSCONF/nsupdate@EXEEXT@
NZD2NZF=$TOP/Build/$VSCONF/named-nzd2nzf@EXEEXT@
PK11DEL="$TOP/Build/$VSCONF/pkcs11-destroy@EXEEXT@ -s ${SLOT:-0} -p ${HSMPIN:-1234} -w 0"
PK11GEN="$TOP/Build/$VSCONF/pkcs11-keygen@EXEEXT@ -q -s ${SLOT:-0} -p ${HSMPIN:-1234}"
PK11LIST="$TOP/Build/$VSCONF/pkcs11-list@EXEEXT@ -s ${SLOT:-0} -p ${HSMPIN:-1234}"
RESOLVE=$TOP/lib/samples/resolve
REVOKE=$TOP/Build/$VSCONF/dnssec-revoke@EXEEXT@
RNDC=$TOP/Build/$VSCONF/rndc@EXEEXT@
RNDCCONFGEN=$TOP/Build/$VSCONF/rndc-confgen@EXEEXT@
RRCHECKER=$TOP/Build/$VSCONF/named-rrchecker@EXEEXT@
SAMPLEUPDATE=$TOP/Build/$VSCONF/update@EXEEXT@
SETTIME=$TOP/Build/$VSCONF/dnssec-settime@EXEEXT@
SIGNER=$TOP/Build/$VSCONF/dnssec-signzone@EXEEXT@
TSIGKEYGEN=$TOP/Build/$VSCONF/tsig-keygen@EXEEXT@
VERIFY=$TOP/Build/$VSCONF/dnssec-verify@EXEEXT@
# to port WIRETEST=$TOP/Build/$VSCONF/wire_test@EXEEXT@
IMPORTKEY=$TOP_BUILDDIR/Build/$VSCONF/dnssec-importkey@EXEEXT@
JOURNALPRINT=$TOP_BUILDDIR/Build/$VSCONF/named-journalprint@EXEEXT@
KEYFRLAB=$TOP_BUILDDIR/Build/$VSCONF/dnssec-keyfromlabel@EXEEXT@
KEYGEN=$TOP_BUILDDIR/Build/$VSCONF/dnssec-keygen@EXEEXT@
MDIG=$TOP_BUILDDIR/Build/$VSCONF/mdig@EXEEXT@
NAMED=$TOP_BUILDDIR/Build/$VSCONF/named@EXEEXT@
NSEC3HASH=$TOP_BUILDDIR/Build/$VSCONF/nsec3hash@EXEEXT@
NSLOOKUP=$TOP_BUILDDIR/Build/$VSCONF/nslookup@EXEEXT@
NSUPDATE=$TOP_BUILDDIR/Build/$VSCONF/nsupdate@EXEEXT@
NZD2NZF=$TOP_BUILDDIR/Build/$VSCONF/named-nzd2nzf@EXEEXT@
PK11DEL="$TOP_BUILDDIR/Build/$VSCONF/pkcs11-destroy@EXEEXT@ -s ${SLOT:-0} -p ${HSMPIN:-1234} -w 0"
PK11GEN="$TOP_BUILDDIR/Build/$VSCONF/pkcs11-keygen@EXEEXT@ -q -s ${SLOT:-0} -p ${HSMPIN:-1234}"
PK11LIST="$TOP_BUILDDIR/Build/$VSCONF/pkcs11-list@EXEEXT@ -s ${SLOT:-0} -p ${HSMPIN:-1234}"
RESOLVE=$TOP_BUILDDIR/lib/samples/resolve
REVOKE=$TOP_BUILDDIR/Build/$VSCONF/dnssec-revoke@EXEEXT@
RNDC=$TOP_BUILDDIR/Build/$VSCONF/rndc@EXEEXT@
RNDCCONFGEN=$TOP_BUILDDIR/Build/$VSCONF/rndc-confgen@EXEEXT@
RRCHECKER=$TOP_BUILDDIR/Build/$VSCONF/named-rrchecker@EXEEXT@
SAMPLEUPDATE=$TOP_BUILDDIR/Build/$VSCONF/update@EXEEXT@
SETTIME=$TOP_BUILDDIR/Build/$VSCONF/dnssec-settime@EXEEXT@
SIGNER=$TOP_BUILDDIR/Build/$VSCONF/dnssec-signzone@EXEEXT@
TSIGKEYGEN=$TOP_BUILDDIR/Build/$VSCONF/tsig-keygen@EXEEXT@
VERIFY=$TOP_BUILDDIR/Build/$VSCONF/dnssec-verify@EXEEXT@
# to port WIRETEST=$TOP_BUILDDIR/Build/$VSCONF/wire_test@EXEEXT@
WIRETEST=
BIGKEY=$TOP/Build/$VSCONF/bigkey@EXEEXT@
GENCHECK=$TOP/Build/$VSCONF/gencheck@EXEEXT@
KEYCREATE=$TOP/Build/$VSCONF/keycreate@EXEEXT@
KEYDELETE=$TOP/Build/$VSCONF/keydelete@EXEEXT@
MAKEJOURNAL=$TOP/Build/$VSCONF/makejournal@EXEEXT@
PIPEQUERIES=$TOP/Build/$VSCONF/pipequeries@EXEEXT@
# to port SAMPLEUPDATE=$TOP/lib/samples/sample-update
BIGKEY=$TOP_BUILDDIR/Build/$VSCONF/bigkey@EXEEXT@
GENCHECK=$TOP_BUILDDIR/Build/$VSCONF/gencheck@EXEEXT@
KEYCREATE=$TOP_BUILDDIR/Build/$VSCONF/keycreate@EXEEXT@
KEYDELETE=$TOP_BUILDDIR/Build/$VSCONF/keydelete@EXEEXT@
MAKEJOURNAL=$TOP_BUILDDIR/Build/$VSCONF/makejournal@EXEEXT@
PIPEQUERIES=$TOP_BUILDDIR/Build/$VSCONF/pipequeries@EXEEXT@
# to port SAMPLEUPDATE=$TOP_BUILDDIR/lib/samples/sample-update
# we don't want a KRB5_CONFIG setting breaking the tests
KRB5_CONFIG=NUL
......@@ -120,6 +118,3 @@ HAVEJSONSTATS=@JSONSTATS@
HAVEZLIB=@ZLIB@
NZD=@NZD_TOOLS@
CRYPTO=@CRYPTO@
# The rest is shared between Windows and Unices
. $TOP/bin/tests/system/conf.sh.common
......@@ -11,7 +11,7 @@
. ./conf.sh
PARALLELS=`echo $PARALLELDIRS | sed "s|\([^ ][^ ]*\)|test-\1|g;" | tr _ -`
PARALLELS=$(echo "$PARALLELDIRS" | sed "s|\([^ ][^ ]*\)|test-\1|g;" | tr _ - | tr "\n" " ")
echo ".PHONY: $PARALLELS"
echo
......@@ -28,7 +28,7 @@ echo "test check: $PARALLELS"
port=${STARTPORT:-5000}
for directory in $PARALLELDIRS ; do
echo
echo "test-`echo $directory | tr _ -`: check_interfaces"
echo " @${SHELL} ./run.sh -p $port $directory 2>&1 | tee test.output.$directory"
port=`expr $port + 100`
echo "test-$(echo "$directory" | tr _ -): check_interfaces"
echo " @${SHELL} ./run.sh -r -p $port $directory 2>&1 | tee test.output.$directory"
port=$((port+100))
done
......@@ -25,7 +25,7 @@
# run sequentially on that platform.)
SYSTEMTESTTOP=.
. $SYSTEMTESTTOP/conf.sh
. "$SYSTEMTESTTOP/conf.sh"
usage="Usage: ./runall.sh [-c] [-n] [numprocesses]"
......@@ -36,22 +36,23 @@ SYSTEMTEST_NO_CLEAN=${SYSTEMTEST_NO_CLEAN:-0}
# Handle command line switches if present.
while getopts "cn" flag; do
while getopts "cn-" flag; do
case "$flag" in
c) SYSTEMTEST_FORCE_COLOR=1 ;;
n) SYSTEMTEST_NO_CLEAN=1 ;;
-) break;;
*) exit 1;;
esac
done
export NOCLEAN
shift `expr $OPTIND - 1`
shift $((OPTIND-1))
# Obtain number of processes to use.
if [ $# -eq 0 ]; then
numproc=1
elif [ $# -eq 1 ]; then
test "$1" -eq "$1" > /dev/null 2>&1
if [ $? -ne 0 ]; then
if [ "$1" -ne "$1" ] 2>&1; then
# Value passed is not numeric
echo "$usage" >&2
exit 1
......@@ -72,14 +73,14 @@ status=0
if [ "$NOPARALLEL" = "" ]; then
if [ "$CYGWIN" = "" ]; then
# Running on Unix, use "make" to run tests in parallel.
make -j $numproc check
make -j "$numproc" check
status=$?
else
# Running on Windows: Cygwin "make" is available, but isn't being
# used for the build. So we create a special makefile for the purpose
# of parallel execution of system tests, and use that.
$SHELL parallel.sh > parallel.mk
make -f parallel.mk -j $numproc check
make -f parallel.mk -j "$numproc" check
$SHELL ./runsequential.sh
$SHELL ./testsummary.sh || status=1
fi
......@@ -95,11 +96,15 @@ else
EOF
exit 1
}
{
(
status=0
for testdir in $SUBDIRS; do
$SHELL run.sh $testdir || status=1
$SHELL run.sh -r "$testdir" || status=1
done
} 2>&1 | tee "systests.output"
echo "$status" > systests.status
) 2>&1 | tee "systests.output"
read -r status < systests.status
rm systests.status
fi
exit $status
exit "$status"
......@@ -17,9 +17,9 @@
#
SYSTEMTESTTOP=.
. $SYSTEMTESTTOP/conf.sh
. "$SYSTEMTESTTOP/conf.sh"
for d in $SEQUENTIALDIRS
do
$SHELL run.sh "${@}" $d 2>&1 | tee test.output.$d
$SHELL run.sh "${@}" "$d" 2>&1 | tee "test.output.$d"
done
......@@ -25,17 +25,18 @@
# 1 - one or more tests failed
SYSTEMTESTTOP=.
. $SYSTEMTESTTOP/conf.sh
. "$SYSTEMTESTTOP/conf.sh"
keepfile=0
while getopts "n" flag; do
case $flag in
n) keepfile=1 ;;
*) exit 1 ;;
esac
done
if [ `ls test.output.* 2> /dev/null | wc -l` -eq 0 ]; then
if [ "$(find . -name 'test.output.*' 2>/dev/null | wc -l)" -eq 0 ]; then
echowarn "I:No 'test.output.*' files were found."
echowarn "I:Printing summary from pre-existing 'systests.output'."
else
......@@ -45,37 +46,42 @@ else
fi
fi
if [ ! -f systests.output ]; then
echowarn "I:No 'systests.output' file found."
exit 1
fi
status=0
echoinfo "I:System test result summary:"
echoinfo "`grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/'`"
echoinfo "$(grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/')"
FAILED_TESTS=`grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output | cut -d':' -f2 | sort | sed -e 's/^/I: /'`
FAILED_TESTS=$(grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output | cut -d':' -f2 | sort | sed -e 's/^/I: /')
if [ -n "${FAILED_TESTS}" ]; then
echoinfo "I:The following system tests failed:"
echoinfo "${FAILED_TESTS}"
status=1
fi
CRASHED_TESTS=`find . -name 'core*' -or -name '*.core' | cut -d'/' -f2 | sort -u | sed -e 's/^/I: /'`
CRASHED_TESTS=$(find . -name 'core*' -or -name '*.core' | cut -d'/' -f2 | sort -u | sed -e 's/^/I: /')
if [ -n "${CRASHED_TESTS}" ]; then
echoinfo "I:Core dumps were found for the following system tests:"
echoinfo "${CRASHED_TESTS}"
fi
ASSERTION_FAILED_TESTS=`find . -name named.run | xargs grep "assertion failure" | cut -d'/' -f2 | sort -u | sed -e 's/^/I: /'`
ASSERTION_FAILED_TESTS=$(find . -name named.run -print0 | xargs -0 grep "assertion failure" | cut -d'/' -f2 | sort -u | sed -e 's/^/I: /')
if [ -n "${ASSERTION_FAILED_TESTS}" ]; then
echoinfo "I:Assertion failures were detected for the following system tests:"
echoinfo "${ASSERTION_FAILED_TESTS}"
fi
TSAN_REPORT_TESTS=`find . -name 'tsan.*' | cut -d'/' -f2 | sort -u | sed -e 's/^/I: /'`
TSAN_REPORT_TESTS=$(find . -name 'tsan.*' | cut -d'/' -f2 | sort -u | sed -e 's/^/I: /')
if [ -n "${TSAN_REPORT_TESTS}" ]; then
echoinfo "I:ThreadSanitizer reported issues for the following system tests:"
echoinfo "${TSAN_REPORT_TESTS}"
fi
RESULTS_FOUND=`grep -c 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output`
TESTS_RUN=`echo "${SUBDIRS}" | wc -w`
RESULTS_FOUND=$(grep -c 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output)
TESTS_RUN=$(echo "${SUBDIRS}" | wc -w)
if [ "${RESULTS_FOUND}" -ne "${TESTS_RUN}" ]; then
echofail "I:Found ${RESULTS_FOUND} test results, but ${TESTS_RUN} tests were run"
status=1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment