Commit 66ed50fb authored by Evan Hunt's avatar Evan Hunt

improve test display, preserve artifacts

- when tests fail, store artifacts from initial run in failed-tests.tar
- store the original test summary output
- after retry, display both the original and the new test summary
- tweak echo_i and cat_i functions to get output spacing and colors right
parent 1607d587
Pipeline #9584 passed with stages
in 11 minutes and 51 seconds
dig.out*
failed-tests.tar
rndc.out*
nsupdate.out*
named.lock
......
......@@ -108,9 +108,9 @@ parallel.mk:
# Targets to run the tests.
test: parallel.mk subdirs
@$(MAKE) -f parallel.mk check
@$(SHELL) ./runsequential.sh -r
@$(SHELL) ./testsummary.sh
+@$(MAKE) -f parallel.mk check
+@$(SHELL) ./testsummary.sh || $(SHELL) ./retry.sh
check: test
......@@ -123,6 +123,8 @@ check: test
testclean clean distclean::
if test -f ./cleanall.sh; then $(SHELL) ./cleanall.sh; fi
rm -f systests.output
rm -f summary.prev
rm -f failed-tests.tar
rm -f random.data
rm -f parallel.mk
......
......@@ -91,32 +91,32 @@ fi
SYSTESTDIR="`basename $PWD`"
echo_i() {
echo "$@" | while read __LINE ; do
echoinfo "I:$SYSTESTDIR:$__LINE"
echo "$@" | while IFS= read -r __LINE ; do
echoinfo "I:$SYSTESTDIR${SYSTESTDIR:+:}$__LINE"
done
}
echo_ic() {
echo "$@" | while read __LINE ; do
echoinfo "I:$SYSTESTDIR: $__LINE"
echoinfo "I:$SYSTESTDIR${SYSTESTDIR:+:} $__LINE"
done
}
cat_i() {
while read __LINE ; do
echoinfo "I:$SYSTESTDIR:$__LINE"
while IFS= read -r __LINE ; do
echoinfo "I:$SYSTESTDIR${SYSTESTDIR:+:}$__LINE"
done
}
echo_d() {
echo "$@" | while read __LINE ; do
echoinfo "D:$SYSTESTDIR:$__LINE"
echoinfo "D:$SYSTESTDIR${SYSTESTDIR:+:}$__LINE"
done
}
cat_d() {
while read __LINE ; do
echoinfo "D:$SYSTESTDIR:$__LINE"
echoinfo "D:$SYSTESTDIR${SYSTESTDIR:+:}$__LINE"
done
}
......
......@@ -24,18 +24,45 @@
SYSTEMTESTTOP=.
. $SYSTEMTESTTOP/conf.sh
numproc=${1:-1}
SYSTESTDIR=""
display () {
while IFS= read -r __LINE ; do
echoinfo "$__LINE"
done
}
if [ ! -f systests.output ]; then
echofail "I:'systests.output' not found."
exit 0
fi
status=0
# first, preserve artifacts from the tests that are failing
fails=$(grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output |
awk -F: 'START { print ". ./conf.sh" }
$1 == "R" && $3 == "FAIL" { printf "%s ", $2 }
END { print "" }')
# if there were no failed tests, we're done
[ -n "$fails" ] || exit 0
tar cf failed-tests.tar $fails
sh testsummary.sh > summary.prev
tar uf failed-tests.tar systests.output
echo_i "Test failures detected"
echo_i "Artifacts from failed tests stored in 'failed-tests.tar'"
echo_i "Rerunning failed tests:"
grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | \
awk -F: 'START { print ". ./conf.sh" }
$1 == "R" && $3 == "FAIL" { retests = retests " test-"$2; }
END { if (retests) { print "make -f parallel.mk -j " retests } }' | \
sed -e "s/NUMPROC/$numproc/" | \
$SHELL
$SHELL testsummary.sh
END { if (retests) { print "make -f parallel.mk " retests } }' | \
$SHELL | display
echo_i "Original test results (after first pass):"
cat summary.prev | display
rm -f summary.prev
echo_i "Updated test results (after second pass):"
sh testsummary.sh
......@@ -27,33 +27,36 @@
SYSTEMTESTTOP=.
. $SYSTEMTESTTOP/conf.sh
keepfile=0
SYSTESTDIR=""
while getopts "n" flag; do
keepfile=0 passes=0
while getopts "np" flag; do
case $flag in
n) keepfile=1 ;;
p) passes=1 ;;
esac
done
if [ `ls */test.output 2> /dev/null | wc -l` -eq 0 ]; then
echowarn "I:No 'test.output' files were found."
echowarn "I:Printing summary from pre-existing 'systests.output'."
else
if [ `ls */test.output 2> /dev/null | wc -l` -ne 0 ]; then
cat */test.output > systests.output
if [ $keepfile -eq 0 ]; then
rm -f */test.output
fi
fi
status=0
echoinfo "I:System test result summary:"
echoinfo "`grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | cut -d':' -f3 | sort | uniq -c | sed -e 's/^/I:/'`"
echo_i "System test result summary:"
grep 'R:[a-z0-9_-][a-z0-9_-]*:[A-Z][A-Z]*' systests.output | \
cut -d':' -f3 | sort | uniq -c | cat_i
FAILED_TESTS=`grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output | cut -d':' -f2 | sort | sed -e 's/^/I: /'`
if [ -n "${FAILED_TESTS}" ]; then
echoinfo "I:The following system tests failed:"
echoinfo "${FAILED_TESTS}"
status=1
if [ "$passes" -eq 1 ]; then
echo_i "The following system tests passed:"
grep 'R:[a-z0-9_-][a-z0-9_-]*:PASS' systests.output | \
cut -d':' -f2 | sort | sed 's/^/ /' | cat_i
fi
exit $status
grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output > /dev/null || exit 0
echo_i "The following system tests failed:"
grep 'R:[a-z0-9_-][a-z0-9_-]*:FAIL' systests.output | \
cut -d':' -f2 | sort | sed 's/^/ /' | cat_i
exit 1
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment