Commit b232e858 authored by Michal Nowak's avatar Michal Nowak Committed by Michał Kępień

Ensure various test issues are treated as failures

Make sure bin/tests/system/run.sh returns a non-zero exit code if any of
the following happens:

  - the test being run produces a core dump,
  - assertion failures are found in the test's logs,
  - ThreadSanitizer reports are found after the test completes,
  - the servers started by the test fail to shut down cleanly.

This change is necessary to always fail a test in such cases (before the
migration to Automake, test failures were determined based on the
presence of "R:<test-name>:FAIL" lines in the test suite output and thus
it was not necessary for bin/tests/system/run.sh to return a non-zero
exit code).
parent f27c0c32
......@@ -131,17 +131,13 @@ start_servers() {
fi
}
stop_servers_failed() {
echoinfo "I:$systest:stopping servers failed"
echofail "R:$systest:FAIL"
echoend "E:$systest:$(date_with_args)"
exit 1
}
stop_servers() {
if $stopservers; then
echoinfo "I:$systest:stopping servers"
$PERL stop.pl "$systest" || stop_servers_failed
if ! $PERL stop.pl "$systest"; then
echoinfo "I:$systest:stopping servers failed"
return 1
fi
fi
}
......@@ -217,7 +213,7 @@ if [ -r "$systest/tests.sh" ]; then
( cd "$systest" && $SHELL tests.sh "$@" )
status=$?
run=$((run+1))
stop_servers
stop_servers || status=1
fi
if [ -n "$PYTEST" ]; then
......@@ -232,7 +228,7 @@ if [ -n "$PYTEST" ]; then
test_status=$(cat "$systest/$test.status")
fi
status=$((status+test_status))
stop_servers
stop_servers || status=1
done
else
echoinfo "I:$systest:pytest not installed, skipping python tests"
......@@ -258,6 +254,7 @@ else
assertion_failures=$(find "$systest/" -name named.run -print0 | xargs -0 grep "assertion failure" | wc -l)
sanitizer_summaries=$(find "$systest/" -name 'tsan.*' | wc -l)
if [ -n "$core_dumps" ]; then
status=1
echoinfo "I:$systest:Test claims success despite crashes: $core_dumps"
echofail "R:$systest:FAIL"
# Do not clean up - we need the evidence.
......@@ -274,12 +271,14 @@ else
echoinfo "D:$systest:backtrace from $coredump end"
done
elif [ "$assertion_failures" -ne 0 ]; then
status=1
SYSTESTDIR="$systest"
echoinfo "I:$systest:Test claims success despite $assertion_failures assertion failure(s)"
find "$systest/" -name 'tsan.*' -print0 | xargs -0 grep "SUMMARY: " | sort -u | cat_d
echofail "R:$systest:FAIL"
# Do not clean up - we need the evidence.
elif [ "$sanitizer_summaries" -ne 0 ]; then
status=1
echoinfo "I:$systest:Test claims success despite $sanitizer_summaries sanitizer reports(s)"
echofail "R:$systest:FAIL"
else
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment