Commit 9dcf2296 authored by Michal Nowak's avatar Michal Nowak

Merge branch 'mnowak/try-harder-to-analyze-cores' into 'main'

Rationalize backtrace logging, fail on core file presence

See merge request !3867
parents f27c0c32 1b13123c
Pipeline #47159 canceled with stages
in 15 minutes and 13 seconds
...@@ -131,17 +131,13 @@ start_servers() { ...@@ -131,17 +131,13 @@ start_servers() {
fi fi
} }
stop_servers_failed() {
echoinfo "I:$systest:stopping servers failed"
echofail "R:$systest:FAIL"
echoend "E:$systest:$(date_with_args)"
exit 1
}
stop_servers() { stop_servers() {
if $stopservers; then if $stopservers; then
echoinfo "I:$systest:stopping servers" echoinfo "I:$systest:stopping servers"
$PERL stop.pl "$systest" || stop_servers_failed if ! $PERL stop.pl "$systest"; then
echoinfo "I:$systest:stopping servers failed"
return 1
fi
fi fi
} }
...@@ -217,7 +213,7 @@ if [ -r "$systest/tests.sh" ]; then ...@@ -217,7 +213,7 @@ if [ -r "$systest/tests.sh" ]; then
( cd "$systest" && $SHELL tests.sh "$@" ) ( cd "$systest" && $SHELL tests.sh "$@" )
status=$? status=$?
run=$((run+1)) run=$((run+1))
stop_servers stop_servers || status=1
fi fi
if [ -n "$PYTEST" ]; then if [ -n "$PYTEST" ]; then
...@@ -232,7 +228,7 @@ if [ -n "$PYTEST" ]; then ...@@ -232,7 +228,7 @@ if [ -n "$PYTEST" ]; then
test_status=$(cat "$systest/$test.status") test_status=$(cat "$systest/$test.status")
fi fi
status=$((status+test_status)) status=$((status+test_status))
stop_servers stop_servers || status=1
done done
else else
echoinfo "I:$systest:pytest not installed, skipping python tests" echoinfo "I:$systest:pytest not installed, skipping python tests"
...@@ -250,54 +246,64 @@ else ...@@ -250,54 +246,64 @@ else
exit $status exit $status
fi fi
if [ $status != 0 ]; then get_core_dumps() {
find "$systest/" \( -name 'core*' -or -name '*.core' \) ! -name '*.gz' ! -name '*.txt' | sort
}
core_dumps=$(get_core_dumps | tr '\n' ' ')
assertion_failures=$(find "$systest/" -name named.run -print0 | xargs -0 grep "assertion failure" | wc -l)
sanitizer_summaries=$(find "$systest/" -name 'tsan.*' | wc -l)
if [ -n "$core_dumps" ]; then
status=1
echoinfo "I:$systest:Core dump(s) found: $core_dumps"
get_core_dumps | while read -r coredump; do
export SYSTESTDIR="$systest"
echoinfo "D:$systest:backtrace from $coredump:"
echoinfo "D:$systest:--------------------------------------------------------------------------------"
binary=$(gdb --batch --core="$coredump" 2>/dev/null | sed -ne "s/Core was generated by \`//;s/ .*'.$//p;")
"${top_builddir}/libtool" --mode=execute gdb \
-batch \
-ex bt \
-core="$coredump" \
-- \
"$binary" 2>/dev/null | sed -n '/^Core was generated by/,$p' | cat_d
echoinfo "D:$systest:--------------------------------------------------------------------------------"
coredump_backtrace=$(basename "${coredump}")-backtrace.txt
echoinfo "D:$systest:full backtrace from $coredump saved in $coredump_backtrace"
"${top_builddir}/libtool" --mode=execute gdb \
-batch \
-command=run.gdb \
-core="$coredump" \
-- \
"$binary" > "$coredump_backtrace" 2>&1
echoinfo "D:$systest:core dump $coredump archived as $coredump.gz"
gzip -1 "${coredump}"
done
elif [ "$assertion_failures" -ne 0 ]; then
status=1
SYSTESTDIR="$systest"
echoinfo "I:$systest:$assertion_failures assertion failure(s) found"
find "$systest/" -name 'tsan.*' -print0 | xargs -0 grep "SUMMARY: " | sort -u | cat_d
elif [ "$sanitizer_summaries" -ne 0 ]; then
status=1
echoinfo "I:$systest:$sanitizer_summaries sanitizer report(s) found"
fi
if [ $status -ne 0 ]; then
echofail "R:$systest:FAIL" echofail "R:$systest:FAIL"
# Do not clean up - we need the evidence.
else else
core_dumps="$(find "$systest/" -name 'core*' -or -name '*.core' | sort | tr '\n' ' ')" echopass "R:$systest:PASS"
assertion_failures=$(find "$systest/" -name named.run -print0 | xargs -0 grep "assertion failure" | wc -l) if $clean; then
sanitizer_summaries=$(find "$systest/" -name 'tsan.*' | wc -l) ( cd "${systest}" && $SHELL clean.sh "$@" )
if [ -n "$core_dumps" ]; then if [ "${srcdir}" != "${builddir}" ]; then
echoinfo "I:$systest:Test claims success despite crashes: $core_dumps" rm -rf "./${systest}" ## FIXME (this also removes compiled binaries)
echofail "R:$systest:FAIL" fi
# Do not clean up - we need the evidence. if test -d ${srcdir}/../../../.git; then
find "$systest/" -name 'core*' -or -name '*.core' | while read -r coredump; do git status -su --ignored "${systest}" 2>/dev/null | \
export SYSTESTDIR="$systest" sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \
echoinfo "D:$systest:backtrace from $coredump start" -e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \
binary=$(gdb --batch --core="$coredump" | sed -ne "s/Core was generated by \`//;s/ .*'.$//p;") -e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p'
"${top_builddir}/libtool" --mode=execute gdb \ fi
--batch \
--command=run.gdb \
--core="$coredump" \
-- \
"$binary"
echoinfo "D:$systest:backtrace from $coredump end"
done
elif [ "$assertion_failures" -ne 0 ]; then
SYSTESTDIR="$systest"
echoinfo "I:$systest:Test claims success despite $assertion_failures assertion failure(s)"
find "$systest/" -name 'tsan.*' -print0 | xargs -0 grep "SUMMARY: " | sort -u | cat_d
echofail "R:$systest:FAIL"
# Do not clean up - we need the evidence.
elif [ "$sanitizer_summaries" -ne 0 ]; then
echoinfo "I:$systest:Test claims success despite $sanitizer_summaries sanitizer reports(s)"
echofail "R:$systest:FAIL"
else
echopass "R:$systest:PASS"
if $clean
then
( cd "${systest}" && $SHELL clean.sh "$@" )
if [ "${srcdir}" != "${builddir}" ]; then
rm -rf "./${systest}" ## FIXME (this also removes compiled binaries)
fi
if test -d ${srcdir}/../../../.git
then
git status -su --ignored "${systest}" 2>/dev/null | \
sed -n -e 's|^?? \(.*\)|I:file \1 not removed|p' \
-e 's|^!! \(.*/named.run\)$|I:file \1 not removed|p' \
-e 's|^!! \(.*/named.memstats\)$|I:file \1 not removed|p'
fi
fi
fi fi
fi fi
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment