Commit cf510593 authored by Ondřej Surý's avatar Ondřej Surý Committed by Ondřej Surý

Add a pytest runner to run.sh

The system tests currently uses patchwork of shell scripts which doesn't
offer proper error handling.

This commit introduced option to write new tests in pytest framework
that also allows easier manipulation of DNS traffic (using dnspython),
native XML and JSON manipulation and proper error reporting.
parent 0bb2bda1
Pipeline #40397 passed with stages
in 45 minutes and 43 seconds
......@@ -14,6 +14,7 @@
*_test
*.ipch # vscode/intellisense precompiled header
*~
__pycache__/
.ccache/
.cproject
.deps/
......
......@@ -199,6 +199,7 @@ stages:
before_script:
- test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}"
- test -n "${OOT_BUILD_WORKSPACE}" && mkdir "${OOT_BUILD_WORKSPACE}" && cd "${OOT_BUILD_WORKSPACE}"
- pip3 install pytest requests || pip install pytest requests || true
script:
- *configure
- make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1
......@@ -256,6 +257,7 @@ stages:
before_script:
- *setup_interfaces
- *setup_softhsm
- pip3 install pytest requests || pip install pytest requests || true
script:
- cd bin/tests/system
- make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 || make -j${TEST_PARALLEL_JOBS:-1} -k recheck V=1
......@@ -441,6 +443,7 @@ pylint:
before_script:
- pip3 install pylint
- PYTHONPATH="$PYTHONPATH:$CI_PROJECT_DIR/bin/python"
- pip3 install pytest requests || pip install pytest requests
script:
- *configure
- pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -v 'ans\.py')
......
__pycache__
dig.out*
rndc.out*
nsupdate.out*
......
......@@ -112,6 +112,7 @@ PERL=$(command -v "@PERL@")
PSSUSPEND=
PYTHON=$(command -v "@PYTHON@")
PYTEST=@PYTEST@
#
# Determine if we support various optional features.
......@@ -127,3 +128,5 @@ HAVEZLIB=${ZLIB_LIBS:+1}
LMDB_LIBS="@LMDB_LIBS@"
NZD=${LMDB_LIBS:+1}
CRYPTO=@CRYPTO@
export HAVEXMLSTATS HAVEJSONSTATS
......@@ -108,6 +108,39 @@ fi
# true, the last digit of EXTRAPORTn is "n".
eval "$(${srcdir}/get_ports.sh -p "$baseport")"
restart=false
start_servers_failed() {
echoinfo "I:$systest:starting servers failed"
echofail "R:$systest:FAIL"
echoend "E:$systest:$(date_with_args)"
exit 1
}
start_servers() {
echoinfo "I:$systest:starting servers"
if $restart; then
$PERL start.pl --restart --port "$PORT" "$systest" || start_fail
else
restart=true
$PERL start.pl --port "$PORT" "$systest" || start_fail
fi
}
stop_servers_failed() {
echoinfo "I:$systest:stopping servers failed"
echofail "R:$systest:FAIL"
echoend "E:$systest:$(date_with_args)"
exit 1
}
stop_servers() {
if $stopservers; then
echoinfo "I:$systest:stopping servers"
$PERL stop.pl "$systest" || stop_servers_failed
fi
}
echostart "S:$systest:$(date_with_args)"
echoinfo "T:$systest:1:A"
echoinfo "A:$systest:System test $systest"
......@@ -161,16 +194,40 @@ then
( cd "${systest}" && $SHELL setup.sh "$@" )
fi
# Start name servers running
if ! $PERL ${srcdir}/start.pl --port "$PORT" "$systest"; then
echofail "R:$systest:FAIL"
echoend "E:$systest:$(date_with_args)"
exit 1
fi
status=0
run=0
# Run the tests
( cd "${systest}" && $SHELL tests.sh "$@" )
status=$?
if [ -r "$systest/tests.sh" ]; then
start_servers
( cd "$systest" && $SHELL tests.sh "$@" )
status=$?
run=$((run+1))
stop_servers
fi
if [ -n "$PYTEST" ]; then
run=$((run+1))
for test in $(cd "${systest}" && find . -name "tests*.py"); do
start_servers
rm -f "$systest/$test.status"
test_status=0
(cd "$systest" && "$PYTEST" -v "$test" "$@" || echo "$?" > "$test.status") | SYSTESTDIR="$systest" cat_d
if [ -f "$systest/$test.status" ]; then
echo_i "FAILED"
test_status=$(cat "$systest/$test.status")
fi
status=$((status+test_status))
stop_servers
done
else
echoinfo "I:$systest:pytest not installed, skipping python tests"
fi
if [ "$run" -eq "0" ]; then
echoinfo "I:$systest:No tests were found and run"
status=255
fi
if $stopservers
then
......@@ -179,11 +236,6 @@ else
exit $status
fi
# Shutdown
$PERL ${srcdir}/stop.pl "$systest"
status=$((status + $?))
if [ $status != 0 ]; then
echofail "R:$systest:FAIL"
# Do not clean up - we need the evidence.
......
......@@ -276,6 +276,11 @@ AM_CONDITIONAL([HAVE_PERLMOD_TIME_HIRES],
AM_PATH_PYTHON([3.4], [], [:])
AM_CONDITIONAL([HAVE_PYTHON], [test "$PYTHON" != ":"])
AC_PATH_PROGS([PYTEST], [pytest-3 pytest pytest-pypy], [])
AS_IF([test -z "$PYTEST"],
[AC_MSG_WARN([pytest not found, some system tests will be skipped])])
AC_SUBST([PYTEST])
AX_PYTHON_MODULE([dns])
AM_CONDITIONAL([HAVE_PYMOD_DNS], [test "$HAVE_PYMOD_DNS" = "yes"])
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment