Commit 12a331dc authored by Jeremy C. Reed's avatar Jeremy C. Reed
Browse files

For trac362

  139.  [build]         jreed
        Introduced configure option and make targets for generating
        Python code coverage report. This adds new make targets:
        report-python-coverage and clean-python-coverage. The C++
        code coverage targets were renamed to clean-cpp-coverage
        and report-cpp-coverage. (Trac #362, svn r4023)


git-svn-id: svn://bind10.isc.org/svn/bind10/trunk@4023 e5f2f494-b856-4b98-b285-d166d9295462
parents bf7abd12 8f985a64
139. [build] jreed
Introduced configure option and make targets for generating
Python code coverage report. This adds new make targets:
report-python-coverage and clean-python-coverage. The C++
code coverage targets were renamed to clean-cpp-coverage
and report-cpp-coverage. (Trac #362, svn r4023)
138. [func]* jinmei
b10-auth: added a configuration interface to support in memory
data sources. For example, the following command to bindctl
......
......@@ -8,20 +8,30 @@ DISTCLEANFILES = config.report
# When running distcheck target, do not install the configurations
DISTCHECK_CONFIGURE_FLAGS = --disable-install-configurations
clean-coverage:
clean-cpp-coverage:
@if [ $(USE_LCOV) = yes ] ; then \
$(LCOV) --directory . --zerocounters; \
rm -rf coverage/; \
else \
echo "Code coverage not enabled at configuration time"; \
exit 1; \
echo "C++ code coverage not enabled at configuration time." ; \
echo "Use: ./configure --with-lcov" ; \
fi
clean-python-coverage:
@if [ $(USE_PYCOVERAGE) = yes ] ; then \
rm -f $(abs_top_srcdir)/.coverage ; \
rm -rf $(abs_top_srcdir)/py-coverage-html ; \
else \
echo "Python code coverage not enabled at configuration time." ; \
echo "Use: ./configure --with-pycoverage" ; \
fi
perform-coverage: check
report-coverage:
$(LCOV) --capture --directory . --output-file all.info
$(LCOV) --remove all.info \
report-cpp-coverage:
@if [ $(USE_LCOV) = yes ] ; then \
$(LCOV) --capture --directory . --output-file all.info ; \
$(LCOV) --remove all.info \
c++/4.4\*/\* \
c++/4.4\*/backward/\* \
c++/4.4\*/bits/\* \
......@@ -36,11 +46,30 @@ report-coverage:
\*_unittests.cc \
\*_unittest.cc \
\*_unittests.h \
--output report.info
$(GENHTML) --legend -o coverage report.info
--output report.info ; \
$(GENHTML) --legend -o $(abs_top_builddir)/coverage-cpp-html report.info ; \
echo "Generated C++ Code Coverage report in HTML at $(abs_top_builddir)/coverage-cpp-html" ; \
else \
echo "C++ code coverage not enabled at configuration time." ; \
echo "Use: ./configure --with-lcov" ; \
fi
report-python-coverage:
@if [ $(USE_PYCOVERAGE) = yes ] ; then \
$(PYCOVERAGE) html -d $(abs_top_builddir)/coverage-python-html --omit=src/bin/bind10/tests/,src/bin/bindctl/tests/,src/bin/cfgmgr/tests/,src/bin/cmdctl/tests/,src/bin/loadzone/tests/,src/bin/msgq/tests/,src/bin/stats/tests/,src/bin/tests/,src/bin/xfrin/tests/,src/bin/xfrout/tests/,src/bin/zonemgr/tests/,src/lib/dns/python/tests/,src/lib/dns/tests/,src/lib/python/isc/cc/tests/,src/lib/python/isc/config/tests/,src/lib/python/isc/datasrc/tests/,src/lib/python/isc/log/tests/,src/lib/python/isc/net/tests/,src/lib/python/isc/notify/tests/,src/lib/python/isc/util/tests/ ; \
echo "Generated Python Code Coverage report in HTML at $(abs_top_builddir)/coverage-python-html" ; \
else \
echo "Python code coverage not enabled at configuration time." ; \
echo "Use: ./configure --with-pycoverage" ; \
fi
# for python and c++ test coverage
coverage: clean-coverage perform-coverage report-coverage
clean-coverage: clean-cpp-coverage clean-python-coverage
report-coverage: report-cpp-coverage report-python-coverage
#### include external sources in the distributed tarball:
EXTRA_DIST = ext/asio/README
EXTRA_DIST += ext/asio/asio/local/stream_protocol.hpp
......
......@@ -93,26 +93,55 @@ Then run "make check" to run these tests.
TEST COVERAGE
Code coverage reports may be generated using make. These are
based on running on the unit tests. The resulting reports are placed
in coverage-cpp-html and coverage-python-html directories for C++
and Python, respectively.
The code coverage report for the C++ tests uses LCOV. It is available
from http://ltp.sourceforge.net/. To generate your own HTML report,
from http://ltp.sourceforge.net/. To generate the HTML report,
first configure BIND 10 with:
./configure --with-lcov
The code coverage report for the Python tests uses coverage.py (aka
pycoverage). It is available from http://nedbatchelder.com/code/coverage/.
To generate the HTML report, first configure BIND 10 with:
./configure --with-pycoverage
Doing code coverage tests:
make coverage
Does the following:
Does the clean, perform, and report targets for C++ and Python.
make clean-coverage
Zeroes the lcov code coverage counters and removes the coverage HTML.
Zeroes the code coverage counters and removes the HTML reports
for C++ and Python.
make perform-coverage
Runs the C++ tests (using googletests framework).
Runs the C++ (using the googletests framework) and Python
tests.
make report-coverage
Generates the coverage HTML, excluding some unrelated headers.
The HTML reports are placed in a directory called coverage/.
Generates the coverage reports in HTML for C++ and Python.
make clean-cpp-coverage
Zeroes the code coverage counters and removes the HTML report
for the C++ tests.
make clean-python-coverage
Zeroes the code coverage counters and removes the HTML report
for the Python tests.
make report-cpp-coverage
Generates the coverage report in HTML for C++, excluding
some unrelated headers. The HTML reports are placed in a
directory called coverage-cpp-html/.
make report-python-coverage
Generates the coverage report in HTML for Python. The HTML
reports are placed in a directory called coverage-python-html/.
DEVELOPERS
......
......@@ -287,6 +287,26 @@ AC_TRY_COMPILE([
AC_DEFINE(HAVE_SA_LEN, 1, [Define to 1 if sockaddr has a sa_len member, and corresponding sin_len and sun_len])],
AC_MSG_RESULT(no))
AC_ARG_WITH(pycoverage,
[ --with-pycoverage[=PROGRAM] enable python code coverage using the specified coverage], pycoverage="$withval", pycoverage="no")
if test "$pycoverage" = "no" ; then
# just run the tests normally with python
PYCOVERAGE_RUN="${PYTHON}"
USE_PYCOVERAGE="no"
elif test "$pycoverage" = "yes" ; then
PYCOVERAGE="coverage"
PYCOVERAGE_RUN="${PYCOVERAGE} run --branch --append"
USE_PYCOVERAGE="yes"
else
PYCOVERAGE="$pycoverage"
PYCOVERAGE_RUN="${PYCOVERAGE} run --branch --append"
USE_PYCOVERAGE="yes"
fi
AM_CONDITIONAL(ENABLE_PYTHON_COVERAGE, test x$USE_PYCOVERAGE != xno)
AC_SUBST(PYCOVERAGE)
AC_SUBST(PYCOVERAGE_RUN)
AC_SUBST(USE_PYCOVERAGE)
AC_ARG_WITH(lcov,
[ --with-lcov[=PROGRAM] enable gtest and coverage target using the specified lcov], lcov="$withval", lcov="no")
......@@ -729,7 +749,8 @@ Features:
Developer:
Google Tests: $gtest_path
Code Coverage: $USE_LCOV
C++ Code Coverage: $USE_LCOV
Python Code Coverage: $USE_PYCOVERAGE
Generate Manuals: $enable_man
END
......
CLEANFILES = *.wire
BUILT_SOURCES = badExampleQuery_fromWire.wire examplequery_fromWire.wire
BUILT_SOURCES += iqueryresponse_fromWire.wire multiquestion_fromWire.wire
BUILT_SOURCES += queryBadEDNS_fromWire.wire shortanswer_fromWire.wire
BUILT_SOURCES += simplequery_fromWire.wire simpleresponse_fromWire.wire
# NOTE: keep this in sync with real file listing
# so is included in tarball
EXTRA_DIST = badExampleQuery_fromWire.spec
EXTRA_DIST += examplequery_fromWire.spec
EXTRA_DIST += iqueryresponse_fromWire.spec
EXTRA_DIST += multiquestion_fromWire.spec
EXTRA_DIST += queryBadEDNS_fromWire.spec
EXTRA_DIST += shortanswer_fromWire.spec
EXTRA_DIST += shortmessage_fromWire
EXTRA_DIST += shortquestion_fromWire
EXTRA_DIST += shortresponse_fromWire
EXTRA_DIST += simplequery_fromWire.spec
EXTRA_DIST += simpleresponse_fromWire.spec
EXTRA_DIST += example.com
EXTRA_DIST += example.sqlite3
.spec.wire:
$(abs_top_builddir)/src/lib/dns/tests/testdata/gen-wiredata.py -o $@ $<
#
# A simple QUERY message for the example.com zone that would hit a broken
# record of the data source.
#
[header]
# use default
[question]
name: broken.example.com
rrtype: AAAA
$TTL 3600
@ SOA ns.example.com. admin.example.com. 1234 3600 1800 2419200 7200
NS ns.example.com.
ns.example.com. A 192.0.2.1
;; bogus RDATA for CNAME RR, but the loadzone tool accepts it. looking up this
;; record will trigger an exception.
broken.example.com. CNAME 0123456789012345678901234567890123456789012345678901234567890123456789.example.com.
#
# A simple QUERY message for the example.com zone
#
[header]
# use default
[question]
# use default
name: ns.example.com
#
# A response to an IQUERY request.
#
[header]
qr: response
opcode: iquery
[question]
# use default
#
# A QUERY message with multiple questions.
#
[custom]
sections: header:question/0:question/1
[header]
qdcount: 2
[question/0]
# use default
[question/1]
rrtype: AAAA
#
# A QUERY message with unsupported version of EDNS..
#
[header]
arcount: 1
# use default
[question]
# use default
[edns]
version: 1
do: 1
#
# A QUERY message with a broken answer section (ancount > 0 but the section
# is empty)
#
[header]
# use default
arcount: 1
[question]
# use default
###
### DNS message-like data but doesn't contain sufficient length of data.
###
# Header Section
# ID=4149 QR=Query Opcode=QUERY(0) Rcode=NOERROR(0)
1035 0000
# QDCNT=1, ANCNT=0, NSCNT=0, (ARCNT is missing)
0001 0000 0000
###
### A query-like data, but missing QCLASS field in the Question section.
###
# Header Section
# ID=4149 QR=Query Opcode=QUERY(0) Rcode=NOERROR(0)
1035 0000
# QDCNT=1, ANCNT=0, NSCNT=0, ARCNT=0
0001 0000 0000 0000
# Question Section
# QNAME=example.com. QTYPE=A(1) (QCLASS missing)
076578616d706c6503636f6d00 0001
###
### A response-like data, but missing QCLASS field in the Question section.
###
# Header Section
# ID=4149 QR=Response Opcode=QUERY(0) Rcode=NOERROR(0)
1035 8000
# QDCNT=1, ANCNT=0, NSCNT=0, ARCNT=0
0001 0000 0000 0000
# Question Section
# QNAME=example.com. QTYPE=A(1) (QCLASS is missing)
076578616d706c6503636f6d00 0001
#
# A simple QUERY message.
#
[header]
# use default
[question]
# use default
#
# A simple response message.
#
[header]
qr: response
[question]
# use default
PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
#PYTESTS = args_test.py bind10_test.py
PYTESTS = bind10_test.py
EXTRA_DIST = $(PYTESTS)
# later will have configure option to choose this, like: coverage run --branch
PYCOVERAGE = $(PYTHON)
# test using command-line arguments, so use check-local target instead of TESTS
check-local:
if ENABLE_PYTHON_COVERAGE
touch $(abs_top_srcdir)/.coverage
rm -f .coverage
${LN_S} $(abs_top_srcdir)/.coverage .coverage
endif
for pytest in $(PYTESTS) ; do \
echo Running test: $$pytest ; \
env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_builddir)/src/bin/bind10 \
$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
done
PYCOVERAGE_RUN = @PYCOVERAGE_RUN@
PYTESTS = bindctl_test.py
EXTRA_DIST = $(PYTESTS)
# later will have configure option to choose this, like: coverage run --branch
PYCOVERAGE = $(PYTHON)
# test using command-line arguments, so use check-local target instead of TESTS
check-local:
if ENABLE_PYTHON_COVERAGE
touch $(abs_top_srcdir)/.coverage
rm -f .coverage
${LN_S} $(abs_top_srcdir)/.coverage .coverage
endif
for pytest in $(PYTESTS) ; do \
echo Running test: $$pytest ; \
env PYTHONPATH=$(abs_top_srcdir)/src/lib/python:$(abs_top_builddir)/src/lib/python:$(abs_top_srcdir)/src/bin \
$(PYCOVERAGE) $(abs_srcdir)/$$pytest || exit ; \
$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
done
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment