Commit e1a0ea8e authored by Naoki Kambe's avatar Naoki Kambe
Browse files

[master] Merge branch 'trac2252'

parents b9608c20 d36c276d
......@@ -213,6 +213,132 @@ operation
-->
</refsect1>
<refsect1>
<title>STATISTICS DATA</title>
<para>
The statistics data collected by the <command>b10-xfrin</command>
daemon for <quote>Xfrin</quote> include:
</para>
<variablelist>
<varlistentry>
<term>zones</term>
<listitem><simpara>
A directory name of per-zone statistics
</simpara>
<variablelist>
<varlistentry>
<term><replaceable>zonename</replaceable></term>
<listitem><simpara>
An actual zone name or special zone name
<quote>_SERVER_</quote> representing the entire server.
Zone classes (e.g. IN, CH, and HS) are mixed and counted so
far. But these will be distinguished in future release.
</simpara>
<variablelist>
<varlistentry>
<term>soaoutv4</term>
<listitem><simpara>
Number of IPv4 SOA queries sent from Xfrin
</simpara></listitem>
</varlistentry>
<varlistentry>
<term>soaoutv6</term>
<listitem><simpara>
Number of IPv6 SOA queries sent from Xfrin
</simpara></listitem>
</varlistentry>
<varlistentry>
<term>axfrreqv4</term>
<listitem><simpara>
Number of IPv4 AXFR requests sent from Xfrin
</simpara></listitem>
</varlistentry>
<varlistentry>
<term>axfrreqv6</term>
<listitem><simpara>
Number of IPv6 AXFR requests sent from Xfrin
</simpara></listitem>
</varlistentry>
<varlistentry>
<term>ixfrreqv4</term>
<listitem><simpara>
Number of IPv4 IXFR requests sent from Xfrin
</simpara></listitem>
</varlistentry>
<varlistentry>
<term>ixfrreqv6</term>
<listitem><simpara>
Number of IPv6 IXFR requests sent from Xfrin
</simpara></listitem>
</varlistentry>
<varlistentry>
<term>xfrsuccess</term>
<listitem><simpara>
Number of zone transfer requests succeeded.
These include the case where the zone turns
out to be the latest as a result of an
initial SOA query (and there is actually no
AXFR or IXFR transaction).
</simpara></listitem>
</varlistentry>
<varlistentry>
<term>xfrfail</term>
<listitem><simpara>
Number of zone transfer requests failed
</simpara></listitem>
</varlistentry>
<varlistentry>
<term>last_axfr_duration</term>
<listitem><simpara>
Duration in seconds of the last successful AXFR. 0.0
means no successful AXFR done or means a successful AXFR
done in less than a microsecond. If an AXFR is aborted
due to some failure, this duration won't be updated.
</simpara></listitem>
</varlistentry>
<varlistentry>
<term>last_ixfr_duration</term>
<listitem><simpara>
Duration in seconds of the last successful IXFR. 0.0
means no successful IXFR done or means a successful IXFR
done in less than a microsecond. If an IXFR is aborted
due to some failure, this duration won't be updated.
</simpara></listitem>
</varlistentry>
</variablelist>
</listitem>
</varlistentry><!-- end of zonename -->
</variablelist>
</listitem>
</varlistentry><!-- end of zones -->
</variablelist>
<para>
In per-zone counters the special zone name <quote>_SERVER_</quote>
exists.
It doesn't mean a specific zone. It represents the entire server
and the counter value of this special zone is the total of the
same counter for all zones.
</para>
</refsect1>
<!--
<refsect1>
......
# Copyright (C) 2009-2011 Internet Systems Consortium.
# Copyright (C) 2009-2013 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
......@@ -19,6 +19,7 @@ import shutil
import socket
import sys
import io
from datetime import datetime
from isc.testutils.tsigctx_mock import MockTSIGContext
from isc.testutils.ccsession_mock import MockModuleCCSession
from isc.testutils.rrset_utils import *
......@@ -717,7 +718,7 @@ class TestXfrinConnection(unittest.TestCase):
self.sock_map = {}
self.conn = MockXfrinConnection(self.sock_map, TEST_ZONE_NAME,
TEST_RRCLASS, None, threading.Event(),
TEST_MASTER_IPV4_ADDRINFO)
self._master_addrinfo)
self.conn.init_socket()
self.soa_response_params = {
'questions': [example_soa_question],
......@@ -749,6 +750,10 @@ class TestXfrinConnection(unittest.TestCase):
os.remove(TEST_DB_FILE)
xfrin.check_zone = self.__orig_check_zone
@property
def _master_addrinfo(self):
return TEST_MASTER_IPV4_ADDRINFO
def __check_zone(self, name, rrclass, rrsets, callbacks):
'''
A mock function used instead of dns.check_zone.
......@@ -1065,6 +1070,20 @@ class TestAXFR(TestXfrinConnection):
self.assertRaises(XfrinProtocolError,
self.conn._handle_xfrin_responses)
def test_ipver_str(self):
addrs = (((socket.AF_INET, socket.SOCK_STREAM), 'v4'),
((socket.AF_INET6, socket.SOCK_STREAM), 'v6'),
((socket.AF_UNIX, socket.SOCK_STREAM), None))
for (info, ver) in addrs:
c = MockXfrinConnection({}, TEST_ZONE_NAME, RRClass.CH, None,
threading.Event(), info)
c.init_socket()
if ver is not None:
self.assertEqual(ver, c._get_ipver_str())
else:
self.assertRaises(ValueError, c._get_ipver_str)
c.close()
def test_soacheck(self):
# we need to defer the creation until we know the QID, which is
# determined in _check_soa_serial(), so we use response_generator.
......@@ -2104,6 +2123,187 @@ class TestXFRSessionWithSQLite3(TestXfrinConnection):
self.assertFalse(self.record_exist(Name('dns01.example.com'),
RRType.A))
class TestStatisticsXfrinConn(TestXfrinConnection):
'''Test class based on TestXfrinConnection and including paramters
and methods related to statistics tests'''
def setUp(self):
super().setUp()
# clear all statistics counters before each test
self.conn._counters.clear_all()
# fake datetime
self.__orig_datetime = isc.statistics.counters.datetime
self.__orig_start_timer = isc.statistics.counters._start_timer
time1 = datetime(2000, 1, 1, 0, 0, 0, 0)
time2 = datetime(2000, 1, 1, 0, 0, 0, 1)
class FakeDateTime:
@classmethod
def now(cls): return time2
isc.statistics.counters.datetime = FakeDateTime
isc.statistics.counters._start_timer = lambda : time1
delta = time2 - time1
self._const_sec = round(delta.days * 86400 + delta.seconds +
delta.microseconds * 1E-6, 6)
# List of statistics counter names and expected initial values
self.__name_to_counter = (('axfrreqv4', 0),
('axfrreqv6', 0),
('ixfrreqv4', 0),
('ixfrreqv6', 0),
('last_axfr_duration', 0.0),
('last_ixfr_duration', 0.0),
('soaoutv4', 0),
('soaoutv6', 0),
('xfrfail', 0),
('xfrsuccess', 0))
self.__zones = 'zones'
def tearDown(self):
super().tearDown()
isc.statistics.counters.datetime = self.__orig_datetime
isc.statistics.counters._start_timer = self.__orig_start_timer
@property
def _ipver(self):
return 'v4'
def _check_init_statistics(self):
'''checks exception being raised if not incremented statistics
counter gotten'''
for (name, exp) in self.__name_to_counter:
self.assertRaises(isc.cc.data.DataNotFoundError,
self.conn._counters.get, self.__zones,
TEST_ZONE_NAME_STR, name)
def _check_updated_statistics(self, overwrite):
'''checks getting expect values after updating the pairs of
statistics counter name and value on to the "overwrite"
dictionary'''
name2count = dict(self.__name_to_counter)
name2count.update(overwrite)
for (name, exp) in name2count.items():
act = self.conn._counters.get(self.__zones,
TEST_ZONE_NAME_STR,
name)
msg = '%s is expected %s but actually %s' % (name, exp, act)
self.assertEqual(exp, act, msg=msg)
class TestStatisticsXfrinAXFRv4(TestStatisticsXfrinConn):
'''Xfrin AXFR tests for IPv4 to check statistics counters'''
def test_soaout(self):
'''tests that an soaoutv4 or soaoutv6 counter is incremented
when an soa query succeeds'''
self.conn.response_generator = self._create_soa_response_data
self._check_init_statistics()
self.assertEqual(self.conn._check_soa_serial(), XFRIN_OK)
self._check_updated_statistics({'soaout' + self._ipver: 1})
def test_axfrreq_xfrsuccess_last_axfr_duration(self):
'''tests that axfrreqv4 or axfrreqv6 and xfrsuccess counters
and last_axfr_duration timer are incremented when xfr succeeds'''
self.conn.response_generator = self._create_normal_response_data
self._check_init_statistics()
self.assertEqual(self.conn.do_xfrin(False), XFRIN_OK)
self._check_updated_statistics({'axfrreq' + self._ipver: 1,
'xfrsuccess': 1,
'last_axfr_duration': self._const_sec})
def test_axfrreq_xfrsuccess_last_axfr_duration2(self):
'''tests that axfrreqv4 or axfrreqv6 and xfrsuccess counters
and last_axfr_duration timer are incremented when raising
XfrinZoneUptodate. The exception is treated as success.'''
def exception_raiser():
raise XfrinZoneUptodate()
self.conn._handle_xfrin_responses = exception_raiser
self._check_init_statistics()
self.assertEqual(self.conn.do_xfrin(False), XFRIN_OK)
self._check_updated_statistics({'axfrreq' + self._ipver: 1,
'xfrsuccess': 1,
'last_axfr_duration':
self._const_sec})
def test_axfrreq_xfrfail(self):
'''tests that axfrreqv4 or axfrreqv6 and xfrfail counters are
incremented even if some failure exceptions are expected to be
raised inside do_xfrin(): XfrinZoneError, XfrinProtocolError,
XfrinException, and Exception'''
self._check_init_statistics()
count = 0
for ex in [XfrinZoneError, XfrinProtocolError, XfrinException,
Exception]:
def exception_raiser():
raise ex()
self.conn._handle_xfrin_responses = exception_raiser
self.assertEqual(self.conn.do_xfrin(False), XFRIN_FAIL)
count += 1
self._check_updated_statistics({'axfrreq' + self._ipver: count,
'xfrfail': count})
class TestStatisticsXfrinIXFRv4(TestStatisticsXfrinConn):
'''Xfrin IXFR tests for IPv4 to check statistics counters'''
def test_ixfrreq_xfrsuccess_last_ixfr_duration(self):
'''tests that ixfrreqv4 or ixfrreqv6 and xfrsuccess counters
and last_ixfr_duration timer are incremented when xfr succeeds'''
def create_ixfr_response():
self.conn.reply_data = self.conn.create_response_data(
questions=[Question(TEST_ZONE_NAME, TEST_RRCLASS,
RRType.IXFR)],
answers=[soa_rrset, begin_soa_rrset, soa_rrset, soa_rrset])
self.conn.response_generator = create_ixfr_response
self._check_init_statistics()
self.assertEqual(XFRIN_OK, self.conn.do_xfrin(False, RRType.IXFR))
self._check_updated_statistics({'ixfrreq' + self._ipver: 1,
'xfrsuccess': 1,
'last_ixfr_duration':
self._const_sec})
def test_ixfrreq_xfrsuccess_last_ixfr_duration2(self):
'''tests that ixfrreqv4 or ixfrreqv6 and xfrsuccess counters
and last_ixfr_duration timer are incremented when raising
XfrinZoneUptodate. The exception is treated as success.'''
def exception_raiser():
raise XfrinZoneUptodate()
self.conn._handle_xfrin_responses = exception_raiser
self._check_init_statistics()
self.assertEqual(self.conn.do_xfrin(False, RRType.IXFR), XFRIN_OK)
self._check_updated_statistics({'ixfrreq' + self._ipver: 1,
'xfrsuccess': 1,
'last_ixfr_duration':
self._const_sec})
def test_ixfrreq_xfrfail(self):
'''tests that ixfrreqv4 or ixfrreqv6 and xfrfail counters are
incremented even if some failure exceptions are expected to be
raised inside do_xfrin(): XfrinZoneError, XfrinProtocolError,
XfrinException, and Exception'''
self._check_init_statistics()
count = 0
for ex in [XfrinZoneError, XfrinProtocolError, XfrinException,
Exception]:
def exception_raiser():
raise ex()
self.conn._handle_xfrin_responses = exception_raiser
self.assertEqual(self.conn.do_xfrin(False, RRType.IXFR), XFRIN_FAIL)
count += 1
self._check_updated_statistics({'ixfrreq' + self._ipver: count,
'xfrfail': count})
class TestStatisticsXfrinAXFRv6(TestStatisticsXfrinAXFRv4):
'''Same tests as TestStatisticsXfrinAXFRv4 for IPv6'''
@property
def _master_addrinfo(self):
return TEST_MASTER_IPV6_ADDRINFO
@property
def _ipver(self):
return 'v6'
class TestStatisticsIXFRv6(TestStatisticsXfrinIXFRv4):
'''Same tests as TestStatisticsXfrinIXFRv4 for IPv6'''
@property
def _master_addrinfo(self):
return TEST_MASTER_IPV6_ADDRINFO
@property
def _ipver(self):
return 'v6'
class TestXfrinRecorder(unittest.TestCase):
def setUp(self):
self.recorder = XfrinRecorder()
......@@ -2512,6 +2712,14 @@ class TestXfrin(unittest.TestCase):
self.assertEqual(self.xfr.config_handler({'transfers_in': 3})['result'][0], 0)
self.assertEqual(self.xfr._max_transfers_in, 3)
def test_command_handler_getstats(self):
module_spec = isc.config.module_spec_from_file(
xfrin.SPECFILE_LOCATION)
ans = isc.config.parse_answer(
self.xfr.command_handler("getstats", None))
self.assertEqual(0, ans[0])
self.assertTrue(module_spec.validate_statistics(False, ans[1]))
def _check_zones_config(self, config_given):
if 'transfers_in' in config_given:
self.assertEqual(config_given['transfers_in'],
......
#!@PYTHON@
# Copyright (C) 2009-2011 Internet Systems Consortium.
# Copyright (C) 2009-2013 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
......@@ -28,6 +28,7 @@ import time
from functools import reduce
from optparse import OptionParser, OptionValueError
from isc.config.ccsession import *
from isc.statistics import Counters
from isc.notify import notify_out
import isc.util.process
from isc.datasrc import DataSourceClient, ZoneFinder
......@@ -612,6 +613,7 @@ class XfrinConnection(asyncore.dispatcher):
# keep a record of this specific transfer to log on success
# (time, rr/s, etc)
self._transfer_stats = XfrinTransferStats()
self._counters = Counters(SPECFILE_LOCATION)
def init_socket(self):
'''Initialize the underlyig socket.
......@@ -891,6 +893,19 @@ class XfrinConnection(asyncore.dispatcher):
# All okay, return it
return soa
def _get_ipver_str(self):
"""Returns a 'v4' or 'v6' string representing a IP version
depending on the socket family. This is for an internal use
only (except for tests). This is supported only for IP sockets.
It raises a ValueError exception on other address families.
"""
if self.socket.family == socket.AF_INET:
return 'v4'
elif self.socket.family == socket.AF_INET6:
return 'v6'
raise ValueError("Invalid address family. "
"This is supported only for IP sockets")
def _check_soa_serial(self):
'''Send SOA query and compare the local and remote serials.
......@@ -902,6 +917,9 @@ class XfrinConnection(asyncore.dispatcher):
'''
self._send_query(RRType.SOA)
# count soaoutv4 or soaoutv6 requests
self._counters.inc('zones', self._zone_name.to_text(),
'soaout' + self._get_ipver_str())
data_len = self._get_request_response(2)
msg_len = socket.htons(struct.unpack('H', data_len)[0])
soa_response = self._get_request_response(msg_len)
......@@ -931,9 +949,7 @@ class XfrinConnection(asyncore.dispatcher):
try:
ret = XFRIN_OK
self._request_type = request_type
# Right now RRType.[IA]XFR().to_text() is 'TYPExxx', so we need
# to hardcode here.
req_str = 'IXFR' if request_type == RRType.IXFR else 'AXFR'
req_str = request_type.to_text()
if check_soa:
self._check_soa_serial()
self.close()
......@@ -941,7 +957,16 @@ class XfrinConnection(asyncore.dispatcher):
if not self.connect_to_master():
raise XfrinException('Unable to reconnect to master')
# start statistics timer
# Note: If the timer for the zone is already started but
# not yet stopped due to some error, the last start time
# is overwritten at this point.
self._counters.start_timer('zones', self._zone_name.to_text(),
'last_' + req_str.lower() + '_duration')
logger.info(XFRIN_XFR_TRANSFER_STARTED, req_str, self.zone_str())
# An AXFR or IXFR is being requested.
self._counters.inc('zones', self._zone_name.to_text(),
req_str.lower() + 'req' + self._get_ipver_str())
self._send_query(self._request_type)
self.__state = XfrinInitialSOA()
self._handle_xfrin_responses()
......@@ -968,7 +993,6 @@ class XfrinConnection(asyncore.dispatcher):
"%.3f" % self._transfer_stats.get_running_time(),
"%.f" % self._transfer_stats.get_bytes_per_second()
)
except XfrinZoneUptodate:
# Eventually we'll probably have to treat this case as a trigger
# of trying another primary server, etc, but for now we treat it
......@@ -1004,11 +1028,21 @@ class XfrinConnection(asyncore.dispatcher):
self.zone_str(), str(e))
ret = XFRIN_FAIL
finally:
# A xfrsuccess or xfrfail counter is incremented depending on
# the result.
result = {XFRIN_OK: 'xfrsuccess', XFRIN_FAIL: 'xfrfail'}[ret]
self._counters.inc('zones', self._zone_name.to_text(), result)
# The started statistics timer is finally stopped only in
# a successful case.
if ret == XFRIN_OK:
self._counters.stop_timer('zones',
self._zone_name.to_text(),
'last_' + req_str.lower() +
'_duration')
# Make sure any remaining transaction in the diff is closed
# (if not yet - possible in case of xfr-level exception) as soon
# as possible
self._diff = None
return ret
def _check_response_header(self, msg):
......@@ -1339,6 +1373,7 @@ class Xfrin:
self._cc_setup()
self.recorder = XfrinRecorder()
self._shutdown_event = threading.Event()
self._counters = Counters(SPECFILE_LOCATION)
def _cc_setup(self):
'''This method is used only as part of initialization, but is
......@@ -1484,6 +1519,7 @@ class Xfrin:
th.join()
def command_handler(self, command, args):
logger.debug(DBG_XFRIN_TRACE, XFRIN_RECEIVED_COMMAND, command)
answer = create_answer(0)
try:
if command == 'shutdown':
......@@ -1552,6 +1588,14 @@ class Xfrin:
(False if command == 'retransfer' else True))
answer = create_answer(ret[0], ret[1])
# return statistics data to the stats daemon
elif command == "getstats":
# The log level is here set to debug in order to avoid
# that a log becomes too verbose. Because the
# b10-stats daemon is periodically asking to the
# b10-xfrin daemon.
answer = create_answer(0, self._counters.get_statistics())
else:
answer = create_answer(1, 'unknown command: ' + command)
except XfrinException as err:
......
......@@ -94,6 +94,119 @@
}
]
}
],
"statistics": [
{
"item_name": "zones",
"item_type": "named_set",
"item_optional": false,
"item_default": {
"_SERVER_" : {
"soaoutv4": 0,
"soaoutv6": 0,
"axfrreqv4": 0,
"axfrreqv6": 0,
"ixfrreqv4": 0,
"ixfrreqv6": 0,
"xfrsuccess": 0,
"xfrfail": 0,
"last_ixfr_duration": 0.0,
"last_axfr_duration": 0.0
}
},
"item_title": "Zone names",
"item_description": "A directory name of per-zone statistics",
"named_set_item_spec": {
"item_name": "zonename",
"item_type": "map",
"item_optional": false,
"item_default": {},
"item_title": "Zone name",
"item_description": "An actual zone name or special zone name _SERVER_ representing the entire server. Zone classes (e.g. IN, CH, and HS) are mixed and counted so far. But these will be distinguished in future release.",
"map_item_spec": [
{
"item_name": "soaoutv4",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "SOAOutv4",
"item_description": "Number of IPv4 SOA queries sent from Xfrin"
},
{
"item_name": "soaoutv6",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "SOAOutv6",
"item_description": "Number of IPv6 SOA queries sent from Xfrin"
},
{
"item_name": "axfrreqv4",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "AXFRReqv4",
"item_description": "Number of IPv4 AXFR requests sent from Xfrin"
},
{
"item_name": "axfrreqv6",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "AXFRReqv6",
"item_description": "Number of IPv6 AXFR requests sent from Xfrin"
},
{
"item_name": "ixfrreqv4",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "IXFRReqv4",
"item_description": "Number of IPv4 IXFR requests sent from Xfrin"
},
{
"item_name": "ixfrreqv6",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "IXFRReqv6",
"item_description": "Number of IPv6 IXFR requests sent from Xfrin"
},
{
"item_name": "xfrsuccess",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "XfrSuccess",
"item_description": "Number of zone transfer requests succeeded. These include the case where the zone turns out to be the latest as a result of an initial SOA query (and there is actually no AXFR or IXFR transaction)."
},
{
"item_name": "xfrfail",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "XfrFail",
"item_description": "Number of zone transfer requests failed"
},
{
"item_name": "last_axfr_duration",
"item_type": "real",
"item_optional": false,
"item_default": 0.0,
"item_title": "Last AXFR duration",
"item_description": "Duration in seconds of the last successful AXFR. 0.0 means no successful AXFR done or means a successful AXFR done in less than a microsecond. If an AXFR is aborted due to some failure, this duration won't be updated."
},
{
"item_name": "last_ixfr_duration",
"item_type": "real",
"item_optional": false,
"item_default": 0.0,
"item_title": "Last IXFR duration",
"item_description": "Duration in seconds of the last successful IXFR. 0.0 means no successful IXFR done or means a successful IXFR done in less than a microsecond. If an IXFR is aborted due to some failure, this duration won't be updated."
}
]
}
}
]