Commit 715419df authored by Naoki Kambe's avatar Naoki Kambe
Browse files

[master] Merge branch 'trac2883'

parents 8fbdef45 8ce213f2
......@@ -300,7 +300,8 @@ class MockXfrinConnection(XfrinConnection):
def __init__(self, sock_map, zone_name, rrclass, datasrc_client,
shutdown_event, master_addr, tsig_key=None):
super().__init__(sock_map, zone_name, rrclass, MockDataSourceClient(),
shutdown_event, master_addr, begin_soa_rrset)
shutdown_event, master_addr, begin_soa_rrset,
xfrin.Counters(xfrin.SPECFILE_LOCATION))
self.query_data = b''
self.reply_data = b''
self.force_time_out = False
......@@ -2129,8 +2130,6 @@ class TestStatisticsXfrinConn(TestXfrinConnection):
and methods related to statistics tests'''
def setUp(self):
super().setUp()
# clear all statistics counters before each test
self.conn._counters.clear_all()
# fake datetime
self.__orig_datetime = isc.statistics.counters.datetime
self.__orig_start_timer = isc.statistics.counters._start_timer
......@@ -3198,7 +3197,9 @@ class TestXfrinProcess(unittest.TestCase):
xfrin.process_xfrin(self, XfrinRecorder(), Name("example.org."),
RRClass.IN, None, zone_soa, None,
TEST_MASTER_IPV4_ADDRINFO, True, None,
request_ixfr, self.__get_connection)
request_ixfr,
xfrin.Counters(xfrin.SPECFILE_LOCATION),
self.__get_connection)
self.assertEqual([], self.__rets)
self.assertEqual(transfers, self.__transfers)
# Create a connection for each attempt
......
......@@ -563,8 +563,8 @@ class XfrinConnection(asyncore.dispatcher):
def __init__(self,
sock_map, zone_name, rrclass, datasrc_client,
shutdown_event, master_addrinfo, zone_soa, tsig_key=None,
idle_timeout=60):
shutdown_event, master_addrinfo, zone_soa, counters,
tsig_key=None, idle_timeout=60):
"""Constructor of the XfirnConnection class.
Parameters:
......@@ -579,6 +579,7 @@ class XfrinConnection(asyncore.dispatcher):
address and port of the master server.
zone_soa (RRset or None): SOA RRset of zone's current SOA or None
if it's not available.
counters (Counters): used for statistics counters
idle_timeout (int): max idle time for read data from socket.
"""
......@@ -617,7 +618,7 @@ class XfrinConnection(asyncore.dispatcher):
# keep a record of this specific transfer to log on success
# (time, rr/s, etc)
self._transfer_stats = XfrinTransferStats()
self._counters = Counters(SPECFILE_LOCATION)
self._counters = counters
def init_socket(self):
'''Initialize the underlyig socket.
......@@ -1107,7 +1108,7 @@ def __get_initial_xfr_type(zone_soa, request_ixfr, zname, zclass, master_addr):
def __process_xfrin(server, zone_name, rrclass, datasrc_client, zone_soa,
shutdown_event, master_addrinfo, check_soa, tsig_key,
request_ixfr, conn_class):
request_ixfr, counters, conn_class):
conn = None
exception = None
ret = XFRIN_FAIL
......@@ -1131,7 +1132,7 @@ def __process_xfrin(server, zone_name, rrclass, datasrc_client, zone_soa,
retry = False
conn = conn_class(sock_map, zone_name, rrclass, datasrc_client,
shutdown_event, master_addrinfo, zone_soa,
tsig_key)
counters, tsig_key)
conn.init_socket()
ret = XFRIN_FAIL
if conn.connect_to_master():
......@@ -1178,7 +1179,7 @@ def __process_xfrin(server, zone_name, rrclass, datasrc_client, zone_soa,
def process_xfrin(server, xfrin_recorder, zone_name, rrclass, datasrc_client,
zone_soa, shutdown_event, master_addrinfo, check_soa,
tsig_key, request_ixfr, conn_class=XfrinConnection):
tsig_key, request_ixfr, counters, conn_class=XfrinConnection):
# Even if it should be rare, the main process of xfrin session can
# raise an exception. In order to make sure the lock in xfrin_recorder
# is released in any cases, we delegate the main part to the helper
......@@ -1188,7 +1189,7 @@ def process_xfrin(server, xfrin_recorder, zone_name, rrclass, datasrc_client,
try:
__process_xfrin(server, zone_name, rrclass, datasrc_client, zone_soa,
shutdown_event, master_addrinfo, check_soa, tsig_key,
request_ixfr, conn_class)
request_ixfr, counters, conn_class)
except Exception as ex:
# don't log it until we complete decrement().
exception = ex
......@@ -1753,7 +1754,8 @@ class Xfrin:
datasrc_client, zone_soa,
self._shutdown_event,
master_addrinfo, check_soa,
tsig_key, request_ixfr))
tsig_key, request_ixfr,
self._counters))
xfrin_thread.start()
return (0, 'zone xfrin is started')
......
......@@ -309,7 +309,8 @@ class TestXfroutSessionBase(unittest.TestCase):
# When not testing ACLs, simply accept
isc.acl.dns.REQUEST_LOADER.load(
[{"action": "ACCEPT"}]),
{})
{},
xfrout.Counters(xfrout.SPECFILE_LOCATION))
self.set_request_type(RRType.AXFR) # test AXFR by default
self.mdata = self.create_request_data()
self.soa_rrset = create_soa(SOA_CURRENT_VERSION)
......@@ -1323,7 +1324,8 @@ class TestUnixSockServer(unittest.TestCase):
# This would be the handler class, but we just check it is passed
# the right parametes, so function is enough for that.
keys = isc.server_common.tsig_keyring.get_keyring()
def handler(sock, data, server, keyring, address, acl, config):
def handler(sock, data, server, keyring, address, acl, config,
counters):
self.assertEqual("sock", sock)
self.assertEqual("data", data)
self.assertEqual(self.unix, server)
......@@ -1331,6 +1333,7 @@ class TestUnixSockServer(unittest.TestCase):
self.assertEqual("Address", address)
self.assertEqual("acl", acl)
self.assertEqual("Zone config", config)
self.assertIs(self.unix._counters, counters)
self.unix.RequestHandlerClass = handler
self.unix.finish_request("sock", "data")
finally:
......@@ -1629,7 +1632,9 @@ class TestUnixSockServerForCounter(unittest.TestCase):
xfrout.ThreadingUnixStreamServer = DummySocketserver
xfrout.super = lambda : DummySocketserver()
xfrout.select.select = lambda x,y,z: ([None],[None],[None])
self.unix = UnixSockServer(None, None, threading.Event(), None, None)
self._counters = xfrout.Counters(xfrout.SPECFILE_LOCATION)
self.unix = UnixSockServer(None, None, threading.Event(), None, None,
self._counters)
def tearDown(self):
( UnixSockServer._remove_unused_sock_file,
......@@ -1659,7 +1664,8 @@ class TestUnixSockServerForCounter(unittest.TestCase):
'socket', 'unixdomain', 'openfail')
xfrout.ThreadingUnixStreamServer = DummySocketserverException
try:
self.unix = UnixSockServer(None, None, None, None, None)
self.unix = UnixSockServer(None, None, None, None, None,
self._counters)
except Exception:
pass
else:
......@@ -1700,7 +1706,7 @@ class TestUnixSockServerForCounter(unittest.TestCase):
self.unix._counters.get,
'socket', 'unixdomain', 'acceptfail')
xfrout.super = lambda : DummyClassException()
self.unix = UnixSockServer(None, None, None, None, None)
self.unix = UnixSockServer(None, None, None, None, None, self._counters)
self.assertRaises(Exception, self.unix.get_request)
self.assertEqual(
self.unix._counters.get('socket', 'unixdomain', 'acceptfail'), 1)
......
......@@ -176,7 +176,8 @@ def make_blocking(filenum, on):
class XfroutSession():
def __init__(self, sock_fd, request_data, server, tsig_key_ring, remote,
default_acl, zone_config, client_class=DataSourceClient):
default_acl, zone_config, counters,
client_class=DataSourceClient):
self._sock_fd = sock_fd
self._request_data = request_data
self._server = server
......@@ -193,7 +194,7 @@ class XfroutSession():
self._jnl_reader = None # will be set to a reader for IXFR
# Creation of self.counters should be done before of
# invoking self._handle()
self._counters = Counters(SPECFILE_LOCATION)
self._counters = counters
self._handle()
def create_tsig_ctx(self, tsig_record, tsig_key_ring):
......@@ -683,11 +684,11 @@ class UnixSockServer(socketserver_mixin.NoPollMixIn,
'''The unix domain socket server which accept xfr query sent from auth server.'''
def __init__(self, sock_file, handle_class, shutdown_event, config_data,
cc):
cc, counters):
self._remove_unused_sock_file(sock_file)
self._sock_file = sock_file
socketserver_mixin.NoPollMixIn.__init__(self)
self._counters = Counters(SPECFILE_LOCATION)
self._counters = counters
try:
ThreadingUnixStreamServer.__init__(self, sock_file, \
handle_class)
......@@ -886,7 +887,8 @@ class UnixSockServer(socketserver_mixin.NoPollMixIn,
self._lock.release()
self.RequestHandlerClass(sock_fd, request_data, self,
isc.server_common.tsig_keyring.get_keyring(),
self._guess_remote(sock_fd), acl, zone_config)
self._guess_remote(sock_fd), acl, zone_config,
self._counters)
def _remove_unused_sock_file(self, sock_file):
'''Try to remove the socket file. If the file is being used
......@@ -1025,12 +1027,12 @@ class XfroutServer:
self._shutdown_event = threading.Event()
self._cc = isc.config.ModuleCCSession(SPECFILE_LOCATION, self.config_handler, self.command_handler)
self._config_data = self._cc.get_full_config()
self._counters = Counters(SPECFILE_LOCATION)
self._cc.start()
self._cc.add_remote_config(AUTH_SPECFILE_LOCATION)
isc.server_common.tsig_keyring.init_keyring(self._cc)
self._start_xfr_query_listener()
self._start_notifier()
self._counters = Counters(SPECFILE_LOCATION)
def _start_xfr_query_listener(self):
'''Start a new thread to accept xfr query. '''
......@@ -1039,13 +1041,13 @@ class XfroutServer:
XfroutSession,
self._shutdown_event,
self._config_data,
self._cc)
self._cc, self._counters)
listener = threading.Thread(target=self._unix_socket_server.serve_forever)
listener.start()
def _start_notifier(self):
datasrc = self._unix_socket_server.get_db_file()
self._notifier = notify_out.NotifyOut(datasrc)
self._notifier = notify_out.NotifyOut(datasrc, counters=self._counters)
if 'also_notify' in self._config_data:
for slave in self._config_data['also_notify']:
address = self._default_notify_address
......
......@@ -128,7 +128,7 @@ class NotifyOut:
notify message to its slaves). notify service can be started by
calling dispatcher(), and it can be stopped by calling shutdown()
in another thread. '''
def __init__(self, datasrc_file, verbose=True):
def __init__(self, datasrc_file, counters=None, verbose=True):
self._notify_infos = {} # key is (zone_name, zone_class)
self._waiting_zones = []
self._notifying_zones = []
......@@ -143,7 +143,7 @@ class NotifyOut:
# Use nonblock event to eliminate busy loop
# If there are no notifying zones, clear the event bit and wait.
self._nonblock_event = threading.Event()
self._counters = Counters()
self._counters = counters
def _init_notify_out(self, datasrc_file):
'''Get all the zones name and its notify target's address.
......@@ -508,16 +508,17 @@ class NotifyOut:
sock = zone_notify_info.create_socket(addrinfo[0])
sock.sendto(render.get_data(), 0, addrinfo)
# count notifying by IPv4 or IPv6 for statistics
if zone_notify_info.get_socket().family == socket.AF_INET:
self._counters.inc('zones',
zone_notify_info.zone_class,
zone_notify_info.zone_name,
'notifyoutv4')
elif zone_notify_info.get_socket().family == socket.AF_INET6:
self._counters.inc('zones',
zone_notify_info.zone_class,
zone_notify_info.zone_name,
'notifyoutv6')
if self._counters is not None:
if zone_notify_info.get_socket().family == socket.AF_INET:
self._counters.inc('zones',
zone_notify_info.zone_class,
zone_notify_info.zone_name,
'notifyoutv4')
elif zone_notify_info.get_socket().family == socket.AF_INET6:
self._counters.inc('zones',
zone_notify_info.zone_class,
zone_notify_info.zone_name,
'notifyoutv6')
logger.info(NOTIFY_OUT_SENDING_NOTIFY, AddressFormatter(addrinfo))
except (socket.error, addr.InvalidAddress) as err:
logger.error(NOTIFY_OUT_SOCKET_ERROR, AddressFormatter(addrinfo),
......
......@@ -5,7 +5,7 @@ EXTRA_DIST += testdata/test.sqlite3 testdata/brokentest.sqlite3
# The rest of the files are actually not necessary, but added for reference
EXTRA_DIST += testdata/example.com testdata/example.net
EXTRA_DIST += testdata/nons.example testdata/nosoa.example
EXTRA_DIST += testdata/multisoa.example
EXTRA_DIST += testdata/multisoa.example testdata/test_spec1.spec
# If necessary (rare cases), explicitly specify paths to dynamic libraries
# required by loadable python modules.
......
......@@ -22,8 +22,10 @@ import socket
from isc.notify import notify_out, SOCK_DATA
import isc.log
from isc.dns import *
from isc.statistics.dns import Counters
TESTDATA_SRCDIR = os.getenv("TESTDATASRCDIR")
SPECFILE_LOCATION = TESTDATA_SRCDIR + os.sep + 'test_spec1.spec'
def get_notify_msgdata(zone_name, qid=0):
"""A helper function to generate a notify response in wire format.
......@@ -128,7 +130,7 @@ class TestZoneNotifyInfo(unittest.TestCase):
class TestNotifyOut(unittest.TestCase):
def setUp(self):
self._db_file = TESTDATA_SRCDIR + '/test.sqlite3'
self._notify = notify_out.NotifyOut(self._db_file)
self._notify = notify_out.NotifyOut(self._db_file, counters=Counters(SPECFILE_LOCATION))
self._notify._notify_infos[('example.com.', 'IN')] = MockZoneNotifyInfo('example.com.', 'IN')
self._notify._notify_infos[('example.com.', 'CH')] = MockZoneNotifyInfo('example.com.', 'CH')
self._notify._notify_infos[('example.net.', 'IN')] = MockZoneNotifyInfo('example.net.', 'IN')
......
{
"module_spec": {
"module_name": "NotifyOutLike",
"module_description": "Test notifier",
"config_data": [],
"commands": [],
"statistics": [
{
"item_name": "zones",
"item_type": "named_set",
"item_optional": false,
"item_default": {
"_SERVER_" : {
"notifyoutv4" : 0,
"notifyoutv6" : 0
}
},
"item_title": "Zone names",
"item_description": "Zone names",
"named_set_item_spec": {
"item_name": "classname",
"item_type": "named_set",
"item_optional": false,
"item_default": {},
"item_title": "RR class name",
"item_description": "RR class name",
"named_set_item_spec": {
"item_name": "zonename",
"item_type": "map",
"item_optional": false,
"item_default": {},
"item_title": "Zone name",
"item_description": "Zone name",
"map_item_spec": [
{
"item_name": "notifyoutv4",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "IPv4 notifies",
"item_description": "Number of IPv4 notifies per zone name sent out"
},
{
"item_name": "notifyoutv6",
"item_type": "integer",
"item_optional": false,
"item_default": 0,
"item_title": "IPv6 notifies",
"item_description": "Number of IPv6 notifies per zone name sent out"
}
]
}
}
}
]
}
}
......@@ -151,15 +151,6 @@ def _concat(*args, sep='/'):
"""
return sep.join(args)
class _Statistics():
"""Statistics data set. This class will be remove in the future
release."""
# default statistics data
_data = {}
# default statistics spec used in case the specfile is omitted when
# constructing a Counters() object
_spec = []
class Counters():
"""A class for holding and manipulating all statistics counters
for a module. A Counters object may be created by specifying a spec
......@@ -174,32 +165,25 @@ class Counters():
timers can be temporarily disabled. If disabled, counter values are
not changed even if methods to update them are invoked."""
# default statistics data set
_statistics = _Statistics()
def __init__(self, spec_file_name=None):
def __init__(self, spec_file_name):
"""A constructor for the Counters class. A path to the spec file
can be specified in spec_file_name. Statistics data based on
statistics spec can be accumulated if spec_file_name is
specified. If omitted, a default statistics spec is used. The
default statistics spec is defined in a hidden class named
_Statistics(). But the hidden class won't be used and
spec_file_name will be required in the future release.
can be specified in spec_file_name, which is required. Statistics data
based on statistics spec can be accumulated. If an invalid argument
including None is specified, ModuleSpecError might be raised.
"""
self._zones_item_list = []
self._start_time = {}
self._disabled = False
self._rlock = threading.RLock()
if not spec_file_name: return
# change the default statistics spec
self._statistics._spec = \
self._statistics_data = {}
self._statistics_spec = \
isc.config.module_spec_from_file(spec_file_name).\
get_statistics_spec()
def clear_all(self):
"""clears all statistics data"""
with self._rlock:
self._statistics._data = {}
self._statistics_data = {}
def disable(self):
"""disables incrementing/decrementing counters"""
......@@ -219,8 +203,8 @@ class Counters():
identifier = _concat(*args)
with self._rlock:
if self._disabled: return
_inc_counter(self._statistics._data,
self._statistics._spec,
_inc_counter(self._statistics_data,
self._statistics_spec,
identifier, step)
def inc(self, *args):
......@@ -240,7 +224,7 @@ class Counters():
of the specified counter. isc.cc.data.DataNotFoundError is
raised when the counter doesn't have a number yet."""
identifier = _concat(*args)
return _get_counter(self._statistics._data, identifier)
return _get_counter(self._statistics_data, identifier)
def start_timer(self, *args):
"""Starts a timer which is identified by args and keeps it
......@@ -271,8 +255,8 @@ class Counters():
# set the end time
_stop_timer(
start_time,
self._statistics._data,
self._statistics._spec,
self._statistics_data,
self._statistics_spec,
identifier)
# A datetime value of once used timer should be deleted
# for a future use.
......@@ -293,5 +277,5 @@ class Counters():
stats module, including each counter. If nothing is counted
yet, then it returns an empty dictionary."""
# entire copy
statistics_data = self._statistics._data.copy()
statistics_data = self._statistics_data.copy()
return statistics_data
......@@ -69,63 +69,6 @@ documentation for isc.statistics.counters for details."""
import isc.config
from isc.statistics import counters
class _Statistics():
"""Statistics data set. This class will be removed in the future
release."""
# default statistics data
_data = {}
# default statistics spec used in case the specfile is omitted when
# constructing a Counters() object
_spec = [
{
"item_name": "zones",
"item_type": "named_set",
"item_optional": False,
"item_default": {
"_SERVER_" : {
"notifyoutv4" : 0,
"notifyoutv6" : 0
}
},
"item_title": "Zone names",
"item_description": "Zone names",
"named_set_item_spec": {
"item_name": "classname",
"item_type": "named_set",
"item_optional": False,
"item_default": {},
"item_title": "RR class name",
"item_description": "RR class name",
"named_set_item_spec": {
"item_name": "zonename",
"item_type": "map",
"item_optional": False,
"item_default": {},
"item_title": "Zone name",
"item_description": "Zone name",
"map_item_spec": [
{
"item_name": "notifyoutv4",
"item_type": "integer",
"item_optional": False,
"item_default": 0,
"item_title": "IPv4 notifies",
"item_description": "Number of IPv4 notifies per zone name sent out"
},
{
"item_name": "notifyoutv6",
"item_type": "integer",
"item_optional": False,
"item_default": 0,
"item_title": "IPv6 notifies",
"item_description": "Number of IPv6 notifies per zone name sent out"
}
]
}
}
}
]
class Counters(counters.Counters):
"""A list of counters which can be handled in the class are like
the following. Also see documentation for
......@@ -176,20 +119,18 @@ class Counters(counters.Counters):
_entire_server = '_SERVER_'
# zone names are contained under this dirname in the spec file.
_perzone_prefix = 'zones'
# default statistics data set
_statistics = _Statistics()
def __init__(self, spec_file_name=None):
def __init__(self, spec_file_name):
"""If the item `zones` is defined in the spec file, it obtains a
list of counter names under it when initiating. For behaviors
other than this, see documentation for
isc.statistics.counters.Counters.__init__()"""
counters.Counters.__init__(self, spec_file_name)
if self._perzone_prefix in \
isc.config.spec_name_list(self._statistics._spec):
isc.config.spec_name_list(self._statistics_spec):
self._zones_item_list = isc.config.spec_name_list(
isc.config.find_spec_part(
self._statistics._spec,
self._statistics_spec,
'%s/%s/%s' % (self._perzone_prefix,
'_CLASS_', self._entire_server)))
......@@ -199,7 +140,7 @@ class Counters(counters.Counters):
counter. If nothing is counted yet, then it returns an empty
dictionary."""
# entire copy
statistics_data = self._statistics._data.copy()
statistics_data = self._statistics_data.copy()
# If there is no 'zones' found in statistics_data,
# i.e. statistics_data contains no per-zone counter, it just
# returns statistics_data because calculating total counts
......@@ -208,7 +149,7 @@ class Counters(counters.Counters):
return statistics_data
zones = statistics_data[self._perzone_prefix]
# Start calculation for '_SERVER_' counts
zones_spec = isc.config.find_spec_part(self._statistics._spec,
zones_spec = isc.config.find_spec_part(self._statistics_spec,
self._perzone_prefix)
zones_data = {}
for cls in zones.keys():
......
......@@ -49,12 +49,8 @@ class TestBasicMethods(unittest.TestCase):
TEST_SPECFILE_LOCATION = TESTDATA_SRCDIR + os.sep + 'test_spec1.spec'
def setUp(self):
imp.reload(counters)
self.counters = counters.Counters(self.TEST_SPECFILE_LOCATION)
def tearDown(self):
self.counters.clear_all()
def test_clear_counters(self):
self.assertRaises(isc.cc.data.DataNotFoundError,
self.counters.get, 'counter')
......@@ -131,15 +127,15 @@ class TestBasicMethods(unittest.TestCase):
start_functor(concurrency, number, self.counters.inc,
counter_name)
counters._stop_timer(start_time,
self.counters._statistics._data,
self.counters._statistics._spec,
self.counters._statistics_data,
self.counters._statistics_spec,
timer_name)
self.assertEqual(
counters._get_counter(self.counters._statistics._data,
counters._get_counter(self.counters._statistics_data,
counter_name),
concurrency * number)
self.assertGreaterEqual(
counters._get_counter(self.counters._statistics._data,
counters._get_counter(self.counters._statistics_data,
timer_name), 0.0)
def test_concat(self):
......@@ -158,16 +154,20 @@ class TestBasicMethods(unittest.TestCase):
b = a + ({},)
self.assertRaises(TypeError, counters._concat, *b)
def test_none_of_arg_of_counters(self):
"""Test Counters raises ModuleSpecError when specifying not valid
argument"""
self.assertRaises(isc.config.module_spec.ModuleSpecError,
counters.Counters, None)
self.assertRaises(isc.config.module_spec.ModuleSpecError,
counters.Counters, '/foo/bar')
class BaseTestCounters():
def setUp(self):
imp.reload(counters)
self._statistics_data = {}
self.counters = counters.Counters(self.TEST_SPECFILE_LOCATION)
def tearDown(self):
self.counters.clear_all()
def check_get_statistics(self):
"""Checks no differences between the value returned from
get_statistics() and locally collected statistics data. Also
......@@ -186,7 +186,7 @@ class BaseTestCounters():
else:
self.assertTrue(isc.config.ModuleSpec(
{'module_name': 'Foo',