Commit 1c6a6aa8 authored by Naoki Kambe's avatar Naoki Kambe
Browse files

[2225] introduced new counter classes and implemented unixsocket counters

 - implemented a base class Counter and a concrete XfroutCounter as an external
   module under isc.statistics. Because it is easy to implement another
   concrete Counter class for other module in future. The caller module can
   statically import it. The new counter class provides a getter method for
   each statistics item. It is intended for making loose relationship between
   the counter class and the caller module.

 - added implementation of unixsocket counter into the existing UnixSockServer
   Class.

 - added new tests for checking counters implemented in UnixSockServer are
   working properly into xfrout_test.py.

 - implemented enabling/disabling counting in the new counter class.
parent bbb0752c
......@@ -1251,6 +1251,8 @@ AC_CONFIG_FILES([Makefile
src/lib/python/isc/server_common/tests/Makefile
src/lib/python/isc/sysinfo/Makefile
src/lib/python/isc/sysinfo/tests/Makefile
src/lib/python/isc/statistics/Makefile
src/lib/python/isc/statistics/tests/Makefile
src/lib/config/Makefile
src/lib/config/tests/Makefile
src/lib/config/tests/testdata/Makefile
......
SUBDIRS = datasrc cc config dns log net notify util testutils acl bind10
SUBDIRS += xfrin log_messages server_common ddns sysinfo
SUBDIRS += xfrin log_messages server_common ddns sysinfo statistics
python_PYTHON = __init__.py
......
SUBDIRS = . tests
python_PYTHON = __init__.py counter.py
pythondir = $(pyexecdir)/isc/statistics
CLEANDIRS = __pycache__
clean-local:
rm -rf $(CLEANDIRS)
from isc.statistics.counter import *
# Copyright (C) 2012 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
'''Statistics counter countainer for modules'''
import threading
import isc.config
_COUNTER = None
def init(spec_file_name):
"""A creator method for a counter class. It creates a counter
object by the module name of the given spec file. An argument is a
specification file name."""
module_spec = isc.config.module_spec_from_file(spec_file_name)
class_name = '%sCounter' % module_spec.get_module_name()
global _COUNTER
_COUNTER = globals()[class_name](module_spec)
# These method are dummies for notify_out in case XfroutCounter is not
# loaded.
def inc_notifyoutv4(self, arg): pass
def inc_notifyoutv6(self, arg): pass
class Counter():
"""A basic counter class for concrete classes"""
_statistics_spec = {}
_statistics_data = {}
_disabled = False
_rlock = threading.RLock()
def __init__(self, module_spec):
self._statistics_spec = module_spec.get_statistics_spec()
global clear_counters
global disable
global enable
clear_counters = self.clear_counters
enable = self.enable
disable = self.disable
def clear_counters(self):
"""clears all statistics data"""
with self._rlock:
self._statistics_data = {}
def disable(self):
"""disables incrementing/decrementing counters"""
self._disabled = True
def enable(self):
"""enables incrementing/decrementing counters"""
self._disabled = False
class XfroutCounter(Counter):
"""A module for holding all statistics counters of Xfrout. The
counter numbers can be accessed by the accesseers defined
according to a spec file. In this class, the structure of per-zone
counters is assumed to be like this:
zones/example.com./notifyoutv4
zones/example.com./notifyoutv6
zones/example.com./xfrrej
zones/example.com./xfrreqdone
ixfr_running
axfr_running
socket/unixdomain/open
socket/unixdomain/openfail
socket/unixdomain/close
socket/unixdomain/bindfail
socket/unixdomain/acceptfail
socket/unixdomain/accept
socket/unixdomain/senderr
socket/unixdomain/recverr
"""
# '_SERVER_' is a special zone name representing an entire
# count. It doesn't mean a specific zone, but it means an
# entire count in the server.
_entire_server = '_SERVER_'
# zone names are contained under this dirname in the spec file.
_perzone_prefix = 'zones'
_xfrrunning_names = []
_unixsocket_names = []
def __init__(self, module_spec):
Counter.__init__(self, module_spec)
self._xfrrunning_names = [ \
n for n in \
isc.config.spec_name_list(self._statistics_spec) \
if 'xfr_running' in n ]
self._unixsocket_names = [ \
n.split('/')[-1] for n in \
isc.config.spec_name_list(\
self._statistics_spec, "", True) \
if n.find('socket/unixdomain/') == 0 ]
self._create_perzone_functors()
self._create_xfrrunning_functors()
self._create_unixsocket_functors()
global dump_default_statistics
global dump_statistics
dump_default_statistics = self.dump_default_statistics
dump_statistics = self.dump_statistics
def _create_perzone_functors(self):
"""Creates increment method of each per-zone counter based on
the spec file. Incrementer can be accessed by name
"inc_${item_name}".Incrementers are passed to the
XfroutSession and NotifyOut class as counter handlers."""
# add a new element under the named_set item for the zone
zones_spec = isc.config.find_spec_part(
self._statistics_spec, self._perzone_prefix)
item_list = isc.config.spec_name_list(\
zones_spec['named_set_item_spec']['map_item_spec'])
# can be accessed by the name 'inc_xxx'
for item in item_list:
def __incrementer(zone_name, counter_name=item, step=1):
"""A per-zone incrementer for counter_name. Locks the
thread because it is considered to be invoked by a
multi-threading caller."""
if self._disabled: return
with self._rlock:
self._add_perzone_counter(zone_name)
self._statistics_data[self._perzone_prefix]\
[zone_name][counter_name] += step
def __getter(zone_name, counter_name=item):
"""A getter method for perzone counters"""
return isc.cc.data.find(
self._statistics_data,
'%s/%s/%s' % ( self._perzone_prefix,
zone_name,
counter_name )
)
globals()['inc_%s' % item] = __incrementer
globals()['get_%s' % item] = __getter
def _create_xfrrunning_functors(self):
"""Creates increment/decrement method of (a|i)xfr_running
based on the spec file. Incrementer can be accessed by name
"inc_${item_name}". Decrementer can be accessed by name
"dec_${item_name}". Both of them are passed to the
XfroutSession as counter handlers."""
# can be accessed by the name 'inc_xxx' or 'dec_xxx'
for item in self._xfrrunning_names:
def __incrementer(counter_name=item, step=1):
"""A incrementer for axfr or ixfr running. Locks the
thread because it is considered to be invoked by a
multi-threading caller."""
if self._disabled: return
with self._rlock:
self._add_xfrrunning_counter(counter_name)
self._statistics_data[counter_name] += step
def __decrementer(counter_name=item, step=-1):
"""A decrementer for axfr or ixfr running. Locks the
thread because it is considered to be invoked by a
multi-threading caller."""
if self._disabled: return
with self._rlock:
self._statistics_data[counter_name] += step
def __getter(counter_name=item):
"""A getter method for xfr_running counters"""
return isc.cc.data.find(
self._statistics_data, counter_name )
globals()['inc_%s' % item] = __incrementer
globals()['dec_%s' % item] = __decrementer
globals()['get_%s' % item] = __getter
def _create_unixsocket_functors(self):
"""Creates increment/decrement method of (a|i)xfr_running
based on the spec file. Incrementer can be accessed by name
"inc_${item_name}". Decrementer can be accessed by name
"dec_${item_name}". Both of them are passed to the
XfroutSession as counter handlers."""
# can be accessed by the name 'inc_xxx' or 'dec_xxx'
for item in self._unixsocket_names:
def __incrementer(counter_name=item, step=1):
"""A incrementer for axfr or ixfr running. Locks the
thread because it is considered to be invoked by a
multi-threading caller."""
if self._disabled: return
with self._rlock:
self._add_unixsocket_counter(counter_name)
self._statistics_data['socket']['unixdomain']\
[counter_name] += step
def __getter(counter_name=item):
"""A getter method for unixsockets counters"""
return isc.cc.data.find(
self._statistics_data,
'socket/unixdomain/%s' % counter_name )
globals()['inc_unixsocket_%s' % item] = __incrementer
globals()['get_unixsocket_%s' % item] = __getter
def _add_perzone_counter(self, zone):
"""Adds a named_set-type counter for each zone name."""
try:
self._statistics_data[self._perzone_prefix][zone]
except KeyError:
# add a new element under the named_set item for the zone
map_spec = isc.config.find_spec_part(
self._statistics_spec, '%s/%s' % \
(self._perzone_prefix, zone))['map_item_spec']
id_list = isc.config.spec_name_list(map_spec)
for id_ in id_list:
spec = isc.config.find_spec_part(map_spec, id_)
isc.cc.data.set(self._statistics_data,
'%s/%s/%s' % \
(self._perzone_prefix, zone, id_),
spec['item_default'])
def _add_xfrrunning_counter(self, counter_name):
"""Adds a counter for counting (a|i)xfr_running"""
try:
self._statistics_data[counter_name]
except KeyError:
# examines the names of xfer running
for n in self._xfrrunning_names:
spec = isc.config.find_spec_part\
(self._statistics_spec, n)
isc.cc.data.set(self._statistics_data, n, \
spec['item_default'])
def _add_unixsocket_counter(self, counter_name):
"""Adds a counter for counting unix sockets"""
try:
self._statistics_data['socket']['unixdomain'][counter_name]
except KeyError:
# examines the name of unixsocket
name = 'socket/unixdomain/%s' % counter_name
spec = isc.config.find_spec_part\
(self._statistics_spec, name)
isc.cc.data.set(self._statistics_data, name, \
spec['item_default'])
def dump_default_statistics(self):
"""Returns default statistics data from the spec file"""
statistics_data = {}
for id_ in isc.config.spec_name_list(self._statistics_spec):
spec = isc.config.find_spec_part(\
self._statistics_spec, id_)
if 'item_default' in spec:
statistics_data.update({id_: spec['item_default']})
return statistics_data
def dump_statistics(self):
"""Calculates an entire server counts, and returns statistics
data format to send out the stats module including each
counter. If there is no counts, then it returns an empty
dictionary."""
# If self.statistics_data contains nothing of zone name, it
# returns an empty dict.
if len(self._statistics_data) == 0: return {}
# for per-zone counter
zones = self._statistics_data[self._perzone_prefix]
# Start calculation for '_SERVER_' counts
attrs = self.dump_default_statistics()\
[self._perzone_prefix][self._entire_server]
statistics_data = {self._perzone_prefix: {}}
for attr in attrs:
sum_ = 0
for name in zones:
if name == self._entire_server: continue
if attr in zones[name]:
if name not in \
statistics_data[self._perzone_prefix]:
statistics_data[self._perzone_prefix][name]\
= {}
statistics_data[self._perzone_prefix][name].\
update({attr: zones[name][attr]})
sum_ += zones[name][attr]
if sum_ > 0:
if self._entire_server not in \
statistics_data[self._perzone_prefix]:
statistics_data[self._perzone_prefix][self._entire_server]\
= {}
statistics_data[self._perzone_prefix][self._entire_server]\
.update({attr:sum_})
# for xfrrunning incrementer/decrementer
for name in self._xfrrunning_names:
if name in self._statistics_data:
statistics_data[name] = self._statistics_data[name]
# for unixsocket incrementer/decrementer
if 'socket' in self._statistics_data:
statistics_data['socket'] = \
self._statistics_data['socket']
return statistics_data
PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
PYTESTS = counter_test.py
EXTRA_DIST = $(PYTESTS)
# If necessary (rare cases), explicitly specify paths to dynamic libraries
# required by loadable python modules.
LIBRARY_PATH_PLACEHOLDER =
if SET_ENV_LIBRARY_PATH
LIBRARY_PATH_PLACEHOLDER += $(ENV_LIBRARY_PATH)=$(abs_top_builddir)/src/lib/cryptolink/.libs:$(abs_top_builddir)/src/lib/dns/.libs:$(abs_top_builddir)/src/lib/dns/python/.libs:$(abs_top_builddir)/src/lib/cc/.libs:$(abs_top_builddir)/src/lib/config/.libs:$(abs_top_builddir)/src/lib/log/.libs:$(abs_top_builddir)/src/lib/util/.libs:$(abs_top_builddir)/src/lib/exceptions/.libs:$(abs_top_builddir)/src/lib/datasrc/.libs:$$$(ENV_LIBRARY_PATH)
else
# Some systems need the ds path even if not all paths are necessary
LIBRARY_PATH_PLACEHOLDER += $(ENV_LIBRARY_PATH)=$(abs_top_builddir)/src/lib/datasrc/.libs
endif
# test using command-line arguments, so use check-local target instead of TESTS
check-local:
if ENABLE_PYTHON_COVERAGE
touch $(abs_top_srcdir)/.coverage
rm -f .coverage
${LN_S} $(abs_top_srcdir)/.coverage .coverage
endif
for pytest in $(PYTESTS) ; do \
echo Running test: $$pytest ; \
PYTHONPATH=$(COMMON_PYTHON_PATH):$(abs_top_builddir)/src/bin/xfrout:$(abs_top_builddir)/src/lib/dns/python/.libs \
$(LIBRARY_PATH_PLACEHOLDER) \
TESTDATASRCDIR=$(abs_top_srcdir)/src/lib/python/isc/statistics/tests/testdata/ \
B10_FROM_BUILD=$(abs_top_builddir) \
$(PYCOVERAGE_RUN) $(abs_srcdir)/$$pytest || exit ; \
done
# Copyright (C) 2012 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
'''Tests for isc.statistics.counter'''
import unittest
import threading
import isc.config
import xfrout
TEST_ZONE_NAME_STR = "example.com."
from isc.statistics import counter
class TestCounter(unittest.TestCase):
def setUp(self):
module_spec = isc.config.module_spec_from_file(\
xfrout.SPECFILE_LOCATION)
self.counter = counter.Counter(module_spec)
def test_clear_counters(self):
self.counter._statistics_data = {'counter': 1}
self.counter.clear_counters()
self.assertEqual(self.counter._statistics_data,
{})
def test_enablediable(self):
self.assertFalse(self.counter._disabled)
self.counter.disable()
self.assertTrue(self.counter._disabled)
self.counter.enable()
self.assertFalse(self.counter._disabled)
class TestXfroutCounter(unittest.TestCase):
_number = 3 # number of the threads
_cycle = 10000 # number of counting per thread
def setUp(self):
self._module_spec = isc.config.module_spec_from_file(\
xfrout.SPECFILE_LOCATION)
self._statistics_spec = \
self._module_spec.get_statistics_spec()
counter.init(xfrout.SPECFILE_LOCATION)
self.xfrout_counter = counter._COUNTER
self._entire_server = self.xfrout_counter._entire_server
self._perzone_prefix = self.xfrout_counter._perzone_prefix
self._xfrrunning_names = self.xfrout_counter._xfrrunning_names
self._unixsocket_names = self.xfrout_counter._unixsocket_names
self._started = threading.Event()
def test_dump_default_statistics(self):
self.assertTrue(\
self._module_spec.validate_statistics(\
True,
counter.dump_default_statistics(),
)
)
def setup_functor(self, incrementer, *args):
self._started.wait()
for i in range(self._cycle): incrementer(*args)
def start_functor(self, incrementer, *args):
threads = []
for i in range(self._number):
threads.append(threading.Thread(\
target=self.setup_functor, \
args=(incrementer,) + args \
))
for th in threads: th.start()
self._started.set()
for th in threads: th.join()
def get_count(self, zone_name, counter_name):
return isc.cc.data.find(\
counter.dump_statistics(),\
'%s/%s/%s' % (self._perzone_prefix,\
zone_name, counter_name))
def test_functors(self):
# for per-zone counters
result = { self._entire_server: {},
TEST_ZONE_NAME_STR: {} }
self._perzone_counters = isc.config.spec_name_list(\
isc.config.find_spec_part(\
self._statistics_spec, self._perzone_prefix)\
['named_set_item_spec']['map_item_spec'])
for counter_name in self._perzone_counters:
incrementer = getattr(counter,'inc_%s' % counter_name)
self.start_functor(incrementer, TEST_ZONE_NAME_STR)
getter = getattr(counter,'get_%s' % counter_name)
self.assertEqual(getter(TEST_ZONE_NAME_STR),
self._number * self._cycle)
self.assertEqual(self.get_count(self._entire_server,
counter_name), self._number * self._cycle)
# checks disable/enable
counter.disable()
incrementer(TEST_ZONE_NAME_STR)
self.assertEqual(getter(TEST_ZONE_NAME_STR),
self._number * self._cycle)
counter.enable()
incrementer(TEST_ZONE_NAME_STR)
self.assertEqual(getter(TEST_ZONE_NAME_STR),
self._number * self._cycle + 1)
result[self._entire_server][counter_name] = \
result[TEST_ZONE_NAME_STR][counter_name] = \
self._number * self._cycle + 1
statistics_data = {self._perzone_prefix: result}
# for {a|i}xfrrunning counters
for counter_name in self._xfrrunning_names:
incrementer = getattr(counter,'inc_%s' % counter_name)
self.start_functor(incrementer)
getter = getattr(counter,'get_%s' % counter_name)
self.assertEqual(getter(), self._number * self._cycle)
decrementer = getattr(counter,'dec_%s' % counter_name)
self.start_functor(decrementer)
self.assertEqual(getter(), 0)
# checks disable/enable
counter.disable()
incrementer()
self.assertEqual(getter(), 0)
counter.enable()
incrementer()
self.assertGreater(getter(), 0)
counter.disable()
decrementer()
self.assertGreater(getter(), 0)
counter.enable()
decrementer()
self.assertEqual(getter(), 0)
statistics_data[counter_name] = 0
# for unixsocket counters
statistics_data.update({'socket': {'unixdomain': {}}})
for counter_name in self._unixsocket_names:
incrementer = getattr(counter, 'inc_unixsocket_%s' % counter_name)
self.start_functor(incrementer)
getter = getattr(counter, 'get_unixsocket_%s' % counter_name)
self.assertEqual(getter(), self._number * self._cycle)
# checks disable/enable
counter.disable()
incrementer()
self.assertEqual(getter(), self._number * self._cycle)
counter.enable()
incrementer()
self.assertEqual(getter(), self._number * self._cycle + 1)
statistics_data['socket']['unixdomain'][counter_name] = \
self._number * self._cycle + 1
# totally chacking
self.assertEqual(counter.dump_statistics(), statistics_data)
self.assertTrue(self._module_spec.validate_statistics(\
True, statistics_data))
if __name__== "__main__":
unittest.main()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment