Commit 775f572c authored by Naoki Kambe's avatar Naoki Kambe
Browse files

[master] Merge branch 'trac2843'

parents 3056c7dd 7a6cf135
......@@ -28,7 +28,7 @@ import time
from functools import reduce
from optparse import OptionParser, OptionValueError
from isc.config.ccsession import *
from isc.statistics import Counters
from isc.statistics.dns import Counters
from isc.notify import notify_out
import isc.util.process
from isc.util.address_formatter import AddressFormatter
......
......@@ -27,7 +27,7 @@ from socketserver import *
import os
from isc.config.ccsession import *
from isc.cc import SessionError, SessionTimeout
from isc.statistics import Counters
from isc.statistics.dns import Counters
from isc.notify import notify_out
import isc.util.process
import fcntl
......
......@@ -25,7 +25,7 @@ from isc.datasrc import DataSourceClient
from isc.net import addr
import isc
from isc.log_messages.notify_out_messages import *
from isc.statistics import Counters
from isc.statistics.dns import Counters
from isc.util.address_formatter import AddressFormatter
logger = isc.log.Logger("notify_out")
......
SUBDIRS = . tests
python_PYTHON = __init__.py counters.py
python_PYTHON = __init__.py counters.py dns.py
pythondir = $(pyexecdir)/isc/statistics
CLEANDIRS = __pycache__
......
......@@ -17,52 +17,34 @@
This module handles the statistics counters for BIND 10 modules. For
using the module `counter.py`, first a counters object should be created
in each module (like b10-xfrin or b10-xfrout) after importing this
module. A spec file can be specified as an argument when creating the
counters object:
in each module like b10-foo after importing this module. A spec file can
be specified as an argument when creating the counters object:
from isc.statistics import Counters
self.counters = Counters("/path/to/foo.spec")
The first argument of Counters() can be specified, which is the location
of the specification file (like src/bin/xfrout/xfrout.spec). If Counters
is constructed this way, statistics counters can be accessed from each
module. For example, in case that the item `xfrreqdone` is defined in
statistics_spec in xfrout.spec, the following methods are
callable. Since these methods require the string of the zone name in the
first argument, if we have the following code in b10-xfrout:
of the specification file. If Counters is constructed this way,
statistics counters can be accessed from each module. For example, in
case that the item `counter1` is defined in statistics_spec in foo.spec,
the following methods are callable.
self.counters.inc('zones', zone_name, 'xfrreqdone')
self.counters.inc('counter1')
then the counter for xfrreqdone corresponding to zone_name is
incremented. For getting the current number of this counter, we can use
the following code:
Then the counter for `counter1` is incremented. For getting the current
number of this counter, we can use the following code:
number = self.counters.get('zones', zone_name, 'xfrreqdone')
number = self.counters.get('counter1')
then the current count is obtained and set in the variable
Then the current count is obtained and set in the variable
`number`. Such a getter method would be mainly used for unit-testing.
As other example, for the item `axfr_running`, the decrementer method is
also callable. This method is used for decrementing a counter. For the
item `axfr_running`, an argument like zone name is not required:
The decrementer method is also callable. This method is used for
decrementing a counter as well as inc().
self.counters.dec('axfr_running')
self.counters.dec('counter2')
These methods are effective in other modules. For example, in case that
this module `counter.py` is once imported in a main module such as
b10-xfrout, then for the item `notifyoutv4`, the `inc()` method can be
invoked in another module such as notify_out.py, which is firstly
imported in the main module.
self.counters.inc('zones', zone_name, 'notifyoutv4')
In this example this is for incrementing the counter of the item
`notifyoutv4`. Thus, such statement can be also written in another
library like isc.notify.notify_out. If this module `counter.py` isn't
imported in the main module but imported in such a library module as
isc.notify.notify_out, in this example, empty methods would be invoked,
which is directly defined in `counter.py`.
"""
Some other methods accessible to a counter are provided by this
module."""
import threading
import isc.config
......@@ -170,60 +152,13 @@ def _concat(*args, sep='/'):
return sep.join(args)
class _Statistics():
"""Statistics data set"""
"""Statistics data set. This class will be remove in the future
release."""
# default statistics data
_data = {}
# default statistics spec used in case the specfile is omitted when
# constructing a Counters() object
_spec = [
{
"item_name": "zones",
"item_type": "named_set",
"item_optional": False,
"item_default": {
"_SERVER_" : {
"notifyoutv4" : 0,
"notifyoutv6" : 0
}
},
"item_title": "Zone names",
"item_description": "Zone names",
"named_set_item_spec": {
"item_name": "classname",
"item_type": "named_set",
"item_optional": False,
"item_default": {},
"item_title": "RR class name",
"item_description": "RR class name",
"named_set_item_spec": {
"item_name": "zonename",
"item_type": "map",
"item_optional": False,
"item_default": {},
"item_title": "Zone name",
"item_description": "Zone name",
"map_item_spec": [
{
"item_name": "notifyoutv4",
"item_type": "integer",
"item_optional": False,
"item_default": 0,
"item_title": "IPv4 notifies",
"item_description": "Number of IPv4 notifies per zone name sent out"
},
{
"item_name": "notifyoutv6",
"item_type": "integer",
"item_optional": False,
"item_default": 0,
"item_title": "IPv6 notifies",
"item_description": "Number of IPv6 notifies per zone name sent out"
}
]
}
}
}
]
_spec = []
class Counters():
"""A class for holding and manipulating all statistics counters
......@@ -237,56 +172,8 @@ class Counters():
stop_timer() and get() are useful for this. Saved counters can be
cleared by the method clear_all(). Manipulating counters and
timers can be temporarily disabled. If disabled, counter values are
not changed even if methods to update them are invoked. Including
per-zone counters, a list of counters which can be handled in the
class are like the following:
zones/IN/example.com./notifyoutv4
zones/IN/example.com./notifyoutv6
zones/IN/example.com./xfrrej
zones/IN/example.com./xfrreqdone
zones/IN/example.com./soaoutv4
zones/IN/example.com./soaoutv6
zones/IN/example.com./axfrreqv4
zones/IN/example.com./axfrreqv6
zones/IN/example.com./ixfrreqv4
zones/IN/example.com./ixfrreqv6
zones/IN/example.com./xfrsuccess
zones/IN/example.com./xfrfail
zones/IN/example.com./last_ixfr_duration
zones/IN/example.com./last_axfr_duration
ixfr_running
axfr_running
socket/unixdomain/open
socket/unixdomain/openfail
socket/unixdomain/close
socket/unixdomain/bindfail
socket/unixdomain/acceptfail
socket/unixdomain/accept
socket/unixdomain/senderr
socket/unixdomain/recverr
socket/ipv4/tcp/open
socket/ipv4/tcp/openfail
socket/ipv4/tcp/close
socket/ipv4/tcp/connfail
socket/ipv4/tcp/conn
socket/ipv4/tcp/senderr
socket/ipv4/tcp/recverr
socket/ipv6/tcp/open
socket/ipv6/tcp/openfail
socket/ipv6/tcp/close
socket/ipv6/tcp/connfail
socket/ipv6/tcp/conn
socket/ipv6/tcp/senderr
socket/ipv6/tcp/recverr
"""
not changed even if methods to update them are invoked."""
# '_SERVER_' is a special zone name representing an entire
# count. It doesn't mean a specific zone, but it means an
# entire count in the server.
_entire_server = '_SERVER_'
# zone names are contained under this dirname in the spec file.
_perzone_prefix = 'zones'
# default statistics data set
_statistics = _Statistics()
......@@ -296,7 +183,8 @@ class Counters():
statistics spec can be accumulated if spec_file_name is
specified. If omitted, a default statistics spec is used. The
default statistics spec is defined in a hidden class named
_Statistics().
_Statistics(). But the hidden class won't be used and
spec_file_name will be required in the future release.
"""
self._zones_item_list = []
self._start_time = {}
......@@ -307,13 +195,6 @@ class Counters():
self._statistics._spec = \
isc.config.module_spec_from_file(spec_file_name).\
get_statistics_spec()
if self._perzone_prefix in \
isc.config.spec_name_list(self._statistics._spec):
self._zones_item_list = isc.config.spec_name_list(
isc.config.find_spec_part(
self._statistics._spec,
'%s/%s/%s' % (self._perzone_prefix,
'_CLASS_', self._entire_server)))
def clear_all(self):
"""clears all statistics data"""
......@@ -408,32 +289,9 @@ class Counters():
del branch_map[leaf]
def get_statistics(self):
"""Calculates an entire server's counts, and returns statistics
data in a format to send out to the stats module, including each
counter. If nothing is counted yet, then it returns an empty
dictionary."""
"""Returns statistics data in a format to send out to the
stats module, including each counter. If nothing is counted
yet, then it returns an empty dictionary."""
# entire copy
statistics_data = self._statistics._data.copy()
# If there is no 'zones' found in statistics_data,
# i.e. statistics_data contains no per-zone counter, it just
# returns statistics_data because calculating total counts
# across the zone names isn't necessary.
if self._perzone_prefix not in statistics_data:
return statistics_data
zones = statistics_data[self._perzone_prefix]
# Start calculation for '_SERVER_' counts
zones_spec = isc.config.find_spec_part(self._statistics._spec,
self._perzone_prefix)
zones_data = {}
for cls in zones.keys():
for zone in zones[cls].keys():
for (attr, val) in zones[cls][zone].items():
id_str1 = '%s/%s/%s' % (cls, zone, attr)
id_str2 = '%s/%s/%s' % (cls, self._entire_server, attr)
_set_counter(zones_data, zones_spec, id_str1, val)
_inc_counter(zones_data, zones_spec, id_str2, val)
# insert entire-server counts
statistics_data[self._perzone_prefix] = dict(
statistics_data[self._perzone_prefix],
**zones_data)
return statistics_data
# Copyright (C) 2013 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
"""BIND 10 statistics counters module for DNS
This module handles the statistics counters for BIND 10 modules for a
DNS-specific purpose. For using the module `counter.py`, first a
counters object should be created in each module (like b10-xfrin or
b10-xfrout) after importing this module. A spec file can be specified as
an argument when creating the counters object:
from isc.statistics.dns import Counters
self.counters = Counters("/path/to/xfrout/xfrout.spec")
The first argument of Counters() can be specified, which is the location
of the specification file. If Counters is constructed this way,
statistics counters can be accessed from each module. For example, in
case that the item `xfrreqdone` is defined in statistics_spec in
xfrout.spec, the following methods are callable. Since these methods
require the string of the zone name in the first argument, if we have
the following code in b10-xfrout:
self.counters.inc('zones', zone_name, 'xfrreqdone')
then the counter for xfrreqdone corresponding to zone_name is
incremented. For getting the current number of this counter, we can use
the following code:
number = self.counters.get('zones', zone_name, 'xfrreqdone')
then the current count is obtained and set in the variable
`number`. Such a getter method would be mainly used for unit-testing.
As other example, for the item `axfr_running`, the decrementer method is
also callable. This method is used for decrementing a counter. For the
item `axfr_running`, an argument like zone name is not required:
self.counters.dec('axfr_running')
These methods are effective in other modules. For example, in case that
this module `counters.py` is once imported in a main module such as
b10-xfrout, then for the item `notifyoutv4`, the `inc()` method can be
invoked in another module such as notify_out.py, which is firstly
imported in the main module.
self.counters.inc('zones', zone_name, 'notifyoutv4')
In this example this is for incrementing the counter of the item
`notifyoutv4`. Thus, such statement can be also written in another
library like isc.notify.notify_out. If this module `counter.py` isn't
imported in the main module but imported in such a library module as
isc.notify.notify_out, in this example, empty methods would be invoked,
which is directly defined in `counter.py`.
This module basically inherits isc.statistics.counters. Also see
documentation for isc.statistics.counters for details."""
import isc.config
from isc.statistics import counters
class _Statistics():
"""Statistics data set. This class will be removed in the future
release."""
# default statistics data
_data = {}
# default statistics spec used in case the specfile is omitted when
# constructing a Counters() object
_spec = [
{
"item_name": "zones",
"item_type": "named_set",
"item_optional": False,
"item_default": {
"_SERVER_" : {
"notifyoutv4" : 0,
"notifyoutv6" : 0
}
},
"item_title": "Zone names",
"item_description": "Zone names",
"named_set_item_spec": {
"item_name": "classname",
"item_type": "named_set",
"item_optional": False,
"item_default": {},
"item_title": "RR class name",
"item_description": "RR class name",
"named_set_item_spec": {
"item_name": "zonename",
"item_type": "map",
"item_optional": False,
"item_default": {},
"item_title": "Zone name",
"item_description": "Zone name",
"map_item_spec": [
{
"item_name": "notifyoutv4",
"item_type": "integer",
"item_optional": False,
"item_default": 0,
"item_title": "IPv4 notifies",
"item_description": "Number of IPv4 notifies per zone name sent out"
},
{
"item_name": "notifyoutv6",
"item_type": "integer",
"item_optional": False,
"item_default": 0,
"item_title": "IPv6 notifies",
"item_description": "Number of IPv6 notifies per zone name sent out"
}
]
}
}
}
]
class Counters(counters.Counters):
"""A list of counters which can be handled in the class are like
the following. Also see documentation for
isc.statistics.counters.Counters for details.
zones/IN/example.com./notifyoutv4
zones/IN/example.com./notifyoutv6
zones/IN/example.com./xfrrej
zones/IN/example.com./xfrreqdone
zones/IN/example.com./soaoutv4
zones/IN/example.com./soaoutv6
zones/IN/example.com./axfrreqv4
zones/IN/example.com./axfrreqv6
zones/IN/example.com./ixfrreqv4
zones/IN/example.com./ixfrreqv6
zones/IN/example.com./xfrsuccess
zones/IN/example.com./xfrfail
zones/IN/example.com./last_ixfr_duration
zones/IN/example.com./last_axfr_duration
ixfr_running
axfr_running
socket/unixdomain/open
socket/unixdomain/openfail
socket/unixdomain/close
socket/unixdomain/bindfail
socket/unixdomain/acceptfail
socket/unixdomain/accept
socket/unixdomain/senderr
socket/unixdomain/recverr
socket/ipv4/tcp/open
socket/ipv4/tcp/openfail
socket/ipv4/tcp/close
socket/ipv4/tcp/connfail
socket/ipv4/tcp/conn
socket/ipv4/tcp/senderr
socket/ipv4/tcp/recverr
socket/ipv6/tcp/open
socket/ipv6/tcp/openfail
socket/ipv6/tcp/close
socket/ipv6/tcp/connfail
socket/ipv6/tcp/conn
socket/ipv6/tcp/senderr
socket/ipv6/tcp/recverr
"""
# '_SERVER_' is a special zone name representing an entire
# count. It doesn't mean a specific zone, but it means an
# entire count in the server.
_entire_server = '_SERVER_'
# zone names are contained under this dirname in the spec file.
_perzone_prefix = 'zones'
# default statistics data set
_statistics = _Statistics()
def __init__(self, spec_file_name=None):
"""If the item `zones` is defined in the spec file, it obtains a
list of counter names under it when initiating. For behaviors
other than this, see documentation for
isc.statistics.counters.Counters.__init__()"""
counters.Counters.__init__(self, spec_file_name)
if self._perzone_prefix in \
isc.config.spec_name_list(self._statistics._spec):
self._zones_item_list = isc.config.spec_name_list(
isc.config.find_spec_part(
self._statistics._spec,
'%s/%s/%s' % (self._perzone_prefix,
'_CLASS_', self._entire_server)))
def get_statistics(self):
"""Calculates an entire server's counts, and returns statistics
data in a format to send out to the stats module, including each
counter. If nothing is counted yet, then it returns an empty
dictionary."""
# entire copy
statistics_data = self._statistics._data.copy()
# If there is no 'zones' found in statistics_data,
# i.e. statistics_data contains no per-zone counter, it just
# returns statistics_data because calculating total counts
# across the zone names isn't necessary.
if self._perzone_prefix not in statistics_data:
return statistics_data
zones = statistics_data[self._perzone_prefix]
# Start calculation for '_SERVER_' counts
zones_spec = isc.config.find_spec_part(self._statistics._spec,
self._perzone_prefix)
zones_data = {}
for cls in zones.keys():
for zone in zones[cls].keys():
for (attr, val) in zones[cls][zone].items():
id_str1 = '%s/%s/%s' % (cls, zone, attr)
id_str2 = '%s/%s/%s' % (cls, self._entire_server, attr)
counters._set_counter(zones_data, zones_spec, id_str1, val)
counters._inc_counter(zones_data, zones_spec, id_str2, val)
# insert entire-server counts
statistics_data[self._perzone_prefix] = dict(
statistics_data[self._perzone_prefix],
**zones_data)
return statistics_data
PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
PYTESTS = counters_test.py
PYTESTS = counters_test.py dns_test.py
EXTRA_DIST = $(PYTESTS)
EXTRA_DIST += testdata/test_spec1.spec
EXTRA_DIST += testdata/test_spec2.spec
......
......@@ -13,7 +13,7 @@
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
'''Tests for isc.statistics.counter'''
'''Tests for isc.statistics.counters'''
import unittest
import threading
......@@ -22,8 +22,6 @@ import os
import imp
import isc.config
TEST_ZONE_NAME_STR = "example.com."
TEST_ZONE_CLASS_STR = "IN"
TESTDATA_SRCDIR = os.getenv("TESTDATASRCDIR")
from isc.statistics import counters
......@@ -166,8 +164,6 @@ class BaseTestCounters():
imp.reload(counters)
self._statistics_data = {}
self.counters = counters.Counters(self.TEST_SPECFILE_LOCATION)
self._entire_server = self.counters._entire_server
self._perzone_prefix = self.counters._perzone_prefix
def tearDown(self):
self.counters.clear_all()
......@@ -194,107 +190,6 @@ class BaseTestCounters():
).validate_statistics(
False, self._statistics_data))
def test_perzone_counters(self):
# for per-zone counters
for name in self.counters._zones_item_list:
args = (self._perzone_prefix, TEST_ZONE_CLASS_STR,
TEST_ZONE_NAME_STR, name)
if name.find('last_') == 0 and name.endswith('_duration'):
self.counters.start_timer(*args)
self.counters.stop_timer(*args)
self.assertGreaterEqual(self.counters.get(*args), 0.0)
sec = self.counters.get(*args)
for zone_str in (self._entire_server, TEST_ZONE_NAME_STR):
isc.cc.data.set(self._statistics_data,
'%s/%s/%s/%s' % (args[0], args[1],
zone_str, name), sec)
# twice exec stopper, then second is not changed
self.counters.stop_timer(*args)
self.assertEqual(self.counters.get(*args), sec)
else:
self.counters.inc(*args)
self.assertEqual(self.counters.get(*args), 1)
# checks disable/enable
self.counters.disable()
self.counters.inc(*args)
self.assertEqual(self.counters.get(*args), 1)
self.counters.enable()
self.counters.inc(*args)
self.assertEqual(self.counters.get(*args), 2)
for zone_str in (self._entire_server, TEST_ZONE_NAME_STR):
isc.cc.data.set(self._statistics_data,
'%s/%s/%s/%s' % (args[0], args[1],
zone_str, name), 2)
self.check_get_statistics()
def test_xfrrunning_counters(self):
# for counters of xfer running
_suffix = 'xfr_running'
_xfrrunning_names = \
isc.config.spec_name_list(self.counters._statistics._spec,
"", True)
for name in _xfrrunning_names:
if name.find(_suffix) != 1: continue
args = name.split('/')
self.counters.inc(*args)
self.assertEqual(self.counters.get(*args), 1)
self.counters.dec(*args)
self.assertEqual(self.counters.get(*args), 0)
# checks disable/enable
self.counters.disable()
self.counters.inc(*args)
self.assertEqual(self.counters.get(*args), 0)
self.counters.enable()
self.counters.inc(*args)
self.assertEqual(self.counters.get(*args), 1)
self.counters.disable()
self.counters.dec(*args)
self.assertEqual(self.counters.get(*args), 1)
self.counters.enable()
self.counters.dec(*args)
self.assertEqual(self.counters.get(*args), 0)
self._statistics_data[name] = 0
self.check_get_statistics()
def test_socket_counters(self):