[master] Merge branch 'trac2843'

src/bin/xfrin/xfrin.py.in

@@ -28,7 +28,7 @@ import time
 from functools import reduce
 from optparse import OptionParser, OptionValueError
 from isc.config.ccsession import *
-from isc.statistics import Counters
+from isc.statistics.dns import Counters
 from isc.notify import notify_out
 import isc.util.process
 from isc.util.address_formatter import AddressFormatter

src/bin/xfrout/xfrout.py.in

@@ -27,7 +27,7 @@ from socketserver import *
 import os
 from isc.config.ccsession import *
 from isc.cc import SessionError, SessionTimeout
-from isc.statistics import Counters
+from isc.statistics.dns import Counters
 from isc.notify import notify_out
 import isc.util.process
 import fcntl

src/lib/python/isc/notify/notify_out.py

@@ -25,7 +25,7 @@ from isc.datasrc import DataSourceClient
 from isc.net import addr
 import isc
 from isc.log_messages.notify_out_messages import *
-from isc.statistics import Counters
+from isc.statistics.dns import Counters
 from isc.util.address_formatter import AddressFormatter
 
 logger = isc.log.Logger("notify_out")

src/lib/python/isc/statistics/Makefile.am

 SUBDIRS = . tests
 
-python_PYTHON = __init__.py counters.py
+python_PYTHON = __init__.py counters.py dns.py
 pythondir = $(pyexecdir)/isc/statistics
 
 CLEANDIRS = __pycache__

src/lib/python/isc/statistics/counters.py

@@ -17,52 +17,34 @@
 
 This module handles the statistics counters for BIND 10 modules.  For
 using the module `counter.py`, first a counters object should be created
-in each module (like b10-xfrin or b10-xfrout) after importing this
-module. A spec file can be specified as an argument when creating the
-counters object:
+in each module like b10-foo after importing this module. A spec file can
+be specified as an argument when creating the counters object:
 
   from isc.statistics import Counters
   self.counters = Counters("/path/to/foo.spec")
 
 The first argument of Counters() can be specified, which is the location
-of the specification file (like src/bin/xfrout/xfrout.spec). If Counters
-is constructed this way, statistics counters can be accessed from each
-module. For example, in case that the item `xfrreqdone` is defined in
-statistics_spec in xfrout.spec, the following methods are
-callable. Since these methods require the string of the zone name in the
-first argument, if we have the following code in b10-xfrout:
+of the specification file. If Counters is constructed this way,
+statistics counters can be accessed from each module. For example, in
+case that the item `counter1` is defined in statistics_spec in foo.spec,
+the following methods are callable.
 
-  self.counters.inc('zones', zone_name, 'xfrreqdone')
+  self.counters.inc('counter1')
 
-then the counter for xfrreqdone corresponding to zone_name is
-incremented. For getting the current number of this counter, we can use
-the following code:
+Then the counter for `counter1` is incremented. For getting the current
+number of this counter, we can use the following code:
 
-  number = self.counters.get('zones', zone_name, 'xfrreqdone')
+  number = self.counters.get('counter1')
 
-then the current count is obtained and set in the variable
+Then the current count is obtained and set in the variable
 `number`. Such a getter method would be mainly used for unit-testing.
-As other example, for the item `axfr_running`, the decrementer method is
-also callable.  This method is used for decrementing a counter.  For the
-item `axfr_running`, an argument like zone name is not required:
+The decrementer method is also callable.  This method is used for
+decrementing a counter as well as inc().
 
-  self.counters.dec('axfr_running')
+  self.counters.dec('counter2')
 
-These methods are effective in other modules. For example, in case that
-this module `counter.py` is once imported in a main module such as
-b10-xfrout, then for the item `notifyoutv4`, the `inc()` method can be
-invoked in another module such as notify_out.py, which is firstly
-imported in the main module.
-
-  self.counters.inc('zones', zone_name, 'notifyoutv4')
-
-In this example this is for incrementing the counter of the item
-`notifyoutv4`. Thus, such statement can be also written in another
-library like isc.notify.notify_out. If this module `counter.py` isn't
-imported in the main module but imported in such a library module as
-isc.notify.notify_out, in this example, empty methods would be invoked,
-which is directly defined in `counter.py`.
-"""
+Some other methods accessible to a counter are provided by this
+module."""
 
 import threading
 import isc.config
@@ -170,60 +152,13 @@ def _concat(*args, sep='/'):
     return sep.join(args)
 
 class _Statistics():
-    """Statistics data set"""
+    """Statistics data set. This class will be remove in the future
+    release."""
     # default statistics data
     _data = {}
     # default statistics spec used in case the specfile is omitted when
     # constructing a Counters() object
-    _spec = [
-      {
-        "item_name": "zones",
-        "item_type": "named_set",
-        "item_optional": False,
-        "item_default": {
-          "_SERVER_" : {
-            "notifyoutv4" : 0,
-            "notifyoutv6" : 0
-          }
-        },
-        "item_title": "Zone names",
-        "item_description": "Zone names",
-        "named_set_item_spec": {
-          "item_name": "classname",
-          "item_type": "named_set",
-          "item_optional": False,
-          "item_default": {},
-          "item_title": "RR class name",
-          "item_description": "RR class name",
-          "named_set_item_spec": {
-            "item_name": "zonename",
-            "item_type": "map",
-            "item_optional": False,
-            "item_default": {},
-            "item_title": "Zone name",
-            "item_description": "Zone name",
-            "map_item_spec": [
-              {
-                "item_name": "notifyoutv4",
-                "item_type": "integer",
-                "item_optional": False,
-                "item_default": 0,
-                "item_title": "IPv4 notifies",
-                "item_description": "Number of IPv4 notifies per zone name sent out"
-              },
-              {
-                "item_name": "notifyoutv6",
-                "item_type": "integer",
-                "item_optional": False,
-                "item_default": 0,
-                "item_title": "IPv6 notifies",
-                "item_description": "Number of IPv6 notifies per zone name sent out"
-              }
-            ]
-          }
-        }
-      }
-    ]
+    _spec = []
 
 class Counters():
     """A class for holding and manipulating all statistics counters
@@ -237,56 +172,8 @@ class Counters():
     stop_timer() and get() are useful for this. Saved counters can be
     cleared by the method clear_all(). Manipulating counters and
     timers can be temporarily disabled.  If disabled, counter values are
-    not changed even if methods to update them are invoked.  Including
-    per-zone counters, a list of counters which can be handled in the
-    class are like the following:
-
-        zones/IN/example.com./notifyoutv4
-        zones/IN/example.com./notifyoutv6
-        zones/IN/example.com./xfrrej
-        zones/IN/example.com./xfrreqdone
-        zones/IN/example.com./soaoutv4
-        zones/IN/example.com./soaoutv6
-        zones/IN/example.com./axfrreqv4
-        zones/IN/example.com./axfrreqv6
-        zones/IN/example.com./ixfrreqv4
-        zones/IN/example.com./ixfrreqv6
-        zones/IN/example.com./xfrsuccess
-        zones/IN/example.com./xfrfail
-        zones/IN/example.com./last_ixfr_duration
-        zones/IN/example.com./last_axfr_duration
-        ixfr_running
-        axfr_running
-        socket/unixdomain/open
-        socket/unixdomain/openfail
-        socket/unixdomain/close
-        socket/unixdomain/bindfail
-        socket/unixdomain/acceptfail
-        socket/unixdomain/accept
-        socket/unixdomain/senderr
-        socket/unixdomain/recverr
-        socket/ipv4/tcp/open
-        socket/ipv4/tcp/openfail
-        socket/ipv4/tcp/close
-        socket/ipv4/tcp/connfail
-        socket/ipv4/tcp/conn
-        socket/ipv4/tcp/senderr
-        socket/ipv4/tcp/recverr
-        socket/ipv6/tcp/open
-        socket/ipv6/tcp/openfail
-        socket/ipv6/tcp/close
-        socket/ipv6/tcp/connfail
-        socket/ipv6/tcp/conn
-        socket/ipv6/tcp/senderr
-        socket/ipv6/tcp/recverr
-    """
+    not changed even if methods to update them are invoked."""
 
-    # '_SERVER_' is a special zone name representing an entire
-    # count. It doesn't mean a specific zone, but it means an
-    # entire count in the server.
-    _entire_server = '_SERVER_'
-    # zone names are contained under this dirname in the spec file.
-    _perzone_prefix = 'zones'
     # default statistics data set
     _statistics = _Statistics()
 
@@ -296,7 +183,8 @@ class Counters():
         statistics spec can be accumulated if spec_file_name is
         specified. If omitted, a default statistics spec is used. The
         default statistics spec is defined in a hidden class named
-        _Statistics().
+        _Statistics(). But the hidden class won't be used and
+        spec_file_name will be required in the future release.
         """
         self._zones_item_list = []
         self._start_time = {}
@@ -307,13 +195,6 @@ class Counters():
         self._statistics._spec = \
             isc.config.module_spec_from_file(spec_file_name).\
             get_statistics_spec()
-        if self._perzone_prefix in \
-                isc.config.spec_name_list(self._statistics._spec):
-            self._zones_item_list = isc.config.spec_name_list(
-                isc.config.find_spec_part(
-                    self._statistics._spec,
-                    '%s/%s/%s' % (self._perzone_prefix,
-                                  '_CLASS_', self._entire_server)))
 
     def clear_all(self):
         """clears all statistics data"""
@@ -408,32 +289,9 @@ class Counters():
             del branch_map[leaf]
 
     def get_statistics(self):
-        """Calculates an entire server's counts, and returns statistics
-        data in a format to send out to the stats module, including each
-        counter. If nothing is counted yet, then it returns an empty
-        dictionary."""
+        """Returns statistics data in a format to send out to the
+        stats module, including each counter. If nothing is counted
+        yet, then it returns an empty dictionary."""
         # entire copy
         statistics_data = self._statistics._data.copy()
-        # If there is no 'zones' found in statistics_data,
-        # i.e. statistics_data contains no per-zone counter, it just
-        # returns statistics_data because calculating total counts
-        # across the zone names isn't necessary.
-        if self._perzone_prefix not in statistics_data:
-            return statistics_data
-        zones = statistics_data[self._perzone_prefix]
-        # Start calculation for '_SERVER_' counts
-        zones_spec = isc.config.find_spec_part(self._statistics._spec,
-                                               self._perzone_prefix)
-        zones_data = {}
-        for cls in zones.keys():
-            for zone in zones[cls].keys():
-                for (attr, val) in zones[cls][zone].items():
-                    id_str1 = '%s/%s/%s' % (cls, zone, attr)
-                    id_str2 = '%s/%s/%s' % (cls, self._entire_server, attr)
-                    _set_counter(zones_data, zones_spec, id_str1, val)
-                    _inc_counter(zones_data, zones_spec, id_str2, val)
-        # insert entire-server counts
-        statistics_data[self._perzone_prefix] = dict(
-            statistics_data[self._perzone_prefix],
-            **zones_data)
         return statistics_data

src/lib/python/isc/statistics/dns.py

+# Copyright (C) 2013  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""BIND 10 statistics counters module for DNS
+
+This module handles the statistics counters for BIND 10 modules for a
+DNS-specific purpose.  For using the module `counter.py`, first a
+counters object should be created in each module (like b10-xfrin or
+b10-xfrout) after importing this module. A spec file can be specified as
+an argument when creating the counters object:
+
+  from isc.statistics.dns import Counters
+  self.counters = Counters("/path/to/xfrout/xfrout.spec")
+
+The first argument of Counters() can be specified, which is the location
+of the specification file. If Counters is constructed this way,
+statistics counters can be accessed from each module. For example, in
+case that the item `xfrreqdone` is defined in statistics_spec in
+xfrout.spec, the following methods are callable. Since these methods
+require the string of the zone name in the first argument, if we have
+the following code in b10-xfrout:
+
+  self.counters.inc('zones', zone_name, 'xfrreqdone')
+
+then the counter for xfrreqdone corresponding to zone_name is
+incremented. For getting the current number of this counter, we can use
+the following code:
+
+  number = self.counters.get('zones', zone_name, 'xfrreqdone')
+
+then the current count is obtained and set in the variable
+`number`. Such a getter method would be mainly used for unit-testing.
+As other example, for the item `axfr_running`, the decrementer method is
+also callable.  This method is used for decrementing a counter.  For the
+item `axfr_running`, an argument like zone name is not required:
+
+  self.counters.dec('axfr_running')
+
+These methods are effective in other modules. For example, in case that
+this module `counters.py` is once imported in a main module such as
+b10-xfrout, then for the item `notifyoutv4`, the `inc()` method can be
+invoked in another module such as notify_out.py, which is firstly
+imported in the main module.
+
+  self.counters.inc('zones', zone_name, 'notifyoutv4')
+
+In this example this is for incrementing the counter of the item
+`notifyoutv4`. Thus, such statement can be also written in another
+library like isc.notify.notify_out. If this module `counter.py` isn't
+imported in the main module but imported in such a library module as
+isc.notify.notify_out, in this example, empty methods would be invoked,
+which is directly defined in `counter.py`.
+
+This module basically inherits isc.statistics.counters.  Also see
+documentation for isc.statistics.counters for details."""
+
+import isc.config
+from isc.statistics import counters
+
+class _Statistics():
+    """Statistics data set. This class will be removed in the future
+    release."""
+    # default statistics data
+    _data = {}
+    # default statistics spec used in case the specfile is omitted when
+    # constructing a Counters() object
+    _spec = [
+      {
+        "item_name": "zones",
+        "item_type": "named_set",
+        "item_optional": False,
+        "item_default": {
+          "_SERVER_" : {
+            "notifyoutv4" : 0,
+            "notifyoutv6" : 0
+          }
+        },
+        "item_title": "Zone names",
+        "item_description": "Zone names",
+        "named_set_item_spec": {
+          "item_name": "classname",
+          "item_type": "named_set",
+          "item_optional": False,
+          "item_default": {},
+          "item_title": "RR class name",
+          "item_description": "RR class name",
+          "named_set_item_spec": {
+            "item_name": "zonename",
+            "item_type": "map",
+            "item_optional": False,
+            "item_default": {},
+            "item_title": "Zone name",
+            "item_description": "Zone name",
+            "map_item_spec": [
+              {
+                "item_name": "notifyoutv4",
+                "item_type": "integer",
+                "item_optional": False,
+                "item_default": 0,
+                "item_title": "IPv4 notifies",
+                "item_description": "Number of IPv4 notifies per zone name sent out"
+              },
+              {
+                "item_name": "notifyoutv6",
+                "item_type": "integer",
+                "item_optional": False,
+                "item_default": 0,
+                "item_title": "IPv6 notifies",
+                "item_description": "Number of IPv6 notifies per zone name sent out"
+              }
+            ]
+          }
+        }
+      }
+    ]
+
+class Counters(counters.Counters):
+    """A list of counters which can be handled in the class are like
+    the following. Also see documentation for
+    isc.statistics.counters.Counters for details.
+
+        zones/IN/example.com./notifyoutv4
+        zones/IN/example.com./notifyoutv6
+        zones/IN/example.com./xfrrej
+        zones/IN/example.com./xfrreqdone
+        zones/IN/example.com./soaoutv4
+        zones/IN/example.com./soaoutv6
+        zones/IN/example.com./axfrreqv4
+        zones/IN/example.com./axfrreqv6
+        zones/IN/example.com./ixfrreqv4
+        zones/IN/example.com./ixfrreqv6
+        zones/IN/example.com./xfrsuccess
+        zones/IN/example.com./xfrfail
+        zones/IN/example.com./last_ixfr_duration
+        zones/IN/example.com./last_axfr_duration
+        ixfr_running
+        axfr_running
+        socket/unixdomain/open
+        socket/unixdomain/openfail
+        socket/unixdomain/close
+        socket/unixdomain/bindfail
+        socket/unixdomain/acceptfail
+        socket/unixdomain/accept
+        socket/unixdomain/senderr
+        socket/unixdomain/recverr
+        socket/ipv4/tcp/open
+        socket/ipv4/tcp/openfail
+        socket/ipv4/tcp/close
+        socket/ipv4/tcp/connfail
+        socket/ipv4/tcp/conn
+        socket/ipv4/tcp/senderr
+        socket/ipv4/tcp/recverr
+        socket/ipv6/tcp/open
+        socket/ipv6/tcp/openfail
+        socket/ipv6/tcp/close
+        socket/ipv6/tcp/connfail
+        socket/ipv6/tcp/conn
+        socket/ipv6/tcp/senderr
+        socket/ipv6/tcp/recverr
+    """
+    # '_SERVER_' is a special zone name representing an entire
+    # count. It doesn't mean a specific zone, but it means an
+    # entire count in the server.
+    _entire_server = '_SERVER_'
+    # zone names are contained under this dirname in the spec file.
+    _perzone_prefix = 'zones'
+    # default statistics data set
+    _statistics = _Statistics()
+
+    def __init__(self, spec_file_name=None):
+        """If the item `zones` is defined in the spec file, it obtains a
+        list of counter names under it when initiating.  For behaviors
+        other than this, see documentation for
+        isc.statistics.counters.Counters.__init__()"""
+        counters.Counters.__init__(self, spec_file_name)
+        if self._perzone_prefix in \
+                isc.config.spec_name_list(self._statistics._spec):
+            self._zones_item_list = isc.config.spec_name_list(
+                isc.config.find_spec_part(
+                    self._statistics._spec,
+                    '%s/%s/%s' % (self._perzone_prefix,
+                                  '_CLASS_', self._entire_server)))
+
+    def get_statistics(self):
+        """Calculates an entire server's counts, and returns statistics
+        data in a format to send out to the stats module, including each
+        counter. If nothing is counted yet, then it returns an empty
+        dictionary."""
+        # entire copy
+        statistics_data = self._statistics._data.copy()
+        # If there is no 'zones' found in statistics_data,
+        # i.e. statistics_data contains no per-zone counter, it just
+        # returns statistics_data because calculating total counts
+        # across the zone names isn't necessary.
+        if self._perzone_prefix not in statistics_data:
+            return statistics_data
+        zones = statistics_data[self._perzone_prefix]
+        # Start calculation for '_SERVER_' counts
+        zones_spec = isc.config.find_spec_part(self._statistics._spec,
+                                               self._perzone_prefix)
+        zones_data = {}
+        for cls in zones.keys():
+            for zone in zones[cls].keys():
+                for (attr, val) in zones[cls][zone].items():
+                    id_str1 = '%s/%s/%s' % (cls, zone, attr)
+                    id_str2 = '%s/%s/%s' % (cls, self._entire_server, attr)
+                    counters._set_counter(zones_data, zones_spec, id_str1, val)
+                    counters._inc_counter(zones_data, zones_spec, id_str2, val)
+        # insert entire-server counts
+        statistics_data[self._perzone_prefix] = dict(
+            statistics_data[self._perzone_prefix],
+            **zones_data)
+        return statistics_data

src/lib/python/isc/statistics/tests/Makefile.am

 PYCOVERAGE_RUN=@PYCOVERAGE_RUN@
-PYTESTS = counters_test.py
+PYTESTS = counters_test.py dns_test.py
 EXTRA_DIST = $(PYTESTS)
 EXTRA_DIST += testdata/test_spec1.spec
 EXTRA_DIST += testdata/test_spec2.spec

src/lib/python/isc/statistics/tests/counters_test.py

@@ -13,7 +13,7 @@
 # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
 # WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 
-'''Tests for isc.statistics.counter'''
+'''Tests for isc.statistics.counters'''
 
 import unittest
 import threading
@@ -22,8 +22,6 @@ import os
 import imp
 import isc.config
 
-TEST_ZONE_NAME_STR = "example.com."
-TEST_ZONE_CLASS_STR = "IN"
 TESTDATA_SRCDIR = os.getenv("TESTDATASRCDIR")
 
 from isc.statistics import counters
@@ -166,8 +164,6 @@ class BaseTestCounters():
         imp.reload(counters)
         self._statistics_data = {}
         self.counters = counters.Counters(self.TEST_SPECFILE_LOCATION)
-        self._entire_server    = self.counters._entire_server
-        self._perzone_prefix   = self.counters._perzone_prefix
 
     def tearDown(self):
         self.counters.clear_all()
@@ -194,107 +190,6 @@ class BaseTestCounters():
                     ).validate_statistics(
                     False, self._statistics_data))
 
-    def test_perzone_counters(self):
-        # for per-zone counters
-        for name in self.counters._zones_item_list:
-            args = (self._perzone_prefix, TEST_ZONE_CLASS_STR,
-                    TEST_ZONE_NAME_STR, name)
-            if name.find('last_') == 0 and name.endswith('_duration'):
-                self.counters.start_timer(*args)
-                self.counters.stop_timer(*args)
-                self.assertGreaterEqual(self.counters.get(*args), 0.0)
-                sec = self.counters.get(*args)
-                for zone_str in (self._entire_server, TEST_ZONE_NAME_STR):
-                    isc.cc.data.set(self._statistics_data,
-                                    '%s/%s/%s/%s' % (args[0], args[1],
-                                                     zone_str, name), sec)
-                # twice exec stopper, then second is not changed
-                self.counters.stop_timer(*args)
-                self.assertEqual(self.counters.get(*args), sec)
-            else:
-                self.counters.inc(*args)
-                self.assertEqual(self.counters.get(*args), 1)
-                # checks disable/enable
-                self.counters.disable()
-                self.counters.inc(*args)
-                self.assertEqual(self.counters.get(*args), 1)
-                self.counters.enable()
-                self.counters.inc(*args)
-                self.assertEqual(self.counters.get(*args), 2)
-                for zone_str in (self._entire_server, TEST_ZONE_NAME_STR):
-                    isc.cc.data.set(self._statistics_data,
-                                    '%s/%s/%s/%s' % (args[0], args[1],
-                                                     zone_str, name), 2)
-        self.check_get_statistics()
-
-    def test_xfrrunning_counters(self):
-        # for counters of xfer running
-        _suffix = 'xfr_running'
-        _xfrrunning_names = \
-            isc.config.spec_name_list(self.counters._statistics._spec,
-                                      "", True)
-        for name in _xfrrunning_names:
-            if name.find(_suffix) != 1: continue
-            args = name.split('/')
-            self.counters.inc(*args)
-            self.assertEqual(self.counters.get(*args), 1)
-            self.counters.dec(*args)
-            self.assertEqual(self.counters.get(*args), 0)
-            # checks disable/enable
-            self.counters.disable()
-            self.counters.inc(*args)
-            self.assertEqual(self.counters.get(*args), 0)
-            self.counters.enable()
-            self.counters.inc(*args)
-            self.assertEqual(self.counters.get(*args), 1)
-            self.counters.disable()
-            self.counters.dec(*args)
-            self.assertEqual(self.counters.get(*args), 1)
-            self.counters.enable()
-            self.counters.dec(*args)
-            self.assertEqual(self.counters.get(*args), 0)
-            self._statistics_data[name] = 0
-        self.check_get_statistics()
-
-    def test_socket_counters(self):
-