Commit fc2610f5 authored by Naoki Kambe's avatar Naoki Kambe
Browse files

- merge trac #191 (Implement a initial version of stats)

- remove obsoleted stats files


git-svn-id: svn://bind10.isc.org/svn/bind10/trunk@3218 e5f2f494-b856-4b98-b285-d166d9295462
parent 2132a059
109. [func] naokikambe
Added the initial version of the stats module for the statistics
feature of BIND 10, which supports the restricted features and
items and reports via bindctl command (Trac #191, rXXXX)
Added the document of the stats module, which is about how stats
module collects the data (Trac #170, [wiki:StatsModule])
108. [func] jerry
src/bin/zonemgr: Provide customizable configurations for
lowerbound_refresh, lowerbound_retry, max_transfer_timeout and
......
......@@ -471,6 +471,8 @@ AC_CONFIG_FILES([Makefile
src/bin/xfrout/tests/Makefile
src/bin/zonemgr/Makefile
src/bin/zonemgr/tests/Makefile
src/bin/stats/Makefile
src/bin/stats/tests/Makefile
src/bin/usermgr/Makefile
src/bin/tests/Makefile
src/lib/Makefile
......@@ -523,6 +525,12 @@ AC_OUTPUT([src/bin/cfgmgr/b10-cfgmgr.py
src/bin/zonemgr/zonemgr.spec.pre
src/bin/zonemgr/tests/zonemgr_test
src/bin/zonemgr/run_b10-zonemgr.sh
src/bin/stats/stats.py
src/bin/stats/stats_stub.py
src/bin/stats/stats.spec.pre
src/bin/stats/run_b10-stats.sh
src/bin/stats/run_b10-stats_stub.sh
src/bin/stats/tests/stats_test
src/bin/bind10/bind10.py
src/bin/bind10/tests/bind10_test
src/bin/bind10/run_bind10.sh
......@@ -556,6 +564,9 @@ AC_OUTPUT([src/bin/cfgmgr/b10-cfgmgr.py
chmod +x src/bin/xfrin/run_b10-xfrin.sh
chmod +x src/bin/xfrout/run_b10-xfrout.sh
chmod +x src/bin/zonemgr/run_b10-zonemgr.sh
chmod +x src/bin/stats/tests/stats_test
chmod +x src/bin/stats/run_b10-stats.sh
chmod +x src/bin/stats/run_b10-stats_stub.sh
chmod +x src/bin/bind10/run_bind10.sh
chmod +x src/bin/cmdctl/tests/cmdctl_test
chmod +x src/bin/xfrin/tests/xfrin_test
......
SUBDIRS = bind10 bindctl cfgmgr loadzone msgq host cmdctl auth xfrin xfrout \
usermgr zonemgr tests
usermgr zonemgr stats tests
check-recursive: all-recursive
......@@ -73,6 +73,9 @@ isc.utils.process.rename(sys.argv[0])
# number, and the overall BIND 10 version number (set in configure.ac).
VERSION = "bind10 20100916 (BIND 10 @PACKAGE_VERSION@)"
# This is for bind10.boottime of stats module
_BASETIME = time.gmtime()
class RestartSchedule:
"""
Keeps state when restarting something (in this case, a process).
......@@ -424,6 +427,27 @@ class BoB:
sys.stdout.write("[bind10] Started b10-zonemgr(PID %d)\n" %
zonemgr.pid)
# start b10-stats
stats_args = ['b10-stats']
if self.verbose:
sys.stdout.write("[bind10] Starting b10-stats\n")
stats_args += ['-v']
try:
statsd = ProcessInfo("b10-stats", stats_args,
c_channel_env)
except Exception as e:
c_channel.process.kill()
bind_cfgd.process.kill()
xfrout.process.kill()
auth.process.kill()
xfrind.process.kill()
zonemgr.process.kill()
return "Unable to start b10-stats; " + str(e)
self.processes[statsd.pid] = statsd
if self.verbose:
sys.stdout.write("[bind10] Started b10-stats (PID %d)\n" % statsd.pid)
# start the b10-cmdctl
# XXX: we hardcode port 8080
cmdctl_args = ['b10-cmdctl']
......@@ -440,6 +464,7 @@ class BoB:
auth.process.kill()
xfrind.process.kill()
zonemgr.process.kill()
statsd.process.kill()
return "Unable to start b10-cmdctl; " + str(e)
self.processes[cmd_ctrld.pid] = cmd_ctrld
if self.verbose:
......@@ -459,6 +484,7 @@ class BoB:
self.cc_session.group_sendmsg(cmd, "Boss", "Xfrout")
self.cc_session.group_sendmsg(cmd, "Boss", "Xfrin")
self.cc_session.group_sendmsg(cmd, "Boss", "Zonemgr")
self.cc_session.group_sendmsg(cmd, "Boss", "Stats")
def stop_process(self, process):
"""Stop the given process, friendly-like."""
......@@ -723,6 +749,17 @@ def main():
sys.exit(1)
sys.stdout.write("[bind10] BIND 10 started\n")
# send "bind10.boot_time" to b10-stats
time.sleep(1) # wait a second
if options.verbose:
sys.stdout.write("[bind10] send \"bind10.boot_time\" to b10-stats\n")
cmd = isc.config.ccsession.create_command('set',
{ "stats_data": {
'bind10.boot_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', _BASETIME)
}
})
boss_of_bind.cc_session.group_sendmsg(cmd, 'Stats')
# In our main loop, we check for dead processes or messages
# on the c-channel.
wakeup_fd = wakeup_pipe[0]
......
......@@ -20,7 +20,7 @@ export PYTHON_EXEC
BIND10_PATH=@abs_top_builddir@/src/bin/bind10
PATH=@abs_top_builddir@/src/bin/msgq:@abs_top_builddir@/src/bin/auth:@abs_top_builddir@/src/bin/cfgmgr:@abs_top_builddir@/src/bin/cmdctl:@abs_top_builddir@/src/bin/xfrin:@abs_top_builddir@/src/bin/xfrout:@abs_top_builddir@/src/bin/zonemgr:$PATH
PATH=@abs_top_builddir@/src/bin/msgq:@abs_top_builddir@/src/bin/auth:@abs_top_builddir@/src/bin/cfgmgr:@abs_top_builddir@/src/bin/cmdctl:@abs_top_builddir@/src/bin/stats:@abs_top_builddir@/src/bin/xfrin:@abs_top_builddir@/src/bin/xfrout:@abs_top_builddir@/src/bin/zonemgr:$PATH
export PATH
PYTHONPATH=@abs_top_builddir@/src/lib/python:@abs_top_builddir@/src/lib/dns/python/.libs:@abs_top_builddir@/src/lib/xfr/.libs
......
SUBDIRS = tests
pkglibexecdir = $(libexecdir)/@PACKAGE@
pkglibexec_SCRIPTS = b10-stats
noinst_SCRIPTS = b10-stats_stub
b10_statsdir = $(DESTDIR)$(pkgdatadir)
b10_stats_DATA = stats.spec
CLEANFILES = stats.spec b10-stats stats.pyc stats.pyo b10-stats_stub stats_stub.pyc stats_stub.pyo
man_MANS = b10-stats.8
EXTRA_DIST = $(man_MANS) b10-stats.xml
if ENABLE_MAN
b10-stats.8: b10-stats.xml
xsltproc --novalid --xinclude --nonet -o $@ http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl $(srcdir)/b10-stats.xml
endif
stats.spec: stats.spec.pre
$(SED) -e "s|@@LOCALSTATEDIR@@|$(localstatedir)|" stats.spec.pre >$@
# TODO: does this need $$(DESTDIR) also?
# this is done here since configure.ac AC_OUTPUT doesn't expand exec_prefix
b10-stats: stats.py
$(SED) -e "s|@@PYTHONPATH@@|@pyexecdir@|" \
-e "s|@@LOCALSTATEDIR@@|$(localstatedir)|" \
-e "s|.*#@@REMOVED@@$$||" stats.py >$@
chmod a+x $@
b10-stats_stub: stats_stub.py stats.py
$(SED) -e "s|@@PYTHONPATH@@|@pyexecdir@|" \
-e "s|@@LOCALSTATEDIR@@|$(localstatedir)|" stats_stub.py >$@
chmod a+x $@
'\" t
.\" Title: b10-stats
.\" Author: [FIXME: author] [see http://docbook.sf.net/el/author]
.\" Generator: DocBook XSL Stylesheets v1.75.2 <http://docbook.sf.net/>
.\" Date: Oct 15, 2010
.\" Manual: BIND10
.\" Source: BIND10
.\" Language: English
.\"
.TH "B10\-STATS" "8" "Oct 15, 2010" "BIND10" "BIND10"
.\" -----------------------------------------------------------------
.\" * set default formatting
.\" -----------------------------------------------------------------
.\" disable hyphenation
.nh
.\" disable justification (adjust text to left margin only)
.ad l
.\" -----------------------------------------------------------------
.\" * MAIN CONTENT STARTS HERE *
.\" -----------------------------------------------------------------
.SH "NAME"
b10-stats \- BIND 10 statistics module
.SH "SYNOPSIS"
.HP \w'\fBb10\-stats\fR\ 'u
\fBb10\-stats\fR [\fB\-v\fR] [\fB\-\-verbose\fR]
.SH "DESCRIPTION"
.PP
The
\fBb10\-stats\fR
is a daemon forked by
\fBbind10\fR\&. Stats module collects statistics data from each module and reports statistics information via
\fBbindctl\fR\&. It communicates by using the Command Channel by
\fBb10\-msgq\fR
with other modules like
\fBbind10\fR,
\fBb10\-auth\fR
and so on\&. It waits for coming data from other modules, then other modules send data to stats module periodically\&. Other modules send stats data to stats module independently from implementation of stats module, so the frequency of sending data may not be constant\&. Stats module collects data and aggregates it\&.
.SH "OPTIONS"
.PP
The arguments are as follows:
.PP
\fB\-v\fR, \fB\-\-verbose\fR
.RS 4
This
\fBb10\-stats\fR
switches to verbose mode\&. It sends verbose messages to STDOUT\&.
.RE
.SH "FILES"
.PP
/usr/local/share/bind10\-devel/stats\&.spec
\(em This is a spec file for
\fBb10\-stats\fR\&. It contains definitions of statistics items of BIND 10 and commands received vi bindctl\&.
.SH "SEE ALSO"
.PP
\fBbind10\fR(8),
\fBbindctl\fR(1),
\fBb10-auth\fR(8),
BIND 10 Guide\&.
.SH "HISTORY"
.PP
The
\fBb10\-stats\fR
daemon was initially designed and implemented by Naoki Kambe of JPRS in Oct 2010\&.
.SH "COPYRIGHT"
.br
Copyright \(co 2010 Internet Systems Consortium, Inc. ("ISC")
.br
<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd"
[<!ENTITY mdash "&#8212;">]>
<!--
- Copyright (C) 2010 Internet Systems Consortium, Inc. ("ISC")
-
- Permission to use, copy, modify, and/or distribute this software for any
- purpose with or without fee is hereby granted, provided that the above
- copyright notice and this permission notice appear in all copies.
-
- THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
- REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
- AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
- INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
- LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
- OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
- PERFORMANCE OF THIS SOFTWARE.
-->
<!-- $Id$ -->
<refentry>
<refentryinfo>
<date>Oct 15, 2010</date>
</refentryinfo>
<refmeta>
<refentrytitle>b10-stats</refentrytitle>
<manvolnum>8</manvolnum>
<refmiscinfo>BIND10</refmiscinfo>
</refmeta>
<refnamediv>
<refname>b10-stats</refname>
<refpurpose>BIND 10 statistics module</refpurpose>
</refnamediv>
<docinfo>
<copyright>
<year>2010</year>
<holder>Internet Systems Consortium, Inc. ("ISC")</holder>
</copyright>
</docinfo>
<refsynopsisdiv>
<cmdsynopsis>
<command>b10-stats</command>
<arg><option>-v</option></arg>
<arg><option>--verbose</option></arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsect1>
<title>DESCRIPTION</title>
<para>
The <command>b10-stats</command> is a daemon forked by
<command>bind10</command>. Stats module collects statistics data
from each module and reports statistics information
via <command>bindctl</command>. It communicates by using the
Command Channel by <command>b10-msgq</command> with other
modules
like <command>bind10</command>, <command>b10-auth</command> and
so on. It waits for coming data from other modules, then other
modules send data to stats module periodically. Other modules
send stats data to stats module independently from
implementation of stats module, so the frequency of sending data
may not be constant. Stats module collects data and aggregates
it.
</para>
</refsect1>
<refsect1>
<title>OPTIONS</title>
<para>The arguments are as follows:</para>
<variablelist>
<varlistentry>
<term><option>-v</option>, <option>--verbose</option></term>
<listitem>
<para>
This <command>b10-stats</command> switches to verbose
mode. It sends verbose messages to STDOUT.
</para>
</listitem>
</varlistentry>
</variablelist>
</refsect1>
<refsect1>
<title>FILES</title>
<para><filename>/usr/local/share/bind10-devel/stats.spec</filename>
&mdash; This is a spec file for <command>b10-stats</command>. It
contains definitions of statistics items of BIND 10 and commands
received vi bindctl.
</para>
</refsect1>
<refsect1>
<title>SEE ALSO</title>
<para>
<citerefentry>
<refentrytitle>bind10</refentrytitle><manvolnum>8</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>bindctl</refentrytitle><manvolnum>1</manvolnum>
</citerefentry>,
<citerefentry>
<refentrytitle>b10-auth</refentrytitle><manvolnum>8</manvolnum>
</citerefentry>,
<citetitle>BIND 10 Guide</citetitle>.
</para>
</refsect1>
<refsect1>
<title>HISTORY</title>
<para>
The <command>b10-stats</command> daemon was initially designed
and implemented by Naoki Kambe of JPRS in Oct 2010.
</para>
</refsect1>
</refentry><!--
- Local variables:
- mode: sgml
- End:
-->
#! /bin/sh
# Copyright (C) 2010 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
PYTHON_EXEC=${PYTHON_EXEC:-@PYTHON@}
export PYTHON_EXEC
PYTHONPATH=@abs_top_builddir@/src/lib/python
export PYTHONPATH
B10_FROM_BUILD=@abs_top_builddir@
export B10_FROM_BUILD
STATS_PATH=@abs_top_builddir@/src/bin/stats
cd ${STATS_PATH}
exec ${PYTHON_EXEC} -O b10-stats $*
#! /bin/sh
# Copyright (C) 2010 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
PYTHON_EXEC=${PYTHON_EXEC:-@PYTHON@}
export PYTHON_EXEC
PYTHONPATH=@abs_top_builddir@/src/lib/python
export PYTHONPATH
B10_FROM_BUILD=@abs_top_srcdir@
export B10_FROM_BUILD
STATS_PATH=@abs_top_builddir@/src/bin/stats
cd ${STATS_PATH}
exec ${PYTHON_EXEC} -O b10-stats_stub $*
#!@PYTHON@
# Copyright (C) 2010 Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
# $Id$
__version__ = "$Revision$"
import sys; sys.path.append ('@@PYTHONPATH@@')
import os
import signal
import select
from time import time, strftime, gmtime
from optparse import OptionParser, OptionValueError
from collections import defaultdict
from isc.config.ccsession import ModuleCCSession, create_answer
from isc.cc import Session, SessionError
# Note: Following lines are removed in b10-stats #@@REMOVED@@
if __name__ == 'stats': #@@REMOVED@@
try: #@@REMOVED@@
from fake_time import time, strftime, gmtime #@@REMOVED@@
except ImportError: #@@REMOVED@@
pass #@@REMOVED@@
# for setproctitle
import isc.utils.process
isc.utils.process.rename()
# If B10_FROM_BUILD is set in the environment, we use data files
# from a directory relative to that, otherwise we use the ones
# installed on the system
if "B10_FROM_BUILD" in os.environ:
SPECFILE_LOCATION = os.environ["B10_FROM_BUILD"] + "/src/bin/stats/stats.spec"
else:
PREFIX = "@prefix@"
DATAROOTDIR = "@datarootdir@"
SPECFILE_LOCATION = "@datadir@/@PACKAGE@/stats.spec".replace("${datarootdir}", DATAROOTDIR).replace("${prefix}", PREFIX)
class Singleton(type):
"""
A abstract class of singleton pattern
"""
# Because of singleton pattern:
# At the beginning of coding, one UNIX domain socket is needed
# for config manager, another socket is needed for stats module,
# then stats module might need two sockets. So I adopted the
# singleton pattern because I avoid creating multiple sockets in
# one stats module. But in the initial version stats module
# reports only via bindctl, so just one socket is needed. To use
# the singleton pattern is not important now. :(
def __init__(self, *args, **kwargs):
type.__init__(self, *args, **kwargs)
self._instances = {}
def __call__(self, *args, **kwargs):
if args not in self._instances:
self._instances[args]={}
kw = tuple(kwargs.items())
if kw not in self._instances[args]:
self._instances[args][kw] = type.__call__(self, *args, **kwargs)
return self._instances[args][kw]
class Callback():
"""
A Callback handler class
"""
def __init__(self, name=None, callback=None, args=(), kwargs={}):
self.name = name
self.callback = callback
self.args = args
self.kwargs = kwargs
def __call__(self, *args, **kwargs):
if not args:
args = self.args
if not kwargs:
kwargs = self.kwargs
if self.callback:
return self.callback(*args, **kwargs)
class Subject():
"""
A abstract subject class of observer pattern
"""
# Because of observer pattern:
# In the initial release, I'm also sure that observer pattern
# isn't definitely needed because the interface between gathering
# and reporting statistics data is single. However in the future
# release, the interfaces may be multiple, that is, multiple
# listeners may be needed. For example, one interface, which
# stats module has, is for between ''config manager'' and stats
# module, another interface is for between ''HTTP server'' and
# stats module, and one more interface is for between ''SNMP
# server'' and stats module. So by considering that stats module
# needs multiple interfaces in the future release, I adopted the
# observer pattern in stats module. But I don't have concrete
# ideas in case of multiple listener currently.
def __init__(self):
self._listeners = []
def attach(self, listener):
if not listener in self._listeners:
self._listeners.append(listener)
def detach(self, listener):
try:
self._listeners.remove(listener)
except ValueError:
pass
def notify(self, event, modifier=None):
for listener in self._listeners:
if modifier != listener:
listener.update(event)
class Listener():
"""
A abstract listener class of observer pattern
"""
def __init__(self, subject):
self.subject = subject
self.subject.attach(self)
self.events = {}
def update(self, name):
if name in self.events:
callback = self.events[name]
return callback()
def add_event(self, event):
self.events[event.name]=event
class SessionSubject(Subject, metaclass=Singleton):
"""
A concrete subject class which creates CC session object
"""
def __init__(self, session=None, verbose=False):
Subject.__init__(self)
self.verbose = verbose
self.session=session
self.running = False
def start(self):
self.running = True
self.notify('start')
def stop(self):
self.running = False
self.notify('stop')
def check(self):
self.notify('check')
class CCSessionListener(Listener):
"""
A concrete listener class which creates SessionSubject object and
ModuleCCSession object
"""
def __init__(self, subject, verbose=False):
Listener.__init__(self, subject)
self.verbose = verbose
self.session = subject.session
self.boot_time = get_datetime()
# create ModuleCCSession object
self.cc_session = ModuleCCSession(SPECFILE_LOCATION,
self.config_handler,
self.command_handler,
self.session)
self.session = self.subject.session = self.cc_session._session
# initialize internal data
self.config_spec = self.cc_session.get_module_spec().get_config_spec()
self.stats_spec = self.config_spec
self.stats_data = self.initialize_data(self.stats_spec)
# add event handler invoked via SessionSubject object
self.add_event(Callback('start', self.start))
self.add_event(Callback('stop', self.stop))
self.add_event(Callback('check', self.check))
# don't add 'command_' suffix to the special commands in
# order to prevent executing internal command via bindctl
# get commands spec
self.commands_spec = self.cc_session.get_module_spec().get_commands_spec()
# add event handler related command_handler of ModuleCCSession
# invoked via bindctl
for cmd in self.commands_spec:
try:
# add prefix "command_"
name = "command_" + cmd["command_name"]
callback = getattr(self, name)
kwargs = self.initialize_data(cmd["command_args"])
self.add_event(Callback(name=name, callback=callback, args=(), kwargs=kwargs))
except AttributeError as ae:
sys.stderr.write("[b10-stats] Caught undefined command while parsing spec file: "
+str(cmd["command_name"])+"\n")
def start(self):
"""