[master] Merge branch 'master' of ssh://git.kea.isc.org/git/kea

ChangeLog

-929	[build]		fdupont
+930.	[func]		tomek
+	Statistics Manager is now implemented. There is a new library
+	libkea-stats that governs statistics collection. Its usage will
+	be added in the upcoming tickets.
+	(Trac #3793, git 68e9554ecabfc2a79731eeec1c706522e4d39332)
+
+929.	[build]		fdupont
 	Corrected problem in build system whereby specifying an
 	installation directory on the "configure" command line that
 	included a "+" in the name caused the build to fail.
@@ -10,7 +16,7 @@
 	(Trac #3812, git cbb135d5f217b0692dcdbc9cfcc04f6a0dbc3922)
 
 927.	[bug]		tmark
-	DHCPv4 no longer attempts to update the lease database with the 
+	DHCPv4 no longer attempts to update the lease database with the
 	generated FQDN when processing DHCPDISCOVERs.
 	(Trac #3779, git 0b413ee8aba1afa1643b216a1e8c35103c6c975b)
 

configure.ac

@@ -1510,6 +1510,8 @@ AC_CONFIG_FILES([compatcheck/Makefile
                  src/lib/testutils/Makefile
                  src/lib/testutils/dhcp_test_lib.sh
                  src/lib/testutils/testdata/Makefile
+                 src/lib/stats/Makefile
+                 src/lib/stats/tests/Makefile
                  src/lib/util/Makefile
                  src/lib/util/io/Makefile
                  src/lib/util/python/Makefile

src/lib/Makefile.am

 # The following build order must be maintained.
-SUBDIRS = exceptions util log hooks cryptolink dns cc config \
-          asiolink asiodns testutils dhcp dhcp_ddns \
-          dhcpsrv
+SUBDIRS = exceptions util log hooks cryptolink dns cc stats config \
+          asiolink asiodns testutils dhcp dhcp_ddns dhcpsrv

src/lib/cc/data.h

@@ -675,6 +675,13 @@ public:
     // a MapElement)
     bool find(const std::string& id, ConstElementPtr& t) const;
 
+    /// @brief Returns number of stored elements
+    ///
+    /// @return number of elements.
+    size_t size() const {
+        return (m.size());
+    }
+
     bool equals(const Element& other) const;
 };
 

src/lib/stats/Makefile.am

+SUBDIRS = . tests
+
+AM_CPPFLAGS = -I$(top_srcdir)/src/lib -I$(top_builddir)/src/lib
+AM_CPPFLAGS += $(BOOST_INCLUDES)
+AM_CXXFLAGS=$(KEA_CXXFLAGS)
+
+lib_LTLIBRARIES = libkea-stats.la
+libkea_stats_la_SOURCES = observation.h observation.cc
+libkea_stats_la_SOURCES += context.h context.cc
+libkea_stats_la_SOURCES += stats_mgr.h stats_mgr.cc
+
+libkea_stats_la_CPPFLAGS = $(AM_CPPFLAGS)
+libkea_stats_la_LIBADD  = $(top_builddir)/src/lib/cc/libkea-cc.la
+libkea_stats_la_LIBADD += $(top_builddir)/src/lib/util/libkea-util.la
+
+libkea_stats_includedir = $(includedir)/$(PACKAGE_NAME)/stats
+libkea_stats_include_HEADERS = stats_mgr.h

src/lib/stats/context.cc

+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <stats/context.h>
+#include <map>
+
+namespace isc {
+namespace stats {
+
+ObservationPtr StatContext::get(const std::string& name) const {
+    std::map<std::string, ObservationPtr>::const_iterator obs = stats_.find(name);
+    if (obs == stats_.end()) {
+        return (ObservationPtr());
+    } else {
+        return (obs->second);
+    }
+}
+
+void StatContext::add(const ObservationPtr& obs) {
+    std::map<std::string, ObservationPtr>::iterator existing = stats_.find(obs->getName());
+    if (existing == stats_.end()) {
+        stats_.insert(make_pair(obs->getName() ,obs));
+    } else {
+        isc_throw(DuplicateStat, "Statistic named " << obs->getName()
+                  << " already exists.");
+    }
+
+}
+
+bool StatContext::del(const std::string& name) {
+    std::map<std::string, ObservationPtr>::iterator obs = stats_.find(name);
+    if (obs == stats_.end()) {
+        return (false);
+    } else {
+        stats_.erase(obs);
+        return (true);
+    }
+}
+
+};
+};

src/lib/stats/context.h

+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef CONTEXT_H
+#define CONTEXT_H
+
+#include <stats/observation.h>
+#include <boost/shared_ptr.hpp>
+#include <string>
+
+namespace isc {
+namespace stats {
+
+/// @brief Exception indicating that a given statistic is duplicated.
+class DuplicateStat : public Exception {
+public:
+    DuplicateStat(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) {}
+};
+
+/// @brief Statistics context
+///
+/// Statistics context is essentially a container used to group statistics
+/// related to a given context together. Two examples of such contexts are
+/// all statistics related to a given subnet or all statistics related to a
+/// given network interface.
+struct StatContext {
+ public:
+
+    /// @brief attempts to get an observation
+    /// @param name name of the statistic
+    /// @return appropriate Observation object (or NULL)
+    ObservationPtr get(const std::string& name) const;
+
+    /// @brief Adds a new observation
+    /// @param obs observation to be added
+    /// @throw DuplicateStat if an observation with the same name exists already
+    void add(const ObservationPtr& obs);
+
+    /// @brief Attempts to delete an observation
+    /// @param name name of the observation to be deleted
+    /// @return true if successful, false if no such statistic was found
+    bool del(const std::string& name);
+
+    /// @brief Statistics container
+    ///
+    /// It is public to allow various operations that require iterating over
+    /// all elements. In particular, two operations (setting all stats to 0;
+    /// reporting all stats) will take advantage of this. Alternatively, we
+    /// could make it protected and then return a pointer to it, but that
+    /// would defeat the purpose of the hermetization in the first place.
+    std::map<std::string, ObservationPtr> stats_;
+};
+
+/// @brief Pointer to the statistics context
+typedef boost::shared_ptr<StatContext> StatContextPtr;
+
+};
+};
+
+#endif // CONTEXT_H

src/lib/stats/observation.cc

+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#include <stats/observation.h>
+#include <util/boost_time_utils.h>
+#include <cc/data.h>
+#include <boost/date_time/posix_time/posix_time.hpp>
+#include <boost/date_time/gregorian/gregorian.hpp>
+#include <utility>
+
+using namespace std;
+using namespace isc::data;
+using namespace boost::posix_time;
+
+namespace isc {
+namespace stats {
+
+Observation::Observation(const std::string& name, const uint64_t value)
+    :name_(name), type_(STAT_INTEGER) {
+    setValue(value);
+}
+
+Observation::Observation(const std::string& name, const double value)
+    :name_(name), type_(STAT_FLOAT) {
+    setValue(value);
+}
+
+Observation::Observation(const std::string& name, const StatsDuration& value)
+    :name_(name), type_(STAT_DURATION) {
+    setValue(value);
+}
+
+Observation::Observation(const std::string& name, const std::string& value)
+    :name_(name), type_(STAT_STRING) {
+    setValue(value);
+}
+
+void Observation::addValue(const uint64_t value) {
+    IntegerSample current = getInteger();
+    setValue(current.first + value);
+}
+
+void Observation::addValue(const double value) {
+    FloatSample current = getFloat();
+    setValue(current.first + value);
+}
+
+void Observation::addValue(const StatsDuration& value) {
+    DurationSample current = getDuration();
+    setValue(current.first + value);
+}
+
+void Observation::addValue(const std::string& value) {
+    StringSample current = getString();
+    setValue(current.first + value);
+}
+
+void Observation::setValue(const uint64_t value) {
+    setValueInternal(value, integer_samples_, STAT_INTEGER);
+}
+
+void Observation::setValue(const double value) {
+    setValueInternal(value, float_samples_, STAT_FLOAT);
+}
+
+void Observation::setValue(const StatsDuration& value) {
+    setValueInternal(value, duration_samples_, STAT_DURATION);
+}
+
+void Observation::setValue(const std::string& value) {
+    setValueInternal(value, string_samples_, STAT_STRING);
+}
+
+template<typename SampleType, typename StorageType>
+void Observation::setValueInternal(SampleType value, StorageType& storage,
+    Type exp_type) {
+    if (type_ != exp_type) {
+        isc_throw(InvalidStatType, "Invalid statistic type requested: "
+                  << typeToText(exp_type) << ", but the actual type is "
+                  << typeToText(type_) );
+    }
+
+    if (storage.empty()) {
+        storage.push_back(make_pair(value, microsec_clock::local_time()));
+    } else {
+
+        /// @todo: Update once more than one sample is supported
+        *storage.begin() = make_pair(value, microsec_clock::local_time());
+    }
+}
+
+IntegerSample Observation::getInteger() const {
+    return (getValueInternal<IntegerSample>(integer_samples_, STAT_INTEGER));
+}
+
+FloatSample Observation::getFloat() const {
+    return (getValueInternal<FloatSample>(float_samples_, STAT_FLOAT));
+}
+
+DurationSample Observation::getDuration() const {
+    return (getValueInternal<DurationSample>(duration_samples_, STAT_DURATION));
+}
+
+StringSample Observation::getString() const {
+    return (getValueInternal<StringSample>(string_samples_, STAT_STRING));
+}
+
+template<typename SampleType, typename Storage>
+SampleType Observation::getValueInternal(Storage& storage, Type exp_type) const {
+    if (type_ != exp_type) {
+        isc_throw(InvalidStatType, "Invalid statistic type requested: "
+                  << typeToText(exp_type) << ", but the actual type is "
+                  << typeToText(type_) );
+    }
+
+    if (storage.empty()) {
+        // That should never happen. The first element is always initialized in
+        // the constructor. reset() sets its value to zero, but the element should
+        // still be there.
+        isc_throw(Unexpected, "Observation storage container empty");
+    }
+    return (*storage.begin());
+}
+
+std::string Observation::typeToText(Type type) {
+    std::stringstream tmp;
+    switch (type) {
+    case STAT_INTEGER:
+        tmp << "integer";
+        break;
+    case STAT_FLOAT:
+        tmp << "float";
+        break;
+    case STAT_DURATION:
+        tmp << "duration";
+        break;
+    case STAT_STRING:
+        tmp << "string";
+        break;
+    default:
+        tmp << "unknown";
+        break;
+    }
+    tmp << "(" << type << ")";
+    return (tmp.str());
+}
+
+isc::data::ConstElementPtr
+Observation::getJSON() const {
+
+    ElementPtr entry = isc::data::Element::createList(); // a single observation
+    ElementPtr value;
+    ElementPtr timestamp;
+
+    /// @todo: Add support for retrieving more than one sample for a given
+    /// observation
+
+    switch (type_) {
+    case STAT_INTEGER: {
+        IntegerSample s = getInteger();
+        value = isc::data::Element::create(static_cast<int64_t>(s.first));
+        timestamp = isc::data::Element::create(isc::util::ptimeToText(s.second));
+        break;
+    }
+    case STAT_FLOAT: {
+        FloatSample s = getFloat();
+        value = isc::data::Element::create(s.first);
+        timestamp = isc::data::Element::create(isc::util::ptimeToText(s.second));
+        break;
+    }
+    case STAT_DURATION: {
+        DurationSample s = getDuration();
+        value = isc::data::Element::create(isc::util::durationToText(s.first));
+        timestamp = isc::data::Element::create(isc::util::ptimeToText(s.second));
+        break;
+    }
+    case STAT_STRING: {
+        StringSample s = getString();
+        value = isc::data::Element::create(s.first);
+        timestamp = isc::data::Element::create(isc::util::ptimeToText(s.second));
+        break;
+    }
+    default:
+        isc_throw(InvalidStatType, "Unknown statistic type: "
+                  << typeToText(type_));
+    };
+
+    entry->add(value);
+    entry->add(timestamp);
+
+    ElementPtr list = isc::data::Element::createList(); // a single observation
+    list->add(entry);
+
+    return (list);
+}
+
+void Observation::reset() {
+    switch(type_) {
+    case STAT_INTEGER: {
+        setValue(static_cast<uint64_t>(0));
+        return;
+    }
+    case STAT_FLOAT: {
+        setValue(0.0);
+        return;
+    }
+    case STAT_DURATION: {
+        setValue(time_duration(0,0,0,0));
+        return;
+    }
+    case STAT_STRING: {
+        setValue(string(""));
+        return;
+    }
+    default:
+        isc_throw(InvalidStatType, "Unknown statistic type: "
+                  << typeToText(type_));
+    };
+}
+
+};
+};

src/lib/stats/observation.h

+// Copyright (C) 2015 Internet Systems Consortium, Inc. ("ISC")
+//
+// Permission to use, copy, modify, and/or distribute this software for any
+// purpose with or without fee is hereby granted, provided that the above
+// copyright notice and this permission notice appear in all copies.
+//
+// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+// PERFORMANCE OF THIS SOFTWARE.
+
+#ifndef OBSERVATION_H
+#define OBSERVATION_H
+
+#include <cc/data.h>
+#include <exceptions/exceptions.h>
+#include <boost/shared_ptr.hpp>
+#include <boost/date_time/time_duration.hpp>
+#include <boost/date_time/posix_time/posix_time_types.hpp>
+#include <list>
+#include <stdint.h>
+
+namespace isc {
+namespace stats {
+
+/// @brief Exception thrown if invalid statistic type is used
+///
+/// For example statistic is of type duration, but methods using
+/// it as integer are called.
+class InvalidStatType : public Exception {
+public:
+    InvalidStatType(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) {}
+};
+
+/// @brief Defines duration resolution
+///
+typedef boost::posix_time::time_duration StatsDuration;
+
+/// @defgroup stat_samples Specifies supported observation types.
+///
+/// @brief The list covers all supported types of observations.
+///
+/// @{
+
+/// @brief Integer (implemented as unsigned 64-bit integer)
+typedef std::pair<uint64_t, boost::posix_time::ptime> IntegerSample;
+
+/// @brief Float (implemented as double precision)
+typedef std::pair<double, boost::posix_time::ptime> FloatSample;
+
+/// @brief Time Duration
+typedef std::pair<StatsDuration, boost::posix_time::ptime> DurationSample;
+
+/// @brief String
+typedef std::pair<std::string, boost::posix_time::ptime> StringSample;
+
+/// @}
+
+/// @brief Represents a single observable characteristic (a 'statistic')
+///
+/// Currently it supports one of four types: integer (implemented as unsigned 64
+/// bit integer), float (implemented as double), time duration (implemented with
+/// millisecond precision) and string. Absolute (setValue) and
+/// incremental (addValue) modes are supported. Statistic type is determined
+/// during its first use. Once type is set, any additional observations recorded
+/// must be of the same type. Attempting to set or extract information about
+/// other types will result in InvalidStateType exception.
+///
+/// Observation can be retrieved in one of @ref getInteger, @ref getFloat,
+/// @ref getDuration, @ref getString (appropriate type must be used) or
+/// @ref getJSON, which is generic and can be used for all types.
+///
+/// @todo: Eventually it will be possible to retain multiple samples for the same
+/// observation, but that is outside of scope for 0.9.2.
+class Observation {
+ public:
+
+    /// @brief Type of available statistics
+    ///
+    /// Note that those will later be exposed using control socket. Therefore
+    /// an easy to understand names were chosen (integer instead of uint64).
+    /// To avoid confusion, we will support only one type of integer and only
+    /// one type of floating points. Initially, these are represented by
+    /// uint64_t and double. If convincing use cases appear to change them
+    /// to something else, we may change the underlying type.
+    enum Type {
+        STAT_INTEGER, ///< this statistic is unsinged 64-bit integer value
+        STAT_FLOAT,   ///< this statistic is a floating point value
+        STAT_DURATION,///< this statistic represents time duration
+        STAT_STRING   ///< this statistic represents a string
+    };
+
+    /// @brief Constructor for integer observations
+    ///
+    /// @param name observation name
+    /// @param value integer value observed.
+    Observation(const std::string& name, const uint64_t value);
+
+    /// @brief Constructor for floating point observations
+    ///
+    /// @param name observation name
+    /// @param value floating point value observed.
+    Observation(const std::string& name, const double value);
+
+    /// @brief Constructor for duration observations
+    ///
+    /// @param name observation name
+    /// @param value duration observed.
+    Observation(const std::string& name, const StatsDuration& value);
+
+    /// @brief Constructor for string observations
+    ///
+    /// @param name observation name
+    /// @param value string observed.
+    Observation(const std::string& name, const std::string& value);
+
+    /// @brief Records absolute integer observation
+    ///
+    /// @param value integer value observed
+    /// @throw InvalidStatType if statistic is not integer
+    void setValue(const uint64_t value);
+
+    /// @brief Records absolute floating point observation
+    ///
+    /// @param value floating point value observed
+    /// @throw InvalidStatType if statistic is not fp
+    void setValue(const double value);
+
+    /// @brief Records absolute duration observation
+    ///
+    /// @param value duration value observed
+    /// @throw InvalidStatType if statistic is not time duration
+    void setValue(const StatsDuration& duration);
+
+    /// @brief Records absolute string observation
+    ///
+    /// @param value string value observed
+    /// @throw InvalidStatType if statistic is not a string
+    void setValue(const std::string& value);
+
+    /// @brief Records incremental integer observation
+    ///
+    /// @param value integer value observed
+    /// @throw InvalidStatType if statistic is not integer
+    void addValue(const uint64_t value);
+
+    /// @brief Records incremental floating point observation
+    ///
+    /// @param value floating point value observed
+    /// @throw InvalidStatType if statistic is not fp
+    void addValue(const double value);
+
+    /// @brief Records incremental duration observation
+    ///
+    /// @param value duration value observed
+    /// @throw InvalidStatType if statistic is not time duration
+    void addValue(const StatsDuration& value);
+
+    /// @brief Records incremental string observation.
+    ///
+    /// @param value string value observed
+    /// @throw InvalidStatType if statistic is not a string
+    void addValue(const std::string& value);
+
+    /// @brief Resets statistic.
+    ///
+    /// Sets statistic to a neutral (0, 0.0 or "") value.
+    void reset();
+
+    /// @brief Returns statistic type
+    /// @return statistic type
+    Type getType() const {
+        return (type_);
+    }
+
+    /// @brief Returns observed integer sample
+    /// @return observed sample (value + timestamp)
+    /// @throw InvalidStatType if statistic is not integer
+    IntegerSample getInteger() const;
+
+    /// @brief Returns observed float sample
+    /// @return observed sample (value + timestamp)
+    /// @throw InvalidStatType if statistic is not fp
+    FloatSample getFloat() const;
+