[116-interface-id-dhcpv4] Merge branch '116-interface-id-dhcpv4' of...

[116-interface-id-dhcpv4] Merge branch '116-interface-id-dhcpv4' of gitlab.isc.org:isc-projects/kea into 116-interface-id-dhcpv4

ChangeLog

+1453.	[func]		marcin
+	Updated MySQL schema to facilitate Kea Configuration Backend
+	feature.
+	(Gitlab #89,!22, git e28c0c7b3e7a7729167cdad993f634ed1f0ac53b)
+
+1452.	[func]		marcin
+	Implemented libkea-cb library which includes basic class
+	hierarchy for the Kea Configuration Backend.
+	(Gitlab #28,!20, git fb5c031ecaf4182e56f62874e9a6bd4c1d755a77)
+
 1451.	[build]		tmark
 	Resolved a namespace issue with std::distance() in libdhcp++.cc
 	when building with Boost 1.68.   Thanks to Huy Vu and Khem Raj

configure.ac

@@ -74,6 +74,9 @@ if test "$cross_compiling" = "yes"; then
 fi
 AM_CONDITIONAL([CROSS_COMPILING], [test "$cross_compiling" = "yes"])
 
+# pkg-config can be required.
+AC_PATH_PROG([PKG_CONFIG], [pkg-config])
+
 # Enable low-performing debugging facilities? This option optionally
 # enables some debugging aids that perform slowly and hence aren't built
 # by default.
@@ -817,7 +820,6 @@ AC_ARG_WITH([cql],
     [cql_config="$withval"])
 
 if test "${cql_config}" = "yes" ; then
-    AC_PATH_PROG([PKG_CONFIG], [pkg-config])
     CQL_CONFIG="$PKG_CONFIG"
 elif test "${cql_config}" != "no" ; then
     CQL_CONFIG="${cql_config}"
@@ -1532,6 +1534,8 @@ AC_CONFIG_FILES([Makefile
                  src/lib/config/tests/Makefile
                  src/lib/config/tests/data_def_unittests_config.h
                  src/lib/config/tests/testdata/Makefile
+                 src/lib/config_backend/Makefile
+                 src/lib/config_backend/tests/Makefile
                  src/lib/cryptolink/Makefile
                  src/lib/cryptolink/tests/Makefile
                  src/lib/database/Makefile
@@ -1597,6 +1601,8 @@ AC_CONFIG_FILES([Makefile
                  src/lib/util/threads/Makefile
                  src/lib/util/threads/tests/Makefile
                  src/lib/util/unittests/Makefile
+                 src/lib/yang/Makefile
+                 src/lib/yang/tests/Makefile
                  src/share/Makefile
                  src/share/database/Makefile
                  src/share/database/scripts/Makefile

doc/Doxyfile

@@ -810,6 +810,7 @@ INPUT                  = ../src/bin/agent \
                          ../src/lib/util/random \
                          ../src/lib/util/threads \
                          ../src/lib/util/unittests \
+                         ../src/lib/yang \
                          devel
 
 # This tag can be used to specify the character encoding of the source files

doc/devel/mainpage.dox

@@ -41,6 +41,7 @@
  *   - @subpage unitTestsIntroduction
  *   - @subpage unitTestsEnvironmentVariables
  *   - @subpage unitTestsDatabaseConfig
+ *   - @subpage unitTestsSysrepo
  *
  * @section performance Performance
  * - @subpage benchmarks
@@ -129,6 +130,7 @@
  * - @subpage libprocess
  *   - @subpage cpl
  *   - @subpage cplSignals
+ * - @subpage libyang
  *
  * @section miscellaneousTopics Miscellaneous Topics
  * - @subpage terminology

doc/guide/netconf.xml

@@ -38,12 +38,12 @@ $ sudo apt-get install git cmake build-essential bison flex libpcre3-dev libev-d
     </para>
 
     <para>STEP 2. Install libyang. Download libyang from
-    https://github.com/CESNET/libyang/releases.  As of writing this document, the latest
-    version was 0.15-r1.
+    https://github.com/CESNET/libyang.git. Checkout the devel branch.
 
 <screen>
-    tar zxvf libyang-0.15-r1.tar.gz
-    cd libyang-0.15-r1/
+    git clone https://github.com/CESNET/libyang.git
+    cd libyang
+    git checkout devel
     mkdir build
     cd build
     cmake ..
@@ -53,12 +53,13 @@ $ sudo apt-get install git cmake build-essential bison flex libpcre3-dev libev-d
 
     For detailed build instructions, see https://github.com/CESNET/libyang/.</para>
 
-    <para>STEP 3. Install syrepo. Download sysrepo from https://github.com/sysrepo/sysrepo/releases.
-    As of writing this document, the 0.7.4 as the latest version.
+    <para>STEP 3. Install syrepo. Download sysrepo from
+    https://github.com/sysrepo/sysrepo.git. Checkout the last devel branch.
 
 <screen>
-tar zxvf sysrepo-0.7.4.tar.gz
-cd sysrepo-0.7.4
+git clone https://github.com/sysrepo/sysrepo.git
+cd sysrepo
+git checkout devel
 mkdir build
 cd build
 cmake -DCMAKE_BUILD_TYPE=Debug -DGEN_LANGUAGE_BINDINGS=ON -DGEN_CPP_BINDINGS=ON\

src/bin/admin/tests/mysql_tests.sh.in

@@ -245,7 +245,7 @@ mysql_upgrade_schema_to_version() {
 
 mysql_upgrade_test() {
 
-    test_start "mysql.host_reservation-upgrade"
+    test_start "mysql.upgrade"
 
     # Let's wipe the whole database
     mysql_wipe
@@ -469,11 +469,126 @@ EOF
     ERRCODE=$?
     assert_eq 0 $ERRCODE "logs table is missing or broken. (expected status code %d, returned %d)"
 
+    # table: modification (upgrade 6.0 -> 7.0)
+    qry="select id, modification_type from modification"
+    run_statement "modification"  "$qry"
+
+    # table: modification table should have 3 entries (upgrade 6.0 -> 7.0)
+    qry="select count(*) from modification"
+    run_statement "modification count" "$qry" 3
+
+    # table: dhcp4_server
+    qry="select id, tag, description, modification_ts from dhcp4_server"
+    run_statement "dhcp4_server" "$qry"
+
+    # table: dhcp4_audit
+    qry="select id, object_type, object_id, modification_type, modification_ts, log_message from dhcp4_audit"
+    run_statement "dhcp4_audit" "$qry"
+
+    # table: dhcp4_global_parameter
+    qry="select id, name, value, modification_ts from dhcp4_global_parameter"
+    run_statement "dhcp4_global_parameter" "$qry"
+
+    # table: dhcp4_global_parameter_server
+    qry="select parameter_id, server_id, modification_ts from dhcp4_global_parameter_server"
+    run_statement "dhcp4_global_parameter_server" "$qry"
+
+    # table: dhcp4_option_def
+    qry="select id, code, space, modification_ts, array, encapsulate, record_types, user_context from dhcp4_option_def"
+    run_statement "dhcp4_option_def" "$qry"
+
+    # table: dhcp4_option_def_server
+    qry="select option_def_id, server_id, modification_ts from dhcp4_option_def_server"
+    run_statement "dhcp4_option_def_server" "$qry"
+
+    # table: dhcp4_shared_network
+    qry="select id, name, client_class, interface, match_client_id, modification_ts, rebind_timer, relay, renew_timer, require_client_classes, reservation_mode, server_hostname, user_context, valid_lifetime from dhcp4_shared_network"
+    run_statement "dhcp4_shared_network" "$qry"
+
+    # table: dhcp4_shared_network_server
+    qry="select shared_network_id, server_id, modification_ts from dhcp4_shared_network_server"
+    run_statement "dhcp4_shared_network_server" "$qry"
+
+    # table: dhcp4_subnet
+    qry="select subnet_prefix, 4o6_interface, 4o6_interface_id, 4o6_subnet, boot_file_name, client_class, interface, match_client_id, modification_ts, next_server, rebind_timer, relay, renew_timer, require_client_classes, reservation_mode, server_hostname, shared_network_name, subnet_id, user_context, valid_lifetime from dhcp4_subnet"
+    run_statement "dhcp4_subnet" "$qry"
+
+    # table: dhcp4_pool
+    qry="select id, start_address, end_address, subnet_id, modification_ts from dhcp4_pool"
+    run_statement "dhcp4_pool" "$qry"
+
+    # table: dhcp4_subnet_server
+    qry="select subnet_id, server_id, modification_ts from dhcp4_subnet_server"
+    run_statement "dhcp4_subnet_server" "$qry"
+
+    # table: dhcp4_options (should include three new columns)
+    qry="select shared_network_name, pool_id, modification_ts from dhcp4_options"
+    run_statement "dhcp4_options" "$qry"
+
+    # table: dhcp4_options_server
+    qry="select option_id, server_id, modification_ts from dhcp4_options_server"
+    run_statement "dhcp4_options_server" "$qry"
+
+    # table: dhcp6_server
+    qry="select id, tag, description, modification_ts from dhcp6_server"
+    run_statement "dhcp6_server" "$qry"
+
+    # table: dhcp6_audit
+    qry="select id, object_type, object_id, modification_type, modification_ts, log_message from dhcp6_audit"
+    run_statement "dhcp6_audit" "$qry"
+
+    # table: dhcp6_global_parameter
+    qry="select id, name, value, modification_ts from dhcp6_global_parameter"
+    run_statement "dhcp6_global_parameter" "$qry"
+
+    # table: dhcp6_global_parameter_server
+    qry="select parameter_id, server_id, modification_ts from dhcp6_global_parameter_server"
+    run_statement "dhcp6_global_parameter_server" "$qry"
+
+    # table: dhcp6_option_def
+    qry="select id, code, space, modification_ts, array, encapsulate, record_types, user_context from dhcp6_option_def"
+    run_statement "dhcp6_option_def" "$qry"
+
+    # table: dhcp6_option_def_server
+    qry="select option_def_id, server_id, modification_ts from dhcp6_option_def_server"
+    run_statement "dhcp6_option_def_server" "$qry"
+
+    # table: dhcp6_shared_network
+    qry="select id, name, client_class, interface, modification_ts, preferred_lifetime, rapid_commit, rebind_timer, relay, renew_timer, require_client_classes, reservation_mode, server_hostname, user_context, valid_lifetime from dhcp6_shared_network"
+    run_statement "dhcp6_shared_network" "$qry"
+
+    # table: dhcp6_shared_network_server
+    qry="select shared_network_id, server_id, modification_ts from dhcp6_shared_network_server"
+    run_statement "dhcp6_shared_network" "$qry"
+
+    # table: dhcp6_subnet
+    qry="select subnet_prefix, client_class, interface, modification_ts, preferred_lifetime, rapid_commit, rebind_timer, relay, renew_timer, require_client_classes, reservation_mode, shared_network_name, subnet_id, user_context, valid_lifetime from dhcp6_subnet"
+    run_statement "dhcp6_subnet" "$qry"
+
+    # table: dhcp6_subnet_server
+    qry="select subnet_id, server_id, modification_ts from dhcp6_subnet_server"
+    run_statement "dhcp6_subnet_server" "$qry"
+
+    # table: dhcp6_pd_pool
+    qry="select id, prefix_length, delegated_prefix_length, dhcp6_subnet_id, modification_ts from dhcp6_pd_pool"
+    run_statement "dhcp6_pd_pool" "$qry"
+
+    # table: dhcp6_pool
+    qry="select id, start_address, end_address, dhcp6_subnet_id, modification_ts from dhcp6_pool"
+    run_statement "dhcp6_pool" "$qry"
+
+    # table: dhcp6_options (should include four new columns)
+    qry="select shared_network_name, pool_id, pd_pool_id, modification_ts from dhcp6_options"
+    run_statement "dhcp6_options" "$qry"
+
+    # table: dhcp6_options_server
+    qry="select option_id, server_id, modification_ts from dhcp6_options_server"
+    run_statement "dhcp6_options_server" "$qry"
+
     # Verify upgraded schema reports version 7.0
     version=$(${keaadmin} lease-version mysql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir)
     assert_str_eq "7.0" ${version} "Expected kea-admin to return %s, returned value was %s"
 
-
     # Let's wipe the whole database
     mysql_wipe
 

src/bin/netconf/Makefile.am

@@ -57,7 +57,7 @@ sbin_PROGRAMS = kea-netconf
 kea_netconf_SOURCES  = main.cc
 
 kea_netconf_LDADD  = libnetconf.la
-kea_netconf_LDADD += $(top_builddir)/src/lib/dhcpsrv/libkea-dhcpsrv.la
+kea_netconf_LDADD += $(top_builddir)/src/lib/process/libkea-process.la
 kea_netconf_LDADD += $(top_builddir)/src/lib/log/libkea-log.la
 kea_netconf_LDADD += $(top_builddir)/src/lib/exceptions/libkea-exceptions.la
 kea_netconf_LDADD += $(LOG4CPLUS_LIBS) $(CRYPTO_LIBS) $(BOOST_LIBS) $(SYSREPO_LIBS)

src/bin/netconf/main.cc

@@ -8,7 +8,7 @@
 
 #include <netconf/netconf_log.h>
 #include <exceptions/exceptions.h>
-#include <dhcpsrv/daemon.h>
+#include <process/daemon.h>
 #include <iostream>
 #include <fstream>
 #include <unistd.h>
@@ -36,7 +36,7 @@ usage() {
     exit(EXIT_FAILURE);
 }
 
-/// @name Temporary code until isc::dhcp::Daemon is used.
+/// @name Temporary code until isc::process::Daemon is used.
 ///
 /// @{
 const char* PID_FILENAME = "kea-netconf.test_config.pid";
@@ -46,7 +46,7 @@ volatile bool SHUTDOWN_FLAG = false;
 void
 createPIDFile(int pid) {
     // This is not a real implemented. We will soon use the one coming
-    // from isc::dhcp::Daemon AFTER it's moved to libprocess.
+    // from isc::process::Daemon AFTER it's moved to libprocess.
 
     ofstream file(PID_FILENAME, ios::trunc);
     file << pid;
@@ -101,7 +101,7 @@ main(int argc, char* argv[]) {
     int ret = EXIT_SUCCESS;
     try {
 
-        // Temporary code. This will be replaced with isc::dhcp::Daemon
+        // Temporary code. This will be replaced with isc::process::Daemon
         // once it is migrated to libprocess. We DO NOT want to bring
         // the whole libdhcpsrv into netconf.
         createPIDFile(getpid());
@@ -111,7 +111,7 @@ main(int argc, char* argv[]) {
 
         // Initialize logging.  If verbose, we'll use maximum verbosity.
         bool verbose_mode = true;
-        isc::dhcp::Daemon::loggerInit(NETCONF_LOGGER_NAME, verbose_mode);
+        isc::process::Daemon::loggerInit(NETCONF_LOGGER_NAME, verbose_mode);
         LOG_INFO(netconf_logger, NETCONF_STARTING).arg(VERSION).arg(getpid());
 
         Connection conn("kea-netconf");

src/lib/Makefile.am

@@ -13,5 +13,10 @@ if HAVE_CQL
 SUBDIRS += cql
 endif
 
-SUBDIRS += testutils hooks dhcp config stats asiodns dhcp_ddns eval \
-	cfgrpt process dhcpsrv http
+SUBDIRS += config_backend testutils hooks dhcp config stats
+
+if HAVE_SYSREPO
+SUBDIRS += yang
+endif
+
+SUBDIRS += asiodns dhcp_ddns eval cfgrpt process dhcpsrv http

src/lib/asiolink/addr_utilities.h

-// Copyright (C) 2012-2015,2017 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2012-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -21,8 +21,6 @@ namespace dhcp {
 /// Example: For 2001:db8:1\::deaf:beef and length /120 the function will return
 /// 2001:db8:1\::dead:be00. See also @ref lastAddrInPrefix.
 ///
-/// @todo It currently works for v6 only and will throw if v4 address is passed.
-///
 /// @param prefix and address that belongs to a prefix
 /// @param len prefix length
 ///
@@ -35,8 +33,6 @@ isc::asiolink::IOAddress firstAddrInPrefix(const isc::asiolink::IOAddress& prefi
 /// Example: For 2001:db8:1\::deaf:beef and length /112 the function will return
 /// 2001:db8:1\::dead:ffff. See also @ref firstAddrInPrefix.
 ///
-/// @todo It currently works for v6 only and will throw if v4 address is passed.
-///
 /// @param prefix and address that belongs to a prefix
 /// @param len prefix length
 ///

src/lib/config_backend/Makefile.am

+SUBDIRS = . tests
+
+AM_CPPFLAGS = -I$(top_srcdir)/src/lib -I$(top_builddir)/src/lib
+AM_CPPFLAGS += $(BOOST_INCLUDES)
+
+AM_CXXFLAGS = $(KEA_CXXFLAGS)
+
+EXTRA_DIST  = base_config_backend.h
+EXTRA_DIST += base_config_backend_mgr.h
+EXTRA_DIST += base_config_backend_pool.h
+
+# The message file should be in the distribution.
+#EXTRA_DIST += config_backend.dox
+
+CLEANFILES = *.gcno *.gcda
+
+# Specify the headers for copying into the installation directory tree.
+libkea_cb_includedir = $(pkgincludedir)/config_backend
+libkea_cb_include_HEADERS = \
+	base_config_backend.h \
+	base_config_backend_mgr.h \
+	base_config_backend_pool.h
+

src/lib/config_backend/base_config_backend.h

+// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef BASE_CONFIG_BACKEND_H
+#define BASE_CONFIG_BACKEND_H
+
+#include <boost/shared_ptr.hpp>
+#include <cstdint>
+#include <set>
+#include <string>
+
+namespace isc {
+namespace cb {
+
+/// @brief Interface for Kea server specific configuration backend
+/// implementations.
+///
+/// Each Kea server (e.g. DHCPv4 server) needs to implement its own
+/// interface to store and fetch its configuration from the databases.
+/// This is because each Kea server uses a different set of
+/// configuration information. This is a base interface which should
+/// be implemented (and extended) by respective Kea servers to provide
+/// API to store and fetch configuration information from a database.
+/// Such implementation is called configuration backend. Each
+/// configuration backend faciliates a single database type, e.g. MySQL
+/// database. In order to support multiple database types, i.e. MySQL,
+/// Posrgres, Cassandra, each Kea server will have to implement
+/// 3 separate configuration backends, one for each database type.
+class BaseConfigBackend {
+public:
+
+    /// @brief Virtual destructor.
+    virtual ~BaseConfigBackend() { }
+
+    /// @brief Returns backend type in the textual format.
+    ///
+    /// @return Name of the storage for configurations, e.g. "mysql",
+    /// "pgsql" and so forth.
+    virtual std::string getType() const = 0;
+
+    /// @brief Returns backend host.
+    ///
+    /// This is used by the @c BaseConfigBackendPool to select backend
+    /// when @c BackendSelector is specified.
+    ///
+    /// @return host on which the database is located.
+    virtual std::string getHost() const = 0;
+
+    /// @brief Returns backend port number.
+    ///
+    /// This is used by the @c BaseConfigBackendPool to select backend
+    /// when @c BackendSelector is specified.
+    ///
+    /// @return Port number on which database service is available.
+    virtual uint16_t getPort() const = 0;
+};
+
+/// @brief Shared pointer to the @c BaseConfigBackend.
+typedef boost::shared_ptr<BaseConfigBackend> BaseConfigBackendPtr;
+
+} // end of namespace isc::cb
+} // end of namespace isc
+
+#endif // BASE_CONFIG_BACKEND_H

src/lib/config_backend/base_config_backend_mgr.h

+// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef BASE_CONFIG_BACKEND_MGR_H
+#define BASE_CONFIG_BACKEND_MGR_H
+
+#include <config_backend/base_config_backend.h>
+#include <database/database_connection.h>
+#include <exceptions/exceptions.h>
+#include <boost/shared_ptr.hpp>
+#include <functional>
+#include <map>
+#include <string>
+
+namespace isc {
+namespace cb {
+
+/// @brief Base class for Configuration Backend Managers (CBM).
+///
+/// Each Kea server supporting Configuration Backend feature implements
+/// a "manager" class which holds information about supported and
+/// configured backends and provides access to the backends. This is
+/// similar to @c HostMgr and @c LeaseMgr singletons being used by the
+/// DHCP servers.
+///
+/// The Config Backend Managers are typically implemented as singletons
+/// which can be accessed from any place within the server code. This
+/// includes server configuration, data fetching during normal server
+/// operation and data management, including processing of control
+/// commands implemented within hooks libraries.
+///
+/// The @c BaseConfigBackendMgr is a base class for all CBMs implemented
+/// for respective Kea servers. It includes mechanisms to register config
+/// backend factory functions and to create instances of the backends using
+/// those factory functions as a result of server configuration. The mechanism
+/// of factory functions registration is useful in cases when the config
+/// backend is implemented within the hook library. Such hook library
+/// registers factory function in its @c load function and the server
+/// simply calls this function to create the instance of this backend when
+/// instructed to do so via configuration. Similar mechanism exists in
+/// DHCP @c HostMgr.
+///
+/// Unlike @c HostMgr, the CBMs do not directly expose API to fetch and
+/// manipulate the data in the database. This is done via, so called,
+/// Configuration Backends Pools. See @c BaseConfigBackendPool for
+/// details. The @c BaseConfigBackendMgr is provided with the pool type
+/// via class template parameter. Respective CBM implementations
+/// use their own pools, which provide APIs appropriate for those
+/// implementation.
+///
+/// @tparam ConfgBackendPoolType Type of the configuration backend pool
+/// to be used by the manager. It must derive from @c BaseConfigBackendPool
+/// template class.
+template<typename ConfigBackendPoolType>
+class BaseConfigBackendMgr {
+public:
+
+    /// @brief Pointer to the configuration backend pool.
+    typedef boost::shared_ptr<ConfigBackendPoolType> ConfigBackendPoolPtr;
+
+    /// @brief Type of the backend factory function.
+    ///
+    /// Factory function returns a pointer to the instance of the configuration
+    /// backend created.
+    typedef std::function<typename ConfigBackendPoolType::ConfigBackendTypePtr
+                          (const db::DatabaseConnection::ParameterMap&)> Factory;
+
+    /// @brief Constructor.
+    BaseConfigBackendMgr()
+        : factories_(), pool_(new ConfigBackendPoolType()) {
+    }
+
+    /// @brief Registers new backend factory function for a given backend type.
+    ///
+    /// The typical usage of this function is to make the CBM aware of a
+    /// configuration backend implementation. This implementation may exist
+    /// in a hooks library. In such case, this function should be called from
+    /// the @c load function in this library. When the backend is registered,
+    /// the server will use it when required by the configuration, i.e. a
+    /// user includes configuration backend of that type in the
+    /// "config-databases" list.
+    ///
+    /// If the backend of the given type has already been registered, perhaps
+    /// by another hooks library, the CBM will refuse to register another
+    /// backend of the same type.
+    ///
+    /// @param db_type Backend type, e.g. "mysql".
+    /// @param factory Pointer to the backend factory function.
+    ///
+    /// @return true if the backend has been successfully registered, false
+    /// if another backend of this type already exists.
+    bool registerBackendFactory(const std::string& db_type,
+                                const Factory& factory) {
+        // Check if this backend has been already registered.
+        if (factories_.count(db_type)) {
+            return (false);
+        }
+
+        // Register the new backend.
+        factories_.insert(std::make_pair(db_type, factory));
+        return (true);
+    }
+