[204-move-models-] Code done, tests to do

src/bin/netconf/netconf.cc

@@ -14,6 +14,7 @@
 #include <netconf/netconf_log.h>
 #include <cc/command_interpreter.h>
 #include <yang/translator_config.h>
+#include <yang/yang_revisions.h>
 #include <boost/algorithm/string.hpp>
 #include <sstream>
 
@@ -28,7 +29,7 @@ using namespace sysrepo;
 
 namespace {
 
-/// @brief Subscription callback.
+/// @brief Module change subscription callback.
 class NetconfAgentCallback : public Callback {
 public:
     /// @brief Constructor.
@@ -94,6 +95,31 @@ public:
     }
 };
 
+/// @brief Module (un)installation subscription callback.
+class NetconfAgentInstallCallback : public Callback {
+public:
+    /// @brief Module (un)installation callback.
+    ///
+    /// This callback is called by sysrepo when a module is (un)installed.
+    ///
+    /// @param module_name The module name.
+    /// @param revision The module revision.
+    /// @param state The new state of the module (ignored).
+    /// @param private_ctx The private context.
+    void module_install(const char* module_name,
+                        const char* revision,
+                        sr_module_state_t /*state*/,
+                        void* /*private_ctx*/) {
+        if (!module_name || !revision) {
+            // Not for us...
+            return;
+        }
+        LOG_WARN(netconf_logger, NETCONF_MODULE_INSTALL)
+            .arg(module_name)
+            .arg(revision);
+    }
+};
+
 } // end of anonymous namespace
 
 namespace isc {
@@ -135,6 +161,31 @@ NetconfAgent::init(NetconfCfgMgrPtr cfg_mgr) {
         return;
     }
 
+    // Check essential modules / revisions.
+    bool can_start = true;
+    for (auto pair : *servers) {
+        can_start = can_start && checkModule(pair.second->getModel());
+        if (NetconfProcess::shut_down) {
+            return;
+        }
+    }
+    if (!can_start) {
+        cerr << "An essential YNAG module / revision is missing."
+             << endl
+             << "The environment is not suitable for running kea-netconf."
+             << endl;
+        exit(EXIT_FAILURE);
+    }
+    if (NetconfProcess::shut_down) {
+        return;
+    }
+
+    // Check modules / revisions.
+    checkModules();
+    if (NetconfProcess::shut_down) {
+        return;
+    }
+
     for (auto pair : *servers) {
         if (NetconfProcess::shut_down) {
             return;
@@ -231,14 +282,107 @@ NetconfAgent::initSysrepo() {
     } catch (const std::exception& ex) {
         isc_throw(Unexpected, "Can't connect to sysrepo: " << ex.what());
     }
+    if (NetconfProcess::shut_down) {
+        return;
+    }
 
     try {
         startup_sess_.reset(new Session(conn_, SR_DS_STARTUP));
+        if (NetconfProcess::shut_down) {
+            return;
+        }
         running_sess_.reset(new Session(conn_, SR_DS_RUNNING));
     } catch (const std::exception& ex) {
         isc_throw(Unexpected,  "Can't establish a sysrepo session: "
                   << ex.what());
     }
+    if (NetconfProcess::shut_down) {
+        return;
+    }
+
+    try {
+        S_Yang_Schemas schemas = startup_sess_->list_schemas();
+        for (size_t i = 0; i < schemas->schema_cnt(); ++i) {
+            if (NetconfProcess::shut_down) {
+                return;
+            }
+            if (!schemas->schema(i) ||
+                !schemas->schema(i)->module_name()) {
+                // Should not happen: skip it.
+                continue;
+            }
+            string module = schemas->schema(i)->module_name();
+            if (!schemas->schema(i)->revision() ||
+                !schemas->schema(i)->revision()->revision()) {
+                // Our modules have revisions: skip it.
+                continue;
+            }
+            string revision = schemas->schema(i)->revision()->revision();
+            modules_.insert(make_pair(module, revision));
+        }
+    } catch (const sysrepo_exception& ex) {
+        isc_throw(Unexpected,  "Can't list schemas: " << ex.what());
+    }
+    if (NetconfProcess::shut_down) {
+        return;
+    }
+
+    try {
+        S_Subscribe subs(new Subscribe(startup_sess_));
+        S_Callback cb(new NetconfAgentInstallCallback());
+        subs->module_install_subscribe(cb);
+        subscriptions_.insert(make_pair("__install__", subs));
+    } catch (const sysrepo_exception& ex) {
+        isc_throw(Unexpected,  "Can't subscribe moduel install: "
+                  << ex.what());
+    }
+}
+
+bool
+NetconfAgent::checkModule(const string& module_name) const {
+    if (module_name.empty()) {
+        return (true);
+    }
+    auto module = modules_.find(module_name);
+    if (module == modules_.end()) {
+        LOG_ERROR(netconf_logger, METCONF_MODULE_MISSING_ERR)
+            .arg(module_name);
+        return (false);
+    }
+    auto modrev = YANG_REVISIONS.find(module_name);
+    if (modrev == YANG_REVISIONS.end()) {
+        // Can't check revision?!
+        return (true);
+    }
+    if (modrev->second != module->second) {
+        LOG_ERROR(netconf_logger, METCONF_MODULE_REVISION_ERR)
+            .arg(module_name)
+            .arg(modrev->second)
+            .arg(module->second);
+        return (false);
+    }
+    return (true);
+}
+
+void
+NetconfAgent::checkModules() const {
+    for (auto modrev : YANG_REVISIONS) {
+        if (NetconfProcess::shut_down) {
+            return;
+        }
+        auto module = modules_.find(modrev.first);
+        if (module == modules_.end()) {
+            LOG_WARN(netconf_logger, METCONF_MODULE_MISSING_WARN)
+                .arg(modrev.first);
+            continue;
+        }
+        if (modrev.second != module->second) {
+            LOG_WARN(netconf_logger, METCONF_MODULE_REVISION_WARN)
+                .arg(modrev.first)
+                .arg(modrev.second)
+                .arg(module->second);
+        }
+    }
 }
 
 void

src/bin/netconf/netconf.h

@@ -52,11 +52,13 @@ public:
 
     /// @brief Initialize sysrepo sessions.
     ///
-    /// Must be called before init.
+    /// Must be called before init. Collect the list of available
+    /// modules with their revisions.
     void initSysrepo();
 
     /// @brief Initialization.
     ///
+    /// Check available modules / revisions.
     /// Get and display Kea server configurations.
     /// Load Kea server configurations from YANG datastore.
     /// Subscribe configuration changes in YANG datastore.
@@ -118,6 +120,22 @@ public:
     bool cancel_;
 
 protected:
+    /// @brief Check essential module availability.
+    ///
+    /// Emit a fatal error if an essential one (i.e. required in
+    /// a further phase) is missing or does not have the expected revision.
+    /// The caller (init) will exit().
+    ///
+    /// @param module_name The module name.
+    /// @return true if available, false if not.
+    bool checkModule(const std::string& module_name) const;
+
+    /// @brief Check module availability.
+    ///
+    /// Emit a warning if a module is missing or does not have
+    /// the expected revision.
+    void checkModules() const;
+
     /// @brief Get and display Kea server configuration.
     ///
     /// Retrieves current configuration via control socket (unix or http)
@@ -166,6 +184,9 @@ protected:
     S_Session running_sess_;
 #endif
 
+    /// @brief Available modules and revisions in Sysrepo.
+    std::map<const std::string, const std::string> modules_;
+
     /// @brief Subscription map.
 #ifndef HAVE_PRE_0_7_6_SYSREPO
     std::map<const std::string, sysrepo::S_Subscribe> subscriptions_;

src/bin/netconf/netconf_messages.mes

@@ -52,6 +52,30 @@ configuration from a Kea server.
 The warning message indicates that the configuration change logging
 encountered an unexpected condition. Details of it will be logged.
 
+% NETCONF_MODULE_INSTALL Sysrepo (un)installs a module: %1 (revision %2)
+This warning message indicates that sysrepo reports the installation
+or uninstallation of a module used by Kea. The name and revision of
+the module are printed.
+
+% METCONF_MODULE_MISSING_ERR Missing essential module %1 in sysrepo
+This fatal error message indicates that a module required by Netconf
+configuration is not available in the sysrepo repository.  The name of
+the module is printed.
+
+% METCONF_MODULE_MISSING_WARN Missing module %1 in sysrepo
+This warning message indicates that a module used by Kea is not
+available in the sysrepo repository. The name of the module is printed.
+
+% METCONF_MODULE_REVISION_ERR Essential module %1 does have the right revision: expected %2, got %3
+This fatal error message indicates that a module required by Netconf
+configuration is not at the right revision in the sysrepo repository.
+The name, expected and available revisions of the module are printed.
+
+% METCONF_MODULE_REVISION_WARN Module %1 does have the right revision: expected %2, got %3
+This warning message indicates that a module used by Kea is not at the
+right revision in the sysrepo repository. The name, expected and
+available revisions of the module are printed.
+
 % NETCONF_RUN_EXIT application is exiting the event loop
 This is a debug message issued when the Netconf application exits its
 event loop. This is a normal step during kea-netconf shutdown.

src/bin/netconf/tests/netconf_unittests.cc

@@ -361,6 +361,7 @@ TEST_F(NetconfAgentTest, initSysrepo) {
     EXPECT_TRUE(agent_->conn_);
     EXPECT_TRUE(agent_->startup_sess_);
     EXPECT_TRUE(agent_->running_sess_);
+    EXPECT_EQ(1, agent_->subscriptions_.size());
 }
 
 /// @brief Default change callback (print changes and return OK).
@@ -771,8 +772,9 @@ TEST_F(NetconfAgentTest, subscribeConfig) {
     // Try subscribeConfig.
     EXPECT_EQ(0, agent_->subscriptions_.size());
     ASSERT_NO_THROW(agent_->initSysrepo());
-    EXPECT_NO_THROW(agent_->subscribeConfig(service_pair));
     EXPECT_EQ(1, agent_->subscriptions_.size());
+    EXPECT_NO_THROW(agent_->subscribeConfig(service_pair));
+    EXPECT_EQ(2, agent_->subscriptions_.size());
 
     /// Unsubscribe.
     EXPECT_NO_THROW(agent_->subscriptions_.clear());
@@ -841,9 +843,9 @@ TEST_F(NetconfAgentTest, update) {
     CfgServersMapPair service_pair = *servers_map->begin();
 
     // Subscribe YANG changes.
-    EXPECT_EQ(0, agent_->subscriptions_.size());
-    EXPECT_NO_THROW(agent_->subscribeConfig(service_pair));
     EXPECT_EQ(1, agent_->subscriptions_.size());
+    EXPECT_NO_THROW(agent_->subscribeConfig(service_pair));
+    EXPECT_EQ(2, agent_->subscriptions_.size());
 
     // Launch server.
     thread_.reset(new Thread([this]() { fakeServer(); signalStopped(); }));
@@ -973,9 +975,9 @@ TEST_F(NetconfAgentTest, validate) {
     CfgServersMapPair service_pair = *servers_map->begin();
 
     // Subscribe YANG changes.
-    EXPECT_EQ(0, agent_->subscriptions_.size());
-    EXPECT_NO_THROW(agent_->subscribeConfig(service_pair));
     EXPECT_EQ(1, agent_->subscriptions_.size());
+    EXPECT_NO_THROW(agent_->subscribeConfig(service_pair));
+    EXPECT_EQ(2, agent_->subscriptions_.size());
 
     // Launch server twice.
     thread_.reset(new Thread([this]()
@@ -1136,9 +1138,9 @@ TEST_F(NetconfAgentTest, noValidate) {
     CfgServersMapPair service_pair = *servers_map->begin();
 
     // Subscribe YANG changes.
-    EXPECT_EQ(0, agent_->subscriptions_.size());
-    EXPECT_NO_THROW(agent_->subscribeConfig(service_pair));
     EXPECT_EQ(1, agent_->subscriptions_.size());
+    EXPECT_NO_THROW(agent_->subscribeConfig(service_pair));
+    EXPECT_EQ(2, agent_->subscriptions_.size());
 
     // Change configuration (add invalid user context).
     const YRTree tree1 = {

src/lib/yang/pretests/sysrepo_setup_tests.cc

@@ -6,6 +6,7 @@
 
 #include <config.h>
 
+#define KEATEST_MODULE
 #include <yang/yang_revisions.h>
 
 #ifndef HAVE_PRE_0_7_6_SYSREPO

src/lib/yang/yang_revisions.h

@@ -15,19 +15,19 @@ namespace yang {
 
 // Table of module name / revision.
 static const std::map<std::string, std::string> YANG_REVISIONS = {
-// Should be generated automatically.
-// cf src/share/yang/modules/README
-{ "ietf-dhcpv6-types", "2018-09-04" },
-{ "ietf-dhcpv6-options", "2018-09-04" },
-{ "ietf-dhcpv6-server", "2018-09-04" },
-{ "kea-types", "2018-11-20" },
-{ "kea-logging", "2018-11-20" },
-{ "kea-dhcp-types", "2018-11-20" },
-{ "kea-dhcp4-server", "2018-11-20" },
-{ "kea-dhcp6-server", "2018-11-20" },
-{ "kea-ctrl-agent", "2018-11-20" },
-{ "kea-dhcp-ddns", "2018-11-20" },
-{ "keatest-module", "2018-11-20" }
+#ifdef KEATEST_MODULE
+    { "keatest-module", "2018-11-20" },
+#endif // KEATEST_MODULE
+    { "ietf-dhcpv6-types", "2018-09-04" },
+    { "ietf-dhcpv6-options", "2018-09-04" },
+    { "ietf-dhcpv6-server", "2018-09-04" },
+    { "kea-types", "2018-11-20" },
+    { "kea-logging", "2018-11-20" },
+    { "kea-dhcp-types", "2018-11-20" },
+    { "kea-dhcp4-server", "2018-11-20" },
+    { "kea-dhcp6-server", "2018-11-20" },
+    { "kea-ctrl-agent", "2018-11-20" },
+    { "kea-dhcp-ddns", "2018-11-20" }
 };
 
 }; // end of namespace isc::yang

src/lib/yang/yang_revisions.h.skel

+// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef ISC_YANG_REVISIONS_H
+#define ISC_YANG_REVISIONS_H 1
+
+#include <map>
+#include <string>
+
+namespace isc {
+namespace yang {
+
+// Table of module name / revision.
+static const std::map<std::string, std::string> YANG_REVISIONS = {
+// Fill this by:
+// cd .../src/share/yang/modules
+// sh utils/gen-revisions.sh > r
+// insert the r file here
+};
+
+}; // end of namespace isc::yang
+}; // end of namespace isc
+
+#endif // ISC_YANG_REVISIONS_H

src/share/yang/modules/README

-Developer tools:
-
-Get revision from file content:
-
-yanglint -f yin xxx | grep '<revision date=' | head -1 | sed \
- 's/.*<revision date="\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)".*/\1/'
-
-Get revision from file name:
-
-echo xxx | sed 's/.*@\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)\..*/\1/'
-
-Get hash:
-
-yanglint -f yin xxx | openssl dgst -sha256
-
-Check revisions:
-
-for m in *.yang
-do
-    rev1=`yanglint -f yin $m | grep '<revision date=' | head -1 | sed \
- 's/.*<revision date="\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)".*/\1/'`
-    rev2=`echo $m | sed \
- 's/.*@\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)\..*/\1/'`
-
-    if test $rev1 != $rev2
-    then
-        echo revision mismatch on $m
-    fi
-done
-
-Check hashes:
-
-for m in *.yang
-do
-    hash1=`yanglint -f yin $m | openssl dgst -sha256`
-    h=hashes/`basename $m .yang`.hash
-    if test -f $h
-    then
-        hash2=`cat $h`
-        if test $hash1 != $hash2
-        then
-            echo hash mismatch on $m
-        fi
-    else
-        echo missing hash for $m
-    fi
-done
-
-Build module / revision table:
-
-for m in ietf-dhcpv6-types*.yang \
-         ietf-dhcpv6-options*.yang \
-         ietf-dhcpv6-server*.yang \
-         kea-types*.yang \
-         kea-logging*.yang \
-         kea-dhcp-types*.yang \
-         kea-dhcp4-server*.yang \
-         kea-dhcp6-server*.yang \
-         kea-ctrl-agent*.yang \
-         kea-dhcp-ddns*.yang \
-         keatest-module*.yang \
-         end-marker
-do
-    if test $m = "end-marker"
-    then
-        echo '{ "", "" }'
-    else
-        b=`echo $m | sed 's/\(.*\)@.*/\1/'`
-        r=`echo $m | sed 's/.*@\(.*\)\.yang/\1/'`
-        echo '{ "'$b'", "'$r'" },'
-    fi
-done
-
-

src/share/yang/modules/utils/check-hashes.sh

+#!/bin/sh
+# Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Check hashes:
+
+for m in *.yang
+do
+    hash1=`yanglint -f yin $m | openssl dgst -sha256 | sed 's/(stdin)= //'`
+    h=hashes/`basename $m .yang`.hash
+    if test -f $h
+    then
+        hash2=`cat $h`
+        if test $hash1 != $hash2
+        then
+            echo hash mismatch on $m expected $hash1 in $h
+        fi
+    else
+        echo missing hash file $h for $m
+    fi
+done

src/share/yang/modules/utils/check-revisions.sh

+#!/bin/sh
+# Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Check revisions:
+
+for m in *.yang
+do
+    rev1=`yanglint -f yin $m | grep '<revision date=' | head -1 | sed \
+ 's/.*<revision date="\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)".*/\1/'`
+    rev2=`echo $m | sed \
+ 's/.*@\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)\..*/\1/'`
+
+    if test $rev1 != $rev2
+    then
+        echo revision mismatch on $m got $rev1
+    fi
+done

src/share/yang/modules/utils/gen-revisions.sh

+#!/bin/sh
+# Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Build module / revision table:
+
+test=keatest-module*.yang
+last=kea-dhcp-ddns*.yang
+
+for m in keatest-module*.yang \
+         ietf-dhcpv6-types*.yang \
+         ietf-dhcpv6-options*.yang \
+         ietf-dhcpv6-server*.yang \
+         kea-types*.yang \
+         kea-logging*.yang \
+         kea-dhcp-types*.yang \
+         kea-dhcp4-server*.yang \
+         kea-dhcp6-server*.yang \
+         kea-ctrl-agent*.yang \
+         kea-dhcp-ddns*.yang
+do
+    if test $m = $test
+    then
+        echo '#ifdef KEATEST_MODULE'
+    fi
+    b=`echo $m | sed 's/\(.*\)@.*/\1/'`
+    r=`echo $m | sed 's/.*@\(.*\)\.yang/\1/'`
+    c=','
+    if test $m = $last
+    then
+        c=''
+    fi
+    echo '    { "'$b'", "'$r'" }'$c
+    if test $m = $test
+    then
+        echo '#endif // KEATEST_MODULE'
+    fi
+done
+
+