[204-move-models-] Checkpoint: moved modules, added revisions, updated pretests

configure.ac

@@ -1659,6 +1659,8 @@ AC_CONFIG_FILES([Makefile
                  src/share/database/scripts/pgsql/upgrade_3.2_to_3.3.sh
                  src/share/database/scripts/pgsql/upgrade_3.3_to_4.0.sh
                  src/share/database/scripts/pgsql/upgrade_4.0_to_5.0.sh
+                 src/share/yang/Makefile
+                 src/share/yang/modules/Makefile
                  tools/Makefile
                  tools/path_replacer.sh
 ])

src/lib/yang/Makefile.am

@@ -64,7 +64,8 @@ libkea_yang_include_HEADERS = \
 	translator_pool.h \
 	translator_shared_network.h \
 	translator_subnet.h \
-	yang_models.h
+	yang_models.h \
+	yang_revisions.h
 
 EXTRA_DIST = yang.dox
 # Distribute yang modules.

src/lib/yang/pretests/Makefile.am

@@ -2,7 +2,10 @@ CLEANFILES = *.gcno *.gcda
 
 TESTS = sysrepo_setup_tests
 sysrepo_setup_tests_SOURCES = sysrepo_setup_tests.cc
-sysrepo_setup_tests_CPPFLAGS = $(SYSREPO_CPPFLAGS) -DSYSREPO_REPO=\"@SYSREPO_REPO@\"
+sysrepo_setup_tests_CPPFLAGS = -I$(top_builddir)/src/lib
+sysrepo_setup_tests_CPPFLAGS += -I$(top_srcdir)/src/lib
+sysrepo_setup_tests_CPPFLAGS += $(SYSREPO_CPPFLAGS)
+sysrepo_setup_tests_CPPFLAGS += -DSYSREPO_REPO=\"@SYSREPO_REPO@\"
 sysrepo_setup_tests_LDFLAGS = $(AM_LDFLAGS)
 sysrepo_setup_tests_LDADD = $(SYSREPO_LIBS)
 

src/lib/yang/pretests/sysrepo_setup_tests.cc

@@ -6,6 +6,8 @@
 
 #include <config.h>
 
+#include <yang/yang_revisions.h>
+
 #ifndef HAVE_PRE_0_7_6_SYSREPO
 #include <sysrepo-cpp/Session.hpp>
 #else
@@ -18,31 +20,46 @@ using namespace std;
 #ifndef HAVE_PRE_0_7_6_SYSREPO
 using namespace sysrepo;
 #endif
+using namespace isc::yang;
 
-/// Not using libyang headers, e.g. yang_models.h.
-
-const string TEST_MODULE = "keatest-module";
-const string IETF_TYPES = "ietf-inet-types";
-const string YANG_TYPES = "ietf-yang-types";
-const string IETF_MODULE = "ietf-dhcpv6-server";
-const string KEA_DHCP4_MODULE = "kea-dhcp4-server";
-const string KEA_DHCP6_MODULE = "kea-dhcp6-server";
-const string KEA_CTRL_AGENT_MODULE = "kea-ctrl-agent";
-const string KEA_D2_MODULE = "kea-dhcp-ddns";
 const string REPOSITORY = SYSREPO_REPO;
 
 /// @brief Returns nicely formed error message if module is missing
 ///
 /// @param name name of the YANG module to complain about
+/// #param revision revision of the YANG module
 /// @return a text explaining what the problem is and how to fix it
-string missingModuleText(const string& name) {
+string missingModuleText(const string& name, const string& revision) {
     stringstream tmp;
-    tmp << "ERROR: YANG model " << name << " is not installed." << endl
+    tmp << "ERROR: YANG module " << name << " is not installed." << endl
         << "The environment is not suitable for running unit-tests." << endl
-        << "Please locate " << name << ".yang, change to its directory and "
-        << "issue the following command:" << endl << endl
+        << "Please locate " << name << "@" << revision << ".yang, "
+        << "change to its directory and issue the following command:"
+        << endl << endl
         << "# sysrepoctl -i -s " << REPOSITORY << "/yang "
-        << "-s . -g " << name << ".yang" << endl
+        << "-s . -g " << name << "@" << revision << ".yang" << endl
+        << endl << endl;
+    return (tmp.str());
+}
+
+/// @brief Returns nicely formed error message if module does not have
+/// the expected revision.
+///
+/// @param name name of the YANG module to complain about
+/// @param expected expected revision of the YANG module
+/// @param got got (bad) revision of the YANG module
+string badRevisionModuleText(const string& name, const string& expected,
+                             const string& got) {
+    stringstream tmp;
+    tmp << "ERROR: YANG module " << name << " is not installed with the right "
+        << "revision: expected " << expected << " but got " << got << endl
+        << "Please remove the module " << name << " and re-install it: "
+        << "Please locate " << name << "@" << expected << ".yang, " << endl
+        << "change to its directory and issue the following commands:"
+        << endl << endl
+        << "# sysrepoctl -u -m " << name << endl
+        << "# sysrepoctl -i -s " << REPOSITORY << "/yang "
+        << "-s . -g " << name << "@" << expected << ".yang" << endl
         << endl << endl;
     return (tmp.str());
 }
@@ -81,86 +98,44 @@ int main() {
         exit(-3);
     }
 
-    bool found_test = false;
-    bool found_ietf_types = false;
-    bool found_yang_types =false;
-    bool found_ietf = false;
-    bool found_kea4 = false;
-    bool found_kea6 = false;
-    bool found_keaca = false;
-    bool found_kea2 = false;
+    map<string, bool> found;
+    map<string, string> got;
+    for (auto modrev : YANG_REVISIONS) {
+        found[modrev.first] = false;
+    }
+
     for (size_t i = 0; i < schemas->schema_cnt(); ++i) {
         string module = schemas->schema(i)->module_name();
-        size_t rev = module.find("@");
-        if (rev != string::npos) {
-            module = module.substr(0, rev);
-        }
-        if (module == TEST_MODULE) {
-            found_test = true;
-        } else if (module == IETF_TYPES) {
-            found_ietf_types = true;
-        } else if (module == YANG_TYPES) {
-            found_yang_types = true;
-        } else if (module == IETF_MODULE) {
-            found_ietf = true;
-        } else if (module == KEA_DHCP4_MODULE) {
-            found_kea4 = true;
-        } else if (module == KEA_DHCP6_MODULE) {
-            found_kea6 = true;
-        } else if (module == KEA_CTRL_AGENT_MODULE) {
-            found_keaca = true;
-        } else if (module == KEA_D2_MODULE) {
-            found_kea2 = true;
+        if (YANG_REVISIONS.count(module)) {
+            found[module] = true;
+            if (!schemas->schema(i)->revision() ||
+                !schemas->schema(i)->revision()->revision()) {
+                got[module] = "none";
+            } else {
+                string rev = schemas->schema(i)->revision()->revision();
+                got[module] = rev;
+            }
         }
     }
 
     int exit_code = 0;
 
-    if (!found_test ||
-        !found_ietf_types || !found_yang_types ||
-        !found_ietf || !found_kea4 || !found_kea6 || !found_keaca ||
-        !found_kea2) {
-        exit_code = -4;
-    }
-
-    if (!found_test) {
-        cerr << missingModuleText(TEST_MODULE);
-        --exit_code;
-    }
-
-    if (!found_ietf_types) {
-        cerr << missingModuleText(IETF_TYPES);
-        --exit_code;
-    }
-
-    if (!found_yang_types) {
-        cerr << missingModuleText(YANG_TYPES);
-        --exit_code;
-    }
-
-    if (!found_ietf) {
-        cerr << missingModuleText(IETF_MODULE);
-        --exit_code;
-    }
-
-    if (!found_kea4) {
-        cerr << missingModuleText(KEA_DHCP4_MODULE);
-        --exit_code;
-    }
-
-    if (!found_kea6) {
-        cerr << missingModuleText(KEA_DHCP6_MODULE);
-        --exit_code;
-    }
-
-    if (!found_keaca) {
-        cerr << missingModuleText(KEA_CTRL_AGENT_MODULE);
-        --exit_code;
-    }
-
-    if (!found_kea2) {
-        cerr << missingModuleText(KEA_D2_MODULE);
-        --exit_code;
+    for (auto modfnd : found) {
+        string rev = YANG_REVISIONS.at(modfnd.first);
+        if (!modfnd.second) {
+            if (exit_code > -4) {
+                exit_code = -4;
+            }
+            --exit_code;
+            cerr << missingModuleText(modfnd.first, rev);
+        } else if (rev != got[modfnd.first]) {
+            if (exit_code > -40) {
+                exit_code += -40;
+            } else {
+                exit_code += -10;
+            }
+            cerr << badRevisionModuleText(modfnd.first, rev, got[modfnd.first]);
+        }
     }
 
     try {

src/lib/yang/tests/Makefile.am

@@ -12,8 +12,6 @@ CLEANFILES = *.gcno *.gcda
 TESTS_ENVIRONMENT = \
 	$(LIBTOOL) --mode=execute $(VALGRIND_COMMAND)
 
-EXTRA_DIST = keatest-module.yang
-
 TESTS =
 if HAVE_GTEST
 TESTS += run_unittests

src/lib/yang/yang_models.h

@@ -4,8 +4,8 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-#ifndef ISC_YANG_MODELS
-#define ISC_YANG_MODELS 1
+#ifndef ISC_YANG_MODELS_H
+#define ISC_YANG_MODELS_H 1
 
 #include <string>
 
@@ -29,9 +29,9 @@ static const std::string KEA_DHCP_DDNS = "kea-dhcp-ddns";
 
 /// This model is currently in prototype phase. It will be developed
 /// by ISC in the near future.
- static const std::string KEA_CTRL_AGENT = "kea-ctrl-agent";
+static const std::string KEA_CTRL_AGENT = "kea-ctrl-agent";
 
 }; // end of namespace isc::yang
 }; // end of namespace isc
 
-#endif // ISC_TRANSLATOR_POOL_H
+#endif // ISC_YANG_MODELS_H

src/lib/yang/yang_revisions.h

+// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef ISC_YANG_REVISIONS_H
+#define ISC_YANG_REVISIONS_H 1
+
+#include <map>
+#include <string>
+
+namespace isc {
+namespace yang {
+
+// Table of module name / revision.
+static const std::map<std::string, std::string> YANG_REVISIONS = {
+// Should be generated automatically.
+// cf src/share/yang/modules/README
+{ "ietf-dhcpv6-types", "2018-09-04" },
+{ "ietf-dhcpv6-options", "2018-09-04" },
+{ "ietf-dhcpv6-server", "2018-09-04" },
+{ "kea-types", "2018-11-20" },
+{ "kea-logging", "2018-11-20" },
+{ "kea-dhcp-types", "2018-11-20" },
+{ "kea-dhcp4-server", "2018-11-20" },
+{ "kea-dhcp6-server", "2018-11-20" },
+{ "kea-ctrl-agent", "2018-11-20" },
+{ "kea-dhcp-ddns", "2018-11-20" },
+{ "keatest-module", "2018-11-20" }
+};
+
+}; // end of namespace isc::yang
+}; // end of namespace isc
+
+#endif // ISC_YANG_REVISIONS_H

src/share/Makefile.am

-SUBDIRS = database
+SUBDIRS = database yang

src/share/yang/Makefile.am

+SUBDIRS = modules

src/share/yang/modules/Makefile.am

+SUBDIRS = .
+
+yangmodulesdir = ${datarootdir}/${PACKAGE_NAME}/yang/modules
+yangmodules_DATA  = ietf-dhcpv6-client@2018-09-04.yang
+yangmodules_DATA += ietf-interfaces@2014-05-08.yang
+yangmodules_DATA += ietf-dhcpv6-options@2018-09-04.yang
+yangmodules_DATA += ietf-dhcpv6-relay@2018-09-04.yang
+yangmodules_DATA += ietf-dhcpv6-server@2018-09-04.yang
+yangmodules_DATA += ietf-dhcpv6-types@2018-09-04.yang
+yangmodules_DATA += ietf-inet-types@2013-07-15.yang
+yangmodules_DATA += ietf-yang-types@2013-07-15.yang
+yangmodules_DATA += kea-ctrl-agent@2018-11-20.yang
+yangmodules_DATA += kea-dhcp-ddns@2018-11-20.yang
+yangmodules_DATA += kea-dhcp-types@2018-11-20.yang
+yangmodules_DATA += kea-dhcp4-server@2018-11-20.yang
+yangmodules_DATA += kea-dhcp6-server@2018-11-20.yang
+yangmodules_DATA += kea-logging@2018-11-20.yang
+yangmodules_DATA += kea-types@2018-11-20.yang
+
+EXTRA_DIST = ${yangmodules_DATA}

src/share/yang/modules/README

+Developer tools:
+
+Get revision from file content:
+
+yanglint -f yin xxx | grep '<revision date=' | head -1 | sed \
+ 's/.*<revision date="\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)".*/\1/'
+
+Get revision from file name:
+
+echo xxx | sed 's/.*@\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)\..*/\1/'
+
+Get hash:
+
+yanglint -f yin xxx | openssl dgst -sha256
+
+Check revisions:
+
+for m in *.yang
+do
+    rev1=`yanglint -f yin $m | grep '<revision date=' | head -1 | sed \
+ 's/.*<revision date="\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)".*/\1/'`
+    rev2=`echo $m | sed \
+ 's/.*@\([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]\)\..*/\1/'`
+
+    if test $rev1 != $rev2
+    then
+        echo revision mismatch on $m
+    fi
+done
+
+Check hashes:
+
+for m in *.yang
+do
+    hash1=`yanglint -f yin $m | openssl dgst -sha256`
+    h=hashes/`basename $m .yang`.hash
+    if test -f $h
+    then
+        hash2=`cat $h`
+        if test $hash1 != $hash2
+        then
+            echo hash mismatch on $m
+        fi
+    else
+        echo missing hash for $m
+    fi
+done
+
+Build module / revision table:
+
+for m in ietf-dhcpv6-types*.yang \
+         ietf-dhcpv6-options*.yang \
+         ietf-dhcpv6-server*.yang \
+         kea-types*.yang \
+         kea-logging*.yang \
+         kea-dhcp-types*.yang \
+         kea-dhcp4-server*.yang \
+         kea-dhcp6-server*.yang \
+         kea-ctrl-agent*.yang \
+         kea-dhcp-ddns*.yang \
+         keatest-module*.yang \
+         end-marker
+do
+    if test $m = "end-marker"
+    then
+        echo '{ "", "" }'
+    else
+        b=`echo $m | sed 's/\(.*\)@.*/\1/'`
+        r=`echo $m | sed 's/.*@\(.*\)\.yang/\1/'`
+        echo '{ "'$b'", "'$r'" },'
+    fi
+done
+
+

src/share/yang/modules/hashes/ietf-dhcpv6-client@2018-09-04.hash

+461ea3c5d65a212413437d52bd328a31591f7cf0b890aec920af7b793deaf749

src/share/yang/modules/hashes/ietf-dhcpv6-options@2018-09-04.hash

+deaf2a9c68b0cf88c6d2080036e50eded5c2356d062f339df44232f5cb240da6

src/share/yang/modules/hashes/ietf-dhcpv6-relay@2018-09-04.hash

+1a18c64a05cf4295864e2e9871b811b9b4fdf30568f0ce14de045a6af9d04450

src/share/yang/modules/hashes/ietf-dhcpv6-server@2018-09-04.hash

+474fcc1cf5a017028796d37240491d72d6a10511539554a709bf877de54c8947

src/share/yang/modules/hashes/ietf-dhcpv6-types@2018-09-04.hash

+2b46b5bb323d062a3432d726187f5f283a67669379af2ef496bb473167552718

src/share/yang/modules/hashes/ietf-inet-types@2013-07-15.hash

+9ba53b11ecce24a3b878a816555efd8f6559bbcbb5179c62c48af91598744b9e

src/share/yang/modules/hashes/ietf-interfaces@2014-05-08.hash

+89386b4a0f7d8b7b56c402b61f577667bf6372c51040cae77a7aa569f63a5b1a

src/share/yang/modules/hashes/ietf-yang-types@2013-07-15.hash

+0e69553a19dff434f563e5f10fac3dd9b87b5d49d73b92fa11453f9f8e6e2c0b

src/share/yang/modules/hashes/kea-ctrl-agent@2018-11-20.hash

+a3e7356b8de165c21ef9bcbc068327cd31e97b59117f93bb18b795e6fdc20dbd