[master] Finishing merge of trac5351 (user context / comment)

ChangeLog

+1343.	[func]		fdupont
+	User-context and comments are now supported in many new scopes:
+	global, shared-network, subnet, pool, host reservation, option,
+	option definition, client-class, control-socket, dhcp-ddns,
+	interfaces, loggers, and for DHCPv6 pd-pool and server-id.
+	(Trac #5351, git xxx)
+
+1342.	[bug]		fdupont
+	Fixed subnets and host reservations returned by config-get and
+	config-write.
+	(Trac #5452, git xxx)
+
 1341.	[func]		razvan, andrei
 	Significant update of Cassandra support. A lot of code for
 	Cassandr Lease Manager has been updated and partially rewritten.

configure.ac

@@ -1320,11 +1320,13 @@ AC_CONFIG_FILES([Makefile
                  src/share/database/scripts/mysql/upgrade_4.0_to_4.1.sh
                  src/share/database/scripts/mysql/upgrade_4.1_to_5.0.sh
                  src/share/database/scripts/mysql/upgrade_5.0_to_5.1.sh
+                 src/share/database/scripts/mysql/upgrade_5.1_to_6.0.sh
                  src/share/database/scripts/pgsql/Makefile
                  src/share/database/scripts/pgsql/upgrade_1.0_to_2.0.sh
                  src/share/database/scripts/pgsql/upgrade_2.0_to_3.0.sh
                  src/share/database/scripts/pgsql/upgrade_3.0_to_3.1.sh
                  src/share/database/scripts/pgsql/upgrade_3.1_to_3.2.sh
+                 src/share/database/scripts/pgsql/upgrade_3.2_to_4.0.sh
                  tools/Makefile
                  tools/path_replacer.sh
 ])

doc/Makefile.am

@@ -18,6 +18,7 @@ nobase_dist_doc_DATA += examples/kea4/advanced.json
 nobase_dist_doc_DATA += examples/kea4/backends.json
 nobase_dist_doc_DATA += examples/kea4/cassandra.json
 nobase_dist_doc_DATA += examples/kea4/classify.json
+nobase_dist_doc_DATA += examples/kea4/comments.json
 nobase_dist_doc_DATA += examples/kea4/dhcpv4-over-dhcpv6.json
 nobase_dist_doc_DATA += examples/kea4/hooks.json
 nobase_dist_doc_DATA += examples/kea4/leases-expiration.json
@@ -33,6 +34,7 @@ nobase_dist_doc_DATA += examples/kea6/advanced.json
 nobase_dist_doc_DATA += examples/kea6/backends.json
 nobase_dist_doc_DATA += examples/kea6/cassandra.json
 nobase_dist_doc_DATA += examples/kea6/classify.json
+nobase_dist_doc_DATA += examples/kea6/comments.json
 nobase_dist_doc_DATA += examples/kea6/dhcpv4-over-dhcpv6.json
 nobase_dist_doc_DATA += examples/kea6/duid.json
 nobase_dist_doc_DATA += examples/kea6/hooks.json

doc/examples/kea4/advanced.json

@@ -101,12 +101,15 @@
     "match-client-id": true,
 
     // The following list defines subnets. Each subnet consists of at
-    // least subnet and pool entries. One extra feature that requires some
-    // explanation is user-context. This is a structure that you can define
-    // in subnets and pools. It is parsed by Kea, but not used directly.
-    // It is intended to keep anything you may want to put there - comments,
-    // extra designations, floor or department names etc. These structures
-    // will be made available to Kea hooks.
+    // least subnet and pool entries. One extra feature that requires
+    // some explanation is user-context. This is a structure that you can
+    // define in subnets, pools and others. It is parsed by Kea, but not
+    // used directly.  It is intended to keep anything you may want to
+    // put there - comments, extra designations, floor or department
+    // names etc. These structures will be made available to Kea hooks.
+    // A comment entry is translated into a user-context with a
+    // "comment" property so you can include comments inside the
+    // configuration itself.
     "subnet4": [
         {
             "pools": [ {
@@ -117,6 +120,8 @@
             "user-context": {
                 "comment": "Our first subnet!"
             }
+            // Equivalent using smart parser
+            // "comment": "Our first subnet!"
         },
         {
             // This particular subnet has match-client-id value changed.

doc/examples/kea4/comments.json

+// This is an example configuration file for the DHCPv4 server in Kea.
+// It uses embedded (i.e., which will be included in configuration objects
+// and not stripped by at lexical analysis) comments.
+
+{ "Dhcp4":
+
+{
+   // Global scope
+   "comment": "A DHCPv4 server",
+
+   // In interface config
+   "interfaces-config": {
+       "comment": "Use wildcard",
+       "interfaces": [ "*" ] },
+
+   // In option definitions
+   "option-def": [ {
+       "comment": "An option definition",
+       "name": "foo",
+       "code": 100,
+       "type": "ipv4-address",
+       "space": "isc"
+    } ],
+
+   // In option data
+   "option-data": [ {
+       "comment": "Set option value",
+       "name": "dhcp-message",
+       "data": "ABCDEF0105",
+       "csv-format": false
+    } ],
+
+   // In client classes
+   "client-classes": [
+       {
+          "comment": "match all",
+          "name": "all",
+          "test": "'' == ''"
+       },
+       // Of course comments are optional
+       {
+          "name": "none"
+       },
+       // A comment and a user-context can be specified
+       {
+          "comment": "a comment",
+          "name": "both",
+          "user-context": {
+              "version": 1
+          }
+       }
+   ],
+
+   // In control socket (more for the agent)
+   "control-socket": {
+       "socket-type": "unix",
+       "socket-name": "/tmp/kea4-ctrl-socket",
+       "user-context": { "comment": "Indirect comment" }
+   },
+
+   // In shared networks
+   "shared-networks": [ {
+       "comment": "A shared network",
+       "name": "foo",
+
+       // In subnets
+       "subnet4": [
+         { 
+           "comment": "A subnet",
+           "subnet": "192.0.1.0/24",
+           "id": 100,
+
+           // In pools
+           "pools": [
+             {
+                "comment": "A pool",
+                "pool": "192.0.1.1-192.0.1.10"
+             }
+           ],
+
+           // In host reservations
+           "reservations": [
+             {
+                "comment": "A host reservation",
+                "hw-address": "AA:BB:CC:DD:EE:FF",
+                "hostname": "foo.example.com",
+
+                // Again in an option data
+                "option-data": [ {
+                    "comment": "An option in a reservation",
+                    "name": "domain-name",
+                    "data": "example.com"
+                } ]
+             }
+           ]
+         }
+       ]
+    } ],
+
+   // In dhcp ddns
+   "dhcp-ddns": {
+       "comment": "No dynamic DNS",
+       "enable-updates": false
+   }
+},
+
+"Logging": {
+   // In loggers
+   "loggers": [ {
+       "comment": "A logger",
+       "name": "kea-dhcp4"
+   } ]
+}
+
+}
+

doc/examples/kea6/advanced.json

@@ -86,6 +86,9 @@
     // structures. You can put anything you want in the user-context
     // as long as it is a valid JSON and it starts with a map (i.e.
     // is enclosed by curly brackets).
+    // A comment entry is translated into a user-context with a
+    // "comment" property so you can include comments inside the
+    // configuration itself.
     "subnet6": [
     {
         "pools": [
@@ -102,6 +105,8 @@
 
         // Here's the user-context for the whole subnet.
         "user-context": { "comment": "Floor one, west wing" },
+        // Equivalent using smart parser
+        // "comment": "Floor one, west wing",
 
         // This defines PD (prefix delegation) pools. In this case
         // we have only one pool. That consists of /64 prefixes

doc/examples/kea6/comments.json

+// This is an example configuration file for the DHCPv6 server in Kea.
+// It uses embedded (i.e., which will be included in configuration objects
+// and not stripped by at lexical analysis) comments.
+
+{ "Dhcp6":
+
+{
+   // Global scope
+   "comment": "A DHCPv6 server",
+
+   // In interface config
+   "interfaces-config": {
+       "comment": "Use wildcard",
+       "interfaces": [ "*" ] },
+
+   // In option definitions
+   "option-def": [ {
+       "comment": "An option definition",
+       "name": "foo",
+       "code": 100,
+       "type": "ipv6-address",
+       "space": "isc"
+    } ],
+
+   // In option data
+   "option-data": [ {
+       "comment": "Set option value",
+       "name": "subscriber-id",
+       "data": "ABCDEF0105",
+       "csv-format": false
+    } ],
+
+   // In client classes
+   "client-classes": [
+       {
+          "comment": "match all",
+          "name": "all",
+          "test": "'' == ''"
+       },
+       // Of course comments are optional
+       {
+          "name": "none"
+       },
+       // A comment and a user-context can be specified
+       {
+          "comment": "a comment",
+          "name": "both",
+          "user-context": {
+              "version": 1
+          }
+       }
+   ],
+
+   // In control socket (more for the agent)
+   "control-socket": {
+       "socket-type": "unix",
+       "socket-name": "/tmp/kea6-ctrl-socket",
+       "user-context": { "comment": "Indirect comment" }
+   },
+
+   // In shared networks
+   "shared-networks": [ {
+       "comment": "A shared network",
+       "name": "foo",
+
+       // In subnets
+       "subnet6": [
+         { 
+           "comment": "A subnet",
+           "subnet": "2001:db1::/64",
+           "id": 100,
+
+           // In pools
+           "pools": [
+             {
+                "comment": "A pool",
+                "pool": "2001:db1::/64"
+             }
+           ],
+
+           // In prefix pools
+           "pd-pools": [
+             {
+                "comment": "A prefix pool",
+                "prefix": "2001:db2::",
+                "prefix-len": 48,
+                "delegated-len": 64
+             }
+           ],
+
+           // In host reservations
+           "reservations": [
+             {
+                "comment": "A host reservation",
+                "hw-address": "AA:BB:CC:DD:EE:FF",
+                "hostname": "foo.example.com",
+
+                // Again in an option data
+                "option-data": [ {
+                    "comment": "An option in a reservation",
+                    "name": "domain-search",
+                    "data": "example.com"
+                } ]
+             }
+           ]
+         }
+       ]
+    } ],
+
+   // In dhcp ddns
+   "dhcp-ddns": {
+       "comment": "No dynamic DNS",
+       "enable-updates": false
+   }
+},
+
+"Logging": {
+   // In loggers
+   "loggers": [ {
+       "comment": "A logger",
+       "name": "kea-dhcp6"
+   } ]
+}
+
+}
+

doc/guide/dhcp4-srv.xml

@@ -4646,8 +4646,10 @@ autogenerated IDs are not stable across configuration changes.</para>
         </para>
 
         <para>
-          User contexts can be specified on either subnet or pool level. One
-          other useful usage is the ability to store comments or descriptions.
+          User contexts can be specified on either global scope,
+          shared network, subnet, pool, client class, option data or
+          definition level, and host reservation. One other useful
+          usage is the ability to store comments or descriptions.
         </para>
 
         <para>
@@ -4693,6 +4695,9 @@ autogenerated IDs are not stable across configuration changes.</para>
           It should be noted that Kea will not use that information, but will
           simply store and make it available to hook libraries. It is up to the
           hook library to extract that information and make use of it.
+          The parser translates a "comment" entry into a user-context
+          with the entry, this allows to attach a comment inside the
+          configuration itself.
         </para>
         <para>
           For more background information, see <xref linkend="user-context"/>.

doc/guide/dhcp6-srv.xml

@@ -4608,8 +4608,10 @@ autogenerated IDs are not stable across configuration changes.
         </para>
 
         <para>
-          User contexts can be specified on either subnet or pool level. One
-          other useful usage is the ability to store comments or descriptions.
+          User contexts can be specified on either global scope,
+          shared network, subnet, pool, client class, option data or
+          definition level, and host reservation. One other useful
+          usage is the ability to store comments or descriptions.
         </para>
 
         <para>
@@ -4661,6 +4663,9 @@ autogenerated IDs are not stable across configuration changes.
           it just stores it, making it available to the hook
           libraries. It is up to each hook library to extract the information
           and make use of it.
+          The parser translates a "comment" entry into a user-context
+          with the entry, this allows to attach a comment inside the
+          configuration itself.
         </para>
         <para>
           For more background information, see <xref linkend="user-context"/>.

doc/guide/hooks.xml

@@ -80,7 +80,9 @@
         hooks-libraries was a list of strings, each string being the name of
         a library.  The change was made in Kea 1.0 to facilitate the
         specification of library-specific parameters, a capability
-        available in Kea 1.1.0 onwards.
+        available in Kea 1.1.0 onwards. Libraries should allow a parameter
+        entry where to put comments as it is done for many configuration
+        scopes with comment and user context.
       </para></note>
 
         <note>
@@ -2634,10 +2636,19 @@ both the command and the response.
         operations for example.
       </para>
       <para>
-        As of Kea 1.3, the structures that allow user contexts are pools of all
-        types (addresses and prefixes) and subnets. These are supported in both
-        DHCPv4 and DHCPv6. It is expected that other structures will be extended
-        in the future to provide the user context capability.
+        If user context is supported in a given context, the parser translates
+        "comment" entries into user context with a "comment" entry. The pretty
+        print of a configuration does the opposite operation and puts "comment"
+        entries at the beginning of maps as it seems to be the common usage.
+      </para>
+      <para>
+        As of Kea 1.3, the structures that allow user contexts are
+        pools of all types (addresses and prefixes) and subnets. Kea
+        1.4 extended user context support to the global scope, interfaces config,
+        shared networks, subnets, client classes, option datas and definitions,
+        host reservations, control socket, dhcp ddns, loggers and server id.
+        These are supported in both DHCPv4 and DHCPv6 at the exception
+        of server id which is DHCPv6 only.
       </para>
     </section>
 

src/bin/admin/tests/mysql_tests.sh.in

@@ -193,7 +193,7 @@ mysql_upgrade_test() {
 
     assert_str_eq "1.0" ${version} "Expected kea-admin to return %s, returned value was %s"
 
-    # Ok, we have a 1.0 database. Let's upgrade it to 5.1
+    # Ok, we have a 1.0 database. Let's upgrade it to 6.0
     ${keaadmin} lease-upgrade mysql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir
     ERRCODE=$?
 
@@ -295,6 +295,7 @@ EOF
     count=`echo $text | grep -ic "order by l\.address"`
     assert_eq 1 $count "lease6DumpData doesn't have order by clause. (returned count %d, expected %d)"
 
+    #table: host_identifier_type (upgrade 4.1 -> 5.0)
     # verify that host_identifier_type table exists.
     qry="select count(*) from host_identifier_type";
     count=`mysql_execute "${qry}"`
@@ -310,6 +311,7 @@ EOF
     assert_eq 0 $ERRCODE "show create table hosts failed. (expected status code %d, returned %d)"
     assert_eq 1 "$count" "show create table hosts did not return correct number of fk_host_identifier_type instances. (expected %d, returned %d)"
 
+    #table: dhcp_option_scope (upgrade 4.1 -> 5.0)
     # verify that dhcp_option_scope table exists.
     qry="select count(*) from dhcp_option_scope";
     count=`mysql_execute "${qry}"`
@@ -317,18 +319,21 @@ EOF
     assert_eq 0 $ERRCODE "select from dhcp_option_scope failed. (expected status code %d, returned %d)"
     assert_eq 4 "$count" "dhcp_option_scope does not contain correct number of entries. (expected %d, returned %d)"
 
+    #table: scope_id columns to dhcp4_options (upgrade 4.1 -> 5.0) 
     # verify that dhcp4_options table includes scope_id
     qry="select scope_id from dhcp4_options";
     count=`mysql_execute "${qry}"`
     ERRCODE=$?
     assert_eq 0 $ERRCODE "select scope_id from dhcp4_options failed. (expected status code %d, returned %d)"
 
+    #table: scope_id columns to dhcp6_options (upgrade 4.1 -> 5.0) 
     # verify that dhcp6_options table includes scope_id
     qry="select scope_id from dhcp6_options";
     count=`mysql_execute "${qry}"`
     ERRCODE=$?
     assert_eq 0 $ERRCODE "select scope_id from dhcp6_options failed. (expected status code %d, returned %d)"
 
+    #table: DHCPv4 fixed field colums (upgrade 4.1 -> 5.0)
     # verify that hosts table has columns holding values for DHCPv4 fixed fields
     qry="select dhcp4_next_server, dhcp4_server_hostname, dhcp4_boot_file_name from hosts";
     count=`mysql_execute "${qry}"`
@@ -351,9 +356,35 @@ EOF
     count=`echo $text | grep -ic unsigned`
     assert_eq 1 $count "dhcp6_subnet_id is not of unsigned type. (expected count %d, returned %d)"
 
-    # Verify upgraded schema reports version 5.1
+    #host_identifier_type should have rows for types 3 and 4 (upgrade 5.0 -> 5.1)
+    qry="select count(*) from host_identifier_type";
+    count=`mysql_execute "${qry}"`
+    ERRCODE=$?
+    assert_eq 0 $ERRCODE "select from host_identifier_type failed. (expected status code %d, returned %d)"
+    assert_eq 5 "$count" "host_identifier_type does not contain correct number of entries. (expected count %d, returned %d)"
+    
+    #table: user_context columns to hosts, dhcp4_options and dhcp6_options (upgrade 5.1 -> 6.0)
+    # verify that hosts table includes user_context
+    qry="select user_context from hosts";
+    count=`mysql_execute "${qry}"`
+    ERRCODE=$?
+    assert_eq 0 $ERRCODE "select user_context from hosts failed. (expected status code %d, returned %d)"
+
+    # verify that dhcp4_options table includes user_context
+    qry="select user_context from dhcp4_options";
+    count=`mysql_execute "${qry}"`
+    ERRCODE=$?
+    assert_eq 0 $ERRCODE "select user_context from dhcp4_options failed. (expected status code %d, returned %d)"
+
+    # verify that dhcp6_options table includes user_context
+    qry="select user_context from dhcp6_options";
+    count=`mysql_execute "${qry}"`
+    ERRCODE=$?
+    assert_eq 0 $ERRCODE "select user_context from dhcp6_options failed. (expected status code %d, returned %d)"
+
+    # Verify upgraded schema reports version 6.0
     version=$(${keaadmin} lease-version mysql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir)
-    assert_str_eq "5.1" ${version} "Expected kea-admin to return %s, returned value was %s"
+    assert_str_eq "6.0" ${version} "Expected kea-admin to return %s, returned value was %s"
 
     # Let's wipe the whole database
     mysql_wipe

src/bin/admin/tests/pgsql_tests.sh.in

@@ -89,7 +89,7 @@ pgsql_lease_version_test() {
 
     # Verify that kea-admin lease-version returns the correct version
     version=$(${keaadmin} lease-version pgsql -u $db_user -p $db_password -n $db_name)
-    assert_str_eq "3.2" ${version} "Expected kea-admin to return %s, returned value was %s"
+    assert_str_eq "4.0" ${version} "Expected kea-admin to return %s, returned value was %s"
 
     # Let's wipe the whole database
     pgsql_wipe
@@ -199,10 +199,10 @@ pgsql_upgrade_2_0_to_3_0() {
     assert_eq 1 "$output" "lease_hwaddr_source does not contain entry for HWADDR_SOURCE_UNKNOWN. (record count %d, expected %d)"
 }
 
-pgsql_upgrade_3_0_to_3_1() {
-    # Verify upgraded schema reports version 3.1.
+pgsql_upgrade_3_0_to_4_0() {
+    # Verify upgraded schema reports version 4.0.
     version=$(${keaadmin} lease-version pgsql -u $db_user -p $db_password -n $db_name -d $db_scripts_dir)
-    assert_str_eq "3.1" ${version} "Expected kea-admin to return %s, returned value was %s"
+    assert_str_eq "4.0" ${version} "Expected kea-admin to return %s, returned value was %s"
 }
 
 pgsql_upgrade_test() {
@@ -224,8 +224,8 @@ pgsql_upgrade_test() {
     # Check 2.0 to 3.0 upgrade
     pgsql_upgrade_2_0_to_3_0
 
-    # Check 3.0 to 3.1 upgrade
-    pgsql_upgrade_3_0_to_3_1
+    # Check 3.0 to 4.0 upgrade
+    pgsql_upgrade_3_0_to_4_0
 
     # Let's wipe the whole database
     pgsql_wipe

src/bin/dhcp4/dhcp4_lexer.ll

@@ -135,6 +135,8 @@ ControlCharacterFill            [^"\\]|\\{JSONEscapeSequence}
             return isc::dhcp::Dhcp4Parser::make_SUB_HOOKS_LIBRARY(driver.loc_);
         case Parser4Context::PARSER_DHCP_DDNS:
             return isc::dhcp::Dhcp4Parser::make_SUB_DHCP_DDNS(driver.loc_);
+        case Parser4Context::PARSER_LOGGING:
+            return isc::dhcp::Dhcp4Parser::make_SUB_LOGGING(driver.loc_);
         }
     }
 %}
@@ -578,14 +580,44 @@ ControlCharacterFill            [^"\\]|\\{JSONEscapeSequence}
 
 \"user-context\" {
     switch(driver.ctx_) {
+    case isc::dhcp::Parser4Context::DHCP4:
+    case isc::dhcp::Parser4Context::INTERFACES_CONFIG:
     case isc::dhcp::Parser4Context::SUBNET4:
     case isc::dhcp::Parser4Context::POOLS:
+    case isc::dhcp::Parser4Context::SHARED_NETWORK:
+    case isc::dhcp::Parser4Context::OPTION_DEF:
+    case isc::dhcp::Parser4Context::OPTION_DATA:
+    case isc::dhcp::Parser4Context::RESERVATIONS:
+    case isc::dhcp::Parser4Context::CLIENT_CLASSES:
+    case isc::dhcp::Parser4Context::CONTROL_SOCKET:
+    case isc::dhcp::Parser4Context::LOGGERS:
+    case isc::dhcp::Parser4Context::DHCP_DDNS:
         return isc::dhcp::Dhcp4Parser::make_USER_CONTEXT(driver.loc_);
     default:
         return isc::dhcp::Dhcp4Parser::make_STRING("user-context", driver.loc_);
     }
 }
 
+\"comment\" {
+    switch(driver.ctx_) {
+    case isc::dhcp::Parser4Context::DHCP4:
+    case isc::dhcp::Parser4Context::INTERFACES_CONFIG:
+    case isc::dhcp::Parser4Context::SUBNET4:
+    case isc::dhcp::Parser4Context::POOLS:
+    case isc::dhcp::Parser4Context::SHARED_NETWORK:
+    case isc::dhcp::Parser4Context::OPTION_DEF:
+    case isc::dhcp::Parser4Context::OPTION_DATA:
+    case isc::dhcp::Parser4Context::RESERVATIONS:
+    case isc::dhcp::Parser4Context::CLIENT_CLASSES:
+    case isc::dhcp::Parser4Context::CONTROL_SOCKET:
+    case isc::dhcp::Parser4Context::LOGGERS:
+    case isc::dhcp::Parser4Context::DHCP_DDNS:
+        return isc::dhcp::Dhcp4Parser::make_COMMENT(driver.loc_);
+    default:
+        return isc::dhcp::Dhcp4Parser::make_STRING("comment", driver.loc_);
+    }
+}
+
 \"subnet\" {
     switch(driver.ctx_) {
     case isc::dhcp::Parser4Context::SUBNET4:

src/bin/dhcp4/dhcp4_parser.h

@@ -410,91 +410,93 @@ namespace isc { namespace dhcp {
         TOKEN_POOLS = 317,
         TOKEN_POOL = 318,
         TOKEN_USER_CONTEXT = 319,
-        TOKEN_SUBNET = 320,
-        TOKEN_INTERFACE = 321,
-        TOKEN_INTERFACE_ID = 322,
-        TOKEN_ID = 323,
-        TOKEN_RAPID_COMMIT = 324,
-        TOKEN_RESERVATION_MODE = 325,
-        TOKEN_DISABLED = 326,