merge back branches/jelte-configuration (branched at rev. 745, up to HEAD)

git-svn-id: svn://bind10.isc.org/svn/bind10/trunk@882 e5f2f494-b856-4b98-b285-d166d9295462

src/bin/auth/auth.spec

 {
-  "data_specification": {
-    "module_name": "Auth"
+  "module_spec": {
+    "module_name": "Auth",
+    "config_data": [
+      { "item_name": "default_name",
+        "item_type": "string",
+        "item_optional": False,
+        "item_default": "Hello, world!"
+      },
+      { "item_name": "zone_list",
+        "item_type": "list",
+        "item_optional": False,
+        "item_default": [],
+        "list_item_spec":
+          { "item_name": "zone_name",
+            "item_type": "string",
+            "item_optional": True,
+            "item_default": ""
+          }
+      }
+    ],
+    "commands": [
+      {
+        "command_name": "print_message",
+        "command_description": "Print the given message to stdout",
+        "command_args": [ {
+          "item_name": "message",
+          "item_type": "string",
+          "item_optional": False,
+          "item_default": ""
+        } ]
+      },
+      {
+        "command_name": "shutdown",
+        "command_description": "Shut down BIND 10",
+        "command_args": []
+      }
+    ]
   }
 }
 

src/bin/auth/auth_srv.cc

@@ -32,6 +32,7 @@
 #include <dns/rrset.h>
 #include <dns/rrttl.h>
 #include <dns/message.h>
+#include <config/ccsession.h>
 
 #include <cc/data.h>
 
@@ -46,6 +47,7 @@ using namespace std;
 using namespace isc::dns;
 using namespace isc::dns::rdata;
 using namespace isc::data;
+using namespace isc::config;
 
 AuthSrv::AuthSrv(int port) {
     int s = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
@@ -120,5 +122,7 @@ AuthSrv::updateConfig(isc::data::ElementPtr config) {
         // todo: what to do with port change. restart automatically?
         // ignore atm
     //}
-    return isc::data::Element::createFromString("{ \"result\": [0] }");
+    std::cout << "[XX] auth: new config " << config << std::endl;
+    
+    return isc::config::createAnswer(0);
 }

src/bin/auth/main.cc

@@ -64,18 +64,15 @@ my_config_handler(isc::data::ElementPtr config)
 
 isc::data::ElementPtr
 my_command_handler(isc::data::ElementPtr command) {
-    isc::data::ElementPtr answer = isc::data::Element::createFromString("{ \"result\": [0] }");
+    isc::data::ElementPtr answer = isc::config::createAnswer(0);
 
     cout << "[XX] Handle command: " << endl << command->str() << endl;
     if (command->get(0)->stringValue() == "print_message") 
     {
         cout << command->get(1)->get("message") << endl;
         /* let's add that message to our answer as well */
-        cout << "[XX] answer was: " << answer->str() << endl;
         answer->get("result")->add(command->get(1));
-        cout << "[XX] answer now: " << answer->str() << endl;
     }
-
     return answer;
 }
 
@@ -106,9 +103,9 @@ main(int argc, char* argv[]) {
         } else {
             specfile = std::string(AUTH_SPECFILE_LOCATION);
         }
-        CommandSession cs = CommandSession(specfile,
-                                           my_config_handler,
-                                           my_command_handler);
+        isc::config::ModuleCCSession cs = isc::config::ModuleCCSession(specfile,
+                                                                       my_config_handler,
+                                                                       my_command_handler);
     
         // main server loop
         fd_set fds;

src/bin/bind10/bind10.py.in

@@ -106,6 +106,7 @@ class BoB:
         self.verbose = verbose
         self.c_channel_port = c_channel_port
         self.cc_session = None
+        self.ccs = None
         self.processes = {}
         self.dead_processes = {}
         self.runnable = False
@@ -114,6 +115,8 @@ class BoB:
         if self.verbose:
             print("[XX] handling new config:")
             print(new_config)
+        answer = isc.config.ccsession.create_answer(0)
+        return answer
         # TODO
 
     def command_handler(self, command):
@@ -121,21 +124,27 @@ class BoB:
         if self.verbose:
             print("[XX] Boss got command:")
             print(command)
-        answer = None
+        answer = [ 1, "Command not implemented" ]
         if type(command) != list or len(command) == 0:
-            answer = { "result": [ 1, "bad command" ] }
+            answer = isc.config.ccsession.create_answer(1, "bad command")
         else:
             cmd = command[0]
             if cmd == "shutdown":
                 print("[XX] got shutdown command")
                 self.runnable = False
-                answer = { "result": [ 0 ] }
+                answer = isc.config.ccsession.create_answer(0)
             elif cmd == "print_message":
                 if len(command) > 1 and type(command[1]) == dict and "message" in command[1]:
                     print(command[1]["message"])
-                answer = { "result": [ 0 ] }
+                answer = isc.config.ccsession.create_answer(0)
+            elif cmd == "print_settings":
+                print("Full Config:")
+                full_config = self.ccs.get_full_config()
+                for item in full_config:
+                    print(item + ": " + str(full_config[item]))
+                answer = isc.config.ccsession.create_answer(0)
             else:
-                answer = { "result": [ 1, "Unknown command" ] }
+                answer = isc.config.ccsession.create_answer(1, "Unknown command")
         return answer
     
     def startup(self):
@@ -190,7 +199,8 @@ class BoB:
         time.sleep(1)
         if self.verbose:
             print("[XX] starting ccsession")
-        self.ccs = isc.config.CCSession(SPECFILE_LOCATION, self.config_handler, self.command_handler)
+        self.ccs = isc.config.ModuleCCSession(SPECFILE_LOCATION, self.config_handler, self.command_handler)
+        self.ccs.start()
         if self.verbose:
             print("[XX] ccsession started")
 
@@ -478,7 +488,7 @@ def main():
 
         for fd in rlist + xlist:
             if fd == ccs_fd:
-                boss_of_bind.ccs.checkCommand()
+                boss_of_bind.ccs.check_command()
             elif fd == wakeup_fd:
                 os.read(wakeup_fd, 32)
 

src/bin/bind10/bob.spec

 {
-  "data_specification": {
+  "module_spec": {
     "module_name": "Boss",
     "config_data": [
       {
@@ -7,6 +7,12 @@
         "item_type": "string",
         "item_optional": False,
         "item_default": "Hi, shane!"
+      },
+      {
+        "item_name": "some_int",
+        "item_type": "integer",
+        "item_optional": False,
+        "item_default": 1
       }
     ],
     "commands": [
@@ -20,6 +26,11 @@
           "item_default": ""
         } ]
       },
+      {
+        "command_name": "print_settings",
+        "command_description": "Print some_string and some_int to stdout",
+        "command_args": []
+      },
       {
         "command_name": "shutdown",
         "command_description": "Shut down BIND 10",

src/bin/bindctl/bindcmd.py

@@ -31,6 +31,7 @@ import os, time, random, re
 import getpass
 from hashlib import sha1
 import csv
+import ast
 
 try:
     from collections import OrderedDict
@@ -85,7 +86,7 @@ class BindCmdInterpreter(Cmd):
                 return False
 
             # Get all module information from cmd-ctrld
-            self.config_data = isc.cc.data.UIConfigData(self)
+            self.config_data = isc.config.UIModuleCCSession(self)
             self.update_commands()
             self.cmdloop()
         except KeyboardInterrupt:
@@ -150,7 +151,8 @@ class BindCmdInterpreter(Cmd):
         if (len(cmd_spec) == 0):
             print('can\'t get any command specification')
         for module_name in cmd_spec.keys():
-            self.prepare_module_commands(module_name, cmd_spec[module_name])
+            if cmd_spec[module_name]:
+                self.prepare_module_commands(module_name, cmd_spec[module_name])
 
     def send_GET(self, url, body = None):
         headers = {"cookie" : self.session_id}
@@ -315,7 +317,7 @@ class BindCmdInterpreter(Cmd):
                     if cmd.module == "config":
                         # grm text has been stripped of slashes...
                         my_text = self.location + "/" + cur_line.rpartition(" ")[2]
-                        list = self.config_data.config.get_item_list(my_text.rpartition("/")[0])
+                        list = self.config_data.get_config_item_list(my_text.rpartition("/")[0])
                         hints.extend([val for val in list if val.startswith(text)])
             except CmdModuleNameFormatError:
                 if not text:
@@ -440,17 +442,28 @@ class BindCmdInterpreter(Cmd):
                         line += "(modified)"
                     print(line)
             elif cmd.command == "add":
-                self.config_data.add(identifier, cmd.params['value'])
+                self.config_data.add_value(identifier, cmd.params['value'])
             elif cmd.command == "remove":
-                self.config_data.remove(identifier, cmd.params['value'])
+                self.config_data.remove_value(identifier, cmd.params['value'])
             elif cmd.command == "set":
-                self.config_data.set(identifier, cmd.params['value'])
+                if 'identifier' not in cmd.params:
+                    print("Error: missing identifier or value")
+                else:
+                    parsed_value = None
+                    try:
+                        parsed_value = ast.literal_eval(cmd.params['value'])
+                    except Exception as exc:
+                        # ok could be an unquoted string, interpret as such
+                        parsed_value = cmd.params['value']
+                    self.config_data.set_value(identifier, parsed_value)
             elif cmd.command == "unset":
                 self.config_data.unset(identifier)
             elif cmd.command == "revert":
                 self.config_data.revert()
             elif cmd.command == "commit":
-                self.config_data.commit(self)
+                self.config_data.commit()
+            elif cmd.command == "diff":
+                print(self.config_data.get_local_changes());
             elif cmd.command == "go":
                 self.go(identifier)
         except isc.cc.data.DataTypeError as dte:

src/bin/bindctl/bindctl.in

@@ -5,7 +5,7 @@ export PYTHON_EXEC
 
 BINDCTL_PATH=@abs_top_srcdir@/src/bin/bindctl
 
-PYTHONPATH=@abs_top_srcdir@/src/lib/cc/python
+PYTHONPATH=@abs_top_builddir@/pyshared
 export PYTHONPATH
 
 cd ${BINDCTL_PATH}

src/bin/bindctl/bindctl.py

@@ -53,6 +53,9 @@ def prepare_config_commands(tool):
     cmd.add_param(param)
     module.add_command(cmd)
 
+    cmd = CommandInfo(name = "diff", desc = "Show all local changes", need_inst_param = False)
+    module.add_command(cmd)
+
     cmd = CommandInfo(name = "revert", desc = "Revert all local changes", need_inst_param = False)
     module.add_command(cmd)
 
@@ -67,12 +70,17 @@ def prepare_config_commands(tool):
     tool.add_module_info(module)
 
 if __name__ == '__main__':
-    try:
-        tool = BindCmdInterpreter("localhost:8080")
-        prepare_config_commands(tool)
-        tool.run()
-    except Exception as e:
-        print(e)
-        print("Failed to connect with b10-cmdctl module, is it running?")
+    tool = BindCmdInterpreter("localhost:8080")
+    prepare_config_commands(tool)
+    tool.run()
+# TODO: put below back, was removed to see errors
+#if __name__ == '__main__':
+    #try:
+        #tool = BindCmdInterpreter("localhost:8080")
+        #prepare_config_commands(tool)
+        #tool.run()
+    #except Exception as e:
+        #print(e)
+        #print("Failed to connect with b10-cmdctl module, is it running?")
 
 

src/bin/cmdctl/b10-cmdctl.py.in

@@ -169,8 +169,8 @@ class SecureHTTPRequestHandler(http.server.BaseHTTPRequestHandler):
             param = json.loads(post_str)
             # TODO, need return some proper return code. 
             # currently always OK.
-            reply = self.server.send_command_to_module(mod, cmd, param)
-            print('b10-cmdctl finish send message \'%s\' to module %s' % (cmd, mod))            
+        reply = self.server.send_command_to_module(mod, cmd, param)
+        print('b10-cmdctl finish send message \'%s\' to module %s' % (cmd, mod))            
 
         return rcode, reply
             
@@ -189,7 +189,7 @@ class CommandControl():
         self.config_data = self.get_config_data()
 
     def get_cmd_specification(self): 
-        return self.send_command('ConfigManager', 'get_commands')
+        return self.send_command('ConfigManager', 'get_commands_spec')
 
     def get_config_data(self):
         return self.send_command('ConfigManager', 'get_config')
@@ -199,7 +199,7 @@ class CommandControl():
             self.config_data = self.get_config_data()
 
     def get_data_specification(self):
-        return self.send_command('ConfigManager', 'get_data_spec')
+        return self.send_command('ConfigManager', 'get_module_spec')
 
     def handle_recv_msg(self):
         # Handle received message, if 'shutdown' is received, return False

src/lib/cc/cpp/data.cc

@@ -829,13 +829,19 @@ ListElement::toWire(std::stringstream& ss, int omit_length)
         (*it)->toWire(ss2, 0);
     }
 
+
     if (omit_length) {
-        ss << ss2.rdbuf();
+        stringbuf *ss2_buf = ss2.rdbuf();
+        if (ss2_buf->in_avail() > 0) {
+            ss << ss2_buf;
+        }
     } else {
         stringbuf *ss2_buf = ss2.rdbuf();
         ss2_buf->pubseekpos(0);
         ss << encode_length(ss2_buf->in_avail(), ITEM_LIST);
-        ss << ss2_buf;
+        if (ss2_buf->in_avail() > 0) {
+            ss << ss2_buf;
+        }
     }
 }
 
@@ -873,13 +879,17 @@ MapElement::toWire(std::stringstream& ss, int omit_length)
     // add length if needed
     //
     if (omit_length) {
-        ss << ss2.rdbuf();
+        stringbuf *ss2_buf = ss2.rdbuf();
+        if (ss2_buf->in_avail() > 0) {
+            ss << ss2_buf;
+        }
     } else {
-        
         stringbuf *ss2_buf = ss2.rdbuf();
         ss2_buf->pubseekpos(0);
         ss << encode_length(ss2_buf->in_avail(), ITEM_HASH);
-        ss << ss2_buf;
+        if (ss2_buf->in_avail() > 0) {
+            ss << ss2_buf;
+        }
     }
 }
 

src/lib/cc/cpp/data.h

@@ -35,7 +35,6 @@ typedef boost::shared_ptr<Element> ElementPtr;
 /// is called for an Element that has a wrong type (e.g. int_value on a
 /// ListElement)
 ///
-// todo: include types and called function in the exception
 class TypeError : public isc::Exception {
 public:
     TypeError(const char* file, size_t line, const char* what) :

src/lib/cc/cpp/data_unittests.cc

@@ -266,5 +266,13 @@ TEST(Element, to_and_from_wire) {
 
     //EXPECT_EQ("\047\0031.2", Element::create(1.2)->toWire(0));
     EXPECT_EQ("\046\0011", Element::createFromString("[ 1 ]")->toWire(1));
+
+    std::string ddef = "{\"data_specification\": {\"config_data\": [ {\"item_default\": \"Hello, world!\", \"item_name\": \"default_name\", \"item_optional\": False, \"item_type\": \"string\"}, {\"item_default\": [  ], \"item_name\": \"zone_list\", \"item_optional\": False, \"item_type\": \"list\", \"list_item_spec\": {\"item_name\": \"zone_name\", \"item_optional\": True, \"item_type\": \"string\"}} ], \"module_name\": \"Auth\"}}";
+    //std::string ddef = "{\"aaa\": 123, \"test\": [  ], \"zzz\": 123}";
+    ElementPtr ddef_el = Element::createFromString(ddef);
+    std::string ddef_wire = ddef_el->toWire();
+    ElementPtr ddef_el2 = Element::fromWire(ddef_wire);
+    std::string ddef2 = ddef_el2->str();
+    EXPECT_EQ(ddef, ddef2);
 }
 

src/lib/cc/python/isc/cc/data.py

-# data, data_definition, config_data, module_config_data and ui_config_data classes
-# we might want to split these up :)
+# Copyright (C) 2010  Internet Systems Consortium.
+#
+# Permission to use, copy, modify, and distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
+# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
+# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
+# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
+# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
+# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+#
+# Helper functions for data elements as used in cc-channel and
+# configuration. There is no python equivalent for the cpp Element
+# class, since data elements are represented by native python types
+# (int, real, bool, string, list and dict respectively)
+#
+
 import ast
 
 class DataNotFoundError(Exception): pass
@@ -9,6 +29,8 @@ def merge(orig, new):
     """Merges the contents of new into orig, think recursive update()
        orig and new must both be dicts. If an element value is None in
        new it will be removed in orig."""
+    if type(orig) != dict or type(new) != dict:
+        raise DataTypeError("Not a dict in merge()")
     for kn in new.keys():
         if kn in orig:
             if new[kn]:
@@ -23,6 +45,10 @@ def merge(orig, new):
 
 def find(element, identifier):
     """Returns the subelement in the given data element, raises DataNotFoundError if not found"""
+    if type(identifier) != str or (type(element) != dict and identifier != ""):
+        raise DataTypeError("identifier in merge() is not a string")
+    if type(identifier) != str or (type(element) != dict and identifier != ""):
+        raise DataTypeError("element in merge() is not a dict")
     id_parts = identifier.split("/")
     id_parts[:] = (value for value in id_parts if value != "")
     cur_el = element
@@ -34,6 +60,17 @@ def find(element, identifier):
     return cur_el
 
 def set(element, identifier, value):
+    """Sets the value at the element specified by identifier to value.
+       If the value is None, it is removed from the dict. If element
+       is not a dict, or if the identifier points to something that is
+       not, a DataTypeError is raised. The element is updated inline,
+       so if the original needs to be kept, you must make a copy before
+       calling set(). The updated base element is returned (so that
+       el.set().set().set() is possible)"""
+    if type(element) != dict:
+        raise DataTypeError("element in set() is not a dict")
+    if type(identifier) != str:
+        raise DataTypeError("identifier in set() is not a string")
     id_parts = identifier.split("/")
     id_parts[:] = (value for value in id_parts if value != "")
     cur_el = element
@@ -41,112 +78,49 @@ def set(element, identifier, value):
         if id in cur_el.keys():
             cur_el = cur_el[id]
         else:
-            cur_el[id] = {}
-            cur_el = cur_el[id]
-    cur_el[id_parts[-1]] = value
+            if value:
+                cur_el[id] = {}
+                cur_el = cur_el[id]
+            else:
+                # set to none, and parent el not found, return
+                return element
+    # value can be an empty list or dict, so check for None eplicitely
+    if value != None:
+        cur_el[id_parts[-1]] = value
+    elif id_parts[-1] in cur_el:
+        del cur_el[id_parts[-1]]
     return element
 
 def unset(element, identifier):
-    id_parts = identifier.split("/")
-    id_parts[:] = (value for value in id_parts if value != "")
-    cur_el = element
-    for id in id_parts[:-1]:
-        if id in cur_el.keys():
-            cur_el = cur_el[id]
-        else:
-            cur_el[id] = {}
-            cur_el = cur_el[id]
-    cur_el[id_parts[-1]] = None
-    return element
+    """Removes the element at the given identifier if it exists. Raises
+       a DataTypeError if element is not a dict or if identifier is not
+       a string. Returns the base element."""
+    # perhaps we can simply do with set none, and remove this whole
+    # function
+    return set(element, identifier, None)
 
 def find_no_exc(element, identifier):
-    """Returns the subelement in the given data element, returns None if not found"""
+    """Returns the subelement in the given data element, returns None
+       if not found, or if an error occurred (i.e. this function should
+       never raise an exception)"""
+    if type(identifier) != str:
+        return None
     id_parts = identifier.split("/")
     id_parts[:] = (value for value in id_parts if value != "")
     cur_el = element
     for id in id_parts:
-        if type(cur_el) == dict and id in cur_el.keys():
+        if (type(cur_el) == dict and id in cur_el.keys()) or id=="":
             cur_el = cur_el[id]
         else:
             return None
     return cur_el
 
-def find_spec(element, identifier):
-    """find the data definition for the given identifier
-       returns either a map with 'item_name' etc, or a list of those"""
-    id_parts = identifier.split("/")
-    id_parts[:] = (value for value in id_parts if value != "")
-    cur_el = element
-    for id in id_parts:
-        if type(cur_el) == dict and id in cur_el.keys():
-            cur_el = cur_el[id]
-        elif type(cur_el) == dict and 'item_name' in cur_el.keys() and cur_el['item_name'] == id:
-            pass
-        elif type(cur_el) == list:
-            found = False
-            for cur_el_item in cur_el:
-                if cur_el_item['item_name'] == id and 'item_default' in cur_el_item.keys():
-                    cur_el = cur_el_item
-                    found = True
-            if not found:
-                raise DataNotFoundError(id + " in " + str(cur_el))
-        else:
-            raise DataNotFoundError(id + " in " + str(cur_el))
-    return cur_el
-
-def check_type(specification, value):
-    """Returns true if the value is of the correct type given the
-       specification"""
-    if type(specification) == list:
-        data_type = "list"
-    else:
-        data_type = specification['item_type']
-
-    if data_type == "integer" and type(value) != int:
-        raise DataTypeError(str(value) + " should be an integer")
-    elif data_type == "real" and type(value) != double:
-        raise DataTypeError(str(value) + " should be a real")
-    elif data_type == "boolean" and type(value) != boolean:
-        raise DataTypeError(str(value) + " should be a boolean")
-    elif data_type == "string" and type(value) != str:
-        raise DataTypeError(str(value) + " should be a string")
-    elif data_type == "list":
-        if type(value) != list:
-            raise DataTypeError(str(value) + " should be a list, not a " + str(value.__class__.__name__))
-        else:
-            # todo: check subtypes etc
-            for element in value:
-                check_type(specification['list_item_spec'], element)
-    elif data_type == "map" and type(value) != dict:
-        # todo: check subtypes etc
-        raise DataTypeError(str(value) + " should be a map")
-
-def spec_name_list(spec, prefix="", recurse=False):
-    """Returns a full list of all possible item identifiers in the
-       specification (part)"""
-    result = []
-    if prefix != "" and not prefix.endswith("/"):
-        prefix += "/"
-    if type(spec) == dict:
-        for name in spec:
-            result.append(prefix + name + "/")
-            if recurse:
-                result.extend(spec_name_list(spec[name],name, recurse))
-    elif type(spec) == list:
-        for list_el in spec:
-            if 'item_name' in list_el:
-                if list_el['item_type'] == dict:
-                    if recurse:
-                        result.extend(spec_name_list(list_el['map_item_spec'], prefix + list_el['item_name'], recurse))
-                else:
-                    name = list_el['item_name']
-                    if list_el['item_type'] in ["list", "map"]:
-                        name += "/"
-                    result.append(name)
-
-    return result
-
 def parse_value_str(value_str):
+    """Parses the given string to a native python object. If the
+       string cannot be parsed, it is returned. If it is not a string,
+       None is returned"""
+    if type(value_str) != str:
+        return None
     try:
         return ast.literal_eval(value_str)
     except ValueError as ve:
@@ -156,181 +130,3 @@ def parse_value_str(value_str):
         # simply return the string itself
         return value_str
 
-class ConfigData:
-    def __init__(self, specification):
-        self.specification = specification
-        self.data = {}
-
-    def get_item_list(self, identifier = None):
-        if identifier: