Commit b302cfa3 authored by Jelte Jansen's avatar Jelte Jansen
Browse files

refactoring of cfgmgr and config in general; they now use the datadefinition...

refactoring of cfgmgr and config in general; they now use the datadefinition class so they could later validate data that is passed around
(refactoring not done yet, though it is now in a working state again, which seemed like a good time to commit)
added a config_data.py with classes for storing definitions and data (for both modules and UIs)
fixed a missed refactoring bug in bob
changed DataDefinition initializer; a string is now parsed instead of seen as a file name; there is a helper function in the module to read a datadef directly from file now
added a temporary example config data specification for auth module
added a temporary second config data element to bob.spec



git-svn-id: svn://bind10.isc.org/svn/bind10/branches/jelte-configuration@814 e5f2f494-b856-4b98-b285-d166d9295462
parent 8e06ee76
{
"data_specification": {
"module_name": "ParkingLot"
"module_name": "Auth",
"config_data": [
{ "item_name": "default_name",
"item_type": "string",
"item_optional": False,
"item_default": "Hello, world!"
},
{ "item_name": "zone_list",
"item_type": "list",
"item_optional": False,
"item_default": [],
"list_item_spec":
{ "item_name": "zone_name",
"item_type": "string",
"item_optional": True,
"item_default": ""
}
}
]
}
}
......@@ -478,7 +478,7 @@ def main():
for fd in rlist + xlist:
if fd == ccs_fd:
boss_of_bind.ccs.checkCommand()
boss_of_bind.ccs.check_command()
elif fd == wakeup_fd:
os.read(wakeup_fd, 32)
......
......@@ -7,6 +7,12 @@
"item_type": "string",
"item_optional": False,
"item_default": "Hi, shane!"
},
{
"item_name": "some_other_string",
"item_type": "string",
"item_optional": False,
"item_default": "Hi, shane!"
}
],
"commands": [
......
......@@ -85,7 +85,7 @@ class BindCmdInterpreter(Cmd):
return False
# Get all module information from cmd-ctrld
self.config_data = isc.cc.data.UIConfigData(self)
self.config_data = isc.config.UIConfigData(self)
self.update_commands()
self.cmdloop()
except KeyboardInterrupt:
......@@ -150,7 +150,8 @@ class BindCmdInterpreter(Cmd):
if (len(cmd_spec) == 0):
print('can\'t get any command specification')
for module_name in cmd_spec.keys():
self.prepare_module_commands(module_name, cmd_spec[module_name])
if cmd_spec[module_name]:
self.prepare_module_commands(module_name, cmd_spec[module_name])
def send_GET(self, url, body = None):
headers = {"cookie" : self.session_id}
......@@ -315,7 +316,7 @@ class BindCmdInterpreter(Cmd):
if cmd.module == "config":
# grm text has been stripped of slashes...
my_text = self.location + "/" + cur_line.rpartition(" ")[2]
list = self.config_data.config.get_item_list(my_text.rpartition("/")[0])
list = self.config_data.get_config_item_list(my_text.rpartition("/")[0])
hints.extend([val for val in list if val.startswith(text)])
except CmdModuleNameFormatError:
if not text:
......@@ -440,17 +441,17 @@ class BindCmdInterpreter(Cmd):
line += "(modified)"
print(line)
elif cmd.command == "add":
self.config_data.add(identifier, cmd.params['value'])
self.config_data.add_value(identifier, cmd.params['value'])
elif cmd.command == "remove":
self.config_data.remove(identifier, cmd.params['value'])
self.config_data.remove_value(identifier, cmd.params['value'])
elif cmd.command == "set":
self.config_data.set(identifier, cmd.params['value'])
self.config_data.set_value(identifier, cmd.params['value'])
elif cmd.command == "unset":
self.config_data.unset(identifier)
elif cmd.command == "revert":
self.config_data.revert()
elif cmd.command == "commit":
self.config_data.commit(self)
self.config_data.commit()
elif cmd.command == "go":
self.go(identifier)
except isc.cc.data.DataTypeError as dte:
......
......@@ -5,7 +5,7 @@ export PYTHON_EXEC
BINDCTL_PATH=@abs_top_srcdir@/src/bin/bindctl
PYTHONPATH=@abs_top_srcdir@/src/lib/cc/python
PYTHONPATH=@abs_top_builddir@/pyshared
export PYTHONPATH
cd ${BINDCTL_PATH}
......
......@@ -67,12 +67,17 @@ def prepare_config_commands(tool):
tool.add_module_info(module)
if __name__ == '__main__':
try:
tool = BindCmdInterpreter("localhost:8080")
prepare_config_commands(tool)
tool.run()
except Exception as e:
print(e)
print("Failed to connect with b10-cmdctl module, is it running?")
tool = BindCmdInterpreter("localhost:8080")
prepare_config_commands(tool)
tool.run()
# TODO: put below back, was removed to see errors
#if __name__ == '__main__':
#try:
#tool = BindCmdInterpreter("localhost:8080")
#prepare_config_commands(tool)
#tool.run()
#except Exception as e:
#print(e)
#print("Failed to connect with b10-cmdctl module, is it running?")
......@@ -84,9 +84,10 @@ def set(element, identifier, value):
else:
# set to none, and parent el not found, return
return element
if value:
# value can be an empty list or dict, so check for None eplicitely
if value != None:
cur_el[id_parts[-1]] = value
else:
elif id_parts[-1] in cur_el:
del cur_el[id_parts[-1]]
return element
......@@ -114,85 +115,6 @@ def find_no_exc(element, identifier):
return None
return cur_el
#
# hmm, these are more relevant for datadefition
# should we (re)move them?
#
def find_spec(element, identifier):
"""find the data definition for the given identifier
returns either a map with 'item_name' etc, or a list of those"""
id_parts = identifier.split("/")
id_parts[:] = (value for value in id_parts if value != "")
cur_el = element
for id in id_parts:
if type(cur_el) == dict and id in cur_el.keys():
cur_el = cur_el[id]
elif type(cur_el) == dict and 'item_name' in cur_el.keys() and cur_el['item_name'] == id:
pass
elif type(cur_el) == list:
found = False
for cur_el_item in cur_el:
if cur_el_item['item_name'] == id and 'item_default' in cur_el_item.keys():
cur_el = cur_el_item
found = True
if not found:
raise DataNotFoundError(id + " in " + str(cur_el))
else:
raise DataNotFoundError(id + " in " + str(cur_el))
return cur_el
def check_type(specification, value):
"""Returns true if the value is of the correct type given the
specification"""
if type(specification) == list:
data_type = "list"
else:
data_type = specification['item_type']
if data_type == "integer" and type(value) != int:
raise DataTypeError(str(value) + " should be an integer")
elif data_type == "real" and type(value) != double:
raise DataTypeError(str(value) + " should be a real")
elif data_type == "boolean" and type(value) != boolean:
raise DataTypeError(str(value) + " should be a boolean")
elif data_type == "string" and type(value) != str:
raise DataTypeError(str(value) + " should be a string")
elif data_type == "list":
if type(value) != list:
raise DataTypeError(str(value) + " should be a list, not a " + str(value.__class__.__name__))
else:
# todo: check subtypes etc
for element in value:
check_type(specification['list_item_spec'], element)
elif data_type == "map" and type(value) != dict:
# todo: check subtypes etc
raise DataTypeError(str(value) + " should be a map")
def spec_name_list(spec, prefix="", recurse=False):
"""Returns a full list of all possible item identifiers in the
specification (part)"""
result = []
if prefix != "" and not prefix.endswith("/"):
prefix += "/"
if type(spec) == dict:
for name in spec:
result.append(prefix + name + "/")
if recurse:
result.extend(spec_name_list(spec[name],name, recurse))
elif type(spec) == list:
for list_el in spec:
if 'item_name' in list_el:
if list_el['item_type'] == dict:
if recurse:
result.extend(spec_name_list(list_el['map_item_spec'], prefix + list_el['item_name'], recurse))
else:
name = list_el['item_name']
if list_el['item_type'] in ["list", "map"]:
name += "/"
result.append(name)
return result
def parse_value_str(value_str):
"""Parses the given string to a native python object. If the
string cannot be parsed, it is returned. If it is not a string,
......@@ -208,181 +130,3 @@ def parse_value_str(value_str):
# simply return the string itself
return value_str
class ConfigData:
def __init__(self, specification):
self.specification = specification
self.data = {}
def get_item_list(self, identifier = None):
if identifier:
spec = find_spec(self.specification, identifier)
return spec_name_list(spec, identifier + "/")
return spec_name_list(self.specification)
def get_value(self, identifier):
"""Returns a tuple where the first item is the value at the
given identifier, and the second item is a bool which is
true if the value is an unset default"""
value = find_no_exc(self.data, identifier)
if value:
return value, False
spec = find_spec(self.specification, identifier)
if spec and 'item_default' in spec:
return spec['item_default'], True
return None, False
class UIConfigData():
def __init__(self, conn, name = ''):
self.module_name = name
data_spec = self.get_data_specification(conn)
self.config = ConfigData(data_spec)
self.get_config_data(conn)
self.config_changes = {}
def get_config_data(self, conn):
self.config.data = conn.send_GET('/config_data')
def send_changes(self, conn):
conn.send_POST('/ConfigManager/set_config', self.config_changes)
# Get latest config data
self.get_config_data(conn)
self.config_changes = {}
def get_data_specification(self, conn):
return conn.send_GET('/config_spec')
def set(self, identifier, value):
# check against definition
spec = find_spec(identifier)
check_type(spec, value)
set(self.config_changes, identifier, value)
def get_value(self, identifier):
"""Returns a three-tuple, where the first item is the value
(or None), the second is a boolean specifying whether
the value is the default value, and the third is a boolean
specifying whether the value is an uncommitted change"""
value = find_no_exc(self.config_changes, identifier)
if value:
return value, False, True
value, default = self.config.get_value(identifier)
if value:
return value, default, False
return None, False, False
def get_value_map_single(self, identifier, entry):
"""Returns a single entry for a value_map, where the value is
not a part of a bigger map"""
result_part = {}
result_part['name'] = entry['item_name']
result_part['type'] = entry['item_type']
value, default, modified = self.get_value(identifier)
# should we check type and only set int, double, bool and string here?
result_part['value'] = value
result_part['default'] = default
result_part['modified'] = modified
return result_part
def get_value_map(self, identifier, entry):
"""Returns a single entry for a value_map, where the value is
a part of a bigger map"""
result_part = {}
result_part['name'] = entry['item_name']
result_part['type'] = entry['item_type']
value, default, modified = self.get_value(identifier + "/" + entry['item_name'])
# should we check type and only set int, double, bool and string here?
result_part['value'] = value
result_part['default'] = default
result_part['modified'] = modified
return result_part
def get_value_maps(self, identifier = None):
"""Returns a list of maps, containing the following values:
name: name of the entry (string)
type: string containing the type of the value (or 'module')
value: value of the entry if it is a string, int, double or bool
modified: true if the value is a local change
default: true if the value has been changed
Throws DataNotFoundError if the identifier is bad
"""
spec = find_spec(self.config.specification, identifier)
result = []
if type(spec) == dict:
# either the top-level list of modules or a spec map
if 'item_name' in spec:
result_part = self.get_value_map_single(identifier, spec)
if result_part['type'] == "list":
values = self.get_value(identifier)[0]
if values:
for value in values:
result_part2 = {}
li_spec = spec['list_item_spec']
result_part2['name'] = li_spec['item_name']
result_part2['value'] = value
result_part2['type'] = li_spec['item_type']
result_part2['default'] = False
result_part2['modified'] = False
result.append(result_part2)
else:
result.append(result_part)
else:
for name in spec:
result_part = {}
result_part['name'] = name
result_part['type'] = "module"
result_part['value'] = None
result_part['default'] = False
result_part['modified'] = False
result.append(result_part)
elif type(spec) == list:
for entry in spec:
if type(entry) == dict and 'item_name' in entry:
result.append(self.get_value_map(identifier, entry))
return result
def add(self, identifier, value_str):
data_spec = find_spec(self.config.specification, identifier)
if (type(data_spec) != dict or "list_item_spec" not in data_spec):
raise DataTypeError(identifier + " is not a list")
value = parse_value_str(value_str)
check_type(data_spec, [value])
cur_list = find_no_exc(self.config_changes, identifier)
if not cur_list:
cur_list = find_no_exc(self.config.data, identifier)
if not cur_list:
cur_list = []
if value not in cur_list:
cur_list.append(value)
set(self.config_changes, identifier, cur_list)
def remove(self, identifier, value_str):
data_spec = find_spec(self.config.specification, identifier)
if (type(data_spec) != dict or "list_item_spec" not in data_spec):
raise DataTypeError(identifier + " is not a list")
value = parse_value_str(value_str)
check_type(data_spec, [value])
cur_list = find_no_exc(self.config_changes, identifier)
if not cur_list:
cur_list = find_no_exc(self.config.data, identifier)
if not cur_list:
cur_list = []
if value in cur_list:
cur_list.remove(value)
set(self.config_changes, identifier, cur_list)
def set(self, identifier, value_str):
data_spec = find_spec(self.config.specification, identifier)
value = parse_value_str(value_str)
check_type(data_spec, value)
set(self.config_changes, identifier, value)
def unset(self, identifier):
# todo: check whether the value is optional?
unset(self.config_changes, identifier)
def revert(self):
self.config_changes = {}
def commit(self, conn):
self.send_changes(conn)
from isc.config.ccsession import *
from isc.config.config_data import *
from isc.config.datadefinition import *
......@@ -27,7 +27,7 @@ import isc
class CCSession:
def __init__(self, spec_file_name, config_handler, command_handler):
self._data_definition = isc.config.DataDefinition(spec_file_name)
self._data_definition = isc.config.data_spec_from_file(spec_file_name)
self._module_name = self._data_definition.get_module_name()
self.set_config_handler(config_handler)
......@@ -83,7 +83,7 @@ class CCSession:
def __send_spec(self):
"""Sends the data specification to the configuration manager"""
self._session.group_sendmsg(self._data_definition.get_definition(), "ConfigManager")
self._session.group_sendmsg({ "data_specification": self._data_definition.get_definition() }, "ConfigManager")
answer, env = self._session.group_recvmsg(False)
def __get_full_config(self):
......
......@@ -106,9 +106,12 @@ class ConfigManager:
The ability to specify a custom session is for testing purposes
and should not be needed for normal usage."""
def __init__(self, data_path, session = None):
self.commands = {}
# remove these and use self.data_specs
#self.commands = {}
self.data_definitions = {}
self.data_path = data_path
self.data_specs = {}
self.config = ConfigManagerData(data_path)
if session:
self.cc = session
......@@ -122,21 +125,40 @@ class ConfigManager:
"""Notifies the Boss module that the Config Manager is running"""
self.cc.group_sendmsg({"running": "configmanager"}, "Boss")
def set_config(self, module_name, data_specification):
"""Set the data specification for the given module"""
self.data_definitions[module_name] = data_specification
def remove_config(self, module_name):
"""Remove the data specification for the given module"""
self.data_definitions[module_name]
def set_data_spec(self, spec):
#data_def = isc.config.DataDefinition(spec)
self.data_specs[spec.get_module_name()] = spec
def set_commands(self, module_name, commands):
"""Set the command list for the given module"""
self.commands[module_name] = commands
def get_data_spec(self, module_name):
if module_name in self.data_specs:
return self.data_specs[module_name]
def remove_commands(self, module_name):
"""Remove the command list for the given module"""
del self.commands[module_name]
def get_config_data(self, name = None):
"""Returns a dict containing 'module_name': config_data for
all modules. If name is specified, only that module will
be included"""
config_data = {}
if name:
if name in self.data_specs:
config_data[name] = self.data_specs[name].get_data
else:
for module_name in self.data_specs.keys():
config_data[module_name] = self.data_specs[module_name].get_config_data()
return config_data
def get_commands(self, name = None):
"""Returns a dict containing 'module_name': commands_dict for
all modules. If name is specified, only that module will
be included"""
commands = {}
if name:
if name in self.data_specs:
commands[name] = self.data_specs[name].get_commands
else:
for module_name in self.data_specs.keys():
print("[XX] add commands for " + module_name)
commands[module_name] = self.data_specs[module_name].get_commands()
return commands
def read_config(self):
"""Read the current configuration from the b10-config.db file
......@@ -158,16 +180,13 @@ class ConfigManager:
if type(cmd[1]) == dict:
if 'module_name' in cmd[1] and cmd[1]['module_name'] != '':
module_name = cmd[1]['module_name']
try:
answer["result"] = [0, self.data_definitions[module_name]]
except KeyError as ke:
answer["result"] = [1, "No specification for module " + module_name]
answer["result"] = [0, self.get_config_data(module_name)]
else:
answer["result"] = [1, "Bad module_name in get_data_spec command"]
else:
answer["result"] = [1, "Bad get_data_spec command, argument not a dict"]
else:
answer["result"] = [0, self.data_definitions]
answer["result"] = [0, self.get_config_data()]
return answer
def _handle_get_config(self, cmd):
......@@ -201,6 +220,8 @@ class ConfigManager:
self.cc.group_sendmsg({ "config_update": conf_part }, module_name)
else:
conf_part = data.set(self.config.data, module_name, {})
print("[XX] SET CONF PART:")
print(conf_part)
data.merge(conf_part[module_name], cmd[2])
# send out changed info
self.cc.group_sendmsg({ "config_update": conf_part[module_name] }, module_name)
......@@ -224,28 +245,37 @@ class ConfigManager:
# todo: use DataDefinition class
# todo: error checking (like keyerrors)
answer = {}
if "config_data" in spec:
self.set_config(spec["module_name"], spec["config_data"])
self.cc.group_sendmsg({ "specification_update": [ spec["module_name"], spec["config_data"] ] }, "Cmd-Ctrld")
if "commands" in spec:
self.set_commands(spec["module_name"], spec["commands"])
self.cc.group_sendmsg({ "commands_update": [ spec["module_name"], spec["commands"] ] }, "Cmd-Ctrld")
print("[XX] CFGMGR got spec:")
print(spec)
self.set_data_spec(spec)
# We should make one general 'spec update for module' that
# passes both specification and commands at once
self.cc.group_sendmsg({ "specification_update": [ spec.get_module_name(), spec.get_config_data() ] }, "Cmd-Ctrld")
self.cc.group_sendmsg({ "commands_update": [ spec.get_module_name(), spec.get_commands() ] }, "Cmd-Ctrld")
answer["result"] = [ 0 ]
return answer
def handle_msg(self, msg):
"""Handle a direct command"""
answer = {}
print("[XX] cfgmgr got msg:")
print(msg)
if "command" in msg:
cmd = msg["command"]
try:
if cmd[0] == "get_commands":
answer["result"] = [ 0, self.commands ]
answer["result"] = [ 0, self.get_commands() ]
print("[XX] get_commands answer:")
print(answer)
elif cmd[0] == "get_data_spec":
answer = self._handle_get_data_spec(cmd)
print("[XX] get_data_spec answer:")
print(answer)
elif cmd[0] == "get_config":
answer = self._handle_get_config(cmd)
print("[XX] get_config answer:")
print(answer)
elif cmd[0] == "set_config":
answer = self._handle_set_config(cmd)
elif cmd[0] == "shutdown":
......@@ -258,12 +288,17 @@ class ConfigManager:
answer["result"] = [ 1, "Missing argument in command: " + str(ie) ]
raise ie
elif "data_specification" in msg:
answer = self._handle_data_specification(msg["data_specification"])
try:
answer = self._handle_data_specification(isc.config.DataDefinition(msg["data_specification"]))
except isc.config.DataDefinitionError as dde:
answer['result'] = [ 1, "Error in data definition: " + str(dde) ]
elif 'result' in msg:
# this seems wrong, might start pingpong
answer['result'] = [0]
else:
answer["result"] = [ 1, "Unknown message format: " + str(msg) ]
print("[XX] cfgmgr sending answer:")
print(answer)