module_spec.py 19.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
# Copyright (C) 2009  Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

Jelte Jansen's avatar
Jelte Jansen committed
16 17 18 19 20 21 22 23
"""Module Specifications

   A module specification holds the information about what configuration
   a module can have, and what commands it understands. It provides
   functions to read it from a .spec file, and to validate a given
   set of data against the specification
"""

24
import json
25
import sys
26
import time
27

28 29
import isc.cc.data

30 31 32
# file objects are passed around as _io.TextIOWrapper objects
# import that so we can check those types

Jelte Jansen's avatar
Jelte Jansen committed
33
class ModuleSpecError(Exception):
Jelte Jansen's avatar
Jelte Jansen committed
34 35 36
    """This exception is raised it the ModuleSpec fails to initialize
       or if there is a failure or parse error reading the specification
       file"""
37 38
    pass

Jelte Jansen's avatar
Jelte Jansen committed
39
def module_spec_from_file(spec_file, check = True):
Jelte Jansen's avatar
Jelte Jansen committed
40 41
    """Returns a ModuleSpec object defined by the file at spec_file.
       If check is True, the contents are verified. If there is an error
42 43 44
       in those contents, a ModuleSpecError is raised.
       A ModuleSpecError is also raised if the file cannot be read, or
       if it is not valid JSON."""
Jelte Jansen's avatar
Jelte Jansen committed
45
    module_spec = None
46 47 48
    try:
        if hasattr(spec_file, 'read'):
            json_str = spec_file.read()
49
            module_spec = json.loads(json_str)
50 51
        elif type(spec_file) == str:
            file = open(spec_file)
52
            json_str = file.read()
53 54 55 56 57 58 59 60 61
            module_spec = json.loads(json_str)
            file.close()
        else:
            raise ModuleSpecError("spec_file not a str or file-like object")
    except ValueError as ve:
        raise ModuleSpecError("JSON parse error: " + str(ve))
    except IOError as ioe:
        raise ModuleSpecError("JSON read error: " + str(ioe))

Jelte Jansen's avatar
Jelte Jansen committed
62 63
    if 'module_spec' not in module_spec:
        raise ModuleSpecError("Data definition has no module_spec element")
64

65 66
    result = ModuleSpec(module_spec['module_spec'], check)
    return result
67

Jelte Jansen's avatar
Jelte Jansen committed
68 69
class ModuleSpec:
    def __init__(self, module_spec, check = True):
Jelte Jansen's avatar
Jelte Jansen committed
70 71 72
        """Initializes a ModuleSpec object from the specification in
           the given module_spec (which must be a dict). If check is
           True, the contents are verified. Raises a ModuleSpec error
73
           if there is something wrong with the contents of the dict"""
Jelte Jansen's avatar
Jelte Jansen committed
74 75
        if type(module_spec) != dict:
            raise ModuleSpecError("module_spec is of type " + str(type(module_spec)) + ", not dict")
76
        if check:
Jelte Jansen's avatar
Jelte Jansen committed
77 78
            _check(module_spec)
        self._module_spec = module_spec
79

80
    def validate_config(self, full, data, errors = None):
81
        """Check whether the given piece of data conforms to this
82 83 84 85
           data definition. If so, it returns True. If not, it will
           return false. If errors is given, and is an array, a string
           describing the error will be appended to it. The current
           version stops as soon as there is one error so this list
86 87 88 89
           will not be exhaustive. If 'full' is true, it also errors on
           non-optional missing values. Set this to False if you want to
           validate only a part of a configuration tree (like a list of
           non-default values)"""
Jelte Jansen's avatar
Jelte Jansen committed
90
        data_def = self.get_config_spec()
91
        if data_def is not None:
92 93 94
            return _validate_spec_list(data_def, full, data, errors)
        else:
            # no spec, always bad
95
            if errors is not None:
96
                errors.append("No config_data specification")
97
            return False
98

99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119
    def validate_command(self, cmd_name, cmd_params, errors = None):
        '''Check whether the given piece of command conforms to this 
        command definition. If so, it reutrns True. If not, it will 
        return False. If errors is given, and is an array, a string
        describing the error will be appended to it. The current version
        stops as soon as there is one error.
           cmd_name is command name to be validated, cmd_params includes 
        command's parameters needs to be validated. cmd_params must 
        be a map, with the format like:
        {param1_name: param1_value, param2_name: param2_value}
        '''
        cmd_spec = self.get_commands_spec()
        if not cmd_spec:
            return False

        for cmd in cmd_spec:
            if cmd['command_name'] != cmd_name:
                continue
            return _validate_spec_list(cmd['command_args'], True, cmd_params, errors)

        return False
120

121 122 123 124 125 126 127 128 129 130 131 132
    def validate_statistics(self, full, stat, errors = None):
        """Check whether the given piece of data conforms to this
           data definition. If so, it returns True. If not, it will
           return false. If errors is given, and is an array, a string
           describing the error will be appended to it. The current
           version stops as soon as there is one error so this list
           will not be exhaustive. If 'full' is true, it also errors on
           non-optional missing values. Set this to False if you want to
           validate only a part of a statistics tree (like a list of
           non-default values). Also it checks 'item_format' in case
           of time"""
        stat_spec = self.get_statistics_spec()
133
        if stat_spec is not None:
134 135 136
            return _validate_spec_list(stat_spec, full, stat, errors)
        else:
            # no spec, always bad
137
            if errors is not None:
138 139 140
                errors.append("No statistics specification")
            return False

141
    def get_module_name(self):
Jelte Jansen's avatar
Jelte Jansen committed
142
        """Returns a string containing the name of the module as
Jelte Jansen's avatar
Jelte Jansen committed
143
           specified by the specification given at __init__()"""
Jelte Jansen's avatar
Jelte Jansen committed
144
        return self._module_spec['module_name']
145

Jelte Jansen's avatar
Jelte Jansen committed
146 147 148 149 150 151 152 153 154 155
    def get_module_description(self):
        """Returns a string containing the description of the module as
           specified by the specification given at __init__().
           Returns an empty string if there is no description.
        """
        if 'module_description' in self._module_spec:
            return self._module_spec['module_description']
        else:
            return ""

Jelte Jansen's avatar
Jelte Jansen committed
156 157 158
    def get_full_spec(self):
        """Returns a dict representation of the full module specification"""
        return self._module_spec
159

160
    def get_config_spec(self):
Jelte Jansen's avatar
Jelte Jansen committed
161 162 163 164
        """Returns a dict representation of the configuration data part
           of the specification, or None if there is none."""
        if 'config_data' in self._module_spec:
            return self._module_spec['config_data']
165 166 167
        else:
            return None
    
Jelte Jansen's avatar
Jelte Jansen committed
168 169 170 171 172
    def get_commands_spec(self):
        """Returns a dict representation of the commands part of the
           specification, or None if there is none."""
        if 'commands' in self._module_spec:
            return self._module_spec['commands']
173 174 175
        else:
            return None
    
176 177 178 179 180 181 182 183
    def get_statistics_spec(self):
        """Returns a dict representation of the statistics part of the
           specification, or None if there is none."""
        if 'statistics' in self._module_spec:
            return self._module_spec['statistics']
        else:
            return None
    
184
    def __str__(self):
Jelte Jansen's avatar
Jelte Jansen committed
185 186
        """Returns a string representation of the full specification"""
        return self._module_spec.__str__()
187

Jelte Jansen's avatar
Jelte Jansen committed
188
def _check(module_spec):
Jelte Jansen's avatar
Jelte Jansen committed
189
    """Checks the full specification. This is a dict that contains the
Jelte Jansen's avatar
Jelte Jansen committed
190
       element "module_spec", which is in itself a dict that
Jelte Jansen's avatar
Jelte Jansen committed
191
       must contain at least a "module_name" (string) and optionally
192 193 194
       a "config_data", a "commands" and a "statistics" element, all
       of which are lists of dicts. Raises a ModuleSpecError if there
       is a problem."""
Jelte Jansen's avatar
Jelte Jansen committed
195 196 197 198
    if type(module_spec) != dict:
        raise ModuleSpecError("data specification not a dict")
    if "module_name" not in module_spec:
        raise ModuleSpecError("no module_name in module_spec")
Jelte Jansen's avatar
Jelte Jansen committed
199 200 201
    if "module_description" in module_spec and \
       type(module_spec["module_description"]) != str:
        raise ModuleSpecError("module_description is not a string")
Jelte Jansen's avatar
Jelte Jansen committed
202 203 204 205
    if "config_data" in module_spec:
        _check_config_spec(module_spec["config_data"])
    if "commands" in module_spec:
        _check_command_spec(module_spec["commands"])
206 207
    if "statistics" in module_spec:
        _check_statistics_spec(module_spec["statistics"])
208

209
def _check_config_spec(config_data):
210 211 212
    # config data is a list of items represented by dicts that contain
    # things like "item_name", depending on the type they can have
    # specific subitems
Jelte Jansen's avatar
Jelte Jansen committed
213
    """Checks a list that contains the configuration part of the
Jelte Jansen's avatar
Jelte Jansen committed
214
       specification. Raises a ModuleSpecError if there is a
Jelte Jansen's avatar
Jelte Jansen committed
215
       problem."""
216
    if type(config_data) != list:
Jelte Jansen's avatar
Jelte Jansen committed
217
        raise ModuleSpecError("config_data is of type " + str(type(config_data)) + ", not a list of items")
218
    for config_item in config_data:
219
        _check_item_spec(config_item)
220

221
def _check_command_spec(commands):
Jelte Jansen's avatar
Jelte Jansen committed
222
    """Checks the list that contains a set of commands. Raises a
Jelte Jansen's avatar
Jelte Jansen committed
223
       ModuleSpecError is there is an error"""
224
    if type(commands) != list:
Jelte Jansen's avatar
Jelte Jansen committed
225
        raise ModuleSpecError("commands is not a list of commands")
226 227
    for command in commands:
        if type(command) != dict:
Jelte Jansen's avatar
Jelte Jansen committed
228
            raise ModuleSpecError("command in commands list is not a dict")
229
        if "command_name" not in command:
Jelte Jansen's avatar
Jelte Jansen committed
230
            raise ModuleSpecError("no command_name in command item")
231 232
        command_name = command["command_name"]
        if type(command_name) != str:
Jelte Jansen's avatar
Jelte Jansen committed
233
            raise ModuleSpecError("command_name not a string: " + str(type(command_name)))
234 235
        if "command_description" in command:
            if type(command["command_description"]) != str:
Jelte Jansen's avatar
Jelte Jansen committed
236
                raise ModuleSpecError("command_description not a string in " + command_name)
237 238
        if "command_args" in command:
            if type(command["command_args"]) != list:
Jelte Jansen's avatar
Jelte Jansen committed
239
                raise ModuleSpecError("command_args is not a list in " + command_name)
240 241
            for command_arg in command["command_args"]:
                if type(command_arg) != dict:
Jelte Jansen's avatar
Jelte Jansen committed
242
                    raise ModuleSpecError("command argument not a dict in " + command_name)
243
                _check_item_spec(command_arg)
244
        else:
Jelte Jansen's avatar
Jelte Jansen committed
245
            raise ModuleSpecError("command_args missing in " + command_name)
246
    pass
247

248
def _check_item_spec(config_item):
Jelte Jansen's avatar
Jelte Jansen committed
249 250
    """Checks the dict that defines one config item
       (i.e. containing "item_name", "item_type", etc.
Jelte Jansen's avatar
Jelte Jansen committed
251
       Raises a ModuleSpecError if there is an error"""
252
    if type(config_item) != dict:
Jelte Jansen's avatar
Jelte Jansen committed
253
        raise ModuleSpecError("item spec not a dict")
254
    if "item_name" not in config_item:
Jelte Jansen's avatar
Jelte Jansen committed
255
        raise ModuleSpecError("no item_name in config item")
256
    if type(config_item["item_name"]) != str:
Jelte Jansen's avatar
Jelte Jansen committed
257
        raise ModuleSpecError("item_name is not a string: " + str(config_item["item_name"]))
258 259
    item_name = config_item["item_name"]
    if "item_type" not in config_item:
Jelte Jansen's avatar
Jelte Jansen committed
260
        raise ModuleSpecError("no item_type in config item")
261 262
    item_type = config_item["item_type"]
    if type(item_type) != str:
Jelte Jansen's avatar
Jelte Jansen committed
263
        raise ModuleSpecError("item_type in " + item_name + " is not a string: " + str(type(item_type)))
264
    if item_type not in ["integer", "real", "boolean", "string", "list", "map", "named_set", "any"]:
Jelte Jansen's avatar
Jelte Jansen committed
265
        raise ModuleSpecError("unknown item_type in " + item_name + ": " + item_type)
266 267
    if "item_optional" in config_item:
        if type(config_item["item_optional"]) != bool:
Jelte Jansen's avatar
Jelte Jansen committed
268
            raise ModuleSpecError("item_default in " + item_name + " is not a boolean")
269
        if not config_item["item_optional"] and "item_default" not in config_item:
Jelte Jansen's avatar
Jelte Jansen committed
270
            raise ModuleSpecError("no default value for non-optional item " + item_name)
271
    else:
Jelte Jansen's avatar
Jelte Jansen committed
272
        raise ModuleSpecError("item_optional not in item " + item_name)
273 274
    if "item_default" in config_item:
        item_default = config_item["item_default"]
275 276
        if (item_type == "integer" and type(item_default) != int) or \
           (item_type == "real" and type(item_default) != float) or \
277 278 279 280
           (item_type == "boolean" and type(item_default) != bool) or \
           (item_type == "string" and type(item_default) != str) or \
           (item_type == "list" and type(item_default) != list) or \
           (item_type == "map" and type(item_default) != dict):
Jelte Jansen's avatar
Jelte Jansen committed
281
            raise ModuleSpecError("Wrong type for item_default in " + item_name)
282 283 284
    # TODO: once we have check_type, run the item default through that with the list|map_item_spec
    if item_type == "list":
        if "list_item_spec" not in config_item:
Jelte Jansen's avatar
Jelte Jansen committed
285
            raise ModuleSpecError("no list_item_spec in list item " + item_name)
286
        if type(config_item["list_item_spec"]) != dict:
Jelte Jansen's avatar
Jelte Jansen committed
287
            raise ModuleSpecError("list_item_spec in " + item_name + " is not a dict")
288
        _check_item_spec(config_item["list_item_spec"])
289 290
    if item_type == "map":
        if "map_item_spec" not in config_item:
Jelte Jansen's avatar
Jelte Jansen committed
291
            raise ModuleSpecError("no map_item_sepc in map item " + item_name)
292
        if type(config_item["map_item_spec"]) != list:
Jelte Jansen's avatar
Jelte Jansen committed
293
            raise ModuleSpecError("map_item_spec in " + item_name + " is not a list")
294 295
        for map_item in config_item["map_item_spec"]:
            if type(map_item) != dict:
Jelte Jansen's avatar
Jelte Jansen committed
296
                raise ModuleSpecError("map_item_spec element is not a dict")
297
            _check_item_spec(map_item)
298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321
    if 'item_format' in config_item and 'item_default' in config_item:
        item_format = config_item["item_format"]
        item_default = config_item["item_default"]
        if not _check_format(item_default, item_format):
            raise ModuleSpecError(
                "Wrong format for " + str(item_default) + " in " + str(item_name))

def _check_statistics_spec(statistics):
    # statistics is a list of items represented by dicts that contain
    # things like "item_name", depending on the type they can have
    # specific subitems
    """Checks a list that contains the statistics part of the
       specification. Raises a ModuleSpecError if there is a
       problem."""
    if type(statistics) != list:
        raise ModuleSpecError("statistics is of type " + str(type(statistics))
                              + ", not a list of items")
    for stat_item in statistics:
        _check_item_spec(stat_item)
        # Additionally checks if there are 'item_title' and
        # 'item_description'
        for item in [ 'item_title',  'item_description' ]:
            if item not in stat_item:
                raise ModuleSpecError("no " + item + " in statistics item")
322

323 324 325 326 327 328 329 330 331 332
def _check_format(value, format_name):
    """Check if specified value and format are correct. Return True if
       is is correct."""
    # TODO: should be added other format types if necessary
    time_formats = { 'date-time' : "%Y-%m-%dT%H:%M:%SZ",
                     'date'      : "%Y-%m-%d",
                     'time'      : "%H:%M:%S" }
    for fmt in time_formats:
        if format_name == fmt:
            try:
333 334 335 336
                # reverse check
                return value == time.strftime(
                    time_formats[fmt],
                    time.strptime(value, time_formats[fmt]))
337 338 339
            except (ValueError, TypeError):
                break
    return False
340 341 342 343 344 345

def _validate_type(spec, value, errors):
    """Returns true if the value is of the correct type given the
       specification"""
    data_type = spec['item_type']
    if data_type == "integer" and type(value) != int:
346
        if errors is not None:
347 348 349
            errors.append(str(value) + " should be an integer")
        return False
    elif data_type == "real" and type(value) != float:
350
        if errors is not None:
351 352 353
            errors.append(str(value) + " should be a real")
        return False
    elif data_type == "boolean" and type(value) != bool:
354
        if errors is not None:
355 356 357
            errors.append(str(value) + " should be a boolean")
        return False
    elif data_type == "string" and type(value) != str:
358
        if errors is not None:
359 360 361
            errors.append(str(value) + " should be a string")
        return False
    elif data_type == "list" and type(value) != list:
362
        if errors is not None:
Jelte Jansen's avatar
Jelte Jansen committed
363
            errors.append(str(value) + " should be a list")
364 365
        return False
    elif data_type == "map" and type(value) != dict:
366
        if errors is not None:
367 368
            errors.append(str(value) + " should be a map")
        return False
369
    elif data_type == "named_set" and type(value) != dict:
Jelte Jansen's avatar
Jelte Jansen committed
370 371 372
        if errors != None:
            errors.append(str(value) + " should be a map")
        return False
373 374 375
    else:
        return True

376 377 378 379 380 381
def _validate_format(spec, value, errors):
    """Returns true if the value is of the correct format given the
       specification. And also return true if no 'item_format'"""
    if "item_format" in spec:
        item_format = spec['item_format']
        if not _check_format(value, item_format):
382
            if errors is not None:
383 384 385 386 387
                errors.append("format type of " + str(value)
                              + " should be " + item_format)
            return False
    return True

388
def _validate_item(spec, full, data, errors):
389 390 391 392 393 394 395
    if not _validate_type(spec, data, errors):
        return False
    elif type(data) == list:
        list_spec = spec['list_item_spec']
        for data_el in data:
            if not _validate_type(list_spec, data_el, errors):
                return False
396 397
            if not _validate_format(list_spec, data_el, errors):
                return False
398
            if list_spec['item_type'] == "map":
399
                if not _validate_item(list_spec, full, data_el, errors):
400 401
                    return False
    elif type(data) == dict:
Jelte Jansen's avatar
Jelte Jansen committed
402 403 404 405
        if 'map_item_spec' in spec:
            if not _validate_spec_list(spec['map_item_spec'], full, data, errors):
                return False
        else:
406
            named_set_spec = spec['named_set_item_spec']
Jelte Jansen's avatar
Jelte Jansen committed
407
            for data_el in data.values():
408
                if not _validate_type(named_set_spec, data_el, errors):
Jelte Jansen's avatar
Jelte Jansen committed
409
                    return False
410
                if not _validate_item(named_set_spec, full, data_el, errors):
Jelte Jansen's avatar
Jelte Jansen committed
411
                    return False
412 413
    elif not _validate_format(spec, data, errors):
        return False
414 415
    return True

416
def _validate_spec(spec, full, data, errors):
417 418 419
    item_name = spec['item_name']
    item_optional = spec['item_optional']

420 421 422
    if not data and item_optional:
        return True
    elif item_name in data:
423
        return _validate_item(spec, full, data[item_name], errors)
424
    elif full and not item_optional:
425
        if errors is not None:
426 427 428 429 430
            errors.append("non-optional item " + item_name + " missing")
        return False
    else:
        return True

Jelte Jansen's avatar
Jelte Jansen committed
431
def _validate_spec_list(module_spec, full, data, errors):
432 433 434 435 436
    # we do not return immediately, there may be more errors
    # so we keep a boolean to keep track if we found errors
    validated = True

    # check if the known items are correct
Jelte Jansen's avatar
Jelte Jansen committed
437
    for spec_item in module_spec:
438
        if not _validate_spec(spec_item, full, data, errors):
439 440 441 442
            validated = False

    # check if there are items in our data that are not in the
    # specification
443 444 445 446 447 448
    if data is not None:
        for item_name in data:
            found = False
            for spec_item in module_spec:
                if spec_item["item_name"] == item_name:
                    found = True
449
            if not found and item_name != "version":
450
                if errors is not None:
451 452
                    errors.append("unknown item " + item_name)
                validated = False
453
    return validated