terrain.py 19.1 KB
Newer Older
1
# Copyright (C) 2011-2014  Internet Systems Consortium.
Jelte Jansen's avatar
Jelte Jansen committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

16
17
18
19
20
21
22
23
24
#
# This is the 'terrain' in which the lettuce lives. By convention, this is
# where global setup and teardown is defined.
#
# We declare some attributes of the global 'world' variables here, so the
# tests can safely assume they are present.
#
# We also use it to provide scenario invariants, such as resetting data.
#
Jelte Jansen's avatar
Jelte Jansen committed
25

26
27
from lettuce import *
import subprocess
28
import os
29
import shutil
30
import re
31
import sys
32
import time
33

34
35
36
37
# lettuce cannot directly pass commands to the terrain, so we need to
# use environment variables to influence behaviour
KEEP_OUTPUT = 'LETTUCE_KEEP_OUTPUT'

38
39
40
41
42
43
44
45
# In order to make sure we start all tests with a 'clean' environment,
# We perform a number of initialization steps, like restoring configuration
# files, and removing generated data files.

# This approach may not scale; if so we should probably provide specific
# initialization steps for scenarios. But until that is shown to be a problem,
# It will keep the scenarios cleaner.

46
47
48
49
# This is a list of files that are freshly copied before each scenario
# The first element is the original, the second is the target that will be
# used by the tests that need them
copylist = [
50
51
    ["configurations/bindctl_commands.config.orig",
     "configurations/bindctl_commands.config"],
52
53
    ["configurations/example.org.config.orig",
     "configurations/example.org.config"],
54
55
    ["configurations/generate.config.orig",
     "configurations/generate.config"],
56
57
    ["configurations/bindctl/bindctl.config.orig",
     "configurations/bindctl/bindctl.config"],
58
59
    ["configurations/auth/auth_basic.config.orig",
     "configurations/auth/auth_basic.config"],
60
61
    ["configurations/auth/auth_badzone.config.orig",
     "configurations/auth/auth_badzone.config"],
62
    ["configurations/resolver/resolver_basic.config.orig",
63
64
     "configurations/resolver/resolver_basic.config"],
    ["configurations/multi_instance/multi_auth.config.orig",
65
     "configurations/multi_instance/multi_auth.config"],
66
67
68
69
    ["configurations/ddns/ddns.config.orig",
     "configurations/ddns/ddns.config"],
    ["configurations/ddns/noddns.config.orig",
     "configurations/ddns/noddns.config"],
70
71
    ["configurations/xfrin/retransfer_master.conf.orig",
     "configurations/xfrin/retransfer_master.conf"],
72
73
    ["configurations/xfrin/retransfer_master_v4.conf.orig",
     "configurations/xfrin/retransfer_master_v4.conf"],
74
75
    ["configurations/xfrin/retransfer_master_nons.conf.orig",
     "configurations/xfrin/retransfer_master_nons.conf"],
76
77
    ["configurations/xfrin/retransfer_slave.conf.orig",
     "configurations/xfrin/retransfer_slave.conf"],
78
79
    ["configurations/xfrin/retransfer_slave_notify.conf.orig",
     "configurations/xfrin/retransfer_slave_notify.conf"],
80
81
    ["configurations/root.config.orig",
     "configurations/root.config"],
82
    ["data/inmem-xfrin.sqlite3.orig",
83
     "data/inmem-xfrin.sqlite3"],
84
85
    ["data/root.sqlite3.orig",
     "data/root.sqlite3"],
86
87
    ["data/xfrin-before-diffs.sqlite3.orig",
     "data/xfrin-before-diffs.sqlite3"],
88
89
    ["data/xfrin-notify.sqlite3.orig",
     "data/xfrin-notify.sqlite3"],
90
    ["data/ddns/example.org.sqlite3.orig",
91
92
93
     "data/ddns/example.org.sqlite3"],
    ["data/empty_db.sqlite3",
     "data/xfrout.sqlite3"]
94
95
]

96
97
98
99
100
101
102
103
104
# This is a list of files that, if present, will be removed before a scenario
removelist = [
"data/test_nonexistent_db.sqlite3"
]

# When waiting for output data of a running process, use OUTPUT_WAIT_INTERVAL
# as the interval in which to check again if it has not been found yet.
# If we have waited OUTPUT_WAIT_MAX_INTERVALS times, we will abort with an
# error (so as not to hang indefinitely)
105
OUTPUT_WAIT_INTERVAL = 0.5
106
OUTPUT_WAIT_MAX_INTERVALS = 120
107

108
# class that keeps track of one running process and the files
109
# we created for it.
110
111
112
class RunningProcess:
    def __init__(self, step, process_name, args):
        # set it to none first so destructor won't error if initializer did
Jelte Jansen's avatar
Jelte Jansen committed
113
114
115
116
117
118
119
120
121
        """
        Initialize the long-running process structure, and start the process.
        Parameters:
        step: The scenario step it was called from. This is used for
              determining the output files for redirection of stdout
              and stderr.
        process_name: The name to refer to this running process later.
        args: Array of arguments to pass to Popen().
        """
122
123
124
        self.process = None
        self.step = step
        self.process_name = process_name
125
        self.remove_files_on_exit = (os.environ.get(KEEP_OUTPUT) != '1')
126
        self._check_output_dir()
127
128
129
        self._create_filenames()
        self._start_process(args)

130
131
132
133
        # used in _wait_for_output_str, map from (filename, (strings))
        # to a file offset.
        self.__file_offsets = {}

134
    def _start_process(self, args):
Jelte Jansen's avatar
Jelte Jansen committed
135
136
137
138
139
140
        """
        Start the process.
        Parameters:
        args:
        Array of arguments to pass to Popen().
        """
141
142
        stderr_write = open(self.stderr_filename, "w")
        stdout_write = open(self.stdout_filename, "w")
143
        self.process = subprocess.Popen(args, 0, None, subprocess.PIPE,
144
145
146
147
148
149
                                        stdout_write, stderr_write)
        # open them again, this time for reading
        self.stderr = open(self.stderr_filename, "r")
        self.stdout = open(self.stdout_filename, "r")

    def mangle_filename(self, filebase, extension):
Jelte Jansen's avatar
Jelte Jansen committed
150
151
152
153
154
155
156
157
158
159
        """
        Remove whitespace and non-default characters from a base string,
        and return the substituted value. Whitespace is replaced by an
        underscore. Any other character that is not an ASCII letter, a
        number, a dot, or a hyphen or underscore is removed.
        Parameter:
        filebase: The string to perform the substitution and removal on
        extension: An extension to append to the result value
        Returns the modified filebase with the given extension
        """
160
        filebase = re.sub("\s+", "_", filebase)
161
        filebase = re.sub("[^a-zA-Z0-9.\-_]", "", filebase)
162
163
        return filebase + "." + extension

164
165
166
167
168
    def _check_output_dir(self):
        # We may want to make this overridable by the user, perhaps
        # through an environment variable. Since we currently expect
        # lettuce to be run from our lettuce dir, we shall just use
        # the relative path 'output/'
Jelte Jansen's avatar
Jelte Jansen committed
169
170
171
172
173
174
        """
        Make sure the output directory for stdout/stderr redirection
        exists.
        Fails if it exists but is not a directory, or if it does not
        and we are unable to create it.
        """
175
176
177
178
179
180
        self._output_dir = os.getcwd() + os.sep + "output"
        if not os.path.exists(self._output_dir):
            os.mkdir(self._output_dir)
        assert os.path.isdir(self._output_dir),\
            self._output_dir + " is not a directory."

181
    def _create_filenames(self):
Jelte Jansen's avatar
Jelte Jansen committed
182
183
184
185
186
        """
        Derive the filenames for stdout/stderr redirection from the
        feature, scenario, and process name. The base will be
        "<Feature>-<Scenario>-<process name>.[stdout|stderr]"
        """
187
188
        filebase = self.step.scenario.feature.name + "-" +\
                   self.step.scenario.name + "-" + self.process_name
189
190
191
192
        self.stderr_filename = self._output_dir + os.sep +\
                               self.mangle_filename(filebase, "stderr")
        self.stdout_filename = self._output_dir + os.sep +\
                               self.mangle_filename(filebase, "stdout")
193
194

    def stop_process(self):
Jelte Jansen's avatar
Jelte Jansen committed
195
196
197
198
199
        """
        Stop this process by calling terminate(). Blocks until process has
        exited. If remove_files_on_exit is True, redirected output files
        are removed.
        """
200
201
202
203
204
205
206
207
        if self.process is not None:
            self.process.terminate()
            self.process.wait()
        self.process = None
        if self.remove_files_on_exit:
            self._remove_files()

    def _remove_files(self):
Jelte Jansen's avatar
Jelte Jansen committed
208
209
210
        """
        Remove the files created for redirection of stdout/stderr output.
        """
211
212
213
        os.remove(self.stderr_filename)
        os.remove(self.stdout_filename)

214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
    def _wait_for_output_str(self, filename, running_file, strings, only_new,
                             matches=1):
        """
        Wait for a line of output in this process. This will (if
        only_new is False) check all output from the process including
        that may have been checked before.  If only_new is True, it
        only checks output that has not been covered in previous calls
        to this method for the file (if there was no such previous call to
        this method, it works same as the case of only_new=False).

        Care should be taken if only_new is to be set to True, as it may cause
        counter-intuitive results.  For example, assume the file is expected
        to contain a line that has XXX and another line has YYY, but the
        ordering is not predictable.  If this method is called with XXX as
        the search string, but the line containing YYY appears before the
        target line, this method remembers the point in the file beyond
        the line that has XXX.  If a next call to this method specifies
        YYY as the search string with only_new being True, the search will
        fail.  If the same string is expected to appear multiple times
        and you want to catch the latest one, a more reliable way is to
        specify the match number and set only_new to False, if the number
        of matches is predictable.

Jelte Jansen's avatar
Jelte Jansen committed
237
238
239
240
241
242
243
        For each line in the output, the given strings array is checked. If
        any output lines checked contains one of the strings in the strings
        array, that string (not the line!) is returned.
        Parameters:
        filename: The filename to read previous output from, if applicable.
        running_file: The open file to read new output from.
        strings: Array of strings to look for.
244
        only_new: See above.
245
        matches: Check for the string this many times.
246
247
        Returns a tuple containing the matched string, and the complete line
        it was found in.
Jelte Jansen's avatar
Jelte Jansen committed
248
249
250
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        """
251
252
253
254
255
256
257
258
        # Identify the start offset of search.  if only_new=True, start from
        # the farthest point we've reached in the file; otherwise start from
        # the beginning.
        if not filename in self.__file_offsets:
            self.__file_offsets[filename] = 0
        offset = self.__file_offsets[filename] if only_new else 0
        running_file.seek(offset)

259
        match_count = 0
260
261
        wait_count = 0
        while wait_count < OUTPUT_WAIT_MAX_INTERVALS:
262
            line = running_file.readline()
263
            where = running_file.tell()
264
265
266
            if line:
                for string in strings:
                    if line.find(string) != -1:
267
268
                        match_count += 1
                        if match_count >= matches:
269
270
271
                            # If we've gone further, update the recorded offset
                            if where > self.__file_offsets[filename]:
                                self.__file_offsets[filename] = where
272
                            return (string, line)
273
            else:
274
275
                wait_count += 1
                time.sleep(OUTPUT_WAIT_INTERVAL)
276
                running_file.seek(where)
277
        assert False, "Timeout waiting for process output: " + str(strings)
278

279
    def wait_for_stderr_str(self, strings, only_new = True, matches = 1):
Jelte Jansen's avatar
Jelte Jansen committed
280
        """
Jelte Jansen's avatar
Jelte Jansen committed
281
        Wait for one of the given strings in this process's stderr output.
Jelte Jansen's avatar
Jelte Jansen committed
282
283
        Parameters:
        strings: Array of strings to look for.
284
        only_new: See _wait_for_output_str.
285
        matches: Check for the string this many times.
286
287
        Returns a tuple containing the matched string, and the complete line
        it was found in.
Jelte Jansen's avatar
Jelte Jansen committed
288
289
290
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        """
291
        return self._wait_for_output_str(self.stderr_filename, self.stderr,
292
                                         strings, only_new, matches)
293

294
    def wait_for_stdout_str(self, strings, only_new = True, matches = 1):
Jelte Jansen's avatar
Jelte Jansen committed
295
        """
Jelte Jansen's avatar
Jelte Jansen committed
296
        Wait for one of the given strings in this process's stdout output.
Jelte Jansen's avatar
Jelte Jansen committed
297
298
        Parameters:
        strings: Array of strings to look for.
299
        only_new: See _wait_for_output_str.
300
        matches: Check for the string this many times.
301
302
        Returns a tuple containing the matched string, and the complete line
        it was found in.
Jelte Jansen's avatar
Jelte Jansen committed
303
304
305
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        """
306
        return self._wait_for_output_str(self.stdout_filename, self.stdout,
307
                                         strings, only_new, matches)
308
309
310
311
312
313

# Container class for a number of running processes
# i.e. servers like bind10, etc
# one-shot programs like dig or bindctl are started and closed separately
class RunningProcesses:
    def __init__(self):
Jelte Jansen's avatar
Jelte Jansen committed
314
315
316
        """
        Initialize with no running processes.
        """
317
        self.processes = {}
318

319
    def add_process(self, step, process_name, args):
Jelte Jansen's avatar
Jelte Jansen committed
320
321
322
323
324
325
326
327
328
329
330
        """
        Start a process with the given arguments, and store it under the given
        name.
        Parameters:
        step: The scenario step it was called from. This is used for
              determining the output files for redirection of stdout
              and stderr.
        process_name: The name to refer to this running process later.
        args: Array of arguments to pass to Popen().
        Fails if a process with the given name is already running.
        """
331
        assert process_name not in self.processes,\
332
            "Process " + process_name + " already running"
333
334
335
        self.processes[process_name] = RunningProcess(step, process_name, args)

    def get_process(self, process_name):
Jelte Jansen's avatar
Jelte Jansen committed
336
337
338
339
340
341
        """
        Return the Process with the given process name.
        Parameters:
        process_name: The name of the process to return.
        Fails if the process is not running.
        """
342
343
344
345
346
        assert process_name in self.processes,\
            "Process " + name + " unknown"
        return self.processes[process_name]

    def stop_process(self, process_name):
Jelte Jansen's avatar
Jelte Jansen committed
347
348
349
350
351
352
        """
        Stop the Process with the given process name.
        Parameters:
        process_name: The name of the process to return.
        Fails if the process is not running.
        """
353
354
355
356
        assert process_name in self.processes,\
            "Process " + name + " unknown"
        self.processes[process_name].stop_process()
        del self.processes[process_name]
357

358
    def stop_all_processes(self):
Jelte Jansen's avatar
Jelte Jansen committed
359
360
361
        """
        Stop all running processes.
        """
362
363
        for process in self.processes.values():
            process.stop_process()
364

365
    def keep_files(self):
Jelte Jansen's avatar
Jelte Jansen committed
366
367
368
369
        """
        Keep the redirection files for stdout/stderr output of all processes
        instead of removing them when they are stopped later.
        """
370
371
372
        for process in self.processes.values():
            process.remove_files_on_exit = False

373
    def wait_for_stderr_str(self, process_name, strings, only_new = True, matches = 1):
Jelte Jansen's avatar
Jelte Jansen committed
374
        """
Jelte Jansen's avatar
Jelte Jansen committed
375
        Wait for one of the given strings in the given process's stderr output.
Jelte Jansen's avatar
Jelte Jansen committed
376
377
378
        Parameters:
        process_name: The name of the process to check the stderr output of.
        strings: Array of strings to look for.
379
        only_new: See _wait_for_output_str.
380
        matches: Check for the string this many times.
Jelte Jansen's avatar
Jelte Jansen committed
381
382
383
384
385
        Returns the matched string.
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        Fails if the process is unknown.
        """
386
387
388
        assert process_name in self.processes,\
           "Process " + process_name + " unknown"
        return self.processes[process_name].wait_for_stderr_str(strings,
389
390
                                                                only_new,
                                                                matches)
391

392
    def wait_for_stdout_str(self, process_name, strings, only_new = True, matches = 1):
Jelte Jansen's avatar
Jelte Jansen committed
393
        """
Jelte Jansen's avatar
Jelte Jansen committed
394
        Wait for one of the given strings in the given process's stdout output.
Jelte Jansen's avatar
Jelte Jansen committed
395
396
397
        Parameters:
        process_name: The name of the process to check the stdout output of.
        strings: Array of strings to look for.
398
        only_new: See _wait_for_output_str.
399
        matches: Check for the string this many times.
Jelte Jansen's avatar
Jelte Jansen committed
400
401
402
403
404
        Returns the matched string.
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        Fails if the process is unknown.
        """
405
406
407
        assert process_name in self.processes,\
           "Process " + process_name + " unknown"
        return self.processes[process_name].wait_for_stdout_str(strings,
408
409
                                                                only_new,
                                                                matches)
410

411
@before.each_scenario
412
def initialize(scenario):
Jelte Jansen's avatar
Jelte Jansen committed
413
414
415
    """
    Global initialization for each scenario.
    """
416
417
418
419
    # Keep track of running processes
    world.processes = RunningProcesses()

    # Convenience variable to access the last query result from querying.py
420
421
    world.last_query_result = None

422
423
424
    # Convenience variable to access the last HTTP response from http.py
    world.last_http_response = None

425
426
427
428
    # For slightly better errors, initialize a process_pids for the relevant
    # steps
    world.process_pids = None

429
430
431
432
433
434
    # Some tests can modify the settings. If the tests fail half-way, or
    # don't clean up, this can leave configurations or data in a bad state,
    # so we copy them from originals before each scenario
    for item in copylist:
        shutil.copy(item[0], item[1])

435
436
437
438
    for item in removelist:
        if os.path.exists(item):
            os.remove(item)

439
@after.each_scenario
440
def cleanup(scenario):
Jelte Jansen's avatar
Jelte Jansen committed
441
442
443
    """
    Global cleanup for each scenario.
    """
444
445
446
    # Keep output files if the scenario failed
    if not scenario.passed:
        world.processes.keep_files()
447
    # Stop any running processes we may have had around
448
    world.processes.stop_all_processes()
449
450
451
452
453
454
455
456
457
458

# Environment check
# Checks if LETTUCE_SETUP_COMPLETED is set in the environment
# If not, abort with an error to use the run-script
if 'LETTUCE_SETUP_COMPLETED' not in os.environ:
    print("Environment check failure; LETTUCE_SETUP_COMPLETED not set")
    print("Please use the run_lettuce.sh script. If you want to test an")
    print("installed version of bind10 with these tests, use")
    print("run_lettuce.sh -I [lettuce arguments]")
    sys.exit(1)