terrain.py 17.7 KB
Newer Older
1

Jelte Jansen's avatar
Jelte Jansen committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# Copyright (C) 2011  Internet Systems Consortium.
#
# Permission to use, copy, modify, and distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SYSTEMS CONSORTIUM
# DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL
# INTERNET SYSTEMS CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT,
# INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING
# FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
# WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

17
18
19
20
21
22
23
24
25
#
# This is the 'terrain' in which the lettuce lives. By convention, this is
# where global setup and teardown is defined.
#
# We declare some attributes of the global 'world' variables here, so the
# tests can safely assume they are present.
#
# We also use it to provide scenario invariants, such as resetting data.
#
Jelte Jansen's avatar
Jelte Jansen committed
26

27
28
from lettuce import *
import subprocess
29
import os
30
import shutil
31
import re
32
import sys
33
import time
34

35
36
37
38
# lettuce cannot directly pass commands to the terrain, so we need to
# use environment variables to influence behaviour
KEEP_OUTPUT = 'LETTUCE_KEEP_OUTPUT'

39
40
41
42
43
44
45
46
# In order to make sure we start all tests with a 'clean' environment,
# We perform a number of initialization steps, like restoring configuration
# files, and removing generated data files.

# This approach may not scale; if so we should probably provide specific
# initialization steps for scenarios. But until that is shown to be a problem,
# It will keep the scenarios cleaner.

47
48
49
50
# This is a list of files that are freshly copied before each scenario
# The first element is the original, the second is the target that will be
# used by the tests that need them
copylist = [
51
52
    ["configurations/bindctl_commands.config.orig",
     "configurations/bindctl_commands.config"],
53
54
    ["configurations/example.org.config.orig",
     "configurations/example.org.config"],
55
56
    ["configurations/bindctl/bindctl.config.orig",
     "configurations/bindctl/bindctl.config"],
57
58
    ["configurations/auth/auth_basic.config.orig",
     "configurations/auth/auth_basic.config"],
59
60
    ["configurations/auth/auth_badzone.config.orig",
     "configurations/auth/auth_badzone.config"],
61
    ["configurations/resolver/resolver_basic.config.orig",
62
63
     "configurations/resolver/resolver_basic.config"],
    ["configurations/multi_instance/multi_auth.config.orig",
64
     "configurations/multi_instance/multi_auth.config"],
65
66
67
68
    ["configurations/ddns/ddns.config.orig",
     "configurations/ddns/ddns.config"],
    ["configurations/ddns/noddns.config.orig",
     "configurations/ddns/noddns.config"],
69
70
    ["configurations/xfrin/retransfer_master.conf.orig",
     "configurations/xfrin/retransfer_master.conf"],
71
72
    ["configurations/xfrin/retransfer_master_nons.conf.orig",
     "configurations/xfrin/retransfer_master_nons.conf"],
73
74
    ["configurations/xfrin/retransfer_slave.conf.orig",
     "configurations/xfrin/retransfer_slave.conf"],
75
    ["data/inmem-xfrin.sqlite3.orig",
76
     "data/inmem-xfrin.sqlite3"],
77
78
    ["data/xfrin-before-diffs.sqlite3.orig",
     "data/xfrin-before-diffs.sqlite3"],
79
80
    ["data/xfrin-notify.sqlite3.orig",
     "data/xfrin-notify.sqlite3"],
81
82
    ["data/ddns/example.org.sqlite3.orig",
     "data/ddns/example.org.sqlite3"]
83
84
]

85
86
87
88
89
90
91
92
93
# This is a list of files that, if present, will be removed before a scenario
removelist = [
"data/test_nonexistent_db.sqlite3"
]

# When waiting for output data of a running process, use OUTPUT_WAIT_INTERVAL
# as the interval in which to check again if it has not been found yet.
# If we have waited OUTPUT_WAIT_MAX_INTERVALS times, we will abort with an
# error (so as not to hang indefinitely)
94
OUTPUT_WAIT_INTERVAL = 0.5
95
OUTPUT_WAIT_MAX_INTERVALS = 120
96

97
# class that keeps track of one running process and the files
98
# we created for it.
99
100
101
class RunningProcess:
    def __init__(self, step, process_name, args):
        # set it to none first so destructor won't error if initializer did
Jelte Jansen's avatar
Jelte Jansen committed
102
103
104
105
106
107
108
109
110
        """
        Initialize the long-running process structure, and start the process.
        Parameters:
        step: The scenario step it was called from. This is used for
              determining the output files for redirection of stdout
              and stderr.
        process_name: The name to refer to this running process later.
        args: Array of arguments to pass to Popen().
        """
111
112
113
        self.process = None
        self.step = step
        self.process_name = process_name
114
        self.remove_files_on_exit = (os.environ.get(KEEP_OUTPUT) != '1')
115
        self._check_output_dir()
116
117
118
119
        self._create_filenames()
        self._start_process(args)

    def _start_process(self, args):
Jelte Jansen's avatar
Jelte Jansen committed
120
121
122
123
124
125
        """
        Start the process.
        Parameters:
        args:
        Array of arguments to pass to Popen().
        """
126
127
128
129
130
131
132
133
134
        stderr_write = open(self.stderr_filename, "w")
        stdout_write = open(self.stdout_filename, "w")
        self.process = subprocess.Popen(args, 1, None, subprocess.PIPE,
                                        stdout_write, stderr_write)
        # open them again, this time for reading
        self.stderr = open(self.stderr_filename, "r")
        self.stdout = open(self.stdout_filename, "r")

    def mangle_filename(self, filebase, extension):
Jelte Jansen's avatar
Jelte Jansen committed
135
136
137
138
139
140
141
142
143
144
        """
        Remove whitespace and non-default characters from a base string,
        and return the substituted value. Whitespace is replaced by an
        underscore. Any other character that is not an ASCII letter, a
        number, a dot, or a hyphen or underscore is removed.
        Parameter:
        filebase: The string to perform the substitution and removal on
        extension: An extension to append to the result value
        Returns the modified filebase with the given extension
        """
145
        filebase = re.sub("\s+", "_", filebase)
146
        filebase = re.sub("[^a-zA-Z0-9.\-_]", "", filebase)
147
148
        return filebase + "." + extension

149
150
151
152
153
    def _check_output_dir(self):
        # We may want to make this overridable by the user, perhaps
        # through an environment variable. Since we currently expect
        # lettuce to be run from our lettuce dir, we shall just use
        # the relative path 'output/'
Jelte Jansen's avatar
Jelte Jansen committed
154
155
156
157
158
159
        """
        Make sure the output directory for stdout/stderr redirection
        exists.
        Fails if it exists but is not a directory, or if it does not
        and we are unable to create it.
        """
160
161
162
163
164
165
        self._output_dir = os.getcwd() + os.sep + "output"
        if not os.path.exists(self._output_dir):
            os.mkdir(self._output_dir)
        assert os.path.isdir(self._output_dir),\
            self._output_dir + " is not a directory."

166
    def _create_filenames(self):
Jelte Jansen's avatar
Jelte Jansen committed
167
168
169
170
171
        """
        Derive the filenames for stdout/stderr redirection from the
        feature, scenario, and process name. The base will be
        "<Feature>-<Scenario>-<process name>.[stdout|stderr]"
        """
172
173
        filebase = self.step.scenario.feature.name + "-" +\
                   self.step.scenario.name + "-" + self.process_name
174
175
176
177
        self.stderr_filename = self._output_dir + os.sep +\
                               self.mangle_filename(filebase, "stderr")
        self.stdout_filename = self._output_dir + os.sep +\
                               self.mangle_filename(filebase, "stdout")
178
179

    def stop_process(self):
Jelte Jansen's avatar
Jelte Jansen committed
180
181
182
183
184
        """
        Stop this process by calling terminate(). Blocks until process has
        exited. If remove_files_on_exit is True, redirected output files
        are removed.
        """
185
186
187
188
189
190
191
192
        if self.process is not None:
            self.process.terminate()
            self.process.wait()
        self.process = None
        if self.remove_files_on_exit:
            self._remove_files()

    def _remove_files(self):
Jelte Jansen's avatar
Jelte Jansen committed
193
194
195
        """
        Remove the files created for redirection of stdout/stderr output.
        """
196
197
198
        os.remove(self.stderr_filename)
        os.remove(self.stdout_filename)

199
    def _wait_for_output_str(self, filename, running_file, strings, only_new, matches = 1):
Jelte Jansen's avatar
Jelte Jansen committed
200
201
202
203
204
205
206
207
208
209
210
211
212
        """
        Wait for a line of output in this process. This will (if only_new is
        False) first check all previous output from the process, and if not
        found, check all output since the last time this method was called.
        For each line in the output, the given strings array is checked. If
        any output lines checked contains one of the strings in the strings
        array, that string (not the line!) is returned.
        Parameters:
        filename: The filename to read previous output from, if applicable.
        running_file: The open file to read new output from.
        strings: Array of strings to look for.
        only_new: If true, only check output since last time this method was
                  called. If false, first check earlier output.
213
        matches: Check for the string this many times.
214
215
        Returns a tuple containing the matched string, and the complete line
        it was found in.
Jelte Jansen's avatar
Jelte Jansen committed
216
217
218
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        """
219
        match_count = 0
220
221
222
223
224
        if not only_new:
            full_file = open(filename, "r")
            for line in full_file:
                for string in strings:
                    if line.find(string) != -1:
225
226
227
228
                        match_count += 1
                        if match_count >= matches:
                            full_file.close()
                            return (string, line)
229
230
        wait_count = 0
        while wait_count < OUTPUT_WAIT_MAX_INTERVALS:
231
232
233
234
235
            where = running_file.tell()
            line = running_file.readline()
            if line:
                for string in strings:
                    if line.find(string) != -1:
236
237
238
                        match_count += 1
                        if match_count >= matches:
                            return (string, line)
239
            else:
240
241
                wait_count += 1
                time.sleep(OUTPUT_WAIT_INTERVAL)
242
                running_file.seek(where)
243
        assert False, "Timeout waiting for process output: " + str(strings)
244

245
    def wait_for_stderr_str(self, strings, only_new = True, matches = 1):
Jelte Jansen's avatar
Jelte Jansen committed
246
        """
Jelte Jansen's avatar
Jelte Jansen committed
247
        Wait for one of the given strings in this process's stderr output.
Jelte Jansen's avatar
Jelte Jansen committed
248
249
250
251
        Parameters:
        strings: Array of strings to look for.
        only_new: If true, only check output since last time this method was
                  called. If false, first check earlier output.
252
        matches: Check for the string this many times.
253
254
        Returns a tuple containing the matched string, and the complete line
        it was found in.
Jelte Jansen's avatar
Jelte Jansen committed
255
256
257
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        """
258
        return self._wait_for_output_str(self.stderr_filename, self.stderr,
259
                                         strings, only_new, matches)
260

261
    def wait_for_stdout_str(self, strings, only_new = True, matches = 1):
Jelte Jansen's avatar
Jelte Jansen committed
262
        """
Jelte Jansen's avatar
Jelte Jansen committed
263
        Wait for one of the given strings in this process's stdout output.
Jelte Jansen's avatar
Jelte Jansen committed
264
265
266
267
        Parameters:
        strings: Array of strings to look for.
        only_new: If true, only check output since last time this method was
                  called. If false, first check earlier output.
268
        matches: Check for the string this many times.
269
270
        Returns a tuple containing the matched string, and the complete line
        it was found in.
Jelte Jansen's avatar
Jelte Jansen committed
271
272
273
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        """
274
        return self._wait_for_output_str(self.stdout_filename, self.stdout,
275
                                         strings, only_new, matches)
276
277
278
279
280
281

# Container class for a number of running processes
# i.e. servers like bind10, etc
# one-shot programs like dig or bindctl are started and closed separately
class RunningProcesses:
    def __init__(self):
Jelte Jansen's avatar
Jelte Jansen committed
282
283
284
        """
        Initialize with no running processes.
        """
285
        self.processes = {}
286

287
    def add_process(self, step, process_name, args):
Jelte Jansen's avatar
Jelte Jansen committed
288
289
290
291
292
293
294
295
296
297
298
        """
        Start a process with the given arguments, and store it under the given
        name.
        Parameters:
        step: The scenario step it was called from. This is used for
              determining the output files for redirection of stdout
              and stderr.
        process_name: The name to refer to this running process later.
        args: Array of arguments to pass to Popen().
        Fails if a process with the given name is already running.
        """
299
        assert process_name not in self.processes,\
300
            "Process " + process_name + " already running"
301
302
303
        self.processes[process_name] = RunningProcess(step, process_name, args)

    def get_process(self, process_name):
Jelte Jansen's avatar
Jelte Jansen committed
304
305
306
307
308
309
        """
        Return the Process with the given process name.
        Parameters:
        process_name: The name of the process to return.
        Fails if the process is not running.
        """
310
311
312
313
314
        assert process_name in self.processes,\
            "Process " + name + " unknown"
        return self.processes[process_name]

    def stop_process(self, process_name):
Jelte Jansen's avatar
Jelte Jansen committed
315
316
317
318
319
320
        """
        Stop the Process with the given process name.
        Parameters:
        process_name: The name of the process to return.
        Fails if the process is not running.
        """
321
322
323
324
        assert process_name in self.processes,\
            "Process " + name + " unknown"
        self.processes[process_name].stop_process()
        del self.processes[process_name]
325

326
    def stop_all_processes(self):
Jelte Jansen's avatar
Jelte Jansen committed
327
328
329
        """
        Stop all running processes.
        """
330
331
        for process in self.processes.values():
            process.stop_process()
332

333
    def keep_files(self):
Jelte Jansen's avatar
Jelte Jansen committed
334
335
336
337
        """
        Keep the redirection files for stdout/stderr output of all processes
        instead of removing them when they are stopped later.
        """
338
339
340
        for process in self.processes.values():
            process.remove_files_on_exit = False

341
    def wait_for_stderr_str(self, process_name, strings, only_new = True, matches = 1):
Jelte Jansen's avatar
Jelte Jansen committed
342
        """
Jelte Jansen's avatar
Jelte Jansen committed
343
        Wait for one of the given strings in the given process's stderr output.
Jelte Jansen's avatar
Jelte Jansen committed
344
345
346
347
348
        Parameters:
        process_name: The name of the process to check the stderr output of.
        strings: Array of strings to look for.
        only_new: If true, only check output since last time this method was
                  called. If false, first check earlier output.
349
        matches: Check for the string this many times.
Jelte Jansen's avatar
Jelte Jansen committed
350
351
352
353
354
        Returns the matched string.
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        Fails if the process is unknown.
        """
355
356
357
        assert process_name in self.processes,\
           "Process " + process_name + " unknown"
        return self.processes[process_name].wait_for_stderr_str(strings,
358
359
                                                                only_new,
                                                                matches)
360

361
    def wait_for_stdout_str(self, process_name, strings, only_new = True, matches = 1):
Jelte Jansen's avatar
Jelte Jansen committed
362
        """
Jelte Jansen's avatar
Jelte Jansen committed
363
        Wait for one of the given strings in the given process's stdout output.
Jelte Jansen's avatar
Jelte Jansen committed
364
365
366
367
368
        Parameters:
        process_name: The name of the process to check the stdout output of.
        strings: Array of strings to look for.
        only_new: If true, only check output since last time this method was
                  called. If false, first check earlier output.
369
        matches: Check for the string this many times.
Jelte Jansen's avatar
Jelte Jansen committed
370
371
372
373
374
        Returns the matched string.
        Fails if none of the strings was read after 10 seconds
        (OUTPUT_WAIT_INTERVAL * OUTPUT_WAIT_MAX_INTERVALS).
        Fails if the process is unknown.
        """
375
376
377
        assert process_name in self.processes,\
           "Process " + process_name + " unknown"
        return self.processes[process_name].wait_for_stdout_str(strings,
378
379
                                                                only_new,
                                                                matches)
380

381
@before.each_scenario
382
def initialize(scenario):
Jelte Jansen's avatar
Jelte Jansen committed
383
384
385
    """
    Global initialization for each scenario.
    """
386
387
388
389
    # Keep track of running processes
    world.processes = RunningProcesses()

    # Convenience variable to access the last query result from querying.py
390
391
    world.last_query_result = None

392
393
394
    # Convenience variable to access the last HTTP response from http.py
    world.last_http_response = None

395
396
397
398
    # For slightly better errors, initialize a process_pids for the relevant
    # steps
    world.process_pids = None

399
400
401
402
403
404
    # Some tests can modify the settings. If the tests fail half-way, or
    # don't clean up, this can leave configurations or data in a bad state,
    # so we copy them from originals before each scenario
    for item in copylist:
        shutil.copy(item[0], item[1])

405
406
407
408
    for item in removelist:
        if os.path.exists(item):
            os.remove(item)

409
@after.each_scenario
410
def cleanup(scenario):
Jelte Jansen's avatar
Jelte Jansen committed
411
412
413
    """
    Global cleanup for each scenario.
    """
414
415
416
    # Keep output files if the scenario failed
    if not scenario.passed:
        world.processes.keep_files()
417
    # Stop any running processes we may have had around
418
    world.processes.stop_all_processes()
419
420
421
422
423
424
425
426
427
428

# Environment check
# Checks if LETTUCE_SETUP_COMPLETED is set in the environment
# If not, abort with an error to use the run-script
if 'LETTUCE_SETUP_COMPLETED' not in os.environ:
    print("Environment check failure; LETTUCE_SETUP_COMPLETED not set")
    print("Please use the run_lettuce.sh script. If you want to test an")
    print("installed version of bind10 with these tests, use")
    print("run_lettuce.sh -I [lettuce arguments]")
    sys.exit(1)