hammer.py 66.3 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11 12 13 14
from __future__ import print_function
import os
import sys
import glob
import time
15
import json
16
import logging
17
import datetime
18 19 20
import platform
import binascii
import argparse
21 22
import textwrap
import functools
23
import subprocess
24
import multiprocessing
25
import xml.etree.ElementTree as ET
26 27 28

# TODO:
# - add docker provider
29
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
30
# - add CCACHE support
31 32 33
# - improve building from tarball
# - improve native-pkg builds
# - avoid using network if possible (e.g. check first if pkgs are installed)
34 35 36 37 38


SYSTEMS = {
    'fedora': ['27', '28', '29'],
    'centos': ['7'],
39 40
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
41
    'debian': ['8', '9'],
42
    'freebsd': ['11.2', '12.0'],
43 44
}

45
# pylint: disable=C0326
46
IMAGE_TEMPLATES = {
47 48 49 50 51 52 53 54
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
    'fedora-28-lxc':           {'bare': 'lxc-fedora-28',               'kea': 'godfryd/kea-fedora-28'},
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
55
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
56 57 58 59 60 61 62 63 64 65 66 67
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
68 69 70 71 72 73
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
74
  config.vm.hostname = "{name}"
75 76

  config.vm.box = "{image_tpl}"
77 78 79

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
80
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
81 82 83
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
84 85 86 87 88 89 90
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
91
  config.vm.hostname = "{name}"
92 93 94 95

  config.vm.box = "{image_tpl}"

  config.vm.provider "virtualbox" do |v|
96
    v.name = "{name}"
97 98 99 100 101 102 103 104 105 106
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
107 108

  config.vm.synced_folder '.', '/vagrant', disabled: true
109 110 111 112 113 114 115
end
"""


log = logging.getLogger()


116
def red(txt):
117
    """Return colorized (if the terminal supports it) or plain text."""
118 119
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
120
    return txt
121 122

def green(txt):
123
    """Return colorized (if the terminal supports it) or plain text."""
124 125
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
126
    return txt
127 128

def blue(txt):
129
    """Return colorized (if the terminal supports it) or plain text."""
130 131
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
132
    return txt
133 134


135
def get_system_revision():
136
    """Return tuple containing system name and its revision."""
137 138
    system = platform.system()
    if system == 'Linux':
139
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
140
        if system == 'debian':
141
            revision = revision[0]
142 143
        elif system == 'redhat':
            system = 'rhel'
144 145 146
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
147 148 149 150 151 152
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


153
class ExecutionError(Exception):
154
    """Exception thrown when execution encountered an error."""
155 156
    pass

157

158 159 160
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
            quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
            sleep_time_after_attempt=None):
161 162 163
    """Execute a command in shell.

    :param str cmd: a command to be executed
164 165
    :param int timeout: timeout in number of seconds, after that time the command is terminated
                        but only if check_times is True
166 167
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
168 169
    :param bool raise_error: if False then in case of error exception is not raised,
                             default: True ie exception is raise
170 171 172 173 174
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
175 176
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
                             used for e.g. SSH
177 178
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
179
    """
180
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
181 182 183 184
    if not check_times:
        timeout = None
    if dry_run:
        return 0
185

186 187 188
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
189

190 191
    if log_file_path:
        log_file = open(log_file_path, "wb")
192

193 194 195 196
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
197

198 199
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
200

201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
                    line_decoded = line.decode(errors='ignore').rstrip() + '\r'
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
221 222 223 224 225 226
                # kill using sudo to be able to kill other sudo commands
                execute('sudo kill -s TERM %s' % p.pid)
                time.sleep(5)
                # if still running, kill harder
                if p.poll() is None:
                    execute('sudo kill -s KILL %s' % p.pid)
227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242
                raise ExecutionError('Execution timeout')
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
243

244
    if exitcode != 0 and raise_error:
245
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
246 247 248

    if capture:
        return exitcode, output
249
    return exitcode
250 251


252
def install_pkgs(pkgs, timeout=60, env=None, check_times=False):
253
    """Install native packages in a system.
254 255 256 257 258 259 260

    :param dict pkgs: specifies a list of packages to be installed
    :param int timeout: timeout in number of seconds, after that time the command
                        is terminated but only if check_times is True
    :param dict env: dictionary with environment variables (optional)
    :param bool check_times: specifies if timeouts should be enabled (optional)
    """
261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276
    system, revision = get_system_revision()

    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'

277 278
    if isinstance(pkgs, list):
        pkgs = ' '.join(pkgs)
279 280 281 282

    cmd += ' ' + pkgs

    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
283 284


285
class VagrantEnv(object):
286 287 288 289 290 291 292
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

293 294 295 296 297 298 299 300 301 302 303 304 305
    def __init__(self, provider, system, revision, features, image_template_variant,
                 dry_run, quiet=False, check_times=False):
        """VagrantEnv initializer.

        :param str provider: indicate backend type: virtualbox or lxc
        :param str system: name of the system eg. ubuntu
        :param str revision: revision of the system e.g. 18.04
        :param list features: list of requested features
        :param str image_template_variant: variant of images' templates: bare or kea
        :param bool dry_run: if False then system commands are not really executed
        :param bool quiet: if True then commands will not trace to stdout
        :param bool check_times: if True then commands will be terminated after given timeout
        """
306
        self.provider = provider
307
        self.system = system
308
        self.revision = revision
309
        self.features = features
310 311 312
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
313

314 315 316 317 318
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

319 320 321 322 323
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

324 325
        key = "%s-%s-%s" % (system, revision, provider)
        image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
326 327
        self.repo_dir = os.getcwd()

328
        sys_dir = "%s-%s" % (system, revision)
329
        if provider == "virtualbox":
330
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
331
        elif provider == "lxc":
332 333 334 335
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
336

337 338
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
339

340
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
341 342 343 344 345

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

346
        crc = binascii.crc32(self.vagrant_dir.encode())
347
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
348 349 350 351

        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
                                             name=self.name)

352 353 354
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

355 356
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

357
    def up(self):
358
        """Do Vagrant up."""
359
        execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run)
360 361
        execute("vagrant up --no-provision --provider %s" % self.provider,
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
362 363

    def package(self):
364 365
        """Package Vagrant system into Vagrant box."""

366
        if self.provider == 'virtualbox':
367
            cmd = "vagrant package --output kea-%s-%s.box" % (self.system, self.revision)
368 369 370 371 372
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

373
            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
374 375 376 377 378 379 380 381
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
382 383 384 385 386 387 388 389 390 391 392 393
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
                    'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
                    '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
                    'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
                    'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
                    'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
                    'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
                    '> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "'
            cmd += 'cd %s '
            cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
            cmd += '"'
394
            execute(cmd % (lxc_container_path, lxc_box_dir))
395 396 397 398 399 400 401 402 403 404 405 406
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
407

408
    def upload(self, src):
409
        """Upload src to Vagrant system, home folder."""
410 411 412 413 414 415 416 417 418 419 420
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

421
    def run_build_and_test(self, tarball_path, jobs):
422
        """Run build and unit tests inside Vagrant system."""
423 424 425
        if self.dry_run:
            return 0, 0

426
        # prepare tarball if needed and upload it to vagrant system
427 428
        if not tarball_path:
            name_ver = 'kea-1.5.0'
429
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
430 431
            cmd += ' --exclude "*~" --exclude .git --exclude .libs '
            cmd += ' --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
432 433
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
434
            tarball_path = '/tmp/%s.tar.gz' % name_ver
435
        self.upload(tarball_path)
436 437 438

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
439 440

        t0 = time.time()
441

442
        # run build command
443
        bld_cmd = "%s hammer.py build -p local -t %s.tar.gz -j %d" % (self.python, name_ver, jobs)
444 445 446 447 448 449
        if self.features_arg:
            bld_cmd += ' ' + self.features_arg
        if self.nofeatures_arg:
            bld_cmd += ' ' + self.nofeatures_arg
        if self.check_times:
            bld_cmd += ' -i'
450 451
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
452 453 454 455 456 457

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
            execute('scp -F %s -r default:/home/vagrant/rpm-root/RPMS/x86_64/ .' % ssh_cfg_path)

458 459
        t1 = time.time()
        dt = int(t1 - t0)
460 461

        log.info('Build log file stored to %s', log_file_path)
462 463 464 465
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

466
        # run unit tests if requested
467 468 469 470
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
471 472
                cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
473 474 475 476 477 478 479
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
480
        except:  # pylint: disable=bare-except
481 482 483 484
            log.exception('ignored issue with parsing unit test results')

        return total, passed

485
    def destroy(self):
486
        """Remove the VM completely."""
487
        cmd = 'vagrant destroy --force'
488
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
489 490

    def ssh(self):
491
        """Open interactive session to the VM."""
492
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
493 494

    def dump_ssh_config(self):
495
        """Dump ssh config that allows getting into Vagrant system via SSH."""
496 497 498 499 500
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
501
        """Execute provided command inside Vagrant system."""
502 503 504
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
505

506 507 508
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
                       raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
                       quiet=quiet, check_times=self.check_times)
509

510
    def prepare_system(self):
511
        """Prepare Vagrant system for building Kea."""
512 513
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
514 515 516
        else:
            self.features_arg = ''

517
        nofeatures = set(DEFAULT_FEATURES) - self.features
518 519 520 521 522
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

523
        # select proper python version for running Hammer inside Vagrant system
524 525
        if (self.system == 'centos' and self.revision == '7' or
            (self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
526 527 528 529 530 531
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

532
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
533 534 535
        if self.system == 'rhel' and self.revision == '8':
            cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
            exitcode = self.execute(cmd, raise_error=False)
536
            if exitcode != 0:
537 538 539 540 541
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
542 543 544 545 546
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

547
        # upload Hammer to Vagrant system
548
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
549
        self.upload(hmr_py_path)
550

551 552 553
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

554
        # run prepare-system inside Vagrant system
555
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times}"
556 557
        cmd = cmd.format(features=self.features_arg,
                         nofeatures=self.nofeatures_arg,
558 559 560
                         python=self.python,
                         check_times='-i' if self.check_times else '')
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
561 562 563


def _install_gtest_sources():
564
    """Install gtest sources."""
565
    # download gtest sources only if it is not present as native package
566
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
567 568 569
        cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
        cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
        execute(cmd)
570
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
571 572 573
        os.unlink('/tmp/gtest.tar.gz')


574
def _configure_mysql(system, revision, features):
575
    """Configure MySQL database."""
576 577 578 579
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
580 581

    if system == 'freebsd':
582 583
        cmd = "echo 'SET PASSWORD = \"\";' "
        cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
584 585
        execute(cmd, raise_error=False)

586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

601 602 603 604 605 606 607 608 609 610 611 612
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

613 614 615 616 617 618 619 620 621 622 623 624 625
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

626

627
def _configure_pgsql(system, features):
628 629
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
630
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
        if exitcode != 0:
            execute('sudo postgresql-setup --initdb --unit postgresql')
    execute('sudo systemctl start postgresql.service')
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

646 647 648 649 650 651 652 653 654 655
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)

656

657
def _install_cassandra_deb(env, check_times):
658
    """Install Cassandra and cpp-driver using DEB package."""
659
    if not os.path.exists('/usr/sbin/cassandra'):
660 661 662 663
        cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
        cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
        execute(cmd, env=env, check_times=check_times)
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
664 665
                env=env, check_times=check_times)
        execute('sudo apt update', env=env, check_times=check_times)
666
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
667 668

    if not os.path.exists('/usr/include/cassandra.h'):
669 670 671 672
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
673 674 675 676
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
677 678


679
def _install_freeradius_client(env, check_times):
680
    """Install FreeRADIUS-client with necessary patches from Francis Dupont."""
681
    execute('rm -rf freeradius-client')
682 683 684 685 686 687
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
    execute('./configure', cwd='freeradius-client', env=env, check_times=check_times)
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
688 689 690
    execute('rm -rf freeradius-client')


691 692
def _install_cassandra_rpm(env, check_times):
    """Install Cassandra and cpp-driver using RPM package."""
693 694 695
    if not os.path.exists('/usr/bin/cassandra'):
        #execute('sudo dnf config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/')
        #execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
696
        install_pkgs('cassandra cassandra-server libuv libuv-devel', env=env, check_times=check_times)
697 698 699 700 701 702 703 704 705 706

    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


707
def prepare_system_local(features, check_times):
708
    """Prepare local system for Kea development based on requested features."""
709 710 711
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

712 713 714
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

715
    # prepare fedora
716
    if system == 'fedora':
717 718
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel']
719 720 721 722

        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'mariadb-connector-c-devel'])

723 724 725 726 727 728 729 730 731
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'community-mysql-devel'])

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

732 733 734
        if 'radius' in features:
            packages.extend(['git'])

735
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
736 737 738 739

        if 'unittest' in features:
            _install_gtest_sources()

740
        execute('sudo dnf clean packages', env=env, check_times=check_times)
741

742
        if 'cql' in features:
743
            _install_cassandra_rpm(env, check_times)
744

745
    # prepare centos
746
    elif system == 'centos':
747
        install_pkgs('epel-release', env=env, check_times=check_times)
748

749 750
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
751 752

        if 'docs' in features:
753 754 755 756
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
757

758 759 760
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

761 762 763
        if 'radius' in features:
            packages.extend(['git'])

764
        install_pkgs(packages, env=env, check_times=check_times)
765 766 767 768

        if 'unittest' in features:
            _install_gtest_sources()

769
        if 'cql' in features:
770
            _install_cassandra_rpm(env, check_times)
771

772
    # prepare rhel
773
    elif system == 'rhel':
774 775 776 777
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

778 779
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
780

781
        # TODO:
782 783 784 785 786 787
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

788 789 790
        if 'radius' in features:
            packages.extend(['git'])

791
        install_pkgs(packages, env=env, timeout=120, check_times=check_times)
792 793 794

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
795 796 797 798
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
799 800
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
801
                    check_times=check_times)
802 803 804 805 806 807
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    env=env, timeout=120, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
808 809 810 811

        if 'unittest' in features:
            _install_gtest_sources()

812
        if 'cql' in features:
813
            _install_cassandra_rpm(env, check_times)
814

815
    # prepare ubuntu
816
    elif system == 'ubuntu':
817
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
818

819 820
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev',
                    'libboost-system-dev']
821 822

        if 'unittest' in features:
823 824 825 826
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
827 828

        if 'docs' in features:
829
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
830 831 832

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
833 834 835
            packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev',
                             'docbook', 'docbook-xsl', 'flex', 'libboost-dev', 'libpq-dev',
                             'postgresql-server-dev-all', 'python3-dev'])
836

837
        if 'mysql' in features:
838 839 840 841
            if revision == '16.04':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
842 843 844 845

        if 'pgsql' in features:
            packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

846 847 848
        if 'radius' in features:
            packages.extend(['git'])

849
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
850 851

        if 'cql' in features:
852
            _install_cassandra_deb(env, check_times)
853

854
    # prepare debian
855
    elif system == 'debian':
856
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
857

858 859
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev',
                    'liblog4cplus-dev', 'libboost-system-dev']
860 861

        if 'docs' in features:
862
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
863 864 865

        if 'unittest' in features:
            if revision == '8':
866
                # libgtest-dev does not work and googletest is not available
867
                _install_gtest_sources()
868 869 870
            else:
                packages.append('googletest')

871
        if 'mysql' in features:
872 873 874 875
            if revision == '8':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
876

877 878 879
        if 'radius' in features:
            packages.extend(['git'])

880
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
881

882 883
        if 'cql' in features and revision != '8':
            # there is no libuv1 package in case of debian 8
884
            _install_cassandra_deb(env, check_times)
885

886
    # prepare freebsd
887 888 889
    elif system == 'freebsd':
        packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs']

890 891
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-xsl'])
892 893 894 895

        if 'unittest' in features:
            _install_gtest_sources()

896 897 898 899 900 901
        if 'mysql' in features:
            packages.extend(['mysql57-server', 'mysql57-client'])

        if 'radius' in features:
            packages.extend(['git'])

902
        install_pkgs(packages, env=env, timeout=6 * 60, check_times=check_times)
903

904 905
        if 'mysql' in features:
            execute('sudo sysrc mysql_enable="yes"', env=env, check_times=check_times)
906 907
            execute('sudo service mysql-server start', env=env, check_times=check_times,
                    raise_error=False)
908

909 910 911
    else:
        raise NotImplementedError

912
    if 'mysql' in features:
913
        _configure_mysql(system, revision, features)
914

915
    if 'pgsql' in features:
916
        _configure_pgsql(system, features)
917

918
    if 'radius' in features:
919
        _install_freeradius_client(env, check_times)
920 921 922

    #execute('sudo rm -rf /usr/share/doc')

923 924
    log.info('Preparing deps completed successfully.')

925

926 927
def prepare_system_in_vagrant(provider, system, revision, features, dry_run, check_times,
                              clean_start):
928
    """Prepare specified system in Vagrant according to specified features."""
929
    ve = VagrantEnv(provider, system, revision, features, 'kea', dry_run, check_times=check_times)
930 931 932 933 934 935
    if clean_start:
        ve.destroy()
    ve.up()
    ve.prepare_system()


936
def _calculate_build_timeout(features):
937
    """Return maximum allowed time for build (in seconds)."""
938 939 940 941 942 943 944
    timeout = 60
    if 'mysql' in features:
        timeout += 60
    timeout *= 60
    return timeout


945
def _build_binaries_and_run_ut(system, revision, features, tarball_path, env, check_times, jobs, dry_run):
946 947
    if tarball_path:
        # unpack tarball with sources
948
        execute('sudo rm -rf kea-src')
949 950 951 952 953 954 955 956 957 958 959 960 961 962
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
        src_path = glob.glob('kea-src/*')[0]
    else:
        src_path = '.'

    execute('autoreconf -f -i', cwd=src_path, env=env, dry_run=dry_run)

    # prepare switches for ./configure
    cmd = './configure'
    if 'mysql' in features:
        cmd += ' --with-mysql'
    if 'pgsql' in features:
        cmd += ' --with-pgsql'
963 964
    if 'cql' in features and not (system == 'debian' and revision == '8'):
        # debian 8 does not have all deps required
965 966 967
        cmd += ' --with-cql=/usr/bin/pkg-config'
    if 'unittest' in features:
        # prepare gtest switch - use downloaded gtest sources only if it is not present as native package
968
        if system in ['centos', 'fedora', 'rhel', 'freebsd']:
969
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
970
        elif system == 'debian' and revision == '8':
971
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
972
        elif system == 'debian':
973
            cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
974
        elif system == 'ubuntu':
975 976 977 978 979 980
            if revision.startswith('16.'):
                cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
            else:
                cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
        else:
            raise NotImplementedError
981
    if 'docs' in features and not (system == 'rhel' and revision == '8'):
982 983 984 985 986 987 988 989 990 991 992 993
        cmd += ' --enable-generate-docs'
    if 'radius' in features:
        cmd += ' --with-freeradius=/usr/local'
    if 'shell' in features:
        cmd += ' --enable-shell'

    # do ./configure
    execute(cmd, cwd=src_path, env=env, timeout=120, check_times=check_times, dry_run=dry_run)

    # estimate number of processes (jobs) to use in compilation if jobs are not provided
    if jobs == 0:
        cpus = multiprocessing.cpu_count() - 1
994
        if system == 'centos':
995 996 997 998 999 1000
            cpus = cpus // 2
        if cpus == 0:
            cpus = 1
    else:
        cpus = jobs

1001

1002
    # do build
1003 1004 1005 1006 1007 1008
    timeout = _calculate_build_timeout(features)
    if 'distcheck' in features:
        cmd = 'make distcheck'
    else:
        cmd = 'make -j%s' % cpus
    execute(cmd, cwd=src_path, env=env, timeout=timeout, check_times=check_times, dry_run=dry_run)
1009 1010 1011 1012 1013 1014 1015 1016 1017

    if 'unittest' in features:
        results_dir = os.path.abspath(os.path.join(src_path, 'tests_result'))
        execute('rm -rf %s' % results_dir, dry_run=dry_run)
        if not os.path.exists(results_dir):
            os.mkdir(results_dir)
        env['GTEST_OUTPUT'] = 'xml:%s/' % results_dir
        env['KEA_SOCKET_TEST_DIR'] = '/tmp/'
        # run unit tests
1018 1019 1020
        execute('make check -k',
                cwd=src_path, env=env, timeout=60 * 60, raise_error=False,
                check_times=check_times, dry_run=dry_run)
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064

        # parse unit tests results
        results = {}
        grand_total = 0
        grand_not_passed = 0
        for fn in os.listdir(results_dir):
            if not fn.endswith('.xml'):
                continue
            fp = os.path.join(results_dir, fn)
            tree = ET.parse(fp)
            root = tree.getroot()
            total = int(root.get('tests'))
            failures = int(root.get('failures'))
            disabled = int(root.get('disabled'))
            errors = int(root.get('errors'))
            results[fn] = dict(total=total, failures=failures, disabled=disabled, errors=errors)
            grand_total += total
            grand_not_passed += failures + errors

        grand_passed = grand_total - grand_not_passed
        results['grand_passed'] = grand_total - grand_not_passed
        results['grand_total'] = grand_total

        result = '%s/%s passed' % (grand_passed, grand_total)
        if grand_not_passed > 0 or grand_total == 0:
            result = red(result)
        else:
            result = green(result)
        log.info('Unit test results: %s', result)

        with open('unit-test-results.json', 'w') as f:
            f.write(json.dumps(results))

    if 'install' in features:
        execute('sudo make install', cwd=src_path, env=env, check_times=check_times, dry_run=dry_run)
        execute('sudo ldconfig', dry_run=dry_run)  # TODO: this shouldn't be needed

        if 'forge' in features:
            if 'mysql' in features:
                execute('kea-admin lease-init mysql -u keauser -p keapass -n keadb', dry_run=dry_run)
            if 'pgsql' in features:
                execute('kea-admin lease-init pgsql -u keauser -p keapass -n keadb', dry_run=dry_run)


1065 1066 1067
def _build_native_pkg(system, features, tarball_path, env, check_times, dry_run):
    """Build native (RPM or DEB) packages."""
    if system in ['fedora', 'centos', 'rhel']:
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078
        # prepare RPM environment
        execute('rm -rf rpm-root', dry_run=dry_run)
        os.mkdir('rpm-root')
        os.mkdir('rpm-root/BUILD')
        os.mkdir('rpm-root/BUILDROOT')
        os.mkdir('rpm-root/RPMS')
        os.mkdir('rpm-root/SOURCES')
        os.mkdir('rpm-root/SPECS')
        os.mkdir('rpm-root/SRPMS')

        # get rpm.spec from tarball
1079
        execute('sudo rm -rf kea-src', dry_run=dry_run)
1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]
        rpm_dir = os.path.join(src_path, 'rpm')
        for f in os.listdir(rpm_dir):
            if f == 'kea.spec':
                continue
            execute('cp %s rpm-root/SOURCES' % os.path.join(rpm_dir, f), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SPECS' % os.path.join(rpm_dir, 'kea.spec'), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SOURCES' % tarball_path, check_times=check_times, dry_run=dry_run)

        # do rpm build
        cmd = "rpmbuild -ba rpm-root/SPECS/kea.spec -D'_topdir /home/vagrant/rpm-root'"
        execute(cmd, env=env, timeout=60 * 40, check_times=check_times, dry_run=dry_run)

        if 'install' in features:
            execute('sudo rpm -i rpm-root/RPMS/x86_64/*rpm', check_times=check_times, dry_run=dry_run)

1098
    elif system in ['ubuntu', 'debian']:
1099
        # unpack tarball
1100
        execute('sudo rm -rf kea-src', check_times=check_times, dry_run=dry_run)
1101 1102 1103 1104 1105
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]

        # do deb build
1106 1107
        execute('debuild -i -us -uc -b', env=env, cwd=src_path,
                timeout=60 * 40, check_times=check_times, dry_run=dry_run)
1108 1109 1110 1111 1112 1113 1114 1115

        if 'install' in features:
            execute('sudo dpkg -i kea-src/*deb', check_times=check_times, dry_run=dry_run)

    else:
        raise NotImplementedError


1116
def build_local(features, tarball_path, check_times, jobs, dry_run):
1117 1118 1119 1120 1121
    """Prepare local system for Kea development based on requested features.

    If tarball_path is provided then instead of Kea sources from current directory
    use provided tarball.
    """
1122 1123 1124
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

1125
    system, revision = get_system_revision()
1126

1127
    execute('df -h', dry_run=dry_run)
1128

1129 1130
    if tarball_path:
        tarball_path = os.path.abspath(tarball_path)