hammer.py 66.6 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11 12 13 14
from __future__ import print_function
import os
import sys
import glob
import time
15
import json
16
import logging
17
import datetime
18 19 20
import platform
import binascii
import argparse
21 22
import textwrap
import functools
23
import subprocess
24
import multiprocessing
25
import xml.etree.ElementTree as ET
26 27 28

# TODO:
# - add docker provider
29
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
30
# - add CCACHE support
31 32 33
# - improve building from tarball
# - improve native-pkg builds
# - avoid using network if possible (e.g. check first if pkgs are installed)
34 35 36 37 38


SYSTEMS = {
    'fedora': ['27', '28', '29'],
    'centos': ['7'],
39 40
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
41
    'debian': ['8', '9'],
42
    'freebsd': ['11.2', '12.0'],
43 44
}

45
# pylint: disable=C0326
46
IMAGE_TEMPLATES = {
47 48 49 50 51 52 53 54
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
    'fedora-28-lxc':           {'bare': 'lxc-fedora-28',               'kea': 'godfryd/kea-fedora-28'},
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
55
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
56 57 58 59 60 61 62 63 64 65 66 67
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
68 69 70 71 72 73
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
74
  config.vm.hostname = "{name}"
75 76

  config.vm.box = "{image_tpl}"
77 78 79

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
80
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
81 82 83
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
84 85 86 87 88 89 90
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
91
  config.vm.hostname = "{name}"
92 93 94 95

  config.vm.box = "{image_tpl}"

  config.vm.provider "virtualbox" do |v|
96
    v.name = "{name}"
97 98 99 100 101 102 103 104 105 106
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
107 108

  config.vm.synced_folder '.', '/vagrant', disabled: true
109 110 111 112 113 114 115
end
"""


log = logging.getLogger()


116
def red(txt):
117
    """Return colorized (if the terminal supports it) or plain text."""
118 119
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
120
    return txt
121 122

def green(txt):
123
    """Return colorized (if the terminal supports it) or plain text."""
124 125
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
126
    return txt
127 128

def blue(txt):
129
    """Return colorized (if the terminal supports it) or plain text."""
130 131
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
132
    return txt
133 134


135
def get_system_revision():
136
    """Return tuple containing system name and its revision."""
137 138
    system = platform.system()
    if system == 'Linux':
139
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
140
        if system == 'debian':
141
            revision = revision[0]
142 143
        elif system == 'redhat':
            system = 'rhel'
144 145 146
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
147 148 149 150 151 152
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


153
class ExecutionError(Exception):
154
    """Exception thrown when execution encountered an error."""
155 156
    pass

157

158 159 160
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
            quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
            sleep_time_after_attempt=None):
161 162 163
    """Execute a command in shell.

    :param str cmd: a command to be executed
164 165
    :param int timeout: timeout in number of seconds, after that time the command is terminated
                        but only if check_times is True
166 167
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
168 169
    :param bool raise_error: if False then in case of error exception is not raised,
                             default: True ie exception is raise
170 171 172 173 174
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
175 176
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
                             used for e.g. SSH
177 178
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
179
    """
180
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
181 182 183 184
    if not check_times:
        timeout = None
    if dry_run:
        return 0
185

186 187 188
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
189

190 191
    if log_file_path:
        log_file = open(log_file_path, "wb")
192

193 194 195 196
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
197

198 199
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
200

201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
                    line_decoded = line.decode(errors='ignore').rstrip() + '\r'
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
221 222 223 224 225 226
                # kill using sudo to be able to kill other sudo commands
                execute('sudo kill -s TERM %s' % p.pid)
                time.sleep(5)
                # if still running, kill harder
                if p.poll() is None:
                    execute('sudo kill -s KILL %s' % p.pid)
227 228 229
                msg = "Execution timeout, %d > %d seconds elapsed (start: %d, stop %d), cmd: '%s'"
                msg = msg % (t1 - t0, timeout, t0, t1, cmd)
                raise ExecutionError(msg)
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
245

246
    if exitcode != 0 and raise_error:
247
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
248 249 250

    if capture:
        return exitcode, output
251
    return exitcode
252 253


254
def install_pkgs(pkgs, timeout=60, env=None, check_times=False):
255
    """Install native packages in a system.
256 257 258 259 260 261 262

    :param dict pkgs: specifies a list of packages to be installed
    :param int timeout: timeout in number of seconds, after that time the command
                        is terminated but only if check_times is True
    :param dict env: dictionary with environment variables (optional)
    :param bool check_times: specifies if timeouts should be enabled (optional)
    """
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278
    system, revision = get_system_revision()

    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'

279 280
    if isinstance(pkgs, list):
        pkgs = ' '.join(pkgs)
281 282 283 284

    cmd += ' ' + pkgs

    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
285 286


287
class VagrantEnv(object):
288 289 290 291 292 293 294
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

295 296 297 298 299 300 301 302 303 304 305 306 307
    def __init__(self, provider, system, revision, features, image_template_variant,
                 dry_run, quiet=False, check_times=False):
        """VagrantEnv initializer.

        :param str provider: indicate backend type: virtualbox or lxc
        :param str system: name of the system eg. ubuntu
        :param str revision: revision of the system e.g. 18.04
        :param list features: list of requested features
        :param str image_template_variant: variant of images' templates: bare or kea
        :param bool dry_run: if False then system commands are not really executed
        :param bool quiet: if True then commands will not trace to stdout
        :param bool check_times: if True then commands will be terminated after given timeout
        """
308
        self.provider = provider
309
        self.system = system
310
        self.revision = revision
311
        self.features = features
312 313 314
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
315

316 317 318 319 320
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

321 322 323 324 325
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

326 327
        key = "%s-%s-%s" % (system, revision, provider)
        image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
328 329
        self.repo_dir = os.getcwd()

330
        sys_dir = "%s-%s" % (system, revision)
331
        if provider == "virtualbox":
332
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
333
        elif provider == "lxc":
334 335 336 337
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
338

339 340
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
341

342
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
343 344 345 346 347

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

348
        crc = binascii.crc32(self.vagrant_dir.encode())
349
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
350 351 352 353

        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
                                             name=self.name)

354 355 356
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

357 358
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

359
    def up(self):
360
        """Do Vagrant up."""
361
        execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run)
362 363
        execute("vagrant up --no-provision --provider %s" % self.provider,
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
364 365

    def package(self):
366 367
        """Package Vagrant system into Vagrant box."""

368
        if self.provider == 'virtualbox':
369
            cmd = "vagrant package --output kea-%s-%s.box" % (self.system, self.revision)
370 371 372 373 374
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

375
            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
376 377 378 379 380 381 382 383
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
384 385 386 387 388 389 390 391 392 393 394 395
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
                    'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
                    '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
                    'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
                    'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
                    'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
                    'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
                    '> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "'
            cmd += 'cd %s '
            cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
            cmd += '"'
396
            execute(cmd % (lxc_container_path, lxc_box_dir))
397 398 399 400 401 402 403 404 405 406 407 408
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
409

410
    def upload(self, src):
411
        """Upload src to Vagrant system, home folder."""
412 413 414 415 416 417 418 419 420 421 422
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

423
    def run_build_and_test(self, tarball_path, jobs):
424
        """Run build and unit tests inside Vagrant system."""
425 426 427
        if self.dry_run:
            return 0, 0

428
        # prepare tarball if needed and upload it to vagrant system
429 430
        if not tarball_path:
            name_ver = 'kea-1.5.0'
431
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
432 433
            cmd += ' --exclude "*~" --exclude .git --exclude .libs '
            cmd += ' --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
434 435
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
436
            tarball_path = '/tmp/%s.tar.gz' % name_ver
437
        self.upload(tarball_path)
438 439 440

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
441 442

        t0 = time.time()
443

444
        # run build command
445
        bld_cmd = "%s hammer.py build -p local -t %s.tar.gz -j %d" % (self.python, name_ver, jobs)
446 447 448 449 450 451
        if self.features_arg:
            bld_cmd += ' ' + self.features_arg
        if self.nofeatures_arg:
            bld_cmd += ' ' + self.nofeatures_arg
        if self.check_times:
            bld_cmd += ' -i'
452 453
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
454 455 456 457 458 459

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
            execute('scp -F %s -r default:/home/vagrant/rpm-root/RPMS/x86_64/ .' % ssh_cfg_path)

460 461
        t1 = time.time()
        dt = int(t1 - t0)
462 463

        log.info('Build log file stored to %s', log_file_path)
464 465 466 467
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

468
        # run unit tests if requested
469 470 471 472
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
473 474
                cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
475 476 477 478 479 480 481
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
482
        except:  # pylint: disable=bare-except
483 484 485 486
            log.exception('ignored issue with parsing unit test results')

        return total, passed

487
    def destroy(self):
488
        """Remove the VM completely."""
489
        cmd = 'vagrant destroy --force'
490
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
491 492

    def ssh(self):
493
        """Open interactive session to the VM."""
494
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
495 496

    def dump_ssh_config(self):
497
        """Dump ssh config that allows getting into Vagrant system via SSH."""
498 499 500 501 502
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
503
        """Execute provided command inside Vagrant system."""
504 505 506
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
507

508 509 510
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
                       raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
                       quiet=quiet, check_times=self.check_times)
511

512
    def prepare_system(self):
513
        """Prepare Vagrant system for building Kea."""
514 515
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
516 517 518
        else:
            self.features_arg = ''

519
        nofeatures = set(DEFAULT_FEATURES) - self.features
520 521 522 523 524
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

525
        # select proper python version for running Hammer inside Vagrant system
526 527
        if (self.system == 'centos' and self.revision == '7' or
            (self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
528 529 530 531 532 533
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

534
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
535 536 537
        if self.system == 'rhel' and self.revision == '8':
            cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
            exitcode = self.execute(cmd, raise_error=False)
538
            if exitcode != 0:
539 540 541 542 543
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
544 545 546 547 548
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

549
        # upload Hammer to Vagrant system
550
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
551
        self.upload(hmr_py_path)
552

553 554 555
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

556
        # run prepare-system inside Vagrant system
557
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times}"
558 559
        cmd = cmd.format(features=self.features_arg,
                         nofeatures=self.nofeatures_arg,
560 561 562
                         python=self.python,
                         check_times='-i' if self.check_times else '')
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
563 564 565


def _install_gtest_sources():
566
    """Install gtest sources."""
567
    # download gtest sources only if it is not present as native package
568
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
569 570 571
        cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
        cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
        execute(cmd)
572
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
573 574 575
        os.unlink('/tmp/gtest.tar.gz')


576
def _configure_mysql(system, revision, features):
577
    """Configure MySQL database."""
578 579 580 581
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
582 583

    if system == 'freebsd':
584 585
        cmd = "echo 'SET PASSWORD = \"\";' "
        cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
586 587
        execute(cmd, raise_error=False)

588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

603 604 605 606 607 608 609 610 611 612 613 614
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

615 616 617 618 619 620 621 622 623 624 625 626 627
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

628

629
def _configure_pgsql(system, features):
630 631
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
632
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
633 634 635 636 637 638 639 640 641 642 643 644 645 646 647
        if exitcode != 0:
            execute('sudo postgresql-setup --initdb --unit postgresql')
    execute('sudo systemctl start postgresql.service')
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

648 649 650 651 652 653 654 655 656 657
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)

658

659
def _install_cassandra_deb(env, check_times):
660
    """Install Cassandra and cpp-driver using DEB package."""
661
    if not os.path.exists('/usr/sbin/cassandra'):
662 663 664 665
        cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
        cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
        execute(cmd, env=env, check_times=check_times)
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
666 667
                env=env, check_times=check_times)
        execute('sudo apt update', env=env, check_times=check_times)
668
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
669 670

    if not os.path.exists('/usr/include/cassandra.h'):
671 672 673 674
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
675 676 677 678
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
679 680


681
def _install_freeradius_client(env, check_times):
682
    """Install FreeRADIUS-client with necessary patches from Francis Dupont."""
683
    execute('rm -rf freeradius-client')
684 685 686 687 688 689
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
    execute('./configure', cwd='freeradius-client', env=env, check_times=check_times)
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
690 691 692
    execute('rm -rf freeradius-client')


693 694
def _install_cassandra_rpm(env, check_times):
    """Install Cassandra and cpp-driver using RPM package."""
695 696 697
    if not os.path.exists('/usr/bin/cassandra'):
        #execute('sudo dnf config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/')
        #execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
698
        install_pkgs('cassandra cassandra-server libuv libuv-devel', env=env, check_times=check_times)
699 700 701 702 703 704 705 706 707 708

    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


709
def prepare_system_local(features, check_times):
710
    """Prepare local system for Kea development based on requested features."""
711 712 713
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

714 715 716
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

717
    # prepare fedora
718
    if system == 'fedora':
719 720
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel']
721 722 723 724

        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'mariadb-connector-c-devel'])

725 726 727 728 729 730 731 732 733
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'community-mysql-devel'])

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

734 735 736
        if 'radius' in features:
            packages.extend(['git'])

737
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
738 739 740 741

        if 'unittest' in features:
            _install_gtest_sources()

742
        execute('sudo dnf clean packages', env=env, check_times=check_times)
743

744
        if 'cql' in features:
745
            _install_cassandra_rpm(env, check_times)
746

747
    # prepare centos
748
    elif system == 'centos':
749
        install_pkgs('epel-release', env=env, check_times=check_times)
750

751 752
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
753 754

        if 'docs' in features:
755 756 757 758
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
759

760 761 762
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

763 764 765
        if 'radius' in features:
            packages.extend(['git'])

766
        install_pkgs(packages, env=env, check_times=check_times)
767 768 769 770

        if 'unittest' in features:
            _install_gtest_sources()

771
        if 'cql' in features:
772
            _install_cassandra_rpm(env, check_times)
773

774
    # prepare rhel
775
    elif system == 'rhel':
776 777 778 779
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

780 781
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
782

783
        # TODO:
784 785 786 787 788 789
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

790 791 792
        if 'radius' in features:
            packages.extend(['git'])

793
        install_pkgs(packages, env=env, timeout=120, check_times=check_times)
794 795 796

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
797 798 799 800
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
801 802
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
803
                    check_times=check_times)
804 805 806 807 808 809
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    env=env, timeout=120, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
810 811 812 813

        if 'unittest' in features:
            _install_gtest_sources()

814
        if 'cql' in features:
815
            _install_cassandra_rpm(env, check_times)
816

817
    # prepare ubuntu
818
    elif system == 'ubuntu':
819
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
820

821 822
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev',
                    'libboost-system-dev']
823 824

        if 'unittest' in features:
825 826 827 828
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
829 830

        if 'docs' in features:
831
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
832 833 834

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
835 836 837
            packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev',
                             'docbook', 'docbook-xsl', 'flex', 'libboost-dev', 'libpq-dev',
                             'postgresql-server-dev-all', 'python3-dev'])
838

839
        if 'mysql' in features:
840 841 842 843
            if revision == '16.04':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
844 845 846 847

        if 'pgsql' in features:
            packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

848 849 850
        if 'radius' in features:
            packages.extend(['git'])

851
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
852 853

        if 'cql' in features:
854
            _install_cassandra_deb(env, check_times)
855

856
    # prepare debian
857
    elif system == 'debian':
858
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
859

860 861
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev',
                    'liblog4cplus-dev', 'libboost-system-dev']
862 863

        if 'docs' in features:
864
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
865 866 867

        if 'unittest' in features:
            if revision == '8':
868
                # libgtest-dev does not work and googletest is not available
869
                _install_gtest_sources()
870 871 872
            else:
                packages.append('googletest')

873
        if 'mysql' in features:
874 875 876 877
            if revision == '8':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
878

879 880 881
        if 'radius' in features:
            packages.extend(['git'])

882
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
883

884 885
        if 'cql' in features and revision != '8':
            # there is no libuv1 package in case of debian 8
886
            _install_cassandra_deb(env, check_times)
887

888
    # prepare freebsd
889 890 891
    elif system == 'freebsd':
        packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs']

892 893
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-xsl'])
894 895 896 897

        if 'unittest' in features:
            _install_gtest_sources()

898 899 900 901 902 903
        if 'mysql' in features:
            packages.extend(['mysql57-server', 'mysql57-client'])

        if 'radius' in features:
            packages.extend(['git'])

904
        install_pkgs(packages, env=env, timeout=6 * 60, check_times=check_times)
905

906 907
        if 'mysql' in features:
            execute('sudo sysrc mysql_enable="yes"', env=env, check_times=check_times)
908 909
            execute('sudo service mysql-server start', env=env, check_times=check_times,
                    raise_error=False)
910

911 912 913
    else:
        raise NotImplementedError

914
    if 'mysql' in features:
915
        _configure_mysql(system, revision, features)
916

917
    if 'pgsql' in features:
918
        _configure_pgsql(system, features)
919

920
    if 'radius' in features:
921
        _install_freeradius_client(env, check_times)
922 923 924

    #execute('sudo rm -rf /usr/share/doc')

925 926
    log.info('Preparing deps completed successfully.')

927

928 929
def prepare_system_in_vagrant(provider, system, revision, features, dry_run, check_times,
                              clean_start):
930
    """Prepare specified system in Vagrant according to specified features."""
931
    ve = VagrantEnv(provider, system, revision, features, 'kea', dry_run, check_times=check_times)
932 933 934 935 936 937
    if clean_start:
        ve.destroy()
    ve.up()
    ve.prepare_system()


938
def _calculate_build_timeout(features):
939
    """Return maximum allowed time for build (in seconds)."""
940 941 942 943 944 945 946
    timeout = 60
    if 'mysql' in features:
        timeout += 60
    timeout *= 60
    return timeout


947
def _build_binaries_and_run_ut(system, revision, features, tarball_path, env, check_times, jobs, dry_run):
948 949
    if tarball_path:
        # unpack tarball with sources
950
        execute('sudo rm -rf kea-src')
951 952 953 954 955 956 957 958 959 960 961 962 963 964
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
        src_path = glob.glob('kea-src/*')[0]
    else:
        src_path = '.'

    execute('autoreconf -f -i', cwd=src_path, env=env, dry_run=dry_run)

    # prepare switches for ./configure
    cmd = './configure'
    if 'mysql' in features:
        cmd += ' --with-mysql'
    if 'pgsql' in features:
        cmd += ' --with-pgsql'
965 966
    if 'cql' in features and not (system == 'debian' and revision == '8'):
        # debian 8 does not have all deps required
967 968 969
        cmd += ' --with-cql=/usr/bin/pkg-config'
    if 'unittest' in features:
        # prepare gtest switch - use downloaded gtest sources only if it is not present as native package
970
        if system in ['centos', 'fedora', 'rhel', 'freebsd']:
971
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
972
        elif system == 'debian' and revision == '8':
973
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
974
        elif system == 'debian':
975
            cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
976
        elif system == 'ubuntu':
977 978 979 980 981 982
            if revision.startswith('16.'):
                cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
            else:
                cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
        else:
            raise NotImplementedError
983
    if 'docs' in features and not (system == 'rhel' and revision == '8'):
984 985 986 987 988
        cmd += ' --enable-generate-docs'
    if 'radius' in features:
        cmd += ' --with-freeradius=/usr/local'
    if 'shell' in features:
        cmd += ' --enable-shell'
989 990
    if 'perfdhcp' in features:
        cmd += ' --enable-perfdhcp'
991 992 993 994 995 996 997

    # do ./configure
    execute(cmd, cwd=src_path, env=env, timeout=120, check_times=check_times, dry_run=dry_run)

    # estimate number of processes (jobs) to use in compilation if jobs are not provided
    if jobs == 0:
        cpus = multiprocessing.cpu_count() - 1
998
        if system == 'centos':
999 1000 1001 1002 1003 1004
            cpus = cpus // 2
        if cpus == 0:
            cpus = 1
    else:
        cpus = jobs

1005

1006
    # do build
1007 1008 1009 1010 1011 1012
    timeout = _calculate_build_timeout(features)
    if 'distcheck' in features:
        cmd = 'make distcheck'
    else:
        cmd = 'make -j%s' % cpus
    execute(cmd, cwd=src_path, env=env, timeout=timeout, check_times=check_times, dry_run=dry_run)
1013 1014 1015 1016 1017 1018 1019 1020 1021

    if 'unittest' in features:
        results_dir = os.path.abspath(os.path.join(src_path, 'tests_result'))
        execute('rm -rf %s' % results_dir, dry_run=dry_run)
        if not os.path.exists(results_dir):
            os.mkdir(results_dir)
        env['GTEST_OUTPUT'] = 'xml:%s/' % results_dir
        env['KEA_SOCKET_TEST_DIR'] = '/tmp/'
        # run unit tests
1022
        execute('make check -k',
1023
                cwd=src_path, env=env, timeout=90 * 60, raise_error=False,
1024
                check_times=check_times, dry_run=dry_run)
1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058

        # parse unit tests results
        results = {}
        grand_total = 0
        grand_not_passed = 0
        for fn in os.listdir(results_dir):
            if not fn.endswith('.xml'):
                continue
            fp = os.path.join(results_dir, fn)
            tree = ET.parse(fp)
            root = tree.getroot()
            total = int(root.get('tests'))
            failures = int(root.get('failures'))
            disabled = int(root.get('disabled'))
            errors = int(root.get('errors'))
            results[fn] = dict(total=total, failures=failures, disabled=disabled, errors=errors)
            grand_total += total
            grand_not_passed += failures + errors

        grand_passed = grand_total - grand_not_passed
        results['grand_passed'] = grand_total - grand_not_passed
        results['grand_total'] = grand_total

        result = '%s/%s passed' % (grand_passed, grand_total)
        if grand_not_passed > 0 or grand_total == 0:
            result = red(result)
        else:
            result = green(result)
        log.info('Unit test results: %s', result)

        with open('unit-test-results.json', 'w') as f:
            f.write(json.dumps(results))

    if 'install' in features:
1059 1060
        execute('sudo make install', timeout=2 * 60,
                cwd=src_path, env=env, check_times=check_times, dry_run=dry_run)
1061 1062 1063 1064 1065 1066 1067 1068 1069
        execute('sudo ldconfig', dry_run=dry_run)  # TODO: this shouldn't be needed

        if 'forge' in features:
            if 'mysql' in features:
                execute('kea-admin lease-init mysql -u keauser -p keapass -n keadb', dry_run=dry_run)
            if 'pgsql' in features:
                execute('kea-admin lease-init pgsql -u keauser -p keapass -n keadb', dry_run=dry_run)


1070 1071 1072
def _build_native_pkg(system, features, tarball_path, env, check_times, dry_run):
    """Build native (RPM or DEB) packages."""
    if system in ['fedora', 'centos', 'rhel']:
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
        # prepare RPM environment
        execute('rm -rf rpm-root', dry_run=dry_run)
        os.mkdir('rpm-root')
        os.mkdir('rpm-root/BUILD')
        os.mkdir('rpm-root/BUILDROOT')
        os.mkdir('rpm-root/RPMS')
        os.mkdir('rpm-root/SOURCES')
        os.mkdir('rpm-root/SPECS')
        os.mkdir('rpm-root/SRPMS')

        # get rpm.spec from tarball
1084
        execute('sudo rm -rf kea-src', dry_run=dry_run)
1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]
        rpm_dir = os.path.join(src_path, 'rpm')
        for f in os.listdir(rpm_dir):
            if f == 'kea.spec':
                continue
            execute('cp %s rpm-root/SOURCES' % os.path.join(rpm_dir, f), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SPECS' % os.path.join(rpm_dir, 'kea.spec'), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SOURCES' % tarball_path, check_times=check_times, dry_run=dry_run)

        # do rpm build
        cmd = "rpmbuild -ba rpm-root/SPECS/kea.spec -D'_topdir /home/vagrant/rpm-root'"
        execute(cmd, env=env, timeout=60 * 40, check_times=check_times, dry_run=dry_run)

        if 'install' in features:
            execute('sudo rpm -i rpm-root/RPMS/x86_64/*rpm', check_times=check_times, dry_run=dry_run)

1103
    elif system in ['ubuntu', 'debian']:
1104
        # unpack tarball
1105
        execute('sudo rm -rf kea-src', check_times=check_times, dry_run=dry_run)
1106 1107 1108 1109 1110
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]

        # do deb build
1111 1112
        execute('debuild -i -us -uc -b', env=env, cwd=src_path,
                timeout=60 * 40, check_times=check_times, dry_run=dry_run)
1113 1114 1115 1116 1117 1118 1119 1120

        if 'install' in features:
            execute('sudo dpkg -i kea-src/*deb', check_times=check_times, dry_run=dry_run)

    else:
        raise NotImplementedError


1121
def build_local(features, tarball_path, check_times, jobs, dry_run):
1122 1123 1124 1125 1126
    """Prepare local system for Kea development based on requested features.

    If tarball_path is provided then instead of Kea sources from current directory
    use provided tarball.
    """