hammer.py 67.1 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11
from __future__ import print_function
import os
12
import re
13 14 15
import sys
import glob
import time
16
import json
17
import logging
18
import datetime
19 20 21
import platform
import binascii
import argparse
22 23
import textwrap
import functools
24
import subprocess
25
import multiprocessing
26
import xml.etree.ElementTree as ET
27 28 29

# TODO:
# - add docker provider
30
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
31
# - add CCACHE support
32 33 34
# - improve building from tarball
# - improve native-pkg builds
# - avoid using network if possible (e.g. check first if pkgs are installed)
35 36 37 38 39


SYSTEMS = {
    'fedora': ['27', '28', '29'],
    'centos': ['7'],
40 41
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
42
    'debian': ['8', '9'],
43
    'freebsd': ['11.2', '12.0'],
44 45
}

46
# pylint: disable=C0326
47
IMAGE_TEMPLATES = {
48 49 50 51 52 53 54 55
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
    'fedora-28-lxc':           {'bare': 'lxc-fedora-28',               'kea': 'godfryd/kea-fedora-28'},
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
56
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
57 58 59 60 61 62 63 64 65 66 67 68
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
69 70 71 72 73 74
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
75
  config.vm.hostname = "{name}"
76 77

  config.vm.box = "{image_tpl}"
78 79 80

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
81
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
82 83 84
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
85 86 87 88 89 90 91
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
92
  config.vm.hostname = "{name}"
93 94 95 96

  config.vm.box = "{image_tpl}"

  config.vm.provider "virtualbox" do |v|
97
    v.name = "{name}"
98 99 100 101 102 103 104 105 106 107
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
108 109

  config.vm.synced_folder '.', '/vagrant', disabled: true
110 111 112 113 114 115 116
end
"""


log = logging.getLogger()


117
def red(txt):
118
    """Return colorized (if the terminal supports it) or plain text."""
119 120
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
121
    return txt
122 123

def green(txt):
124
    """Return colorized (if the terminal supports it) or plain text."""
125 126
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
127
    return txt
128 129

def blue(txt):
130
    """Return colorized (if the terminal supports it) or plain text."""
131 132
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
133
    return txt
134 135


136
def get_system_revision():
137
    """Return tuple containing system name and its revision."""
138 139
    system = platform.system()
    if system == 'Linux':
140
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
141
        if system == 'debian':
142
            revision = revision[0]
143 144
        elif system == 'redhat':
            system = 'rhel'
145 146 147
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
148 149 150 151 152 153
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


154
class ExecutionError(Exception):
155
    """Exception thrown when execution encountered an error."""
156 157
    pass

158

159 160 161
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
            quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
            sleep_time_after_attempt=None):
162 163 164
    """Execute a command in shell.

    :param str cmd: a command to be executed
165 166
    :param int timeout: timeout in number of seconds, after that time the command is terminated
                        but only if check_times is True
167 168
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
169 170
    :param bool raise_error: if False then in case of error exception is not raised,
                             default: True ie exception is raise
171 172 173 174 175
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
176 177
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
                             used for e.g. SSH
178 179
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
180
    """
181
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
182 183 184 185
    if not check_times:
        timeout = None
    if dry_run:
        return 0
186

187 188 189
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
190

191 192
    if log_file_path:
        log_file = open(log_file_path, "wb")
193

194 195 196 197
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
198

199 200
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
201

202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
                    line_decoded = line.decode(errors='ignore').rstrip() + '\r'
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
222 223 224 225 226 227
                # kill using sudo to be able to kill other sudo commands
                execute('sudo kill -s TERM %s' % p.pid)
                time.sleep(5)
                # if still running, kill harder
                if p.poll() is None:
                    execute('sudo kill -s KILL %s' % p.pid)
228 229 230
                msg = "Execution timeout, %d > %d seconds elapsed (start: %d, stop %d), cmd: '%s'"
                msg = msg % (t1 - t0, timeout, t0, t1, cmd)
                raise ExecutionError(msg)
231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
246

247
    if exitcode != 0 and raise_error:
248
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
249 250 251

    if capture:
        return exitcode, output
252
    return exitcode
253 254


255
def install_pkgs(pkgs, timeout=60, env=None, check_times=False):
256
    """Install native packages in a system.
257 258 259 260 261 262 263

    :param dict pkgs: specifies a list of packages to be installed
    :param int timeout: timeout in number of seconds, after that time the command
                        is terminated but only if check_times is True
    :param dict env: dictionary with environment variables (optional)
    :param bool check_times: specifies if timeouts should be enabled (optional)
    """
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
    system, revision = get_system_revision()

    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'

280 281
    if isinstance(pkgs, list):
        pkgs = ' '.join(pkgs)
282 283 284 285

    cmd += ' ' + pkgs

    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
286 287


288
class VagrantEnv(object):
289 290 291 292 293 294 295
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

296 297 298 299 300 301 302 303 304 305 306 307 308
    def __init__(self, provider, system, revision, features, image_template_variant,
                 dry_run, quiet=False, check_times=False):
        """VagrantEnv initializer.

        :param str provider: indicate backend type: virtualbox or lxc
        :param str system: name of the system eg. ubuntu
        :param str revision: revision of the system e.g. 18.04
        :param list features: list of requested features
        :param str image_template_variant: variant of images' templates: bare or kea
        :param bool dry_run: if False then system commands are not really executed
        :param bool quiet: if True then commands will not trace to stdout
        :param bool check_times: if True then commands will be terminated after given timeout
        """
309
        self.provider = provider
310
        self.system = system
311
        self.revision = revision
312
        self.features = features
313 314 315
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
316

317 318 319 320 321
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

322 323 324 325 326
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

327 328
        key = "%s-%s-%s" % (system, revision, provider)
        image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
329 330
        self.repo_dir = os.getcwd()

331
        sys_dir = "%s-%s" % (system, revision)
332
        if provider == "virtualbox":
333
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
334
        elif provider == "lxc":
335 336 337 338
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
339

340 341
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
342

343
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
344 345 346 347 348

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

349
        crc = binascii.crc32(self.vagrant_dir.encode())
350
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
351 352 353 354

        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
                                             name=self.name)

355 356 357
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

358 359
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

360
    def up(self):
361
        """Do Vagrant up."""
362
        execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run)
363 364
        execute("vagrant up --no-provision --provider %s" % self.provider,
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
365 366

    def package(self):
367 368
        """Package Vagrant system into Vagrant box."""

369
        if self.provider == 'virtualbox':
370
            cmd = "vagrant package --output kea-%s-%s.box" % (self.system, self.revision)
371 372 373 374 375
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

376
            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
377 378 379 380 381 382 383 384
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
385 386 387 388 389 390 391 392 393 394 395 396
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
                    'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
                    '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
                    'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
                    'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
                    'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
                    'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
                    '> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "'
            cmd += 'cd %s '
            cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
            cmd += '"'
397
            execute(cmd % (lxc_container_path, lxc_box_dir))
398 399 400 401 402 403 404 405 406 407 408 409
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
410

411
    def upload(self, src):
412
        """Upload src to Vagrant system, home folder."""
413 414 415 416 417 418 419 420 421 422 423
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

424
    def run_build_and_test(self, tarball_path, jobs):
425
        """Run build and unit tests inside Vagrant system."""
426 427 428
        if self.dry_run:
            return 0, 0

429
        # prepare tarball if needed and upload it to vagrant system
430 431
        if not tarball_path:
            name_ver = 'kea-1.5.0'
432
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
433 434
            cmd += ' --exclude "*~" --exclude .git --exclude .libs '
            cmd += ' --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
435 436
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
437
            tarball_path = '/tmp/%s.tar.gz' % name_ver
438
        self.upload(tarball_path)
439 440 441

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
442 443

        t0 = time.time()
444

445
        # run build command
446
        bld_cmd = "%s hammer.py build -p local -t %s.tar.gz -j %d" % (self.python, name_ver, jobs)
447 448 449 450 451 452
        if self.features_arg:
            bld_cmd += ' ' + self.features_arg
        if self.nofeatures_arg:
            bld_cmd += ' ' + self.nofeatures_arg
        if self.check_times:
            bld_cmd += ' -i'
453 454
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
455 456 457 458 459 460

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
            execute('scp -F %s -r default:/home/vagrant/rpm-root/RPMS/x86_64/ .' % ssh_cfg_path)

461 462
        t1 = time.time()
        dt = int(t1 - t0)
463 464

        log.info('Build log file stored to %s', log_file_path)
465 466 467 468
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

469
        # run unit tests if requested
470 471 472 473
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
474 475
                cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
476 477 478 479 480 481 482
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
483
        except:  # pylint: disable=bare-except
484 485 486 487
            log.exception('ignored issue with parsing unit test results')

        return total, passed

488
    def destroy(self):
489
        """Remove the VM completely."""
490
        cmd = 'vagrant destroy --force'
491
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
492 493

    def ssh(self):
494
        """Open interactive session to the VM."""
495
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
496 497

    def dump_ssh_config(self):
498
        """Dump ssh config that allows getting into Vagrant system via SSH."""
499 500 501 502 503
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
504
        """Execute provided command inside Vagrant system."""
505 506 507
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
508

509 510 511
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
                       raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
                       quiet=quiet, check_times=self.check_times)
512

513
    def prepare_system(self):
514
        """Prepare Vagrant system for building Kea."""
515 516
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
517 518 519
        else:
            self.features_arg = ''

520
        nofeatures = set(DEFAULT_FEATURES) - self.features
521 522 523 524 525
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

526
        # select proper python version for running Hammer inside Vagrant system
527 528
        if (self.system == 'centos' and self.revision == '7' or
            (self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
529 530 531 532 533 534
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

535
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
536 537 538
        if self.system == 'rhel' and self.revision == '8':
            cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
            exitcode = self.execute(cmd, raise_error=False)
539
            if exitcode != 0:
540 541 542 543 544
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
545 546 547 548 549
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

550
        # upload Hammer to Vagrant system
551
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
552
        self.upload(hmr_py_path)
553

554 555 556
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

557
        # run prepare-system inside Vagrant system
558
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times}"
559 560
        cmd = cmd.format(features=self.features_arg,
                         nofeatures=self.nofeatures_arg,
561 562 563
                         python=self.python,
                         check_times='-i' if self.check_times else '')
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
564 565 566


def _install_gtest_sources():
567
    """Install gtest sources."""
568
    # download gtest sources only if it is not present as native package
569
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
570 571 572
        cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
        cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
        execute(cmd)
573
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
574 575 576
        os.unlink('/tmp/gtest.tar.gz')


577
def _configure_mysql(system, revision, features):
578
    """Configure MySQL database."""
579 580 581 582
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
583 584

    if system == 'freebsd':
585 586
        cmd = "echo 'SET PASSWORD = \"\";' "
        cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
587 588
        execute(cmd, raise_error=False)

589 590 591 592 593 594 595 596 597 598 599 600 601 602 603
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

604 605 606 607 608 609 610 611 612 613 614 615
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

616 617 618 619 620 621 622 623 624 625 626 627 628
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

629

630
def _configure_pgsql(system, features):
631 632
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
633
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
634 635 636 637 638 639 640 641 642 643 644 645 646 647 648
        if exitcode != 0:
            execute('sudo postgresql-setup --initdb --unit postgresql')
    execute('sudo systemctl start postgresql.service')
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

649 650 651 652 653 654 655 656 657 658
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)

659

660
def _install_cassandra_deb(env, check_times):
661
    """Install Cassandra and cpp-driver using DEB package."""
662
    if not os.path.exists('/usr/sbin/cassandra'):
663 664 665 666
        cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
        cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
        execute(cmd, env=env, check_times=check_times)
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
667 668
                env=env, check_times=check_times)
        execute('sudo apt update', env=env, check_times=check_times)
669
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
670 671

    if not os.path.exists('/usr/include/cassandra.h'):
672 673 674 675
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
676 677 678 679
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
680 681


682
def _install_freeradius_client(env, check_times):
683
    """Install FreeRADIUS-client with necessary patches from Francis Dupont."""
684
    execute('rm -rf freeradius-client')
685 686 687 688 689 690
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
    execute('./configure', cwd='freeradius-client', env=env, check_times=check_times)
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
691 692 693
    execute('rm -rf freeradius-client')


694 695
def _install_cassandra_rpm(env, check_times):
    """Install Cassandra and cpp-driver using RPM package."""
696 697 698
    if not os.path.exists('/usr/bin/cassandra'):
        #execute('sudo dnf config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/')
        #execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
699
        install_pkgs('cassandra cassandra-server libuv libuv-devel', env=env, check_times=check_times)
700 701 702 703 704 705 706 707 708 709

    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


710
def prepare_system_local(features, check_times):
711
    """Prepare local system for Kea development based on requested features."""
712 713 714
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

715 716 717
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

718
    # prepare fedora
719
    if system == 'fedora':
720 721
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel']
722 723 724 725

        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'mariadb-connector-c-devel'])

726 727 728 729 730 731 732 733 734
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'community-mysql-devel'])

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

735 736 737
        if 'radius' in features:
            packages.extend(['git'])

738
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
739 740 741 742

        if 'unittest' in features:
            _install_gtest_sources()

743
        execute('sudo dnf clean packages', env=env, check_times=check_times)
744

745
        if 'cql' in features:
746
            _install_cassandra_rpm(env, check_times)
747

748
    # prepare centos
749
    elif system == 'centos':
750
        install_pkgs('epel-release', env=env, check_times=check_times)
751

752 753
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
754 755

        if 'docs' in features:
756 757 758 759
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
760

761 762 763
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

764 765 766
        if 'radius' in features:
            packages.extend(['git'])

767
        install_pkgs(packages, env=env, check_times=check_times)
768 769 770 771

        if 'unittest' in features:
            _install_gtest_sources()

772
        if 'cql' in features:
773
            _install_cassandra_rpm(env, check_times)
774

775
    # prepare rhel
776
    elif system == 'rhel':
777 778 779 780
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

781 782
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
783

784
        # TODO:
785 786 787 788 789 790
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

791 792 793
        if 'radius' in features:
            packages.extend(['git'])

794
        install_pkgs(packages, env=env, timeout=120, check_times=check_times)
795 796 797

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
798 799 800 801
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
802 803
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
804
                    check_times=check_times)
805 806 807 808 809 810
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    env=env, timeout=120, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
811 812 813 814

        if 'unittest' in features:
            _install_gtest_sources()

815
        if 'cql' in features:
816
            _install_cassandra_rpm(env, check_times)
817

818
    # prepare ubuntu
819
    elif system == 'ubuntu':
820
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
821

822 823
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev',
                    'libboost-system-dev']
824 825

        if 'unittest' in features:
826 827 828 829
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
830 831

        if 'docs' in features:
832
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
833 834 835

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
836 837 838
            packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev',
                             'docbook', 'docbook-xsl', 'flex', 'libboost-dev', 'libpq-dev',
                             'postgresql-server-dev-all', 'python3-dev'])
839

840
        if 'mysql' in features:
841 842 843 844
            if revision == '16.04':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
845 846 847 848

        if 'pgsql' in features:
            packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

849 850 851
        if 'radius' in features:
            packages.extend(['git'])

852
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
853 854

        if 'cql' in features:
855
            _install_cassandra_deb(env, check_times)
856

857
    # prepare debian
858
    elif system == 'debian':
859
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
860

861 862
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev',
                    'liblog4cplus-dev', 'libboost-system-dev']
863 864

        if 'docs' in features:
865
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
866 867 868

        if 'unittest' in features:
            if revision == '8':
869
                # libgtest-dev does not work and googletest is not available
870
                _install_gtest_sources()
871 872 873
            else:
                packages.append('googletest')

874
        if 'mysql' in features:
875 876 877 878
            if revision == '8':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
879

880 881 882
        if 'radius' in features:
            packages.extend(['git'])

883
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
884

885 886
        if 'cql' in features and revision != '8':
            # there is no libuv1 package in case of debian 8
887
            _install_cassandra_deb(env, check_times)
888

889
    # prepare freebsd
890 891 892
    elif system == 'freebsd':
        packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs']

893 894
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-xsl'])
895 896 897 898

        if 'unittest' in features:
            _install_gtest_sources()

899 900 901 902 903 904
        if 'mysql' in features:
            packages.extend(['mysql57-server', 'mysql57-client'])

        if 'radius' in features:
            packages.extend(['git'])

905
        install_pkgs(packages, env=env, timeout=6 * 60, check_times=check_times)
906

907 908
        if 'mysql' in features:
            execute('sudo sysrc mysql_enable="yes"', env=env, check_times=check_times)
909 910
            execute('sudo service mysql-server start', env=env, check_times=check_times,
                    raise_error=False)
911

912 913 914
    else:
        raise NotImplementedError

915
    if 'mysql' in features:
916
        _configure_mysql(system, revision, features)
917

918
    if 'pgsql' in features:
919
        _configure_pgsql(system, features)
920

921
    if 'radius' in features:
922
        _install_freeradius_client(env, check_times)
923 924 925

    #execute('sudo rm -rf /usr/share/doc')

926 927
    log.info('Preparing deps completed successfully.')

928

929 930
def prepare_system_in_vagrant(provider, system, revision, features, dry_run, check_times,
                              clean_start):
931
    """Prepare specified system in Vagrant according to specified features."""
932
    ve = VagrantEnv(provider, system, revision, features, 'kea', dry_run, check_times=check_times)
933 934 935 936 937 938
    if clean_start:
        ve.destroy()
    ve.up()
    ve.prepare_system()


939
def _calculate_build_timeout(features):
940
    """Return maximum allowed time for build (in seconds)."""
941 942 943 944 945 946 947
    timeout = 60
    if 'mysql' in features:
        timeout += 60
    timeout *= 60
    return timeout


948
def _build_binaries_and_run_ut(system, revision, features, tarball_path, env, check_times, jobs, dry_run):
949 950
    if tarball_path:
        # unpack tarball with sources
951
        execute('sudo rm -rf kea-src')
952 953 954 955 956 957 958 959 960 961 962 963 964 965
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
        src_path = glob.glob('kea-src/*')[0]
    else:
        src_path = '.'

    execute('autoreconf -f -i', cwd=src_path, env=env, dry_run=dry_run)

    # prepare switches for ./configure
    cmd = './configure'
    if 'mysql' in features:
        cmd += ' --with-mysql'
    if 'pgsql' in features:
        cmd += ' --with-pgsql'
966 967
    if 'cql' in features and not (system == 'debian' and revision == '8'):
        # debian 8 does not have all deps required
968 969 970
        cmd += ' --with-cql=/usr/bin/pkg-config'
    if 'unittest' in features:
        # prepare gtest switch - use downloaded gtest sources only if it is not present as native package
971
        if system in ['centos', 'fedora', 'rhel', 'freebsd']:
972
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
973
        elif system == 'debian' and revision == '8':
974
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
975
        elif system == 'debian':
976
            cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
977
        elif system == 'ubuntu':
978 979 980 981 982 983
            if revision.startswith('16.'):
                cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
            else:
                cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
        else:
            raise NotImplementedError
984
    if 'docs' in features and not (system == 'rhel' and revision == '8'):
985 986 987 988 989
        cmd += ' --enable-generate-docs'
    if 'radius' in features:
        cmd += ' --with-freeradius=/usr/local'
    if 'shell' in features:
        cmd += ' --enable-shell'
990 991
    if 'perfdhcp' in features:
        cmd += ' --enable-perfdhcp'
992 993 994 995 996 997 998

    # do ./configure
    execute(cmd, cwd=src_path, env=env, timeout=120, check_times=check_times, dry_run=dry_run)

    # estimate number of processes (jobs) to use in compilation if jobs are not provided
    if jobs == 0:
        cpus = multiprocessing.cpu_count() - 1
999
        if system == 'centos':
1000 1001 1002 1003 1004 1005
            cpus = cpus // 2
        if cpus == 0:
            cpus = 1
    else:
        cpus = jobs

1006

1007
    # do build
1008 1009 1010 1011 1012 1013
    timeout = _calculate_build_timeout(features)
    if 'distcheck' in features:
        cmd = 'make distcheck'
    else:
        cmd = 'make -j%s' % cpus
    execute(cmd, cwd=src_path, env=env, timeout=timeout, check_times=check_times, dry_run=dry_run)
1014 1015 1016 1017 1018 1019 1020 1021 1022

    if 'unittest' in features:
        results_dir = os.path.abspath(os.path.join(src_path, 'tests_result'))
        execute('rm -rf %s' % results_dir, dry_run=dry_run)
        if not os.path.exists(results_dir):
            os.mkdir(results_dir)
        env['GTEST_OUTPUT'] = 'xml:%s/' % results_dir
        env['KEA_SOCKET_TEST_DIR'] = '/tmp/'
        # run unit tests
1023
        execute('make check -k',
1024
                cwd=src_path, env=env, timeout=90 * 60, raise_error=False,
1025
                check_times=check_times, dry_run=dry_run)
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059

        # parse unit tests results
        results = {}
        grand_total = 0
        grand_not_passed = 0
        for fn in os.listdir(results_dir):
            if not fn.endswith('.xml'):
                continue
            fp = os.path.join(results_dir, fn)
            tree = ET.parse(fp)
            root = tree.getroot()
            total = int(root.get('tests'))
            failures = int(root.get('failures'))
            disabled = int(root.get('disabled'))
            errors = int(root.get('errors'))
            results[fn] = dict(total=total, failures=failures, disabled=disabled, errors=errors)
            grand_total += total
            grand_not_passed += failures + errors

        grand_passed = grand_total - grand_not_passed
        results['grand_passed'] = grand_total - grand_not_passed
        results['grand_total'] = grand_total

        result = '%s/%s passed' % (grand_passed, grand_total)
        if grand_not_passed > 0 or grand_total == 0:
            result = red(result)
        else:
            result = green(result)
        log.info('Unit test results: %s', result)

        with open('unit-test-results.json', 'w') as f:
            f.write(json.dumps(results))

    if 'install' in features:
1060 1061
        execute('sudo make install', timeout=2 * 60,
                cwd=src_path, env=env, check_times=check_times, dry_run=dry_run)
1062 1063 1064 1065 1066 1067 1068 1069 1070
        execute('sudo ldconfig', dry_run=dry_run)  # TODO: this shouldn't be needed

        if 'forge' in features:
            if 'mysql' in features:
                execute('kea-admin lease-init mysql -u keauser -p keapass -n keadb', dry_run=dry_run)
            if 'pgsql' in features:
                execute('kea-admin lease-init pgsql -u keauser -p keapass -n keadb', dry_run=dry_run)


1071 1072 1073
def _build_native_pkg(system, features, tarball_path, env, check_times, dry_run):
    """Build native (RPM or DEB) packages."""
    if system in ['fedora', 'centos', 'rhel']:
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
        # prepare RPM environment
        execute('rm -rf rpm-root', dry_run=dry_run)
        os.mkdir('rpm-root')
        os.mkdir('rpm-root/BUILD')
        os.mkdir('rpm-root/BUILDROOT')
        os.mkdir('rpm-root/RPMS')
        os.mkdir('rpm-root/SOURCES')
        os.mkdir('rpm-root/SPECS')
        os.mkdir('rpm-root/SRPMS')

        # get rpm.spec from tarball
1085
        execute('sudo rm -rf kea-src', dry_run=dry_run)
1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]
        rpm_dir = os.path.join(src_path, 'rpm')
        for f in os.listdir(rpm_dir):
            if f == 'kea.spec':
                continue
            execute('cp %s rpm-root/SOURCES' % os.path.join(rpm_dir, f), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SPECS' % os.path.join(rpm_dir, 'kea.spec'), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SOURCES' % tarball_path, check_times=check_times, dry_run=dry_run)

        # do rpm build
        cmd = "rpmbuild -ba rpm-root/SPECS/kea.spec -D'_topdir /home/vagrant/rpm-root'"
        execute(cmd, env=env, timeout=60 * 40, check_times=check_times, dry_run=dry_run)

        if 'install' in features:
            execute('sudo rpm -i rpm-root/RPMS/x86_64/*rpm', check_times=check_times, dry_run=dry_run)

1104
    elif system in ['ubuntu', 'debian']:
1105
        # unpack tarball
1106
        execute('sudo rm -rf kea-src', check_times=check_times, dry_run=dry_run)
1107 1108 1109 1110 1111
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]

        # do deb build
1112 1113
        execute('debuild -i -us -uc -b', env=env, cwd=src_path,
                timeout=60 * 40, check_times=check_times, dry_run=dry_run)
1114 1115 1116 1117 1118 1119 1120 1121

        if 'install' in features:
            execute('sudo dpkg -i kea-src/*deb', check_times=check_times, dry_run=dry_run)

    else:
        raise NotImplementedError


1122
def build_local(features, tarball_path, check_times, jobs, dry_run):