hammer.py 62.5 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11 12 13 14
from __future__ import print_function
import os
import sys
import glob
import time
15
import json
16
import logging
17
import datetime
18 19 20
import platform
import binascii
import argparse
21 22
import textwrap
import functools
23
import subprocess
24
import multiprocessing
25
import xml.etree.ElementTree as ET
26 27 28

# TODO:
# - add docker provider
29
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
30
# - add CCACHE support
31 32 33
# - improve building from tarball
# - improve native-pkg builds
# - avoid using network if possible (e.g. check first if pkgs are installed)
34 35 36 37 38


SYSTEMS = {
    'fedora': ['27', '28', '29'],
    'centos': ['7'],
39 40
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
41
    'debian': ['8', '9'],
42
    'freebsd': ['11.2', '12.0'],
43 44
}

45
# pylint: disable=C0326
46
IMAGE_TEMPLATES = {
47 48 49 50 51 52 53 54
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
    'fedora-28-lxc':           {'bare': 'lxc-fedora-28',               'kea': 'godfryd/kea-fedora-28'},
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
55
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
56 57 58 59 60 61 62 63 64 65 66 67
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
68 69 70 71 72 73
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
74
  config.vm.hostname = "{name}"
75 76

  config.vm.box = "{image_tpl}"
77 78 79

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
80
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
81 82 83
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
84 85 86 87 88 89 90
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
91
  config.vm.hostname = "{name}"
92 93 94 95

  config.vm.box = "{image_tpl}"

  config.vm.provider "virtualbox" do |v|
96
    v.name = "{name}"
97 98 99 100 101 102 103 104 105 106
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
107 108

  config.vm.synced_folder '.', '/vagrant', disabled: true
109 110 111 112 113 114 115
end
"""


log = logging.getLogger()


116 117 118
def red(txt):
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
119
    return txt
120 121 122 123

def green(txt):
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
124
    return txt
125 126 127 128

def blue(txt):
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
129
    return txt
130 131


132
def get_system_revision():
133
    """Return tuple containing system name and its revision."""
134 135
    system = platform.system()
    if system == 'Linux':
136
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
137
        if system == 'debian':
138
            revision = revision[0]
139 140
        elif system == 'redhat':
            system = 'rhel'
141 142 143
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
144 145 146 147 148 149
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


150 151 152
class ExecutionError(Exception):
    pass

153

154
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None, quiet=False, check_times=False, capture=False,
155
            interactive=False, attempts=1, sleep_time_after_attempt=None):
156 157 158 159 160 161 162 163 164 165 166 167 168
    """Execute a command in shell.

    :param str cmd: a command to be executed
    :param int timeout: timeout in number of seconds, after that time the command is terminated but only if check_times is True
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
    :param bool raise_error: if False then in case of error exception is not raised, default: True ie exception is raise
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled, used for e.g. SSH
169 170
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
171
    """
172
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
173 174 175 176
    if not check_times:
        timeout = None
    if dry_run:
        return 0
177

178 179 180
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
181

182 183
    if log_file_path:
        log_file = open(log_file_path, "wb")
184

185 186 187 188
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
189

190 191
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
192

193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
                    line_decoded = line.decode(errors='ignore').rstrip() + '\r'
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
                p.terminate()
                raise ExecutionError('Execution timeout')
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
230

231
    if exitcode != 0 and raise_error:
232
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
233 234 235

    if capture:
        return exitcode, output
236
    return exitcode
237 238


239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
def install_pkgs(pkgs, timeout=60, env=None, check_times=False):
    system, revision = get_system_revision()

    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'

256 257
    if isinstance(pkgs, list):
        pkgs = ' '.join(pkgs)
258 259 260 261

    cmd += ' ' + pkgs

    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
262 263


264
class VagrantEnv(object):
265 266 267 268 269 270 271
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

272 273
    def __init__(self, provider, system, sys_revision, features, image_template_variant, dry_run, quiet=False, check_times=False):
        self.provider = provider
274 275 276
        self.system = system
        self.sys_revision = sys_revision
        self.features = features
277 278 279
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
280

281 282 283 284 285
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

286 287 288 289 290 291 292 293 294 295
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

        image_tpl = IMAGE_TEMPLATES["%s-%s-%s" % (system, sys_revision, provider)][image_template_variant]
        self.repo_dir = os.getcwd()

        sys_dir = "%s-%s" % (system, sys_revision)
        if provider == "virtualbox":
296
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
297
        elif provider == "lxc":
298 299 300 301
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
302

303 304
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
305

306
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
307 308 309 310 311

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

312 313 314 315 316 317
        crc = binascii.crc32(self.vagrant_dir.encode())
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, sys_revision.replace('.', '-'), crc)

        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
                                             name=self.name)

318 319 320
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

321 322
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

323
    def up(self):
324 325
        execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run)
        execute("vagrant up --no-provision --provider %s" % self.provider, cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
326 327

    def package(self):
328 329
        """Package Vagrant system into Vagrant box."""

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345
        if self.provider == 'virtualbox':
            cmd = "vagrant package --output kea-%s-%s.box" % (self.system, self.sys_revision)
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.sys_revision))
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
346 347 348 349 350 351 352
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+'
                    'kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo'
                    '3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2M'
                    'WZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEg'
                    'E98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" > %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "cd %s && tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*"'
            execute(cmd % (lxc_container_path, lxc_box_dir))
353 354 355 356 357 358 359 360 361 362 363 364
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
365

366
    def upload(self, src):
367
        """Upload src to Vagrant system, home folder."""
368 369 370 371 372 373 374 375 376 377 378
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

379
    def run_build_and_test(self, tarball_path, jobs):
380
        """Run build and unit tests inside Vagrant system."""
381 382 383
        if self.dry_run:
            return 0, 0

384
        # prepare tarball if needed and upload it to vagrant system
385 386
        if not tarball_path:
            name_ver = 'kea-1.5.0'
387 388 389 390
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
            cmd += ' --exclude "*~" --exclude .git --exclude .libs --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
391
            tarball_path = '/tmp/%s.tar.gz' % name_ver
392
        self.upload(tarball_path)
393 394 395

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
396 397

        t0 = time.time()
398

399
        # run build command
400
        bld_cmd = "%s hammer.py build -p local -t %s.tar.gz -j %d" % (self.python, name_ver, jobs)
401 402 403 404 405 406
        if self.features_arg:
            bld_cmd += ' ' + self.features_arg
        if self.nofeatures_arg:
            bld_cmd += ' ' + self.nofeatures_arg
        if self.check_times:
            bld_cmd += ' -i'
407 408
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
409 410 411 412 413 414

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
            execute('scp -F %s -r default:/home/vagrant/rpm-root/RPMS/x86_64/ .' % ssh_cfg_path)

415 416
        t1 = time.time()
        dt = int(t1 - t0)
417 418

        log.info('Build log file stored to %s', log_file_path)
419 420 421 422
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

423
        # run unit tests if requested
424 425 426 427 428 429 430 431 432 433 434 435
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
                execute('scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path, cwd=self.vagrant_dir)
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
436
        except:  # pylint: disable=bare-except
437 438 439 440
            log.exception('ignored issue with parsing unit test results')

        return total, passed

441 442
    def destroy(self):
        cmd = 'vagrant destroy --force'
443
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
444 445

    def ssh(self):
446
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
447 448

    def dump_ssh_config(self):
449
        """Dump ssh config that allows getting into Vagrant system via SSH."""
450 451 452 453 454
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
455
        """Execute provided command inside Vagrant system."""
456 457 458
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
459

460 461
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout, raise_error=raise_error,
                       dry_run=self.dry_run, log_file_path=log_file_path, quiet=quiet, check_times=self.check_times)
462

463
    def prepare_system(self):
464
        """Prepare Vagrant system for building Kea."""
465 466
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
467 468 469
        else:
            self.features_arg = ''

470
        nofeatures = set(DEFAULT_FEATURES) - self.features
471 472 473 474 475
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

476
        # select proper python version for running Hammer inside Vagrant system
477
        if self.system == 'centos' and self.sys_revision == '7' or (self.system == 'debian' and self.sys_revision == '8' and self.provider != 'lxc'):
478 479 480 481 482 483
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

484
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
485 486 487
        if self.system == 'rhel' and self.sys_revision == '8':
            exitcode = self.execute("sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms", raise_error=False)
            if exitcode != 0:
488 489 490 491 492
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
493 494 495 496 497
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

498
        # upload Hammer to Vagrant system
499
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
500
        self.upload(hmr_py_path)
501

502 503 504
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

505
        # run prepare-system inside Vagrant system
506
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times}"
507 508
        cmd = cmd.format(features=self.features_arg,
                         nofeatures=self.nofeatures_arg,
509 510 511
                         python=self.python,
                         check_times='-i' if self.check_times else '')
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
512 513 514


def _install_gtest_sources():
515
    # download gtest sources only if it is not present as native package
516 517
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
        execute('wget --no-verbose -O /tmp/gtest.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz')
518
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
519 520 521
        os.unlink('/tmp/gtest.tar.gz')


522
def _configure_mysql(system, revision, features):
523 524 525 526
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
527 528 529 530 531

    if system == 'freebsd':
        cmd = "echo 'SET PASSWORD = \"\";' | sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
        execute(cmd, raise_error=False)

532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

547 548 549 550 551 552 553 554 555 556 557 558
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

559 560 561 562 563 564 565 566 567 568 569 570 571
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

572

573
def _configure_pgsql(system, features):
574 575
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
576
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
577 578 579 580 581 582 583 584 585 586 587 588 589 590 591
        if exitcode != 0:
            execute('sudo postgresql-setup --initdb --unit postgresql')
    execute('sudo systemctl start postgresql.service')
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

592 593 594 595 596 597 598 599 600 601
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)

602

603
def _install_cassandra_deb(env, check_times):
604
    if not os.path.exists('/usr/sbin/cassandra'):
605 606
        execute('echo "deb http://www.apache.org/dist/cassandra/debian 311x main" | sudo tee /etc/apt/sources.list.d/cassandra.sources.list',
                env=env, check_times=check_times)
607
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -', env=env, check_times=check_times)
608
        execute('sudo apt update', env=env, check_times=check_times)
609
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
610 611

    if not os.path.exists('/usr/include/cassandra.h'):
612 613 614 615 616 617
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb', env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb', env=env, check_times=check_times)
618 619


620
def _install_freeradius_client(env, check_times):
621
    execute('rm -rf freeradius-client')
622 623 624 625 626 627
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
    execute('./configure', cwd='freeradius-client', env=env, check_times=check_times)
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
628 629 630
    execute('rm -rf freeradius-client')


631
def _install_cassandra_rpm(system, env, check_times):
632 633 634
    if not os.path.exists('/usr/bin/cassandra'):
        #execute('sudo dnf config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/')
        #execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
635
        install_pkgs('cassandra cassandra-server libuv libuv-devel', env=env, check_times=check_times)
636 637 638 639 640 641 642 643 644 645

    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


646
def prepare_system_local(features, check_times):
647
    """Prepare local system for Kea development based on requested features."""
648 649 650
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

651 652 653
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

654
    # prepare fedora
655
    if system == 'fedora':
656
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel']
657 658 659 660

        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'mariadb-connector-c-devel'])

661 662 663 664 665 666 667 668 669
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'community-mysql-devel'])

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

670 671 672
        if 'radius' in features:
            packages.extend(['git'])

673
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
674 675 676 677

        if 'unittest' in features:
            _install_gtest_sources()

678
        execute('sudo dnf clean packages', env=env, check_times=check_times)
679

680
        if 'cql' in features:
681
            _install_cassandra_rpm(system, env, check_times)
682

683
    # prepare centos
684
    elif system == 'centos':
685
        install_pkgs('epel-release', env=env, check_times=check_times)
686 687 688 689 690

        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']

        if 'docs' in features:
691 692 693 694
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
695

696 697 698
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

699 700 701
        if 'radius' in features:
            packages.extend(['git'])

702
        install_pkgs(packages, env=env, check_times=check_times)
703 704 705 706

        if 'unittest' in features:
            _install_gtest_sources()

707
        if 'cql' in features:
708
            _install_cassandra_rpm(system, env, check_times)
709

710
    # prepare rhel
711
    elif system == 'rhel':
712 713 714 715
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

716 717
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
718

719
        # TODO:
720 721 722 723 724 725
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

726 727 728
        if 'radius' in features:
            packages.extend(['git'])

729
        install_pkgs(packages, env=env, timeout=120, check_times=check_times)
730 731 732

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
733 734 735 736
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
737 738
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
739
                    check_times=check_times)
740
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm', env=env, timeout=120, check_times=check_times)
741 742
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times)
743 744 745 746

        if 'unittest' in features:
            _install_gtest_sources()

747
        if 'cql' in features:
748
            _install_cassandra_rpm(system, env, check_times)
749

750
    # prepare ubuntu
751
    elif system == 'ubuntu':
752
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
753 754 755 756

        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev']

        if 'unittest' in features:
757 758 759 760
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
761 762

        if 'docs' in features:
763
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
764 765 766 767 768 769

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
            packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev', 'docbook', 'docbook-xsl', 'flex', 'libboost-dev',
                             'libpq-dev', 'postgresql-server-dev-all', 'python3-dev'])

770
        if 'mysql' in features:
771 772 773 774
            if revision == '16.04':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
775 776 777 778

        if 'pgsql' in features:
            packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

779 780 781
        if 'radius' in features:
            packages.extend(['git'])

782
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
783 784

        if 'cql' in features:
785
            _install_cassandra_deb(env, check_times)
786

787
    # prepare debian
788
    elif system == 'debian':
789
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
790 791 792 793

        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev']

        if 'docs' in features:
794
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
795 796 797

        if 'unittest' in features:
            if revision == '8':
798
                # libgtest-dev does not work and googletest is not available
799
                _install_gtest_sources()
800 801 802
            else:
                packages.append('googletest')

803
        if 'mysql' in features:
804 805 806 807
            if revision == '8':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
808

809 810 811
        if 'radius' in features:
            packages.extend(['git'])

812
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
813

814 815
        if 'cql' in features and revision != '8':
            # there is no libuv1 package in case of debian 8
816
            _install_cassandra_deb(env, check_times)
817

818
    # prepare freebsd
819 820 821
    elif system == 'freebsd':
        packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs']

822
        # TODO:
823 824 825 826 827 828
        #execute('sudo portsnap --interactive fetch', timeout=240, check_times=check_times)
        #execute('sudo portsnap extract /usr/ports/devel/log4cplus', timeout=240, check_times=check_times)
        #execute('sudo make -C /usr/ports/devel/log4cplus install clean BATCH=yes', timeout=240, check_times=check_times)

        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-xsl'])
829 830 831 832

        if 'unittest' in features:
            _install_gtest_sources()

833 834 835 836 837 838
        if 'mysql' in features:
            packages.extend(['mysql57-server', 'mysql57-client'])

        if 'radius' in features:
            packages.extend(['git'])

839
        install_pkgs(packages, env=env, timeout=6 * 60, check_times=check_times)
840

841 842 843 844
        if 'mysql' in features:
            execute('sudo sysrc mysql_enable="yes"', env=env, check_times=check_times)
            execute('sudo service mysql-server start', env=env, check_times=check_times, raise_error=False)

845 846 847
    else:
        raise NotImplementedError

848
    if 'mysql' in features:
849
        _configure_mysql(system, revision, features)
850

851
    if 'pgsql' in features:
852
        _configure_pgsql(system, features)
853

854
    if 'radius' in features:
855
        _install_freeradius_client(env, check_times)
856 857 858

    #execute('sudo rm -rf /usr/share/doc')

859 860
    log.info('Preparing deps completed successfully.')

861

862 863 864 865 866 867 868 869 870
def prepare_system_in_vagrant(provider, system, sys_revision, features, dry_run, check_times, clean_start):
    """Prepare specified system in Vagrant according to specified features."""
    ve = VagrantEnv(provider, system, sys_revision, features, 'kea', dry_run, check_times=check_times)
    if clean_start:
        ve.destroy()
    ve.up()
    ve.prepare_system()


871 872 873 874 875 876 877 878
def _calculate_build_timeout(features):
    timeout = 60
    if 'mysql' in features:
        timeout += 60
    timeout *= 60
    return timeout


879 880 881
def _build_just_binaries(distro, revision, features, tarball_path, env, check_times, jobs, dry_run):
    if tarball_path:
        # unpack tarball with sources
882
        execute('sudo rm -rf kea-src')
883 884 885 886 887 888 889 890 891 892 893 894 895 896
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
        src_path = glob.glob('kea-src/*')[0]
    else:
        src_path = '.'

    execute('autoreconf -f -i', cwd=src_path, env=env, dry_run=dry_run)

    # prepare switches for ./configure
    cmd = './configure'
    if 'mysql' in features:
        cmd += ' --with-mysql'
    if 'pgsql' in features:
        cmd += ' --with-pgsql'
897 898
    if 'cql' in features and not (system == 'debian' and revision == '8'):
        # debian 8 does not have all deps required
899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934
        cmd += ' --with-cql=/usr/bin/pkg-config'
    if 'unittest' in features:
        # prepare gtest switch - use downloaded gtest sources only if it is not present as native package
        if distro in ['centos', 'fedora', 'rhel', 'freebsd']:
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
        elif distro == 'debian' and revision == '8':
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
        elif distro == 'debian':
            cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
        elif distro == 'ubuntu':
            if revision.startswith('16.'):
                cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
            else:
                cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
        else:
            raise NotImplementedError
    if 'docs' in features and not (distro == 'rhel' and revision == '8'):
        cmd += ' --enable-generate-docs'
    if 'radius' in features:
        cmd += ' --with-freeradius=/usr/local'
    if 'shell' in features:
        cmd += ' --enable-shell'

    # do ./configure
    execute(cmd, cwd=src_path, env=env, timeout=120, check_times=check_times, dry_run=dry_run)

    # estimate number of processes (jobs) to use in compilation if jobs are not provided
    if jobs == 0:
        cpus = multiprocessing.cpu_count() - 1
        if distro == 'centos':
            cpus = cpus // 2
        if cpus == 0:
            cpus = 1
    else:
        cpus = jobs

935

936
    # do build
937 938 939 940 941 942
    timeout = _calculate_build_timeout(features)
    if 'distcheck' in features:
        cmd = 'make distcheck'
    else:
        cmd = 'make -j%s' % cpus
    execute(cmd, cwd=src_path, env=env, timeout=timeout, check_times=check_times, dry_run=dry_run)
943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009

    if 'unittest' in features:
        results_dir = os.path.abspath(os.path.join(src_path, 'tests_result'))
        execute('rm -rf %s' % results_dir, dry_run=dry_run)
        if not os.path.exists(results_dir):
            os.mkdir(results_dir)
        env['GTEST_OUTPUT'] = 'xml:%s/' % results_dir
        env['KEA_SOCKET_TEST_DIR'] = '/tmp/'
        # run unit tests
        execute('make check -k', cwd=src_path, env=env, timeout=60 * 60, raise_error=False, check_times=check_times, dry_run=dry_run)

        # parse unit tests results
        results = {}
        grand_total = 0
        grand_not_passed = 0
        for fn in os.listdir(results_dir):
            if not fn.endswith('.xml'):
                continue
            fp = os.path.join(results_dir, fn)
            tree = ET.parse(fp)
            root = tree.getroot()
            total = int(root.get('tests'))
            failures = int(root.get('failures'))
            disabled = int(root.get('disabled'))
            errors = int(root.get('errors'))
            results[fn] = dict(total=total, failures=failures, disabled=disabled, errors=errors)
            grand_total += total
            grand_not_passed += failures + errors

        grand_passed = grand_total - grand_not_passed
        results['grand_passed'] = grand_total - grand_not_passed
        results['grand_total'] = grand_total

        result = '%s/%s passed' % (grand_passed, grand_total)
        if grand_not_passed > 0 or grand_total == 0:
            result = red(result)
        else:
            result = green(result)
        log.info('Unit test results: %s', result)

        with open('unit-test-results.json', 'w') as f:
            f.write(json.dumps(results))

    if 'install' in features:
        execute('sudo make install', cwd=src_path, env=env, check_times=check_times, dry_run=dry_run)
        execute('sudo ldconfig', dry_run=dry_run)  # TODO: this shouldn't be needed

        if 'forge' in features:
            if 'mysql' in features:
                execute('kea-admin lease-init mysql -u keauser -p keapass -n keadb', dry_run=dry_run)
            if 'pgsql' in features:
                execute('kea-admin lease-init pgsql -u keauser -p keapass -n keadb', dry_run=dry_run)


def _build_native_pkg(distro, features, tarball_path, env, check_times, dry_run):
    if distro in ['fedora', 'centos', 'rhel']:
        # prepare RPM environment
        execute('rm -rf rpm-root', dry_run=dry_run)
        os.mkdir('rpm-root')
        os.mkdir('rpm-root/BUILD')
        os.mkdir('rpm-root/BUILDROOT')
        os.mkdir('rpm-root/RPMS')
        os.mkdir('rpm-root/SOURCES')
        os.mkdir('rpm-root/SPECS')
        os.mkdir('rpm-root/SRPMS')

        # get rpm.spec from tarball
1010
        execute('sudo rm -rf kea-src', dry_run=dry_run)
1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]
        rpm_dir = os.path.join(src_path, 'rpm')
        for f in os.listdir(rpm_dir):
            if f == 'kea.spec':
                continue
            execute('cp %s rpm-root/SOURCES' % os.path.join(rpm_dir, f), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SPECS' % os.path.join(rpm_dir, 'kea.spec'), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SOURCES' % tarball_path, check_times=check_times, dry_run=dry_run)

        # do rpm build
        cmd = "rpmbuild -ba rpm-root/SPECS/kea.spec -D'_topdir /home/vagrant/rpm-root'"
        execute(cmd, env=env, timeout=60 * 40, check_times=check_times, dry_run=dry_run)

        if 'install' in features:
            execute('sudo rpm -i rpm-root/RPMS/x86_64/*rpm', check_times=check_times, dry_run=dry_run)

    elif distro in ['ubuntu', 'debian']:
        # unpack tarball
1031
        execute('sudo rm -rf kea-src', check_times=check_times, dry_run=dry_run)
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]

        # do deb build
        execute('debuild -i -us -uc -b', env=env, cwd=src_path, timeout=60 * 40, check_times=check_times, dry_run=dry_run)

        if 'install' in features:
            execute('sudo dpkg -i kea-src/*deb', check_times=check_times, dry_run=dry_run)

    else:
        raise NotImplementedError


1046
def build_local(features, tarball_path, check_times, jobs, dry_run):
1047 1048 1049 1050 1051
    """Prepare local system for Kea development based on requested features.

    If tarball_path is provided then instead of Kea sources from current directory
    use provided tarball.
    """
1052 1053 1054 1055 1056
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

    distro, revision = get_system_revision()

1057
    execute('df -h', dry_run=dry_run)
1058

1059 1060
    if tarball_path:
        tarball_path = os.path.abspath(tarball_path)
1061 1062 1063

    if 'native-pkg' in features:
        # native pkg build
1064
        _build_native_pkg(distro, features, tarball_path, env, check_times, dry_run)
1065 1066
    else:
        # build straight from sources
1067
        _build_just_binaries(distro, revision, features, tarball_path, env, check_times, jobs, dry_run)
1068

1069
    execute('df -h', dry_run=dry_run)
1070 1071


1072
def build_in_vagrant(provider, system, sys_revision, features, leave_system, tarball_path, dry_run, quiet, clean_start, check_times, jobs):
1073
    """Build Kea via Vagrant in specified system with specified features."""
1074
    log.info('')
1075
    log.info(">>> Building %s, %s, %s", provider, system, sys_revision)
1076 1077 1078 1079
    log.info('')

    t0 = time.time()

1080
    ve = None
1081 1082 1083
    error = None
    total = 0
    passed = 0
1084
    try:
1085 1086
        ve = VagrantEnv(provider, system, sys_revision, features, 'kea', dry_run, quiet, check_times)
        if clean_start:
1087
            ve.destroy()
1088
        ve.up()
1089
        ve.prepare_system()
1090
        total, passed = ve.run_build_and_test(tarball_path, jobs)
1091 1092 1093 1094 1095 1096 1097
        msg = ' - ' + green('all ok')
    except KeyboardInterrupt as e:
        error = e
        msg = ' - keyboard interrupt'
    except ExecutionError as e:
        error = e
        msg = ' - ' + red(str(e))
1098
    except Exception as e:  # pylint: disable=broad-except
1099 1100 1101 1102
        log.exception('Building erred')
        error = e
        msg = ' - ' + red(str(e))
    finally:
1103
        if not leave_system and ve:
1104
            ve.destroy()
1105 1106 1107 1108 1109

    t1 = time.time()
    dt = int(t1 - t0)

    log.info('')
1110
    log.info(">>> Building %s, %s, %s completed in %s:%s%s", provider, system, sys_revision, dt // 60, dt % 60, msg)
1111 1112
    log.info('')

1113
    return dt, error, total, passed
1114 1115


1116
def package_box(provider, system, sys_revision, features, dry_run, check_times):
1117
    """Prepare Vagrant box of specified system."""
1118
    ve = VagrantEnv(provider, system, sys_revision, features, 'bare', dry_run, check_times=check_times)
1119
    ve.destroy()
1120
    ve.up()
1121
    ve.prepare_system()
1122
    # TODO cleanup
1123 1124 1125
    ve.package()


1126 1127
def ssh(provider, system, sys_revision):
    ve = VagrantEnv(provider, system, sys_revision, [], 'kea', False)
1128 1129 1130 1131
    ve.up()
    ve.ssh()


1132
def ensure_hammer_deps():
1133
    """Install Hammer dependencies onto current, host system."""
1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
    distro, _ = get_system_revision()

    exitcode = execute('vagrant version', raise_error=False)
    if exitcode != 0:
        if distro in ['fedora', 'centos', 'rhel']:
            execute('wget --no-verbose -O /tmp/vagrant_2.2.2_x86_64.rpm https://releases.hashicorp.com/vagrant/2.2.2/vagrant_2.2.2_x86_64.rpm')
            execute('sudo rpm -i /tmp/vagrant_2.2.2_x86_64.rpm')
            os.unlink('/tmp/vagrant_2.2.2_x86_64.rpm')
        elif distro in ['debian', 'ubuntu']:
            execute('wget --no-verbose -O /tmp/vagrant_2.2.2_x86_64.deb https://releases.hashicorp.com/vagrant/2.2.2/vagrant_2.2.2_x86_64.deb')
            execute('sudo dpkg -i /tmp/vagrant_2.2.2_x86_64.deb')
            os.unlink('/tmp/vagrant_2.2.2_x86_64.deb')
        else:
            # TODO: check for packages here: https://www.vagrantup.com/downloads.html
            raise NotImplementedError

    exitcode = execute('vagrant plugin list | grep vagrant-lxc', raise_error=False)
    if exitcode != 0:
        execute('vagrant plugin install vagrant-lxc')


1155
class CollectCommaSeparatedArgsAction(argparse.Action):
1156 1157
    """Helper argparse action class that can split multi-argument options by space and by comma."""

1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170
    def __call__(self, parser, namespace, values, option_string=None):
        values2 = []
        for v1 in values:
            for v2 in v1.split():
                values2.extend(v2.split(','))

        for v in values2:
            if v not in ALL_FEATURES:
                raise argparse.ArgumentError(self, "feature '%s' is not supported. List of supported features: %s." % (v, ", ".join(ALL_FEATURES)))

        setattr(namespace, self.dest, values2)


1171
DEFAULT_FEATURES = ['install', 'unittest', 'docs']
1172
ALL_FEATURES = ['install', 'distcheck', 'unittest', 'docs', 'mysql', 'pgsql', 'cql', 'native-pkg', 'radius', 'shell', 'forge']
1173

1174

1175
def parse_args():
1176 1177
    fl = functools.partial(lambda w, t: textwrap.fill(t, w), 80)  # used lambda to change args order and able to substitute width
    description = [
1178
        "Hammer - Kea development environment management tool.\n",
1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207
        fl("At first it is required to install Hammer dependencies which is Vagrant and either "
           "VirtualBox or LXC. To make life easier Hammer can install Vagrant and required "
           "Vagrant plugins using the command:"),
        "\n  ./hammer.py ensure-hammer-deps\n",
        "Still VirtualBox and LXC need to be installed manually.",
        fl("Basic functionality provided by Hammer is preparing building environment and "
           "performing actual build and running unit tests locally, in current system. "
           "This can be achieved by running the command:"),
        "\n  ./hammer.py build -p local\n",
        fl("The scope of the process can be defined using --with (-w) and --without (-x) options. "
           "By default the build command will build Kea with documentation, install it locally "
           "and run unit tests."),
        "To exclude installation and generating docs do:",
        "\n  ./hammer.py build -p local -x install docs\n",
        fl("The whole list of available features is: %s." % ", ".join(ALL_FEATURES)),
        fl("Hammer can be told to set up a new virtual machine with specified operating system "
           "and not running the build:"),
        "\n  ./hammer.py prepare-system -p virtualbox -s freebsd -r 12.0\n",
        fl("This way we can prepare a system for our own use. To get to such system using SSH invoke:"),
        "\n  ./hammer.py ssh -p virtualbox -s freebsd -r 12.0\n",
        "To list all created system on a host invoke:",
        "\n  ./hammer.py created-systems\n",
        "And then to destroy a given system run:",
        "\n  ./hammer.py destroy -d /path/to/dir/with/Vagrantfile\n",
    ]
    description = "\n".join(description)
    main_parser = argparse.ArgumentParser(description=description,
                                          formatter_class=argparse.RawDescriptionHelpFormatter)

1208 1209 1210 1211 1212
    main_parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose mode.')
    main_parser.add_argument('-q', '--quiet', action='store_true', help='Enable quiet mode.')

    subparsers = main_parser.add_subparsers(dest='command',
                                            title="Hammer commands",
1213 1214
                                            description=fl("The following commands are provided by Hammer. "
                                                           "To get more information about particular command invoke: ./hammer.py <command> -h."))
1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228

    parent_parser1 = argparse.ArgumentParser(add_help=False)
    parent_parser1.add_argument('-p', '--provider', default='virtualbox', choices=['lxc', 'virtualbox', 'local', 'all'],
                                help="Backend build executor. If 'all' then build is executed several times on all providers. "
                                "If 'local' then build is executed on current system. Default is 'virtualbox'.")
    parent_parser1.add_argument(