hammer.py 59.8 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7 8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

9 10 11 12 13
from __future__ import print_function
import os
import sys
import glob
import time
14
import json
15
import logging
16
import datetime
17 18 19
import platform
import binascii
import argparse
20 21
import textwrap
import functools
22
import subprocess
23
import multiprocessing
24
import xml.etree.ElementTree as ET
25 26 27

# TODO:
# - add docker provider
28
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
29
# - add CCACHE support
30 31 32
# - improve building from tarball
# - improve native-pkg builds
# - avoid using network if possible (e.g. check first if pkgs are installed)
33 34 35 36 37


SYSTEMS = {
    'fedora': ['27', '28', '29'],
    'centos': ['7'],
38 39
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
40
    'debian': ['8', '9'],
41
    'freebsd': ['11.2', '12.0'],
42 43 44
}

IMAGE_TEMPLATES = {
45 46 47 48 49 50 51 52
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
    'fedora-28-lxc':           {'bare': 'lxc-fedora-28',               'kea': 'godfryd/kea-fedora-28'},
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
53
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
54 55 56 57 58 59 60 61 62 63 64 65
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
66 67 68 69 70 71
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
72
  config.vm.hostname = "{name}"
73 74

  config.vm.box = "{image_tpl}"
75 76 77

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
78
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
79 80 81
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
82 83 84 85 86 87 88
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
89
  config.vm.hostname = "{name}"
90 91 92 93

  config.vm.box = "{image_tpl}"

  config.vm.provider "virtualbox" do |v|
94
    v.name = "{name}"
95 96 97 98 99 100 101 102 103 104
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
105 106

  config.vm.synced_folder '.', '/vagrant', disabled: true
107 108 109 110 111 112 113
end
"""


log = logging.getLogger()


114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132
def red(txt):
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
    else:
        return txt

def green(txt):
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
    else:
        return txt

def blue(txt):
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
    else:
        return txt


133
def get_system_revision():
134
    """Return tuple containing system name and its revision."""
135 136 137 138 139 140
    system = platform.system()
    if system == 'Linux':
        system, revision, _ = platform.dist()
        if system == 'debian':
            if revision.startswith('8.'):
                revision = '8'
141 142
            if revision.startswith('9.'):
                revision = '9'
143 144 145 146 147 148 149 150 151 152
        elif system == 'redhat':
            system = 'rhel'
            if revision.startswith('8.'):
                revision = '8'
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


153 154
class ExecutionError(Exception): pass

155 156
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None, quiet=False, check_times=False, capture=False,
            interactive=False):
157 158 159 160 161 162 163 164 165 166 167 168 169 170
    """Execute a command in shell.

    :param str cmd: a command to be executed
    :param int timeout: timeout in number of seconds, after that time the command is terminated but only if check_times is True
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
    :param bool raise_error: if False then in case of error exception is not raised, default: True ie exception is raise
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled, used for e.g. SSH
    """
171
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
172 173 174 175
    if not check_times:
        timeout = None
    if dry_run:
        return 0
176 177 178 179 180

    if interactive:
        p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
        ver = platform.python_version()
        exitcode = p.wait()
181

182 183 184 185
    else:
        if log_file_path:
            log_file = open(log_file_path, "wb")

186
        p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
187 188 189 190 191

        if capture:
            output = ''
        t0 = time.time()
        t1 = time.time()
192
        # repeat until process is running or timeout not occured
193 194 195 196 197 198 199 200 201
        while p.poll() is None and (timeout is None or t1 - t0 < timeout):
            line = p.stdout.readline()
            if line:
                line_decoded = line.decode(errors='ignore').rstrip() + '\r'
                if not quiet:
                    print(line_decoded)
                if capture:
                    output += line_decoded
                if log_file_path:
202
                    log_file.write(line)
203
            t1 = time.time()
204

205 206
        if log_file_path:
            log_file.close()
207

208 209
        # If no exitcode yet, ie. process is still running then it means that timeout occured.
        # In such case terminate the process and raise an exception.
210
        if p.poll() is None:
211
            p.terminate()
212 213
            raise ExecutionError('Execution timeout')
        exitcode = p.returncode
214

215
    if exitcode != 0 and raise_error:
216
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
217 218 219 220 221

    if capture:
        return exitcode, output
    else:
        return exitcode
222 223


224 225 226
def install_yum(pkgs, env=None, check_times=False):
    if isinstance(pkgs, list):
        pkgs = ' '.join(pkgs)
227 228
    # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
    # but we want an error
229 230 231 232
    cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False %s' % pkgs
    execute(cmd, env=env, check_times=check_times)


233
class VagrantEnv(object):
234 235 236 237 238 239 240
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

241 242
    def __init__(self, provider, system, sys_revision, features, image_template_variant, dry_run, quiet=False, check_times=False):
        self.provider = provider
243 244 245
        self.system = system
        self.sys_revision = sys_revision
        self.features = features
246 247 248
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
249 250 251 252 253 254 255 256 257 258 259

        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

        image_tpl = IMAGE_TEMPLATES["%s-%s-%s" % (system, sys_revision, provider)][image_template_variant]
        self.repo_dir = os.getcwd()

        sys_dir = "%s-%s" % (system, sys_revision)
        if provider == "virtualbox":
260
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
261
        elif provider == "lxc":
262 263 264 265
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
266

267 268
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
269

270
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
271 272 273 274 275

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

276 277 278 279 280 281
        crc = binascii.crc32(self.vagrant_dir.encode())
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, sys_revision.replace('.', '-'), crc)

        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
                                             name=self.name)

282 283 284
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

285 286
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

287
    def up(self):
288 289
        execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run)
        execute("vagrant up --no-provision --provider %s" % self.provider, cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
290 291

    def package(self):
292 293
        """Package Vagrant system into Vagrant box."""

294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
        if self.provider == 'virtualbox':
            cmd = "vagrant package --output kea-%s-%s.box" % (self.system, self.sys_revision)
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.sys_revision))
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" > %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            execute('sudo bash -c "cd %s && tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*"' % (lxc_container_path, lxc_box_dir))
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
324

325 326 327 328 329 330 331 332 333 334 335 336
    def upload(self, src):
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

337
    def run_build_and_test(self, tarball_path, jobs):
338
        """Run build and unit tests inside Vagrant system."""
339 340 341
        if self.dry_run:
            return 0, 0

342
        # prepare tarball if needed and upload it to vagrant system
343 344
        if not tarball_path:
            name_ver = 'kea-1.5.0'
345 346 347 348
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
            cmd += ' --exclude "*~" --exclude .git --exclude .libs --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
349
            tarball_path = '/tmp/%s.tar.gz' % name_ver
350
        self.upload(tarball_path)
351 352 353

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
354 355

        t0 = time.time()
356

357
        # run build command
358
        bld_cmd = "%s hammer.py build -p local -t %s.tar.gz -j %d" % (self.python, name_ver, jobs)
359 360 361 362 363 364 365 366 367 368 369 370 371
        if self.features_arg:
            bld_cmd += ' ' + self.features_arg
        if self.nofeatures_arg:
            bld_cmd += ' ' + self.nofeatures_arg
        if self.check_times:
            bld_cmd += ' -i'
        self.execute(bld_cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
            execute('scp -F %s -r default:/home/vagrant/rpm-root/RPMS/x86_64/ .' % ssh_cfg_path)

372 373
        t1 = time.time()
        dt = int(t1 - t0)
374 375

        log.info('Build log file stored to %s', log_file_path)
376 377 378 379
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

380
        # run unit tests if requested
381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
                execute('scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path, cwd=self.vagrant_dir)
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
        except:
            log.exception('ignored issue with parsing unit test results')

        return total, passed

398 399
    def destroy(self):
        cmd = 'vagrant destroy --force'
400
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
401 402

    def ssh(self):
403
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
404 405

    def dump_ssh_config(self):
406
        """Dump ssh config that allows getting into Vagrant system via SSH."""
407 408 409 410 411
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
412
        """Execute provided command inside Vagrant system."""
413 414 415
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
416

417 418
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout, raise_error=raise_error,
                       dry_run=self.dry_run, log_file_path=log_file_path, quiet=quiet, check_times=self.check_times)
419

420
    def prepare_system(self):
421
        """Prepare Vagrant system for building Kea."""
422 423
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
424 425 426
        else:
            self.features_arg = ''

427
        nofeatures = set(DEFAULT_FEATURES) - self.features
428 429 430 431 432
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

433
        # select proper python version for running Hammer inside Vagrant system
434
        if self.system == 'centos' and self.sys_revision == '7' or (self.system == 'debian' and self.sys_revision == '8' and self.provider != 'lxc'):
435 436 437 438 439 440
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

441
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
442 443 444
        if self.system == 'rhel' and self.sys_revision == '8':
            exitcode = self.execute("sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms", raise_error=False)
            if exitcode != 0:
445 446 447 448 449
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
450 451 452 453 454
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

455
        # upload Hammer to Vagrant system
456
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
457
        self.upload(hmr_py_path)
458

459 460 461
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

462
        # run prepare-system inside Vagrant system
463
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times}"
464 465
        cmd = cmd.format(features=self.features_arg,
                         nofeatures=self.nofeatures_arg,
466 467 468
                         python=self.python,
                         check_times='-i' if self.check_times else '')
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
469 470 471


def _install_gtest_sources():
472
    # download gtest sources only if it is not present as native package
473 474
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
        execute('wget --no-verbose -O /tmp/gtest.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz')
475
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
476 477 478
        os.unlink('/tmp/gtest.tar.gz')


479
def _configure_mysql(system, revision, features):
480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

499 500 501 502 503 504 505 506 507 508 509 510
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

511 512 513 514 515 516 517 518 519 520 521 522 523
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

524

525
def _configure_pgsql(system, features):
526 527
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
528
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
        if exitcode != 0:
            execute('sudo postgresql-setup --initdb --unit postgresql')
    execute('sudo systemctl start postgresql.service')
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

544 545 546 547 548 549 550 551 552 553
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)

554

555
def _install_cassandra_deb(env, check_times):
556
    if not os.path.exists('/usr/sbin/cassandra'):
557 558 559 560 561
        execute('echo "deb http://www.apache.org/dist/cassandra/debian 311x main" | sudo tee /etc/apt/sources.list.d/cassandra.sources.list',
                env=env, check_times=check_times)
        execute('curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -', env=env, check_times=check_times)
        execute('sudo apt update', env=env, check_times=check_times)
        execute('sudo apt install -y cassandra libuv1 pkgconf', env=env, check_times=check_times)
562 563

    if not os.path.exists('/usr/include/cassandra.h'):
564 565 566 567 568 569
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb', env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb', env=env, check_times=check_times)
570 571


572
def _install_freeradius_client(env, check_times):
573
    execute('rm -rf freeradius-client')
574 575 576 577 578 579
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
    execute('./configure', cwd='freeradius-client', env=env, check_times=check_times)
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
580 581 582
    execute('rm -rf freeradius-client')


583
def _install_cassandra_rpm(system, env, check_times):
584 585 586 587
    if not os.path.exists('/usr/bin/cassandra'):
        #execute('sudo dnf config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/')
        #execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
        if system == 'fedora':
588
            install_yum('cassandra cassandra-server libuv libuv-devel', env=env, check_times=check_times)
589 590 591 592 593 594 595 596 597 598 599 600 601
        #elif system == 'centos':
        else:
            raise NotImplementedError

    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


602
def prepare_system_local(features, check_times):
603
    """Prepare local system for Kea development based on requested features."""
604 605 606
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

607 608 609
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

610
    # prepare fedora
611
    if system == 'fedora':
612
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel']
613 614 615 616

        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'mariadb-connector-c-devel'])

617 618 619 620 621 622 623 624 625
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'community-mysql-devel'])

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

626 627 628
        if 'radius' in features:
            packages.extend(['git'])

629
        cmd = 'sudo dnf -y install %s' % ' '.join(packages)
Michal Nowikowski's avatar
Michal Nowikowski committed
630
        execute(cmd, env=env, timeout=300, check_times=check_times)
631 632 633 634

        if 'unittest' in features:
            _install_gtest_sources()

635
        execute('sudo dnf clean packages', env=env, check_times=check_times)
636

637
        if 'cql' in features:
638
            _install_cassandra_rpm(system, env, check_times)
639

640
    # prepare centos
641 642
    elif system == 'centos':
        install_yum('epel-release', env=env, check_times=check_times)
643 644 645 646 647

        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']

        if 'docs' in features:
648 649 650 651
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
652

653 654 655
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

656 657 658
        if 'radius' in features:
            packages.extend(['git'])

659
        install_yum(packages, env=env, check_times=check_times)
660 661 662 663

        if 'unittest' in features:
            _install_gtest_sources()

664
        if 'cql' in features:
665
            _install_cassandra_rpm(system, env, check_times)
666

667
    # prepare rhel
668
    elif system == 'rhel':
669 670 671 672
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

673 674
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
675

676
        # TODO:
677 678 679 680 681 682
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

683 684 685
        if 'radius' in features:
            packages.extend(['git'])

686
        install_cmd = 'sudo dnf -y install %s'
687
        execute(install_cmd % ' '.join(packages), env=env, timeout=120, check_times=check_times)
688 689 690

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
691 692 693 694 695 696
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    check_times=check_times)
697
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm', env=env, timeout=120, check_times=check_times)
698 699
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times)
700 701 702 703

        if 'unittest' in features:
            _install_gtest_sources()

704
        if 'cql' in features:
705
            _install_cassandra_rpm(system, env, check_times)
706

707
    # prepare ubuntu
708
    elif system == 'ubuntu':
709
        execute('sudo apt update', env=env, check_times=check_times)
710 711 712 713

        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev']

        if 'unittest' in features:
714 715 716 717
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
718 719

        if 'docs' in features:
720
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
721 722 723 724 725 726

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
            packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev', 'docbook', 'docbook-xsl', 'flex', 'libboost-dev',
                             'libpq-dev', 'postgresql-server-dev-all', 'python3-dev'])

727 728 729 730 731 732
        if 'mysql' in features:
            packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        if 'pgsql' in features:
            packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

733 734 735
        if 'radius' in features:
            packages.extend(['git'])

736 737 738 739 740 741 742 743 744 745
        done = False
        while not done:
            try:
                execute('sudo apt install --no-install-recommends -y %s' % ' '.join(packages), env=env, timeout=240, check_times=check_times)
                done = True
            except:
                log.exception('ble')
                time.sleep(20)

        if 'cql' in features:
746
            _install_cassandra_deb(env, check_times)
747

748
    # prepare debian
749
    elif system == 'debian':
750
        execute('sudo apt update', env=env, check_times=check_times)
751 752 753 754

        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev']

        if 'docs' in features:
755
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
756 757 758

        if 'unittest' in features:
            if revision == '8':
759
                # libgtest-dev does not work and googletest is not available
760
                _install_gtest_sources()
761 762 763
            else:
                packages.append('googletest')

764 765 766
        if 'mysql' in features:
            packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

767 768 769
        if 'radius' in features:
            packages.extend(['git'])

770 771 772
        execute('sudo apt install --no-install-recommends -y %s' % ' '.join(packages), env=env, timeout=240, check_times=check_times)

        if 'cql' in features:
773
            _install_cassandra_deb(env, check_times)
774

775
    # prepare freebsd
776 777 778
    elif system == 'freebsd':
        packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs']

779
        # TODO:
780 781 782 783 784 785
        #execute('sudo portsnap --interactive fetch', timeout=240, check_times=check_times)
        #execute('sudo portsnap extract /usr/ports/devel/log4cplus', timeout=240, check_times=check_times)
        #execute('sudo make -C /usr/ports/devel/log4cplus install clean BATCH=yes', timeout=240, check_times=check_times)

        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-xsl'])
786

787 788 789
        if 'radius' in features:
            packages.extend(['git'])

790 791 792
        if 'unittest' in features:
            _install_gtest_sources()

793 794
        execute('sudo pkg install -y %s' % ' '.join(packages), env=env, timeout=6 * 60, check_times=check_times)

795 796 797
    else:
        raise NotImplementedError

798
    if 'mysql' in features:
799
        _configure_mysql(system, revision, features)
800

801
    if 'pgsql' in features:
802
        _configure_pgsql(system, features)
803

804
    if 'radius' in features:
805
        _install_freeradius_client(env, check_times)
806 807 808

    #execute('sudo rm -rf /usr/share/doc')

809 810
    log.info('Preparing deps completed successfully.')

811

812 813 814 815 816 817 818 819 820
def prepare_system_in_vagrant(provider, system, sys_revision, features, dry_run, check_times, clean_start):
    """Prepare specified system in Vagrant according to specified features."""
    ve = VagrantEnv(provider, system, sys_revision, features, 'kea', dry_run, check_times=check_times)
    if clean_start:
        ve.destroy()
    ve.up()
    ve.prepare_system()


821
def build_local(features, tarball_path, check_times, jobs, dry_run):
822 823 824 825 826
    """Prepare local system for Kea development based on requested features.

    If tarball_path is provided then instead of Kea sources from current directory
    use provided tarball.
    """
827 828 829 830 831
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

    distro, revision = get_system_revision()

832
    execute('df -h', dry_run=dry_run)
833

834 835
    if tarball_path:
        tarball_path = os.path.abspath(tarball_path)
836 837 838 839 840

    if 'native-pkg' in features:
        # native pkg build

        if distro in ['fedora', 'centos', 'rhel']:
841
            # prepare RPM environment
842 843 844 845 846 847 848 849 850
            execute('rm -rf rpm-root')
            os.mkdir('rpm-root')
            os.mkdir('rpm-root/BUILD')
            os.mkdir('rpm-root/BUILDROOT')
            os.mkdir('rpm-root/RPMS')
            os.mkdir('rpm-root/SOURCES')
            os.mkdir('rpm-root/SPECS')
            os.mkdir('rpm-root/SRPMS')

851
            # get rpm.spec from tarball
852 853
            execute('rm -rf kea-src')
            os.mkdir('kea-src')
854
            execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
855 856 857 858 859
            src_path = glob.glob('kea-src/*')[0]
            rpm_dir = os.path.join(src_path, 'rpm')
            for f in os.listdir(rpm_dir):
                if f == 'kea.spec':
                    continue
860 861 862
                execute('cp %s rpm-root/SOURCES' % os.path.join(rpm_dir, f), check_times=check_times)
            execute('cp %s rpm-root/SPECS' % os.path.join(rpm_dir, 'kea.spec'), check_times=check_times)
            execute('cp %s rpm-root/SOURCES' % tarball_path, check_times=check_times)
863

864
            # do rpm build
865
            cmd = "rpmbuild -ba rpm-root/SPECS/kea.spec -D'_topdir /home/vagrant/rpm-root'"
866
            execute(cmd, env=env, timeout=60 * 40, check_times=check_times)
867 868

            if 'install' in features:
869
                execute('sudo rpm -i rpm-root/RPMS/x86_64/*rpm', check_times=check_times)
870 871

        elif distro in ['ubuntu', 'debian']:
872
            # unpack tarball
873
            execute('rm -rf kea-src', check_times=check_times)
874
            os.mkdir('kea-src')
875
            execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
876 877
            src_path = glob.glob('kea-src/*')[0]

878
            # do deb build
879
            execute('debuild -i -us -uc -b', env=env, cwd=src_path, timeout=60 * 40, check_times=check_times)
880 881

            if 'install' in features:
882
                execute('sudo dpkg -i kea-src/*deb', check_times=check_times)
883 884 885 886 887 888 889 890

        else:
            raise NotImplementedError

    else:
        # build straight from sources

        if tarball_path:
891
            # unpack tarball with sources
892 893
            execute('rm -rf kea-src')
            os.mkdir('kea-src')
894
            execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
895 896 897 898
            src_path = glob.glob('kea-src/*')[0]
        else:
            src_path = '.'

899
        execute('autoreconf -f -i', cwd=src_path, env=env, dry_run=dry_run)
900

901
        # prepare switches for ./configure
902 903 904 905 906
        cmd = './configure'
        if 'mysql' in features:
            cmd += ' --with-mysql'
        if 'pgsql' in features:
            cmd += ' --with-pgsql'
907 908
        if 'cql' in features:
            cmd += ' --with-cql=/usr/bin/pkg-config'
909
        if 'unittest' in features:
910
            # prepare gtest switch - use downloaded gtest sources only if it is not present as native package
911
            if distro in ['centos', 'fedora', 'rhel', 'freebsd']:
912 913
                cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
            elif distro == 'debian' and revision == '8':
914 915
                cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
            elif distro == 'debian':
916
                cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
917 918 919 920 921
            elif distro == 'ubuntu':
                if revision.startswith('16.'):
                    cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
                else:
                    cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
922 923
            else:
                raise NotImplementedError
924
        if 'docs' in features and not (distro == 'rhel' and revision == '8'):
925
            cmd += ' --enable-generate-docs'
926 927
        if 'radius' in features:
            cmd += ' --with-freeradius=/usr/local'
928 929
        if 'shell' in features:
            cmd += ' --enable-shell'
930

931
        # do ./configure
932
        execute(cmd, cwd=src_path, env=env, timeout=120, check_times=check_times, dry_run=dry_run)
933

934
        # estimate number of processes (jobs) to use in compilation if jobs are not provided
935 936 937 938 939 940 941 942
        if jobs == 0:
            cpus = multiprocessing.cpu_count() - 1
            if distro == 'centos':
                cpus = cpus // 2
            if cpus == 0:
                cpus = 1
        else:
            cpus = jobs
943 944

        # do build
945
        cmd = 'make -j%s' % cpus
946
        execute(cmd, cwd=src_path, env=env, timeout=40 * 60, check_times=check_times, dry_run=dry_run)
947 948

        if 'unittest' in features:
949
            results_dir = os.path.abspath(os.path.join(src_path, 'tests_result'))
950
            execute('rm -rf %s' % results_dir, dry_run=dry_run)
951 952 953 954
            if not os.path.exists(results_dir):
                os.mkdir(results_dir)
            env['GTEST_OUTPUT'] = 'xml:%s/' % results_dir
            env['KEA_SOCKET_TEST_DIR'] = '/tmp/'
955
            # run unit tests
956
            execute('make check -k', cwd=src_path, env=env, timeout=60 * 60, raise_error=False, check_times=check_times, dry_run=dry_run)
957

958
            # parse unit tests results
959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988
            results = {}
            grand_total = 0
            grand_not_passed = 0
            for fn in os.listdir(results_dir):
                if not fn.endswith('.xml'):
                    continue
                fp = os.path.join(results_dir, fn)
                tree = ET.parse(fp)
                root = tree.getroot()
                total = int(root.get('tests'))
                failures = int(root.get('failures'))
                disabled = int(root.get('disabled'))
                errors = int(root.get('errors'))
                results[fn] = dict(total=total, failures=failures, disabled=disabled, errors=errors)
                grand_total += total
                grand_not_passed += failures + errors

            grand_passed = grand_total - grand_not_passed
            results['grand_passed'] = grand_total - grand_not_passed
            results['grand_total'] = grand_total

            result = '%s/%s passed' % (grand_passed, grand_total)
            if grand_not_passed > 0 or grand_total == 0:
                result = red(result)
            else:
                result = green(result)
            log.info('Unit test results: %s', result)

            with open('unit-test-results.json', 'w') as f:
                f.write(json.dumps(results))
989 990

        if 'install' in features:
991 992 993 994 995 996 997 998
            execute('sudo make install', cwd=src_path, env=env, check_times=check_times, dry_run=dry_run)
            execute('sudo ldconfig', dry_run=dry_run)  # TODO: this shouldn't be needed

            if 'forge' in features:
                if 'mysql' in features:
                    execute('kea-admin lease-init mysql -u keauser -p keapass -n keadb', dry_run=dry_run)
                if 'pgsql' in features:
                    execute('kea-admin lease-init pgsql -u keauser -p keapass -n keadb', dry_run=dry_run)
999

1000
    execute('df -h', dry_run=dry_run)
1001 1002


1003
def build_in_vagrant(provider, system, sys_revision, features, leave_system, tarball_path, dry_run, quiet, clean_start, check_times, jobs):
1004
    """Build Kea via Vagrant in specified system with specified features."""
1005 1006 1007 1008 1009 1010
    log.info('')
    log.info(">>> Building %s, %s, %s" % (provider, system, sys_revision))
    log.info('')

    t0 = time.time()

1011
    ve = None
1012 1013 1014
    error = None
    total = 0
    passed = 0
1015
    try:
1016 1017
        ve = VagrantEnv(provider, system, sys_revision, features, 'kea', dry_run, quiet, check_times)
        if clean_start:
1018
            ve.destroy()
1019
        ve.up()
1020
        ve.prepare_system()
1021
        total, passed = ve.run_build_and_test(tarball_path, jobs)
1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
        msg = ' - ' + green('all ok')
    except KeyboardInterrupt as e:
        error = e
        msg = ' - keyboard interrupt'
    except ExecutionError as e:
        error = e
        msg = ' - ' + red(str(e))
    except Exception as e:
        log.exception('Building erred')
        error = e
        msg = ' - ' + red(str(e))
    finally:
1034
        if not leave_system and ve:
1035
            ve.destroy()
1036 1037 1038 1039 1040

    t1 = time.time()
    dt = int(t1 - t0)

    log.info('')
1041
    log.info(">>> Building %s, %s, %s completed in %s:%s%s", provider, system, sys_revision, dt // 60, dt % 60, msg)
1042 1043
    log.info('')

1044
    return dt, error, total, passed
1045 1046


1047
def package_box(provider, system, sys_revision, features, dry_run, check_times):
1048
    """Prepare Vagrant box of specified system."""
1049
    ve = VagrantEnv(provider, system, sys_revision, features, 'bare', dry_run, check_times=check_times)
1050
    ve.destroy()
1051
    ve.up()
1052
    ve.prepare_system()
1053
    # TODO cleanup
1054 1055 1056
    ve.package()


1057 1058
def ssh(provider, system, sys_revision):
    ve = VagrantEnv(provider, system, sys_revision, [], 'kea', False)
1059 1060 1061 1062
    ve.up()
    ve.ssh()


1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084
def ensure_hammer_deps():
    distro, _ = get_system_revision()

    exitcode = execute('vagrant version', raise_error=False)
    if exitcode != 0:
        if distro in ['fedora', 'centos', 'rhel']:
            execute('wget --no-verbose -O /tmp/vagrant_2.2.2_x86_64.rpm https://releases.hashicorp.com/vagrant/2.2.2/vagrant_2.2.2_x86_64.rpm')
            execute('sudo rpm -i /tmp/vagrant_2.2.2_x86_64.rpm')
            os.unlink('/tmp/vagrant_2.2.2_x86_64.rpm')
        elif distro in ['debian', 'ubuntu']:
            execute('wget --no-verbose -O /tmp/vagrant_2.2.2_x86_64.deb https://releases.hashicorp.com/vagrant/2.2.2/vagrant_2.2.2_x86_64.deb')
            execute('sudo dpkg -i /tmp/vagrant_2.2.2_x86_64.deb')
            os.unlink('/tmp/vagrant_2.2.2_x86_64.deb')
        else:
            # TODO: check for packages here: https://www.vagrantup.com/downloads.html
            raise NotImplementedError

    exitcode = execute('vagrant plugin list | grep vagrant-lxc', raise_error=False)
    if exitcode != 0:
        execute('vagrant plugin install vagrant-lxc')


1085
class CollectCommaSeparatedArgsAction(argparse.Action):
1086 1087
    """Helper argparse action class that can split multi-argument options by space and by comma."""

1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100
    def __call__(self, parser, namespace, values, option_string=None):
        values2 = []
        for v1 in values:
            for v2 in v1.split():
                values2.extend(v2.split(','))

        for v in values2:
            if v not in ALL_FEATURES:
                raise argparse.ArgumentError(self, "feature '%s' is not supported. List of supported features: %s." % (v, ", ".join(ALL_FEATURES)))

        setattr(namespace, self.dest, values2)


1101
DEFAULT_FEATURES = ['install', 'unittest', 'docs']
1102
ALL_FEATURES = ['install', 'unittest', 'docs', 'mysql', 'pgsql', 'cql', 'native-pkg', 'radius', 'shell', 'forge']
1103 1104

def parse_args():
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
    fl = functools.partial(lambda w, t: textwrap.fill(t, w), 80)  # used lambda to change args order and able to substitute width
    description = [
        "Hammer - Kea develepment environment management tool.\n",
        fl("At first it is required to install Hammer dependencies which is Vagrant and either "
           "VirtualBox or LXC. To make life easier Hammer can install Vagrant and required "
           "Vagrant plugins using the command:"),
        "\n  ./hammer.py ensure-hammer-deps\n",
        "Still VirtualBox and LXC need to be installed manually.",
        fl("Basic functionality provided by Hammer is preparing building environment and "
           "performing actual build and running unit tests locally, in current system. "
           "This can be achieved by running the command:"),
        "\n  ./hammer.py build -p local\n",
        fl("The scope of the process can be defined using --with (-w) and --without (-x) options. "
           "By default the build command will build Kea with documentation, install it locally "
           "and run unit tests."),
        "To exclude installation and generating docs do:",
        "\n  ./hammer.py build -p local -x install docs\n",
        fl("The whole list of available features is: %s." % ", ".join(ALL_FEATURES)),
        fl("Hammer can be told to set up a new virtual machine with specified operating system "
           "and not running the build:"),
        "\n  ./hammer.py prepare-system -p virtualbox -s freebsd -r 12.0\n",
        fl("This way we can prepare a system for our own use. To get to such system using SSH invoke:"),
        "\n  ./hammer.py ssh -p virtualbox -s freebsd -r 12.0\n",
        "To list all created system on a host invoke:",
        "\n  ./hammer.py created-systems\n",
        "And then to destroy a given system run:",
        "\n  ./hammer.py destroy -d /path/to/dir/with/Vagrantfile\n",
    ]
    description = "\n".join(description)
    main_parser = argparse.ArgumentParser(description=description,
                                          formatter_class=argparse.RawDescriptionHelpFormatter)

1137 1138 1139 1140 1141
    main_parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose mode.')
    main_parser.add_argument('-q', '--quiet', action='store_true', help='Enable quiet mode.')

    subparsers = main_parser.add_subparsers(dest='command',
                                            title="Hammer commands",
1142 1143
                                            description=fl("The following commands are provided by Hammer. "
                                                           "To get more information about particular command invoke: ./hammer.py <command> -h."))
1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157

    parent_parser1 = argparse.ArgumentParser(add_help=False)
    parent_parser1.add_argument('-p', '--provider', default='virtualbox', choices=['lxc', 'virtualbox', 'local', 'all'],
                                help="Backend build executor. If 'all' then build is executed several times on all providers. "
                                "If 'local' then build is executed on current system. Default is 'virtualbox'.")
    parent_parser1.add_argument('-s', '--system', default='all', choices=list(SYSTEMS.keys()) + ['all'],
                                help="Build is executed on selected system. If 'all' then build is executed several times on all systems. "
                                "If provider is 'local' then this option is ignored. Default is 'all'.")
    parent_parser1.add_argument('-r', '--revision', default='all',
                                help="Revision of selected system. If 'all' then build is executed several times "
                                "on all revisions of selected system. To list supported systems and their revisions invoke 'supported-systems'. "
                                "Default is 'all'.")

    parent_parser2 = argparse.ArgumentParser(add_help=False)
1158 1159 1160 1161 1162
    parent_parser2.add_argument('-w', '--with', metavar='FEATURE', nargs='+', default=set(), action=CollectCommaSeparatedArgsAction,
                                help="Enable features. Separate them by space or comma. List of available features: %s. Default is '%s'." % (", ".join(ALL_FEATURES),
                                                                                                                                             ' '.join(DEFAULT_FEATURES)))
    parent_parser2.add_argument('-x', '--without', metavar='FEATURE', nargs='+', default=set(), action=CollectCommaSeparatedArgsAction,
                                help="Disable features. Separate them by space or comma. List of available features: %s. Default is ''." % ", ".join(ALL_FEATURES))
1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
    parent_parser2.add_argument('-l', '--leave-system', action='store_true',
                                help='At the end of the command do not destroy vagrant system. Default behavior is destroing the system.')
    parent_parser2.add_argument('-c', '--clean-start', action='store_true', help='If there is pre-existing system then it is destroyed first.')
    parent_parser2.add_argument('-i', '--check-times', action='store_true', help='Do not allow executing commands infinitelly.')
    parent_parser2.add_argument('-n', '--dry-run', action='store_true', help='Print only what would be done.')


    parser = subparsers.add_parser('ensure-hammer-deps', help="Install Hammer dependencies on current, host system.")
    parser = subparsers.add_parser('supported-systems', help="List system supported by Hammer for doing Kea development.")
    parser = subparsers.add_parser('build', help="Prepare system and run Kea build in indicated system.",
                                   parents=[parent_parser1, parent_parser2])
    parser.add_argument('-j', '--jobs', default=0, help='Number of processes used in compilation. Override make -j default value.')
1175
    parser.add_argument('-t', '--from-tarball', metavar='TARBALL_PATH',
1176
                        help='Instead of building sources in current folder use provided tarball package (e.g. tar.gz).')
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196
    parser = subparsers.add_parser('prepare-system',
                                   help="Prepare system for doing Kea development i.e. install all required dependencies "
                                   "and pre-configure the system. build command always first calls prepare-system internally.",
                                   parents=[parent_parser1, parent_parser2])
    parser = subparsers.add_parser('ssh', help="SSH to indicated system.",
                                   formatter_class=argparse.RawDescriptionHelpFormatter,
                                   description="Allows getting into the system using SSH. If the system is not present then it will be created first "
                                   "but not prepared. The command can be run in 2 way: "
                                   "\n1) ./hammer.py ssh -p <provider> -s <system> -r <revision>\n2) ./hammer.py ssh -d <path-to-vagrant-dir>",
                                   parents=[parent_parser1])
    parser.add_argument('-d', '--directory', help='Path to directory with Vagrantfile.')
    parser = subparsers.add_parser('created-systems', help="List ALL systems created by Hammer.")
    parser = subparsers.add_parser('destroy', help="Destroy indicated system.",