hammer.py 60.1 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11 12 13 14
from __future__ import print_function
import os
import sys
import glob
import time
15
import json
16
import logging
17
import datetime
18 19 20
import platform
import binascii
import argparse
21 22
import textwrap
import functools
23
import subprocess
24
import multiprocessing
25
import xml.etree.ElementTree as ET
26 27 28

# TODO:
# - add docker provider
29
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
30
# - add CCACHE support
31 32 33
# - improve building from tarball
# - improve native-pkg builds
# - avoid using network if possible (e.g. check first if pkgs are installed)
34 35 36 37 38


SYSTEMS = {
    'fedora': ['27', '28', '29'],
    'centos': ['7'],
39 40
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
41
    'debian': ['8', '9'],
42
    'freebsd': ['11.2', '12.0'],
43 44
}

45
# pylint: disable=C0326
46
IMAGE_TEMPLATES = {
47 48 49 50 51 52 53 54
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
    'fedora-28-lxc':           {'bare': 'lxc-fedora-28',               'kea': 'godfryd/kea-fedora-28'},
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
55
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
56 57 58 59 60 61 62 63 64 65 66 67
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
68 69 70 71 72 73
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
74
  config.vm.hostname = "{name}"
75 76

  config.vm.box = "{image_tpl}"
77 78 79

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
80
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
81 82 83
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
84 85 86 87 88 89 90
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
91
  config.vm.hostname = "{name}"
92 93 94 95

  config.vm.box = "{image_tpl}"

  config.vm.provider "virtualbox" do |v|
96
    v.name = "{name}"
97 98 99 100 101 102 103 104 105 106
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
107 108

  config.vm.synced_folder '.', '/vagrant', disabled: true
109 110 111 112 113 114 115
end
"""


log = logging.getLogger()


116 117 118
def red(txt):
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
119
    return txt
120 121 122 123

def green(txt):
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
124
    return txt
125 126 127 128

def blue(txt):
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
129
    return txt
130 131


132
def get_system_revision():
133
    """Return tuple containing system name and its revision."""
134 135
    system = platform.system()
    if system == 'Linux':
136
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
137 138 139
        if system == 'debian':
            if revision.startswith('8.'):
                revision = '8'
140 141
            if revision.startswith('9.'):
                revision = '9'
142 143 144 145 146 147 148 149 150 151
        elif system == 'redhat':
            system = 'rhel'
            if revision.startswith('8.'):
                revision = '8'
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


152 153 154
class ExecutionError(Exception):
    pass

155

156 157
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None, quiet=False, check_times=False, capture=False,
            interactive=False):
158 159 160 161 162 163 164 165 166 167 168 169 170 171
    """Execute a command in shell.

    :param str cmd: a command to be executed
    :param int timeout: timeout in number of seconds, after that time the command is terminated but only if check_times is True
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
    :param bool raise_error: if False then in case of error exception is not raised, default: True ie exception is raise
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled, used for e.g. SSH
    """
172
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
173 174 175 176
    if not check_times:
        timeout = None
    if dry_run:
        return 0
177 178 179 180

    if interactive:
        p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
        exitcode = p.wait()
181

182 183 184 185
    else:
        if log_file_path:
            log_file = open(log_file_path, "wb")

186
        p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
187 188 189 190 191

        if capture:
            output = ''
        t0 = time.time()
        t1 = time.time()
192
        # repeat until process is running or timeout not occured
193 194 195 196 197 198 199 200 201
        while p.poll() is None and (timeout is None or t1 - t0 < timeout):
            line = p.stdout.readline()
            if line:
                line_decoded = line.decode(errors='ignore').rstrip() + '\r'
                if not quiet:
                    print(line_decoded)
                if capture:
                    output += line_decoded
                if log_file_path:
202
                    log_file.write(line)
203
            t1 = time.time()
204

205 206
        if log_file_path:
            log_file.close()
207

208 209
        # If no exitcode yet, ie. process is still running then it means that timeout occured.
        # In such case terminate the process and raise an exception.
210
        if p.poll() is None:
211
            p.terminate()
212 213
            raise ExecutionError('Execution timeout')
        exitcode = p.returncode
214

215
    if exitcode != 0 and raise_error:
216
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
217 218 219

    if capture:
        return exitcode, output
220
    return exitcode
221 222


223 224 225
def install_yum(pkgs, env=None, check_times=False):
    if isinstance(pkgs, list):
        pkgs = ' '.join(pkgs)
226 227
    # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
    # but we want an error
228 229 230 231
    cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False %s' % pkgs
    execute(cmd, env=env, check_times=check_times)


232
class VagrantEnv(object):
233 234 235 236 237 238 239
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

240 241
    def __init__(self, provider, system, sys_revision, features, image_template_variant, dry_run, quiet=False, check_times=False):
        self.provider = provider
242 243 244
        self.system = system
        self.sys_revision = sys_revision
        self.features = features
245 246 247
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
248

249 250 251 252 253
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

254 255 256 257 258 259 260 261 262 263
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

        image_tpl = IMAGE_TEMPLATES["%s-%s-%s" % (system, sys_revision, provider)][image_template_variant]
        self.repo_dir = os.getcwd()

        sys_dir = "%s-%s" % (system, sys_revision)
        if provider == "virtualbox":
264
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
265
        elif provider == "lxc":
266 267 268 269
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
270

271 272
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
273

274
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
275 276 277 278 279

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

280 281 282 283 284 285
        crc = binascii.crc32(self.vagrant_dir.encode())
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, sys_revision.replace('.', '-'), crc)

        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
                                             name=self.name)

286 287 288
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

289 290
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

291
    def up(self):
292 293
        execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run)
        execute("vagrant up --no-provision --provider %s" % self.provider, cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
294 295

    def package(self):
296 297
        """Package Vagrant system into Vagrant box."""

298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313
        if self.provider == 'virtualbox':
            cmd = "vagrant package --output kea-%s-%s.box" % (self.system, self.sys_revision)
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.sys_revision))
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
314 315 316 317 318 319 320
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+'
                    'kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo'
                    '3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2M'
                    'WZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEg'
                    'E98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" > %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "cd %s && tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*"'
            execute(cmd % (lxc_container_path, lxc_box_dir))
321 322 323 324 325 326 327 328 329 330 331 332
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
333

334
    def upload(self, src):
335
        """Upload src to Vagrant system, home folder."""
336 337 338 339 340 341 342 343 344 345 346
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

347
    def run_build_and_test(self, tarball_path, jobs):
348
        """Run build and unit tests inside Vagrant system."""
349 350 351
        if self.dry_run:
            return 0, 0

352
        # prepare tarball if needed and upload it to vagrant system
353 354
        if not tarball_path:
            name_ver = 'kea-1.5.0'
355 356 357 358
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
            cmd += ' --exclude "*~" --exclude .git --exclude .libs --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
359
            tarball_path = '/tmp/%s.tar.gz' % name_ver
360
        self.upload(tarball_path)
361 362 363

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
364 365

        t0 = time.time()
366

367
        # run build command
368
        bld_cmd = "%s hammer.py build -p local -t %s.tar.gz -j %d" % (self.python, name_ver, jobs)
369 370 371 372 373 374 375 376 377 378 379 380 381
        if self.features_arg:
            bld_cmd += ' ' + self.features_arg
        if self.nofeatures_arg:
            bld_cmd += ' ' + self.nofeatures_arg
        if self.check_times:
            bld_cmd += ' -i'
        self.execute(bld_cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
            execute('scp -F %s -r default:/home/vagrant/rpm-root/RPMS/x86_64/ .' % ssh_cfg_path)

382 383
        t1 = time.time()
        dt = int(t1 - t0)
384 385

        log.info('Build log file stored to %s', log_file_path)
386 387 388 389
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

390
        # run unit tests if requested
391 392 393 394 395 396 397 398 399 400 401 402
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
                execute('scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path, cwd=self.vagrant_dir)
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
403
        except:  # pylint: disable=bare-except
404 405 406 407
            log.exception('ignored issue with parsing unit test results')

        return total, passed

408 409
    def destroy(self):
        cmd = 'vagrant destroy --force'
410
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
411 412

    def ssh(self):
413
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
414 415

    def dump_ssh_config(self):
416
        """Dump ssh config that allows getting into Vagrant system via SSH."""
417 418 419 420 421
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
422
        """Execute provided command inside Vagrant system."""
423 424 425
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
426

427 428
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout, raise_error=raise_error,
                       dry_run=self.dry_run, log_file_path=log_file_path, quiet=quiet, check_times=self.check_times)
429

430
    def prepare_system(self):
431
        """Prepare Vagrant system for building Kea."""
432 433
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
434 435 436
        else:
            self.features_arg = ''

437
        nofeatures = set(DEFAULT_FEATURES) - self.features
438 439 440 441 442
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

443
        # select proper python version for running Hammer inside Vagrant system
444
        if self.system == 'centos' and self.sys_revision == '7' or (self.system == 'debian' and self.sys_revision == '8' and self.provider != 'lxc'):
445 446 447 448 449 450
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

451
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
452 453 454
        if self.system == 'rhel' and self.sys_revision == '8':
            exitcode = self.execute("sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms", raise_error=False)
            if exitcode != 0:
455 456 457 458 459
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
460 461 462 463 464
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

465
        # upload Hammer to Vagrant system
466
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
467
        self.upload(hmr_py_path)
468

469 470 471
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

472
        # run prepare-system inside Vagrant system
473
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times}"
474 475
        cmd = cmd.format(features=self.features_arg,
                         nofeatures=self.nofeatures_arg,
476 477 478
                         python=self.python,
                         check_times='-i' if self.check_times else '')
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
479 480 481


def _install_gtest_sources():
482
    # download gtest sources only if it is not present as native package
483 484
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
        execute('wget --no-verbose -O /tmp/gtest.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz')
485
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
486 487 488
        os.unlink('/tmp/gtest.tar.gz')


489
def _configure_mysql(system, revision, features):
490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

509 510 511 512 513 514 515 516 517 518 519 520
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

521 522 523 524 525 526 527 528 529 530 531 532 533
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

534

535
def _configure_pgsql(system, features):
536 537
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
538
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
539 540 541 542 543 544 545 546 547 548 549 550 551 552 553
        if exitcode != 0:
            execute('sudo postgresql-setup --initdb --unit postgresql')
    execute('sudo systemctl start postgresql.service')
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

554 555 556 557 558 559 560 561 562 563
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)

564

565
def _install_cassandra_deb(env, check_times):
566
    if not os.path.exists('/usr/sbin/cassandra'):
567 568 569 570 571
        execute('echo "deb http://www.apache.org/dist/cassandra/debian 311x main" | sudo tee /etc/apt/sources.list.d/cassandra.sources.list',
                env=env, check_times=check_times)
        execute('curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -', env=env, check_times=check_times)
        execute('sudo apt update', env=env, check_times=check_times)
        execute('sudo apt install -y cassandra libuv1 pkgconf', env=env, check_times=check_times)
572 573

    if not os.path.exists('/usr/include/cassandra.h'):
574 575 576 577 578 579
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb', env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb', env=env, check_times=check_times)
580 581


582
def _install_freeradius_client(env, check_times):
583
    execute('rm -rf freeradius-client')
584 585 586 587 588 589
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
    execute('./configure', cwd='freeradius-client', env=env, check_times=check_times)
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
590 591 592
    execute('rm -rf freeradius-client')


593
def _install_cassandra_rpm(system, env, check_times):
594 595 596 597
    if not os.path.exists('/usr/bin/cassandra'):
        #execute('sudo dnf config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/')
        #execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
        if system == 'fedora':
598
            install_yum('cassandra cassandra-server libuv libuv-devel', env=env, check_times=check_times)
599 600 601 602 603 604 605 606 607 608 609 610 611
        #elif system == 'centos':
        else:
            raise NotImplementedError

    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


612
def prepare_system_local(features, check_times):
613
    """Prepare local system for Kea development based on requested features."""
614 615 616
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

617 618 619
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

620
    # prepare fedora
621
    if system == 'fedora':
622
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel']
623 624 625 626

        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'mariadb-connector-c-devel'])

627 628 629 630 631 632 633 634 635
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'community-mysql-devel'])

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

636 637 638
        if 'radius' in features:
            packages.extend(['git'])

639
        cmd = 'sudo dnf -y install %s' % ' '.join(packages)
Michal Nowikowski's avatar
Michal Nowikowski committed
640
        execute(cmd, env=env, timeout=300, check_times=check_times)
641 642 643 644

        if 'unittest' in features:
            _install_gtest_sources()

645
        execute('sudo dnf clean packages', env=env, check_times=check_times)
646

647
        if 'cql' in features:
648
            _install_cassandra_rpm(system, env, check_times)
649

650
    # prepare centos
651 652
    elif system == 'centos':
        install_yum('epel-release', env=env, check_times=check_times)
653 654 655 656 657

        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']

        if 'docs' in features:
658 659 660 661
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
662

663 664 665
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

666 667 668
        if 'radius' in features:
            packages.extend(['git'])

669
        install_yum(packages, env=env, check_times=check_times)
670 671 672 673

        if 'unittest' in features:
            _install_gtest_sources()

674
        if 'cql' in features:
675
            _install_cassandra_rpm(system, env, check_times)
676

677
    # prepare rhel
678
    elif system == 'rhel':
679 680 681 682
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

683 684
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
685

686
        # TODO:
687 688 689 690 691 692
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

693 694 695
        if 'radius' in features:
            packages.extend(['git'])

696
        install_cmd = 'sudo dnf -y install %s'
697
        execute(install_cmd % ' '.join(packages), env=env, timeout=120, check_times=check_times)
698 699 700

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
701 702 703 704
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
705 706
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
707
                    check_times=check_times)
708
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm', env=env, timeout=120, check_times=check_times)
709 710
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times)
711 712 713 714

        if 'unittest' in features:
            _install_gtest_sources()

715
        if 'cql' in features:
716
            _install_cassandra_rpm(system, env, check_times)
717

718
    # prepare ubuntu
719
    elif system == 'ubuntu':
720
        execute('sudo apt update', env=env, check_times=check_times)
721 722 723 724

        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev']

        if 'unittest' in features:
725 726 727 728
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
729 730

        if 'docs' in features:
731
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
732 733 734 735 736 737

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
            packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev', 'docbook', 'docbook-xsl', 'flex', 'libboost-dev',
                             'libpq-dev', 'postgresql-server-dev-all', 'python3-dev'])

738 739 740 741 742 743
        if 'mysql' in features:
            packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        if 'pgsql' in features:
            packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

744 745 746
        if 'radius' in features:
            packages.extend(['git'])

747 748 749 750 751
        done = False
        while not done:
            try:
                execute('sudo apt install --no-install-recommends -y %s' % ' '.join(packages), env=env, timeout=240, check_times=check_times)
                done = True
752
            except:  # pylint: disable=bare-except
753 754 755 756
                log.exception('ble')
                time.sleep(20)

        if 'cql' in features:
757
            _install_cassandra_deb(env, check_times)
758

759
    # prepare debian
760
    elif system == 'debian':
761
        execute('sudo apt update', env=env, check_times=check_times)
762 763 764 765

        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev']

        if 'docs' in features:
766
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
767 768 769

        if 'unittest' in features:
            if revision == '8':
770
                # libgtest-dev does not work and googletest is not available
771
                _install_gtest_sources()
772 773 774
            else:
                packages.append('googletest')

775 776 777
        if 'mysql' in features:
            packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

778 779 780
        if 'radius' in features:
            packages.extend(['git'])

781 782 783
        execute('sudo apt install --no-install-recommends -y %s' % ' '.join(packages), env=env, timeout=240, check_times=check_times)

        if 'cql' in features:
784
            _install_cassandra_deb(env, check_times)
785

786
    # prepare freebsd
787 788 789
    elif system == 'freebsd':
        packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs']

790
        # TODO:
791 792 793 794 795 796
        #execute('sudo portsnap --interactive fetch', timeout=240, check_times=check_times)
        #execute('sudo portsnap extract /usr/ports/devel/log4cplus', timeout=240, check_times=check_times)
        #execute('sudo make -C /usr/ports/devel/log4cplus install clean BATCH=yes', timeout=240, check_times=check_times)

        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-xsl'])
797

798 799 800
        if 'radius' in features:
            packages.extend(['git'])

801 802 803
        if 'unittest' in features:
            _install_gtest_sources()

804 805
        execute('sudo pkg install -y %s' % ' '.join(packages), env=env, timeout=6 * 60, check_times=check_times)

806 807 808
    else:
        raise NotImplementedError

809
    if 'mysql' in features:
810
        _configure_mysql(system, revision, features)
811

812
    if 'pgsql' in features:
813
        _configure_pgsql(system, features)
814

815
    if 'radius' in features:
816
        _install_freeradius_client(env, check_times)
817 818 819

    #execute('sudo rm -rf /usr/share/doc')

820 821
    log.info('Preparing deps completed successfully.')

822

823 824 825 826 827 828 829 830 831
def prepare_system_in_vagrant(provider, system, sys_revision, features, dry_run, check_times, clean_start):
    """Prepare specified system in Vagrant according to specified features."""
    ve = VagrantEnv(provider, system, sys_revision, features, 'kea', dry_run, check_times=check_times)
    if clean_start:
        ve.destroy()
    ve.up()
    ve.prepare_system()


832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992
def _build_just_binaries(distro, revision, features, tarball_path, env, check_times, jobs, dry_run):
    if tarball_path:
        # unpack tarball with sources
        execute('rm -rf kea-src')
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
        src_path = glob.glob('kea-src/*')[0]
    else:
        src_path = '.'

    execute('autoreconf -f -i', cwd=src_path, env=env, dry_run=dry_run)

    # prepare switches for ./configure
    cmd = './configure'
    if 'mysql' in features:
        cmd += ' --with-mysql'
    if 'pgsql' in features:
        cmd += ' --with-pgsql'
    if 'cql' in features:
        cmd += ' --with-cql=/usr/bin/pkg-config'
    if 'unittest' in features:
        # prepare gtest switch - use downloaded gtest sources only if it is not present as native package
        if distro in ['centos', 'fedora', 'rhel', 'freebsd']:
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
        elif distro == 'debian' and revision == '8':
            cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
        elif distro == 'debian':
            cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
        elif distro == 'ubuntu':
            if revision.startswith('16.'):
                cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
            else:
                cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
        else:
            raise NotImplementedError
    if 'docs' in features and not (distro == 'rhel' and revision == '8'):
        cmd += ' --enable-generate-docs'
    if 'radius' in features:
        cmd += ' --with-freeradius=/usr/local'
    if 'shell' in features:
        cmd += ' --enable-shell'

    # do ./configure
    execute(cmd, cwd=src_path, env=env, timeout=120, check_times=check_times, dry_run=dry_run)

    # estimate number of processes (jobs) to use in compilation if jobs are not provided
    if jobs == 0:
        cpus = multiprocessing.cpu_count() - 1
        if distro == 'centos':
            cpus = cpus // 2
        if cpus == 0:
            cpus = 1
    else:
        cpus = jobs

    # do build
    cmd = 'make -j%s' % cpus
    execute(cmd, cwd=src_path, env=env, timeout=40 * 60, check_times=check_times, dry_run=dry_run)

    if 'unittest' in features:
        results_dir = os.path.abspath(os.path.join(src_path, 'tests_result'))
        execute('rm -rf %s' % results_dir, dry_run=dry_run)
        if not os.path.exists(results_dir):
            os.mkdir(results_dir)
        env['GTEST_OUTPUT'] = 'xml:%s/' % results_dir
        env['KEA_SOCKET_TEST_DIR'] = '/tmp/'
        # run unit tests
        execute('make check -k', cwd=src_path, env=env, timeout=60 * 60, raise_error=False, check_times=check_times, dry_run=dry_run)

        # parse unit tests results
        results = {}
        grand_total = 0
        grand_not_passed = 0
        for fn in os.listdir(results_dir):
            if not fn.endswith('.xml'):
                continue
            fp = os.path.join(results_dir, fn)
            tree = ET.parse(fp)
            root = tree.getroot()
            total = int(root.get('tests'))
            failures = int(root.get('failures'))
            disabled = int(root.get('disabled'))
            errors = int(root.get('errors'))
            results[fn] = dict(total=total, failures=failures, disabled=disabled, errors=errors)
            grand_total += total
            grand_not_passed += failures + errors

        grand_passed = grand_total - grand_not_passed
        results['grand_passed'] = grand_total - grand_not_passed
        results['grand_total'] = grand_total

        result = '%s/%s passed' % (grand_passed, grand_total)
        if grand_not_passed > 0 or grand_total == 0:
            result = red(result)
        else:
            result = green(result)
        log.info('Unit test results: %s', result)

        with open('unit-test-results.json', 'w') as f:
            f.write(json.dumps(results))

    if 'install' in features:
        execute('sudo make install', cwd=src_path, env=env, check_times=check_times, dry_run=dry_run)
        execute('sudo ldconfig', dry_run=dry_run)  # TODO: this shouldn't be needed

        if 'forge' in features:
            if 'mysql' in features:
                execute('kea-admin lease-init mysql -u keauser -p keapass -n keadb', dry_run=dry_run)
            if 'pgsql' in features:
                execute('kea-admin lease-init pgsql -u keauser -p keapass -n keadb', dry_run=dry_run)


def _build_native_pkg(distro, features, tarball_path, env, check_times, dry_run):
    if distro in ['fedora', 'centos', 'rhel']:
        # prepare RPM environment
        execute('rm -rf rpm-root', dry_run=dry_run)
        os.mkdir('rpm-root')
        os.mkdir('rpm-root/BUILD')
        os.mkdir('rpm-root/BUILDROOT')
        os.mkdir('rpm-root/RPMS')
        os.mkdir('rpm-root/SOURCES')
        os.mkdir('rpm-root/SPECS')
        os.mkdir('rpm-root/SRPMS')

        # get rpm.spec from tarball
        execute('rm -rf kea-src', dry_run=dry_run)
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]
        rpm_dir = os.path.join(src_path, 'rpm')
        for f in os.listdir(rpm_dir):
            if f == 'kea.spec':
                continue
            execute('cp %s rpm-root/SOURCES' % os.path.join(rpm_dir, f), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SPECS' % os.path.join(rpm_dir, 'kea.spec'), check_times=check_times, dry_run=dry_run)
        execute('cp %s rpm-root/SOURCES' % tarball_path, check_times=check_times, dry_run=dry_run)

        # do rpm build
        cmd = "rpmbuild -ba rpm-root/SPECS/kea.spec -D'_topdir /home/vagrant/rpm-root'"
        execute(cmd, env=env, timeout=60 * 40, check_times=check_times, dry_run=dry_run)

        if 'install' in features:
            execute('sudo rpm -i rpm-root/RPMS/x86_64/*rpm', check_times=check_times, dry_run=dry_run)

    elif distro in ['ubuntu', 'debian']:
        # unpack tarball
        execute('rm -rf kea-src', check_times=check_times, dry_run=dry_run)
        os.mkdir('kea-src')
        execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times, dry_run=dry_run)
        src_path = glob.glob('kea-src/*')[0]

        # do deb build
        execute('debuild -i -us -uc -b', env=env, cwd=src_path, timeout=60 * 40, check_times=check_times, dry_run=dry_run)

        if 'install' in features:
            execute('sudo dpkg -i kea-src/*deb', check_times=check_times, dry_run=dry_run)

    else:
        raise NotImplementedError


993
def build_local(features, tarball_path, check_times, jobs, dry_run):
994 995 996 997 998
    """Prepare local system for Kea development based on requested features.

    If tarball_path is provided then instead of Kea sources from current directory
    use provided tarball.
    """
999 1000 1001 1002 1003
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

    distro, revision = get_system_revision()

1004
    execute('df -h', dry_run=dry_run)
1005

1006 1007
    if tarball_path:
        tarball_path = os.path.abspath(tarball_path)
1008 1009 1010

    if 'native-pkg' in features:
        # native pkg build
1011
        _build_native_pkg(distro, features, tarball_path, env, check_times, dry_run)
1012 1013
    else:
        # build straight from sources
1014
        _build_just_binaries(distro, revision, features, tarball_path, env, check_times, jobs, dry_run)
1015

1016
    execute('df -h', dry_run=dry_run)
1017 1018


1019
def build_in_vagrant(provider, system, sys_revision, features, leave_system, tarball_path, dry_run, quiet, clean_start, check_times, jobs):
1020
    """Build Kea via Vagrant in specified system with specified features."""
1021
    log.info('')
1022
    log.info(">>> Building %s, %s, %s", provider, system, sys_revision)
1023 1024 1025 1026
    log.info('')

    t0 = time.time()

1027
    ve = None
1028 1029 1030
    error = None
    total = 0
    passed = 0
1031
    try:
1032 1033
        ve = VagrantEnv(provider, system, sys_revision, features, 'kea', dry_run, quiet, check_times)
        if clean_start:
1034
            ve.destroy()
1035
        ve.up()
1036
        ve.prepare_system()
1037
        total, passed = ve.run_build_and_test(tarball_path, jobs)
1038 1039 1040 1041 1042 1043 1044
        msg = ' - ' + green('all ok')
    except KeyboardInterrupt as e:
        error = e
        msg = ' - keyboard interrupt'
    except ExecutionError as e:
        error = e
        msg = ' - ' + red(str(e))
1045
    except Exception as e:  # pylint: disable=broad-except
1046 1047 1048 1049
        log.exception('Building erred')
        error = e
        msg = ' - ' + red(str(e))
    finally:
1050
        if not leave_system and ve:
1051
            ve.destroy()
1052 1053 1054 1055 1056

    t1 = time.time()
    dt = int(t1 - t0)

    log.info('')
1057
    log.info(">>> Building %s, %s, %s completed in %s:%s%s", provider, system, sys_revision, dt // 60, dt % 60, msg)
1058 1059
    log.info('')

1060
    return dt, error, total, passed
1061 1062


1063
def package_box(provider, system, sys_revision, features, dry_run, check_times):
1064
    """Prepare Vagrant box of specified system."""
1065
    ve = VagrantEnv(provider, system, sys_revision, features, 'bare', dry_run, check_times=check_times)
1066
    ve.destroy()
1067
    ve.up()
1068
    ve.prepare_system()
1069
    # TODO cleanup
1070 1071 1072
    ve.package()


1073 1074
def ssh(provider, system, sys_revision):
    ve = VagrantEnv(provider, system, sys_revision, [], 'kea', False)
1075 1076 1077 1078
    ve.up()
    ve.ssh()


1079
def ensure_hammer_deps():
1080
    """Install Hammer dependencies onto current, host system."""
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
    distro, _ = get_system_revision()

    exitcode = execute('vagrant version', raise_error=False)
    if exitcode != 0:
        if distro in ['fedora', 'centos', 'rhel']:
            execute('wget --no-verbose -O /tmp/vagrant_2.2.2_x86_64.rpm https://releases.hashicorp.com/vagrant/2.2.2/vagrant_2.2.2_x86_64.rpm')
            execute('sudo rpm -i /tmp/vagrant_2.2.2_x86_64.rpm')
            os.unlink('/tmp/vagrant_2.2.2_x86_64.rpm')
        elif distro in ['debian', 'ubuntu']:
            execute('wget --no-verbose -O /tmp/vagrant_2.2.2_x86_64.deb https://releases.hashicorp.com/vagrant/2.2.2/vagrant_2.2.2_x86_64.deb')
            execute('sudo dpkg -i /tmp/vagrant_2.2.2_x86_64.deb')
            os.unlink('/tmp/vagrant_2.2.2_x86_64.deb')
        else:
            # TODO: check for packages here: https://www.vagrantup.com/downloads.html
            raise NotImplementedError

    exitcode = execute('vagrant plugin list | grep vagrant-lxc', raise_error=False)
    if exitcode != 0:
        execute('vagrant plugin install vagrant-lxc')


1102
class CollectCommaSeparatedArgsAction(argparse.Action):
1103 1104
    """Helper argparse action class that can split multi-argument options by space and by comma."""

1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117
    def __call__(self, parser, namespace, values, option_string=None):
        values2 = []
        for v1 in values:
            for v2 in v1.split():
                values2.extend(v2.split(','))

        for v in values2:
            if v not in ALL_FEATURES:
                raise argparse.ArgumentError(self, "feature '%s' is not supported. List of supported features: %s." % (v, ", ".join(ALL_FEATURES)))

        setattr(namespace, self.dest, values2)


1118
DEFAULT_FEATURES = ['install', 'unittest', 'docs']
1119
ALL_FEATURES = ['install', 'unittest', 'docs', 'mysql', 'pgsql', 'cql', 'native-pkg', 'radius', 'shell', 'forge']
1120

1121

1122
def parse_args():
1123 1124
    fl = functools.partial(lambda w, t: textwrap.fill(t, w), 80)  # used lambda to change args order and able to substitute width
    description = [
1125
        "Hammer - Kea development environment management tool.\n",
1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
        fl("At first it is required to install Hammer dependencies which is Vagrant and either "
           "VirtualBox or LXC. To make life easier Hammer can install Vagrant and required "
           "Vagrant plugins using the command:"),
        "\n  ./hammer.py ensure-hammer-deps\n",
        "Still VirtualBox and LXC need to be installed manually.",
        fl("Basic functionality provided by Hammer is preparing building environment and "
           "performing actual build and running unit tests locally, in current system. "
           "This can be achieved by running the command:"),
        "\n  ./hammer.py build -p local\n",
        fl("The scope of the process can be defined using --with (-w) and --without (-x) options. "
           "By default the build command will build Kea with documentation, install it locally "
           "and run unit tests."),
        "To exclude installation and generating docs do:",
        "\n  ./hammer.py build -p local -x install docs\n",
        fl("The whole list of available features is: %s." % ", ".join(ALL_FEATURES)),
        fl("Hammer can be told to set up a new virtual machine with specified operating system "
           "and not running the build:"),
        "\n  ./hammer.py prepare-system -p virtualbox -s freebsd -r 12.0\n",
        fl("This way we can prepare a system for our own use. To get to such system using SSH invoke:"),
        "\n  ./hammer.py ssh -p virtualbox -s freebsd -r 12.0\n",
        "To list all created system on a host invoke:",
        "\n  ./hammer.py created-systems\n",
        "And then to destroy a given system run:",
        "\n  ./hammer.py destroy -d /path/to/dir/with/Vagrantfile\n",
    ]
    description = "\n".join(description)
    main_parser = argparse.ArgumentParser(description=description,
                                          formatter_class=argparse.RawDescriptionHelpFormatter)

1155 1156 1157 1158 1159
    main_parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose mode.')
    main_parser.add_argument('-q', '--quiet', action='store_true', help='Enable quiet mode.')

    subparsers = main_parser.add_subparsers(dest='command',
                                            title="Hammer commands",
1160 1161
                                            description=fl("The following commands are provided by Hammer. "
                                                           "To get more information about particular command invoke: ./hammer.py <command> -h."))
1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175

    parent_parser1 = argparse.ArgumentParser(add_help=False)
    parent_parser1.add_argument('-p', '--provider', default='virtualbox', choices=['lxc', 'virtualbox', 'local', 'all'],
                                help="Backend build executor. If 'all' then build is executed several times on all providers. "
                                "If 'local' then build is executed on current system. Default is 'virtualbox'.")
    parent_parser1.add_argument('-s', '--system', default='all', choices=list(SYSTEMS.keys()) + ['all'],
                                help="Build is executed on selected system. If 'all' then build is executed several times on all systems. "
                                "If provider is 'local' then this option is ignored. Default is 'all'.")
    parent_parser1.add_argument('-r', '--revision', default='all',
                                help="Revision of selected system. If 'all' then build is executed several times "
                                "on all revisions of selected system. To list supported systems and their revisions invoke 'supported-systems'. "
                                "Default is 'all'.")

    parent_parser2 = argparse.ArgumentParser(add_help=False)
1176 1177 1178 1179 1180
    hlp = "Enable features. Separate them by space or comma. List of available features: %s. Default is '%s'."
    hlp = hlp % (", ".join(ALL_FEATURES), ' '.join(DEFAULT_FEATURES))
    parent_parser2.add_argument('-w', '--with', metavar='FEATURE', nargs='+', default=set(), action=CollectCommaSeparatedArgsAction, help=hlp)
    hlp = "Disable features. Separate them by space or comma. List of available features: %s. Default is ''." % ", ".join(ALL_FEATURES)
    parent_parser2.add_argument('-x', '--without', metavar='FEATURE', nargs='+', default=set(), action=CollectCommaSeparatedArgsAction, help=hlp)
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
    parent_parser2.add_argument('-l', '--leave-system', action='store_true',
                                help='At the end of the command do not destroy vagrant system. Default behavior is destroing the system.')
    parent_parser2.add_argument('-c', '--clean-start', action='store_true', help='If there is pre-existing system then it is destroyed first.')
    parent_parser2.add_argument('-i', '--check-times', action='store_true', help='Do not allow executing commands infinitelly.')
    parent_parser2.add_argument('-n', '--dry-run', action='store_true', help='Print only what would be done.')


    parser = subparsers.add_parser('ensure-hammer-deps', help="Install Hammer dependencies on current, host system.")
    parser = subparsers.add_parser('supported-systems', help="List system supported by Hammer for doing Kea development.")
    parser = subparsers.add_parser('build', help="Prepare system and run Kea build in indicated system.",
                                   parents=[parent_parser1, parent_parser2])
    parser.add_argument('-j', '--jobs', default=0, help='Number of processes used in compilation. Override make -j default value.')
1193
    parser.add_argument('-t', '--from-tarball', metavar='TARBALL_PATH',
1194
                        help='Instead of building sources in current folder use provided tarball package (e.g. tar.gz).')
1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210
    parser = subparsers.add_parser('prepare-system',
                                   help="Prepare system for doing Kea development i.e. install all required dependencies "
                                   "and pre-configure the system. build command always first calls prepare-system internally.",
                                   parents=[parent_parser1, parent_parser2])
    parser = subparsers.add_parser('ssh', help="SSH to indicated system.",
                                   formatter_class=argparse.RawDescriptionHelpFormatter,
                                   description="Allows getting into the system using SSH. If the system is not present then it will be created first "
                                   "but not prepared. The command can be run in 2 way: "
                                   "\n1) ./hammer.py ssh -p <provider> -s <system> -r <revision>\n2) ./hammer.py ssh -d <path-to-vagrant-dir>",
                                   parents=[parent_parser1])
    parser.add_argument('-d', '--directory', help='Path to directory with Vagrantfile.')
    parser = subparsers.add_parser('created-systems', help="List ALL systems created by Hammer.")
    parser = subparsers.add_parser('destroy', help="Destroy indicated system.",
                                   description="Destroys system indicated by a path to directory with Vagrantfile. "
                                   "To get the list of created systems run: ./hammer.py created-systems.")
    parser.add_argument('-d', '--directory', help='Path to directory with Vagrantfile.')
1211 1212
    parser = subparsers.add_parser('package-box',
                                   help="Package currently running system into Vagrant Box. Prepared box can be later deployed to Vagrant Cloud.",
1213 1214 1215
                                   parents=[parent_parser1, parent_parser2])

    args = main_parser.parse_args()
1216 1217 1218 1219

    return args


1220
def list_supported_systems():
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232
    for system, revisions in SYSTEMS.items():
        print('%s:' % system)
        for r in revisions:
            providers = []
            for p in ['lxc', 'virtualbox']:
                k = '%s-%s-%s' % (system, r, p)
                if k in IMAGE_TEMPLATES:
                    providers.append(p)
            providers = ', '.join(providers)
            print('  - %s: %s' % (r, providers))


1233
def list_created_systems():
1234
    _, output = execute('vagrant global-status', quiet=True, capture=True)
1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257
    systems = []
    for line in output.splitlines():
        if 'hammer' not in line:
            continue
        elems = line.split()
        state = elems[3]
        path = elems[4]
        systems.append([path, state])

    print('')
    print('%-10s %s' % ('State', 'Path'))
    print('-' * 80)
    for path, state, in sorted(systems):
        print('%-10s %s' % (state, path))
    print('-' * 80)
    print('To destroy a system run: ./hammer.py destroy -d <path>')
    print('')


def destroy_system(path):
    execute('vagrant destroy', cwd=path, interactive=True)