hammer.py 76.9 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11
from __future__ import print_function
import os
12
import re
13 14 15
import sys
import glob
import time
16
import json
17
import logging
18
import datetime
19 20 21
import platform
import binascii
import argparse
22 23
import textwrap
import functools
24
import subprocess
25
import multiprocessing
Michal Nowikowski's avatar
Michal Nowikowski committed
26 27 28 29
try:
    import urllib.request
except:
    pass
30
import xml.etree.ElementTree as ET
31 32 33

# TODO:
# - add docker provider
34 35 36 37
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
# - improve building from tarball
# - improve native-pkg builds
# - avoid using network if possible (e.g. check first if pkgs are installed)
38 39 40 41 42


SYSTEMS = {
    'fedora': ['27', '28', '29'],
    'centos': ['7'],
43 44
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
45
    'debian': ['8', '9'],
46
    'freebsd': ['11.2', '12.0'],
47 48
}

49
# pylint: disable=C0326
50
IMAGE_TEMPLATES = {
51 52 53 54 55 56 57 58
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
    'fedora-28-lxc':           {'bare': 'lxc-fedora-28',               'kea': 'godfryd/kea-fedora-28'},
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
59
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
60 61 62 63 64 65 66 67 68 69 70 71
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
72 73 74 75
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
76
ENV["LC_ALL"] = "C"
77 78

Vagrant.configure("2") do |config|
79
  config.vm.hostname = "{name}"
80 81

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
82
  {box_version}
83 84 85

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
86
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
87 88 89
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
90
  config.vm.synced_folder '{ccache_dir}', '/ccache'
91 92 93 94 95
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
96
ENV["LC_ALL"] = "C"
97 98

Vagrant.configure("2") do |config|
99
  config.vm.hostname = "{name}"
100 101

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
102
  {box_version}
103 104

  config.vm.provider "virtualbox" do |v|
105
    v.name = "{name}"
106 107 108 109 110 111 112 113 114 115
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
116 117

  config.vm.synced_folder '.', '/vagrant', disabled: true
118 119 120 121 122 123 124
end
"""


log = logging.getLogger()


125
def red(txt):
126
    """Return colorized (if the terminal supports it) or plain text."""
127 128
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
129
    return txt
130 131

def green(txt):
132
    """Return colorized (if the terminal supports it) or plain text."""
133 134
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
135
    return txt
136 137

def blue(txt):
138
    """Return colorized (if the terminal supports it) or plain text."""
139 140
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
141
    return txt
142 143


144
def get_system_revision():
145
    """Return tuple containing system name and its revision."""
146 147
    system = platform.system()
    if system == 'Linux':
148
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
149
        if system == 'debian':
150
            revision = revision[0]
151 152
        elif system == 'redhat':
            system = 'rhel'
153 154 155
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
156 157 158 159 160 161
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


162
class ExecutionError(Exception):
163
    """Exception thrown when execution encountered an error."""
164 165
    pass

166

167 168 169
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
            quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
            sleep_time_after_attempt=None):
170 171 172
    """Execute a command in shell.

    :param str cmd: a command to be executed
173 174
    :param int timeout: timeout in number of seconds, after that time the command is terminated
                        but only if check_times is True
175 176
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
177 178
    :param bool raise_error: if False then in case of error exception is not raised,
                             default: True ie exception is raise
179 180 181 182 183
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
184 185
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
                             used for e.g. SSH
186 187
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
188
    """
189
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
190 191 192 193
    if not check_times:
        timeout = None
    if dry_run:
        return 0
194

195 196 197
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
198

199 200
    if log_file_path:
        log_file = open(log_file_path, "wb")
201

202 203 204 205
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
206

207 208
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
209

210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
                    line_decoded = line.decode(errors='ignore').rstrip() + '\r'
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
230 231 232 233 234 235
                # kill using sudo to be able to kill other sudo commands
                execute('sudo kill -s TERM %s' % p.pid)
                time.sleep(5)
                # if still running, kill harder
                if p.poll() is None:
                    execute('sudo kill -s KILL %s' % p.pid)
236 237 238
                msg = "Execution timeout, %d > %d seconds elapsed (start: %d, stop %d), cmd: '%s'"
                msg = msg % (t1 - t0, timeout, t0, t1, cmd)
                raise ExecutionError(msg)
239 240 241 242 243 244 245 246 247 248 249 250 251 252 253
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
254

255
    if exitcode != 0 and raise_error:
Michal Nowikowski's avatar
Michal Nowikowski committed
256 257
        if capture and quiet:
            log.error(output)
258
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
259 260 261

    if capture:
        return exitcode, output
262
    return exitcode
263 264


Michal Nowikowski's avatar
Michal Nowikowski committed
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
def _prepare_installed_packages_cache_for_debs():
    pkg_cache = {}

    _, out = execute("dpkg -l", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        line = line.strip()
        m = re.search('^([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+(.+)', line)
        if not m:
            continue
        status, name, version, arch, descr = m.groups()
        name = name.split(':')[0]
        pkg_cache[name] = dict(status=status, version=version, arch=arch, descr=descr)

    return pkg_cache


def _prepare_installed_packages_cache_for_rpms():
    pkg_cache = {}

    _, out = execute("rpm -qa --qf '%{NAME}\\n'", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        name = line.strip()
        pkg_cache[name] = dict(status='ii')

    return pkg_cache


def install_pkgs(pkgs, timeout=60, env=None, check_times=False, pkg_cache={}):
295
    """Install native packages in a system.
296 297 298 299 300 301 302

    :param dict pkgs: specifies a list of packages to be installed
    :param int timeout: timeout in number of seconds, after that time the command
                        is terminated but only if check_times is True
    :param dict env: dictionary with environment variables (optional)
    :param bool check_times: specifies if timeouts should be enabled (optional)
    """
303 304
    system, revision = get_system_revision()

Michal Nowikowski's avatar
Michal Nowikowski committed
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325
    if not isinstance(pkgs, list):
        pkgs = pkgs.split()

    # prepare cache if needed
    if not pkg_cache and system in ['centos', 'rhel', 'fedora', 'debian', 'ubuntu']:
        if system in ['centos', 'rhel', 'fedora']:
            pkg_cache.update(_prepare_installed_packages_cache_for_rpms())
        elif system in ['debian', 'ubuntu']:
            pkg_cache.update(_prepare_installed_packages_cache_for_debs())

        # check if packages actually need to be installed
        pkgs_to_install = []
        for pkg in pkgs:
            if pkg not in pkg_cache or pkg_cache[pkg]['status'] != 'ii':
                pkgs_to_install.append(pkg)
        pkgs = pkgs_to_install

    if not pkgs:
        log.info('all packages already installed')
        return

326 327 328 329 330 331 332
    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
Michal Nowikowski's avatar
Michal Nowikowski committed
333
        # prepare the command for ubuntu/debian
334 335 336 337 338 339
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'
Michal Nowikowski's avatar
Michal Nowikowski committed
340 341
    else:
        raise NotImplementedError
342

Michal Nowikowski's avatar
Michal Nowikowski committed
343
    pkgs = ' '.join(pkgs)
344 345
    cmd += ' ' + pkgs
    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
346 347


348
class VagrantEnv(object):
349 350 351 352 353 354 355
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

356
    def __init__(self, provider, system, revision, features, image_template_variant,
357
                 dry_run, quiet=False, check_times=False, ccache_dir=None):
358 359 360 361 362 363 364 365 366 367 368
        """VagrantEnv initializer.

        :param str provider: indicate backend type: virtualbox or lxc
        :param str system: name of the system eg. ubuntu
        :param str revision: revision of the system e.g. 18.04
        :param list features: list of requested features
        :param str image_template_variant: variant of images' templates: bare or kea
        :param bool dry_run: if False then system commands are not really executed
        :param bool quiet: if True then commands will not trace to stdout
        :param bool check_times: if True then commands will be terminated after given timeout
        """
369
        self.provider = provider
370
        self.system = system
371
        self.revision = revision
372
        self.features = features
373 374 375
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
376

377 378 379 380 381
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

382 383 384 385 386
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

Michal Nowikowski's avatar
Michal Nowikowski committed
387 388
        self.key = key = "%s-%s-%s" % (system, revision, provider)
        self.image_tpl = image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
389 390
        self.repo_dir = os.getcwd()

391
        sys_dir = "%s-%s" % (system, revision)
392
        if provider == "virtualbox":
393
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
394
        elif provider == "lxc":
395 396 397 398
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
399

400 401
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
402

403
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
404 405 406 407 408

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

409
        crc = binascii.crc32(self.vagrant_dir.encode())
410
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
411

412 413 414 415 416 417
        if ccache_dir is None:
            ccache_dir = '/'
            self.ccache_enabled = False
        else:
            self.ccache_enabled = True

Michal Nowikowski's avatar
Michal Nowikowski committed
418 419 420 421 422 423 424
        if '/' in image_tpl:
            self.latest_version = self._get_latest_cloud_version()
            box_version = 'config.vm.box_version = "%s"' % self.latest_version
        else:
            self.latest_version = None
            box_version = ""

425
        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
426
                                             name=self.name,
Michal Nowikowski's avatar
Michal Nowikowski committed
427 428
                                             ccache_dir=ccache_dir,
                                             box_version=box_version)
429

430 431 432
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

433 434
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

435
    def up(self):
436 437 438
        """Do Vagrant up."""
        execute("vagrant up --no-provision --provider %s" % self.provider,
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
439

Michal Nowikowski's avatar
Michal Nowikowski committed
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501
    def _get_cloud_meta(self, image_tpl=None):
        if '/' not in self.image_tpl:
            return {}
        url = 'https://app.vagrantup.com/api/v1/box/' + (image_tpl if image_tpl else self.image_tpl)
        try:
            with urllib.request.urlopen(url) as response:
                data = response.read()
        except:
            log.exception('ignored exception')
            return {}
        data = json.loads(data)
        return data

    def _get_local_meta(self):
        meta_file = os.path.join(self.vagrant_dir, '.vagrant/machines/default', self.provider, 'box_meta')
        if not os.path.exists(meta_file):
            return {}
        with open(meta_file) as f:
            data = f.read()
        data = json.loads(data)
        return data

    def _get_latest_cloud_version(self, image_tpl=None):
        cloud_meta = self._get_cloud_meta(image_tpl)
        if not cloud_meta and 'versions' not in cloud_meta:
            return 0
        latest_version = 0
        for ver in cloud_meta['versions']:
            provider_found = False
            for p in ver['providers']:
                if p['name'] == self.provider:
                    provider_found = True
                    break
            if provider_found:
                v = int(ver['number'])
                if v > latest_version:
                    latest_version = v
        return latest_version

    def get_status(self):
        """Return system status.

        Status can be: 'not created', 'running', 'stopped', etc.
        """
        _, out = execute("vagrant status", cwd=self.vagrant_dir, timeout=15, capture=True, quiet=True)
        m = re.search('default\s+(.+)\(', out)
        if not m:
            raise Exception('cannot get status in:\n%s' % out)
        return m.group(1).strip()

    def bring_up_latest_box(self):
        if self.get_status() == 'running':
            self.reload()
        else:
            self.up()

    def reload(self):
        """Do Vagrant reload."""
        execute("vagrant reload --no-provision --force",
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)


502
    def package(self):
503 504
        """Package Vagrant system into Vagrant box."""

505
        if self.provider == 'virtualbox':
Michal Nowikowski's avatar
Michal Nowikowski committed
506 507
            box_path = "kea-%s-%s.box" % (self.system, self.revision)
            cmd = "vagrant package --output %s" % box_path
508 509 510 511 512
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

513
            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
514 515 516 517 518 519 520 521
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
522 523 524 525 526 527 528 529 530 531 532 533
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
                    'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
                    '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
                    'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
                    'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
                    'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
                    'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
                    '> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "'
            cmd += 'cd %s '
            cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
            cmd += '"'
534
            execute(cmd % (lxc_container_path, lxc_box_dir))
535 536 537 538 539 540 541 542 543 544 545 546
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
547

Michal Nowikowski's avatar
Michal Nowikowski committed
548 549 550 551 552 553 554 555 556 557 558 559 560 561 562
        return box_path

    def upload_to_cloud(self, box_path):
        image_tpl = IMAGE_TEMPLATES[self.key]['kea']
        if '/' not in image_tpl:
            return

        latest_version = self._get_latest_cloud_version(image_tpl)
        new_version = latest_version + 1

        cmd = "vagrant cloud publish -f -r %s %s %s %s"
        cmd = cmd % (image_tpl, new_version, self.provider, box_path)

        execute(cmd, cwd=self.vagrant_dir, timeout=60 * 60)

563
    def upload(self, src):
564
        """Upload src to Vagrant system, home folder."""
565 566 567 568 569 570 571 572 573 574 575
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

576
    def run_build_and_test(self, tarball_path, jobs):
577
        """Run build and unit tests inside Vagrant system."""
578 579 580
        if self.dry_run:
            return 0, 0

581
        # prepare tarball if needed and upload it to vagrant system
582 583
        if not tarball_path:
            name_ver = 'kea-1.5.0'
584
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
585 586
            cmd += ' --exclude "*~" --exclude .git --exclude .libs '
            cmd += ' --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
587 588
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
589
            tarball_path = '/tmp/%s.tar.gz' % name_ver
590
        self.upload(tarball_path)
591 592 593

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
594 595

        t0 = time.time()
596

597
        # run build command
598
        bld_cmd = "%s hammer.py build -p local -t %s.tar.gz -j %d" % (self.python, name_ver, jobs)
599 600 601 602 603 604
        if self.features_arg:
            bld_cmd += ' ' + self.features_arg
        if self.nofeatures_arg:
            bld_cmd += ' ' + self.nofeatures_arg
        if self.check_times:
            bld_cmd += ' -i'
605 606
        if self.ccache_enabled:
            bld_cmd += ' --ccache-dir /ccache'
607 608
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
609 610 611 612 613 614

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
            execute('scp -F %s -r default:/home/vagrant/rpm-root/RPMS/x86_64/ .' % ssh_cfg_path)

615 616
        t1 = time.time()
        dt = int(t1 - t0)
617 618

        log.info('Build log file stored to %s', log_file_path)
619 620 621 622
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

623
        # run unit tests if requested
624 625 626 627
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
628 629
                cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
630 631 632 633 634 635 636
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
Michal Nowikowski's avatar
Michal Nowikowski committed
637 638 639

                cmd = 'scp -F %s -r default:/home/vagrant/aggregated_tests.xml .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
640
        except:  # pylint: disable=bare-except
641 642 643 644
            log.exception('ignored issue with parsing unit test results')

        return total, passed

645
    def destroy(self):
646
        """Remove the VM completely."""
647
        cmd = 'vagrant destroy --force'
648
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
649 650

    def ssh(self):
651
        """Open interactive session to the VM."""
652
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
653 654

    def dump_ssh_config(self):
655
        """Dump ssh config that allows getting into Vagrant system via SSH."""
656 657 658 659 660
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
661
        """Execute provided command inside Vagrant system."""
662 663 664
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
665

666 667 668
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
                       raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
                       quiet=quiet, check_times=self.check_times)
669

670
    def prepare_system(self):
671
        """Prepare Vagrant system for building Kea."""
672 673
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
674 675 676
        else:
            self.features_arg = ''

677
        nofeatures = set(DEFAULT_FEATURES) - self.features
678 679 680 681 682
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

683
        # select proper python version for running Hammer inside Vagrant system
684 685
        if (self.system == 'centos' and self.revision == '7' or
            (self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
686 687 688 689 690 691
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

692
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
693 694 695
        if self.system == 'rhel' and self.revision == '8':
            cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
            exitcode = self.execute(cmd, raise_error=False)
696
            if exitcode != 0:
697 698 699 700 701
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
702 703 704 705 706
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

707
        # upload Hammer to Vagrant system
708
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
709
        self.upload(hmr_py_path)
710

711 712 713
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

Michal Nowikowski's avatar
Michal Nowikowski committed
714 715
        t0 = time.time()

716
        # run prepare-system inside Vagrant system
717
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times} {ccache}"
718 719
        cmd = cmd.format(features=self.features_arg,
                         nofeatures=self.nofeatures_arg,
720
                         python=self.python,
721 722
                         check_times='-i' if self.check_times else '',
                         ccache='--ccache-dir /ccache' if self.ccache_enabled else '')
723
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
724

Michal Nowikowski's avatar
Michal Nowikowski committed
725 726 727 728 729 730 731 732
        t1 = time.time()
        dt = int(t1 - t0)

        log.info('')
        log.info(">>> Preparing %s, %s, %s completed in %s:%s", self.provider, self.system, self.revision,
                 dt // 60, dt % 60)
        log.info('')

733 734

def _install_gtest_sources():
735
    """Install gtest sources."""
736
    # download gtest sources only if it is not present as native package
737
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
738 739 740
        cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
        cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
        execute(cmd)
741
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
742 743 744
        os.unlink('/tmp/gtest.tar.gz')


745
def _configure_mysql(system, revision, features):
746
    """Configure MySQL database."""
747 748 749 750
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
751 752

    if system == 'freebsd':
753 754
        cmd = "echo 'SET PASSWORD = \"\";' "
        cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
755 756
        execute(cmd, raise_error=False)

757 758 759 760 761 762 763 764 765 766 767 768 769 770 771
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

772 773 774 775 776 777 778 779 780 781 782 783
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

784 785 786 787 788 789 790 791 792 793 794 795 796
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

797

798
def _configure_pgsql(system, features):
799 800
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
801
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
802
        if exitcode != 0:
Michal Nowikowski's avatar
Michal Nowikowski committed
803 804 805 806
            if system == 'centos':
                execute('sudo postgresql-setup initdb')
            else:
                execute('sudo postgresql-setup --initdb --unit postgresql')
807
    execute('sudo systemctl start postgresql.service')
Michal Nowikowski's avatar
Michal Nowikowski committed
808
    execute('sudo systemctl enable postgresql.service')
809 810 811 812 813 814 815 816 817 818 819 820
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

821 822 823 824 825 826 827 828 829
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)
Michal Nowikowski's avatar
Michal Nowikowski committed
830 831 832 833 834 835
        # TODO: in /etc/postgresql/10/main/pg_hba.conf
        # change:
        #    local   all             all                                     peer
        # to:
        #    local   all             all                                     md5
    log.info('postgresql just configured')
836

837

Michal Nowikowski's avatar
Michal Nowikowski committed
838
def _install_cassandra_deb(system, revision, env, check_times):
839
    """Install Cassandra and cpp-driver using DEB package."""
840
    if not os.path.exists('/usr/sbin/cassandra'):
841 842 843 844
        cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
        cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
        execute(cmd, env=env, check_times=check_times)
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
845 846
                env=env, check_times=check_times)
        execute('sudo apt update', env=env, check_times=check_times)
847
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
848 849

    if not os.path.exists('/usr/include/cassandra.h'):
Michal Nowikowski's avatar
Michal Nowikowski committed
850 851 852 853 854 855 856 857 858 859
        if system == 'ubuntu' and revision == '16.04':
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
        else:
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
860 861 862 863
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
864 865


Michal Nowikowski's avatar
Michal Nowikowski committed
866
def _install_cassandra_rpm(system, env, check_times):
867
    """Install Cassandra and cpp-driver using RPM package."""
868
    if not os.path.exists('/usr/bin/cassandra'):
Michal Nowikowski's avatar
Michal Nowikowski committed
869 870 871 872 873 874 875 876
        if system == 'centos':
            install_pkgs('yum-utils', env=env, check_times=check_times)
            execute('sudo yum-config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/')
            execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
            pkgs = 'cassandra cassandra-tools libuv libuv-devel openssl'
        else:
            pkgs = 'cassandra cassandra-server libuv libuv-devel'
        install_pkgs(pkgs, env=env, check_times=check_times)
877

Michal Nowikowski's avatar
Michal Nowikowski committed
878 879
    if system == 'centos':
        execute('sudo systemctl daemon-reload')
880 881 882 883 884 885 886 887 888
    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


Michal Nowikowski's avatar
Michal Nowikowski committed
889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905
def _install_freeradius_client(env, check_times):
    """Install FreeRADIUS-client with necessary patches from Francis Dupont."""
    if (os.path.exists('/usr/local/lib/libfreeradius-client.so.2.0.0') and
        os.path.exists('/usr/local/include/freeradius-client.h')):
        log.info('freeradius is already installed')
        return
    execute('rm -rf freeradius-client')
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
    execute('./configure', cwd='freeradius-client', env=env, check_times=check_times)
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
    execute('rm -rf freeradius-client')
    log.info('freeradius just installed')


906
def prepare_system_local(features, check_times):
907
    """Prepare local system for Kea development based on requested features."""
908 909 910
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

911 912 913
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

914
    # prepare fedora
915
    if system == 'fedora':
916 917
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel']
918 919 920 921

        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'mariadb-connector-c-devel'])

922 923 924 925 926 927 928 929 930
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'community-mysql-devel'])

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

931 932 933
        if 'radius' in features:
            packages.extend(['git'])

934 935 936
        if 'ccache' in features:
            packages.extend(['ccache'])

937
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
938 939 940 941

        if 'unittest' in features:
            _install_gtest_sources()

942
        execute('sudo dnf clean packages', env=env, check_times=check_times)
943

944
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
945
            _install_cassandra_rpm(system, env, check_times)
946

947
    # prepare centos
948
    elif system == 'centos':
949
        install_pkgs('epel-release', env=env, check_times=check_times)
950

951 952
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
953 954

        if 'docs' in features:
955 956 957 958
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
959

960 961 962
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

963 964 965
        if 'radius' in features:
            packages.extend(['git'])

966 967 968
        if 'ccache' in features:
            packages.extend(['ccache'])

969
        install_pkgs(packages, env=env, check_times=check_times)
970 971 972 973

        if 'unittest' in features:
            _install_gtest_sources()

974
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
975
            _install_cassandra_rpm(system, env, check_times)
976

977
    # prepare rhel
978
    elif system == 'rhel':
979 980 981 982
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

983 984
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
985

986
        # TODO:
987 988 989 990 991 992
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

993 994 995
        if 'radius' in features:
            packages.extend(['git'])

996 997 998
        if 'ccache' in features:
            packages.extend(['ccache'])

999
        install_pkgs(packages, env=env, timeout=120, check_times=check_times)
1000 1001 1002

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
1003 1004 1005 1006
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
1007 1008
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
1009
                    check_times=check_times)
1010 1011 1012 1013 1014 1015
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    env=env, timeout=120, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
1016 1017 1018 1019

        if 'unittest' in features:
            _install_gtest_sources()

1020
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1021
            _install_cassandra_rpm(system, env, check_times)
1022

1023
    # prepare ubuntu
1024
    elif system == 'ubuntu':
1025
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
1026

1027 1028
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev',
                    'libboost-system-dev']
1029 1030

        if 'unittest' in features:
1031 1032 1033 1034
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
1035 1036

        if 'docs' in features:
1037
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
1038 1039 1040

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
1041 1042 1043
            packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev',
                             'docbook', 'docbook-xsl', 'flex', 'libboost-dev', 'libpq-dev',
                             'postgresql-server-dev-all', 'python3-dev'])
1044

1045
        if 'mysql' in features:
1046 1047 1048 1049
            if revision == '16.04':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
1050 1051

        if 'pgsql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1052 1053 1054 1055
            if revision == '16.04':
                packages.extend(['postgresql-client', 'libpq-dev', 'postgresql', 'postgresql-server-dev-all'])
            else:
                packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])
1056

1057 1058 1059
        if 'radius' in features:
            packages.extend(['git'])

1060 1061 1062
        if 'ccache' in features:
            packages.extend(['ccache'])

1063
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
1064 1065

        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1066
            _install_cassandra_deb(system, revision, env, check_times)
1067

1068
    # prepare debian
1069
    elif system == 'debian':
1070
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
1071

1072 1073
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev',
                    'liblog4cplus-dev', 'libboost-system-dev']
1074 1075

        if 'docs' in features:
1076
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
1077 1078 1079

        if 'unittest' in features:
            if revision == '8':
1080
                # libgtest-dev does not work and googletest is not available
1081
                _install_gtest_sources()
1082 1083 1084
            else:
                packages.append('googletest')

1085
        if 'mysql' in features:
1086 1087 1088 1089
            if revision == '8':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
1090

Michal Nowikowski's avatar
Michal Nowikowski committed
1091 1092 1093
        if 'pgsql' in features:
            packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

1094 1095 1096
        if 'radius' in features:
            packages.extend(['git'])

1097 1098 1099
        if 'ccache' in features:
            packages.extend(['ccache'])

1100
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
1101

1102 1103
        if 'cql' in features and revision != '8':
            # there is no libuv1 package in case of debian 8
Michal Nowikowski's avatar
Michal Nowikowski committed
1104
            _install_cassandra_deb(system, revision, env, check_times)
1105

1106
    # prepare freebsd
1107 1108 1109
    elif system == 'freebsd':
        packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs']

1110 1111
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-xsl'])
1112 1113 1114 1115

        if 'unittest' in features:
            _install_gtest_sources()

1116 1117 1118 1119 1120 1121
        if 'mysql' in features:
            packages.extend(['mysql57-server', 'mysql57-client'])

        if 'radius' in features:
            packages.extend(['git'])

1122
        install_pkgs(packages, env=env, timeout=6 * 60, check_times=check_times)
1123

1124 1125
        if 'mysql' in features:
            execute('sudo sysrc mysql_enable="yes"', env=env, check_times=check_times)
1126 1127
            execute('sudo service mysql-server start', env=env, check_times=check_times,
                    raise_error=False)
1128

1129 1130 1131
    else:
        raise NotImplementedError

1132
    if 'mysql' in features:
1133
        _configure_mysql(system, revision, features)
1134

1135
    if 'pgsql' in features:
1136
        _configure_pgsql(system, features)
1137

1138
    if 'radius' in features:
1139
        _install_freeradius_client(env, check_times)
1140 1141 1142

    #execute('sudo rm -rf /usr/share/doc')

1143 1144
    log.info('Preparing deps completed successfully.')

1145

1146
def prepare_system_in_vagrant(provider, system, revision, features, dry_run, check_times,
1147
                              clean_start, ccache_dir=None):
1148
    """Prepare specified system in Vagrant according to specified features."""
1149 1150
    ve = VagrantEnv(provider, system, revision, features, 'kea', dry_run, check_times=check_times,
                    ccache_dir=ccache_dir)