hammer.py 84.5 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11
from __future__ import print_function
import os
12
import re
13 14 15
import sys
import glob
import time
16
import json
17
import logging
18
import datetime
19 20 21
import platform
import binascii
import argparse
22 23
import textwrap
import functools
24
import subprocess
25
import multiprocessing
Michal Nowikowski's avatar
Michal Nowikowski committed
26 27 28 29
try:
    import urllib.request
except:
    pass
30 31 32 33
try:
    from urllib.parse import urljoin
except:
    from urlparse import urljoin
34
import xml.etree.ElementTree as ET
35 36 37

# TODO:
# - add docker provider
38 39
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
# - avoid using network if possible (e.g. check first if pkgs are installed)
40 41 42 43 44


SYSTEMS = {
    'fedora': ['27', '28', '29'],
    'centos': ['7'],
45 46
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
47
    'debian': ['8', '9'],
48
    'freebsd': ['11.2', '12.0'],
49 50
}

51
# pylint: disable=C0326
52
IMAGE_TEMPLATES = {
53 54 55 56 57 58 59 60
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
    'fedora-28-lxc':           {'bare': 'lxc-fedora-28',               'kea': 'godfryd/kea-fedora-28'},
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
61
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
62 63 64 65 66 67 68 69 70 71 72 73
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
74 75 76 77
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
78
ENV["LC_ALL"] = "C"
79 80

Vagrant.configure("2") do |config|
81
  config.vm.hostname = "{name}"
82 83

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
84
  {box_version}
85 86 87

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
88
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
89 90 91
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
92
  config.vm.synced_folder '{ccache_dir}', '/ccache'
93 94 95 96 97
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
98
ENV["LC_ALL"] = "C"
99 100

Vagrant.configure("2") do |config|
101
  config.vm.hostname = "{name}"
102 103

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
104
  {box_version}
105 106

  config.vm.provider "virtualbox" do |v|
107
    v.name = "{name}"
108 109 110 111 112 113 114 115 116 117
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
118 119

  config.vm.synced_folder '.', '/vagrant', disabled: true
120 121 122 123 124 125 126
end
"""


log = logging.getLogger()


127
def red(txt):
128
    """Return colorized (if the terminal supports it) or plain text."""
129 130
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
131
    return txt
132 133

def green(txt):
134
    """Return colorized (if the terminal supports it) or plain text."""
135 136
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
137
    return txt
138 139

def blue(txt):
140
    """Return colorized (if the terminal supports it) or plain text."""
141 142
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
143
    return txt
144 145


146
def get_system_revision():
147
    """Return tuple containing system name and its revision."""
148 149
    system = platform.system()
    if system == 'Linux':
150
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
151
        if system == 'debian':
152
            revision = revision[0]
153 154
        elif system == 'redhat':
            system = 'rhel'
155 156 157
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
158 159 160 161 162 163
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


164
class ExecutionError(Exception):
165
    """Exception thrown when execution encountered an error."""
166 167
    pass

168

169 170 171
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
            quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
            sleep_time_after_attempt=None):
172 173 174
    """Execute a command in shell.

    :param str cmd: a command to be executed
175 176
    :param int timeout: timeout in number of seconds, after that time the command is terminated
                        but only if check_times is True
177 178
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
179 180
    :param bool raise_error: if False then in case of error exception is not raised,
                             default: True ie exception is raise
181 182 183 184 185
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
186 187
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
                             used for e.g. SSH
188 189
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
190
    """
191
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
192 193 194 195
    if not check_times:
        timeout = None
    if dry_run:
        return 0
196

197 198 199
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
200

201 202
    if log_file_path:
        log_file = open(log_file_path, "wb")
203

204 205 206 207
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
208

209 210
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
211

212 213 214 215 216 217 218 219
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
220
                    line_decoded = line.decode(encoding='ascii', errors='ignore').rstrip() + '\r'
221 222 223 224 225 226 227 228 229 230 231
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
232 233 234 235 236 237
                # kill using sudo to be able to kill other sudo commands
                execute('sudo kill -s TERM %s' % p.pid)
                time.sleep(5)
                # if still running, kill harder
                if p.poll() is None:
                    execute('sudo kill -s KILL %s' % p.pid)
238 239 240
                msg = "Execution timeout, %d > %d seconds elapsed (start: %d, stop %d), cmd: '%s'"
                msg = msg % (t1 - t0, timeout, t0, t1, cmd)
                raise ExecutionError(msg)
241 242 243 244 245 246 247 248 249 250 251 252 253 254 255
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
256

257
    if exitcode != 0 and raise_error:
Michal Nowikowski's avatar
Michal Nowikowski committed
258 259
        if capture and quiet:
            log.error(output)
260
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
261 262 263

    if capture:
        return exitcode, output
264
    return exitcode
265 266


Michal Nowikowski's avatar
Michal Nowikowski committed
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
def _prepare_installed_packages_cache_for_debs():
    pkg_cache = {}

    _, out = execute("dpkg -l", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        line = line.strip()
        m = re.search('^([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+(.+)', line)
        if not m:
            continue
        status, name, version, arch, descr = m.groups()
        name = name.split(':')[0]
        pkg_cache[name] = dict(status=status, version=version, arch=arch, descr=descr)

    return pkg_cache


def _prepare_installed_packages_cache_for_rpms():
    pkg_cache = {}

    _, out = execute("rpm -qa --qf '%{NAME}\\n'", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        name = line.strip()
        pkg_cache[name] = dict(status='ii')

    return pkg_cache


def install_pkgs(pkgs, timeout=60, env=None, check_times=False, pkg_cache={}):
297
    """Install native packages in a system.
298 299 300 301 302 303 304

    :param dict pkgs: specifies a list of packages to be installed
    :param int timeout: timeout in number of seconds, after that time the command
                        is terminated but only if check_times is True
    :param dict env: dictionary with environment variables (optional)
    :param bool check_times: specifies if timeouts should be enabled (optional)
    """
305 306
    system, revision = get_system_revision()

Michal Nowikowski's avatar
Michal Nowikowski committed
307 308 309 310 311 312 313 314 315 316
    if not isinstance(pkgs, list):
        pkgs = pkgs.split()

    # prepare cache if needed
    if not pkg_cache and system in ['centos', 'rhel', 'fedora', 'debian', 'ubuntu']:
        if system in ['centos', 'rhel', 'fedora']:
            pkg_cache.update(_prepare_installed_packages_cache_for_rpms())
        elif system in ['debian', 'ubuntu']:
            pkg_cache.update(_prepare_installed_packages_cache_for_debs())

317 318
    # check if packages actually need to be installed
    if pkg_cache:
Michal Nowikowski's avatar
Michal Nowikowski committed
319 320 321 322 323 324 325 326 327 328
        pkgs_to_install = []
        for pkg in pkgs:
            if pkg not in pkg_cache or pkg_cache[pkg]['status'] != 'ii':
                pkgs_to_install.append(pkg)
        pkgs = pkgs_to_install

    if not pkgs:
        log.info('all packages already installed')
        return

329 330 331 332 333 334 335
    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
Michal Nowikowski's avatar
Michal Nowikowski committed
336
        # prepare the command for ubuntu/debian
337 338 339 340 341 342
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'
Michal Nowikowski's avatar
Michal Nowikowski committed
343 344
    else:
        raise NotImplementedError
345

Michal Nowikowski's avatar
Michal Nowikowski committed
346
    pkgs = ' '.join(pkgs)
347 348
    cmd += ' ' + pkgs
    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
349 350


351 352 353 354 355 356 357 358 359
def _get_full_repo_url(repository_url, system, revision, pkg_version):
    if not repository_url:
        return None
    repo_name = 'kea-%s-%s-%s' % (pkg_version.rsplit('.', 1)[0], system, revision)
    repo_url = urljoin(repository_url, 'repository')
    repo_url += '/%s-ci/' % repo_name
    return repo_url


360
class VagrantEnv(object):
361 362 363 364 365 366 367
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

368
    def __init__(self, provider, system, revision, features, image_template_variant,
369
                 dry_run, quiet=False, check_times=False, ccache_dir=None):
370 371 372 373 374 375 376 377 378 379 380
        """VagrantEnv initializer.

        :param str provider: indicate backend type: virtualbox or lxc
        :param str system: name of the system eg. ubuntu
        :param str revision: revision of the system e.g. 18.04
        :param list features: list of requested features
        :param str image_template_variant: variant of images' templates: bare or kea
        :param bool dry_run: if False then system commands are not really executed
        :param bool quiet: if True then commands will not trace to stdout
        :param bool check_times: if True then commands will be terminated after given timeout
        """
381
        self.provider = provider
382
        self.system = system
383
        self.revision = revision
384
        self.features = features
385 386 387
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
388

389 390 391 392 393
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

394 395 396 397 398
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

Michal Nowikowski's avatar
Michal Nowikowski committed
399 400
        self.key = key = "%s-%s-%s" % (system, revision, provider)
        self.image_tpl = image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
401 402
        self.repo_dir = os.getcwd()

403
        sys_dir = "%s-%s" % (system, revision)
404
        if provider == "virtualbox":
405
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
406
        elif provider == "lxc":
407 408 409 410
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
411

412 413
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
414

415
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
416 417 418 419 420

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

421
        crc = binascii.crc32(self.vagrant_dir.encode())
422
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
423

424 425 426 427 428 429
        if ccache_dir is None:
            ccache_dir = '/'
            self.ccache_enabled = False
        else:
            self.ccache_enabled = True

Michal Nowikowski's avatar
Michal Nowikowski committed
430 431 432 433 434 435 436
        if '/' in image_tpl:
            self.latest_version = self._get_latest_cloud_version()
            box_version = 'config.vm.box_version = "%s"' % self.latest_version
        else:
            self.latest_version = None
            box_version = ""

437
        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
438
                                             name=self.name,
Michal Nowikowski's avatar
Michal Nowikowski committed
439 440
                                             ccache_dir=ccache_dir,
                                             box_version=box_version)
441

442 443 444
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

445 446
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

447
    def up(self):
448 449 450
        """Do Vagrant up."""
        execute("vagrant up --no-provision --provider %s" % self.provider,
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
451

Michal Nowikowski's avatar
Michal Nowikowski committed
452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
    def _get_cloud_meta(self, image_tpl=None):
        if '/' not in self.image_tpl:
            return {}
        url = 'https://app.vagrantup.com/api/v1/box/' + (image_tpl if image_tpl else self.image_tpl)
        try:
            with urllib.request.urlopen(url) as response:
                data = response.read()
        except:
            log.exception('ignored exception')
            return {}
        data = json.loads(data)
        return data

    def _get_local_meta(self):
        meta_file = os.path.join(self.vagrant_dir, '.vagrant/machines/default', self.provider, 'box_meta')
        if not os.path.exists(meta_file):
            return {}
        with open(meta_file) as f:
            data = f.read()
        data = json.loads(data)
        return data

    def _get_latest_cloud_version(self, image_tpl=None):
        cloud_meta = self._get_cloud_meta(image_tpl)
        if not cloud_meta and 'versions' not in cloud_meta:
            return 0
        latest_version = 0
        for ver in cloud_meta['versions']:
            provider_found = False
            for p in ver['providers']:
                if p['name'] == self.provider:
                    provider_found = True
                    break
            if provider_found:
                v = int(ver['number'])
                if v > latest_version:
                    latest_version = v
        return latest_version

    def get_status(self):
        """Return system status.

        Status can be: 'not created', 'running', 'stopped', etc.
        """
        _, out = execute("vagrant status", cwd=self.vagrant_dir, timeout=15, capture=True, quiet=True)
        m = re.search('default\s+(.+)\(', out)
        if not m:
            raise Exception('cannot get status in:\n%s' % out)
        return m.group(1).strip()

    def bring_up_latest_box(self):
        if self.get_status() == 'running':
            self.reload()
        else:
            self.up()

    def reload(self):
        """Do Vagrant reload."""
        execute("vagrant reload --no-provision --force",
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)


514
    def package(self):
515 516
        """Package Vagrant system into Vagrant box."""

517
        if self.provider == 'virtualbox':
Michal Nowikowski's avatar
Michal Nowikowski committed
518 519
            box_path = "kea-%s-%s.box" % (self.system, self.revision)
            cmd = "vagrant package --output %s" % box_path
520 521 522 523 524
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

525
            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
526 527 528 529 530 531 532 533
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
534 535 536 537 538 539 540 541 542 543 544 545
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
                    'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
                    '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
                    'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
                    'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
                    'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
                    'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
                    '> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "'
            cmd += 'cd %s '
            cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
            cmd += '"'
546
            execute(cmd % (lxc_container_path, lxc_box_dir))
547 548 549 550 551 552 553 554 555 556 557 558
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
559

Michal Nowikowski's avatar
Michal Nowikowski committed
560 561 562 563 564 565 566 567 568 569 570 571 572 573 574
        return box_path

    def upload_to_cloud(self, box_path):
        image_tpl = IMAGE_TEMPLATES[self.key]['kea']
        if '/' not in image_tpl:
            return

        latest_version = self._get_latest_cloud_version(image_tpl)
        new_version = latest_version + 1

        cmd = "vagrant cloud publish -f -r %s %s %s %s"
        cmd = cmd % (image_tpl, new_version, self.provider, box_path)

        execute(cmd, cwd=self.vagrant_dir, timeout=60 * 60)

575
    def upload(self, src):
576
        """Upload src to Vagrant system, home folder."""
577 578 579 580 581 582 583 584 585 586 587
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

588
    def run_build_and_test(self, tarball_path, jobs, pkg_version, pkg_isc_version, upload, repository_url):
589
        """Run build and unit tests inside Vagrant system."""
590 591 592
        if self.dry_run:
            return 0, 0

593
        # prepare tarball if needed and upload it to vagrant system
594
        if not tarball_path:
595
            name_ver = 'kea-%s' % pkg_version
596
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
597 598
            cmd += ' --exclude "*~" --exclude .git --exclude .libs '
            cmd += ' --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
599 600
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
601
            tarball_path = '/tmp/%s.tar.gz' % name_ver
602
        self.upload(tarball_path)
603 604 605

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
606 607

        t0 = time.time()
608

609
        # run build command
610 611 612 613 614 615 616 617 618 619 620 621 622
        bld_cmd = "{python} hammer.py build -p local {features} {nofeatures} {check_times} {ccache}"
        bld_cmd += " {tarball} {jobs} {pkg_version} {pkg_isc_version} {repository_url}"
        bld_cmd = bld_cmd.format(python=self.python,
                                 features=self.features_arg,
                                 nofeatures=self.nofeatures_arg,
                                 check_times='-i' if self.check_times else '',
                                 ccache='--ccache-dir /ccache' if self.ccache_enabled else '',
                                 tarball='-t %s.tar.gz' % name_ver,
                                 jobs='-j %d' % jobs,
                                 pkg_version='--pkg-version %s' % pkg_version,
                                 pkg_isc_version='--pkg-isc-version %s' % pkg_isc_version,
                                 repository_url=('--repository-url %s' % repository_url) if repository_url else '')

623 624
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
625 626 627 628

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663
            pkgs_dir = os.path.join(self.vagrant_dir, 'pkgs')
            if os.path.exists(pkgs_dir):
                execute('rm -rf %s' % pkgs_dir)
            os.makedirs(pkgs_dir)

            if self.system in ['ubuntu', 'debian']:
                # TODO: change to pkgs folder
                execute('scp -F %s -r default:/home/vagrant/kea-src/isc-kea_* .' % ssh_cfg_path, cwd=pkgs_dir)
                execute('scp -F %s -r default:/home/vagrant/kea-src/*deb .' % ssh_cfg_path, cwd=pkgs_dir)
            elif self.system in ['fedora', 'centos', 'rhel']:
                execute('scp -F %s -r default:/home/vagrant/pkgs/* .' % ssh_cfg_path, cwd=pkgs_dir)
            else:
                raise NotImplementedError

            if upload:
                repo_url = _get_full_repo_url(repository_url, self.system, self.revision, pkg_version)
                assert repo_url is not None
                upload_cmd = 'curl -v --netrc -f'

                if self.system in ['ubuntu', 'debian']:
                    upload_cmd += ' -X POST -H "Content-Type: multipart/form-data" --data-binary "@%s" '
                    file_ext = '.deb'

                elif self.system in ['fedora', 'centos', 'rhel']:
                    upload_cmd += ' --upload-file %s '
                    file_ext = '.rpm'

                upload_cmd += ' ' + repo_url

                for fn in os.listdir(pkgs_dir):
                    if not fn.endswith(file_ext):
                        continue
                    fp = os.path.join(pkgs_dir, fn)
                    cmd = upload_cmd % fp
                    execute(cmd)
664

665 666
        t1 = time.time()
        dt = int(t1 - t0)
667 668

        log.info('Build log file stored to %s', log_file_path)
669 670 671 672
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

673
        # run unit tests if requested
674 675 676 677
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
678 679
                cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
680 681 682 683 684 685 686
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
Michal Nowikowski's avatar
Michal Nowikowski committed
687 688 689

                cmd = 'scp -F %s -r default:/home/vagrant/aggregated_tests.xml .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
690
        except:  # pylint: disable=bare-except
691 692 693 694
            log.exception('ignored issue with parsing unit test results')

        return total, passed

695
    def destroy(self):
696
        """Remove the VM completely."""
697
        cmd = 'vagrant destroy --force'
698
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
699 700

    def ssh(self):
701
        """Open interactive session to the VM."""
702
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
703 704

    def dump_ssh_config(self):
705
        """Dump ssh config that allows getting into Vagrant system via SSH."""
706 707 708 709 710
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
711
        """Execute provided command inside Vagrant system."""
712 713 714
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
715

716 717 718
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
                       raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
                       quiet=quiet, check_times=self.check_times)
719

720
    def prepare_system(self):
721
        """Prepare Vagrant system for building Kea."""
722 723
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
724 725 726
        else:
            self.features_arg = ''

727
        nofeatures = set(DEFAULT_FEATURES) - self.features
728 729 730 731 732
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

733
        # select proper python version for running Hammer inside Vagrant system
734 735
        if (self.system == 'centos' and self.revision == '7' or
            (self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
736 737 738 739 740 741
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

742
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
743 744 745
        if self.system == 'rhel' and self.revision == '8':
            cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
            exitcode = self.execute(cmd, raise_error=False)
746
            if exitcode != 0:
747 748 749 750 751
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
752 753 754 755 756
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

757
        # upload Hammer to Vagrant system
758
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
759
        self.upload(hmr_py_path)
760

761 762 763
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

Michal Nowikowski's avatar
Michal Nowikowski committed
764 765
        t0 = time.time()

766
        # run prepare-system inside Vagrant system
767
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times} {ccache}"
768 769
        cmd = cmd.format(python=self.python,
                         features=self.features_arg,
770
                         nofeatures=self.nofeatures_arg,
771 772
                         check_times='-i' if self.check_times else '',
                         ccache='--ccache-dir /ccache' if self.ccache_enabled else '')
773
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
774

Michal Nowikowski's avatar
Michal Nowikowski committed
775 776 777 778 779 780 781 782
        t1 = time.time()
        dt = int(t1 - t0)

        log.info('')
        log.info(">>> Preparing %s, %s, %s completed in %s:%s", self.provider, self.system, self.revision,
                 dt // 60, dt % 60)
        log.info('')

783 784

def _install_gtest_sources():
785
    """Install gtest sources."""
786
    # download gtest sources only if it is not present as native package
787
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
788 789 790
        cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
        cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
        execute(cmd)
791
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
792 793 794
        os.unlink('/tmp/gtest.tar.gz')


795
def _configure_mysql(system, revision, features):
796
    """Configure MySQL database."""
797 798 799 800
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
801 802

    if system == 'freebsd':
803 804
        cmd = "echo 'SET PASSWORD = \"\";' "
        cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
805 806
        execute(cmd, raise_error=False)

807 808 809 810 811 812 813 814 815 816 817 818 819 820 821
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

822 823 824 825 826 827 828 829 830 831 832 833
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

834 835 836 837 838 839 840 841 842 843 844 845 846
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

847

848
def _configure_pgsql(system, features):
849 850
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
851
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
852
        if exitcode != 0:
Michal Nowikowski's avatar
Michal Nowikowski committed
853 854 855 856
            if system == 'centos':
                execute('sudo postgresql-setup initdb')
            else:
                execute('sudo postgresql-setup --initdb --unit postgresql')
857
    execute('sudo systemctl start postgresql.service')
Michal Nowikowski's avatar
Michal Nowikowski committed
858
    execute('sudo systemctl enable postgresql.service')
859 860 861 862 863 864 865 866 867 868 869 870
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

871 872 873 874 875 876 877 878 879
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)
Michal Nowikowski's avatar
Michal Nowikowski committed
880 881 882 883 884 885
        # TODO: in /etc/postgresql/10/main/pg_hba.conf
        # change:
        #    local   all             all                                     peer
        # to:
        #    local   all             all                                     md5
    log.info('postgresql just configured')
886

887

Michal Nowikowski's avatar
Michal Nowikowski committed
888
def _install_cassandra_deb(system, revision, env, check_times):
889
    """Install Cassandra and cpp-driver using DEB package."""
890
    if not os.path.exists('/usr/sbin/cassandra'):
891 892 893 894
        cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
        cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
        execute(cmd, env=env, check_times=check_times)
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
895 896
                env=env, check_times=check_times)
        execute('sudo apt update', env=env, check_times=check_times)
897
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
898 899

    if not os.path.exists('/usr/include/cassandra.h'):
Michal Nowikowski's avatar
Michal Nowikowski committed
900 901 902 903 904 905 906 907 908 909
        if system == 'ubuntu' and revision == '16.04':
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
        else:
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
910 911 912 913
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
914 915


Michal Nowikowski's avatar
Michal Nowikowski committed
916
def _install_cassandra_rpm(system, env, check_times):
917
    """Install Cassandra and cpp-driver using RPM package."""
918
    if not os.path.exists('/usr/bin/cassandra'):
Michal Nowikowski's avatar
Michal Nowikowski committed
919 920
        if system == 'centos':
            install_pkgs('yum-utils', env=env, check_times=check_times)
921
            execute('sudo yum-config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/', raise_error=False)
Michal Nowikowski's avatar
Michal Nowikowski committed
922 923 924 925 926
            execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
            pkgs = 'cassandra cassandra-tools libuv libuv-devel openssl'
        else:
            pkgs = 'cassandra cassandra-server libuv libuv-devel'
        install_pkgs(pkgs, env=env, check_times=check_times)
927

Michal Nowikowski's avatar
Michal Nowikowski committed
928 929
    if system == 'centos':
        execute('sudo systemctl daemon-reload')
930 931 932 933 934 935 936 937 938
    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


939
def _install_freeradius_client(system, revision, features, env, check_times):
Michal Nowikowski's avatar
Michal Nowikowski committed
940
    """Install FreeRADIUS-client with necessary patches from Francis Dupont."""
941
    # check if it is already installed
Michal Nowikowski's avatar
Michal Nowikowski committed
942 943 944 945
    if (os.path.exists('/usr/local/lib/libfreeradius-client.so.2.0.0') and
        os.path.exists('/usr/local/include/freeradius-client.h')):
        log.info('freeradius is already installed')
        return
946 947 948 949 950 951 952 953 954 955

    # install freeradius dependencies
    if system in ['centos', 'rhel', 'fedora']:
        install_pkgs('nettle-devel', env=env, check_times=check_times)
    elif system in ['debian', 'ubuntu']:
        install_pkgs('nettle-dev', env=env, check_times=check_times)
    else:
        raise NotImplementedError

    # checkout sources, build them and install
Michal Nowikowski's avatar
Michal Nowikowski committed
956 957 958
    execute('rm -rf freeradius-client')
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
959
    execute('./configure --with-nettle', cwd='freeradius-client', env=env, check_times=check_times)
Michal Nowikowski's avatar
Michal Nowikowski committed
960 961 962 963 964 965 966
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
    execute('rm -rf freeradius-client')
    log.info('freeradius just installed')


967
def prepare_system_local(features, check_times):
968
    """Prepare local system for Kea development based on requested features."""
969 970 971
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

972 973 974
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

975
    # prepare fedora
976
    if system == 'fedora':
977 978
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel']
979 980

        if 'native-pkg' in features:
981
            packages.extend(['rpm-build', 'python2-devel', 'python3-devel'])
982

983 984 985 986
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
987
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-connector-c-devel'])
988 989 990 991

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

992 993 994
        if 'radius' in features:
            packages.extend(['git'])

995 996 997
        if 'ccache' in features:
            packages.extend(['ccache'])

998
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
999 1000 1001 1002

        if 'unittest' in features:
            _install_gtest_sources()

1003
        execute('sudo dnf clean packages', env=env, check_times=check_times)
1004

1005
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1006
            _install_cassandra_rpm(system, env, check_times)
1007

1008
    # prepare centos
1009
    elif system == 'centos':
1010
        install_pkgs('epel-release', env=env, check_times=check_times)
1011

1012 1013
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
1014

1015 1016 1017
        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'python2-devel'])

1018
        if 'docs' in features:
1019 1020 1021 1022
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
1023

1024 1025 1026
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

1027 1028 1029
        if 'radius' in features:
            packages.extend(['git'])

1030 1031 1032
        if 'ccache' in features:
            packages.extend(['ccache'])

1033
        install_pkgs(packages, env=env, check_times=check_times)
1034 1035 1036 1037

        if 'unittest' in features:
            _install_gtest_sources()

1038
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1039
            _install_cassandra_rpm(system, env, check_times)
1040

1041
    # prepare rhel
1042
    elif system == 'rhel':
1043 1044 1045 1046
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

1047 1048
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
1049

1050
        # TODO:
1051 1052 1053 1054 1055 1056
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

1057 1058 1059
        if 'radius' in features:
            packages.extend(['git'])

1060 1061 1062
        if 'ccache' in features:
            packages.extend(['ccache'])

1063
        install_pkgs(packages, env=env, timeout=120, check_times=check_times)
1064 1065 1066

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
1067 1068 1069 1070
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
1071 1072
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
1073
                    check_times=check_times)
1074 1075 1076 1077 1078 1079
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    env=env, timeout=120, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
1080 1081 1082 1083

        if 'unittest' in features:
            _install_gtest_sources()

1084
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1085
            _install_cassandra_rpm(system, env, check_times)
1086

1087
    # prepare ubuntu
1088
    elif system == 'ubuntu':
1089
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
1090

1091 1092
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev',
                    'libboost-system-dev']
1093 1094

        if 'unittest' in features:
1095 1096 1097 1098
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
1099 1100

        if 'docs' in features:
1101
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
1102 1103 1104

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
1105
            packages.extend(['bison', 'debhelper', 'docbook', 'flex', 'libboost-dev', 'python3-dev'])
1106

1107
        if 'mysql' in features:
1108 1109 1110 1111
            if revision == '16.04':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
1112 1113

        if 'pgsql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1114 1115 1116 1117
            if revision == '16.04':
                packages.extend(['postgresql-client', 'libpq-dev', 'postgresql', 'postgresql-server-dev-all'])
            else:
                packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])
1118

1119 1120 1121
        if 'radius' in features:
            packages.extend(['git'])

1122 1123 1124
        if 'ccache' in features:
            packages.extend(['ccache'])

1125
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
1126 1127

        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1128
            _install_cassandra_deb(system, revision, env, check_times)
1129

1130
    # prepare debian
1131
    elif system == 'debian':
1132
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
1133

1134 1135
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev',
                    'liblog4cplus-dev', 'libboost-system-dev']
1136 1137 1138

        if 'unittest' in features:
            if revision == '8':
1139
                # libgtest-dev does not work and googletest is not available
1140
                _install_gtest_sources()
1141 1142 1143
            else:
                packages.append('googletest')

1144 1145 1146 1147 1148 1149 1150
        if 'docs' in features:
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
            packages.extend(['bison', 'debhelper', 'docbook', 'flex', 'libboost-dev', 'python3-dev'])

1151
        if 'mysql' in features:
1152 1153 1154 1155
            if revision == '8':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
1156

Michal Nowikowski's avatar
Michal Nowikowski committed
1157 1158 1159
        if 'pgsql' in features:
            packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

1160 1161 1162
        if 'radius' in features:
            packages.extend(['git'])

1163 1164 1165
        if 'ccache' in features:
            packages.extend(['ccache'])

1166
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
1167

1168 1169
        if 'cql' in features and revision != '8':
            # there is no libuv1 package in case of debian 8
Michal Nowikowski's avatar
Michal Nowikowski committed