hammer.py 89.8 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11
from __future__ import print_function
import os
12
import re
13 14 15
import sys
import glob
import time
16
import json
17
import logging
18
import datetime
19 20 21
import platform
import binascii
import argparse
22 23
import textwrap
import functools
24
import subprocess
25
import multiprocessing
Michal Nowikowski's avatar
Michal Nowikowski committed
26 27 28 29
try:
    import urllib.request
except:
    pass
30 31 32 33
try:
    from urllib.parse import urljoin
except:
    from urlparse import urljoin
34
import xml.etree.ElementTree as ET
35 36 37

# TODO:
# - add docker provider
38 39
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
# - avoid using network if possible (e.g. check first if pkgs are installed)
40 41 42


SYSTEMS = {
43 44 45 46
    'fedora': ['27',  # EOLed
               '28',  # EOLed
               '29',
               '30'],
47
    'centos': ['7'],
48
    'rhel': ['8'],
49 50 51 52 53 54 55 56 57
    'ubuntu': ['16.04',
               '18.04',
               '18.10',  # EOLed
               '19.04'],
    'debian': ['8',
               '9',
               '10'],
    'freebsd': ['11.2',
                '12.0'],
58 59
}

60
# pylint: disable=C0326
61
IMAGE_TEMPLATES = {
62 63
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
64
    'fedora-28-lxc':           {'bare': 'godfryd/lxc-fedora-28',       'kea': 'godfryd/kea-fedora-28'},
65 66 67
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
68 69
    'fedora-30-lxc':           {'bare': 'godfryd/lxc-fedora-30',       'kea': 'godfryd/kea-fedora-30'},
    'fedora-30-virtualbox':    {'bare': 'generic/fedora30',            'kea': 'godfryd/kea-fedora-30'},
70 71
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
72
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
73 74 75 76 77 78
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
79 80
    'ubuntu-19.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-19.04',    'kea': 'godfryd/kea-ubuntu-19.04'},
    'ubuntu-19.04-virtualbox': {'bare': 'ubuntu/disco64',              'kea': 'godfryd/kea-ubuntu-19.04'},
81 82 83 84
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
85 86
    'debian-10-lxc':           {'bare': 'godfryd/lxc-debian-10',       'kea': 'godfryd/kea-debian-10'},
    'debian-10-virtualbox':    {'bare': 'debian/buster64',             'kea': 'godfryd/kea-debian-10'},
87 88
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
89 90 91 92
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
93
ENV["LC_ALL"] = "C"
94 95

Vagrant.configure("2") do |config|
96
  config.vm.hostname = "{name}"
97 98

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
99
  {box_version}
100 101 102

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
103
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
104 105 106
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
107
  config.vm.synced_folder '{ccache_dir}', '/ccache'
108 109 110 111 112
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
113
ENV["LC_ALL"] = "C"
114 115

Vagrant.configure("2") do |config|
116
  config.vm.hostname = "{name}"
117 118

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
119
  {box_version}
120 121

  config.vm.provider "virtualbox" do |v|
122
    v.name = "{name}"
123 124 125 126 127 128 129 130 131 132
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
133 134

  config.vm.synced_folder '.', '/vagrant', disabled: true
135 136 137 138 139 140 141
end
"""


log = logging.getLogger()


142
def red(txt):
143
    """Return colorized (if the terminal supports it) or plain text."""
144 145
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
146
    return txt
147 148

def green(txt):
149
    """Return colorized (if the terminal supports it) or plain text."""
150 151
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
152
    return txt
153 154

def blue(txt):
155
    """Return colorized (if the terminal supports it) or plain text."""
156 157
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
158
    return txt
159 160


161
def get_system_revision():
162
    """Return tuple containing system name and its revision."""
163 164
    system = platform.system()
    if system == 'Linux':
165
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
166
        if system == 'debian':
167
            revision = revision.split('.')[0]
168 169
        elif system == 'redhat':
            system = 'rhel'
170 171 172
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
173 174 175 176 177 178
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


179
class ExecutionError(Exception):
180
    """Exception thrown when execution encountered an error."""
181 182
    pass

183

184 185 186
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
            quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
            sleep_time_after_attempt=None):
187 188 189
    """Execute a command in shell.

    :param str cmd: a command to be executed
190 191
    :param int timeout: timeout in number of seconds, after that time the command is terminated
                        but only if check_times is True
192 193
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
194 195
    :param bool raise_error: if False then in case of error exception is not raised,
                             default: True ie exception is raise
196 197 198 199 200
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
201 202
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
                             used for e.g. SSH
203 204
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
205
    """
206
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
207 208 209 210
    if not check_times:
        timeout = None
    if dry_run:
        return 0
211

212 213 214
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
215

216 217
    if log_file_path:
        log_file = open(log_file_path, "wb")
218

219 220 221 222
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
223

224 225
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
226

227 228 229 230 231 232 233 234
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
235
                    line_decoded = line.decode(encoding='ascii', errors='ignore').rstrip() + '\r'
236 237 238 239 240 241 242 243 244 245 246
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
247 248 249 250 251 252
                # kill using sudo to be able to kill other sudo commands
                execute('sudo kill -s TERM %s' % p.pid)
                time.sleep(5)
                # if still running, kill harder
                if p.poll() is None:
                    execute('sudo kill -s KILL %s' % p.pid)
253 254 255
                msg = "Execution timeout, %d > %d seconds elapsed (start: %d, stop %d), cmd: '%s'"
                msg = msg % (t1 - t0, timeout, t0, t1, cmd)
                raise ExecutionError(msg)
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
271

272
    if exitcode != 0 and raise_error:
Michal Nowikowski's avatar
Michal Nowikowski committed
273 274
        if capture and quiet:
            log.error(output)
275
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
276 277 278

    if capture:
        return exitcode, output
279
    return exitcode
280 281


Michal Nowikowski's avatar
Michal Nowikowski committed
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311
def _prepare_installed_packages_cache_for_debs():
    pkg_cache = {}

    _, out = execute("dpkg -l", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        line = line.strip()
        m = re.search('^([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+(.+)', line)
        if not m:
            continue
        status, name, version, arch, descr = m.groups()
        name = name.split(':')[0]
        pkg_cache[name] = dict(status=status, version=version, arch=arch, descr=descr)

    return pkg_cache


def _prepare_installed_packages_cache_for_rpms():
    pkg_cache = {}

    _, out = execute("rpm -qa --qf '%{NAME}\\n'", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        name = line.strip()
        pkg_cache[name] = dict(status='ii')

    return pkg_cache


def install_pkgs(pkgs, timeout=60, env=None, check_times=False, pkg_cache={}):
312
    """Install native packages in a system.
313 314 315 316 317 318 319

    :param dict pkgs: specifies a list of packages to be installed
    :param int timeout: timeout in number of seconds, after that time the command
                        is terminated but only if check_times is True
    :param dict env: dictionary with environment variables (optional)
    :param bool check_times: specifies if timeouts should be enabled (optional)
    """
320 321
    system, revision = get_system_revision()

Michal Nowikowski's avatar
Michal Nowikowski committed
322 323 324 325 326 327 328 329 330 331
    if not isinstance(pkgs, list):
        pkgs = pkgs.split()

    # prepare cache if needed
    if not pkg_cache and system in ['centos', 'rhel', 'fedora', 'debian', 'ubuntu']:
        if system in ['centos', 'rhel', 'fedora']:
            pkg_cache.update(_prepare_installed_packages_cache_for_rpms())
        elif system in ['debian', 'ubuntu']:
            pkg_cache.update(_prepare_installed_packages_cache_for_debs())

332 333
    # check if packages actually need to be installed
    if pkg_cache:
Michal Nowikowski's avatar
Michal Nowikowski committed
334 335 336 337 338 339 340 341 342 343
        pkgs_to_install = []
        for pkg in pkgs:
            if pkg not in pkg_cache or pkg_cache[pkg]['status'] != 'ii':
                pkgs_to_install.append(pkg)
        pkgs = pkgs_to_install

    if not pkgs:
        log.info('all packages already installed')
        return

344 345 346 347 348 349 350
    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
Michal Nowikowski's avatar
Michal Nowikowski committed
351
        # prepare the command for ubuntu/debian
352 353 354 355 356 357
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'
Michal Nowikowski's avatar
Michal Nowikowski committed
358 359
    else:
        raise NotImplementedError
360

Michal Nowikowski's avatar
Michal Nowikowski committed
361
    pkgs = ' '.join(pkgs)
362 363
    cmd += ' ' + pkgs
    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
364 365


366 367 368 369 370 371 372 373 374
def _get_full_repo_url(repository_url, system, revision, pkg_version):
    if not repository_url:
        return None
    repo_name = 'kea-%s-%s-%s' % (pkg_version.rsplit('.', 1)[0], system, revision)
    repo_url = urljoin(repository_url, 'repository')
    repo_url += '/%s-ci/' % repo_name
    return repo_url


375
class VagrantEnv(object):
376 377 378 379 380 381 382
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

383
    def __init__(self, provider, system, revision, features, image_template_variant,
384
                 dry_run, quiet=False, check_times=False, ccache_dir=None):
385 386 387 388 389 390 391 392 393 394 395
        """VagrantEnv initializer.

        :param str provider: indicate backend type: virtualbox or lxc
        :param str system: name of the system eg. ubuntu
        :param str revision: revision of the system e.g. 18.04
        :param list features: list of requested features
        :param str image_template_variant: variant of images' templates: bare or kea
        :param bool dry_run: if False then system commands are not really executed
        :param bool quiet: if True then commands will not trace to stdout
        :param bool check_times: if True then commands will be terminated after given timeout
        """
396
        self.provider = provider
397
        self.system = system
398
        self.revision = revision
399
        self.features = features
400 401 402
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
403

404 405 406 407 408
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

409 410 411 412 413
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

Michal Nowikowski's avatar
Michal Nowikowski committed
414 415
        self.key = key = "%s-%s-%s" % (system, revision, provider)
        self.image_tpl = image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
416 417
        self.repo_dir = os.getcwd()

418
        sys_dir = "%s-%s" % (system, revision)
419
        if provider == "virtualbox":
420
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
421
        elif provider == "lxc":
422 423 424 425
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
426

427 428
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
429

430
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
431 432 433 434 435

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

436
        crc = binascii.crc32(self.vagrant_dir.encode())
437
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
438

439 440 441 442 443 444
        if ccache_dir is None:
            ccache_dir = '/'
            self.ccache_enabled = False
        else:
            self.ccache_enabled = True

Michal Nowikowski's avatar
Michal Nowikowski committed
445 446 447 448 449 450 451
        if '/' in image_tpl:
            self.latest_version = self._get_latest_cloud_version()
            box_version = 'config.vm.box_version = "%s"' % self.latest_version
        else:
            self.latest_version = None
            box_version = ""

452
        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
453
                                             name=self.name,
Michal Nowikowski's avatar
Michal Nowikowski committed
454 455
                                             ccache_dir=ccache_dir,
                                             box_version=box_version)
456

457 458 459
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

460 461
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

462
    def up(self):
463
        """Do Vagrant up."""
464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480
        exitcode, out = execute("vagrant up --no-provision --provider %s" % self.provider,
                                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run,
                                capture=True, raise_error=False)
        if exitcode != 0:
            if 'There is container on your system' in out and 'lxc-destroy' in out:
                m = re.search('`lxc-destroy.*?`', out)
                if m:
                    # destroy some old container
                    cmd = m.group(0)[1:-1]
                    cmd = 'sudo ' + cmd + ' -f'
                    execute(cmd, timeout=60)

                    # try again spinning up new
                    execute("vagrant up --no-provision --provider %s" % self.provider,
                            cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
                    return
            raise ExecutionError('There is a problem with putting up a system')
481

Michal Nowikowski's avatar
Michal Nowikowski committed
482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515
    def _get_cloud_meta(self, image_tpl=None):
        if '/' not in self.image_tpl:
            return {}
        url = 'https://app.vagrantup.com/api/v1/box/' + (image_tpl if image_tpl else self.image_tpl)
        try:
            with urllib.request.urlopen(url) as response:
                data = response.read()
        except:
            log.exception('ignored exception')
            return {}
        data = json.loads(data)
        return data

    def _get_local_meta(self):
        meta_file = os.path.join(self.vagrant_dir, '.vagrant/machines/default', self.provider, 'box_meta')
        if not os.path.exists(meta_file):
            return {}
        with open(meta_file) as f:
            data = f.read()
        data = json.loads(data)
        return data

    def _get_latest_cloud_version(self, image_tpl=None):
        cloud_meta = self._get_cloud_meta(image_tpl)
        if not cloud_meta and 'versions' not in cloud_meta:
            return 0
        latest_version = 0
        for ver in cloud_meta['versions']:
            provider_found = False
            for p in ver['providers']:
                if p['name'] == self.provider:
                    provider_found = True
                    break
            if provider_found:
516 517 518 519
                try:
                    v = int(ver['number'])
                except:
                    return ver['number']
Michal Nowikowski's avatar
Michal Nowikowski committed
520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546
                if v > latest_version:
                    latest_version = v
        return latest_version

    def get_status(self):
        """Return system status.

        Status can be: 'not created', 'running', 'stopped', etc.
        """
        _, out = execute("vagrant status", cwd=self.vagrant_dir, timeout=15, capture=True, quiet=True)
        m = re.search('default\s+(.+)\(', out)
        if not m:
            raise Exception('cannot get status in:\n%s' % out)
        return m.group(1).strip()

    def bring_up_latest_box(self):
        if self.get_status() == 'running':
            self.reload()
        else:
            self.up()

    def reload(self):
        """Do Vagrant reload."""
        execute("vagrant reload --no-provision --force",
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)


547
    def package(self):
548
        """Package Vagrant system into Vagrant box."""
549
        execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
550

551 552 553
        box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
        if os.path.exists(box_path):
            os.unlink(box_path)
554

555
        if self.provider == 'virtualbox':
Michal Nowikowski's avatar
Michal Nowikowski committed
556
            cmd = "vagrant package --output %s" % box_path
557 558 559 560 561 562 563 564
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
565 566 567 568 569 570 571 572 573 574 575 576
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
                    'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
                    '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
                    'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
                    'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
                    'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
                    'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
                    '> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "'
            cmd += 'cd %s '
            cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
            cmd += '"'
577
            execute(cmd % (lxc_container_path, lxc_box_dir))
578 579 580 581 582 583 584 585 586 587 588 589
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
590

Michal Nowikowski's avatar
Michal Nowikowski committed
591 592 593 594 595 596 597 598 599 600 601 602 603 604 605
        return box_path

    def upload_to_cloud(self, box_path):
        image_tpl = IMAGE_TEMPLATES[self.key]['kea']
        if '/' not in image_tpl:
            return

        latest_version = self._get_latest_cloud_version(image_tpl)
        new_version = latest_version + 1

        cmd = "vagrant cloud publish -f -r %s %s %s %s"
        cmd = cmd % (image_tpl, new_version, self.provider, box_path)

        execute(cmd, cwd=self.vagrant_dir, timeout=60 * 60)

606
    def upload(self, src):
607
        """Upload src to Vagrant system, home folder."""
608 609 610 611 612 613 614 615 616 617 618
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

619
    def run_build_and_test(self, tarball_path, jobs, pkg_version, pkg_isc_version, upload, repository_url):
620
        """Run build and unit tests inside Vagrant system."""
621 622 623
        if self.dry_run:
            return 0, 0

624
        # prepare tarball if needed and upload it to vagrant system
625
        if not tarball_path:
626
            name_ver = 'kea-%s' % pkg_version
627
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
628 629
            cmd += ' --exclude "*~" --exclude .git --exclude .libs '
            cmd += ' --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
630 631
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
632
            tarball_path = '/tmp/%s.tar.gz' % name_ver
633
        self.upload(tarball_path)
634 635 636

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
637 638

        t0 = time.time()
639

640
        # run build command
641 642 643 644 645 646 647 648 649 650 651 652 653
        bld_cmd = "{python} hammer.py build -p local {features} {nofeatures} {check_times} {ccache}"
        bld_cmd += " {tarball} {jobs} {pkg_version} {pkg_isc_version} {repository_url}"
        bld_cmd = bld_cmd.format(python=self.python,
                                 features=self.features_arg,
                                 nofeatures=self.nofeatures_arg,
                                 check_times='-i' if self.check_times else '',
                                 ccache='--ccache-dir /ccache' if self.ccache_enabled else '',
                                 tarball='-t %s.tar.gz' % name_ver,
                                 jobs='-j %d' % jobs,
                                 pkg_version='--pkg-version %s' % pkg_version,
                                 pkg_isc_version='--pkg-isc-version %s' % pkg_isc_version,
                                 repository_url=('--repository-url %s' % repository_url) if repository_url else '')

654 655
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
656 657 658 659

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694
            pkgs_dir = os.path.join(self.vagrant_dir, 'pkgs')
            if os.path.exists(pkgs_dir):
                execute('rm -rf %s' % pkgs_dir)
            os.makedirs(pkgs_dir)

            if self.system in ['ubuntu', 'debian']:
                # TODO: change to pkgs folder
                execute('scp -F %s -r default:/home/vagrant/kea-src/isc-kea_* .' % ssh_cfg_path, cwd=pkgs_dir)
                execute('scp -F %s -r default:/home/vagrant/kea-src/*deb .' % ssh_cfg_path, cwd=pkgs_dir)
            elif self.system in ['fedora', 'centos', 'rhel']:
                execute('scp -F %s -r default:/home/vagrant/pkgs/* .' % ssh_cfg_path, cwd=pkgs_dir)
            else:
                raise NotImplementedError

            if upload:
                repo_url = _get_full_repo_url(repository_url, self.system, self.revision, pkg_version)
                assert repo_url is not None
                upload_cmd = 'curl -v --netrc -f'

                if self.system in ['ubuntu', 'debian']:
                    upload_cmd += ' -X POST -H "Content-Type: multipart/form-data" --data-binary "@%s" '
                    file_ext = '.deb'

                elif self.system in ['fedora', 'centos', 'rhel']:
                    upload_cmd += ' --upload-file %s '
                    file_ext = '.rpm'

                upload_cmd += ' ' + repo_url

                for fn in os.listdir(pkgs_dir):
                    if not fn.endswith(file_ext):
                        continue
                    fp = os.path.join(pkgs_dir, fn)
                    cmd = upload_cmd % fp
                    execute(cmd)
695

696 697
        t1 = time.time()
        dt = int(t1 - t0)
698 699

        log.info('Build log file stored to %s', log_file_path)
700 701 702 703
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

704
        # run unit tests if requested
705 706 707 708
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
709 710
                cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
711 712 713 714 715 716 717
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
Michal Nowikowski's avatar
Michal Nowikowski committed
718 719 720

                cmd = 'scp -F %s -r default:/home/vagrant/aggregated_tests.xml .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
721
        except:  # pylint: disable=bare-except
722 723 724 725
            log.exception('ignored issue with parsing unit test results')

        return total, passed

726
    def destroy(self):
727
        """Remove the VM completely."""
728
        cmd = 'vagrant destroy --force'
729
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
730 731

    def ssh(self):
732
        """Open interactive session to the VM."""
733
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
734 735

    def dump_ssh_config(self):
736
        """Dump ssh config that allows getting into Vagrant system via SSH."""
737 738 739 740 741
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
742
        """Execute provided command inside Vagrant system."""
743 744 745
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
746

747 748 749
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
                       raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
                       quiet=quiet, check_times=self.check_times)
750

751
    def prepare_system(self):
752
        """Prepare Vagrant system for building Kea."""
753 754
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
755 756 757
        else:
            self.features_arg = ''

758
        nofeatures = set(DEFAULT_FEATURES) - self.features
759 760 761 762 763
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

764
        # select proper python version for running Hammer inside Vagrant system
765 766
        if (self.system == 'centos' and self.revision == '7' or
            (self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
767 768 769 770 771 772
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

773
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
774 775 776
        if self.system == 'rhel' and self.revision == '8':
            cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
            exitcode = self.execute(cmd, raise_error=False)
777
            if exitcode != 0:
778 779 780 781 782
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
783 784 785 786 787
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

788
        # upload Hammer to Vagrant system
789
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
790
        self.upload(hmr_py_path)
791

792 793 794
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

Michal Nowikowski's avatar
Michal Nowikowski committed
795 796
        t0 = time.time()

797
        # run prepare-system inside Vagrant system
798
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times} {ccache}"
799 800
        cmd = cmd.format(python=self.python,
                         features=self.features_arg,
801
                         nofeatures=self.nofeatures_arg,
802 803
                         check_times='-i' if self.check_times else '',
                         ccache='--ccache-dir /ccache' if self.ccache_enabled else '')
804
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
805

Michal Nowikowski's avatar
Michal Nowikowski committed
806 807 808 809 810 811 812 813
        t1 = time.time()
        dt = int(t1 - t0)

        log.info('')
        log.info(">>> Preparing %s, %s, %s completed in %s:%s", self.provider, self.system, self.revision,
                 dt // 60, dt % 60)
        log.info('')

814 815

def _install_gtest_sources():
816
    """Install gtest sources."""
817
    # download gtest sources only if it is not present as native package
818
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
819 820 821
        cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
        cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
        execute(cmd)
822
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
823 824 825
        os.unlink('/tmp/gtest.tar.gz')


826
def _configure_mysql(system, revision, features):
827
    """Configure MySQL database."""
828 829 830 831
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
832 833

    if system == 'freebsd':
834 835
        cmd = "echo 'SET PASSWORD = \"\";' "
        cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
836 837
        execute(cmd, raise_error=False)

838 839 840 841 842 843 844 845 846 847 848 849 850 851 852
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

853 854 855 856 857 858 859 860 861 862 863 864
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

865 866 867 868 869 870 871 872 873 874 875 876 877
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

878

879
def _configure_pgsql(system, features):
880 881
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
882
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
883
        if exitcode != 0:
Michal Nowikowski's avatar
Michal Nowikowski committed
884 885 886 887
            if system == 'centos':
                execute('sudo postgresql-setup initdb')
            else:
                execute('sudo postgresql-setup --initdb --unit postgresql')
888
    execute('sudo systemctl start postgresql.service')
Michal Nowikowski's avatar
Michal Nowikowski committed
889
    execute('sudo systemctl enable postgresql.service')
890 891 892 893 894 895 896 897 898 899
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
900
    execute(cmd, cwd='/tmp')  # CWD to avoid: could not change as posgres user directory to "/home/jenkins": Permission denied
901

902 903 904 905 906 907 908 909
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
910
        execute(cmd, cwd='/tmp')  # CWD to avoid: could not change as posgres user directory to "/home/jenkins": Permission denied
Michal Nowikowski's avatar
Michal Nowikowski committed
911 912 913 914 915 916
        # TODO: in /etc/postgresql/10/main/pg_hba.conf
        # change:
        #    local   all             all                                     peer
        # to:
        #    local   all             all                                     md5
    log.info('postgresql just configured')
917

918

919 920 921 922 923 924 925 926 927
def _apt_update(system, revision, env=None, check_times=False, attempts=1, sleep_time_after_attempt=None,
                capture=False):
    cmd = 'sudo apt update'
    if system == 'debian' and int(revision) >= 10:
        cmd += ' --allow-releaseinfo-change'
    return execute(cmd, env=env, check_times=check_times, attempts=attempts,
                   sleep_time_after_attempt=sleep_time_after_attempt, capture=capture)


Michal Nowikowski's avatar
Michal Nowikowski committed
928
def _install_cassandra_deb(system, revision, env, check_times):
929
    """Install Cassandra and cpp-driver using DEB package."""
930
    if not os.path.exists('/usr/sbin/cassandra'):
931 932 933 934
        cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
        cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
        execute(cmd, env=env, check_times=check_times)
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
935
                env=env, check_times=check_times)
936
        _apt_update(system, revision, env=env, check_times=check_times)
937
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
938 939

    if not os.path.exists('/usr/include/cassandra.h'):
Michal Nowikowski's avatar
Michal Nowikowski committed
940 941 942 943 944 945 946 947 948 949
        if system == 'ubuntu' and revision == '16.04':
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
        else:
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
950 951 952
            if system == 'debian' and revision == '10':
                install_pkgs('multiarch-support', env=env, check_times=check_times)

953 954 955 956
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
957 958


959
def _install_cassandra_rpm(system, revision, env, check_times):
960
    """Install Cassandra and cpp-driver using RPM package."""
961
    if not os.path.exists('/usr/bin/cassandra'):
Michal Nowikowski's avatar
Michal Nowikowski committed
962 963
        if system == 'centos':
            install_pkgs('yum-utils', env=env, check_times=check_times)
964
            execute('sudo yum-config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/', raise_error=False)
Michal Nowikowski's avatar
Michal Nowikowski committed
965 966 967 968 969
            execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
            pkgs = 'cassandra cassandra-tools libuv libuv-devel openssl'
        else:
            pkgs = 'cassandra cassandra-server libuv libuv-devel'
        install_pkgs(pkgs, env=env, check_times=check_times)
970

Michal Nowikowski's avatar
Michal Nowikowski committed
971 972
    if system == 'centos':
        execute('sudo systemctl daemon-reload')
973 974 975

    if system == 'fedora' and revision == '30':
        execute("echo '-Xms1G -Xmx1G' | sudo tee -a /etc/cassandra/jvm.options")
976 977 978 979 980
    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
981 982 983 984
        if system == 'centos':
            execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        else:
            execute('sudo dnf install -y cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
985 986 987
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


988
def _install_freeradius_client(system, revision, features, env, check_times):
Michal Nowikowski's avatar
Michal Nowikowski committed
989
    """Install FreeRADIUS-client with necessary patches from Francis Dupont."""
990
    # check if it is already installed
Michal Nowikowski's avatar
Michal Nowikowski committed
991 992 993 994
    if (os.path.exists('/usr/local/lib/libfreeradius-client.so.2.0.0') and
        os.path.exists('/usr/local/include/freeradius-client.h')):
        log.info('freeradius is already installed')
        return
995 996 997 998 999 1000 1001 1002 1003 1004

    # install freeradius dependencies
    if system in ['centos', 'rhel', 'fedora']:
        install_pkgs('nettle-devel', env=env, check_times=check_times)
    elif system in ['debian', 'ubuntu']:
        install_pkgs('nettle-dev', env=env, check_times=check_times)
    else:
        raise NotImplementedError

    # checkout sources, build them and install
Michal Nowikowski's avatar
Michal Nowikowski committed
1005 1006 1007
    execute('rm -rf freeradius-client')
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
1008
    execute('./configure --with-nettle', cwd='freeradius-client', env=env, check_times=check_times)
Michal Nowikowski's avatar
Michal Nowikowski committed
1009 1010 1011 1012 1013 1014 1015
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
    execute('rm -rf freeradius-client')
    log.info('freeradius just installed')


1016
def prepare_system_local(features, check_times):
1017
    """Prepare local system for Kea development based on requested features."""
1018 1019 1020
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

1021 1022 1023
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

1024
    # prepare fedora
1025
    if system == 'fedora':
1026 1027
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel']
1028 1029

        if 'native-pkg' in features:
1030
            packages.extend(['rpm-build', 'python2-devel', 'python3-devel'])
1031

1032
        if 'docs' in features:
1033
            packages.extend(['python3-sphinx', 'texlive', 'texlive-collection-latexextra'])
1034 1035

        if 'mysql' in features:
1036
            execute('sudo dnf remove -y community-mysql-devel || true')
1037
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-connector-c-devel'])
1038 1039 1040

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])
1041 1042
            if revision in ['30']:
                packages.extend(['postgresql-server-devel'])
1043

1044 1045 1046
        if 'radius' in features:
            packages.extend(['git'])

1047 1048 1049
        if 'ccache' in features:
            packages.extend(['ccache'])

1050
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
1051 1052 1053 1054

        if 'unittest' in features:
            _install_gtest_sources()

1055
        execute('sudo dnf clean packages', env=env, check_times=check_times)
1056

1057
        if 'cql' in features:
1058
            _install_cassandra_rpm(system, revision, env, check_times)
1059

1060
    # prepare centos
1061
    elif system == 'centos':
1062
        install_pkgs('epel-release', env=env, check_times=check_times)
1063

1064 1065
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
1066

1067 1068 1069
        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'python2-devel'])

1070
        if 'docs' in features:
1071
            packages.extend(['python36-virtualenv'])
1072 1073 1074

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
1075

1076 1077 1078
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

1079 1080 1081
        if 'radius' in features:
            packages.extend(['git'])

1082 1083 1084
        if 'ccache' in features:
            packages.extend(['ccache'])

1085
        install_pkgs(packages, env=env, check_times=check_times)
1086

1087 1088 1089 1090 1091 1092
        if 'docs' in features:
            execute('virtualenv-3 ~/venv',
                    env=env, timeout=60, check_times=check_times)
            execute('~/venv/bin/pip install sphinx sphinx-rtd-theme',
                    env=env, timeout=120, check_times=check_times)

1093 1094 1095
        if 'unittest' in features:
            _install_gtest_sources()

1096
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1097
            _install_cassandra_rpm(system, env, check_times)
1098

1099
    # prepare rhel
1100
    elif system == 'rhel':
1101 1102 1103 1104
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

1105 1106
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
1107

1108
        # TODO:
1109 1110 1111 1112 1113 1114
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

1115 1116
        if 'radius' in features:
            packages.extend(['git'])
1117 1118
            if 'forge' in features:
                packages.extend(['freeradius'])
1119

1120 1121 1122
        if 'ccache' in features:
            packages.extend(['ccache'])

1123
        install_pkgs(packages, env=env, timeout=120, check_times=check_times)
1124 1125 1126

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
1127 1128 1129 1130
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
1131 1132
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
1133
                    check_times=check_times)
1134 1135 1136 1137 1138 1139
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    env=env, timeout=120, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
1140 1141 1142 1143

        if 'unittest' in features:
            _install_gtest_sources()

1144
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1145
            _install_cassandra_rpm(system, env, check_times)
1146

1147
    # prepare ubuntu
1148
    elif system == 'ubuntu':
1149
        _apt_update(system, revision, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
1150

1151 1152
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev',
                    'libboost-system-dev']
1153 1154

        if 'unittest' in features:
1155 1156 1157 1158
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
1159 1160

        if 'docs' in features:
1161
            packages.extend(['python3-sphinx', 'python3-sphinx-rtd-theme'])
1162 1163 1164

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])