hammer.py 86.7 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11
from __future__ import print_function
import os
12
import re
13 14 15
import sys
import glob
import time
16
import json
17
import logging
18
import datetime
19 20 21
import platform
import binascii
import argparse
22 23
import textwrap
import functools
24
import subprocess
25
import multiprocessing
Michal Nowikowski's avatar
Michal Nowikowski committed
26 27 28 29
try:
    import urllib.request
except:
    pass
30 31 32 33
try:
    from urllib.parse import urljoin
except:
    from urlparse import urljoin
34
import xml.etree.ElementTree as ET
35 36 37

# TODO:
# - add docker provider
38 39
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
# - avoid using network if possible (e.g. check first if pkgs are installed)
40 41 42


SYSTEMS = {
43
    'fedora': ['27', '28', '29', '30'],
44
    'centos': ['7'],
45 46
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
47
    'debian': ['8', '9'],
48
    'freebsd': ['11.2', '12.0'],
49 50
}

51
# pylint: disable=C0326
52
IMAGE_TEMPLATES = {
53 54
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
55
    'fedora-28-lxc':           {'bare': 'godfryd/lxc-fedora-28',       'kea': 'godfryd/kea-fedora-28'},
56 57 58
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
59 60
    'fedora-30-lxc':           {'bare': 'godfryd/lxc-fedora-30',       'kea': 'godfryd/kea-fedora-30'},
    'fedora-30-virtualbox':    {'bare': 'generic/fedora30',            'kea': 'godfryd/kea-fedora-30'},
61 62
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
63
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
64 65 66 67 68 69 70 71 72 73 74 75
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
76 77 78 79
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
80
ENV["LC_ALL"] = "C"
81 82

Vagrant.configure("2") do |config|
83
  config.vm.hostname = "{name}"
84 85

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
86
  {box_version}
87 88 89

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
90
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
91 92 93
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
94
  config.vm.synced_folder '{ccache_dir}', '/ccache'
95 96 97 98 99
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
100
ENV["LC_ALL"] = "C"
101 102

Vagrant.configure("2") do |config|
103
  config.vm.hostname = "{name}"
104 105

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
106
  {box_version}
107 108

  config.vm.provider "virtualbox" do |v|
109
    v.name = "{name}"
110 111 112 113 114 115 116 117 118 119
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
120 121

  config.vm.synced_folder '.', '/vagrant', disabled: true
122 123 124 125 126 127 128
end
"""


log = logging.getLogger()


129
def red(txt):
130
    """Return colorized (if the terminal supports it) or plain text."""
131 132
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
133
    return txt
134 135

def green(txt):
136
    """Return colorized (if the terminal supports it) or plain text."""
137 138
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
139
    return txt
140 141

def blue(txt):
142
    """Return colorized (if the terminal supports it) or plain text."""
143 144
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
145
    return txt
146 147


148
def get_system_revision():
149
    """Return tuple containing system name and its revision."""
150 151
    system = platform.system()
    if system == 'Linux':
152
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
153
        if system == 'debian':
154
            revision = revision[0]
155 156
        elif system == 'redhat':
            system = 'rhel'
157 158 159
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
160 161 162 163 164 165
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


166
class ExecutionError(Exception):
167
    """Exception thrown when execution encountered an error."""
168 169
    pass

170

171 172 173
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
            quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
            sleep_time_after_attempt=None):
174 175 176
    """Execute a command in shell.

    :param str cmd: a command to be executed
177 178
    :param int timeout: timeout in number of seconds, after that time the command is terminated
                        but only if check_times is True
179 180
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
181 182
    :param bool raise_error: if False then in case of error exception is not raised,
                             default: True ie exception is raise
183 184 185 186 187
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
188 189
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
                             used for e.g. SSH
190 191
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
192
    """
193
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
194 195 196 197
    if not check_times:
        timeout = None
    if dry_run:
        return 0
198

199 200 201
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
202

203 204
    if log_file_path:
        log_file = open(log_file_path, "wb")
205

206 207 208 209
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
210

211 212
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
213

214 215 216 217 218 219 220 221
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
222
                    line_decoded = line.decode(encoding='ascii', errors='ignore').rstrip() + '\r'
223 224 225 226 227 228 229 230 231 232 233
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
234 235 236 237 238 239
                # kill using sudo to be able to kill other sudo commands
                execute('sudo kill -s TERM %s' % p.pid)
                time.sleep(5)
                # if still running, kill harder
                if p.poll() is None:
                    execute('sudo kill -s KILL %s' % p.pid)
240 241 242
                msg = "Execution timeout, %d > %d seconds elapsed (start: %d, stop %d), cmd: '%s'"
                msg = msg % (t1 - t0, timeout, t0, t1, cmd)
                raise ExecutionError(msg)
243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
258

259
    if exitcode != 0 and raise_error:
Michal Nowikowski's avatar
Michal Nowikowski committed
260 261
        if capture and quiet:
            log.error(output)
262
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
263 264 265

    if capture:
        return exitcode, output
266
    return exitcode
267 268


Michal Nowikowski's avatar
Michal Nowikowski committed
269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
def _prepare_installed_packages_cache_for_debs():
    pkg_cache = {}

    _, out = execute("dpkg -l", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        line = line.strip()
        m = re.search('^([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+(.+)', line)
        if not m:
            continue
        status, name, version, arch, descr = m.groups()
        name = name.split(':')[0]
        pkg_cache[name] = dict(status=status, version=version, arch=arch, descr=descr)

    return pkg_cache


def _prepare_installed_packages_cache_for_rpms():
    pkg_cache = {}

    _, out = execute("rpm -qa --qf '%{NAME}\\n'", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        name = line.strip()
        pkg_cache[name] = dict(status='ii')

    return pkg_cache


def install_pkgs(pkgs, timeout=60, env=None, check_times=False, pkg_cache={}):
299
    """Install native packages in a system.
300 301 302 303 304 305 306

    :param dict pkgs: specifies a list of packages to be installed
    :param int timeout: timeout in number of seconds, after that time the command
                        is terminated but only if check_times is True
    :param dict env: dictionary with environment variables (optional)
    :param bool check_times: specifies if timeouts should be enabled (optional)
    """
307 308
    system, revision = get_system_revision()

Michal Nowikowski's avatar
Michal Nowikowski committed
309 310 311 312 313 314 315 316 317 318
    if not isinstance(pkgs, list):
        pkgs = pkgs.split()

    # prepare cache if needed
    if not pkg_cache and system in ['centos', 'rhel', 'fedora', 'debian', 'ubuntu']:
        if system in ['centos', 'rhel', 'fedora']:
            pkg_cache.update(_prepare_installed_packages_cache_for_rpms())
        elif system in ['debian', 'ubuntu']:
            pkg_cache.update(_prepare_installed_packages_cache_for_debs())

319 320
    # check if packages actually need to be installed
    if pkg_cache:
Michal Nowikowski's avatar
Michal Nowikowski committed
321 322 323 324 325 326 327 328 329 330
        pkgs_to_install = []
        for pkg in pkgs:
            if pkg not in pkg_cache or pkg_cache[pkg]['status'] != 'ii':
                pkgs_to_install.append(pkg)
        pkgs = pkgs_to_install

    if not pkgs:
        log.info('all packages already installed')
        return

331 332 333 334 335 336 337
    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
Michal Nowikowski's avatar
Michal Nowikowski committed
338
        # prepare the command for ubuntu/debian
339 340 341 342 343 344
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'
Michal Nowikowski's avatar
Michal Nowikowski committed
345 346
    else:
        raise NotImplementedError
347

Michal Nowikowski's avatar
Michal Nowikowski committed
348
    pkgs = ' '.join(pkgs)
349 350
    cmd += ' ' + pkgs
    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
351 352


353 354 355 356 357 358 359 360 361
def _get_full_repo_url(repository_url, system, revision, pkg_version):
    if not repository_url:
        return None
    repo_name = 'kea-%s-%s-%s' % (pkg_version.rsplit('.', 1)[0], system, revision)
    repo_url = urljoin(repository_url, 'repository')
    repo_url += '/%s-ci/' % repo_name
    return repo_url


362
class VagrantEnv(object):
363 364 365 366 367 368 369
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

370
    def __init__(self, provider, system, revision, features, image_template_variant,
371
                 dry_run, quiet=False, check_times=False, ccache_dir=None):
372 373 374 375 376 377 378 379 380 381 382
        """VagrantEnv initializer.

        :param str provider: indicate backend type: virtualbox or lxc
        :param str system: name of the system eg. ubuntu
        :param str revision: revision of the system e.g. 18.04
        :param list features: list of requested features
        :param str image_template_variant: variant of images' templates: bare or kea
        :param bool dry_run: if False then system commands are not really executed
        :param bool quiet: if True then commands will not trace to stdout
        :param bool check_times: if True then commands will be terminated after given timeout
        """
383
        self.provider = provider
384
        self.system = system
385
        self.revision = revision
386
        self.features = features
387 388 389
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
390

391 392 393 394 395
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

396 397 398 399 400
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

Michal Nowikowski's avatar
Michal Nowikowski committed
401 402
        self.key = key = "%s-%s-%s" % (system, revision, provider)
        self.image_tpl = image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
403 404
        self.repo_dir = os.getcwd()

405
        sys_dir = "%s-%s" % (system, revision)
406
        if provider == "virtualbox":
407
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
408
        elif provider == "lxc":
409 410 411 412
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
413

414 415
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
416

417
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
418 419 420 421 422

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

423
        crc = binascii.crc32(self.vagrant_dir.encode())
424
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
425

426 427 428 429 430 431
        if ccache_dir is None:
            ccache_dir = '/'
            self.ccache_enabled = False
        else:
            self.ccache_enabled = True

Michal Nowikowski's avatar
Michal Nowikowski committed
432 433 434 435 436 437 438
        if '/' in image_tpl:
            self.latest_version = self._get_latest_cloud_version()
            box_version = 'config.vm.box_version = "%s"' % self.latest_version
        else:
            self.latest_version = None
            box_version = ""

439
        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
440
                                             name=self.name,
Michal Nowikowski's avatar
Michal Nowikowski committed
441 442
                                             ccache_dir=ccache_dir,
                                             box_version=box_version)
443

444 445 446
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

447 448
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

449
    def up(self):
450
        """Do Vagrant up."""
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
        exitcode, out = execute("vagrant up --no-provision --provider %s" % self.provider,
                                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run,
                                capture=True, raise_error=False)
        if exitcode != 0:
            if 'There is container on your system' in out and 'lxc-destroy' in out:
                m = re.search('`lxc-destroy.*?`', out)
                if m:
                    # destroy some old container
                    cmd = m.group(0)[1:-1]
                    cmd = 'sudo ' + cmd + ' -f'
                    execute(cmd, timeout=60)

                    # try again spinning up new
                    execute("vagrant up --no-provision --provider %s" % self.provider,
                            cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
                    return
            raise ExecutionError('There is a problem with putting up a system')
468

Michal Nowikowski's avatar
Michal Nowikowski committed
469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
    def _get_cloud_meta(self, image_tpl=None):
        if '/' not in self.image_tpl:
            return {}
        url = 'https://app.vagrantup.com/api/v1/box/' + (image_tpl if image_tpl else self.image_tpl)
        try:
            with urllib.request.urlopen(url) as response:
                data = response.read()
        except:
            log.exception('ignored exception')
            return {}
        data = json.loads(data)
        return data

    def _get_local_meta(self):
        meta_file = os.path.join(self.vagrant_dir, '.vagrant/machines/default', self.provider, 'box_meta')
        if not os.path.exists(meta_file):
            return {}
        with open(meta_file) as f:
            data = f.read()
        data = json.loads(data)
        return data

    def _get_latest_cloud_version(self, image_tpl=None):
        cloud_meta = self._get_cloud_meta(image_tpl)
        if not cloud_meta and 'versions' not in cloud_meta:
            return 0
        latest_version = 0
        for ver in cloud_meta['versions']:
            provider_found = False
            for p in ver['providers']:
                if p['name'] == self.provider:
                    provider_found = True
                    break
            if provider_found:
503 504 505 506
                try:
                    v = int(ver['number'])
                except:
                    return ver['number']
Michal Nowikowski's avatar
Michal Nowikowski committed
507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
                if v > latest_version:
                    latest_version = v
        return latest_version

    def get_status(self):
        """Return system status.

        Status can be: 'not created', 'running', 'stopped', etc.
        """
        _, out = execute("vagrant status", cwd=self.vagrant_dir, timeout=15, capture=True, quiet=True)
        m = re.search('default\s+(.+)\(', out)
        if not m:
            raise Exception('cannot get status in:\n%s' % out)
        return m.group(1).strip()

    def bring_up_latest_box(self):
        if self.get_status() == 'running':
            self.reload()
        else:
            self.up()

    def reload(self):
        """Do Vagrant reload."""
        execute("vagrant reload --no-provision --force",
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)


534
    def package(self):
535 536
        """Package Vagrant system into Vagrant box."""

537
        if self.provider == 'virtualbox':
Michal Nowikowski's avatar
Michal Nowikowski committed
538 539
            box_path = "kea-%s-%s.box" % (self.system, self.revision)
            cmd = "vagrant package --output %s" % box_path
540 541 542 543 544
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

545
            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
546 547 548 549 550 551 552 553
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
554 555 556 557 558 559 560 561 562 563 564 565
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
                    'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
                    '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
                    'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
                    'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
                    'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
                    'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
                    '> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "'
            cmd += 'cd %s '
            cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
            cmd += '"'
566
            execute(cmd % (lxc_container_path, lxc_box_dir))
567 568 569 570 571 572 573 574 575 576 577 578
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
579

Michal Nowikowski's avatar
Michal Nowikowski committed
580 581 582 583 584 585 586 587 588 589 590 591 592 593 594
        return box_path

    def upload_to_cloud(self, box_path):
        image_tpl = IMAGE_TEMPLATES[self.key]['kea']
        if '/' not in image_tpl:
            return

        latest_version = self._get_latest_cloud_version(image_tpl)
        new_version = latest_version + 1

        cmd = "vagrant cloud publish -f -r %s %s %s %s"
        cmd = cmd % (image_tpl, new_version, self.provider, box_path)

        execute(cmd, cwd=self.vagrant_dir, timeout=60 * 60)

595
    def upload(self, src):
596
        """Upload src to Vagrant system, home folder."""
597 598 599 600 601 602 603 604 605 606 607
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

608
    def run_build_and_test(self, tarball_path, jobs, pkg_version, pkg_isc_version, upload, repository_url):
609
        """Run build and unit tests inside Vagrant system."""
610 611 612
        if self.dry_run:
            return 0, 0

613
        # prepare tarball if needed and upload it to vagrant system
614
        if not tarball_path:
615
            name_ver = 'kea-%s' % pkg_version
616
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
617 618
            cmd += ' --exclude "*~" --exclude .git --exclude .libs '
            cmd += ' --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
619 620
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
621
            tarball_path = '/tmp/%s.tar.gz' % name_ver
622
        self.upload(tarball_path)
623 624 625

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
626 627

        t0 = time.time()
628

629
        # run build command
630 631 632 633 634 635 636 637 638 639 640 641 642
        bld_cmd = "{python} hammer.py build -p local {features} {nofeatures} {check_times} {ccache}"
        bld_cmd += " {tarball} {jobs} {pkg_version} {pkg_isc_version} {repository_url}"
        bld_cmd = bld_cmd.format(python=self.python,
                                 features=self.features_arg,
                                 nofeatures=self.nofeatures_arg,
                                 check_times='-i' if self.check_times else '',
                                 ccache='--ccache-dir /ccache' if self.ccache_enabled else '',
                                 tarball='-t %s.tar.gz' % name_ver,
                                 jobs='-j %d' % jobs,
                                 pkg_version='--pkg-version %s' % pkg_version,
                                 pkg_isc_version='--pkg-isc-version %s' % pkg_isc_version,
                                 repository_url=('--repository-url %s' % repository_url) if repository_url else '')

643 644
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
645 646 647 648

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683
            pkgs_dir = os.path.join(self.vagrant_dir, 'pkgs')
            if os.path.exists(pkgs_dir):
                execute('rm -rf %s' % pkgs_dir)
            os.makedirs(pkgs_dir)

            if self.system in ['ubuntu', 'debian']:
                # TODO: change to pkgs folder
                execute('scp -F %s -r default:/home/vagrant/kea-src/isc-kea_* .' % ssh_cfg_path, cwd=pkgs_dir)
                execute('scp -F %s -r default:/home/vagrant/kea-src/*deb .' % ssh_cfg_path, cwd=pkgs_dir)
            elif self.system in ['fedora', 'centos', 'rhel']:
                execute('scp -F %s -r default:/home/vagrant/pkgs/* .' % ssh_cfg_path, cwd=pkgs_dir)
            else:
                raise NotImplementedError

            if upload:
                repo_url = _get_full_repo_url(repository_url, self.system, self.revision, pkg_version)
                assert repo_url is not None
                upload_cmd = 'curl -v --netrc -f'

                if self.system in ['ubuntu', 'debian']:
                    upload_cmd += ' -X POST -H "Content-Type: multipart/form-data" --data-binary "@%s" '
                    file_ext = '.deb'

                elif self.system in ['fedora', 'centos', 'rhel']:
                    upload_cmd += ' --upload-file %s '
                    file_ext = '.rpm'

                upload_cmd += ' ' + repo_url

                for fn in os.listdir(pkgs_dir):
                    if not fn.endswith(file_ext):
                        continue
                    fp = os.path.join(pkgs_dir, fn)
                    cmd = upload_cmd % fp
                    execute(cmd)
684

685 686
        t1 = time.time()
        dt = int(t1 - t0)
687 688

        log.info('Build log file stored to %s', log_file_path)
689 690 691 692
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

693
        # run unit tests if requested
694 695 696 697
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
698 699
                cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
700 701 702 703 704 705 706
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
Michal Nowikowski's avatar
Michal Nowikowski committed
707 708 709

                cmd = 'scp -F %s -r default:/home/vagrant/aggregated_tests.xml .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
710
        except:  # pylint: disable=bare-except
711 712 713 714
            log.exception('ignored issue with parsing unit test results')

        return total, passed

715
    def destroy(self):
716
        """Remove the VM completely."""
717
        cmd = 'vagrant destroy --force'
718
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
719 720

    def ssh(self):
721
        """Open interactive session to the VM."""
722
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
723 724

    def dump_ssh_config(self):
725
        """Dump ssh config that allows getting into Vagrant system via SSH."""
726 727 728 729 730
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
731
        """Execute provided command inside Vagrant system."""
732 733 734
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
735

736 737 738
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
                       raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
                       quiet=quiet, check_times=self.check_times)
739

740
    def prepare_system(self):
741
        """Prepare Vagrant system for building Kea."""
742 743
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
744 745 746
        else:
            self.features_arg = ''

747
        nofeatures = set(DEFAULT_FEATURES) - self.features
748 749 750 751 752
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

753
        # select proper python version for running Hammer inside Vagrant system
754 755
        if (self.system == 'centos' and self.revision == '7' or
            (self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
756 757 758 759 760 761
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

762
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
763 764 765
        if self.system == 'rhel' and self.revision == '8':
            cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
            exitcode = self.execute(cmd, raise_error=False)
766
            if exitcode != 0:
767 768 769 770 771
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
772 773 774 775 776
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

777
        # upload Hammer to Vagrant system
778
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
779
        self.upload(hmr_py_path)
780

781 782 783
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

Michal Nowikowski's avatar
Michal Nowikowski committed
784 785
        t0 = time.time()

786
        # run prepare-system inside Vagrant system
787
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times} {ccache}"
788 789
        cmd = cmd.format(python=self.python,
                         features=self.features_arg,
790
                         nofeatures=self.nofeatures_arg,
791 792
                         check_times='-i' if self.check_times else '',
                         ccache='--ccache-dir /ccache' if self.ccache_enabled else '')
793
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
794

Michal Nowikowski's avatar
Michal Nowikowski committed
795 796 797 798 799 800 801 802
        t1 = time.time()
        dt = int(t1 - t0)

        log.info('')
        log.info(">>> Preparing %s, %s, %s completed in %s:%s", self.provider, self.system, self.revision,
                 dt // 60, dt % 60)
        log.info('')

803 804

def _install_gtest_sources():
805
    """Install gtest sources."""
806
    # download gtest sources only if it is not present as native package
807
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
808 809 810
        cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
        cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
        execute(cmd)
811
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
812 813 814
        os.unlink('/tmp/gtest.tar.gz')


815
def _configure_mysql(system, revision, features):
816
    """Configure MySQL database."""
817 818 819 820
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
821 822

    if system == 'freebsd':
823 824
        cmd = "echo 'SET PASSWORD = \"\";' "
        cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
825 826
        execute(cmd, raise_error=False)

827 828 829 830 831 832 833 834 835 836 837 838 839 840 841
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

842 843 844 845 846 847 848 849 850 851 852 853
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

854 855 856 857 858 859 860 861 862 863 864 865 866
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

867

868
def _configure_pgsql(system, features):
869 870
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
871
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
872
        if exitcode != 0:
Michal Nowikowski's avatar
Michal Nowikowski committed
873 874 875 876
            if system == 'centos':
                execute('sudo postgresql-setup initdb')
            else:
                execute('sudo postgresql-setup --initdb --unit postgresql')
877
    execute('sudo systemctl start postgresql.service')
Michal Nowikowski's avatar
Michal Nowikowski committed
878
    execute('sudo systemctl enable postgresql.service')
879 880 881 882 883 884 885 886 887 888 889 890
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

891 892 893 894 895 896 897 898 899
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)
Michal Nowikowski's avatar
Michal Nowikowski committed
900 901 902 903 904 905
        # TODO: in /etc/postgresql/10/main/pg_hba.conf
        # change:
        #    local   all             all                                     peer
        # to:
        #    local   all             all                                     md5
    log.info('postgresql just configured')
906

907

Michal Nowikowski's avatar
Michal Nowikowski committed
908
def _install_cassandra_deb(system, revision, env, check_times):
909
    """Install Cassandra and cpp-driver using DEB package."""
910
    if not os.path.exists('/usr/sbin/cassandra'):
911 912 913 914
        cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
        cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
        execute(cmd, env=env, check_times=check_times)
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
915 916
                env=env, check_times=check_times)
        execute('sudo apt update', env=env, check_times=check_times)
917
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
918 919

    if not os.path.exists('/usr/include/cassandra.h'):
Michal Nowikowski's avatar
Michal Nowikowski committed
920 921 922 923 924 925 926 927 928 929
        if system == 'ubuntu' and revision == '16.04':
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
        else:
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
930 931 932 933
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
934 935


936
def _install_cassandra_rpm(system, revision, env, check_times):
937
    """Install Cassandra and cpp-driver using RPM package."""
938
    if not os.path.exists('/usr/bin/cassandra'):
Michal Nowikowski's avatar
Michal Nowikowski committed
939 940
        if system == 'centos':
            install_pkgs('yum-utils', env=env, check_times=check_times)
941
            execute('sudo yum-config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/', raise_error=False)
Michal Nowikowski's avatar
Michal Nowikowski committed
942 943 944 945 946
            execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
            pkgs = 'cassandra cassandra-tools libuv libuv-devel openssl'
        else:
            pkgs = 'cassandra cassandra-server libuv libuv-devel'
        install_pkgs(pkgs, env=env, check_times=check_times)
947

Michal Nowikowski's avatar
Michal Nowikowski committed
948 949
    if system == 'centos':
        execute('sudo systemctl daemon-reload')
950 951 952

    if system == 'fedora' and revision == '30':
        execute("echo '-Xms1G -Xmx1G' | sudo tee -a /etc/cassandra/jvm.options")
953 954 955 956 957
    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
958 959 960 961
        if system == 'centos':
            execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        else:
            execute('sudo dnf install -y cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
962 963 964
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


965
def _install_freeradius_client(system, revision, features, env, check_times):
Michal Nowikowski's avatar
Michal Nowikowski committed
966
    """Install FreeRADIUS-client with necessary patches from Francis Dupont."""
967
    # check if it is already installed
Michal Nowikowski's avatar
Michal Nowikowski committed
968 969 970 971
    if (os.path.exists('/usr/local/lib/libfreeradius-client.so.2.0.0') and
        os.path.exists('/usr/local/include/freeradius-client.h')):
        log.info('freeradius is already installed')
        return
972 973 974 975 976 977 978 979 980 981

    # install freeradius dependencies
    if system in ['centos', 'rhel', 'fedora']:
        install_pkgs('nettle-devel', env=env, check_times=check_times)
    elif system in ['debian', 'ubuntu']:
        install_pkgs('nettle-dev', env=env, check_times=check_times)
    else:
        raise NotImplementedError

    # checkout sources, build them and install
Michal Nowikowski's avatar
Michal Nowikowski committed
982 983 984
    execute('rm -rf freeradius-client')
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
985
    execute('./configure --with-nettle', cwd='freeradius-client', env=env, check_times=check_times)
Michal Nowikowski's avatar
Michal Nowikowski committed
986 987 988 989 990 991 992
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
    execute('rm -rf freeradius-client')
    log.info('freeradius just installed')


993
def prepare_system_local(features, check_times):
994
    """Prepare local system for Kea development based on requested features."""
995 996 997
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

998 999 1000
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

1001
    # prepare fedora
1002
    if system == 'fedora':
1003 1004
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel']
1005 1006

        if 'native-pkg' in features:
1007
            packages.extend(['rpm-build', 'python2-devel', 'python3-devel'])
1008

1009 1010 1011 1012
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
1013
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-connector-c-devel'])
1014 1015 1016

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])
1017 1018
            if revision in ['30']:
                packages.extend(['postgresql-server-devel'])
1019

1020 1021 1022
        if 'radius' in features:
            packages.extend(['git'])

1023 1024 1025
        if 'ccache' in features:
            packages.extend(['ccache'])

1026
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
1027 1028 1029 1030

        if 'unittest' in features:
            _install_gtest_sources()

1031
        execute('sudo dnf clean packages', env=env, check_times=check_times)
1032

1033
        if 'cql' in features:
1034
            _install_cassandra_rpm(system, revision, env, check_times)
1035

1036
    # prepare centos
1037
    elif system == 'centos':
1038
        install_pkgs('epel-release', env=env, check_times=check_times)
1039

1040 1041
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
1042

1043 1044 1045
        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'python2-devel'])

1046
        if 'docs' in features:
1047 1048 1049 1050
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
1051

1052 1053 1054
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

1055 1056 1057
        if 'radius' in features:
            packages.extend(['git'])

1058 1059 1060
        if 'ccache' in features:
            packages.extend(['ccache'])

1061
        install_pkgs(packages, env=env, check_times=check_times)
1062 1063 1064 1065

        if 'unittest' in features:
            _install_gtest_sources()

1066
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1067
            _install_cassandra_rpm(system, env, check_times)
1068

1069
    # prepare rhel
1070
    elif system == 'rhel':
1071 1072 1073 1074
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

1075 1076
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
1077

1078
        # TODO:
1079 1080 1081 1082 1083 1084
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

1085 1086
        if 'radius' in features:
            packages.extend(['git'])
1087 1088
            if 'forge' in features:
                packages.extend(['freeradius'])
1089

1090 1091 1092
        if 'ccache' in features:
            packages.extend(['ccache'])

1093
        install_pkgs(packages, env=env, timeout=120, check_times=check_times)
1094 1095 1096

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
1097 1098 1099 1100
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
1101 1102
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
1103
                    check_times=check_times)
1104 1105 1106 1107 1108 1109
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    env=env, timeout=120, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
1110 1111 1112 1113

        if 'unittest' in features:
            _install_gtest_sources()

1114
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1115
            _install_cassandra_rpm(system, env, check_times)
1116

1117
    # prepare ubuntu
1118
    elif system == 'ubuntu':
1119
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
1120

1121 1122
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev',
                    'libboost-system-dev']
1123 1124

        if 'unittest' in features:
1125 1126 1127 1128
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
1129 1130

        if 'docs' in features:
1131
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
1132 1133 1134

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
1135
            packages.extend(['bison', 'debhelper', 'docbook', 'flex', 'libboost-dev', 'python3-dev'])
1136

1137
        if 'mysql' in features:
1138 1139 1140 1141
            if revision == '16.04':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
1142 1143

        if 'pgsql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1144 1145 1146 1147
            if revision == '16.04':
                packages.extend(['postgresql-client', 'libpq-dev', 'postgresql', 'postgresql-server-dev-all'])
            else:
                packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])
1148

1149 1150 1151
        if 'radius' in features:
            packages.extend(['git'])

1152 1153 1154
        if 'ccache' in features:
            packages.extend(['ccache'])

1155
        install_pkgs(packages, env=env, timeout=240, check_times=check_times)
1156 1157

        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1158
            _install_cassandra_deb(system, revision, env, check_times)
1159

1160
    # prepare debian
1161
    elif system == 'debian':
1162
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
1163