hammer.py 88.2 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11
from __future__ import print_function
import os
12
import re
13 14 15
import sys
import glob
import time
16
import json
17
import logging
18
import datetime
19 20 21
import platform
import binascii
import argparse
22 23
import textwrap
import functools
24
import subprocess
25
import multiprocessing
Michal Nowikowski's avatar
Michal Nowikowski committed
26 27 28 29
try:
    import urllib.request
except:
    pass
30 31 32 33
try:
    from urllib.parse import urljoin
except:
    from urlparse import urljoin
34
import xml.etree.ElementTree as ET
35 36 37

# TODO:
# - add docker provider
38 39
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
# - avoid using network if possible (e.g. check first if pkgs are installed)
40 41 42


SYSTEMS = {
43
    'fedora': ['27', '28', '29', '30'],
44
    'centos': ['7'],
45
    'rhel': ['8'],
46
    'ubuntu': ['16.04', '18.04', '18.10', '19.04'],
47
    'debian': ['8', '9', '10'],
48
    'freebsd': ['11.2', '12.0'],
49 50
}

51
# pylint: disable=C0326
52
IMAGE_TEMPLATES = {
53 54
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
55
    'fedora-28-lxc':           {'bare': 'godfryd/lxc-fedora-28',       'kea': 'godfryd/kea-fedora-28'},
56 57 58
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
59 60
    'fedora-30-lxc':           {'bare': 'godfryd/lxc-fedora-30',       'kea': 'godfryd/kea-fedora-30'},
    'fedora-30-virtualbox':    {'bare': 'generic/fedora30',            'kea': 'godfryd/kea-fedora-30'},
61 62
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
63
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
64 65 66 67 68 69
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
70 71
    'ubuntu-19.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-19.04',    'kea': 'godfryd/kea-ubuntu-19.04'},
    'ubuntu-19.04-virtualbox': {'bare': 'ubuntu/disco64',              'kea': 'godfryd/kea-ubuntu-19.04'},
72 73 74 75
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
76 77
    'debian-10-lxc':           {'bare': 'godfryd/lxc-debian-10',       'kea': 'godfryd/kea-debian-10'},
    'debian-10-virtualbox':    {'bare': 'debian/buster64',             'kea': 'godfryd/kea-debian-10'},
78 79
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
80 81 82 83
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
84
ENV["LC_ALL"] = "C"
85 86

Vagrant.configure("2") do |config|
87
  config.vm.hostname = "{name}"
88 89

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
90
  {box_version}
91 92 93

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
94
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
95 96 97
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
98
  config.vm.synced_folder '{ccache_dir}', '/ccache'
99 100 101 102 103
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
104
ENV["LC_ALL"] = "C"
105 106

Vagrant.configure("2") do |config|
107
  config.vm.hostname = "{name}"
108 109

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
110
  {box_version}
111 112

  config.vm.provider "virtualbox" do |v|
113
    v.name = "{name}"
114 115 116 117 118 119 120 121 122 123
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
124 125

  config.vm.synced_folder '.', '/vagrant', disabled: true
126 127 128 129 130 131 132
end
"""


log = logging.getLogger()


133
def red(txt):
134
    """Return colorized (if the terminal supports it) or plain text."""
135 136
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
137
    return txt
138 139

def green(txt):
140
    """Return colorized (if the terminal supports it) or plain text."""
141 142
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
143
    return txt
144 145

def blue(txt):
146
    """Return colorized (if the terminal supports it) or plain text."""
147 148
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
149
    return txt
150 151


152
def get_system_revision():
153
    """Return tuple containing system name and its revision."""
154 155
    system = platform.system()
    if system == 'Linux':
156
        system, revision, _ = platform.dist()  # pylit: disable=deprecated-method
157
        if system == 'debian':
158
            revision = revision.split('.')[0]
159 160
        elif system == 'redhat':
            system = 'rhel'
161 162 163
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
164 165 166 167 168 169
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


170
class ExecutionError(Exception):
171
    """Exception thrown when execution encountered an error."""
172 173
    pass

174

175 176 177
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
            quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
            sleep_time_after_attempt=None):
178 179 180
    """Execute a command in shell.

    :param str cmd: a command to be executed
181 182
    :param int timeout: timeout in number of seconds, after that time the command is terminated
                        but only if check_times is True
183 184
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
185 186
    :param bool raise_error: if False then in case of error exception is not raised,
                             default: True ie exception is raise
187 188 189 190 191
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
192 193
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
                             used for e.g. SSH
194 195
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
196
    """
197
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
198 199 200 201
    if not check_times:
        timeout = None
    if dry_run:
        return 0
202

203 204 205
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
206

207 208
    if log_file_path:
        log_file = open(log_file_path, "wb")
209

210 211 212 213
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
214

215 216
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
217

218 219 220 221 222 223 224 225
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
226
                    line_decoded = line.decode(encoding='ascii', errors='ignore').rstrip() + '\r'
227 228 229 230 231 232 233 234 235 236 237
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
238 239 240 241 242 243
                # kill using sudo to be able to kill other sudo commands
                execute('sudo kill -s TERM %s' % p.pid)
                time.sleep(5)
                # if still running, kill harder
                if p.poll() is None:
                    execute('sudo kill -s KILL %s' % p.pid)
244 245 246
                msg = "Execution timeout, %d > %d seconds elapsed (start: %d, stop %d), cmd: '%s'"
                msg = msg % (t1 - t0, timeout, t0, t1, cmd)
                raise ExecutionError(msg)
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
262

263
    if exitcode != 0 and raise_error:
Michal Nowikowski's avatar
Michal Nowikowski committed
264 265
        if capture and quiet:
            log.error(output)
266
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
267 268 269

    if capture:
        return exitcode, output
270
    return exitcode
271 272


Michal Nowikowski's avatar
Michal Nowikowski committed
273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
def _prepare_installed_packages_cache_for_debs():
    pkg_cache = {}

    _, out = execute("dpkg -l", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        line = line.strip()
        m = re.search('^([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+(.+)', line)
        if not m:
            continue
        status, name, version, arch, descr = m.groups()
        name = name.split(':')[0]
        pkg_cache[name] = dict(status=status, version=version, arch=arch, descr=descr)

    return pkg_cache


def _prepare_installed_packages_cache_for_rpms():
    pkg_cache = {}

    _, out = execute("rpm -qa --qf '%{NAME}\\n'", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        name = line.strip()
        pkg_cache[name] = dict(status='ii')

    return pkg_cache


def install_pkgs(pkgs, timeout=60, env=None, check_times=False, pkg_cache={}):
303
    """Install native packages in a system.
304 305 306 307 308 309 310

    :param dict pkgs: specifies a list of packages to be installed
    :param int timeout: timeout in number of seconds, after that time the command
                        is terminated but only if check_times is True
    :param dict env: dictionary with environment variables (optional)
    :param bool check_times: specifies if timeouts should be enabled (optional)
    """
311 312
    system, revision = get_system_revision()

Michal Nowikowski's avatar
Michal Nowikowski committed
313 314 315 316 317 318 319 320 321 322
    if not isinstance(pkgs, list):
        pkgs = pkgs.split()

    # prepare cache if needed
    if not pkg_cache and system in ['centos', 'rhel', 'fedora', 'debian', 'ubuntu']:
        if system in ['centos', 'rhel', 'fedora']:
            pkg_cache.update(_prepare_installed_packages_cache_for_rpms())
        elif system in ['debian', 'ubuntu']:
            pkg_cache.update(_prepare_installed_packages_cache_for_debs())

323 324
    # check if packages actually need to be installed
    if pkg_cache:
Michal Nowikowski's avatar
Michal Nowikowski committed
325 326 327 328 329 330 331 332 333 334
        pkgs_to_install = []
        for pkg in pkgs:
            if pkg not in pkg_cache or pkg_cache[pkg]['status'] != 'ii':
                pkgs_to_install.append(pkg)
        pkgs = pkgs_to_install

    if not pkgs:
        log.info('all packages already installed')
        return

335 336 337 338 339 340 341
    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
Michal Nowikowski's avatar
Michal Nowikowski committed
342
        # prepare the command for ubuntu/debian
343 344 345 346 347 348
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'
Michal Nowikowski's avatar
Michal Nowikowski committed
349 350
    else:
        raise NotImplementedError
351

Michal Nowikowski's avatar
Michal Nowikowski committed
352
    pkgs = ' '.join(pkgs)
353 354
    cmd += ' ' + pkgs
    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
355 356


357 358 359 360 361 362 363 364 365
def _get_full_repo_url(repository_url, system, revision, pkg_version):
    if not repository_url:
        return None
    repo_name = 'kea-%s-%s-%s' % (pkg_version.rsplit('.', 1)[0], system, revision)
    repo_url = urljoin(repository_url, 'repository')
    repo_url += '/%s-ci/' % repo_name
    return repo_url


366
class VagrantEnv(object):
367 368 369 370 371 372 373
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

374
    def __init__(self, provider, system, revision, features, image_template_variant,
375
                 dry_run, quiet=False, check_times=False, ccache_dir=None):
376 377 378 379 380 381 382 383 384 385 386
        """VagrantEnv initializer.

        :param str provider: indicate backend type: virtualbox or lxc
        :param str system: name of the system eg. ubuntu
        :param str revision: revision of the system e.g. 18.04
        :param list features: list of requested features
        :param str image_template_variant: variant of images' templates: bare or kea
        :param bool dry_run: if False then system commands are not really executed
        :param bool quiet: if True then commands will not trace to stdout
        :param bool check_times: if True then commands will be terminated after given timeout
        """
387
        self.provider = provider
388
        self.system = system
389
        self.revision = revision
390
        self.features = features
391 392 393
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
394

395 396 397 398 399
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

400 401 402 403 404
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

Michal Nowikowski's avatar
Michal Nowikowski committed
405 406
        self.key = key = "%s-%s-%s" % (system, revision, provider)
        self.image_tpl = image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
407 408
        self.repo_dir = os.getcwd()

409
        sys_dir = "%s-%s" % (system, revision)
410
        if provider == "virtualbox":
411
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
412
        elif provider == "lxc":
413 414 415 416
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
417

418 419
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
420

421
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
422 423 424 425 426

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

427
        crc = binascii.crc32(self.vagrant_dir.encode())
428
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
429

430 431 432 433 434 435
        if ccache_dir is None:
            ccache_dir = '/'
            self.ccache_enabled = False
        else:
            self.ccache_enabled = True

Michal Nowikowski's avatar
Michal Nowikowski committed
436 437 438 439 440 441 442
        if '/' in image_tpl:
            self.latest_version = self._get_latest_cloud_version()
            box_version = 'config.vm.box_version = "%s"' % self.latest_version
        else:
            self.latest_version = None
            box_version = ""

443
        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
444
                                             name=self.name,
Michal Nowikowski's avatar
Michal Nowikowski committed
445 446
                                             ccache_dir=ccache_dir,
                                             box_version=box_version)
447

448 449 450
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

451 452
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

453
    def up(self):
454
        """Do Vagrant up."""
455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471
        exitcode, out = execute("vagrant up --no-provision --provider %s" % self.provider,
                                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run,
                                capture=True, raise_error=False)
        if exitcode != 0:
            if 'There is container on your system' in out and 'lxc-destroy' in out:
                m = re.search('`lxc-destroy.*?`', out)
                if m:
                    # destroy some old container
                    cmd = m.group(0)[1:-1]
                    cmd = 'sudo ' + cmd + ' -f'
                    execute(cmd, timeout=60)

                    # try again spinning up new
                    execute("vagrant up --no-provision --provider %s" % self.provider,
                            cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
                    return
            raise ExecutionError('There is a problem with putting up a system')
472

Michal Nowikowski's avatar
Michal Nowikowski committed
473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506
    def _get_cloud_meta(self, image_tpl=None):
        if '/' not in self.image_tpl:
            return {}
        url = 'https://app.vagrantup.com/api/v1/box/' + (image_tpl if image_tpl else self.image_tpl)
        try:
            with urllib.request.urlopen(url) as response:
                data = response.read()
        except:
            log.exception('ignored exception')
            return {}
        data = json.loads(data)
        return data

    def _get_local_meta(self):
        meta_file = os.path.join(self.vagrant_dir, '.vagrant/machines/default', self.provider, 'box_meta')
        if not os.path.exists(meta_file):
            return {}
        with open(meta_file) as f:
            data = f.read()
        data = json.loads(data)
        return data

    def _get_latest_cloud_version(self, image_tpl=None):
        cloud_meta = self._get_cloud_meta(image_tpl)
        if not cloud_meta and 'versions' not in cloud_meta:
            return 0
        latest_version = 0
        for ver in cloud_meta['versions']:
            provider_found = False
            for p in ver['providers']:
                if p['name'] == self.provider:
                    provider_found = True
                    break
            if provider_found:
507 508 509 510
                try:
                    v = int(ver['number'])
                except:
                    return ver['number']
Michal Nowikowski's avatar
Michal Nowikowski committed
511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537
                if v > latest_version:
                    latest_version = v
        return latest_version

    def get_status(self):
        """Return system status.

        Status can be: 'not created', 'running', 'stopped', etc.
        """
        _, out = execute("vagrant status", cwd=self.vagrant_dir, timeout=15, capture=True, quiet=True)
        m = re.search('default\s+(.+)\(', out)
        if not m:
            raise Exception('cannot get status in:\n%s' % out)
        return m.group(1).strip()

    def bring_up_latest_box(self):
        if self.get_status() == 'running':
            self.reload()
        else:
            self.up()

    def reload(self):
        """Do Vagrant reload."""
        execute("vagrant reload --no-provision --force",
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)


538
    def package(self):
539
        """Package Vagrant system into Vagrant box."""
540
        execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
541

542 543 544
        box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
        if os.path.exists(box_path):
            os.unlink(box_path)
545

546
        if self.provider == 'virtualbox':
Michal Nowikowski's avatar
Michal Nowikowski committed
547
            cmd = "vagrant package --output %s" % box_path
548 549 550 551 552 553 554 555
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
556 557 558 559 560 561 562 563 564 565 566 567
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
                    'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
                    '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
                    'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
                    'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
                    'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
                    'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
                    '> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "'
            cmd += 'cd %s '
            cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
            cmd += '"'
568
            execute(cmd % (lxc_container_path, lxc_box_dir))
569 570 571 572 573 574 575 576 577 578 579 580
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
581

Michal Nowikowski's avatar
Michal Nowikowski committed
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
        return box_path

    def upload_to_cloud(self, box_path):
        image_tpl = IMAGE_TEMPLATES[self.key]['kea']
        if '/' not in image_tpl:
            return

        latest_version = self._get_latest_cloud_version(image_tpl)
        new_version = latest_version + 1

        cmd = "vagrant cloud publish -f -r %s %s %s %s"
        cmd = cmd % (image_tpl, new_version, self.provider, box_path)

        execute(cmd, cwd=self.vagrant_dir, timeout=60 * 60)

597
    def upload(self, src):
598
        """Upload src to Vagrant system, home folder."""
599 600 601 602 603 604 605 606 607 608 609
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

610
    def run_build_and_test(self, tarball_path, jobs, pkg_version, pkg_isc_version, upload, repository_url):
611
        """Run build and unit tests inside Vagrant system."""
612 613 614
        if self.dry_run:
            return 0, 0

615
        # prepare tarball if needed and upload it to vagrant system
616
        if not tarball_path:
617
            name_ver = 'kea-%s' % pkg_version
618
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
619 620
            cmd += ' --exclude "*~" --exclude .git --exclude .libs '
            cmd += ' --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
621 622
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
623
            tarball_path = '/tmp/%s.tar.gz' % name_ver
624
        self.upload(tarball_path)
625 626 627

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
628 629

        t0 = time.time()
630

631
        # run build command
632 633 634 635 636 637 638 639 640 641 642 643 644
        bld_cmd = "{python} hammer.py build -p local {features} {nofeatures} {check_times} {ccache}"
        bld_cmd += " {tarball} {jobs} {pkg_version} {pkg_isc_version} {repository_url}"
        bld_cmd = bld_cmd.format(python=self.python,
                                 features=self.features_arg,
                                 nofeatures=self.nofeatures_arg,
                                 check_times='-i' if self.check_times else '',
                                 ccache='--ccache-dir /ccache' if self.ccache_enabled else '',
                                 tarball='-t %s.tar.gz' % name_ver,
                                 jobs='-j %d' % jobs,
                                 pkg_version='--pkg-version %s' % pkg_version,
                                 pkg_isc_version='--pkg-isc-version %s' % pkg_isc_version,
                                 repository_url=('--repository-url %s' % repository_url) if repository_url else '')

645 646
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
647 648 649 650

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685
            pkgs_dir = os.path.join(self.vagrant_dir, 'pkgs')
            if os.path.exists(pkgs_dir):
                execute('rm -rf %s' % pkgs_dir)
            os.makedirs(pkgs_dir)

            if self.system in ['ubuntu', 'debian']:
                # TODO: change to pkgs folder
                execute('scp -F %s -r default:/home/vagrant/kea-src/isc-kea_* .' % ssh_cfg_path, cwd=pkgs_dir)
                execute('scp -F %s -r default:/home/vagrant/kea-src/*deb .' % ssh_cfg_path, cwd=pkgs_dir)
            elif self.system in ['fedora', 'centos', 'rhel']:
                execute('scp -F %s -r default:/home/vagrant/pkgs/* .' % ssh_cfg_path, cwd=pkgs_dir)
            else:
                raise NotImplementedError

            if upload:
                repo_url = _get_full_repo_url(repository_url, self.system, self.revision, pkg_version)
                assert repo_url is not None
                upload_cmd = 'curl -v --netrc -f'

                if self.system in ['ubuntu', 'debian']:
                    upload_cmd += ' -X POST -H "Content-Type: multipart/form-data" --data-binary "@%s" '
                    file_ext = '.deb'

                elif self.system in ['fedora', 'centos', 'rhel']:
                    upload_cmd += ' --upload-file %s '
                    file_ext = '.rpm'

                upload_cmd += ' ' + repo_url

                for fn in os.listdir(pkgs_dir):
                    if not fn.endswith(file_ext):
                        continue
                    fp = os.path.join(pkgs_dir, fn)
                    cmd = upload_cmd % fp
                    execute(cmd)
686

687 688
        t1 = time.time()
        dt = int(t1 - t0)
689 690

        log.info('Build log file stored to %s', log_file_path)
691 692 693 694
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

695
        # run unit tests if requested
696 697 698 699
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
700 701
                cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
702 703 704 705 706 707 708
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
Michal Nowikowski's avatar
Michal Nowikowski committed
709 710 711

                cmd = 'scp -F %s -r default:/home/vagrant/aggregated_tests.xml .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
712
        except:  # pylint: disable=bare-except
713 714 715 716
            log.exception('ignored issue with parsing unit test results')

        return total, passed

717
    def destroy(self):
718
        """Remove the VM completely."""
719
        cmd = 'vagrant destroy --force'
720
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
721 722

    def ssh(self):
723
        """Open interactive session to the VM."""
724
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
725 726

    def dump_ssh_config(self):
727
        """Dump ssh config that allows getting into Vagrant system via SSH."""
728 729 730 731 732
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
733
        """Execute provided command inside Vagrant system."""
734 735 736
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
737

738 739 740
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
                       raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
                       quiet=quiet, check_times=self.check_times)
741

742
    def prepare_system(self):
743
        """Prepare Vagrant system for building Kea."""
744 745
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
746 747 748
        else:
            self.features_arg = ''

749
        nofeatures = set(DEFAULT_FEATURES) - self.features
750 751 752 753 754
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

755
        # select proper python version for running Hammer inside Vagrant system
756 757
        if (self.system == 'centos' and self.revision == '7' or
            (self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
758 759 760 761 762 763
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

764
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
765 766 767
        if self.system == 'rhel' and self.revision == '8':
            cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
            exitcode = self.execute(cmd, raise_error=False)
768
            if exitcode != 0:
769 770 771 772 773
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
774 775 776 777 778
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

779
        # upload Hammer to Vagrant system
780
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
781
        self.upload(hmr_py_path)
782

783 784 785
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

Michal Nowikowski's avatar
Michal Nowikowski committed
786 787
        t0 = time.time()

788
        # run prepare-system inside Vagrant system
789
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times} {ccache}"
790 791
        cmd = cmd.format(python=self.python,
                         features=self.features_arg,
792
                         nofeatures=self.nofeatures_arg,
793 794
                         check_times='-i' if self.check_times else '',
                         ccache='--ccache-dir /ccache' if self.ccache_enabled else '')
795
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
796

Michal Nowikowski's avatar
Michal Nowikowski committed
797 798 799 800 801 802 803 804
        t1 = time.time()
        dt = int(t1 - t0)

        log.info('')
        log.info(">>> Preparing %s, %s, %s completed in %s:%s", self.provider, self.system, self.revision,
                 dt // 60, dt % 60)
        log.info('')

805 806

def _install_gtest_sources():
807
    """Install gtest sources."""
808
    # download gtest sources only if it is not present as native package
809
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
810 811 812
        cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
        cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
        execute(cmd)
813
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
814 815 816
        os.unlink('/tmp/gtest.tar.gz')


817
def _configure_mysql(system, revision, features):
818
    """Configure MySQL database."""
819 820 821 822
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
823 824

    if system == 'freebsd':
825 826
        cmd = "echo 'SET PASSWORD = \"\";' "
        cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
827 828
        execute(cmd, raise_error=False)

829 830 831 832 833 834 835 836 837 838 839 840 841 842 843
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

844 845 846 847 848 849 850 851 852 853 854 855
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

856 857 858 859 860 861 862 863 864 865 866 867 868
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

869

870
def _configure_pgsql(system, features):
871 872
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
873
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
874
        if exitcode != 0:
Michal Nowikowski's avatar
Michal Nowikowski committed
875 876 877 878
            if system == 'centos':
                execute('sudo postgresql-setup initdb')
            else:
                execute('sudo postgresql-setup --initdb --unit postgresql')
879
    execute('sudo systemctl start postgresql.service')
Michal Nowikowski's avatar
Michal Nowikowski committed
880
    execute('sudo systemctl enable postgresql.service')
881 882 883 884 885 886 887 888 889 890
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
891
    execute(cmd, cwd='/tmp')  # CWD to avoid: could not change as posgres user directory to "/home/jenkins": Permission denied
892

893 894 895 896 897 898 899 900
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
901
        execute(cmd, cwd='/tmp')  # CWD to avoid: could not change as posgres user directory to "/home/jenkins": Permission denied
Michal Nowikowski's avatar
Michal Nowikowski committed
902 903 904 905 906 907
        # TODO: in /etc/postgresql/10/main/pg_hba.conf
        # change:
        #    local   all             all                                     peer
        # to:
        #    local   all             all                                     md5
    log.info('postgresql just configured')
908

909

Michal Nowikowski's avatar
Michal Nowikowski committed
910
def _install_cassandra_deb(system, revision, env, check_times):
911
    """Install Cassandra and cpp-driver using DEB package."""
912
    if not os.path.exists('/usr/sbin/cassandra'):
913 914 915 916
        cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
        cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
        execute(cmd, env=env, check_times=check_times)
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
917 918
                env=env, check_times=check_times)
        execute('sudo apt update', env=env, check_times=check_times)
919
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
920 921

    if not os.path.exists('/usr/include/cassandra.h'):
Michal Nowikowski's avatar
Michal Nowikowski committed
922 923 924 925 926 927 928 929 930 931
        if system == 'ubuntu' and revision == '16.04':
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
        else:
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
932 933 934
            if system == 'debian' and revision == '10':
                install_pkgs('multiarch-support', env=env, check_times=check_times)

935 936 937 938
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
939 940


941
def _install_cassandra_rpm(system, revision, env, check_times):
942
    """Install Cassandra and cpp-driver using RPM package."""
943
    if not os.path.exists('/usr/bin/cassandra'):
Michal Nowikowski's avatar
Michal Nowikowski committed
944 945
        if system == 'centos':
            install_pkgs('yum-utils', env=env, check_times=check_times)
946
            execute('sudo yum-config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/', raise_error=False)
Michal Nowikowski's avatar
Michal Nowikowski committed
947 948 949 950 951
            execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
            pkgs = 'cassandra cassandra-tools libuv libuv-devel openssl'
        else:
            pkgs = 'cassandra cassandra-server libuv libuv-devel'
        install_pkgs(pkgs, env=env, check_times=check_times)
952

Michal Nowikowski's avatar
Michal Nowikowski committed
953 954
    if system == 'centos':
        execute('sudo systemctl daemon-reload')
955 956 957

    if system == 'fedora' and revision == '30':
        execute("echo '-Xms1G -Xmx1G' | sudo tee -a /etc/cassandra/jvm.options")
958 959 960 961 962
    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
963 964 965 966
        if system == 'centos':
            execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        else:
            execute('sudo dnf install -y cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
967 968 969
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


970
def _install_freeradius_client(system, revision, features, env, check_times):
Michal Nowikowski's avatar
Michal Nowikowski committed
971
    """Install FreeRADIUS-client with necessary patches from Francis Dupont."""
972
    # check if it is already installed
Michal Nowikowski's avatar
Michal Nowikowski committed
973 974 975 976
    if (os.path.exists('/usr/local/lib/libfreeradius-client.so.2.0.0') and
        os.path.exists('/usr/local/include/freeradius-client.h')):
        log.info('freeradius is already installed')
        return
977 978 979 980 981 982 983 984 985 986

    # install freeradius dependencies
    if system in ['centos', 'rhel', 'fedora']:
        install_pkgs('nettle-devel', env=env, check_times=check_times)
    elif system in ['debian', 'ubuntu']:
        install_pkgs('nettle-dev', env=env, check_times=check_times)
    else:
        raise NotImplementedError

    # checkout sources, build them and install
Michal Nowikowski's avatar
Michal Nowikowski committed
987 988 989
    execute('rm -rf freeradius-client')
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
990
    execute('./configure --with-nettle', cwd='freeradius-client', env=env, check_times=check_times)
Michal Nowikowski's avatar
Michal Nowikowski committed
991 992 993 994 995 996 997
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
    execute('rm -rf freeradius-client')
    log.info('freeradius just installed')


998
def prepare_system_local(features, check_times):
999
    """Prepare local system for Kea development based on requested features."""
1000 1001 1002
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

1003 1004 1005
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

1006
    # prepare fedora
1007
    if system == 'fedora':
1008 1009
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel']
1010 1011

        if 'native-pkg' in features:
1012
            packages.extend(['rpm-build', 'python2-devel', 'python3-devel'])
1013

1014 1015 1016 1017
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
1018
            execute('sudo dnf remove -y community-mysql-devel || true')
1019
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-connector-c-devel'])
1020 1021 1022

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])
1023 1024
            if revision in ['30']:
                packages.extend(['postgresql-server-devel'])
1025

1026 1027 1028
        if 'radius' in features:
            packages.extend(['git'])

1029 1030 1031
        if 'ccache' in features:
            packages.extend(['ccache'])

1032
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
1033 1034 1035 1036

        if 'unittest' in features:
            _install_gtest_sources()

1037
        execute('sudo dnf clean packages', env=env, check_times=check_times)
1038

1039
        if 'cql' in features:
1040
            _install_cassandra_rpm(system, revision, env, check_times)
1041

1042
    # prepare centos
1043
    elif system == 'centos':
1044
        install_pkgs('epel-release', env=env, check_times=check_times)
1045

1046 1047
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
1048

1049 1050 1051
        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'python2-devel'])

1052
        if 'docs' in features:
1053 1054 1055 1056
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
1057

1058 1059 1060
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

1061 1062 1063
        if 'radius' in features:
            packages.extend(['git'])

1064 1065 1066
        if 'ccache' in features:
            packages.extend(['ccache'])

1067
        install_pkgs(packages, env=env, check_times=check_times)
1068 1069 1070 1071

        if 'unittest' in features:
            _install_gtest_sources()

1072
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1073
            _install_cassandra_rpm(system, env, check_times)
1074

1075
    # prepare rhel
1076
    elif system == 'rhel':
1077 1078 1079 1080
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

1081 1082
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
1083

1084
        # TODO:
1085 1086 1087 1088 1089 1090
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

1091 1092
        if 'radius' in features:
            packages.extend(['git'])
1093 1094
            if 'forge' in features:
                packages.extend(['freeradius'])
1095

1096 1097 1098
        if 'ccache' in features:
            packages.extend(['ccache'])

1099
        install_pkgs(packages, env=env, timeout=120, check_times=check_times)
1100 1101 1102

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
1103 1104 1105 1106
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
1107 1108
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
                    'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
1109
                    check_times=check_times)
1110 1111 1112 1113 1114 1115
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    env=env, timeout=120, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm',
                    env=env, check_times=check_times)
1116 1117 1118 1119

        if 'unittest' in features:
            _install_gtest_sources()

1120
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1121
            _install_cassandra_rpm(system, env, check_times)
1122

1123
    # prepare ubuntu
1124
    elif system == 'ubuntu':
1125
        execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
1126

1127 1128
        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev',
                    'libboost-system-dev']
1129 1130

        if 'unittest' in features:
1131 1132 1133 1134
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
1135 1136

        if 'docs' in features:
1137
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
1138 1139 1140

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
1141
            packages.extend(['bison', 'debhelper', 'docbook', 'flex', 'libboost-dev', 'python3-dev'])
1142

1143
        if 'mysql' in features:
1144 1145 1146 1147
            if revision == '16.04':
                packages.extend(['mysql-client', 'libmysqlclient-dev', 'mysql-server'])
            else:
                packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])
1148 1149

        if 'pgsql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
1150 1151 1152 1153
            if revision == '16.04':
                packages.extend(['postgresql-client', 'libpq-dev', 'postgresql', 'postgresql-server-dev-all'])
            else:
                packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])
1154