hammer.py 98.9 KB
Newer Older
1
#!/usr/bin/env python3
2

3
# Copyright (C) 2018-2019 Internet Systems Consortium, Inc. ("ISC")
4 5 6 7
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
"""Hammer - Kea development environment management tool."""
9

10 11
from __future__ import print_function
import os
12
import re
13 14 15
import sys
import glob
import time
16
import json
17
import logging
18
import datetime
19 20 21
import platform
import binascii
import argparse
22 23
import textwrap
import functools
24
import subprocess
25
import multiprocessing
Michal Nowikowski's avatar
Michal Nowikowski committed
26 27 28 29
try:
    import urllib.request
except:
    pass
30 31 32 33
try:
    from urllib.parse import urljoin
except:
    from urlparse import urljoin
34
import xml.etree.ElementTree as ET
35 36 37

# TODO:
# - add docker provider
38
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
39 40 41


SYSTEMS = {
42 43
    'fedora': [#'27',  # EOLed
               #'28',  # EOLed
44
               '29',
45 46
               '30',
               '31'],
47
    'centos': ['7', '8'],
48
    'rhel': ['8'],
49
    'ubuntu': [#'16.04',
50
               '18.04',
51
               #'18.10',  # EOLed
52 53
               '19.04',
               '19.10'],
54
    'debian': [#'8',
55 56 57 58
               '9',
               '10'],
    'freebsd': ['11.2',
                '12.0'],
59
    'alpine': ['3.10']
60 61
}

62
# pylint: disable=C0326
63
IMAGE_TEMPLATES = {
64 65
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
66
    'fedora-28-lxc':           {'bare': 'godfryd/lxc-fedora-28',       'kea': 'godfryd/kea-fedora-28'},
67 68 69
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
70 71
    'fedora-30-lxc':           {'bare': 'godfryd/lxc-fedora-30',       'kea': 'godfryd/kea-fedora-30'},
    'fedora-30-virtualbox':    {'bare': 'generic/fedora30',            'kea': 'godfryd/kea-fedora-30'},
72 73
    'fedora-31-lxc':           {'bare': 'isc/lxc-fedora-31',           'kea': 'isc/kea-fedora-31'},
    'fedora-31-virtualbox':    {'bare': 'isc/vbox-fedora-31',          'kea': 'isc/kea-fedora-31'},
74 75
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
76 77
    'centos-8-lxc':            {'bare': 'isc/lxc-centos-8',            'kea': 'isc/kea-centos-8'},
    'centos-8-virtualbox':     {'bare': 'generic/centos8',             'kea': 'isc/kea-centos-8'},
78
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
79 80 81 82 83 84
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
85 86
    'ubuntu-19.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-19.04',    'kea': 'godfryd/kea-ubuntu-19.04'},
    'ubuntu-19.04-virtualbox': {'bare': 'ubuntu/disco64',              'kea': 'godfryd/kea-ubuntu-19.04'},
87 88
    'ubuntu-19.10-lxc':        {'bare': 'isc/lxc-ubuntu-19.10',        'kea': 'isc/kea-ubuntu-19.10'},
    'ubuntu-19.10-virtualbox': {'bare': 'generic/ubuntu1910',          'kea': 'isc/kea-ubuntu-19.10'},
89 90 91 92
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
93 94
    'debian-10-lxc':           {'bare': 'godfryd/lxc-debian-10',       'kea': 'godfryd/kea-debian-10'},
    'debian-10-virtualbox':    {'bare': 'debian/buster64',             'kea': 'godfryd/kea-debian-10'},
95 96
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
97
    'alpine-3.10-lxc':         {'bare': 'godfryd/lxc-alpine-3.10',     'kea': 'godfryd/kea-alpine-3.10'},
98 99 100 101
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
102
ENV["LC_ALL"] = "C"
103 104

Vagrant.configure("2") do |config|
105
  config.vm.hostname = "{name}"
106 107

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
108
  {box_version}
109 110 111

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
112
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
113 114 115
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
116
  config.vm.synced_folder '{ccache_dir}', '/ccache'
117 118 119 120 121
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :
Michal Nowikowski's avatar
Michal Nowikowski committed
122
ENV["LC_ALL"] = "C"
123 124

Vagrant.configure("2") do |config|
125
  config.vm.hostname = "{name}"
126 127

  config.vm.box = "{image_tpl}"
Michal Nowikowski's avatar
Michal Nowikowski committed
128
  {box_version}
129 130

  config.vm.provider "virtualbox" do |v|
131
    v.name = "{name}"
132 133 134 135 136 137 138 139 140 141
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
142 143

  config.vm.synced_folder '.', '/vagrant', disabled: true
144 145 146 147 148 149 150
end
"""


log = logging.getLogger()


151
def red(txt):
152
    """Return colorized (if the terminal supports it) or plain text."""
153 154
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
155
    return txt
156 157

def green(txt):
158
    """Return colorized (if the terminal supports it) or plain text."""
159 160
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
161
    return txt
162 163

def blue(txt):
164
    """Return colorized (if the terminal supports it) or plain text."""
165 166
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
167
    return txt
168 169


170
def get_system_revision():
171
    """Return tuple containing system name and its revision."""
172 173
    system = platform.system()
    if system == 'Linux':
174
        system, revision, _ = platform.dist()  # pylint: disable=deprecated-method
175
        if system == 'debian':
176
            revision = revision.split('.')[0]
177 178
        elif system == 'redhat':
            system = 'rhel'
179 180 181
            revision = revision[0]
        elif system == 'centos':
            revision = revision[0]
182 183 184 185 186 187 188 189 190 191 192 193
        else:
            if os.path.exists('/etc/os-release'):
                vals = {}
                with open('/etc/os-release') as f:
                    for l in f.readlines():
                        if '=' in l:
                            key, val = l.split('=', 1)
                            vals[key.strip()] = val.strip()
                system = vals['ID']
                revision = vals['VERSION_ID']
                if system == 'alpine':
                    revision = revision.rsplit('.', 1)[0]
194 195 196
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
197 198
    if '"' in revision:
        revision = revision.replace('"', '')
199 200 201
    return system.lower(), revision


202
class ExecutionError(Exception):
203
    """Exception thrown when execution encountered an error."""
204 205
    pass

206

207 208 209
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
            quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
            sleep_time_after_attempt=None):
210 211 212
    """Execute a command in shell.

    :param str cmd: a command to be executed
213 214
    :param int timeout: timeout in number of seconds, after that time the command is terminated
                        but only if check_times is True
215 216
    :param str cwd: current working directory for the command
    :param dict env: dictionary with environment variables
217 218
    :param bool raise_error: if False then in case of error exception is not raised,
                             default: True ie exception is raise
219 220 221 222 223
    :param bool dry_run: if True then the command is not executed
    :param str log_file_path: if provided then all traces from the command are stored in indicated file
    :param bool quiet: if True then the command's traces are not printed to stdout
    :param bool check_times: if True then timeout is taken into account
    :param bool capture: if True then the command's traces are captured and returned by the function
224 225
    :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
                             used for e.g. SSH
226 227
    :param int attemts: number of attempts to run the command if it fails
    :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
228
    """
229
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
230 231 232 233
    if not check_times:
        timeout = None
    if dry_run:
        return 0
234

235 236 237
    if 'sudo' in cmd and env:
        # if sudo is used and env is overridden then to preserve env add -E to sudo
        cmd = cmd.replace('sudo', 'sudo -E')
238

239 240
    if log_file_path:
        log_file = open(log_file_path, "wb")
241

242 243 244 245
    for attempt in range(attempts):
        if interactive:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
            exitcode = p.wait()
246

247 248
        else:
            p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
249

250 251 252 253 254 255 256 257
            if capture:
                output = ''
            t0 = time.time()
            t1 = time.time()
            # repeat until process is running or timeout not occured
            while p.poll() is None and (timeout is None or t1 - t0 < timeout):
                line = p.stdout.readline()
                if line:
258
                    line_decoded = line.decode(encoding='ascii', errors='ignore').rstrip() + '\r'
259 260 261 262 263 264 265 266 267 268 269
                    if not quiet:
                        print(line_decoded)
                    if capture:
                        output += line_decoded
                    if log_file_path:
                        log_file.write(line)
                t1 = time.time()

            # If no exitcode yet, ie. process is still running then it means that timeout occured.
            # In such case terminate the process and raise an exception.
            if p.poll() is None:
270 271 272 273 274 275
                # kill using sudo to be able to kill other sudo commands
                execute('sudo kill -s TERM %s' % p.pid)
                time.sleep(5)
                # if still running, kill harder
                if p.poll() is None:
                    execute('sudo kill -s KILL %s' % p.pid)
276 277 278
                msg = "Execution timeout, %d > %d seconds elapsed (start: %d, stop %d), cmd: '%s'"
                msg = msg % (t1 - t0, timeout, t0, t1, cmd)
                raise ExecutionError(msg)
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293
            exitcode = p.returncode

        if exitcode == 0:
            break
        elif attempt < attempts - 1:
            txt = 'command failed, retry, attempt %d/%d' % (attempt, attempts)
            if log_file_path:
                txt_to_file = '\n\n[HAMMER] %s\n\n\n' % txt
                log_file.write(txt_to_file.encode('ascii'))
            log.info(txt)
            if sleep_time_after_attempt:
                time.sleep(sleep_time_after_attempt)

    if log_file_path:
        log_file.close()
294

295
    if exitcode != 0 and raise_error:
Michal Nowikowski's avatar
Michal Nowikowski committed
296 297
        if capture and quiet:
            log.error(output)
298
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
299 300 301

    if capture:
        return exitcode, output
302
    return exitcode
303 304


Michal Nowikowski's avatar
Michal Nowikowski committed
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333
def _prepare_installed_packages_cache_for_debs():
    pkg_cache = {}

    _, out = execute("dpkg -l", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        line = line.strip()
        m = re.search('^([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+([^\s]+)\s+(.+)', line)
        if not m:
            continue
        status, name, version, arch, descr = m.groups()
        name = name.split(':')[0]
        pkg_cache[name] = dict(status=status, version=version, arch=arch, descr=descr)

    return pkg_cache


def _prepare_installed_packages_cache_for_rpms():
    pkg_cache = {}

    _, out = execute("rpm -qa --qf '%{NAME}\\n'", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        name = line.strip()
        pkg_cache[name] = dict(status='ii')

    return pkg_cache


334 335 336 337 338 339 340 341 342 343 344 345
def _prepare_installed_packages_cache_for_alpine():
    pkg_cache = {}

    _, out = execute("apk list -I\\n'", timeout=15, capture=True, quiet=True)

    for line in out.splitlines():
        name = line.strip()
        pkg_cache[name] = dict(status='ii')

    return pkg_cache


Michal Nowikowski's avatar
Michal Nowikowski committed
346
def install_pkgs(pkgs, timeout=60, env=None, check_times=False, pkg_cache={}):
347
    """Install native packages in a system.
348 349 350 351 352 353 354

    :param dict pkgs: specifies a list of packages to be installed
    :param int timeout: timeout in number of seconds, after that time the command
                        is terminated but only if check_times is True
    :param dict env: dictionary with environment variables (optional)
    :param bool check_times: specifies if timeouts should be enabled (optional)
    """
355 356
    system, revision = get_system_revision()

Michal Nowikowski's avatar
Michal Nowikowski committed
357 358 359 360
    if not isinstance(pkgs, list):
        pkgs = pkgs.split()

    # prepare cache if needed
361
    if not pkg_cache and system in ['centos', 'rhel', 'fedora', 'debian', 'ubuntu']:#, 'alpine']: # TODO: complete caching support for alpine
Michal Nowikowski's avatar
Michal Nowikowski committed
362 363 364 365
        if system in ['centos', 'rhel', 'fedora']:
            pkg_cache.update(_prepare_installed_packages_cache_for_rpms())
        elif system in ['debian', 'ubuntu']:
            pkg_cache.update(_prepare_installed_packages_cache_for_debs())
366 367
        elif system in ['alpine']:
            pkg_cache.update(_prepare_installed_packages_cache_for_alpine())
Michal Nowikowski's avatar
Michal Nowikowski committed
368

369 370
    # check if packages actually need to be installed
    if pkg_cache:
Michal Nowikowski's avatar
Michal Nowikowski committed
371 372 373 374 375 376 377 378 379 380
        pkgs_to_install = []
        for pkg in pkgs:
            if pkg not in pkg_cache or pkg_cache[pkg]['status'] != 'ii':
                pkgs_to_install.append(pkg)
        pkgs = pkgs_to_install

    if not pkgs:
        log.info('all packages already installed')
        return

381 382 383 384 385 386 387
    if system in ['centos', 'rhel'] and revision == '7':
        # skip_missing_names_on_install used to detect case when one packet is not found and no error is returned
        # but we want an error
        cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False'
    elif system == 'fedora' or (system in ['centos', 'rhel'] and revision == '8'):
        cmd = 'sudo dnf -y install'
    elif system in ['debian', 'ubuntu']:
Michal Nowikowski's avatar
Michal Nowikowski committed
388
        # prepare the command for ubuntu/debian
389 390 391 392 393 394
        if not env:
            env = os.environ.copy()
        env['DEBIAN_FRONTEND'] = 'noninteractive'
        cmd = 'sudo apt install --no-install-recommends -y'
    elif system == 'freebsd':
        cmd = 'sudo pkg install -y'
395 396
    elif system == 'alpine':
        cmd = 'sudo apk add'
Michal Nowikowski's avatar
Michal Nowikowski committed
397
    else:
398
        raise NotImplementedError('no implementation for %s' % system)
399

Michal Nowikowski's avatar
Michal Nowikowski committed
400
    pkgs = ' '.join(pkgs)
401 402
    cmd += ' ' + pkgs
    execute(cmd, timeout=timeout, env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
403 404


405 406 407 408 409 410 411 412 413
def _get_full_repo_url(repository_url, system, revision, pkg_version):
    if not repository_url:
        return None
    repo_name = 'kea-%s-%s-%s' % (pkg_version.rsplit('.', 1)[0], system, revision)
    repo_url = urljoin(repository_url, 'repository')
    repo_url += '/%s-ci/' % repo_name
    return repo_url


414
class VagrantEnv(object):
415 416 417 418 419 420 421
    """Helper class that makes interacting with Vagrant easier.

    It creates Vagrantfile according to specified system. It exposes basic Vagrant functions
    like up, upload, destro, ssh. It also provides more complex function for preparing system
    for Kea build and building Kea.
    """

422
    def __init__(self, provider, system, revision, features, image_template_variant,
423
                 dry_run, quiet=False, check_times=False, ccache_dir=None):
424 425 426 427 428 429 430 431 432 433 434
        """VagrantEnv initializer.

        :param str provider: indicate backend type: virtualbox or lxc
        :param str system: name of the system eg. ubuntu
        :param str revision: revision of the system e.g. 18.04
        :param list features: list of requested features
        :param str image_template_variant: variant of images' templates: bare or kea
        :param bool dry_run: if False then system commands are not really executed
        :param bool quiet: if True then commands will not trace to stdout
        :param bool check_times: if True then commands will be terminated after given timeout
        """
435
        self.provider = provider
436
        self.system = system
437
        self.revision = revision
438
        self.features = features
439 440 441
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
442

443 444 445 446 447
        # set properly later
        self.features_arg = None
        self.nofeatures_arg = None
        self.python = None

448 449 450 451 452
        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

Michal Nowikowski's avatar
Michal Nowikowski committed
453 454
        self.key = key = "%s-%s-%s" % (system, revision, provider)
        self.image_tpl = image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
455 456
        self.repo_dir = os.getcwd()

457
        sys_dir = "%s-%s" % (system, revision)
458
        if provider == "virtualbox":
459
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
460
        elif provider == "lxc":
461 462 463 464
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
465

466 467
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
468

469
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
470 471 472 473 474

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

475
        crc = binascii.crc32(self.vagrant_dir.encode())
476
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
477

478 479 480 481 482 483
        if ccache_dir is None:
            ccache_dir = '/'
            self.ccache_enabled = False
        else:
            self.ccache_enabled = True

Michal Nowikowski's avatar
Michal Nowikowski committed
484 485 486 487 488 489 490
        if '/' in image_tpl:
            self.latest_version = self._get_latest_cloud_version()
            box_version = 'config.vm.box_version = "%s"' % self.latest_version
        else:
            self.latest_version = None
            box_version = ""

491
        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
492
                                             name=self.name,
Michal Nowikowski's avatar
Michal Nowikowski committed
493 494
                                             ccache_dir=ccache_dir,
                                             box_version=box_version)
495

496 497 498
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

499 500
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

501
    def up(self):
502
        """Do Vagrant up."""
503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519
        exitcode, out = execute("vagrant up --no-provision --provider %s" % self.provider,
                                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run,
                                capture=True, raise_error=False)
        if exitcode != 0:
            if 'There is container on your system' in out and 'lxc-destroy' in out:
                m = re.search('`lxc-destroy.*?`', out)
                if m:
                    # destroy some old container
                    cmd = m.group(0)[1:-1]
                    cmd = 'sudo ' + cmd + ' -f'
                    execute(cmd, timeout=60)

                    # try again spinning up new
                    execute("vagrant up --no-provision --provider %s" % self.provider,
                            cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
                    return
            raise ExecutionError('There is a problem with putting up a system')
520

Michal Nowikowski's avatar
Michal Nowikowski committed
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554
    def _get_cloud_meta(self, image_tpl=None):
        if '/' not in self.image_tpl:
            return {}
        url = 'https://app.vagrantup.com/api/v1/box/' + (image_tpl if image_tpl else self.image_tpl)
        try:
            with urllib.request.urlopen(url) as response:
                data = response.read()
        except:
            log.exception('ignored exception')
            return {}
        data = json.loads(data)
        return data

    def _get_local_meta(self):
        meta_file = os.path.join(self.vagrant_dir, '.vagrant/machines/default', self.provider, 'box_meta')
        if not os.path.exists(meta_file):
            return {}
        with open(meta_file) as f:
            data = f.read()
        data = json.loads(data)
        return data

    def _get_latest_cloud_version(self, image_tpl=None):
        cloud_meta = self._get_cloud_meta(image_tpl)
        if not cloud_meta and 'versions' not in cloud_meta:
            return 0
        latest_version = 0
        for ver in cloud_meta['versions']:
            provider_found = False
            for p in ver['providers']:
                if p['name'] == self.provider:
                    provider_found = True
                    break
            if provider_found:
555 556 557 558
                try:
                    v = int(ver['number'])
                except:
                    return ver['number']
Michal Nowikowski's avatar
Michal Nowikowski committed
559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
                if v > latest_version:
                    latest_version = v
        return latest_version

    def get_status(self):
        """Return system status.

        Status can be: 'not created', 'running', 'stopped', etc.
        """
        _, out = execute("vagrant status", cwd=self.vagrant_dir, timeout=15, capture=True, quiet=True)
        m = re.search('default\s+(.+)\(', out)
        if not m:
            raise Exception('cannot get status in:\n%s' % out)
        return m.group(1).strip()

    def bring_up_latest_box(self):
        if self.get_status() == 'running':
            self.reload()
        else:
            self.up()

    def reload(self):
        """Do Vagrant reload."""
        execute("vagrant reload --no-provision --force",
                cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)


586
    def package(self):
587
        """Package Vagrant system into Vagrant box."""
588
        execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
589

590 591 592
        box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
        if os.path.exists(box_path):
            os.unlink(box_path)
593

594
        if self.provider == 'virtualbox':
Michal Nowikowski's avatar
Michal Nowikowski committed
595
            cmd = "vagrant package --output %s" % box_path
596 597 598 599 600 601 602 603
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
604 605 606 607 608 609 610 611 612 613 614 615
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
                    'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
                    '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
                    'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
                    'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
                    'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
                    'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
                    '> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            cmd = 'sudo bash -c "'
            cmd += 'cd %s '
            cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
            cmd += '"'
616
            execute(cmd % (lxc_container_path, lxc_box_dir))
617 618 619 620 621 622 623 624 625 626 627 628
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
629

Michal Nowikowski's avatar
Michal Nowikowski committed
630 631 632 633 634 635 636 637 638 639 640 641 642 643 644
        return box_path

    def upload_to_cloud(self, box_path):
        image_tpl = IMAGE_TEMPLATES[self.key]['kea']
        if '/' not in image_tpl:
            return

        latest_version = self._get_latest_cloud_version(image_tpl)
        new_version = latest_version + 1

        cmd = "vagrant cloud publish -f -r %s %s %s %s"
        cmd = cmd % (image_tpl, new_version, self.provider, box_path)

        execute(cmd, cwd=self.vagrant_dir, timeout=60 * 60)

645
    def upload(self, src):
646
        """Upload src to Vagrant system, home folder."""
647 648 649 650 651 652 653 654 655 656 657
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

658
    def run_build_and_test(self, tarball_path, jobs, pkg_version, pkg_isc_version, upload, repository_url):
659
        """Run build and unit tests inside Vagrant system."""
660 661 662
        if self.dry_run:
            return 0, 0

663
        # prepare tarball if needed and upload it to vagrant system
664
        if not tarball_path:
665
            name_ver = 'kea-%s' % pkg_version
666
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
667 668
            cmd += ' --exclude "*~" --exclude .git --exclude .libs '
            cmd += ' --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
669 670
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
671
            tarball_path = '/tmp/%s.tar.gz' % name_ver
672
        self.upload(tarball_path)
673 674 675

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
676 677

        t0 = time.time()
678

679
        # run build command
680 681 682 683 684 685 686 687 688 689 690 691 692
        bld_cmd = "{python} hammer.py build -p local {features} {nofeatures} {check_times} {ccache}"
        bld_cmd += " {tarball} {jobs} {pkg_version} {pkg_isc_version} {repository_url}"
        bld_cmd = bld_cmd.format(python=self.python,
                                 features=self.features_arg,
                                 nofeatures=self.nofeatures_arg,
                                 check_times='-i' if self.check_times else '',
                                 ccache='--ccache-dir /ccache' if self.ccache_enabled else '',
                                 tarball='-t %s.tar.gz' % name_ver,
                                 jobs='-j %d' % jobs,
                                 pkg_version='--pkg-version %s' % pkg_version,
                                 pkg_isc_version='--pkg-isc-version %s' % pkg_isc_version,
                                 repository_url=('--repository-url %s' % repository_url) if repository_url else '')

693 694
        timeout = _calculate_build_timeout(self.features) + 5 * 60
        self.execute(bld_cmd, timeout=timeout, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes
695 696 697 698

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
699 700 701 702 703 704 705 706 707 708
            pkgs_dir = os.path.join(self.vagrant_dir, 'pkgs')
            if os.path.exists(pkgs_dir):
                execute('rm -rf %s' % pkgs_dir)
            os.makedirs(pkgs_dir)

            if self.system in ['ubuntu', 'debian']:
                execute('scp -F %s -r default:/home/vagrant/kea-src/isc-kea_* .' % ssh_cfg_path, cwd=pkgs_dir)
                execute('scp -F %s -r default:/home/vagrant/kea-src/*deb .' % ssh_cfg_path, cwd=pkgs_dir)
            elif self.system in ['fedora', 'centos', 'rhel']:
                execute('scp -F %s -r default:/home/vagrant/pkgs/* .' % ssh_cfg_path, cwd=pkgs_dir)
709 710
            elif self.system in ['alpine']:
                execute('scp -F %s -r default:/home/vagrant/packages/vagrant/x86_64/* .' % ssh_cfg_path, cwd=pkgs_dir)
711
            else:
712
                raise NotImplementedError('no implementation for %s' % self.system)
713 714 715 716 717 718 719 720 721 722 723 724 725 726

            if upload:
                repo_url = _get_full_repo_url(repository_url, self.system, self.revision, pkg_version)
                assert repo_url is not None
                upload_cmd = 'curl -v --netrc -f'

                if self.system in ['ubuntu', 'debian']:
                    upload_cmd += ' -X POST -H "Content-Type: multipart/form-data" --data-binary "@%s" '
                    file_ext = '.deb'

                elif self.system in ['fedora', 'centos', 'rhel']:
                    upload_cmd += ' --upload-file %s '
                    file_ext = '.rpm'

727 728 729 730 731
                elif self.system == 'alpine':
                    upload_cmd += ' --upload-file %s '
                    file_ext = ''
                    repo_url = urljoin(repo_url, '%s/v%s/x86_64/' % (pkg_isc_version, self.revision))

732 733 734
                upload_cmd += ' ' + repo_url

                for fn in os.listdir(pkgs_dir):
735
                    if file_ext and not fn.endswith(file_ext):
736 737 738 739
                        continue
                    fp = os.path.join(pkgs_dir, fn)
                    cmd = upload_cmd % fp
                    execute(cmd)
740

741 742
        t1 = time.time()
        dt = int(t1 - t0)
743 744

        log.info('Build log file stored to %s', log_file_path)
745 746 747 748
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

749
        # run unit tests if requested
750 751 752 753
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
754 755
                cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
756 757 758 759 760 761 762
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
Michal Nowikowski's avatar
Michal Nowikowski committed
763 764 765

                cmd = 'scp -F %s -r default:/home/vagrant/aggregated_tests.xml .' % ssh_cfg_path
                execute(cmd, cwd=self.vagrant_dir)
766
        except:  # pylint: disable=bare-except
767 768 769 770
            log.exception('ignored issue with parsing unit test results')

        return total, passed

771
    def destroy(self):
772
        """Remove the VM completely."""
773
        cmd = 'vagrant destroy --force'
774
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
775 776

    def ssh(self):
777
        """Open interactive session to the VM."""
778
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
779 780

    def dump_ssh_config(self):
781
        """Dump ssh config that allows getting into Vagrant system via SSH."""
782 783 784 785 786
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
787
        """Execute provided command inside Vagrant system."""
788 789 790
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
791

792 793 794
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
                       raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
                       quiet=quiet, check_times=self.check_times)
795

796
    def prepare_system(self):
797
        """Prepare Vagrant system for building Kea."""
798 799
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
800 801 802
        else:
            self.features_arg = ''

803
        nofeatures = set(DEFAULT_FEATURES) - self.features
804 805 806 807 808
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

809 810 811 812 813 814 815 816 817 818 819 820 821 822
        # install python3 for centos 8
        if self.system == 'centos' and self.revision == '8':
            # we need log4cplus that is in the nexus
            cmd = 'bash -c \'cat <<EOF | sudo tee /etc/yum.repos.d/isc.repo\n'
            cmd += '[nexus]\n'
            cmd += 'name=ISC Repo\n'
            cmd += 'baseurl=https://packages.isc.org/repository/kea-1.7-centos-8-ci/\n'
            cmd += 'enabled=1\n'
            cmd += 'gpgcheck=0\n'
            cmd += "EOF\n\'"
            self.execute(cmd)
            self.execute("sudo dnf install -y python36 rpm-build python3-virtualenv")
            self.python = 'python3'

823
        # select proper python version for running Hammer inside Vagrant system
824 825
        if (self.system == 'centos' and self.revision == '7' or
            (self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
826 827 828 829 830 831
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

832
        # to get python in RHEL 8 beta it is required first register machine in RHEL account
833 834 835
        if self.system == 'rhel' and self.revision == '8':
            cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
            exitcode = self.execute(cmd, raise_error=False)
836
            if exitcode != 0:
837 838 839 840 841
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
842 843 844 845 846
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

847
        # upload Hammer to Vagrant system
848
        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
849
        self.upload(hmr_py_path)
850

851 852 853
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

Michal Nowikowski's avatar
Michal Nowikowski committed
854 855
        t0 = time.time()

856
        # run prepare-system inside Vagrant system
857
        cmd = "{python} hammer.py prepare-system -p local {features} {nofeatures} {check_times} {ccache}"
858 859
        cmd = cmd.format(python=self.python,
                         features=self.features_arg,
860
                         nofeatures=self.nofeatures_arg,
861 862
                         check_times='-i' if self.check_times else '',
                         ccache='--ccache-dir /ccache' if self.ccache_enabled else '')
863
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
864

Michal Nowikowski's avatar
Michal Nowikowski committed
865 866 867 868 869 870 871 872
        t1 = time.time()
        dt = int(t1 - t0)

        log.info('')
        log.info(">>> Preparing %s, %s, %s completed in %s:%s", self.provider, self.system, self.revision,
                 dt // 60, dt % 60)
        log.info('')

873 874

def _install_gtest_sources():
875
    """Install gtest sources."""
876
    # download gtest sources only if it is not present as native package
877
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
878 879 880
        cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
        cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
        execute(cmd)
881
        execute('sudo mkdir -p /usr/src')
882
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
883 884 885
        os.unlink('/tmp/gtest.tar.gz')


886
def _configure_mysql(system, revision, features):
887
    """Configure MySQL database."""
888 889 890 891
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
892

893
    elif system == 'freebsd':
894 895
        cmd = "echo 'SET PASSWORD = \"\";' "
        cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
896 897
        execute(cmd, raise_error=False)

898 899 900 901 902
    elif system == 'alpine':
        execute('sudo rc-update add mariadb')
        execute('sudo /etc/init.d/mariadb setup', raise_error=False)
        execute('sudo /etc/init.d/mariadb start')

903 904 905 906 907 908 909 910 911 912 913 914 915 916 917
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

918 919 920 921 922 923 924 925 926 927 928 929
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

930 931 932 933 934 935 936 937 938 939 940 941 942
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

943

944
def _configure_pgsql(system, features):
945
    """ Configure PostgreSQL DB """
946 947
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
948
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
949
        if exitcode != 0:
Michal Nowikowski's avatar
Michal Nowikowski committed
950 951 952 953
            if system == 'centos':
                execute('sudo postgresql-setup initdb')
            else:
                execute('sudo postgresql-setup --initdb --unit postgresql')
954 955 956 957 958 959 960 961
    elif system == 'freebsd':
        # pgsql must be enabled before runnig initdb
        execute('sudo sysrc postgresql_enable="yes"')
        execute('sudo /usr/local/etc/rc.d/postgresql initdb')

    if system == 'freebsd':
        # echo or redirection to stdout is needed otherwise the script will hang at "line = p.stdout.readline()"
        execute('sudo service postgresql start && echo "PostgreSQL started"')
962 963 964
    elif system == 'alpine':
        execute('sudo rc-update add postgresql')
        execute('sudo /etc/init.d/postgresql start')
965 966 967 968
    else:
        execute('sudo systemctl enable postgresql.service')
        execute('sudo systemctl start postgresql.service')

969 970 971 972 973 974 975 976 977 978
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
979
    execute(cmd, cwd='/tmp')  # CWD to avoid: could not change as posgres user directory to "/home/jenkins": Permission denied
980

981 982 983 984 985 986 987 988
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
989
        execute(cmd, cwd='/tmp')  # CWD to avoid: could not change as posgres user directory to "/home/jenkins": Permission denied
Michal Nowikowski's avatar
Michal Nowikowski committed
990 991 992 993 994 995
        # TODO: in /etc/postgresql/10/main/pg_hba.conf
        # change:
        #    local   all             all                                     peer
        # to:
        #    local   all             all                                     md5
    log.info('postgresql just configured')
996

997

998 999 1000 1001 1002 1003 1004 1005 1006
def _apt_update(system, revision, env=None, check_times=False, attempts=1, sleep_time_after_attempt=None,
                capture=False):
    cmd = 'sudo apt update'
    if system == 'debian' and int(revision) >= 10:
        cmd += ' --allow-releaseinfo-change'
    return execute(cmd, env=env, check_times=check_times, attempts=attempts,
                   sleep_time_after_attempt=sleep_time_after_attempt, capture=capture)


Michal Nowikowski's avatar
Michal Nowikowski committed
1007
def _install_cassandra_deb(system, revision, env, check_times):
1008
    """Install Cassandra and cpp-driver using DEB package."""
1009
    if not os.path.exists('/usr/sbin/cassandra'):
1010 1011 1012 1013
        cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
        cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
        execute(cmd, env=env, check_times=check_times)
        execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
1014
                env=env, check_times=check_times)
1015
        _apt_update(system, revision, env=env, check_times=check_times)
1016 1017
        # ca-certificates-java needs to be installed first because it fails if installed together with cassandra
        install_pkgs('ca-certificates-java', env=env, check_times=check_times)
1018
        install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
1019 1020

    if not os.path.exists('/usr/include/cassandra.h'):
Michal Nowikowski's avatar
Michal Nowikowski committed
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
        if system == 'ubuntu' and revision == '16.04':
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/16.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
        else:
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
            execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
                    env=env, check_times=check_times)
1031 1032 1033
            if system == 'debian' and revision == '10':
                install_pkgs('multiarch-support', env=env, check_times=check_times)

1034 1035 1036 1037
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
                env=env, check_times=check_times)
1038 1039


1040
def _install_cassandra_rpm(system, revision, env, check_times):
1041
    """Install Cassandra and cpp-driver using RPM package."""
1042
    if not os.path.exists('/usr/bin/cassandra'):
Michal Nowikowski's avatar
Michal Nowikowski committed
1043 1044
        if system == 'centos':
            install_pkgs('yum-utils', env=env, check_times=check_times)
1045
            execute('sudo yum-config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/', raise_error=False)
Michal Nowikowski's avatar
Michal Nowikowski committed
1046 1047 1048 1049 1050
            execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
            pkgs = 'cassandra cassandra-tools libuv libuv-devel openssl'
        else:
            pkgs = 'cassandra cassandra-server libuv libuv-devel'
        install_pkgs(pkgs, env=env, check_times=check_times)
1051

Michal Nowikowski's avatar
Michal Nowikowski committed
1052 1053
    if system == 'centos':
        execute('sudo systemctl daemon-reload')
1054

1055
    if system == 'fedora' and int(revision) >= 30:
1056
        execute("echo '-Xms1G -Xmx1G' | sudo tee -a /etc/cassandra/jvm.options")
1057 1058 1059 1060 1061
    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
1062 1063 1064 1065
        if system == 'centos':
            execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        else:
            execute('sudo dnf install -y cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
1066 1067 1068
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


1069
def _install_freeradius_client(system, revision, features, env, check_times):
Michal Nowikowski's avatar
Michal Nowikowski committed
1070
    """Install FreeRADIUS-client with necessary patches from Francis Dupont."""
1071
    # check if it is already installed
Michal Nowikowski's avatar
Michal Nowikowski committed
1072 1073 1074 1075
    if (os.path.exists('/usr/local/lib/libfreeradius-client.so.2.0.0') and
        os.path.exists('/usr/local/include/freeradius-client.h')):
        log.info('freeradius is already installed')
        return
1076 1077 1078 1079 1080 1081 1082

    # install freeradius dependencies
    if system in ['centos', 'rhel', 'fedora']:
        install_pkgs('nettle-devel', env=env, check_times=check_times)
    elif system in ['debian', 'ubuntu']:
        install_pkgs('nettle-dev', env=env, check_times=check_times)
    else:
1083
        raise NotImplementedError('no implementation for %s' % system)
1084 1085

    # checkout sources, build them and install
Michal Nowikowski's avatar
Michal Nowikowski committed
1086 1087 1088
    execute('rm -rf freeradius-client')
    execute('git clone https://github.com/fxdupont/freeradius-client.git', env=env, check_times=check_times)
    execute('git checkout iscdev', cwd='freeradius-client', env=env, check_times=check_times)
1089
    execute('./configure --with-nettle', cwd='freeradius-client', env=env, check_times=check_times)
Michal Nowikowski's avatar
Michal Nowikowski committed
1090 1091 1092 1093 1094 1095 1096
    execute('make', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo make install', cwd='freeradius-client', env=env, check_times=check_times)
    execute('sudo ldconfig', env=env, check_times=check_times)
    execute('rm -rf freeradius-client')
    log.info('freeradius just installed')


1097
def prepare_system_local(features, check_times):
1098
    """Prepare local system for Kea development based on requested features."""
1099 1100 1101
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

1102 1103 1104
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

1105
    # prepare fedora
1106
    if system == 'fedora':
1107 1108
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel']
1109 1110

        if 'native-pkg' in features:
1111
            packages.extend(['rpm-build', 'python2-devel', 'python3-devel'])
1112

1113
        if 'docs' in features:
1114
            packages.extend(['python3-sphinx', 'texlive', 'texlive-collection-latexextra'])
1115 1116
            if int(revision) >= 31:
                packages.extend(['python3-sphinx_rtd_theme'])
1117 1118

        if 'mysql' in features:
1119
            execute('sudo dnf remove -y community-mysql-devel || true')
1120
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-connector-c-devel'])
1121 1122 1123

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])
1124
            if int(revision) >= 30:
1125
                packages.extend(['postgresql-server-devel'])
1126

1127 1128 1129
        if 'radius' in features:
            packages.extend(['git'])

1130 1131 1132
        if 'ccache' in features:
            packages.extend(['ccache'])

1133
        install_pkgs(packages, timeout=300, env=env, check_times=check_times)
1134 1135 1136 1137

        if 'unittest' in features:
            _install_gtest_sources()

1138
        execute('sudo dnf clean packages', env=env, check_times=check_times)
1139

1140
        if 'cql' in features:
1141
            _install_cassandra_rpm(system, revision, env, check_times)
1142

1143
    # prepare centos
1144
    elif system == 'centos':
1145
        install_pkgs('epel-release', env=env, check_times=check_times)
1146

1147 1148
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
                    'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
1149