hammer.py 52.4 KB
Newer Older
1
#!/usr/bin/env python3
2 3 4 5 6 7 8

# Copyright (C) 2014-2017 Internet Systems Consortium, Inc. ("ISC")
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

9 10 11 12 13
from __future__ import print_function
import os
import sys
import glob
import time
14
import json
15
import logging
16
import datetime
17 18 19 20
import platform
import binascii
import argparse
import subprocess
21
import multiprocessing
22
import xml.etree.ElementTree as ET
23 24 25

# TODO:
# - add docker provider
26
#   https://developer.fedoraproject.org/tools/docker/docker-installation.html
27
# - add CCACHE support
28 29 30
# - improve building from tarball
# - improve native-pkg builds
# - avoid using network if possible (e.g. check first if pkgs are installed)
31 32 33 34 35


SYSTEMS = {
    'fedora': ['27', '28', '29'],
    'centos': ['7'],
36 37 38
    #'rhel': ['7', '8'],
    'rhel': ['8'],
    'ubuntu': ['16.04', '18.04', '18.10'],
39 40
    'debian': ['8', '9'],
    #'freebsd': ['11.0', '11.1', '11.2', '12.0'],
41
    'freebsd': ['11.2', '12.0'],
42 43 44
}

IMAGE_TEMPLATES = {
45 46 47 48 49 50 51 52
    'fedora-27-lxc':           {'bare': 'lxc-fedora-27',               'kea': 'godfryd/kea-fedora-27'},
    'fedora-27-virtualbox':    {'bare': 'generic/fedora27',            'kea': 'godfryd/kea-fedora-27'},
    'fedora-28-lxc':           {'bare': 'lxc-fedora-28',               'kea': 'godfryd/kea-fedora-28'},
    'fedora-28-virtualbox':    {'bare': 'generic/fedora28',            'kea': 'godfryd/kea-fedora-28'},
    'fedora-29-lxc':           {'bare': 'godfryd/lxc-fedora-29',       'kea': 'godfryd/kea-fedora-29'},
    'fedora-29-virtualbox':    {'bare': 'generic/fedora29',            'kea': 'godfryd/kea-fedora-29'},
    'centos-7-lxc':            {'bare': 'godfryd/lxc-centos-7',        'kea': 'godfryd/kea-centos-7'},
    'centos-7-virtualbox':     {'bare': 'generic/centos7',             'kea': 'godfryd/kea-centos-7'},
53 54
#    'rhel-7-virtualbox':       {'bare': 'generic/rhel7',               'kea': 'generic/rhel7'},   # TODO: subsciption needed
    'rhel-8-virtualbox':       {'bare': 'generic/rhel8',               'kea': 'generic/rhel8'},
55 56 57 58 59 60 61 62 63 64 65 66
    'ubuntu-16.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-16.04',    'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-16.04-virtualbox': {'bare': 'ubuntu/xenial64',             'kea': 'godfryd/kea-ubuntu-16.04'},
    'ubuntu-18.04-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.04',    'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.04-virtualbox': {'bare': 'ubuntu/bionic64',             'kea': 'godfryd/kea-ubuntu-18.04'},
    'ubuntu-18.10-lxc':        {'bare': 'godfryd/lxc-ubuntu-18.10',    'kea': 'godfryd/kea-ubuntu-18.10'},
    'ubuntu-18.10-virtualbox': {'bare': 'ubuntu/cosmic64',             'kea': 'godfryd/kea-ubuntu-18.10'},
    'debian-8-lxc':            {'bare': 'godfryd/lxc-debian-8',        'kea': 'godfryd/kea-debian-8'},
    'debian-8-virtualbox':     {'bare': 'debian/jessie64',             'kea': 'godfryd/kea-debian-8'},
    'debian-9-lxc':            {'bare': 'godfryd/lxc-debian-9',        'kea': 'godfryd/kea-debian-9'},
    'debian-9-virtualbox':     {'bare': 'debian/stretch64',            'kea': 'godfryd/kea-debian-9'},
    'freebsd-11.2-virtualbox': {'bare': 'generic/freebsd11',           'kea': 'godfryd/kea-freebsd-11.2'},
    'freebsd-12.0-virtualbox': {'bare': 'generic/freebsd12',           'kea': 'godfryd/kea-freebsd-12.0'},
67 68 69 70 71 72
}

LXC_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
73
  config.vm.hostname = "{name}"
74 75

  config.vm.box = "{image_tpl}"
76 77 78

  config.vm.provider "lxc" do |lxc|
    lxc.container_name = "{name}"
79
    lxc.customize 'rootfs.path', "/var/lib/lxc/{name}/rootfs"
80 81 82
  end

  config.vm.synced_folder '.', '/vagrant', disabled: true
83 84 85 86 87 88 89
end
"""

VBOX_VAGRANTFILE_TPL = """# -*- mode: ruby -*-
# vi: set ft=ruby :

Vagrant.configure("2") do |config|
90
  config.vm.hostname = "{name}"
91 92 93 94

  config.vm.box = "{image_tpl}"

  config.vm.provider "virtualbox" do |v|
95
    v.name = "{name}"
96 97 98 99 100 101 102 103 104 105
    v.memory = 8192

    nproc = Etc.nprocessors
    if nproc > 8
      nproc -= 2
    elsif nproc > 1
      nproc -= 1
    end
    v.cpus = nproc
  end
106 107

  config.vm.synced_folder '.', '/vagrant', disabled: true
108 109 110 111 112 113 114
end
"""


log = logging.getLogger()


115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
def red(txt):
    if sys.stdout.isatty():
        return '\033[1;31m%s\033[0;0m' % txt
    else:
        return txt

def green(txt):
    if sys.stdout.isatty():
        return '\033[0;32m%s\033[0;0m' % txt
    else:
        return txt

def blue(txt):
    if sys.stdout.isatty():
        return '\033[0;34m%s\033[0;0m' % txt
    else:
        return txt


134 135 136 137 138 139 140
def get_system_revision():
    system = platform.system()
    if system == 'Linux':
        system, revision, _ = platform.dist()
        if system == 'debian':
            if revision.startswith('8.'):
                revision = '8'
141 142
            if revision.startswith('9.'):
                revision = '9'
143 144 145 146 147 148 149 150 151 152
        elif system == 'redhat':
            system = 'rhel'
            if revision.startswith('8.'):
                revision = '8'
    elif system == 'FreeBSD':
        system = system.lower()
        revision = platform.release()
    return system.lower(), revision


153 154
class ExecutionError(Exception): pass

155 156
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None, quiet=False, check_times=False, capture=False,
            interactive=False):
157
    log.info('>>>>> Executing %s in %s', cmd, cwd if cwd else os.getcwd())
158 159 160 161
    if not check_times:
        timeout = None
    if dry_run:
        return 0
162 163 164 165 166 167 168 169 170

    if interactive:
        p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True)
        ver = platform.python_version()
        exitcode = p.wait()
    else:
        if log_file_path:
            log_file = open(log_file_path, "wb")

171
        p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
172 173 174 175 176 177 178 179 180 181 182 183 184 185

        if capture:
            output = ''
        t0 = time.time()
        t1 = time.time()
        while p.poll() is None and (timeout is None or t1 - t0 < timeout):
            line = p.stdout.readline()
            if line:
                line_decoded = line.decode(errors='ignore').rstrip() + '\r'
                if not quiet:
                    print(line_decoded)
                if capture:
                    output += line_decoded
                if log_file_path:
186
                    log_file.write(line)
187
            t1 = time.time()
188

189 190
        if log_file_path:
            log_file.close()
191

192 193 194
        if p.poll() is None:
            raise ExecutionError('Execution timeout')
        exitcode = p.returncode
195

196
    if exitcode != 0 and raise_error:
197
        raise ExecutionError("The command return non-zero exitcode %s, cmd: '%s'" % (exitcode, cmd))
198 199 200 201 202

    if capture:
        return exitcode, output
    else:
        return exitcode
203 204


205 206 207 208 209 210 211 212
def install_yum(pkgs, env=None, check_times=False):
    if isinstance(pkgs, list):
        pkgs = ' '.join(pkgs)
    # skip.... to detect case when one packet is not found and no error is returned
    cmd = 'sudo yum install -y --setopt=skip_missing_names_on_install=False %s' % pkgs
    execute(cmd, env=env, check_times=check_times)


213
class VagrantEnv(object):
214 215
    def __init__(self, provider, system, sys_revision, features, image_template_variant, dry_run, quiet=False, check_times=False):
        self.provider = provider
216 217 218
        self.system = system
        self.sys_revision = sys_revision
        self.features = features
219 220 221
        self.dry_run = dry_run
        self.quiet = quiet
        self.check_times = check_times
222 223 224 225 226 227 228 229 230 231 232

        if provider == "virtualbox":
            vagrantfile_tpl = VBOX_VAGRANTFILE_TPL
        elif provider == "lxc":
            vagrantfile_tpl = LXC_VAGRANTFILE_TPL

        image_tpl = IMAGE_TEMPLATES["%s-%s-%s" % (system, sys_revision, provider)][image_template_variant]
        self.repo_dir = os.getcwd()

        sys_dir = "%s-%s" % (system, sys_revision)
        if provider == "virtualbox":
233
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
234
        elif provider == "lxc":
235 236 237 238
            self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'lxc')

        if dry_run:
            return
239

240 241
        if not os.path.exists(self.vagrant_dir):
            os.makedirs(self.vagrant_dir)
242

243
        vagrantfile_path = os.path.join(self.vagrant_dir, "Vagrantfile")
244 245 246 247 248

        if os.path.exists(vagrantfile_path):
            # TODO: destroy any existing VM
            pass

249 250 251 252 253 254
        crc = binascii.crc32(self.vagrant_dir.encode())
        self.name = "hmr-%s-%s-kea-srv-%08d" % (system, sys_revision.replace('.', '-'), crc)

        vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
                                             name=self.name)

255 256 257
        with open(vagrantfile_path, "w") as f:
            f.write(vagrantfile)

258 259
        log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)

260
    def up(self):
261 262
        execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run)
        execute("vagrant up --no-provision --provider %s" % self.provider, cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
263 264

    def package(self):
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294
        if self.provider == 'virtualbox':
            cmd = "vagrant package --output kea-%s-%s.box" % (self.system, self.sys_revision)
            execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)

        elif self.provider == 'lxc':
            execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)

            box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.sys_revision))
            if os.path.exists(box_path):
                os.unlink(box_path)

            lxc_box_dir = os.path.join(self.vagrant_dir, 'lxc-box')
            if os.path.exists(lxc_box_dir):
                execute('sudo rm -rf %s' % lxc_box_dir)
            os.mkdir(lxc_box_dir)
            lxc_container_path = os.path.join('/var/lib/lxc', self.name)
            execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" > %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
            execute('sudo bash -c "cd %s && tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*"' % (lxc_container_path, lxc_box_dir))
            execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
            execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
            with open(os.path.join(lxc_box_dir, 'metadata.json'), 'w') as f:
                now = datetime.datetime.now()
                f.write('{\n')
                f.write('  "provider": "lxc",\n')
                f.write('  "version":  "1.0.0",\n')
                f.write('  "built-on": "%s"\n' % now.strftime('%c'))
                f.write('}\n')

            execute('tar -czf %s ./*' % box_path, cwd=lxc_box_dir)
            execute('sudo rm -rf %s' % lxc_box_dir)
295

296 297 298 299 300 301 302 303 304 305 306 307
    def upload(self, src):
        attempt = 4
        while attempt > 0:
            exitcode = execute('vagrant upload %s' % src, cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
            if exitcode == 0:
                break
            attempt -= 1
        if exitcode != 0:
            msg = 'cannot upload %s' % src
            log.error(msg)
            raise ExecutionError(msg)

308
    def run_build_and_test(self, tarball_path, jobs):
309 310 311
        if self.dry_run:
            return 0, 0

312 313
        if not tarball_path:
            name_ver = 'kea-1.5.0'
314 315 316 317
            cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
            cmd += ' --exclude "*~" --exclude .git --exclude .libs --exclude .deps --exclude \'*.o\'  --exclude \'*.lo\' '
            cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
            execute(cmd)
318
            tarball_path = '/tmp/%s.tar.gz' % name_ver
319
        self.upload(tarball_path)
320 321 322

        log_file_path = os.path.join(self.vagrant_dir, 'build.log')
        log.info('Build log file stored to %s', log_file_path)
323 324

        t0 = time.time()
325

326
        bld_cmd = "%s hammer.py build -p local -t %s.tar.gz -j %d" % (self.python, name_ver, jobs)
327 328 329 330 331 332 333 334 335 336 337 338 339
        if self.features_arg:
            bld_cmd += ' ' + self.features_arg
        if self.nofeatures_arg:
            bld_cmd += ' ' + self.nofeatures_arg
        if self.check_times:
            bld_cmd += ' -i'
        self.execute(bld_cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)  # timeout: 40 minutes

        ssh_cfg_path = self.dump_ssh_config()

        if 'native-pkg' in self.features:
            execute('scp -F %s -r default:/home/vagrant/rpm-root/RPMS/x86_64/ .' % ssh_cfg_path)

340 341
        t1 = time.time()
        dt = int(t1 - t0)
342 343

        log.info('Build log file stored to %s', log_file_path)
344 345 346 347
        log.info("")
        log.info(">>>>>> Build time %s:%s", dt // 60, dt % 60)
        log.info("")

348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364
        total = 0
        passed = 0
        try:
            if 'unittest' in self.features:
                execute('scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path, cwd=self.vagrant_dir)
                results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
                if os.path.exists(results_file):
                    with open(results_file) as f:
                        txt = f.read()
                        results = json.loads(txt)
                        total = results['grand_total']
                        passed = results['grand_passed']
        except:
            log.exception('ignored issue with parsing unit test results')

        return total, passed

365 366
    def destroy(self):
        cmd = 'vagrant destroy --force'
367
        execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run)  # timeout: 3 minutes
368 369

    def ssh(self):
370
        execute('vagrant ssh', cwd=self.vagrant_dir, timeout=None, dry_run=self.dry_run, interactive=True)
371 372 373 374 375 376 377 378 379 380

    def dump_ssh_config(self):
        ssh_cfg_path = os.path.join(self.vagrant_dir, 'ssh.cfg')
        execute('vagrant ssh-config > %s' % ssh_cfg_path, cwd=self.vagrant_dir)
        return ssh_cfg_path

    def execute(self, cmd, timeout=None, raise_error=True, log_file_path=None, quiet=False, env=None):
        if not env:
            env = os.environ.copy()
        env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
381

382 383
        return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout, raise_error=raise_error,
                       dry_run=self.dry_run, log_file_path=log_file_path, quiet=quiet, check_times=self.check_times)
384

385
    def prepare_system(self):
386 387
        if self.features:
            self.features_arg = '--with ' + ' '.join(self.features)
388 389 390
        else:
            self.features_arg = ''

391
        nofeatures = set(DEFAULT_FEATURES) - self.features
392 393 394 395 396
        if nofeatures:
            self.nofeatures_arg = '--without ' + ' '.join(nofeatures)
        else:
            self.nofeatures_arg = ''

397
        if self.system == 'centos' and self.sys_revision == '7' or (self.system == 'debian' and self.sys_revision == '8' and self.provider != 'lxc'):
398 399 400 401 402 403 404 405 406
            self.python = 'python'
        elif self.system == 'freebsd':
            self.python = 'python3.6'
        else:
            self.python = 'python3'

        if self.system == 'rhel' and self.sys_revision == '8':
            exitcode = self.execute("sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms", raise_error=False)
            if exitcode != 0:
407 408 409 410 411
                env = os.environ.copy()
                with open(os.path.expanduser('~/rhel-creds.txt')) as f:
                    env['RHEL_USER'] = f.readline().strip()
                    env['RHEL_PASSWD'] = f.readline().strip()
                self.execute('sudo subscription-manager register --user $RHEL_USER --password "$RHEL_PASSWD"', env=env)
412 413 414 415 416 417
                self.execute("sudo subscription-manager refresh")
                self.execute("sudo subscription-manager attach --pool 8a85f99a67cdc3e70167e45c85f47429")
                self.execute("sudo subscription-manager repos --enable rhel-8-for-x86_64-baseos-beta-rpms")
                self.execute("sudo dnf install -y python36")

        hmr_py_path = os.path.join(self.repo_dir, 'hammer.py')
418
        self.upload(hmr_py_path)
419

420 421 422 423
        log_file_path = os.path.join(self.vagrant_dir, 'prepare.log')
        log.info('Prepare log file stored to %s', log_file_path)

        cmd = "{python} hammer.py prepare-deps {features} {nofeatures} {check_times}"
424 425
        cmd = cmd.format(features=self.features_arg,
                         nofeatures=self.nofeatures_arg,
426 427 428
                         python=self.python,
                         check_times='-i' if self.check_times else '')
        self.execute(cmd, timeout=40 * 60, log_file_path=log_file_path, quiet=self.quiet)
429 430 431 432 433


def _install_gtest_sources():
    if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
        execute('wget --no-verbose -O /tmp/gtest.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz')
434
        execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
435 436 437
        os.unlink('/tmp/gtest.tar.gz')


438
def _configure_mysql(system, revision, features):
439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
    if system in ['fedora', 'centos']:
        execute('sudo systemctl enable mariadb.service')
        execute('sudo systemctl start mariadb.service')
        time.sleep(5)
    cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
    execute(cmd)
    cmd = "echo 'DROP USER 'keatest'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "echo 'DROP USER 'keatest_readonly'@'localhost';' | sudo mysql -u root"
    execute(cmd, raise_error=False)
    cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "CREATE USER 'keatest'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "CREATE USER 'keatest_readonly'@'localhost' IDENTIFIED BY 'keatest';\n"
    cmd += "GRANT ALL ON keatest.* TO 'keatest'@'localhost';\n"
    cmd += "GRANT SELECT ON keatest.* TO 'keatest_readonly'@'localhost';\n"
    cmd += "EOF\n\""
    execute(cmd)

458 459 460 461 462 463 464 465 466 467 468 469
    if 'forge' in features:
        cmd = "echo 'DROP DATABASE IF EXISTS keadb;' | sudo mysql -u root"
        execute(cmd)
        cmd = "echo 'DROP USER 'keauser'@'localhost';' | sudo mysql -u root"
        execute(cmd, raise_error=False)
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "CREATE USER 'keauser'@'localhost' IDENTIFIED BY 'keapass';\n"
        cmd += "GRANT ALL ON keadb.* TO 'keauser'@'localhost';\n"
        cmd += "EOF\n\""
        execute(cmd)

470 471 472 473 474 475 476 477 478 479 480 481 482
    log.info("FIX FOR ISSUE: %s %s", system, revision)
    if system == 'debian' and revision == '9':
        log.info("FIX FOR ISSUE 2: %s %s", system, revision)
        # fix for issue: https://gitlab.isc.org/isc-projects/kea/issues/389
        cmd = "bash -c \"cat <<EOF | sudo mysql -u root\n"
        cmd += "use keatest;\n"
        cmd += "set global innodb_large_prefix=on;\n"
        cmd += "set global innodb_file_format=Barracuda;\n"
        cmd += "set global innodb_file_per_table=true;\n"
        cmd += "set global innodb_default_row_format=dynamic;\n"
        cmd += "EOF\n\""
        execute(cmd)

483

484
def _configure_pgsql(system, features):
485 486
    if system in ['fedora', 'centos']:
        # https://fedoraproject.org/wiki/PostgreSQL
Michal Nowikowski's avatar
Michal Nowikowski committed
487
        exitcode = execute('sudo ls /var/lib/pgsql/data/postgresql.conf', raise_error=False)
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
        if exitcode != 0:
            execute('sudo postgresql-setup --initdb --unit postgresql')
    execute('sudo systemctl start postgresql.service')
    cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
    cmd += "DROP DATABASE IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest;\n"
    cmd += "DROP USER IF EXISTS keatest_readonly;\n"
    cmd += "CREATE USER keatest WITH PASSWORD 'keatest';\n"
    cmd += "CREATE USER keatest_readonly WITH PASSWORD 'keatest';\n"
    cmd += "CREATE DATABASE keatest;\n"
    cmd += "GRANT ALL PRIVILEGES ON DATABASE keatest TO keatest;\n"
    cmd += "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES to keatest_readonly;\n"
    cmd += "EOF\n\""
    execute(cmd)

503 504 505 506 507 508 509 510 511 512
    if 'forge' in features:
        cmd = "bash -c \"cat <<EOF | sudo -u postgres psql postgres\n"
        cmd += "DROP DATABASE IF EXISTS keadb;\n"
        cmd += "DROP USER IF EXISTS keauser;\n"
        cmd += "CREATE USER keauser WITH PASSWORD 'keapass';\n"
        cmd += "CREATE DATABASE keadb;\n"
        cmd += "GRANT ALL PRIVILEGES ON DATABASE keauser TO keadb;\n"
        cmd += "EOF\n\""
        execute(cmd)

513 514 515 516 517 518 519 520 521 522 523 524 525 526 527

def _install_cassandra_deb():
    if not os.path.exists('/usr/sbin/cassandra'):
        execute('echo "deb http://www.apache.org/dist/cassandra/debian 311x main" | sudo tee /etc/apt/sources.list.d/cassandra.sources.list')
        execute('curl https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -')
        execute('sudo apt update')
        execute('sudo apt install -y cassandra libuv1 pkgconf')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver-dev_2.11.0-1_amd64.deb')
        execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb')
        execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb')
        execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb')


528 529 530 531 532 533 534 535 536 537 538
def _install_freeradius_client():
    execute('rm -rf freeradius-client')
    execute('git clone https://github.com/fxdupont/freeradius-client.git')
    execute('git checkout iscdev', cwd='freeradius-client')
    execute('./configure', cwd='freeradius-client')
    execute('make', cwd='freeradius-client')
    execute('sudo make install', cwd='freeradius-client')
    execute('sudo ldconfig')
    execute('rm -rf freeradius-client')


539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557
def _install_cassandra_rpm(system):
    if not os.path.exists('/usr/bin/cassandra'):
        #execute('sudo dnf config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/')
        #execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
        if system == 'fedora':
            install_yum('cassandra cassandra-server libuv libuv-devel')
        #elif system == 'centos':
        else:
            raise NotImplementedError

    execute('sudo systemctl start cassandra')

    if not os.path.exists('/usr/include/cassandra.h'):
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm')
        execute('wget http://downloads.datastax.com/cpp-driver/centos/7/cassandra/v2.11.0/cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('sudo rpm -i cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')
        execute('rm -rf cassandra-cpp-driver-2.11.0-1.el7.x86_64.rpm cassandra-cpp-driver-devel-2.11.0-1.el7.x86_64.rpm')


558
def prepare_system_local(features, check_times):
559 560 561
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

562 563 564 565
    system, revision = get_system_revision()
    log.info('Preparing deps for %s %s', system, revision)

    if system == 'fedora':
566
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel']
567 568 569 570

        if 'native-pkg' in features:
            packages.extend(['rpm-build', 'mariadb-connector-c-devel'])

571 572 573 574 575 576 577 578 579
        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'community-mysql-devel'])

        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

580 581 582
        if 'radius' in features:
            packages.extend(['git'])

583
        cmd = 'sudo dnf -y install %s' % ' '.join(packages)
Michal Nowikowski's avatar
Michal Nowikowski committed
584
        execute(cmd, env=env, timeout=300, check_times=check_times)
585 586 587 588

        if 'unittest' in features:
            _install_gtest_sources()

589
        execute('sudo dnf clean packages', env=env, check_times=check_times)
590

591
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
592
            _install_cassandra_rpm(system)
593 594 595

    elif system == 'centos':
        install_yum('epel-release', env=env, check_times=check_times)
596 597 598 599 600

        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']

        if 'docs' in features:
601 602 603 604
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])

        if 'mysql' in features:
            packages.extend(['mariadb', 'mariadb-server', 'mariadb-devel'])
605

606 607 608
        if 'pgsql' in features:
            packages.extend(['postgresql-devel', 'postgresql-server'])

609 610 611
        if 'radius' in features:
            packages.extend(['git'])

612
        install_yum(packages, env=env, check_times=check_times)
613 614 615 616

        if 'unittest' in features:
            _install_gtest_sources()

617
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
618
            _install_cassandra_rpm(system)
619 620

    elif system == 'rhel':
621 622 623 624
        packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'boost-devel',
                    'mariadb-devel', 'postgresql-devel']
        packages.extend(['rpm-build'])

625 626
        if 'docs' in features and not revision == '8':
            packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
627

628 629 630 631 632 633
        # if 'mysql' in features:
        #     packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        # if 'pgsql' in features:
        #     packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

634 635 636
        if 'radius' in features:
            packages.extend(['git'])

637
        install_cmd = 'sudo dnf -y install %s'
638
        execute(install_cmd % ' '.join(packages), env=env, timeout=120, check_times=check_times)
639 640 641

        # prepare lib4cplus as epel repos are not available for rhel 8 yet
        if revision == '8' and not os.path.exists('/usr/include/log4cplus/logger.h'):
642 643 644 645 646 647
            if not os.path.exists('srpms'):
                execute('mkdir srpms')
            execute('rm -rf srpms/*')
            execute('rm -rf rpmbuild')
            execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
                    check_times=check_times)
648
            execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm', env=env, timeout=120, check_times=check_times)
649 650
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times)
            execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times)
651 652 653 654

        if 'unittest' in features:
            _install_gtest_sources()

655
        if 'cql' in features:
Michal Nowikowski's avatar
Michal Nowikowski committed
656
            _install_cassandra_rpm(system)
657

658
    elif system == 'ubuntu':
659
        execute('sudo apt update', env=env, check_times=check_times)
660 661 662 663

        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev']

        if 'unittest' in features:
664 665 666 667
            if revision.startswith('16.'):
                _install_gtest_sources()
            else:
                packages.append('googletest')
668 669

        if 'docs' in features:
670
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
671 672 673 674 675 676

        if 'native-pkg' in features:
            packages.extend(['build-essential', 'fakeroot', 'devscripts'])
            packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev', 'docbook', 'docbook-xsl', 'flex', 'libboost-dev',
                             'libpq-dev', 'postgresql-server-dev-all', 'python3-dev'])

677 678 679 680 681 682
        if 'mysql' in features:
            packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

        if 'pgsql' in features:
            packages.extend(['postgresql-client', 'libpq-dev', 'postgresql-all'])

683 684 685
        if 'radius' in features:
            packages.extend(['git'])

686 687 688 689 690 691 692 693 694 695 696
        done = False
        while not done:
            try:
                execute('sudo apt install --no-install-recommends -y %s' % ' '.join(packages), env=env, timeout=240, check_times=check_times)
                done = True
            except:
                log.exception('ble')
                time.sleep(20)

        if 'cql' in features:
            _install_cassandra_deb()
697 698

    elif system == 'debian':
699
        execute('sudo apt update', env=env, check_times=check_times)
700 701 702 703

        packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev']

        if 'docs' in features:
704
            packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
705 706 707

        if 'unittest' in features:
            if revision == '8':
708 709
                _install_gtest_sources()
                #packages.append('libgtest-dev') # it does not work
710 711 712
            else:
                packages.append('googletest')

713 714 715
        if 'mysql' in features:
            packages.extend(['default-mysql-client-core', 'default-libmysqlclient-dev', 'mysql-server'])

716 717 718
        if 'radius' in features:
            packages.extend(['git'])

719 720 721 722
        execute('sudo apt install --no-install-recommends -y %s' % ' '.join(packages), env=env, timeout=240, check_times=check_times)

        if 'cql' in features:
            _install_cassandra_deb()
723 724 725 726

    elif system == 'freebsd':
        packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs']

727 728
        #execute('sudo portsnap --interactive fetch', timeout=240, check_times=check_times)
        #execute('sudo portsnap extract /usr/ports/devel/log4cplus', timeout=240, check_times=check_times)
729

730 731 732 733
        #execute('sudo make -C /usr/ports/devel/log4cplus install clean BATCH=yes', timeout=240, check_times=check_times)

        if 'docs' in features:
            packages.extend(['libxslt', 'elinks', 'docbook-xsl'])
734

735 736 737
        if 'radius' in features:
            packages.extend(['git'])

738 739 740
        if 'unittest' in features:
            _install_gtest_sources()

741 742
        execute('sudo pkg install -y %s' % ' '.join(packages), env=env, timeout=6 * 60, check_times=check_times)

743 744 745
    else:
        raise NotImplementedError

746
    if 'mysql' in features:
747
        _configure_mysql(system, revision, features)
748

749
    if 'pgsql' in features:
750
        _configure_pgsql(system, features)
751

752 753 754 755 756
    if 'radius' in features:
        _install_freeradius_client()

    #execute('sudo rm -rf /usr/share/doc')

757 758
    log.info('Preparing deps completed successfully.')

759

760
def build_local(features, tarball_path, check_times, jobs, dry_run):
761 762 763 764 765
    env = os.environ.copy()
    env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'

    distro, revision = get_system_revision()

766
    execute('df -h', dry_run=dry_run)
767

768 769
    if tarball_path:
        tarball_path = os.path.abspath(tarball_path)
770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785

    if 'native-pkg' in features:
        # native pkg build

        if distro in ['fedora', 'centos', 'rhel']:
            execute('rm -rf rpm-root')
            os.mkdir('rpm-root')
            os.mkdir('rpm-root/BUILD')
            os.mkdir('rpm-root/BUILDROOT')
            os.mkdir('rpm-root/RPMS')
            os.mkdir('rpm-root/SOURCES')
            os.mkdir('rpm-root/SPECS')
            os.mkdir('rpm-root/SRPMS')

            execute('rm -rf kea-src')
            os.mkdir('kea-src')
786
            execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
787 788 789 790 791
            src_path = glob.glob('kea-src/*')[0]
            rpm_dir = os.path.join(src_path, 'rpm')
            for f in os.listdir(rpm_dir):
                if f == 'kea.spec':
                    continue
792 793 794
                execute('cp %s rpm-root/SOURCES' % os.path.join(rpm_dir, f), check_times=check_times)
            execute('cp %s rpm-root/SPECS' % os.path.join(rpm_dir, 'kea.spec'), check_times=check_times)
            execute('cp %s rpm-root/SOURCES' % tarball_path, check_times=check_times)
795 796

            cmd = "rpmbuild -ba rpm-root/SPECS/kea.spec -D'_topdir /home/vagrant/rpm-root'"
797
            execute(cmd, env=env, timeout=60 * 40, check_times=check_times)
798 799

            if 'install' in features:
800
                execute('sudo rpm -i rpm-root/RPMS/x86_64/*rpm', check_times=check_times)
801 802

        elif distro in ['ubuntu', 'debian']:
803
            execute('rm -rf kea-src', check_times=check_times)
804
            os.mkdir('kea-src')
805
            execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
806 807
            src_path = glob.glob('kea-src/*')[0]

808
            execute('debuild -i -us -uc -b', env=env, cwd=src_path, timeout=60 * 40, check_times=check_times)
809 810

            if 'install' in features:
811
                execute('sudo dpkg -i kea-src/*deb', check_times=check_times)
812 813 814 815 816 817 818 819 820 821

        else:
            raise NotImplementedError

    else:
        # build straight from sources

        if tarball_path:
            execute('rm -rf kea-src')
            os.mkdir('kea-src')
822
            execute('tar -zxf %s' % tarball_path, cwd='kea-src', check_times=check_times)
823 824 825 826
            src_path = glob.glob('kea-src/*')[0]
        else:
            src_path = '.'

827
        execute('autoreconf -f -i', cwd=src_path, env=env, dry_run=dry_run)
828 829 830 831 832 833

        cmd = './configure'
        if 'mysql' in features:
            cmd += ' --with-mysql'
        if 'pgsql' in features:
            cmd += ' --with-pgsql'
834 835
        if 'cql' in features:
            cmd += ' --with-cql=/usr/bin/pkg-config'
836
        if 'unittest' in features:
837
            if distro in ['centos', 'fedora', 'rhel', 'freebsd']:
838 839
                cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
            elif distro == 'debian' and revision == '8':
840 841 842
                #cmd += ' --with-gtest-source=/usr/src/gtest' # it does not work
                cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
            elif distro == 'debian':
843
                cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
844 845 846 847 848
            elif distro == 'ubuntu':
                if revision.startswith('16.'):
                    cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
                else:
                    cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
849 850
            else:
                raise NotImplementedError
851
        if 'docs' in features and not (distro == 'rhel' and revision == '8'):
852
            cmd += ' --enable-generate-docs'
853 854
        if 'radius' in features:
            cmd += ' --with-freeradius=/usr/local'
855 856
        if 'shell' in features:
            cmd += ' --enable-shell'
857

858
        execute(cmd, cwd=src_path, env=env, timeout=120, check_times=check_times, dry_run=dry_run)
859

860 861 862 863 864 865 866 867
        if jobs == 0:
            cpus = multiprocessing.cpu_count() - 1
            if distro == 'centos':
                cpus = cpus // 2
            if cpus == 0:
                cpus = 1
        else:
            cpus = jobs
868
        cmd = 'make -j%s' % cpus
869
        execute(cmd, cwd=src_path, env=env, timeout=40 * 60, check_times=check_times, dry_run=dry_run)
870 871

        if 'unittest' in features:
872
            results_dir = os.path.abspath(os.path.join(src_path, 'tests_result'))
873
            execute('rm -rf %s' % results_dir, dry_run=dry_run)
874 875 876 877
            if not os.path.exists(results_dir):
                os.mkdir(results_dir)
            env['GTEST_OUTPUT'] = 'xml:%s/' % results_dir
            env['KEA_SOCKET_TEST_DIR'] = '/tmp/'
878
            execute('make check -k', cwd=src_path, env=env, timeout=60 * 60, raise_error=False, check_times=check_times, dry_run=dry_run)
879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909

            results = {}
            grand_total = 0
            grand_not_passed = 0
            for fn in os.listdir(results_dir):
                if not fn.endswith('.xml'):
                    continue
                fp = os.path.join(results_dir, fn)
                tree = ET.parse(fp)
                root = tree.getroot()
                total = int(root.get('tests'))
                failures = int(root.get('failures'))
                disabled = int(root.get('disabled'))
                errors = int(root.get('errors'))
                results[fn] = dict(total=total, failures=failures, disabled=disabled, errors=errors)
                grand_total += total
                grand_not_passed += failures + errors

            grand_passed = grand_total - grand_not_passed
            results['grand_passed'] = grand_total - grand_not_passed
            results['grand_total'] = grand_total

            result = '%s/%s passed' % (grand_passed, grand_total)
            if grand_not_passed > 0 or grand_total == 0:
                result = red(result)
            else:
                result = green(result)
            log.info('Unit test results: %s', result)

            with open('unit-test-results.json', 'w') as f:
                f.write(json.dumps(results))
910 911

        if 'install' in features:
912 913 914 915 916 917 918 919
            execute('sudo make install', cwd=src_path, env=env, check_times=check_times, dry_run=dry_run)
            execute('sudo ldconfig', dry_run=dry_run)  # TODO: this shouldn't be needed

            if 'forge' in features:
                if 'mysql' in features:
                    execute('kea-admin lease-init mysql -u keauser -p keapass -n keadb', dry_run=dry_run)
                if 'pgsql' in features:
                    execute('kea-admin lease-init pgsql -u keauser -p keapass -n keadb', dry_run=dry_run)
920

921
    execute('df -h', dry_run=dry_run)
922 923


924
def build_in_vagrant(provider, system, sys_revision, features, leave_system, tarball_path, dry_run, quiet, clean_start, check_times, jobs):
925 926 927 928 929 930
    log.info('')
    log.info(">>> Building %s, %s, %s" % (provider, system, sys_revision))
    log.info('')

    t0 = time.time()

931
    ve = None
932 933 934
    error = None
    total = 0
    passed = 0
935
    try:
936 937
        ve = VagrantEnv(provider, system, sys_revision, features, 'kea', dry_run, quiet, check_times)
        if clean_start:
938
            ve.destroy()
939
        ve.up()
940
        ve.prepare_system()
941
        total, passed = ve.run_build_and_test(tarball_path, jobs)
942 943 944 945 946 947 948 949 950 951 952 953
        msg = ' - ' + green('all ok')
    except KeyboardInterrupt as e:
        error = e
        msg = ' - keyboard interrupt'
    except ExecutionError as e:
        error = e
        msg = ' - ' + red(str(e))
    except Exception as e:
        log.exception('Building erred')
        error = e
        msg = ' - ' + red(str(e))
    finally:
954
        if not leave_system and ve:
955
            ve.destroy()
956 957 958 959 960

    t1 = time.time()
    dt = int(t1 - t0)

    log.info('')
961
    log.info(">>> Building %s, %s, %s completed in %s:%s%s", provider, system, sys_revision, dt // 60, dt % 60, msg)
962 963
    log.info('')

964
    return dt, error, total, passed
965 966


967 968
def package_box(provider, system, sys_revision, features, dry_run, check_times):
    ve = VagrantEnv(provider, system, sys_revision, features, 'bare', dry_run, check_times=check_times)
969
    ve.destroy()
970
    ve.up()
971
    ve.prepare_system()
972
    # TODO cleanup
973 974 975
    ve.package()


976
def prepare_system_in_vagrant(provider, system, sys_revision, features, dry_run, check_times, clean_start):
977 978
    ve = VagrantEnv(provider, system, sys_revision, features, 'kea', dry_run, check_times=check_times)
    if clean_start:
979
        ve.destroy()
980
    ve.up()
981
    ve.prepare_system()
982 983


984 985
def ssh(provider, system, sys_revision):
    ve = VagrantEnv(provider, system, sys_revision, [], 'kea', False)
986 987 988 989
    ve.up()
    ve.ssh()


990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
def ensure_hammer_deps():
    distro, _ = get_system_revision()

    exitcode = execute('vagrant version', raise_error=False)
    if exitcode != 0:
        if distro in ['fedora', 'centos', 'rhel']:
            execute('wget --no-verbose -O /tmp/vagrant_2.2.2_x86_64.rpm https://releases.hashicorp.com/vagrant/2.2.2/vagrant_2.2.2_x86_64.rpm')
            execute('sudo rpm -i /tmp/vagrant_2.2.2_x86_64.rpm')
            os.unlink('/tmp/vagrant_2.2.2_x86_64.rpm')
        elif distro in ['debian', 'ubuntu']:
            execute('wget --no-verbose -O /tmp/vagrant_2.2.2_x86_64.deb https://releases.hashicorp.com/vagrant/2.2.2/vagrant_2.2.2_x86_64.deb')
            execute('sudo dpkg -i /tmp/vagrant_2.2.2_x86_64.deb')
            os.unlink('/tmp/vagrant_2.2.2_x86_64.deb')
        else:
            # TODO: check for packages here: https://www.vagrantup.com/downloads.html
            raise NotImplementedError

    exitcode = execute('vagrant plugin list | grep vagrant-lxc', raise_error=False)
    if exitcode != 0:
        execute('vagrant plugin install vagrant-lxc')


1012
DEFAULT_FEATURES = ['install', 'unittest', 'docs']
1013
ALL_FEATURES = ['install', 'unittest', 'docs', 'mysql', 'pgsql', 'cql', 'native-pkg', 'radius', 'shell', 'forge']
1014 1015

def parse_args():
1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053
    main_parser = argparse.ArgumentParser(description='Hammer - Kea develepment environment management tool.')
    main_parser.add_argument('-v', '--verbose', action='store_true', help='Enable verbose mode.')
    main_parser.add_argument('-q', '--quiet', action='store_true', help='Enable quiet mode.')

    subparsers = main_parser.add_subparsers(dest='command',
                                            title="Hammer commands",
                                            description="The following commands are provided by Hammer. "
                                            "To get more information about particular command invoke: ./hammer.py <command> -h.")

    parent_parser1 = argparse.ArgumentParser(add_help=False)
    parent_parser1.add_argument('-p', '--provider', default='virtualbox', choices=['lxc', 'virtualbox', 'local', 'all'],
                                help="Backend build executor. If 'all' then build is executed several times on all providers. "
                                "If 'local' then build is executed on current system. Default is 'virtualbox'.")
    parent_parser1.add_argument('-s', '--system', default='all', choices=list(SYSTEMS.keys()) + ['all'],
                                help="Build is executed on selected system. If 'all' then build is executed several times on all systems. "
                                "If provider is 'local' then this option is ignored. Default is 'all'.")
    parent_parser1.add_argument('-r', '--revision', default='all',
                                help="Revision of selected system. If 'all' then build is executed several times "
                                "on all revisions of selected system. To list supported systems and their revisions invoke 'supported-systems'. "
                                "Default is 'all'.")

    parent_parser2 = argparse.ArgumentParser(add_help=False)
    parent_parser2.add_argument('-w', '--with', nargs='+', default=set(), choices=ALL_FEATURES,
                                help="Enabled, comma-separated features. Default is '%s'." % ' '.join(DEFAULT_FEATURES))
    parent_parser2.add_argument('-x', '--without', nargs='+', default=set(), choices=ALL_FEATURES,
                                help="Disabled, comma-separated features. Default is ''.")
    parent_parser2.add_argument('-l', '--leave-system', action='store_true',
                                help='At the end of the command do not destroy vagrant system. Default behavior is destroing the system.')
    parent_parser2.add_argument('-c', '--clean-start', action='store_true', help='If there is pre-existing system then it is destroyed first.')
    parent_parser2.add_argument('-i', '--check-times', action='store_true', help='Do not allow executing commands infinitelly.')
    parent_parser2.add_argument('-n', '--dry-run', action='store_true', help='Print only what would be done.')


    parser = subparsers.add_parser('ensure-hammer-deps', help="Install Hammer dependencies on current, host system.")
    parser = subparsers.add_parser('supported-systems', help="List system supported by Hammer for doing Kea development.")
    parser = subparsers.add_parser('build', help="Prepare system and run Kea build in indicated system.",
                                   parents=[parent_parser1, parent_parser2])
    parser.add_argument('-j', '--jobs', default=0, help='Number of processes used in compilation. Override make -j default value.')
1054 1055
    parser.add_argument('-t', '--from-tarball',
                        help='Instead of building sources in current folder use provided tarball package (e.g. tar.gz).')
1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
    parser = subparsers.add_parser('prepare-system',
                                   help="Prepare system for doing Kea development i.e. install all required dependencies "
                                   "and pre-configure the system. build command always first calls prepare-system internally.",
                                   parents=[parent_parser1, parent_parser2])
    parser = subparsers.add_parser('ssh', help="SSH to indicated system.",
                                   formatter_class=argparse.RawDescriptionHelpFormatter,
                                   description="Allows getting into the system using SSH. If the system is not present then it will be created first "
                                   "but not prepared. The command can be run in 2 way: "
                                   "\n1) ./hammer.py ssh -p <provider> -s <system> -r <revision>\n2) ./hammer.py ssh -d <path-to-vagrant-dir>",
                                   parents=[parent_parser1])
    parser.add_argument('-d', '--directory', help='Path to directory with Vagrantfile.')
    parser = subparsers.add_parser('created-systems', help="List ALL systems created by Hammer.")
    parser = subparsers.add_parser('destroy', help="Destroy indicated system.",
                                   description="Destroys system indicated by a path to directory with Vagrantfile. "
                                   "To get the list of created systems run: ./hammer.py created-systems.")
    parser.add_argument('-d', '--directory', help='Path to directory with Vagrantfile.')
    parser = subparsers.add_parser('package-box', help="Package currently running system into Vagrant Box. Prepared box can be later deployed to Vagrant Cloud.",
                                   parents=[parent_parser1, parent_parser2])

    args = main_parser.parse_args()
1076 1077 1078 1079

    return args


1080
def list_supported_systems():
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
    for system, revisions in SYSTEMS.items():
        print('%s:' % system)
        for r in revisions:
            providers = []
            for p in ['lxc', 'virtualbox']:
                k = '%s-%s-%s' % (system, r, p)
                if k in IMAGE_TEMPLATES:
                    providers.append(p)
            providers = ', '.join(providers)
            print('  - %s: %s' % (r, providers))


1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117
def list_created_systems():
    exitcode, output = execute('vagrant global-status', quiet=True, capture=True)
    systems = []
    for line in output.splitlines():
        if 'hammer' not in line:
            continue
        elems = line.split()
        state = elems[3]
        path = elems[4]
        systems.append([path, state])

    print('')
    print('%-10s %s' % ('State', 'Path'))
    print('-' * 80)
    for path, state, in sorted(systems):
        print('%-10s %s' % (state, path))
    print('-' * 80)
    print('To destroy a system run: ./hammer.py destroy -d <path>')
    print('')


def destroy_system(path):
    execute('vagrant destroy', cwd=path, interactive=True)


1118 1119 1120 1121 1122 1123 1124 1125 1126
def _what_features(args):
    features = set(vars(args)['with'])
    features = features.union(DEFAULT_FEATURES)
    nofeatures = set(args.without)
    features = features.difference(nofeatures)

    return features


1127
def _print_summary(results, features):
1128
    print("")
1129 1130 1131
    print("+===== Hammer Summary ====================================================+")
    print("|   provider |     system | revision |  duration |  status |   unit tests |")
    print("+------------+------------+----------+-----------+---------+--------------+")
1132 1133 1134
    total_dt = 0
    for key, result in results.items():
        provider, system, revision = key
1135 1136
        dt, error, ut_total, ut_passed = result

1137
        total_dt += dt
1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157
        if error is None:
            status = '     %s' % green('ok')
        elif error == 'not run':
            status = blue('not run')
        else:
            status = '  %s' % red('error')

        if 'unittest' in features:
            ut_results = '%s/%s' % (ut_passed, ut_total)
            padding = ' ' * (12 - len(ut_results))
            if ut_passed < ut_total or ut_total == 0:
                ut_results = padding + red(ut_results)
            else:
                ut_results = padding + green(ut_results)
        else:
            ut_results = ' not planned'
        print('| %10s | %10s | %8s | %6d:%02d | %s | %s |' % (provider, system, revision, dt // 60, dt % 60, status, ut_results))
    print("+------------+------------+----------+-----------+---------+--------------+")
    print("|                               Total: %6d:%02d |                        |" % (total_dt // 60, total_dt % 60))
    print("+=========================================================================+")
1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169


def main():
    args = parse_args()

    level = logging.INFO
    if args.verbose:
        level = logging.DEBUG

    format = '[HAMMER]  %(asctime)-15s  %(message)s'
    logging.basicConfig(format=format, level=level)

1170 1171
    if args.command == 'supported-systems':
        list_supported_systems()
1172

1173 1174
    elif args.command == 'created-systems':
        list_created_systems()
1175 1176

    elif args.command == "package-box":
1177
        features = _what_features(args)
1178
        log.info('Enabled features: %s', ' '.join(features))
1179
        package_box(args.provider, args.system, args.revision, features, args.dry_run, args.check_times)
1180 1181

    elif args.command == "prepare-system":
1182 1183 1184 1185 1186 1187 1188
        if args.system == 'all' or args.revision == 'all':
            print('Please provide required system and its version.')
            print('Example: ./hammer.py prepare-system -s fedora -r 28.')
            print('To get list of supported systems run: ./hammer.py supported-systems.')
            sys.exit(1)

        features = _what_features(args)
1189
        log.info('Enabled features: %s', ' '.join(features))
1190 1191 1192 1193 1194 1195

        if args.provider == 'local':
            prepare_system_local(features, args.check_times)
            return

        prepare_system_in_vagrant(args.provider, args.system, args.revision, features, args.dry_run, args.check_times, args.clean_start)
1196 1197

    elif args.command == "build":
1198
        features = _what_features(args)
1199 1200
        log.info('Enabled features: %s', ' '.join(features))
        if args.provider == 'local':
1201
            build_local(features, args.from_tarball, args.check_times, int(args.jobs), args.dry_run)
1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213
            return

        if args.provider == 'all':
            providers = ['lxc', 'virtualbox']
        else:
            providers = [args.provider]

        if args.system == 'all':
            systems = SYSTEMS.keys()
        else:
            systems = [args.system]

1214
        plan = []
1215
        results = {}
1216
        log.info('Build plan:')
1217 1218 1219 1220 1221 1222 1223 1224
        for provider in providers:
            for system in systems:
                if args.revision == 'all':
                    revisions = SYSTEMS[system]
                else:
                    revisions = [args.revision]

                for revision in revisions:
1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
                    if args.revision == 'all':
                        key = '%s-%s-%s' % (system, revision, provider)
                        if key not in IMAGE_TEMPLATES:
                            continue
                    plan.append((provider, system, revision))
                    log.info(' - %s, %s, %s', provider, system, revision)
                    results[(provider, system, revision)] = (0, 'not run')

        fail = False
        for provider, system, revision in plan:
            duration, error, total, passed = build_in_vagrant(provider, system, revision, features, args.leave_system, args.from_tarball,
1236
                                                              args.dry_run, args.quiet, args.clean_start, args.check_times, int(args.jobs))
1237 1238 1239 1240 1241 1242
            results[(provider, system, revision)] = (duration, error, total, passed)

            if error:
                fail = True
                if isinstance(error, KeyboardInterrupt):
                    break
1243

1244
        _print_summary(results, features)
1245 1246 1247 1248 1249

        if fail:
            sys.exit(1)

    elif args.command == "ssh":
1250
        ssh(args.provider, args.system, args.revision)
1251 1252 1253

    elif args.command == "ensure-hammer-deps":
        ensure_hammer_deps()
1254

1255 1256 1257
    elif args.command == "destroy":
        destroy_system(args.directory)

1258 1259 1260 1261 1262 1263 1264 1265 1266

if __name__ == '__main__':
    # results = {
    #     ('virtualbox', 'centos', '7'): (920, False),
    #     ('lxc', 'fedora', '29'): (120, False),
    # }
    # _print_summary(results)

    main()