Commit 8b1f0055 authored by Michal Nowikowski's avatar Michal Nowikowski

[#372,!181] hammer: reduced line lenghts, added docstrings, unified naming,...

[#372,!181] hammer: reduced line lenghts, added docstrings, unified naming, fixed terminating timed out process
parent 2a66c9b1
...@@ -114,19 +114,19 @@ log = logging.getLogger() ...@@ -114,19 +114,19 @@ log = logging.getLogger()
def red(txt): def red(txt):
"""Return colorized (if the terminal supports it) or plain text""" """Return colorized (if the terminal supports it) or plain text."""
if sys.stdout.isatty(): if sys.stdout.isatty():
return '\033[1;31m%s\033[0;0m' % txt return '\033[1;31m%s\033[0;0m' % txt
return txt return txt
def green(txt): def green(txt):
"""Return colorized (if the terminal supports it) or plain text""" """Return colorized (if the terminal supports it) or plain text."""
if sys.stdout.isatty(): if sys.stdout.isatty():
return '\033[0;32m%s\033[0;0m' % txt return '\033[0;32m%s\033[0;0m' % txt
return txt return txt
def blue(txt): def blue(txt):
"""Return colorized (if the terminal supports it) or plain text""" """Return colorized (if the terminal supports it) or plain text."""
if sys.stdout.isatty(): if sys.stdout.isatty():
return '\033[0;34m%s\033[0;0m' % txt return '\033[0;34m%s\033[0;0m' % txt
return txt return txt
...@@ -155,21 +155,25 @@ class ExecutionError(Exception): ...@@ -155,21 +155,25 @@ class ExecutionError(Exception):
pass pass
def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None, quiet=False, check_times=False, capture=False, def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False, log_file_path=None,
interactive=False, attempts=1, sleep_time_after_attempt=None): quiet=False, check_times=False, capture=False, interactive=False, attempts=1,
sleep_time_after_attempt=None):
"""Execute a command in shell. """Execute a command in shell.
:param str cmd: a command to be executed :param str cmd: a command to be executed
:param int timeout: timeout in number of seconds, after that time the command is terminated but only if check_times is True :param int timeout: timeout in number of seconds, after that time the command is terminated
but only if check_times is True
:param str cwd: current working directory for the command :param str cwd: current working directory for the command
:param dict env: dictionary with environment variables :param dict env: dictionary with environment variables
:param bool raise_error: if False then in case of error exception is not raised, default: True ie exception is raise :param bool raise_error: if False then in case of error exception is not raised,
default: True ie exception is raise
:param bool dry_run: if True then the command is not executed :param bool dry_run: if True then the command is not executed
:param str log_file_path: if provided then all traces from the command are stored in indicated file :param str log_file_path: if provided then all traces from the command are stored in indicated file
:param bool quiet: if True then the command's traces are not printed to stdout :param bool quiet: if True then the command's traces are not printed to stdout
:param bool check_times: if True then timeout is taken into account :param bool check_times: if True then timeout is taken into account
:param bool capture: if True then the command's traces are captured and returned by the function :param bool capture: if True then the command's traces are captured and returned by the function
:param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled, used for e.g. SSH :param bool interactive: if True then stdin and stdout are not redirected, traces handling is disabled,
used for e.g. SSH
:param int attemts: number of attempts to run the command if it fails :param int attemts: number of attempts to run the command if it fails
:param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt :param int sleep_time_after_attempt: number of seconds to sleep before taking next attempt
""" """
...@@ -214,7 +218,12 @@ def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False ...@@ -214,7 +218,12 @@ def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False
# If no exitcode yet, ie. process is still running then it means that timeout occured. # If no exitcode yet, ie. process is still running then it means that timeout occured.
# In such case terminate the process and raise an exception. # In such case terminate the process and raise an exception.
if p.poll() is None: if p.poll() is None:
p.terminate() # kill using sudo to be able to kill other sudo commands
execute('sudo kill -s TERM %s' % p.pid)
time.sleep(5)
# if still running, kill harder
if p.poll() is None:
execute('sudo kill -s KILL %s' % p.pid)
raise ExecutionError('Execution timeout') raise ExecutionError('Execution timeout')
exitcode = p.returncode exitcode = p.returncode
...@@ -241,7 +250,7 @@ def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False ...@@ -241,7 +250,7 @@ def execute(cmd, timeout=60, cwd=None, env=None, raise_error=True, dry_run=False
def install_pkgs(pkgs, timeout=60, env=None, check_times=False): def install_pkgs(pkgs, timeout=60, env=None, check_times=False):
"""Installs native packages in a system. """Install native packages in a system.
:param dict pkgs: specifies a list of packages to be installed :param dict pkgs: specifies a list of packages to be installed
:param int timeout: timeout in number of seconds, after that time the command :param int timeout: timeout in number of seconds, after that time the command
...@@ -281,10 +290,22 @@ class VagrantEnv(object): ...@@ -281,10 +290,22 @@ class VagrantEnv(object):
for Kea build and building Kea. for Kea build and building Kea.
""" """
def __init__(self, provider, system, sys_revision, features, image_template_variant, dry_run, quiet=False, check_times=False): def __init__(self, provider, system, revision, features, image_template_variant,
dry_run, quiet=False, check_times=False):
"""VagrantEnv initializer.
:param str provider: indicate backend type: virtualbox or lxc
:param str system: name of the system eg. ubuntu
:param str revision: revision of the system e.g. 18.04
:param list features: list of requested features
:param str image_template_variant: variant of images' templates: bare or kea
:param bool dry_run: if False then system commands are not really executed
:param bool quiet: if True then commands will not trace to stdout
:param bool check_times: if True then commands will be terminated after given timeout
"""
self.provider = provider self.provider = provider
self.system = system self.system = system
self.sys_revision = sys_revision self.revision = revision
self.features = features self.features = features
self.dry_run = dry_run self.dry_run = dry_run
self.quiet = quiet self.quiet = quiet
...@@ -300,10 +321,11 @@ class VagrantEnv(object): ...@@ -300,10 +321,11 @@ class VagrantEnv(object):
elif provider == "lxc": elif provider == "lxc":
vagrantfile_tpl = LXC_VAGRANTFILE_TPL vagrantfile_tpl = LXC_VAGRANTFILE_TPL
image_tpl = IMAGE_TEMPLATES["%s-%s-%s" % (system, sys_revision, provider)][image_template_variant] key = "%s-%s-%s" % (system, revision, provider)
image_tpl = IMAGE_TEMPLATES[key][image_template_variant]
self.repo_dir = os.getcwd() self.repo_dir = os.getcwd()
sys_dir = "%s-%s" % (system, sys_revision) sys_dir = "%s-%s" % (system, revision)
if provider == "virtualbox": if provider == "virtualbox":
self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox') self.vagrant_dir = os.path.join(self.repo_dir, 'hammer', sys_dir, 'vbox')
elif provider == "lxc": elif provider == "lxc":
...@@ -322,7 +344,7 @@ class VagrantEnv(object): ...@@ -322,7 +344,7 @@ class VagrantEnv(object):
pass pass
crc = binascii.crc32(self.vagrant_dir.encode()) crc = binascii.crc32(self.vagrant_dir.encode())
self.name = "hmr-%s-%s-kea-srv-%08d" % (system, sys_revision.replace('.', '-'), crc) self.name = "hmr-%s-%s-kea-srv-%08d" % (system, revision.replace('.', '-'), crc)
vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl, vagrantfile = vagrantfile_tpl.format(image_tpl=image_tpl,
name=self.name) name=self.name)
...@@ -333,20 +355,22 @@ class VagrantEnv(object): ...@@ -333,20 +355,22 @@ class VagrantEnv(object):
log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir) log.info('Prepared vagrant system %s in %s', self.name, self.vagrant_dir)
def up(self): def up(self):
"""Do Vagrant up."""
execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run) execute("vagrant box update", cwd=self.vagrant_dir, timeout=20 * 60, dry_run=self.dry_run)
execute("vagrant up --no-provision --provider %s" % self.provider, cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run) execute("vagrant up --no-provision --provider %s" % self.provider,
cwd=self.vagrant_dir, timeout=15 * 60, dry_run=self.dry_run)
def package(self): def package(self):
"""Package Vagrant system into Vagrant box.""" """Package Vagrant system into Vagrant box."""
if self.provider == 'virtualbox': if self.provider == 'virtualbox':
cmd = "vagrant package --output kea-%s-%s.box" % (self.system, self.sys_revision) cmd = "vagrant package --output kea-%s-%s.box" % (self.system, self.revision)
execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run) execute(cmd, cwd=self.vagrant_dir, timeout=4 * 60, dry_run=self.dry_run)
elif self.provider == 'lxc': elif self.provider == 'lxc':
execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False) execute('vagrant halt', cwd=self.vagrant_dir, dry_run=self.dry_run, raise_error=False)
box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.sys_revision)) box_path = os.path.join(self.vagrant_dir, 'kea-%s-%s.box' % (self.system, self.revision))
if os.path.exists(box_path): if os.path.exists(box_path):
os.unlink(box_path) os.unlink(box_path)
...@@ -355,12 +379,18 @@ class VagrantEnv(object): ...@@ -355,12 +379,18 @@ class VagrantEnv(object):
execute('sudo rm -rf %s' % lxc_box_dir) execute('sudo rm -rf %s' % lxc_box_dir)
os.mkdir(lxc_box_dir) os.mkdir(lxc_box_dir)
lxc_container_path = os.path.join('/var/lib/lxc', self.name) lxc_container_path = os.path.join('/var/lib/lxc', self.name)
execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8iallvQVp22WDkTkyrtvp9eWW6A8YVr+' execute('sudo bash -c \'echo "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA6NF8ia'
'kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo' 'llvQVp22WDkTkyrtvp9eWW6A8YVr+kz4TjGYe7gHzIw+niNltGEFHzD8+v1I2YJ'
'3MlTBckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6kivAIhyfHilFR61RGL+GPXQ2M' '6oXevct1YeS0o9HZyN1Q9qgCgzUFtdOKLv6IedplqoPkcmF0aYet2PkEDo3MlTB'
'WZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmBYSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEg' 'ckFXPITAMzF8dJSIFo9D8HfdOV0IAdx4O7PtixWKn5y2hMNG0zQPyUecp4pzC6k'
'E98OHlnVYCzRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key" > %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path) 'ivAIhyfHilFR61RGL+GPXQ2MWZWFYbAGjyiYJnAmCP3NOTd0jMZEnDkbUvxhMmB'
cmd = 'sudo bash -c "cd %s && tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*"' 'YSdETk1rRgm+R4LOzFUGaHqHDLKLX+FIPKcF96hrucXzcWyLbIbEgE98OHlnVYC'
'zRdK8jlqm8tehUc9c9WhQ== vagrant insecure public key"'
'> %s/rootfs/home/vagrant/.ssh/authorized_keys\'' % lxc_container_path)
cmd = 'sudo bash -c "'
cmd += 'cd %s '
cmd += '&& tar --numeric-owner --anchored --exclude=./rootfs/dev/log -czf %s/rootfs.tar.gz ./rootfs/*'
cmd += '"'
execute(cmd % (lxc_container_path, lxc_box_dir)) execute(cmd % (lxc_container_path, lxc_box_dir))
execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir)) execute('sudo cp %s/config %s/lxc-config' % (lxc_container_path, lxc_box_dir))
execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir) execute('sudo chown `id -un`:`id -gn` *', cwd=lxc_box_dir)
...@@ -397,7 +427,8 @@ class VagrantEnv(object): ...@@ -397,7 +427,8 @@ class VagrantEnv(object):
if not tarball_path: if not tarball_path:
name_ver = 'kea-1.5.0' name_ver = 'kea-1.5.0'
cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver cmd = 'tar --transform "flags=r;s|^|%s/|" --exclude hammer ' % name_ver
cmd += ' --exclude "*~" --exclude .git --exclude .libs --exclude .deps --exclude \'*.o\' --exclude \'*.lo\' ' cmd += ' --exclude "*~" --exclude .git --exclude .libs '
cmd += ' --exclude .deps --exclude \'*.o\' --exclude \'*.lo\' '
cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver cmd += ' -zcf /tmp/%s.tar.gz .' % name_ver
execute(cmd) execute(cmd)
tarball_path = '/tmp/%s.tar.gz' % name_ver tarball_path = '/tmp/%s.tar.gz' % name_ver
...@@ -437,7 +468,8 @@ class VagrantEnv(object): ...@@ -437,7 +468,8 @@ class VagrantEnv(object):
passed = 0 passed = 0
try: try:
if 'unittest' in self.features: if 'unittest' in self.features:
execute('scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path, cwd=self.vagrant_dir) cmd = 'scp -F %s -r default:/home/vagrant/unit-test-results.json .' % ssh_cfg_path
execute(cmd, cwd=self.vagrant_dir)
results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json') results_file = os.path.join(self.vagrant_dir, 'unit-test-results.json')
if os.path.exists(results_file): if os.path.exists(results_file):
with open(results_file) as f: with open(results_file) as f:
...@@ -451,7 +483,7 @@ class VagrantEnv(object): ...@@ -451,7 +483,7 @@ class VagrantEnv(object):
return total, passed return total, passed
def destroy(self): def destroy(self):
"""Removes the VM completely.""" """Remove the VM completely."""
cmd = 'vagrant destroy --force' cmd = 'vagrant destroy --force'
execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run) # timeout: 3 minutes execute(cmd, cwd=self.vagrant_dir, timeout=3 * 60, dry_run=self.dry_run) # timeout: 3 minutes
...@@ -471,8 +503,9 @@ class VagrantEnv(object): ...@@ -471,8 +503,9 @@ class VagrantEnv(object):
env = os.environ.copy() env = os.environ.copy()
env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C' env['LANGUAGE'] = env['LANG'] = env['LC_ALL'] = 'C'
return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout, raise_error=raise_error, return execute('vagrant ssh -c "%s"' % cmd, env=env, cwd=self.vagrant_dir, timeout=timeout,
dry_run=self.dry_run, log_file_path=log_file_path, quiet=quiet, check_times=self.check_times) raise_error=raise_error, dry_run=self.dry_run, log_file_path=log_file_path,
quiet=quiet, check_times=self.check_times)
def prepare_system(self): def prepare_system(self):
"""Prepare Vagrant system for building Kea.""" """Prepare Vagrant system for building Kea."""
...@@ -488,7 +521,8 @@ class VagrantEnv(object): ...@@ -488,7 +521,8 @@ class VagrantEnv(object):
self.nofeatures_arg = '' self.nofeatures_arg = ''
# select proper python version for running Hammer inside Vagrant system # select proper python version for running Hammer inside Vagrant system
if self.system == 'centos' and self.sys_revision == '7' or (self.system == 'debian' and self.sys_revision == '8' and self.provider != 'lxc'): if (self.system == 'centos' and self.revision == '7' or
(self.system == 'debian' and self.revision == '8' and self.provider != 'lxc')):
self.python = 'python' self.python = 'python'
elif self.system == 'freebsd': elif self.system == 'freebsd':
self.python = 'python3.6' self.python = 'python3.6'
...@@ -496,8 +530,9 @@ class VagrantEnv(object): ...@@ -496,8 +530,9 @@ class VagrantEnv(object):
self.python = 'python3' self.python = 'python3'
# to get python in RHEL 8 beta it is required first register machine in RHEL account # to get python in RHEL 8 beta it is required first register machine in RHEL account
if self.system == 'rhel' and self.sys_revision == '8': if self.system == 'rhel' and self.revision == '8':
exitcode = self.execute("sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms", raise_error=False) cmd = "sudo subscription-manager repos --list-enabled | grep rhel-8-for-x86_64-baseos-beta-rpms"
exitcode = self.execute(cmd, raise_error=False)
if exitcode != 0: if exitcode != 0:
env = os.environ.copy() env = os.environ.copy()
with open(os.path.expanduser('~/rhel-creds.txt')) as f: with open(os.path.expanduser('~/rhel-creds.txt')) as f:
...@@ -529,20 +564,23 @@ def _install_gtest_sources(): ...@@ -529,20 +564,23 @@ def _install_gtest_sources():
"""Install gtest sources.""" """Install gtest sources."""
# download gtest sources only if it is not present as native package # download gtest sources only if it is not present as native package
if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'): if not os.path.exists('/usr/src/googletest-release-1.8.0/googletest'):
execute('wget --no-verbose -O /tmp/gtest.tar.gz https://github.com/google/googletest/archive/release-1.8.0.tar.gz') cmd = 'wget --no-verbose -O /tmp/gtest.tar.gz '
cmd += 'https://github.com/google/googletest/archive/release-1.8.0.tar.gz'
execute(cmd)
execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz') execute('sudo tar -C /usr/src -zxf /tmp/gtest.tar.gz')
os.unlink('/tmp/gtest.tar.gz') os.unlink('/tmp/gtest.tar.gz')
def _configure_mysql(system, revision, features): def _configure_mysql(system, revision, features):
"""Configures MySQL database.""" """Configure MySQL database."""
if system in ['fedora', 'centos']: if system in ['fedora', 'centos']:
execute('sudo systemctl enable mariadb.service') execute('sudo systemctl enable mariadb.service')
execute('sudo systemctl start mariadb.service') execute('sudo systemctl start mariadb.service')
time.sleep(5) time.sleep(5)
if system == 'freebsd': if system == 'freebsd':
cmd = "echo 'SET PASSWORD = \"\";' | sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password" cmd = "echo 'SET PASSWORD = \"\";' "
cmd += "| sudo mysql -u root --password=\"$(sudo cat /root/.mysql_secret | grep -v '#')\" --connect-expired-password"
execute(cmd, raise_error=False) execute(cmd, raise_error=False)
cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root" cmd = "echo 'DROP DATABASE IF EXISTS keatest;' | sudo mysql -u root"
...@@ -617,11 +655,13 @@ def _configure_pgsql(system, features): ...@@ -617,11 +655,13 @@ def _configure_pgsql(system, features):
def _install_cassandra_deb(env, check_times): def _install_cassandra_deb(env, check_times):
"""Installs Cassandra and cpp-driver using DEB package.""" """Install Cassandra and cpp-driver using DEB package."""
if not os.path.exists('/usr/sbin/cassandra'): if not os.path.exists('/usr/sbin/cassandra'):
execute('echo "deb http://www.apache.org/dist/cassandra/debian 311x main" | sudo tee /etc/apt/sources.list.d/cassandra.sources.list', cmd = 'echo "deb http://www.apache.org/dist/cassandra/debian 311x main" '
cmd += '| sudo tee /etc/apt/sources.list.d/cassandra.sources.list'
execute(cmd, env=env, check_times=check_times)
execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -',
env=env, check_times=check_times) env=env, check_times=check_times)
execute('wget -qO- https://www.apache.org/dist/cassandra/KEYS | sudo apt-key add -', env=env, check_times=check_times)
execute('sudo apt update', env=env, check_times=check_times) execute('sudo apt update', env=env, check_times=check_times)
install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times) install_pkgs('cassandra libuv1 pkgconf', env=env, check_times=check_times)
...@@ -630,8 +670,10 @@ def _install_cassandra_deb(env, check_times): ...@@ -630,8 +670,10 @@ def _install_cassandra_deb(env, check_times):
env=env, check_times=check_times) env=env, check_times=check_times)
execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb', execute('wget http://downloads.datastax.com/cpp-driver/ubuntu/18.04/cassandra/v2.11.0/cassandra-cpp-driver_2.11.0-1_amd64.deb',
env=env, check_times=check_times) env=env, check_times=check_times)
execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb', env=env, check_times=check_times) execute('sudo dpkg -i cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb', env=env, check_times=check_times) env=env, check_times=check_times)
execute('rm -rf cassandra-cpp-driver-dev_2.11.0-1_amd64.deb cassandra-cpp-driver_2.11.0-1_amd64.deb',
env=env, check_times=check_times)
def _install_freeradius_client(env, check_times): def _install_freeradius_client(env, check_times):
...@@ -646,8 +688,8 @@ def _install_freeradius_client(env, check_times): ...@@ -646,8 +688,8 @@ def _install_freeradius_client(env, check_times):
execute('rm -rf freeradius-client') execute('rm -rf freeradius-client')
def _install_cassandra_rpm(system, env, check_times): def _install_cassandra_rpm(env, check_times):
"""Installs Cassandra and cpp-driver using RPM package.""" """Install Cassandra and cpp-driver using RPM package."""
if not os.path.exists('/usr/bin/cassandra'): if not os.path.exists('/usr/bin/cassandra'):
#execute('sudo dnf config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/') #execute('sudo dnf config-manager --add-repo https://www.apache.org/dist/cassandra/redhat/311x/')
#execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS') #execute('sudo rpm --import https://www.apache.org/dist/cassandra/KEYS')
...@@ -672,7 +714,8 @@ def prepare_system_local(features, check_times): ...@@ -672,7 +714,8 @@ def prepare_system_local(features, check_times):
# prepare fedora # prepare fedora
if system == 'fedora': if system == 'fedora':
packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel'] packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
'log4cplus-devel', 'boost-devel']
if 'native-pkg' in features: if 'native-pkg' in features:
packages.extend(['rpm-build', 'mariadb-connector-c-devel']) packages.extend(['rpm-build', 'mariadb-connector-c-devel'])
...@@ -697,14 +740,14 @@ def prepare_system_local(features, check_times): ...@@ -697,14 +740,14 @@ def prepare_system_local(features, check_times):
execute('sudo dnf clean packages', env=env, check_times=check_times) execute('sudo dnf clean packages', env=env, check_times=check_times)
if 'cql' in features: if 'cql' in features:
_install_cassandra_rpm(system, env, check_times) _install_cassandra_rpm(env, check_times)
# prepare centos # prepare centos
elif system == 'centos': elif system == 'centos':
install_pkgs('epel-release', env=env, check_times=check_times) install_pkgs('epel-release', env=env, check_times=check_times)
packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel', 'log4cplus-devel', 'boost-devel', packages = ['make', 'autoconf', 'automake', 'libtool', 'gcc-c++', 'openssl-devel',
'mariadb-devel', 'postgresql-devel'] 'log4cplus-devel', 'boost-devel', 'mariadb-devel', 'postgresql-devel']
if 'docs' in features: if 'docs' in features:
packages.extend(['libxslt', 'elinks', 'docbook-style-xsl']) packages.extend(['libxslt', 'elinks', 'docbook-style-xsl'])
...@@ -724,7 +767,7 @@ def prepare_system_local(features, check_times): ...@@ -724,7 +767,7 @@ def prepare_system_local(features, check_times):
_install_gtest_sources() _install_gtest_sources()
if 'cql' in features: if 'cql' in features:
_install_cassandra_rpm(system, env, check_times) _install_cassandra_rpm(env, check_times)
# prepare rhel # prepare rhel
elif system == 'rhel': elif system == 'rhel':
...@@ -756,21 +799,25 @@ def prepare_system_local(features, check_times): ...@@ -756,21 +799,25 @@ def prepare_system_local(features, check_times):
execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm ' execute('wget --no-verbose -O srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm '
'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm', 'https://rpmfind.net/linux/epel/7/SRPMS/Packages/l/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
check_times=check_times) check_times=check_times)
execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm', env=env, timeout=120, check_times=check_times) execute('rpmbuild --rebuild srpms/log4cplus-1.1.3-0.4.rc3.el7.src.rpm',
execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times) env=env, timeout=120, check_times=check_times)
execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm', env=env, check_times=check_times) execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-1.1.3-0.4.rc3.el8.x86_64.rpm',
env=env, check_times=check_times)
execute('sudo rpm -i rpmbuild/RPMS/x86_64/log4cplus-devel-1.1.3-0.4.rc3.el8.x86_64.rpm',
env=env, check_times=check_times)
if 'unittest' in features: if 'unittest' in features:
_install_gtest_sources() _install_gtest_sources()
if 'cql' in features: if 'cql' in features:
_install_cassandra_rpm(system, env, check_times) _install_cassandra_rpm(env, check_times)
# prepare ubuntu # prepare ubuntu
elif system == 'ubuntu': elif system == 'ubuntu':
execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10) execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev'] packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev',
'libboost-system-dev']
if 'unittest' in features: if 'unittest' in features:
if revision.startswith('16.'): if revision.startswith('16.'):
...@@ -783,8 +830,9 @@ def prepare_system_local(features, check_times): ...@@ -783,8 +830,9 @@ def prepare_system_local(features, check_times):
if 'native-pkg' in features: if 'native-pkg' in features:
packages.extend(['build-essential', 'fakeroot', 'devscripts']) packages.extend(['build-essential', 'fakeroot', 'devscripts'])
packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev', 'docbook', 'docbook-xsl', 'flex', 'libboost-dev', packages.extend(['bison', 'debhelper', 'default-libmysqlclient-dev', 'libmysqlclient-dev',
'libpq-dev', 'postgresql-server-dev-all', 'python3-dev']) 'docbook', 'docbook-xsl', 'flex', 'libboost-dev', 'libpq-dev',
'postgresql-server-dev-all', 'python3-dev'])
if 'mysql' in features: if 'mysql' in features:
if revision == '16.04': if revision == '16.04':
...@@ -807,7 +855,8 @@ def prepare_system_local(features, check_times): ...@@ -807,7 +855,8 @@ def prepare_system_local(features, check_times):
elif system == 'debian': elif system == 'debian':
execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10) execute('sudo apt update', env=env, check_times=check_times, attempts=3, sleep_time_after_attempt=10)
packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev', 'liblog4cplus-dev', 'libboost-system-dev'] packages = ['gcc', 'g++', 'make', 'autoconf', 'automake', 'libtool', 'libssl-dev',
'liblog4cplus-dev', 'libboost-system-dev']
if 'docs' in features: if 'docs' in features:
packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl']) packages.extend(['dblatex', 'xsltproc', 'elinks', 'docbook-xsl'])
...@@ -838,11 +887,6 @@ def prepare_system_local(features, check_times): ...@@ -838,11 +887,6 @@ def prepare_system_local(features, check_times):
elif system == 'freebsd': elif system == 'freebsd':
packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs'] packages = ['autoconf', 'automake', 'libtool', 'openssl', 'log4cplus', 'boost-libs']
# TODO:
#execute('sudo portsnap --interactive fetch', timeout=240, check_times=check_times)
#execute('sudo portsnap extract /usr/ports/devel/log4cplus', timeout=240, check_times=check_times)
#execute('sudo make -C /usr/ports/devel/log4cplus install clean BATCH=yes', timeout=240, check_times=check_times)
if 'docs' in features: if 'docs' in features:
packages.extend(['libxslt', 'elinks', 'docbook-xsl']) packages.extend(['libxslt', 'elinks', 'docbook-xsl'])
...@@ -859,7 +903,8 @@ def prepare_system_local(features, check_times): ...@@ -859,7 +903,8 @@ def prepare_system_local(features, check_times):
if 'mysql' in features: if 'mysql' in features:
execute('sudo sysrc mysql_enable="yes"', env=env, check_times=check_times) execute('sudo sysrc mysql_enable="yes"', env=env, check_times=check_times)
execute('sudo service mysql-server start', env=env, check_times=check_times, raise_error=False) execute('sudo service mysql-server start', env=env, check_times=check_times,
raise_error=False)
else: else:
raise NotImplementedError raise NotImplementedError
...@@ -878,9 +923,10 @@ def prepare_system_local(features, check_times): ...@@ -878,9 +923,10 @@ def prepare_system_local(features, check_times):
log.info('Preparing deps completed successfully.') log.info('Preparing deps completed successfully.')
def prepare_system_in_vagrant(provider, system, sys_revision, features, dry_run, check_times, clean_start): def prepare_system_in_vagrant(provider, system, revision, features, dry_run, check_times,
clean_start):
"""Prepare specified system in Vagrant according to specified features.""" """Prepare specified system in Vagrant according to specified features."""
ve = VagrantEnv(provider, system, sys_revision, features, 'kea', dry_run, check_times=check_times) ve = VagrantEnv(provider, system, revision, features, 'kea', dry_run, check_times=check_times)
if clean_start: if clean_start:
ve.destroy() ve.destroy()
ve.up() ve.up()
...@@ -888,7 +934,7 @@ def prepare_system_in_vagrant(provider, system, sys_revision, features, dry_run, ...@@ -888,7 +934,7 @@ def prepare_system_in_vagrant(provider, system, sys_revision, features, dry_run,
def _calculate_build_timeout(features): def _calculate_build_timeout(features):
"""Returns maximum allowed time for build (in seconds)""" """Return maximum allowed time for build (in seconds)."""
timeout = 60 timeout = 60
if 'mysql' in features: if 'mysql' in features:
timeout += 60 timeout += 60
...@@ -896,7 +942,7 @@ def _calculate_build_timeout(features): ...@@ -896,7 +942,7 @@ def _calculate_build_timeout(features):
return timeout return timeout
def _build_just_binaries(distro, revision, features, tarball_path, env, check_times, jobs, dry_run): def _build_binaries_and_run_ut(system, revision, features, tarball_path, env, check_times, jobs, dry_run):
if tarball_path: if tarball_path:
# unpack tarball with sources # unpack tarball with sources
execute('sudo rm -rf kea-src') execute('sudo rm -rf kea-src')
...@@ -919,20 +965,20 @@ def _build_just_binaries(distro, revision, features, tarball_path, env, check_ti ...@@ -919,20 +965,20 @@ def _build_just_binaries(distro, revision, features, tarball_path, env, check_ti
cmd += ' --with-cql=/usr/bin/pkg-config' cmd += ' --with-cql=/usr/bin/pkg-config'
if 'unittest' in features: if 'unittest' in features:
# prepare gtest switch - use downloaded gtest sources only if it is not present as native package # prepare gtest switch - use downloaded gtest sources only if it is not present as native package
if distro in ['centos', 'fedora', 'rhel', 'freebsd']: if system in ['centos', 'fedora', 'rhel', 'freebsd']:
cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/' cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
elif distro == 'debian' and revision == '8': elif system == 'debian' and revision == '8':
cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/' cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
elif distro == 'debian': elif system == 'debian':
cmd += ' --with-gtest-source=/usr/src/googletest/googletest' cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
elif distro == 'ubuntu': elif system == 'ubuntu':
if revision.startswith('16.'): if revision.startswith('16.'):
cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/' cmd += ' --with-gtest-source=/usr/src/googletest-release-1.8.0/googletest/'
else: else:
cmd += ' --with-gtest-source=/usr/src/googletest/googletest' cmd += ' --with-gtest-source=/usr/src/googletest/googletest'
else: else:
raise NotImplementedError raise NotImplementedError
if 'docs' in features and not (distro == 'rhel' and revision == '8'): if 'docs' in features and not (system == 'rhel' and revision == '8'):
cmd += ' --enable-generate-docs' cmd += ' --enable-generate-docs'
if 'radius' in features: if 'radius' in features:
cmd += ' --with-freeradius=/usr/local' cmd += ' --with-freeradius=/usr/local'
...@@ -945,7 +991,7 @@ def _build_just_binaries(distro, revision, features, tarball_path, env, check_ti ...@@ -945,7 +991,7 @@ def _build_just_binaries(distro, revision, features, tarball_path, env, check_ti
# estimate number of processes (jobs) to use in compilation if jobs are not provided # estimate number of processes (jobs) to use in compilation if jobs are not provided
if jobs == 0: if jobs == 0:
cpus = multiprocessing.cpu_count() - 1 cpus = multiprocessing.cpu_count() - 1
if distro == 'centos': if system == 'centos':
cpus = cpus // 2 cpus = cpus // 2
if cpus == 0: if cpus == 0:
cpus = 1 cpus = 1
...@@ -969,7 +1015,9 @@ def _build_just_binaries(distro, revision, features, tarball_path, env, check_ti ...@@ -969,7 +1015,9 @@ def _build_just_binaries(distro, revision, features, tarball_path, env, check_ti
env['GTEST_OUTPUT'] = 'xml:%s/' % results_dir env['GTEST_OUTPUT'] = 'xml:%s/' % results_dir
env['KEA_SOCKET_TEST_DIR'] = '/tmp/' env['KEA_SOCKET_TEST_DIR'] = '/tmp/'
# run unit tests # run unit tests
execute('make check -k', cwd=src_path, env=env, timeout=60 * 60, raise_error=False, check_times=check_times, dry_run=dry_run) execute('make check -k',
cwd=src_path, env=env, timeout=60 * 60, raise_error=False,
check_times=check_times, dry_run=dry_run)
# parse unit tests results # parse unit tests results
results = {} results = {}
...@@ -1014,9 +1062,9 @@ def _build_just_binaries(distro, revision, features, tarball_path, env, check_ti ...@@ -1014,9 +1062,9 @@ def _build_just_binaries(distro, revision, features, tarball_path, env, check_ti
execute('kea-admin lease-init pgsql -u keauser -p keapass -n keadb', dry_run=