From aafe8fbc431d243b6c9d4f8225ad092c4719b71e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20K=C4=99pie=C5=84?= Date: Thu, 19 Mar 2020 07:21:46 +0100 Subject: [PATCH] Add QEMU-based libvirt executor --- libvirt-qemu/README.md | 144 ++++++++++++++++++++++++ libvirt-qemu/domain-template.xml | 27 +++++ libvirt-qemu/executor.conf.sample | 6 + libvirt-qemu/executor.sh | 175 ++++++++++++++++++++++++++++++ libvirt-qemu/pull-images.sh | 61 +++++++++++ 5 files changed, 413 insertions(+) create mode 100644 libvirt-qemu/README.md create mode 100644 libvirt-qemu/domain-template.xml create mode 100644 libvirt-qemu/executor.conf.sample create mode 100755 libvirt-qemu/executor.sh create mode 100755 libvirt-qemu/pull-images.sh diff --git a/libvirt-qemu/README.md b/libvirt-qemu/README.md new file mode 100644 index 0000000..e42c26e --- /dev/null +++ b/libvirt-qemu/README.md @@ -0,0 +1,144 @@ +# GitLab Runner Custom Executor for QEMU-based libvirt Domains + +## Overview + +This shell script allows GitLab CI jobs to be run inside QEMU-based +libvirt domains (over SSH). + +## Required Tools + +- `arp` +- `qemu-img` +- `ssh-keyscan` +- `sshpass` +- `virsh` + +## Installation + +- Clone this repository to the libvirt host which will be running GitLab + CI jobs. + +- Copy the sample configuration file (`executor.conf.sample`) to + `/etc/gitlab-runner/executor-libvirt-qemu.conf`. + +- Adjust configuration file contents to match your preferences. In + particular, set `BASE_DIR` to the path to the executor's working + directory where all temporary files will be stored. + +- Adjust the libvirt domain template (a sample is provided as + `domain-template.xml`) to match your preferences. + +- Put the operating system QCOW2 images to be used by the executor in + the `${BASE_DIR}/qcow2` directory. + +- Register a new GitLab CI runner using `gitlab-runner register`, + specifying `custom` as the executor to use. + +- Adjust the relevant section of `/etc/gitlab-runner/config.toml` so + that the executor gets invoked properly: + + ```toml + [runners.custom] + config_exec = "/path/to/libvirt-qemu/executor.sh" + config_args = ["config"] + prepare_exec = "/path/to/libvirt-qemu/executor.sh" + prepare_args = ["prepare"] + run_exec = "/path/to/libvirt-qemu/executor.sh" + run_args = ["run"] + cleanup_exec = "/path/to/libvirt-qemu/executor.sh" + cleanup_args = ["cleanup"] + ``` + +- In `/etc/gitlab-runner/config.toml`, set the `limit` [runner + variable][1] to the maximum number of GitLab CI jobs to run (i.e. + virtual machines to create) simultaneously at any given time, based on + the amount of memory available on your host and the load it is capable + of handling. + +## Design + +- A separate libvirt domain is created for each GitLab CI job. A + common XML template is used as a base for all domain definitions. + +- Every supported operating system has a corresponding QCOW2 image. + These images are *not* prepared by the executor script. For some + ideas on how to do this, see the `packer` directory in the [ISC + images][2] Git repository. + +- For each GitLab CI job, a separate QCOW2 image is created with the + given operating system's QCOW2 image specified as the backing file + (`qemu-img create -b ...`). This speeds up domain creation and saves + disk space as it is *not* necessary to copy the whole QCOW2 image + before starting each virtual machine. + +- Every created domain is assigned a single virtual network interface. + The `default` libvirt network needs to be started before using the + executor script. + +- The IP address assigned to the given domain is determined by looking + up its MAC address in the output of the `arp` tool. + +- Job scripts are executed by connecting to virtual machines using + password-based SSH. Access credentials are configured by setting the + `SSH_USER` and `SSH_PASSWORD` configuration variables appropriately. + The username can be overridden on a per-job basis by setting the + `USER` variable in a given job's YAML definition. + +- While not strictly necessary, each domain is expected to use the first + serial port (COM1) as its system console. The sample domain template + redirects COM1 to a file which can be inspected on the host in case of + boot issues. + +- [GitLab Runner Docker executor][3] uses the `image` keyword for + determining which Docker image to use. The contents of that keyword + are not exposed as an environment variable by the [GitLab Runner + Custom executor][4] and thus it cannot be used for determining the + operating system image to use. Instead, the latter is extracted from + the job name. The QCOW2 image file path is derived from the detected + operating system name and release (see the `do_prepare()` function in + `executor.sh` for hints on expected file naming). + +## QCOW2 Image Synchronization + +Whenever [GitLab Runner Docker executor][3] runs a GitLab CI job, it +first checks whether a newer version of the Docker image to be used is +available; if so, the latest version of the image is automatically +downloaded, used, and cached locally. No similar mechanisms are built +into [GitLab Runner Custom executor][4]. + +Since GitLab comes with a built-in [Container Registry][5] and +maintaining a separate infrastructure just for hosting QCOW2 images +would be cumbersome, for the time being, ISC uses a simple trick to +solve the image distribution issue: the QCOW2 images built are wrapped +into Docker containers and distributed through the GitLab Container +Registry. + +However, this still does not cause QCOW2 images to be updated +automatically whenever a GitLab CI job is run. As implementing such a +mechanism in a safe manner is a non-trivial task and we generally update +our QCOW2 images only when a new version of a given operating system is +released (i.e. rarely), our GitLab CI runners execute a rather crude +helper script (`pull-images.sh`) as an hourly cron job. This script +pulls the latest Docker images available from the specified Docker +repository, extracts the QCOW2 images embedded in them and replaces the +locally stored QCOW2 images with their up-to-date versions. All QCOW2 +images are expected to be published in the same Docker repository, +specified using the `DOCKER_REPOSITORY` variable in the executor's +configuration file. + +The biggest caveat with this approach is that removing a QCOW2 backing +file while it is used by a libvirt domain is obviously more than likely +to cause a given GitLab CI job to fail. Yet, given the low frequency of +QCOW2 image updates, this approach works for us for the time being. +Running the script on an hourly basis on all runners ensures they pull +updated QCOW2 images automatically and that it happens relatively +quickly after these updated QCOW2 images are published. The helper +script can also be run manually before the first use of the executor in +order to download all QCOW2 images available in the specified Docker +repository to `${BASE_DIR}/qcow2`. + +[1]: https://docs.gitlab.com/runner/configuration/advanced-configuration.html#the-runners-section +[2]: https://gitlab.isc.org/isc-projects/images/-/tree/master/packer +[3]: https://docs.gitlab.com/runner/executors/docker.html +[4]: https://docs.gitlab.com/runner/executors/custom.html +[5]: https://docs.gitlab.com/ce/user/packages/container_registry/index.html diff --git a/libvirt-qemu/domain-template.xml b/libvirt-qemu/domain-template.xml new file mode 100644 index 0000000..e998eb3 --- /dev/null +++ b/libvirt-qemu/domain-template.xml @@ -0,0 +1,27 @@ + + @RUNNER_NAME@ + + hvm + + 4 + 4096 + + + + + + + + + + + + + + + + + + + + diff --git a/libvirt-qemu/executor.conf.sample b/libvirt-qemu/executor.conf.sample new file mode 100644 index 0000000..c5d6306 --- /dev/null +++ b/libvirt-qemu/executor.conf.sample @@ -0,0 +1,6 @@ +DOCKER_REPOSITORY="registry.gitlab.isc.org/isc-projects/images/qcow2" +BASE_DIR="/home/gitlab-runner" +DOMAIN_DEFINITION_TEMPLATE_PATH="${BASE_DIR}/gitlab-runner-scripts/libvirt-qemu/domain-template.xml" +BOOT_TIMEOUT=180 +SSH_USER="root" +SSH_PASSWORD="vagrant" diff --git a/libvirt-qemu/executor.sh b/libvirt-qemu/executor.sh new file mode 100755 index 0000000..aa0a60e --- /dev/null +++ b/libvirt-qemu/executor.sh @@ -0,0 +1,175 @@ +#!/bin/sh +# +# executor.sh - GitLab Runner Custom Executor for QEMU-based libvirt Domains +# +# Written in 2020 by Michał Kępień +# +# To the extent possible under law, the author(s) have dedicated all copyright +# and related and neighboring rights to this software to the public domain +# worldwide. This software is distributed without any warranty. +# +# You should have received a copy of the CC0 Public Domain Dedication along with +# this software. If not, see http://creativecommons.org/publicdomain/zero/1.0/. + +set -e + +CONFIG_PATH="/etc/gitlab-runner/executor-libvirt-qemu.conf" + +# shellcheck source=executor.conf.sample +. "${CONFIG_PATH}" + +CENTOS_RELEASE="$(expr "${CUSTOM_ENV_CI_JOB_NAME}" : ".*:el\([6-8]\)\(:\|$\)" || :)" +OPENBSD_RELEASE="$(expr "${CUSTOM_ENV_CI_JOB_NAME}" : ".*:openbsd\([0-9][0-9.]*\)\(:\|$\)" || :)" +FREEBSD_RELEASE="$(expr "${CUSTOM_ENV_CI_JOB_NAME}" : ".*:freebsd\([0-9][0-9.]*\)\(:\|$\)" || :)" + +BASE_IMAGE_DIR="${BASE_DIR}/qcow2" +RUNNER_NAME="runner-${CUSTOM_ENV_CI_JOB_ID}" +DOMAIN_DEFINITION_PATH="${BASE_DIR}/${RUNNER_NAME}.xml" +DOMAIN_LOG_PATH="${BASE_DIR}/${RUNNER_NAME}.log" +IMAGE_PATH="${BASE_DIR}/${RUNNER_NAME}.qcow2" +SSH_KNOWN_HOSTS_PATH="${BASE_DIR}/${RUNNER_NAME}.pub" + +fatal_error() { + echo "${1}" >&2 + exit 1 +} + +get_runner_ip() { + MAC_ADDRESS="$(virsh dumpxml "${RUNNER_NAME}" | grep -oE "[0-9a-f:]{17}")" + arp -n | awk '$3 == "'"${MAC_ADDRESS}"'" { print $1 }' +} + +do_config() { + cat <<-EOF + { + "builds_dir": "/builds", + "cache_dir": "/tmp", + "builds_dir_is_shared": false + } + EOF +} + +do_prepare() { + if [ -n "${CENTOS_RELEASE}" ]; then + BASE_IMAGE_PATH="$(readlink -f "${BASE_IMAGE_DIR}/centos-${CENTOS_RELEASE}-x86_64")" + elif [ -n "${FREEBSD_RELEASE}" ]; then + BASE_IMAGE_PATH="$(readlink -f "${BASE_IMAGE_DIR}/freebsd-${FREEBSD_RELEASE}-x86_64")" + elif [ -n "${OPENBSD_RELEASE}" ]; then + BASE_IMAGE_PATH="$(readlink -f "${BASE_IMAGE_DIR}/openbsd-${OPENBSD_RELEASE}-x86_64")" + else + fatal_error "Unable to extract OS to use from job name (${CUSTOM_ENV_CI_JOB_NAME})" + fi + + if [ ! -f "${BASE_IMAGE_PATH}" ]; then + fatal_error "Base image ${BASE_IMAGE_PATH} does not exist" + fi + + cp "${DOMAIN_DEFINITION_TEMPLATE_PATH}" "${DOMAIN_DEFINITION_PATH}" + for TOKEN_NAME in DOMAIN_LOG_PATH IMAGE_PATH RUNNER_NAME; do + eval "TOKEN_VALUE=\"\$${TOKEN_NAME}\"" + sed -i "s|@${TOKEN_NAME}@|${TOKEN_VALUE}|g" "${DOMAIN_DEFINITION_PATH}" + done + + qemu-img create -f qcow2 -F qcow2 -b "${BASE_IMAGE_PATH}" "${IMAGE_PATH}" + + virsh define "${DOMAIN_DEFINITION_PATH}" + virsh start "${RUNNER_NAME}" + + BOOT_OK=0 + BOOT_START="$(date +%s)" + while [ "$(($(date +%s) - BOOT_START))" -lt "${BOOT_TIMEOUT}" ]; do + IPV4_ADDRESS="$(get_runner_ip)" + if [ -z "${IPV4_ADDRESS}" ]; then + sleep 1 + continue + fi + SSH_KEYS="$(ssh-keyscan -T 1 "${IPV4_ADDRESS}" 2>/dev/null)" + if [ -z "${SSH_KEYS}" ]; then + sleep 1 + continue + fi + BOOT_OK=1 + break + done + if [ "${BOOT_OK}" -eq 0 ]; then + cp "${DOMAIN_LOG_PATH}" "${DOMAIN_LOG_PATH}.boot-timeout" + fatal_error "VM boot timed out" + fi +} + +do_run() { + SCRIPT_PATH="${1}" + RUN_STAGE="${2}" + + case "${RUN_STAGE}" in + "prepare_script") ;; + "get_sources") ;; + "restore_cache") ;; + "download_artifacts") ;; + "build_script") ;; + "after_script") ;; + "archive_cache") ;; + "upload_artifacts_on_success"|"upload_artifacts_on_failure") ;; + *) fatal_error "Unsupported run stage '${RUN_STAGE}'" ;; + esac + + IPV4_ADDRESS="$(get_runner_ip)" + + if [ -n "${CUSTOM_ENV_USER}" ]; then + SSH_USER="${CUSTOM_ENV_USER}" + fi + + sshpass -p "${SSH_PASSWORD}" ssh \ + -o "StrictHostKeyChecking=no" \ + -o "UserKnownHostsFile=${SSH_KNOWN_HOSTS_PATH}" \ + "${SSH_USER}@${IPV4_ADDRESS}" \ + "bash" < "${SCRIPT_PATH}" +} + +do_cleanup() { + virsh destroy "${RUNNER_NAME}" + virsh undefine "${RUNNER_NAME}" + + rm -f "${DOMAIN_DEFINITION_PATH}" + rm -f "${DOMAIN_LOG_PATH}" + rm -f "${IMAGE_PATH}" + rm -f "${SSH_KNOWN_HOSTS_PATH}" +} + +ensure_variable_set() { + eval VALUE="\${${1}+set}" + if [ -z "${VALUE}" ]; then + fatal_error "${1} not set, please check contents of ${CONFIG_PATH}" + fi +} + +ensure_program_available() { + if ! command -v "${1}" > /dev/null 2>&1; then + fatal_error "'${1}' not found in PATH" + fi +} + +ensure_variable_set BASE_DIR +ensure_variable_set BOOT_TIMEOUT +ensure_variable_set DOMAIN_DEFINITION_TEMPLATE_PATH +ensure_variable_set SSH_PASSWORD +ensure_variable_set SSH_USER + +ensure_program_available arp +ensure_program_available qemu-img +ensure_program_available ssh-keyscan +ensure_program_available sshpass +ensure_program_available virsh + +MODE="${1}" + +case "${MODE}" in + "config") ;; + "prepare") ;; + "run") ;; + "cleanup") ;; + *) fatal_error "Usage: $0 config|prepare|run|cleanup [args...]" ;; +esac + +shift +eval "do_${MODE}" "$@" diff --git a/libvirt-qemu/pull-images.sh b/libvirt-qemu/pull-images.sh new file mode 100755 index 0000000..87f5cb5 --- /dev/null +++ b/libvirt-qemu/pull-images.sh @@ -0,0 +1,61 @@ +#!/bin/sh +# +# pull-images.sh - Pull QCOW2 Images Embedded in Docker Images From a Registry +# +# Written in 2020 by Michał Kępień +# +# To the extent possible under law, the author(s) have dedicated all copyright +# and related and neighboring rights to this software to the public domain +# worldwide. This software is distributed without any warranty. +# +# You should have received a copy of the CC0 Public Domain Dedication along with +# this software. If not, see http://creativecommons.org/publicdomain/zero/1.0/. + +set -e + +CONFIG_PATH="/etc/gitlab-runner/executor-libvirt-qemu.conf" + +# shellcheck source=executor.conf.sample +. "${CONFIG_PATH}" + +fatal_error() { + echo "${1}" >&2 + exit 1 +} + +ensure_variable_set() { + eval VALUE="\${${1}+set}" + if [ -z "${VALUE}" ]; then + fatal_error "${1} not set, please check contents of ${CONFIG_PATH}" + fi +} + +ensure_variable_set BASE_DIR +ensure_variable_set DOCKER_REPOSITORY + +if ! command -v "docker" > /dev/null 2>&1; then + fatal_error "docker not available, aborting" +fi + +BASE_IMAGE_DIR="${BASE_DIR}/qcow2" + +get_local_qcow2_image_list() { + docker image ls "${DOCKER_REPOSITORY}" | tail -n +2 | awk '$2 != "" { print $2 "=" $3 }' | sort +} + +mkdir -p "${BASE_IMAGE_DIR}" +ORIGINAL_IMAGES="$(mktemp)" +LATEST_IMAGES="$(mktemp)" +get_local_qcow2_image_list > "${ORIGINAL_IMAGES}" +docker pull -a "${DOCKER_REPOSITORY}" +get_local_qcow2_image_list > "${LATEST_IMAGES}" +diff -u "${ORIGINAL_IMAGES}" "${LATEST_IMAGES}" | sed -nE "s|^\+([^+=][^=]*)=([0-9a-f]+$)|\1 \2|p" | while read -r QCOW2_IMAGE VERSION; do + CONTAINER_ID="$(docker create "${DOCKER_REPOSITORY}:${QCOW2_IMAGE}" /bin/sh)" + docker cp "${CONTAINER_ID}:${QCOW2_IMAGE}.qcow2" "${BASE_IMAGE_DIR}/${QCOW2_IMAGE}-${VERSION}.qcow2" + docker rm "${CONTAINER_ID}" + ln -sf "${QCOW2_IMAGE}-${VERSION}.qcow2" "${BASE_IMAGE_DIR}/${QCOW2_IMAGE}" +done +rm -f "${ORIGINAL_IMAGES}" "${LATEST_IMAGES}" +docker image ls | awk '$1 == "'"${DOCKER_REPOSITORY}"'" && $2 == "" { print $3 }' | while read -r IMAGE_ID; do + find "${BASE_IMAGE_DIR}" -type f -name "*-${IMAGE_ID}.qcow2" -delete +done -- GitLab