Commit 8a94d972 authored by Ondřej Surý's avatar Ondřej Surý
Browse files

Rework the way we build and use Debian/Ubuntu images

Following changes have been made to the Makefile and Dockerfile for
Debian/Ubuntu images:

* With introduction of arm64 builder, we needed more fine grained
  switches that define whether to install specific software bundle into
  the image.  The switches are now defined as a tripplets:
  <software>_<version>_<architecture>, e.g. dnstap_buster_amd64

* Change the install/no-install logic for abi-compliance-checker,
  shellcheck, and compiledb from Dockerfile to the Makefile

* It was hard to test whether the images build correctly and not mess
  with production environment.  Now, the images are pushed only if the
  pipeline is run on the default repository branch (e.g. master).

* Install only a single LLVM/Clang version - 10 at the time of this
  commit.  This has to match with bind9's .gitlab-ci.yml.

* Don't install the LLVM/Clang to Debian sid, this is more often broken
  than it's not.

* Pick a "base_image" in the bind9's .gitlab-ci.yml that will have the
  extra software.  At the time of writing this, The Debian amd64 buster
  image was picked.  This means needs to get the latest version of the
  clang, coccinelle, abicc, cppcheck and others.

* If `@FOO@` in the `if [ -n "@FOO@" ]` construct didn't get replaced it
  would default to true.  Change this to `if [ "@FOO@" = "yes" ]` that
  defaults to false.

* Install the abi-compliance-checker and it's dependencies from the git
  repository instead from the Debian buster repository.
parent e071c7b5
......@@ -24,20 +24,23 @@ ubuntu_VERSIONS ?= $(default_ubuntu_VERSIONS)
endif
# Define where softhsm and what version gets installed
softhsm_buster := 2
softhsm_buster_amd64 := 2
# Define extras
dnstap_centos7 := yes
dnstap_buster := yes
dnstap_sid := yes
uncrustify_sid := yes
abi_check_buster_amd64 := yes
coccinelle_buster_amd64 := yes
compiledb_buster_amd64 := yes
cppcheck_buster_amd64 := yes
dnstap_buster_amd64 := yes
dnstap_centos7_amd64 := yes
dnstap_sid_amd64 := yes
dnstap_sid_i386 := yes
respdiff_buster_amd64 := yes
shellcheck_sid_amd64 := yes
# Install development version of LLVM/Clang
clang_stretch := yes
clang_buster := yes
clang_sid := yes
coccinelle_buster := yes
# Install latest stable version of LLVM/Clang
clang_stretch_amd64 := 10
clang_buster_amd64 := 10
TARGETS := $(foreach arch,$(ARCHS),$(foreach distro,$(DISTROS),$(foreach version,$($(distro)_VERSIONS),$(distro)-$(version)-$(arch))))
......@@ -58,33 +61,49 @@ website-rsync_ssh_curl:
docker build --no-cache -t "$(CI_REGISTRY_IMAGE)/$(CI_REGISTRY_NAME)" "docker/$(SOURCE_DIR)"
docker push "$(CI_REGISTRY_IMAGE)/$(CI_REGISTRY_NAME)"
# Only push into the main image registry when built in the master branch
CI_REGISTRY_PREFIX ?= bind9-staging
ifneq (,$(CI_COMMIT_REF_NAME))
ifeq ($(CI_COMMIT_REF_NAME),$(CI_DEFAULT_BRANCH))
CI_REGISTRY_PREFIX := bind9
endif
endif
$(TARGETS): ARCH = $(word 3,$(subst -, ,$@))
$(TARGETS): VENDOR = $(word 1,$(subst -, ,$@))
$(TARGETS): VERSION = $(word 2,$(subst -, ,$@))
$(TARGETS): ARCH = $(word 3,$(subst -, ,$@))
$(TARGETS): SOURCE_IMAGE = $(subst arm64,arm64v8/,$(subst amd64,,$(subst i386,i386/,$(ARCH))))$(VENDOR):$(VERSION)
$(TARGETS): CI_REGISTRY_NAME = bind9:$@
$(TARGETS): SOFTHSM = $(softhsm_$(VERSION))
$(TARGETS): DNSTAP = $(dnstap_$(VERSION))
$(TARGETS): UNCRUSTIFY = $(uncrustify_$(VERSION))
$(TARGETS): CLANG = $(clang_$(VERSION))
$(TARGETS): COCCINELLE = $(coccinelle_$(VERSION))
$(TARGETS): CI_REGISTRY_NAME = $(CI_REGISTRY_PREFIX):$@
$(TARGETS): ABI_CHECK = $(abi_check_$(VERSION)_$(ARCH))
$(TARGETS): CLANG = $(clang_$(VERSION)_$(ARCH))
$(TARGETS): COCCINELLE = $(coccinelle_$(VERSION)_$(ARCH))
$(TARGETS): COMPILEDB = $(compiledb_$(VERSION)_$(ARCH))
$(TARGETS): CPPCHECK = $(cppcheck_$(VERSION)_$(ARCH))
$(TARGETS): DNSTAP = $(dnstap_$(VERSION)_$(ARCH))
$(TARGETS): RESPDIFF = $(respdiff_$(VERSION)_$(ARCH))
$(TARGETS): SHELLCHECK = $(shellcheck_$(VERSION)_$(ARCH))
$(TARGETS): SOFTHSM = $(softhsm_$(VERSION)_$(ARCH))
$(TARGETS):
$(info Building $(CI_REGISTRY_IMAGE)/$(CI_REGISTRY_NAME) from $(SOURCE_IMAGE))
mkdir -p docker/bind9/$@
sed \
-e 's,@VENDOR@,$(VENDOR),g' \
-e 's,@VERSION@,$(VERSION),g' \
-e 's,@ABI_CHECK@,$(ABI_CHECK),g' \
-e 's,@ARCH@,$(ARCH),g' \
-e 's,@SOURCE_IMAGE@,$(SOURCE_IMAGE),g' \
-e 's,@SOFTHSM@,$(SOFTHSM),g' \
-e 's,@DNSTAP@,$(DNSTAP),g' \
-e 's,@UNCRUSTIFY@,$(UNCRUSTIFY),g' \
-e 's,@CLANG@,$(CLANG),g' \
-e 's,@COCCINELLE@,$(COCCINELLE),g' \
-e 's,@COMPILEDB@,$(COMPILEDB),g' \
-e 's,@CPPCHECK@,$(CPPCHECK),g' \
-e 's,@DNSTAP@,$(DNSTAP),g' \
-e 's,@KYUA_GIT_COMMIT@,$(KYUA_GIT_COMMIT),g' \
-e 's,@RESPDIFF@,$(RESPDIFF),g' \
-e 's,@SHELLCHECK@,$(SHELLCHECK),g' \
-e 's,@SOFTHSM@,$(SOFTHSM),g' \
-e 's,@SOURCE_IMAGE@,$(SOURCE_IMAGE),g' \
-e 's,@VENDOR@,$(VENDOR),g' \
-e 's,@VERSION@,$(VERSION),g' \
-e 's,@[^@]*@,,g' \
< docker/bind9/$(VENDOR)-template/Dockerfile \
> docker/bind9/$@/Dockerfile
docker build --no-cache -t "$(CI_REGISTRY_IMAGE)/$(CI_REGISTRY_NAME)" "docker/bind9/$@"
docker push "$(CI_REGISTRY_IMAGE)/$(CI_REGISTRY_NAME)"
if [ "$CI_COMMIT_REF_NAME" = "$CI_DEFAULT_BRANCH" ]; then docker push "$(CI_REGISTRY_IMAGE)/$(CI_REGISTRY_NAME)"; fi
rm -rf docker/bind9/$@
......@@ -6,25 +6,24 @@ ENV LUTOK_VERSION 0.4
ENV KYUA_GIT_COMMIT @KYUA_GIT_COMMIT@
RUN apt-get -y update
RUN apt-get -y dist-upgrade
RUN apt-get -y install curl
RUN if [ -n "@CLANG@" ] && [ "@VERSION@" != "sid" ]; then \
RUN apt-get -y install apt-utils curl
RUN if [ "@CLANG@" -ge "9" ] 2>/dev/null; then \
apt-get -y install apt-transport-https; \
curl -sSL -o /etc/apt/trusted.gpg.d/llvm.asc https://apt.llvm.org/llvm-snapshot.gpg.key; \
echo "deb http://apt.llvm.org/@VERSION@/ llvm-toolchain-@VERSION@ main" > /etc/apt/sources.list.d/llvm.list; \
echo "deb http://apt.llvm.org/@VERSION@/ llvm-toolchain-@VERSION@-9 main" >> /etc/apt/sources.list.d/llvm.list; \
echo "deb http://apt.llvm.org/@VERSION@/ llvm-toolchain-@VERSION@-10 main" >> /etc/apt/sources.list.d/llvm.list; \
echo "deb http://apt.llvm.org/@VERSION@/ llvm-toolchain-@VERSION@-@CLANG@ main" >> /etc/apt/sources.list.d/llvm.list; \
fi
RUN if [ -n "@CLANG@" ] && [ "@VERSION@" = "sid" ]; then \
apt-get -y install apt-transport-https; \
curl -sSL -o /etc/apt/trusted.gpg.d/llvm.asc https://apt.llvm.org/llvm-snapshot.gpg.key; \
echo "deb http://apt.llvm.org/unstable/ llvm-toolchain main" > /etc/apt/sources.list.d/llvm.list; \
echo "deb http://apt.llvm.org/unstable/ llvm-toolchain-9 main" >> /etc/apt/sources.list.d/llvm.list; \
echo "deb http://apt.llvm.org/unstable/ llvm-toolchain-10 main" >> /etc/apt/sources.list.d/llvm.list; \
fi
RUN if [ -n "@COCCINELLE@" ]; then \
RUN if [ "@COCCINELLE@" = "yes" ]; then \
curl -sSL -o /etc/apt/trusted.gpg.d/coccinelle.gpg https://packages.sury.org/coccinelle/apt.gpg; \
echo "deb https://packages.sury.org/coccinelle/ @VERSION@ main" > /etc/apt/sources.list.d/coccinelle.list; \
fi
RUN if [ "@CPPCHECK@" = "yes" ]; then \
curl -sSL -o /etc/apt/trusted.gpg.d/cppcheck.gpg https://packages.sury.org/cppcheck/apt.gpg; \
echo "deb https://packages.sury.org/cppcheck/ @VERSION@ main" > /etc/apt/sources.list.d/cppcheck.list; \
fi
RUN if [ "@RESPDIFF@" = "yes" ]; then \
curl -sSL -o /etc/apt/trusted.gpg.d/lmdb.gpg https://packages.sury.org/lmdb/apt.gpg; \
echo "deb https://packages.sury.org/lmdb/ @VERSION@ main" > /etc/apt/sources.list.d/lmdb.list; \
fi
RUN apt-get -y update
RUN apt-get -y dist-upgrade
RUN apt-get -y install \
......@@ -37,8 +36,6 @@ RUN apt-get -y install \
bison \
build-essential \
ccache \
clang \
cppcheck \
docbook-xsl \
gdb \
git \
......@@ -81,49 +78,31 @@ RUN apt-get -y install \
xsltproc \
zip \
zlib1g-dev
RUN if [ -n "@COCCINELLE@" ]; then apt -y install coccinelle; fi
RUN if [ -n "@CLANG@" ]; then \
RUN if [ "@COMPILEDB@" = "yes" ]; then apt-get -y install python3-pip && pip3 install compiledb; fi
RUN if [ "@COCCINELLE@" = "yes" ]; then apt-get -y install coccinelle; fi
RUN if [ "@CPPCHECK@" = "yes" ]; then apt-get -y install cppcheck; fi
RUN if [ "@CLANG@" -ge "9" ] 2>/dev/null; then \
apt-get -y install \
llvm-9 \
lldb-9 \
clang-9 \
clang-tools-9 \
clang-format-9 \
clang-tidy-9; \
llvm-@CLANG@ \
lldb-@CLANG@ \
clang-@CLANG@ \
clang-tools-@CLANG@ \
clang-format-@CLANG@ \
clang-tidy-@CLANG@; \
fi
RUN if [ -n "@CLANG@" ]; then \
apt-get -y install \
llvm-10 \
lldb-10 \
clang-10 \
clang-tools-10 \
clang-format-10 \
clang-tidy-10; \
fi
RUN if [ -n "@CLANG@" ] && [ "@ARCH@" != "arm64" ]; then \
apt-get -y install \
llvm-11 \
lldb-11 \
clang-11 \
clang-tools-11 \
clang-format-11 \
clang-tidy-11; \
fi
RUN if [ "@VERSION@" = "sid" ]; then \
apt-get -y install \
llvm \
lldb \
clang \
clang-tools \
clang-format \
clang-tidy; \
RUN if [ "@RESPDIFF@" = "yes" ]; then apt-get -y install python3-dpkt python3-lmdb python3-numpy python3-scipy python3-tabulate; fi
RUN if [ "@DNSTAP@" = "yes" ]; then apt-get -y install libfstrm-dev libprotobuf-c-dev protobuf-c-compiler; fi
RUN if [ "@SOFTHSM@" -ge "2" ] 2>/dev/null; then apt-get -y install libsofthsm2 softhsm2; fi
RUN if [ "@SOFTHSM@" -lt "2" ] 2>/dev/null; then apt-get -y install libsofthsm softhsm; fi
RUN if [ "@SHELLCHECK@" = "yes" ]; then apt-get -y install shellcheck; fi
# FIXME: This is Debian bug https://bugs.debian.org/952596. Before
# it gets fixed we patch with a Fedora version of the upstream fix
# (the upstream one does not apply cleanly).
RUN if [ "@ABI_CHECK@" = "yes" ]; then \
apt-get -y install abi-compliance-checker abi-dumper w3m; \
curl -s https://src.fedoraproject.org/rpms/abi-dumper/raw/master/f/0001-Support-for-new-elfutils-Fedora-30.patch | patch /usr/bin/abi-dumper; \
fi
RUN if [ "@VERSION@" = "sid" ]; then apt-get -y install python3-dpkt python3-lmdb python3-numpy python3-scipy python3-tabulate; fi
RUN if [ -n "@UNCRUSTIFY@" ]; then apt-get -y install uncrustify; fi
RUN if [ -n "@DNSTAP@" ]; then apt-get -y install libfstrm-dev libprotobuf-c-dev protobuf-c-compiler; fi
RUN if [ "@SOFTHSM@" -ge "2" ]; then apt-get -y install libsofthsm2 softhsm2; fi
RUN if [ "@SOFTHSM@" -lt "2" ]; then apt-get -y install libsofthsm softhsm; fi
RUN if [ "@VERSION@" = "sid" ]; then apt-get -y install shellcheck; fi
# Kyua and dependencies
RUN curl -sSL https://github.com/jmmv/atf/releases/download/atf-$ATF_VERSION/atf-$ATF_VERSION.tar.gz | tar -xz -C /usr/src
RUN cd /usr/src/atf-$ATF_VERSION && autoreconf -fi && ./configure --prefix /usr && make && make install
RUN ldconfig
......@@ -133,11 +112,3 @@ RUN ldconfig
RUN curl -sSL "https://github.com/Mno-hime/kyua/archive/${KYUA_GIT_COMMIT}.tar.gz" | tar -xz -C /usr/src
RUN cd "/usr/src/kyua-${KYUA_GIT_COMMIT}" && ./configure --prefix /usr && make && make install && cd .. && rm -rf /usr/tests "/usr/src/kyua-${KYUA_GIT_COMMIT}"
RUN ldconfig
# FIXME: This is Debian bug https://bugs.debian.org/952596. Before
# it gets fixed we patch with a Fedora version of the upstream fix
# (the upstream one does not apply cleanly).
RUN if [ "@VERSION@" = "sid" ]; then \
apt-get -y install abi-compliance-checker abi-dumper w3m; \
curl -s https://src.fedoraproject.org/rpms/abi-dumper/raw/master/f/0001-Support-for-new-elfutils-Fedora-30.patch | patch /usr/bin/abi-dumper; \
fi
RUN if [ "@VERSION@" = "sid" ]; then pip3 install compiledb; fi
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment