diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1072902ae3dc7a280050043504298c26b11f1231..71c4fb9ba4fc9970ceef147178ff77de554ab483 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,23 +6,26 @@ variables: CI_REGISTRY_IMAGE: registry.gitlab.isc.org/isc-projects/images/bind9 CCACHE_DIR: "/ccache" + SOFTHSM2_CONF: "/var/tmp/softhsm2/softhsm2.conf" + OPENSSL_ENGINES: "/usr/lib/x86_64-linux-gnu/engines-1.1" + DEFAULT_OPENSSL_CONF: "/etc/ssl/openssl.cnf" + OPENSSL_CONF: "/var/tmp/etc/openssl.cnf" GIT_DEPTH: 1 BUILD_PARALLEL_JOBS: 6 TEST_PARALLEL_JOBS: 6 CONFIGURE: ./configure - CLANG_VERSION: 14 - CLANG: "clang-${CLANG_VERSION}" - SCAN_BUILD: "scan-build-${CLANG_VERSION}" - ASAN_SYMBOLIZER_PATH: "/usr/lib/llvm-${CLANG_VERSION}/bin/llvm-symbolizer" - CLANG_FORMAT: "clang-format-${CLANG_VERSION}" + CLANG: clang-13 + SCAN_BUILD: scan-build-13 + ASAN_SYMBOLIZER_PATH: /usr/lib/llvm-13/bin/llvm-symbolizer + CLANG_FORMAT: clang-format-13 - CFLAGS_COMMON: -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1 -g -Wall -Wextra + CFLAGS_COMMON: -fno-omit-frame-pointer -fno-optimize-sibling-calls -O1 -g -Wall -Wextra # Pass run-time flags to AddressSanitizer to get core dumps on error. ASAN_OPTIONS: abort_on_error=1:disable_coredump=0:unmap_shadow_on_exit=1 - TSAN_OPTIONS_COMMON: "disable_coredump=0 second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan" + TSAN_OPTIONS_COMMON: "disable_coredump=0 second_deadlock_stack=1 history_size=7 log_exe_name=true log_path=tsan external_symbolizer_path=/usr/lib/llvm-12/bin/llvm-symbolizer" TARBALL_EXTENSION: xz @@ -62,6 +65,7 @@ stages: - system - performance - docs + - push - postcheck - release @@ -102,8 +106,8 @@ stages: # Alpine Linux -.alpine-3.16-amd64: &alpine_3_16_amd64_image - image: "$CI_REGISTRY_IMAGE:alpine-3.16-amd64" +.alpine-3.15-amd64: &alpine_3_15_amd64_image + image: "$CI_REGISTRY_IMAGE:alpine-3.15-amd64" <<: *linux_amd64 # Oracle Linux @@ -116,11 +120,15 @@ stages: image: "$CI_REGISTRY_IMAGE:oraclelinux-8-amd64" <<: *linux_amd64 -.oraclelinux-9-amd64: &oraclelinux_9_amd64_image - image: "$CI_REGISTRY_IMAGE:oraclelinux-9-amd64" +# Debian + +.debian-stretch-amd64: &debian_stretch_amd64_image + image: "$CI_REGISTRY_IMAGE:debian-stretch-amd64" <<: *linux_amd64 -# Debian +.debian-buster-amd64: &debian_buster_amd64_image + image: "$CI_REGISTRY_IMAGE:debian-buster-amd64" + <<: *linux_amd64 .debian-bullseye-amd64: &debian_bullseye_amd64_image image: "$CI_REGISTRY_IMAGE:debian-bullseye-amd64" @@ -142,22 +150,22 @@ stages: # Fedora -.fedora-36-amd64: &fedora_36_amd64_image - image: "$CI_REGISTRY_IMAGE:fedora-36-amd64" +.fedora-35-amd64: &fedora_35_amd64_image + image: "$CI_REGISTRY_IMAGE:fedora-35-amd64" <<: *linux_amd64 -.fedora-36-arm64: &fedora_36_arm64_image - image: "$CI_REGISTRY_IMAGE:fedora-36-arm64" +.fedora-35-arm64: &fedora_35_arm64_image + image: "$CI_REGISTRY_IMAGE:fedora-35-arm64" <<: *linux_stress_arm64 # Ubuntu -.ubuntu-focal-amd64: &ubuntu_focal_amd64_image - image: "$CI_REGISTRY_IMAGE:ubuntu-focal-amd64" +.ubuntu-bionic-amd64: &ubuntu_bionic_amd64_image + image: "$CI_REGISTRY_IMAGE:ubuntu-bionic-amd64" <<: *linux_amd64 -.ubuntu-jammy-amd64: &ubuntu_jammy_amd64_image - image: "$CI_REGISTRY_IMAGE:ubuntu-jammy-amd64" +.ubuntu-focal-amd64: &ubuntu_focal_amd64_image + image: "$CI_REGISTRY_IMAGE:ubuntu-focal-amd64" <<: *linux_amd64 # Base image @@ -173,46 +181,16 @@ stages: <<: *libvirt_amd64 .freebsd-13-amd64: &freebsd_13_amd64_image - image: "freebsd-13.1-x86_64" + image: "freebsd-13.0-x86_64" <<: *libvirt_amd64 .openbsd-amd64: &openbsd_amd64_image - image: "openbsd-7.1-x86_64" + image: "openbsd-7.0-x86_64" <<: *libvirt_amd64 ### Job Templates -.api-schedules-tags-triggers-web-triggering-rules: &api_schedules_tags_triggers_web_triggering_rules - only: - - api - - schedules - - tags - - triggers - - web - -.api-schedules-triggers-web-triggering-rules: &api_schedules_triggers_web_triggering_rules - only: - - api - - schedules - - triggers - - web - -.default-triggering-rules: &default_triggering_rules - only: - - api - - merge_requests - - schedules - - tags - - triggers - - web - -.precheck: &precheck_job - <<: *default_triggering_rules - <<: *base_image - stage: precheck - .autoconf: &autoconf_job - <<: *default_triggering_rules <<: *base_image stage: autoconf script: @@ -230,15 +208,9 @@ stages: --with-cmocka --with-libxml2 --with-json-c - --enable-leak-detection $EXTRA_CONFIGURE || (test -s config.log && cat config.log; exit 1) -# change directory to the workspace before including this -.find_python: &find_python - - PYTHON="$(source bin/tests/system/conf.sh; echo $PYTHON)" - - test -x "$PYTHON" - .check_readline_setup: &check_readline_setup - if [[ -n "${WITHOUT_READLINE}" ]]; then ! grep "^#define HAVE_READLINE" config.h; @@ -260,8 +232,19 @@ stages: - rm -f bind-*.tar.${TARBALL_EXTENSION} - cd bind-* +# Move the out-of-tree workspace to CI project dir to save it for use in +# dependent jobs. +.save_out_of_tree_workspace: &save_out_of_tree_workspace + - test -n "${OUT_OF_TREE_WORKSPACE}" && mv "${OUT_OF_TREE_WORKSPACE}" "${CI_PROJECT_DIR}" + +# Move the artifacts from the out-of-tree build job to their original +# location (the out-of-tree workspace) and then continue work in the +# out-of-tree workspace. +.retrieve_out_of_tree_workspace: &retrieve_out_of_tree_workspace + - test -n "${OUT_OF_TREE_WORKSPACE}" && mv "$(basename "${OUT_OF_TREE_WORKSPACE}")" "${OUT_OF_TREE_WORKSPACE}" + - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" + .build: &build_job - <<: *default_triggering_rules stage: build before_script: - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}" @@ -277,7 +260,8 @@ stages: - test -z "${CROSS_COMPILATION}" || file lib/dns/gen | grep -F -q "ELF 64-bit LSB" - test -z "${CROSS_COMPILATION}" || ( ! git ls-files -z --others --exclude lib/dns/gen | xargs -0 file | grep "ELF 64-bit LSB" ) - if test -z "${OUT_OF_TREE_WORKSPACE}" && test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi - - bin/named/named -V + after_script: + - *save_out_of_tree_workspace needs: - job: autoreconf artifacts: true @@ -286,135 +270,19 @@ stages: expire_in: "1 day" when: always -.setup_interfaces: &setup_interfaces - - if [ "$(id -u)" -eq "0" ]; then - sh -x bin/tests/system/ifconfig.sh up; - else - sudo sh -x bin/tests/system/ifconfig.sh up; - fi - -.system_test_common: &system_test_common - <<: *default_triggering_rules - stage: system - before_script: - - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" - - *setup_interfaces - script: - - cd bin/tests/system - - make -j${TEST_PARALLEL_JOBS:-1} -k check V=1 - - if git rev-parse > /dev/null 2>&1; then ( ! grep "^I:.*:file.*not removed$" *.log ); fi - after_script: - - test -d bind-* && cd bind-* - - REALSOURCEDIR="$PWD" - - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" - - cat bin/tests/system/test-suite.log - - *find_python - - > - "$PYTHON" "$REALSOURCEDIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - -.system_test: &system_test_job - <<: *system_test_common - artifacts: - untracked: true - expire_in: "1 day" - when: always - reports: - junit: junit.xml - -.system_test_gcov: &system_test_gcov_job - <<: *system_test_common - artifacts: - untracked: true - expire_in: "1 day" - when: always - -.system_test_tsan: &system_test_tsan_job - <<: *system_test_common - after_script: - - cat bin/tests/system/test-suite.log - - *find_python - - find -name 'tsan.*' -exec "$PYTHON" util/parse_tsan.py {} \; - - > - "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - artifacts: - untracked: true - expire_in: "1 day" - when: always - reports: - junit: junit.xml - -.unit_test_common: &unit_test_common - <<: *default_triggering_rules - stage: unit - before_script: - - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" - script: - - make -j${TEST_PARALLEL_JOBS:-1} -k unit V=1 - after_script: - - test -d bind-* && cd bind-* - - REALSOURCEDIR="$PWD" - - test -n "${OUT_OF_TREE_WORKSPACE}" && cd "${OUT_OF_TREE_WORKSPACE}" - - *find_python - - > - "$PYTHON" "$REALSOURCEDIR"/bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - -.unit_test: &unit_test_job - <<: *unit_test_common - artifacts: - untracked: true - expire_in: "1 day" - when: always - reports: - junit: junit.xml - -.unit_test_gcov: &unit_test_gcov_job - <<: *unit_test_common - artifacts: - untracked: true - expire_in: "1 day" - when: always - -.unit_test_tsan: &unit_test_tsan_job - <<: *unit_test_common - after_script: - - *find_python - - find -name 'tsan.*' -exec "$PYTHON" util/parse_tsan.py {} \; - - > - "$PYTHON" bin/tests/convert-trs-to-junit.py . > "$CI_PROJECT_DIR"/junit.xml - artifacts: - untracked: true - expire_in: "1 day" - when: always - reports: - junit: junit.xml - .docs: &docs_job stage: docs script: - *configure - make -j${BUILD_PARALLEL_JOBS:-1} -k doc V=1 - - qpdf --check doc/arm/_build/latex/Bv9ARM.pdf - - find doc/man/ -maxdepth 1 -name "*.[0-9]" -exec mandoc -T lint "{}" \; | ( ! grep -v -e "skipping paragraph macro. sp after" -e "unknown font, skipping request. ft C" ) - -.respdiff: &respdiff_job - <<: *base_image - stage: system - before_script: - - autoreconf -fi - - *configure - - make -j${BUILD_PARALLEL_JOBS:-1} V=1 - - *setup_interfaces - - git clone --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.isc.org/isc-private/bind-qa.git - - cd bind-qa/bind9/respdiff - needs: [] artifacts: + expire_in: "1 week" paths: - - bind-qa/bind9/respdiff + - doc/ exclude: - - bind-qa/bind9/respdiff/rspworkdir/data.mdb # Exclude a 10 GB file. - untracked: true - expire_in: "1 day" - when: always + - doc/arm/_build/.doctrees/**/* + - doc/arm/_build/epub/**/* + - doc/arm/_build/singlehtml/**/* ### Job Definitions @@ -423,1048 +291,69 @@ stages: autoreconf: <<: *autoconf_job -misc: - <<: *precheck_job - script: - - sh util/checklibs.sh > checklibs.out - - sh util/tabify-changes < CHANGES > CHANGES.tmp - - diff -urNap CHANGES CHANGES.tmp - - perl util/check-changes CHANGES - - sh util/check-line-length.sh CHANGES - - test ! -f CHANGES.SE || sh util/tabify-changes < CHANGES.SE > CHANGES.tmp - - test ! -f CHANGES.SE || diff -urNap CHANGES.SE CHANGES.tmp - - test ! -f CHANGES.SE || perl util/check-changes master=0 CHANGES.SE - - test ! -f CHANGES.SE || sh util/check-line-length.sh CHANGES.SE - - rm CHANGES.tmp - - sh util/check-categories.sh - - if git grep SYSTEMTESTTOP -- ':!.gitlab-ci.yml'; then echo 'Please use relative paths instead of $SYSTEMTESTTOP.'; exit 1; fi - - bash util/unused-headers.sh - - bash util/xmllint-html.sh - needs: [] - artifacts: - paths: - - checklibs.out - expire_in: "1 day" - when: on_failure - -black: - <<: *precheck_job - needs: [] - script: - - black $(git ls-files '*.py') - - git diff > black.patch - - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi - artifacts: - paths: - - black.patch - expire_in: "1 week" - when: on_failure - -clang-format: - <<: *precheck_job - needs: [] - script: - - if [ -r .clang-format ]; then "${CLANG_FORMAT}" -i -style=file $(git ls-files '*.c' '*.h'); fi - - git diff > clang-format.patch - - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi - artifacts: - paths: - - clang-format.patch - expire_in: "1 week" - when: on_failure - -coccinelle: - <<: *precheck_job - needs: [] - script: - - util/check-cocci - - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi - -pylint: - <<: *precheck_job - needs: [] - script: - - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -vE '(ans\.py|dangerfile\.py|^bin/tests/system/)') - # Ignore Pylint wrong-import-position error in system test to enable use of pytest.importorskip - - pylint --rcfile $CI_PROJECT_DIR/.pylintrc --disable=wrong-import-position $(git ls-files 'bin/tests/system/*.py' | grep -vE 'ans\.py') - -reuse: - <<: *precheck_job - needs: [] - image: - name: docker.io/fsfe/reuse:latest - entrypoint: [""] - script: - - reuse lint - -danger: - <<: *precheck_job - needs: [] - script: - - danger-python ci -f - only: - refs: - - merge_requests - variables: - - $DANGER_GITLAB_API_TOKEN - -tarball-create: - stage: precheck - <<: *base_image - <<: *default_triggering_rules - script: - - ./configure --enable-maintainer-mode - - make maintainer-clean - - autoreconf -fi - - ./configure --enable-maintainer-mode - - make -j${BUILD_PARALLEL_JOBS:-1} all V=1 - - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; git diff > diff.patch; exit 1; fi - - make -j${BUILD_PARALLEL_JOBS:-1} dist V=1 - artifacts: - paths: - - diff.patch - - bind-*.tar.${TARBALL_EXTENSION} - when: always - needs: - - job: autoreconf - artifacts: true + #flake8: + # <<: *base_image + # stage: postcheck + # needs: + # - job: autoreconf + # artifacts: true + # script: + # - *configure + # - flake8 --max-line-length=80 $(git ls-files '*.py' | grep -vE '(ans\.py|dangerfile\.py|^bin/tests/system/)') + # # Ignore Flake8 E402 error (module level import not at top of file) in system test to enable use of pytest.importorskip + # - flake8 --max-line-length=80 --extend-ignore=E402 $(git ls-files 'bin/tests/system/*.py' | grep -vE 'ans\.py') + # + #pylint: + # <<: *base_image + # stage: postcheck + # needs: + # - job: autoreconf + # artifacts: true + # script: + # - *configure + # - pylint --rcfile $CI_PROJECT_DIR/.pylintrc $(git ls-files '*.py' | grep -vE '(ans\.py|dangerfile\.py|^bin/tests/system/)') + # # Ignore Pylint wrong-import-position error in system test to enable use of pytest.importorskip + # - pylint --rcfile $CI_PROJECT_DIR/.pylintrc --disable=wrong-import-position $(git ls-files 'bin/tests/system/*.py' | grep -vE 'ans\.py') + # + #tarball-create: + # stage: precheck + # <<: *base_image + # script: + # - ./configure --enable-maintainer-mode + # - make maintainer-clean + # - autoreconf -fi + # - ./configure --enable-maintainer-mode + # - make -j${BUILD_PARALLEL_JOBS:-1} all V=1 + # - if test "$(git status --porcelain | grep -Ev '\?\?' | wc -l)" -gt "0"; then git status --short; exit 1; fi + # - make -j${BUILD_PARALLEL_JOBS:-1} dist V=1 + # artifacts: + # paths: + # - bind-*.tar.${TARBALL_EXTENSION} + # needs: + # - job: autoreconf + # artifacts: true + # allow_failure: true # not to scare Ron - and not to make him touch Makefiles # Jobs for doc builds on Debian 11 "bullseye" (amd64) docs: - <<: *default_triggering_rules - <<: *base_image - <<: *docs_job - needs: - - job: autoreconf - artifacts: true - artifacts: - untracked: true - -docs:tarball: - <<: *default_triggering_rules <<: *base_image <<: *docs_job before_script: - - *unpack_release_tarball - needs: - - job: tarball-create - artifacts: true - -# Jobs for regular GCC builds on Alpine Linux 3.16 (amd64) - -gcc:alpine3.16:amd64: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON}" - EXTRA_CONFIGURE: "${WITHOUT_READLINE}" - <<: *alpine_3_16_amd64_image - <<: *build_job - -system:gcc:alpine3.16:amd64: - <<: *alpine_3_16_amd64_image - <<: *system_test_job - needs: - - job: gcc:alpine3.16:amd64 - artifacts: true - -unit:gcc:alpine3.16:amd64: - <<: *alpine_3_16_amd64_image - <<: *unit_test_job - needs: - - job: gcc:alpine3.16:amd64 - artifacts: true - -# Jobs for regular GCC builds on Oracle Linux 7 (amd64) - -gcc:oraclelinux7:amd64: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON}" - EXTRA_CONFIGURE: "--with-libidn2" - <<: *oraclelinux_7_amd64_image - <<: *build_job - -system:gcc:oraclelinux7:amd64: - <<: *oraclelinux_7_amd64_image - <<: *system_test_job - needs: - - job: gcc:oraclelinux7:amd64 - artifacts: true - -unit:gcc:oraclelinux7:amd64: - <<: *oraclelinux_7_amd64_image - <<: *unit_test_job - needs: - - job: gcc:oraclelinux7:amd64 - artifacts: true - -# Jobs for regular GCC builds on Oracle Linux 8 (amd64) - -gcc:oraclelinux8:amd64: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON}" - EXTRA_CONFIGURE: "--with-libidn2" - <<: *oraclelinux_8_amd64_image - <<: *build_job - -system:gcc:oraclelinux8:amd64: - <<: *oraclelinux_8_amd64_image - <<: *system_test_job - needs: - - job: gcc:oraclelinux8:amd64 - artifacts: true - -unit:gcc:oraclelinux8:amd64: - <<: *oraclelinux_8_amd64_image - <<: *unit_test_job - needs: - - job: gcc:oraclelinux8:amd64 - artifacts: true - -# Jobs for regular GCC builds on Oracle Linux 9 (amd64) - -gcc:oraclelinux9:amd64: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON}" - EXTRA_CONFIGURE: "--with-libidn2" - <<: *oraclelinux_9_amd64_image - <<: *build_job - -system:gcc:oraclelinux9:amd64: - <<: *oraclelinux_9_amd64_image - <<: *system_test_job - needs: - - job: gcc:oraclelinux9:amd64 - artifacts: true - -unit:gcc:oraclelinux9:amd64: - <<: *oraclelinux_9_amd64_image - <<: *unit_test_job - needs: - - job: gcc:oraclelinux9:amd64 - artifacts: true - -# Jobs for regular GCC builds on Debian 11 "bullseye" (amd64) -# (The second unit test job also executes unstable unit tests.) - -gcc:bullseye:amd64: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} --coverage -O0" - EXTRA_CONFIGURE: "--with-libidn2 ${WITH_READLINE_LIBEDIT}" - <<: *debian_bullseye_amd64_image - <<: *build_job - -system:gcc:bullseye:amd64: - <<: *debian_bullseye_amd64_image - <<: *system_test_gcov_job - needs: - - job: unit:gcc:bullseye:amd64 - artifacts: true - -system:gcc:bullseye:unstable:amd64: - <<: *debian_bullseye_amd64_image - <<: *system_test_job - <<: *api_schedules_triggers_web_triggering_rules - variables: - CI_ENABLE_ALL_TESTS: 1 - needs: - - job: gcc:bullseye:amd64 - artifacts: true - -unit:gcc:bullseye:amd64: - <<: *debian_bullseye_amd64_image - <<: *unit_test_gcov_job - needs: - - job: gcc:bullseye:amd64 - artifacts: true - -unit:gcc:bullseye:unstable:amd64: - <<: *debian_bullseye_amd64_image - <<: *unit_test_job - <<: *api_schedules_triggers_web_triggering_rules - variables: - CI_ENABLE_ALL_TESTS: 1 - needs: - - job: gcc:bullseye:amd64 - artifacts: true - -# Jobs for cross-compiled GCC builds on Debian 11 "bullseye" (amd64) with -# 32-bit libraries - -gcc:bullseye:amd64cross32: - variables: - CFLAGS: "${CFLAGS_COMMON}" - CROSS_COMPILATION: 1 - EXTRA_CONFIGURE: "--build=x86_64-linux-gnu --host=i686-linux-gnu --with-libidn2 ${WITH_READLINE_LIBEDIT}" - <<: *debian_bullseye_amd64cross32_image - <<: *build_job - -system:gcc:bullseye:amd64cross32: - <<: *debian_bullseye_amd64cross32_image - <<: *system_test_job - needs: - - job: gcc:bullseye:amd64cross32 - artifacts: true - -unit:gcc:bullseye:amd64cross32: - <<: *debian_bullseye_amd64cross32_image - <<: *unit_test_job - needs: - - job: gcc:bullseye:amd64cross32 - artifacts: true - -# Jobs for scan-build builds on Debian 11 "bullseye" (amd64) - -.scan_build: &scan_build - - ${SCAN_BUILD} --html-title="BIND 9 ($CI_COMMIT_SHORT_SHA)" - --keep-cc - --status-bugs - --keep-going - -o scan-build.reports make -j${BUILD_PARALLEL_JOBS:-1} all V=1 - -scan-build: - <<: *default_triggering_rules - <<: *base_image - stage: postcheck - variables: - CC: "${CLANG}" - CFLAGS: "${CFLAGS_COMMON}" - CONFIGURE: "${SCAN_BUILD} ./configure" - EXTRA_CONFIGURE: "--with-libidn2" - script: - - *configure - - *scan_build + - test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}" needs: - job: autoreconf artifacts: true - artifacts: - paths: - - scan-build.reports/ - expire_in: "1 day" - when: on_failure - -# Jobs for regular GCC builds on Debian "sid" (amd64) -# Also tests configration option: --without-lmdb. - -gcc:sid:amd64: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -O3 -DOPENSSL_API_COMPAT=10100" - # For the jemalloc ./configure option, see https://gitlab.isc.org/isc-projects/bind9/-/issues/3444 - EXTRA_CONFIGURE: "--with-libidn2 --without-lmdb --without-jemalloc --disable-leak-detection ${WITH_READLINE}" - RUN_MAKE_INSTALL: 1 - <<: *debian_sid_amd64_image - <<: *build_job - -system:gcc:sid:amd64: - # Set up environment variables that allow the "keyfromlabel" system test to be run - variables: - DEFAULT_OPENSSL_CONF: "/etc/ssl/openssl.cnf" - OPENSSL_CONF: "/var/tmp/etc/openssl.cnf" - SOFTHSM2_CONF: "/var/tmp/softhsm2/softhsm2.conf" - SOFTHSM2_MODULE: "/usr/lib/softhsm/libsofthsm2.so" - <<: *debian_sid_amd64_image - <<: *system_test_job - needs: - - job: gcc:sid:amd64 - artifacts: true - -unit:gcc:sid:amd64: - <<: *debian_sid_amd64_image - <<: *unit_test_job - needs: - - job: gcc:sid:amd64 - artifacts: true - -# Job for out-of-tree GCC build on Debian 11 "bullseye" (amd64) -# Also tests configration option: --with-lmdb. -gcc:out-of-tree: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -Og" - CONFIGURE: "${CI_PROJECT_DIR}/configure" - EXTRA_CONFIGURE: "--with-libidn2 --with-lmdb" - RUN_MAKE_INSTALL: 1 - OUT_OF_TREE_WORKSPACE: workspace +pages: + stage: push <<: *base_image - <<: *build_job - -system:gcc:out-of-tree: - variables: - OUT_OF_TREE_WORKSPACE: workspace - needs: - - job: gcc:out-of-tree - artifacts: true - <<: *base_image - <<: *system_test_job - <<: *api_schedules_tags_triggers_web_triggering_rules - -unit:gcc:out-of-tree: - variables: - OUT_OF_TREE_WORKSPACE: workspace - needs: - - job: gcc:out-of-tree - artifacts: true - <<: *base_image - <<: *unit_test_job - <<: *api_schedules_tags_triggers_web_triggering_rules - -# Jobs for tarball GCC builds on Debian 11 "bullseye" (amd64) - -gcc:tarball: - variables: - CC: gcc - EXTRA_CONFIGURE: "--with-libidn2" - RUN_MAKE_INSTALL: 1 - <<: *base_image - <<: *build_job - before_script: - - *unpack_release_tarball - needs: - - job: tarball-create - artifacts: true - -system:gcc:tarball: - <<: *base_image - <<: *system_test_job - <<: *api_schedules_tags_triggers_web_triggering_rules - before_script: - - cd bind-* - - *setup_interfaces - needs: - - job: gcc:tarball - artifacts: true - -unit:gcc:tarball: - <<: *base_image - <<: *unit_test_job - <<: *api_schedules_tags_triggers_web_triggering_rules - before_script: - - cd bind-* - needs: - - job: gcc:tarball - artifacts: true - -# Jobs for debug GCC builds on openSUSE Tumbleweed (amd64) - -gcc:tumbleweed:amd64: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -DDEBUG" - EXTRA_CONFIGURE: "--with-libidn2 ${WITH_READLINE_READLINE}" - <<: *tumbleweed_latest_amd64_image - <<: *build_job - -system:gcc:tumbleweed:amd64: - <<: *tumbleweed_latest_amd64_image - <<: *system_test_job - needs: - - job: gcc:tumbleweed:amd64 - artifacts: true - -unit:gcc:tumbleweed:amd64: - <<: *tumbleweed_latest_amd64_image - <<: *unit_test_job - needs: - - job: gcc:tumbleweed:amd64 - artifacts: true - -# Jobs for regular GCC builds on Ubuntu 20.04 Focal Fossa (amd64) - -gcc:focal:amd64: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -Og" - EXTRA_CONFIGURE: "--disable-dnstap --without-cmocka --without-gssapi --with-libidn2" - <<: *ubuntu_focal_amd64_image - <<: *build_job - -system:gcc:focal:amd64: - <<: *ubuntu_focal_amd64_image - <<: *system_test_job - needs: - - job: gcc:focal:amd64 - artifacts: true - -unit:gcc:focal:amd64: - <<: *ubuntu_focal_amd64_image - <<: *unit_test_job - needs: - - job: gcc:focal:amd64 - artifacts: true - -# Jobs for regular GCC builds on Ubuntu 22.04 Jammy Jellyfish (amd64) - -gcc:jammy:amd64: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON}" - EXTRA_CONFIGURE: "--disable-geoip --with-libidn2 --disable-doh" - <<: *ubuntu_jammy_amd64_image - <<: *build_job - -system:gcc:jammy:amd64: - <<: *ubuntu_jammy_amd64_image - <<: *system_test_job - needs: - - job: gcc:jammy:amd64 - artifacts: true - -unit:gcc:jammy:amd64: - <<: *ubuntu_jammy_amd64_image - <<: *unit_test_job - needs: - - job: gcc:jammy:amd64 - artifacts: true - -# Jobs for ASAN builds on Fedora 36 (amd64) - -gcc:asan: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -fsanitize=address,undefined" - LDFLAGS: "-fsanitize=address,undefined" - EXTRA_CONFIGURE: "--with-libidn2 --without-jemalloc" - <<: *fedora_36_amd64_image - <<: *build_job - -system:gcc:asan: - variables: - LSAN_OPTIONS: "suppressions=$CI_PROJECT_DIR/suppr-lsan.txt" - <<: *fedora_36_amd64_image - <<: *system_test_job - needs: - - job: gcc:asan - artifacts: true - -unit:gcc:asan: - <<: *fedora_36_amd64_image - <<: *unit_test_job - needs: - - job: gcc:asan - artifacts: true - -clang:asan: - variables: - CC: ${CLANG} - CFLAGS: "${CFLAGS_COMMON} -fsanitize=address,undefined" - LDFLAGS: "-fsanitize=address,undefined" - EXTRA_CONFIGURE: "--with-libidn2 --without-jemalloc" - <<: *base_image - <<: *build_job - -system:clang:asan: - <<: *base_image - <<: *system_test_job - needs: - - job: clang:asan - artifacts: true - -unit:clang:asan: - <<: *base_image - <<: *unit_test_job - needs: - - job: clang:asan - artifacts: true - -# Jobs for TSAN builds on Fedora 36 (amd64) - -gcc:tsan: - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -fsanitize=thread" - LDFLAGS: "-fsanitize=thread" - EXTRA_CONFIGURE: "--with-libidn2 --enable-pthread-rwlock --without-jemalloc" - <<: *fedora_36_amd64_image - <<: *build_job - -system:gcc:tsan: - variables: - TSAN_OPTIONS: "${TSAN_OPTIONS_COMMON} external_symbolizer_path=/usr/bin/llvm-symbolizer" - <<: *fedora_36_amd64_image - <<: *system_test_tsan_job - needs: - - job: gcc:tsan - artifacts: true - -unit:gcc:tsan: - variables: - TSAN_OPTIONS: "${TSAN_OPTIONS_COMMON} external_symbolizer_path=/usr/bin/llvm-symbolizer" - <<: *fedora_36_amd64_image - <<: *unit_test_tsan_job - needs: - - job: gcc:tsan - artifacts: true - -clang:tsan: - <<: *base_image - <<: *build_job - variables: - CC: "${CLANG}" - CFLAGS: "${CFLAGS_COMMON} -fsanitize=thread" - LDFLAGS: "-fsanitize=thread" - EXTRA_CONFIGURE: "--with-libidn2 --enable-pthread-rwlock --without-jemalloc" - -system:clang:tsan: - variables: - TSAN_OPTIONS: "${TSAN_OPTIONS_COMMON} external_symbolizer_path=/usr/lib/llvm-${CLANG_VERSION}/bin/llvm-symbolizer" - <<: *base_image - <<: *system_test_tsan_job - needs: - - job: clang:tsan - artifacts: true - -unit:clang:tsan: - variables: - TSAN_OPTIONS: "${TSAN_OPTIONS_COMMON} external_symbolizer_path=/usr/lib/llvm-${CLANG_VERSION}/bin/llvm-symbolizer suppressions=$CI_PROJECT_DIR/tsan-suppressions.txt" - <<: *base_image - <<: *unit_test_tsan_job - needs: - - job: clang:tsan - artifacts: true - -# Jobs for Clang builds on Debian 11 "bullseye" (amd64) - -clang:bullseye:amd64: - variables: - CC: ${CLANG} - CFLAGS: "${CFLAGS_COMMON} -Wenum-conversion" - # See https://gitlab.isc.org/isc-projects/bind9/-/issues/3444 - EXTRA_CONFIGURE: "--without-jemalloc --disable-leak-detection" - <<: *debian_bullseye_amd64_image - <<: *build_job - -system:clang:bullseye:amd64: - # Set up environment variables that allow the "keyfromlabel" system test to be run - variables: - DEFAULT_OPENSSL_CONF: "/etc/ssl/openssl.cnf" - OPENSSL_CONF: "/var/tmp/etc/openssl.cnf" - SOFTHSM2_CONF: "/var/tmp/softhsm2/softhsm2.conf" - SOFTHSM2_MODULE: "/usr/lib/softhsm/libsofthsm2.so" - <<: *debian_bullseye_amd64_image - <<: *system_test_job - needs: - - job: clang:bullseye:amd64 - artifacts: true - -unit:clang:bullseye:amd64: - <<: *debian_bullseye_amd64_image - <<: *unit_test_job - needs: - - job: clang:bullseye:amd64 - artifacts: true - -# Jobs for Clang builds on FreeBSD 12 (amd64) - -clang:freebsd12:amd64: - variables: - CFLAGS: "${CFLAGS_COMMON}" - EXTRA_CONFIGURE: "${WITH_READLINE_EDITLINE}" - USER: gitlab-runner - <<: *freebsd_12_amd64_image - <<: *build_job - -system:clang:freebsd12:amd64: - <<: *freebsd_12_amd64_image - <<: *system_test_job - variables: - USER: gitlab-runner - TEST_PARALLEL_JOBS: 4 - needs: - - job: clang:freebsd12:amd64 - artifacts: true - -unit:clang:freebsd12:amd64: - <<: *freebsd_12_amd64_image - <<: *unit_test_job - needs: - - job: clang:freebsd12:amd64 - artifacts: true - -# Jobs for Clang builds on FreeBSD 13 (amd64) - -clang:freebsd13:amd64: - variables: - CFLAGS: "${CFLAGS_COMMON}" - EXTRA_CONFIGURE: "${WITH_READLINE_LIBEDIT}" - USER: gitlab-runner - <<: *freebsd_13_amd64_image - <<: *build_job - -system:clang:freebsd13:amd64: - <<: *freebsd_13_amd64_image - <<: *system_test_job - variables: - USER: gitlab-runner - TEST_PARALLEL_JOBS: 4 - needs: - - job: clang:freebsd13:amd64 - artifacts: true - -unit:clang:freebsd13:amd64: - <<: *freebsd_13_amd64_image - <<: *unit_test_job - needs: - - job: clang:freebsd13:amd64 - artifacts: true - -# Jobs for Clang builds on OpenBSD (amd64) - -clang:openbsd:amd64: - variables: - CC: clang - USER: gitlab-runner - EXTRA_CONFIGURE: "--disable-dnstap" - <<: *openbsd_amd64_image - <<: *build_job - -system:clang:openbsd:amd64: - <<: *openbsd_amd64_image - <<: *system_test_job - <<: *api_schedules_triggers_web_triggering_rules - variables: - USER: gitlab-runner - needs: - - job: clang:openbsd:amd64 - artifacts: true - -unit:clang:openbsd:amd64: - <<: *openbsd_amd64_image - <<: *unit_test_job - variables: - USER: gitlab-runner - needs: - - job: clang:openbsd:amd64 - artifacts: true - -# Job producing a release tarball - -release: - <<: *base_image - stage: release script: - - export BIND_DIRECTORY="$(basename "$(find . -name "bind-*.tar.*" -printf "%f")" ".tar.${TARBALL_EXTENSION}")" - # Prepare release tarball contents (tarballs + documentation) - - mkdir -p release/doc/arm - - pushd release - - mv "../${BIND_DIRECTORY}.tar.${TARBALL_EXTENSION}" . - - tar --extract --file="${BIND_DIRECTORY}.tar.${TARBALL_EXTENSION}" - - mv "${BIND_DIRECTORY}"/{CHANGES*,COPYRIGHT,LICENSE,README.md,srcid} . - - rm -rf "${BIND_DIRECTORY}" - - mv "../doc/arm/_build/html" doc/arm/ - - mv "../doc/arm/_build/latex/Bv9ARM.pdf" doc/arm/ - - mv "../doc/arm/_build/epub/Bv9ARM.epub" doc/arm/ - - echo 'Redirect' > "RELEASE-NOTES-${BIND_DIRECTORY}.html" - - popd - # Create release tarball - - tar --create --file="${CI_COMMIT_TAG}.tar.gz" --gzip release/ - needs: - - job: tarball-create - artifacts: true - - job: docs - artifacts: true - only: - - tags + - mv -v doc/arm/_build/html public artifacts: paths: - - "*.tar.gz" - expire_in: "1 day" - -# Coverity Scan analysis upload - -.coverity_prep: &coverity_prep - - curl --output /tmp/cov-analysis-linux64.md5 https://scan.coverity.com/download/linux64 - --form project=$COVERITY_SCAN_PROJECT_NAME - --form token=$COVERITY_SCAN_TOKEN - --form md5=1 - - curl --output /tmp/cov-analysis-linux64.tgz https://scan.coverity.com/download/linux64 - --form project=$COVERITY_SCAN_PROJECT_NAME - --form token=$COVERITY_SCAN_TOKEN - - test "$(md5sum /tmp/cov-analysis-linux64.tgz | awk '{ print $1 }')" = "$(cat /tmp/cov-analysis-linux64.md5)" - - tar --extract --gzip --file=/tmp/cov-analysis-linux64.tgz --directory=/tmp - - test -d /tmp/cov-analysis-linux64-2022.6.0 - -.coverity_build: &coverity_build - - /tmp/cov-analysis-linux64-2022.6.0/bin/cov-build --dir /tmp/cov-int --fs-capture-search . sh -c 'make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1' - - tar --create --gzip --file=/tmp/cov-int.tar.gz --directory /tmp cov-int - - curl -v https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME - --form token=$COVERITY_SCAN_TOKEN - --form email=bind-changes@isc.org - --form file=@/tmp/cov-int.tar.gz - --form version="$(git rev-parse --short HEAD)" - --form description="$(git rev-parse --short HEAD) / $CI_COMMIT_TITLE / $CI_COMMIT_REF_NAME:$CI_PIPELINE_ID" 2>&1 - | tee curl-response.txt - - grep -q 'Build successfully submitted' curl-response.txt - -coverity: - <<: *base_image - stage: postcheck - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -Og" - EXTRA_CONFIGURE: "--with-libidn2" - script: - - *coverity_prep - - *configure - - *coverity_build - after_script: - - mv -v /tmp/cov-int.tar.gz ${CI_PROJECT_DIR}/ + - public/ needs: - - job: autoreconf - artifacts: true - artifacts: - paths: - - curl-response.txt - - cov-int.tar.gz - expire_in: "1 week" - when: on_failure - only: - variables: - - $COVERITY_SCAN_PROJECT_NAME - - $COVERITY_SCAN_TOKEN - -# Respdiff tests - -respdiff-short: - <<: *respdiff_job - <<: *default_triggering_rules - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -Og -DISC_TRACK_PTHREADS_OBJECTS" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" - script: - - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/10k_a.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" - -respdiff-long: - <<: *respdiff_job - <<: *api_schedules_tags_triggers_web_triggering_rules - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -Og -DISC_TRACK_PTHREADS_OBJECTS" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" - script: - - bash respdiff.sh -m /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 -s named -q "${PWD}/100k_mixed.txt" -c 3 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" "/usr/local/respdiff-reference-bind/sbin/named" - -respdiff-long-third-party: - <<: *respdiff_job - <<: *api_schedules_tags_triggers_web_triggering_rules - variables: - CC: gcc - CFLAGS: "${CFLAGS_COMMON} -Og" - MAX_DISAGREEMENTS_PERCENTAGE: "0.5" - script: - - bash respdiff.sh -s third_party -q "${PWD}/100k_mixed.txt" -c 1 -w "${PWD}/rspworkdir" "${CI_PROJECT_DIR}" - -# "Stress" tests - -.stress: &stress_job - stage: performance - script: - - *configure - - *setup_interfaces - - make -j${BUILD_PARALLEL_JOBS:-1} -k all V=1 - - make DESTDIR="${INSTALL_PATH}" install - - git clone --depth 1 https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.isc.org/isc-private/bind-qa.git - - cd bind-qa/bind9/stress - - LD_LIBRARY_PATH="${INSTALL_PATH}/usr/local/lib" BIND_INSTALL_PATH="${INSTALL_PATH}/usr/local" WORKSPACE="${CI_PROJECT_DIR}" bash stress.sh - needs: - - job: autoreconf - artifacts: true - artifacts: - untracked: true - expire_in: "1 day" - when: always - timeout: 2h - -stress:authoritative:fedora:36:amd64: - <<: *fedora_36_amd64_image - <<: *linux_stress_amd64 - <<: *stress_job - variables: - CC: gcc - FLAME: /usr/bin/flame - MODE: authoritative - RATE: 10000 - RUN_TIME: 1 - only: - variables: - - $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /authoritative/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i) - -stress:recursive:fedora:36:amd64: - <<: *fedora_36_amd64_image - <<: *linux_stress_amd64 - <<: *stress_job - variables: - CC: gcc - FLAME: /usr/bin/flame - MODE: recursive - RATE: 10000 - RUN_TIME: 1 - only: - variables: - - $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /recursive/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i) - -stress:rpz:fedora:36:amd64: - <<: *fedora_36_amd64_image - <<: *linux_stress_amd64 - <<: *stress_job - variables: - CC: gcc - FLAME: /usr/bin/flame - MODE: rpz - RATE: 1500 - RUN_TIME: 1 - only: - variables: - - $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /rpz/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i) - -stress:authoritative:fedora:36:arm64: - <<: *fedora_36_arm64_image - <<: *linux_stress_arm64 - <<: *stress_job - variables: - CC: gcc - FLAME: /usr/bin/flame - MODE: authoritative - RATE: 10000 - RUN_TIME: 1 - only: - variables: - - $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /authoritative/i && $BIND_STRESS_TEST_ARCH =~ /arm64/i) - -stress:recursive:fedora:36:arm64: - <<: *fedora_36_arm64_image - <<: *linux_stress_arm64 - <<: *stress_job - variables: - CC: gcc - FLAME: /usr/bin/flame - MODE: recursive - RATE: 10000 - RUN_TIME: 1 - only: - variables: - - $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /recursive/i && $BIND_STRESS_TEST_ARCH =~ /arm64/i) - -stress:rpz:fedora:36:arm64: - <<: *fedora_36_arm64_image - <<: *linux_stress_arm64 - <<: *stress_job - variables: - CC: gcc - FLAME: /usr/bin/flame - MODE: rpz - RATE: 1500 - RUN_TIME: 1 - only: - variables: - - $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /linux/i && $BIND_STRESS_TEST_MODE =~ /rpz/i && $BIND_STRESS_TEST_ARCH =~ /arm64/i) - -stress:authoritative:freebsd12:amd64: - <<: *freebsd_12_amd64_image - <<: *freebsd_stress_amd64 - <<: *stress_job - variables: - CC: clang - FLAME: /usr/local/bin/flame - MODE: authoritative - RATE: 10000 - RUN_TIME: 1 - only: - variables: - - $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /freebsd/i && $BIND_STRESS_TEST_MODE =~ /authoritative/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i) - -stress:recursive:freebsd12:amd64: - <<: *freebsd_12_amd64_image - <<: *freebsd_stress_amd64 - <<: *stress_job - variables: - CC: clang - FLAME: /usr/local/bin/flame - MODE: recursive - RATE: 10000 - RUN_TIME: 1 - only: - variables: - - $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /freebsd/i && $BIND_STRESS_TEST_MODE =~ /recursive/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i) - -stress:rpz:freebsd12:amd64: - <<: *freebsd_12_amd64_image - <<: *freebsd_stress_amd64 - <<: *stress_job - variables: - CC: clang - FLAME: /usr/local/bin/flame - MODE: rpz - RATE: 1500 - RUN_TIME: 1 - only: - variables: - - $CI_COMMIT_TAG || ($BIND_STRESS_TEST_OS =~ /freebsd/i && $BIND_STRESS_TEST_MODE =~ /rpz/i && $BIND_STRESS_TEST_ARCH =~ /amd64/i) - -gcov: - <<: *base_image - <<: *default_triggering_rules - stage: postcheck - needs: - - job: system:gcc:bullseye:amd64 + - job: docs artifacts: true - script: - # *.gcno and *.gcda files generated for shared library objects are created - # in directories in which gcovr is unable to process them properly - # (.../.libs/...). Move such *.gcno and *.gcda files one level higher. - - find . -regex ".*/\.libs/.*\.\(gcda\|gcno\)" -execdir mv "{}" .. \; - # Help gcovr process the nasty tricks in lib/dns/code.h, where we include C - # source files from lib/dns/rdata/*/, using an even nastier trick. - - find lib/dns/rdata/* -name "*.c" -execdir cp -f "{}" ../../ \; - # Help gcovr process inline functions in headers - - cp -f lib/dns/include/dns/*.h lib/dns/ - - cp -f lib/dns/include/dns/*.h lib/ns/ - - cp -f lib/isc/include/isc/*.h lib/isc/ - - cp -f lib/isc/include/isc/*.h lib/dns/ - - cp -f lib/isc/include/isc/*.h lib/ns/ - - find bin lib -maxdepth 1 -mindepth 1 -type d -exec cp -f lib/isc/include/isc/buffer.h "{}" \; - - cp -f lib/isc/include/isc/buffer.h fuzz/ - - cp -f lib/isc/include/isc/buffer.h lib/isc/netmgr/buffer.h - - cp -f lib/isc/include/isc/hash.h lib/dns/hash.h - # Generate XML file in the Cobertura XML format suitable for use by GitLab - # for the purpose of displaying code coverage information in the diff view - # of a given merge request. - - gcovr --root . --exclude-directories bin/tests --exclude-directories doc --exclude-directories lib/samples --exclude tests --xml -o coverage.xml - - gcovr --root . --exclude-directories bin/tests --exclude-directories doc --exclude-directories lib/samples --exclude tests --html-details -o coverage.html - - gcovr --root . --exclude-directories bin/tests --exclude-directories doc --exclude-directories lib/samples --exclude tests -o coverage.txt - - tail -n 3 coverage.txt - artifacts: - paths: - - coverage*.html - - coverage.txt - - coverage.xml - reports: - coverage_report: - coverage_format: cobertura - path: coverage.xml - -# Pairwise testing of ./configure options -pairwise: - <<: *base_image - stage: build - needs: - - job: autoreconf - artifacts: true - script: - - util/pairwise-testing.sh - artifacts: - paths: - - pairwise-commands.txt - - pairwise-model.txt - - pairwise-output.*.txt - when: on_failure - only: - variables: - - $PAIRWISE_TESTING diff --git a/doc/arm/chapter3.rst b/doc/arm/chapter3.rst index 509dc489e6da1557fb491feebba29f85e4794e58..d011d98b151dc687aa6bb3bc912d594e0f7c7aee 100644 --- a/doc/arm/chapter3.rst +++ b/doc/arm/chapter3.rst @@ -14,4 +14,5 @@ .. include:: config-intro.inc.rst .. include:: config-auth.inc.rst .. include:: config-resolve.inc.rst +.. include:: load-balance.inc.rst .. include:: zones.inc.rst diff --git a/doc/arm/config-resolve.inc.rst b/doc/arm/config-resolve.inc.rst index fcfa97e20d7f79ebbc11d6d54e77cfbf0095595a..50fd7c4f3982baf55a9c932335b3f1f9a86d37e3 100644 --- a/doc/arm/config-resolve.inc.rst +++ b/doc/arm/config-resolve.inc.rst @@ -19,7 +19,7 @@ complete answers; that is, they issue one or more :ref:`iterative queries ` to the DNS hierarchy. Having obtained a complete answer (or an error), a resolver passes the answer to the user and places it in its cache. Subsequent user requests for the same query will be answered from the -resolver's cache until the :term:`TTL` of the cached answer has expired, when +resolver's cache until the time-to-live (TTL) of the cached answer has expired, when it will be flushed from the cache; the next user query that requests the same information results in a new series of queries to the DNS hierarchy. @@ -529,38 +529,5 @@ hierarchy (does not use the public network) for any recursive query for which: 6. Is a query for the domain name **example.com**, in which case it will be forwarded to either 192.168.250.3 or 192.168.230.27 (zone example.com). -All other recursive queries will result in access to the DNS hierarchy to +All other recursive queries result in access to the DNS hierarchy to resolve the query. - -.. _load_balancing: - -Load Balancing --------------- - -A primitive form of load balancing can be achieved in the DNS by using multiple -resource records (RRs) in a :ref:`zone file` (such as multiple A -records) for one name. - -For example, assuming three HTTP servers with network addresses of -10.0.0.1, 10.0.0.2, and 10.0.0.3, a set of records such as the following -means that clients will connect to each machine one-third of the time: - -+-----------+------+----------+----------+----------------------------+ -| Name | TTL | CLASS | TYPE | Resource Record (RR) Data | -+-----------+------+----------+----------+----------------------------+ -| www | 600 | IN | A | 10.0.0.1 | -+-----------+------+----------+----------+----------------------------+ -| | 600 | IN | A | 10.0.0.2 | -+-----------+------+----------+----------+----------------------------+ -| | 600 | IN | A | 10.0.0.3 | -+-----------+------+----------+----------+----------------------------+ - -When a resolver queries for these records, BIND rotates them and -responds to the query with the records in a random order. In the -example above, clients randomly receive records in the order 1, 2, -3; 2, 3, 1; and 3, 1, 2. Most clients use the first record returned -and discard the rest. - -For more detail on ordering responses, refer to the -:ref:`rrset-order` statement in the -:namedconf:ref:`options` block. diff --git a/doc/arm/load-balance.inc.rst b/doc/arm/load-balance.inc.rst new file mode 100644 index 0000000000000000000000000000000000000000..8cecb1e1942b5c1be31a009630c75b16f851e35d --- /dev/null +++ b/doc/arm/load-balance.inc.rst @@ -0,0 +1,192 @@ +.. Copyright (C) Internet Systems Consortium, Inc. ("ISC") +.. +.. SPDX-License-Identifier: MPL-2.0 +.. +.. This Source Code Form is subject to the terms of the Mozilla Public +.. License, v. 2.0. If a copy of the MPL was not distributed with this +.. file, you can obtain one at https://mozilla.org/MPL/2.0/. +.. +.. See the COPYRIGHT file distributed with this work for additional +.. information regarding copyright ownership. +.. _load_balancing: + +Load Balancing +-------------- + +Load balancing distributes client service requests across a group of server machines, +to reduce the overall load on any one server. There are many ways to achieve +load balancing: + + - The client-side method is to publish DNS records with support for load-balancing + (such as :ref:`HTTPS ` records, :ref:`MX ` + records, or :ref:`SRV ` records). + - The :ref:`tailored-response method ` takes + advantage of built-in BIND 9 features, such as GeoIPs within :any:`acl` + blocks and :any:`view` blocks, to send different clients tailored responses. + - The :ref:`last-resort ` option uses the :any:`rrset-order` and + :any:`sortlist` features in BIND. + +Each approach is described in the following sections and the limits to each are +identified. Generic limitations are described in the section +:ref:`balancing_caveats`. + +.. note:: + This section deals with the use of DNS to balance end-user services. + Load balancing of the DNS service itself is not addressed by these techniques. + +.. _https_balance: + +Balancing Web Traffic (HTTPS Records) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +TODO + +.. _mx_balance: + +Balancing Mail (MX Records) +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Sharing load between multiple mail servers is controlled by +:ref:`MX` resource records. These records contain a *preference* +value. One primary use of this value is to achieve resilience of the mail +service by designating a primary server and one or more secondary, or backup, +servers. The :ref:`MX` resource record of the primary server is +given a low *preference* value and the :ref:`MX` resource record of +the secondary server(s) is given higher *preference* values. *Preference* can +therefore be regarded more like a *cost*; the lowest-cost server is preferred. + +However, *preference* can also be used to achieve load balancing between two or +more mail servers by assigning them the same value; for example: + +.. code-block:: none + + ; zone file fragment + @ MX 10 mail.example.com. + @ MX 10 mail1.example.com. + @ MX 10 mail2.example.com. + ... + mail A 192.168.0.4 + mail1 A 192.168.0.5 + mail2 A 192.168.0.6 + +**mail**, **mail1**, and **mail2** are all considered to have equal preference, +or cost. The authoritative name server delivers the MX records in the order +defined by the :any:`rrset-order` statement, and the receiving +SMTP software selects one based on its algorithm. In some cases the SMTP +selection algorithm may work against the definition of the RRset-order +statement. + +To avoid problems if the receiving mail system does +reverse lookups as a spam check, define the PTR records for 192.168.0.4, +192.168.0.5, and 192.168.0.6 to mail.example.com. + +.. note:: + In the above situation, each mail server must be capable of handling and + synchronizing the load for all the mailboxes served by the domain, This + can be accomplished either using some appropriate back-end or by access + to a common file system (NAS, NFS, etc.), or by defining all but one + server to be a mail relay or forwarder. + +.. _srv_balance: + +Balancing Other Services (SRV Records) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The :ref:`SRV` resource record allows an application to **discover** +the server name or names (and optional port number) on which a service - SIP or +LDAP, for example - is provided. As such, it offers another approach to load +balancing. SRV RRs contain both *priority* and *weight* fields, allowing a fine +level of granular configuration as well as providing some level of failover. +However, the end application must be **SRV-aware** for this approach to work. +Application support for SRV is patchy at best - varying from very high in SIP +(VoIP) to non-existent (browsers). + +.. _last_resort_balance: + +Last-Resort Option (A/AAAA records) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some services do not have a service-specific record type or domain name, and rely +on A/AAAA records to map the service name to addresses. +If the requirement is to load-share services without a specialized resource record, +then defining multiple A/AAAA records with the same name and different IP +addresses, as in the example below, can be used as an **imperfect workaround**. +Please note this technique relies on quirks in client implementations and is +not reliable. + +.. note:: + This legacy method is still in use for HTTP traffic, but is + becoming obsolete as :ref:`HTTPS ` resource record support in + clients is rolled out. + +It is best illustrated in a simple zone file: + +.. code-block:: none + + ; zone file fragment + + ftp A 192.168.0.4 + A 192.168.0.5 + A 192.168.0.6 + ... + www A 192.168.0.7 + A 192.168.0.8 + +The authoritative name server delivers all the IP addresses from the zone file; +the first IP address in the returned list is defined according to the value +of the :any:`rrset-order` or :any:`sortlist` statements. The **ftp** and **www** +servers must all be exact (synchronized) replicas of each other in this scenario. + +.. warning:: + Use this method only as a last-resort option. + Resource record sets, by DNS protocol definition, can be reordered at any + time. Intermediate resolvers might reorder records and ruin any + load-balancing attempts. Similarly, the client side is allowed to reorder records + at will. + +.. _tailored_responses: + +Balancing Services with Split Horizon (GeoIP) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +All application-specific approaches listed above can be combined with BIND's +:any:`view` feature to create a split-horizon (or GeoIP-aware) configuration. +Split horizon uses the client's source IP address to respond with a specific +set of records, thus balancing for geographic or even service +provider-specific traffic sources (please see :ref:`Example Split-Horizon +Configuration`). + +.. _balancing_caveats: + +Effectiveness of DNS Service Load Balancing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The previous sections have addressed some of the techniques that may be used +to balance service load using DNS functionality. However, the following points +should also be considered: + +1. Data supplied from the authoritative name server will reflect both the +zone file content, such as multiple RRs, and any BIND 9 operational control +statements, such as :any:`rrset-order` and :any:`sortlist`. + +2. When this data is cached by a resolver and subsequently supplied from its +cache, two consequences apply: + + a. The order in which multiple IPs appear **can change** within + the resolver's cache; it is no longer controlled by the authoritative name + server's policies. If data is supplied from too few + caches, any balancing effect may become distorted. + + b. The resolver may be configured with its own policies using + :any:`rrset-order` or the (relatively rare) :any:`sortlist` + statement, which may distort the aims of the authoritative name server. + + c. Changes on the authoritative side might not take effect until the :term:`TTL` + expires. + +3. To account for server load or availability, data on the authoritative server + must be modified using :ref:`dynamic_update`. For instance, certain + transactions may generate very high CPU or resource loads, or certain servers + in a set may simply be unavailable. For this type of control only a local load + balancer - one which measures service response times, server loading, and + potentially other metrics - must modify the content of the DNS zone, and the + dynamically modified records should use sufficiently low :term:`TTL` values. diff --git a/doc/arm/reference.rst b/doc/arm/reference.rst index ff36672cc4d8f1d60ca79bf3a9940ffcb41c1db0..1e1b060c7445a50e37fb3a4047862381bb19dfb5 100644 --- a/doc/arm/reference.rst +++ b/doc/arm/reference.rst @@ -4003,13 +4003,20 @@ Periodic Task Intervals gone away. For convenience, TTL-style time-unit suffixes may be used to specify the value. It also accepts ISO 8601 duration formats. -The :any:`sortlist` Statement -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +RRset Ordering +^^^^^^^^^^^^^^ + +.. note:: + + While alternating the order of records in a DNS response between + subsequent queries is a known load-distribution technique, certain + caveats apply which usually make it a suboptimal choice for load balancing + purposes when used on its own. See :ref:`balancing_caveats`. The response to a DNS query may consist of multiple resource records (RRs) forming a resource record set (RRset). The name server normally returns the RRs within the RRset in an indeterminate order (but -see the :any:`rrset-order` statement in :ref:`rrset_ordering`). The client resolver code should +see the :any:`rrset-order` statement). The client resolver code should rearrange the RRs as appropriate: that is, using any addresses on the local net in preference to other addresses. However, not all resolvers can do this or are correctly configured. When a client is using a local @@ -4089,18 +4096,6 @@ are not sorted. { localnets; }; }; -.. _rrset_ordering: - -RRset Ordering -^^^^^^^^^^^^^^ - -.. note:: - - While alternating the order of records in a DNS response between - subsequent queries is a known load distribution technique, certain - caveats apply (mostly stemming from caching) which usually make it a - suboptimal choice for load balancing purposes when used on its own. - .. namedconf:statement:: rrset-order :tags: query :short: Defines the order in which equal RRs (RRsets) are returned. diff --git a/doc/arm/zones.inc.rst b/doc/arm/zones.inc.rst index 1807029cac15476850309216ad46d337d407e5d1..2fda5a99c58f89b13a3d9d3cf69d1c20a2eacbeb 100644 --- a/doc/arm/zones.inc.rst +++ b/doc/arm/zones.inc.rst @@ -13,11 +13,15 @@ .. _soa_rr: +.. _srv_rr: + Zone File --------- This section, largely borrowed from :rfc:`1034`, describes the concept of a -Resource Record (RR) and explains how to use them. +Resource Record (RR) and explains when each type is used. Since the +publication of :rfc:`1034`, dozens of new RRs have been identified and +implemented in the DNS. Most notable ones are included here. Resource Records ~~~~~~~~~~~~~~~~ @@ -26,10 +30,7 @@ A domain name identifies a node in the DNS tree namespace. Each node has a set o information, which may be empty. The set of resource information associated with a particular name is composed of separate RRs. The order of RRs in a set is not significant and need not be preserved by name -servers, resolvers, or other parts of the DNS. However, sorting of -multiple RRs is permitted for optimization purposes: for example, to -specify that a particular nearby server be tried first. See -:any:`sortlist` and :ref:`rrset_ordering`. +servers, resolvers, or other parts of the DNS. The components of a Resource Record are: