.gitlab-ci.yml 10 KB
Newer Older
1
image: registry.gitlab.isc.org/isc-projects/stork/ci-base:latest
2

3 4 5 6 7 8
# Only create pipelines for merge requests and pushes/merges to the main branch
workflow:
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'

9
variables:
10
  POSTGRES_ADDR: postgres:5432
11 12 13
  POSTGRES_DB: storktest
  POSTGRES_USER: storktest
  POSTGRES_PASSWORD: storktest
14
  POSTGRES_IN_DOCKER: 'yes'
15

16 17 18 19 20
# this is used to build docker images of stork (using DIND)
  DOCKER_DRIVER: overlay2
  DOCKER_TLS_CERTDIR: ""  # this is workaround for TLS problem https://about.gitlab.com/blog/2019/07/31/docker-in-docker-with-docker-19-dot-03/

# stages order
21 22
stages:
  - build
23
  - test
24
  - deploy
25

26
# cache
27
cache:
28
  key: one-shared-key-2
29 30 31 32
  paths:
  - webui/node_modules/
  - tools/

33

34
### build jobs ###
35

36 37
# common parts used in build jobs
.base_build:
38
  stage: build
39 40
  rules:
    - when: always
41 42 43
  tags:
    - linux
    - amd64
44
    - ssd
45
  before_script:
46 47
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
48 49 50
    - apt-get update
    - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends wget xz-utils rake openjdk-11-jre-headless gcc g++ chromium-browser unzip python3-sphinx python3-sphinx-rtd-theme iputils-ping
    - rake prepare_env
51 52 53

lint_go:
  extends: .base_build
54 55 56
  script:
    - rake lint_go

57
unittest_backend:
58
  extends: .base_build
59
  services:
60 61
    - name: registry.gitlab.isc.org/isc-projects/stork/ci-postgres:11
      alias: postgres
62
  script:
63 64 65
    - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends postgresql-client
    - echo 'postgres:*:*:storktest:storktest' > ~/.pgpass
    - chmod 600 ~/.pgpass
66
    - rake unittest_backend
67

68
ci_ui:
69
  extends: .base_build
70 71 72 73
  script:
    - rake ci_ui

build_webui:
74
  extends: .base_build
75 76
  script:
    - rake build_ui
77 78 79 80 81 82 83
  artifacts:
    name: "webui"
    expire_in: 1 week
    paths:
      - webui/dist/stork/

build_backend:
84
  extends: .base_build
85 86 87 88 89 90 91 92 93
  script:
    - rake build_backend
  artifacts:
    name: "backend"
    expire_in: 1 week
    paths:
      - backend/cmd/stork-agent/stork-agent
      - backend/cmd/stork-server/stork-server
      - backend/cmd/stork-db-migrate/stork-db-migrate
Michal Nowikowski's avatar
Michal Nowikowski committed
94 95 96

danger:
  stage: build
97 98
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
99
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'
100 101 102
  tags:
    - linux
    - amd64
103
    - ssd
104
  image: registry.gitlab.isc.org/isc-projects/stork/ci-danger
Michal Nowikowski's avatar
Michal Nowikowski committed
105 106 107 108 109
  before_script:
    - export CI_MERGE_REQUEST_ID=$(git ls-remote -q origin merge-requests\*\head | grep $CI_COMMIT_SHA | sed 's/.*refs\/merge-requests\/\([0-9]*\)\/head/\1/g')
    - export CI_PROJECT_PATH=$CI_PROJECT_ID #some version of gitlab has problems with searching by project path
    - export DANGER_GITLAB_HOST=gitlab.isc.org
    - export DANGER_GITLAB_API_BASE_URL=https://gitlab.isc.org/api/v4
110
  script:
Michal Nowikowski's avatar
Michal Nowikowski committed
111 112 113 114
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
    - gem install danger-commit_lint
    - danger --fail-on-errors=true --new-comment
115 116 117

tarball:
  stage: build
118 119
  rules:
    - when: always
120 121 122
  tags:
    - linux
    - amd64
123
    - ssd
124 125 126 127 128 129
  script:
    - rake tarball
  artifacts:
    paths:
      - stork-*.tar.gz
    expire_in: 1 week
130 131


132 133
### build rpm & deb packages ###

134
debs_and_rpms:
135
  stage: build
136 137
  rules:
    - when: always
138
  image: stork-tests-ubuntu-18.04-x86_64
139
  tags:
140
    - libvirt
141
    - amd64
142 143 144
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
145 146 147 148 149 150
    - apt-get update
    - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ruby ruby-dev rubygems build-essential git wget unzip apt-transport-https ca-certificates curl gnupg-agent software-properties-common
    - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
    - add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
    - apt-get update
    - DEBIAN_FRONTEND=noninteractive apt-get install -y docker-ce docker-ce-cli containerd.io
151
  script:
152 153
    - docker info
    - rake build_pkgs_in_docker
154 155
  artifacts:
    paths:
156
      - isc-stork-*
157 158 159
    expire_in: 1 week


160 161 162 163
### system testing ###

system_testing:
  stage: test
164 165
  rules:
    - when: on_success
166
  image: stork-tests-ubuntu-18.04-x86_64
167
  tags:
168
    - libvirt
169
    - amd64
170
  needs:
171
    - debs_and_rpms
172 173 174 175 176
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
  script:
    - apt-get update
177
    - DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3-venv rake net-tools firefox
178 179 180 181 182 183 184 185 186 187 188
    # - ifconfig
    # - lxc --version
    # - lxc network list
    # - lxc network show lxdbr0
    # - lxc network set lxdbr0 ipv4.address auto
    # - lxc network set lxdbr0 ipv4.dhcp true
    # - lxc network set lxdbr0 ipv6.address none
    # - lxc network show lxdbr0
    # - lxc profile show default
    # - cat /etc/netplan/*
    # - cat /etc/network/interfaces
189
    # - rake system_tests_ui
190
    - rake system_tests
191 192


193 194 195 196 197
### upload release notes and tarball to repo.isc.org ###

deploy_to_repo:
  stage: deploy
  when: manual
198
  allow_failure: true
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239
  tags:
    - linux
    - amd64
    - ssd
    - stork-repo
  needs:
    - tarball
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
  script:
    - export LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
    - ls -al
    # prepare release notes file
    - git clone https://gitlab.isc.org/isc-projects/stork.wiki
    - stork_version=`perl -lne 'print $1 if /(\d+\.\d+\.\d+)/' < backend/version.go`
    - stork_version_minor=`perl -lne 'print $1 if /"(\d+\.\d+)\.\d+/' < backend/version.go`
    - output=Stork-${stork_version}-ReleaseNotes.txt
    - fold -sw 73 < stork.wiki/*eleases/*${stork_version_minor}.md > $output
    # upload release notes and tarball to repo.isc.org
    - eval $(ssh-agent -s)
    - echo "$REPO_SSH_PRIVATE_KEY" | base64 -d | ssh-add -
    - mkdir -p ~/.ssh
    - chmod 0700 ~/.ssh
    - touch ~/.ssh/known_hosts
    - ssh-keyscan repo.isc.org >> ~/.ssh/known_hosts
    - chmod 644 ~/.ssh/known_hosts
    - destdir=/data/shared/sweng/stork/releases/${stork_version}
    - ssh -4 storkpush@repo.isc.org mkdir -p $destdir
    - scp -4 stork-${stork_version}.tar.gz $output storkpush@repo.isc.org:$destdir/
    - ssh -4 storkpush@repo.isc.org chmod -R g+w $destdir
    - ssh -4 storkpush@repo.isc.org ls -al $destdir
    - ssh -4 storkpush@repo.isc.org sha256 $destdir/stork-*.tar.gz
    # TODO:
    # - automate pasting ChangeLog.md to release notes
  artifacts:
    name: "release-notes"
    expire_in: 1 week
    paths:
      - Stork-*-ReleaseNotes.txt

240
### demo deploy jobs ###
241 242

deploy_demo:
243
  stage: deploy
244 245 246 247
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
      when: manual
    - if: '$CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH'
248 249
      when: always
  allow_failure: true
250 251 252 253 254
  image: docker:latest
  tags:
    - linux
    - docker
    - amd64
255
    - ssd
256 257
  needs:
    - danger
258 259 260 261 262
  services:
    - docker:dind
  before_script:
    - ip -6 route del default
    - docker info
263
    - docker login -u stork-ci -p "$STORK_CI_TOKEN_FOR_REGISTRY" "$CI_REGISTRY"
264 265
    - apk update
    - apk upgrade
266
    - apk add docker-compose ruby-rake
267 268 269
    - eval $(ssh-agent -s)
    - echo "$SSH_PRIVATE_KEY_STORK" | base64 -d | ssh-add -
  script:
270
    - rake build_all_in_container
271
    - docker-compose -f docker-compose.yaml -f docker-compose-premium.yaml build --build-arg CS_REPO_ACCESS_TOKEN=$CLOUDSMITH_REPO_ACCESS_TOKEN
272
    - docker-compose -f docker-compose.yaml -f docker-compose-premium.yaml push
273
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org docker-compose -f deploy/docker-compose.yaml -f deploy/docker-compose-premium.yaml down -v --remove-orphans
274
    - scp -o StrictHostKeyChecking=no docker-compose.yaml jenkins@stork.lab.isc.org:~/deploy
275
    - scp -o StrictHostKeyChecking=no docker-compose-premium.yaml jenkins@stork.lab.isc.org:~/deploy
276 277 278 279 280
    - scp -o StrictHostKeyChecking=no docker/prometheus.yml jenkins@stork.lab.isc.org:~/deploy/docker/prometheus.yml
    - scp -o StrictHostKeyChecking=no grafana/kea-dhcp4.json jenkins@stork.lab.isc.org:~/deploy/grafana/kea-dhcp4.json
    - scp -o StrictHostKeyChecking=no docker/grafana/dashboards.yaml jenkins@stork.lab.isc.org:~/deploy/docker/grafana/dashboards.yaml
    - scp -o StrictHostKeyChecking=no docker/grafana/datasource.yaml jenkins@stork.lab.isc.org:~/deploy/docker/grafana/datasource.yaml
    - scp -o StrictHostKeyChecking=no docker/grafana/grafana.ini jenkins@stork.lab.isc.org:~/deploy/docker/grafana/grafana.ini
281 282
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org docker-compose -f deploy/docker-compose.yaml -f deploy/docker-compose-premium.yaml pull
    - ssh -o StrictHostKeyChecking=no jenkins@stork.lab.isc.org docker-compose -f deploy/docker-compose.yaml -f deploy/docker-compose-premium.yaml up -d --no-build
283 284 285 286 287 288


### deploy pkgs to cloudsmith.io ###

deploy_pkgs:
  stage: deploy
289 290 291
  rules:
    - when: manual
  allow_failure: true
292 293 294 295 296
  image: registry.gitlab.isc.org/isc-projects/stork/pkgs-cloudsmith:latest
  tags:
    - linux
    - docker
    - amd64
297
    - ssd
298
  needs:
299
    - debs_and_rpms
300 301 302 303 304 305 306 307 308 309 310 311 312
  before_script:
    - sysctl -w net.ipv6.conf.all.disable_ipv6=1
    - sysctl -w net.ipv6.conf.default.disable_ipv6=1
  script:
    - ls -al
    - export LANG='en_US.UTF-8' LANGUAGE='en_US:en' LC_ALL='en_US.UTF-8'
    - cloudsmith check service
    - cloudsmith whoami -k $CLOUDSMITH_API_KEY
    - if [ -z "$REPO" ]; then export REPO="stork-testing"; fi
    - cloudsmith upload deb -k $CLOUDSMITH_API_KEY -W --republish isc/$REPO/any-distro/any-version isc-stork-agent*deb
    - cloudsmith upload deb -k $CLOUDSMITH_API_KEY -W --republish isc/$REPO/any-distro/any-version isc-stork-server*deb
    - cloudsmith upload rpm -k $CLOUDSMITH_API_KEY -W --republish isc/$REPO/any-distro/any-version isc-stork-agent*rpm
    - cloudsmith upload rpm -k $CLOUDSMITH_API_KEY -W --republish isc/$REPO/any-distro/any-version isc-stork-server*rpm