Rakefile 29.9 KB
Newer Older
1
# coding: utf-8
2 3 4
require 'rake'

# Tool Versions
5
NODE_VER = '12.16.2'
6
SWAGGER_CODEGEN_VER = '2.4.13'
7
GOSWAGGER_VER = 'v0.23.0'
8
GOLANGCILINT_VER = '1.21.0'
9
GO_VER = '1.13.5'
10 11
PROTOC_VER = '3.11.2'
PROTOC_GEN_GO_VER = 'v1.3.3'
12

13 14 15 16 17 18
# Check host OS
UNAME=`uname -s`

case UNAME.rstrip
  when "Darwin"
    OS="macos"
19
    GOSWAGGER_BIN="swagger_darwin_amd64"
20 21 22 23
    GO_SUFFIX="darwin-amd64"
    PROTOC_ZIP_SUFFIX="osx-x86_64"
    NODE_SUFFIX="darwin-x64"
    GOLANGCILINT_SUFFIX="darwin-amd64"
24
    CHROME_BIN="/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome"
25 26
    puts "WARNING: MacOS is not officially supported, the provisions for building on MacOS are made"
    puts "WARNING: for the developers' convenience only."
27 28
  when "Linux"
    OS="linux"
29
    GOSWAGGER_BIN="swagger_linux_amd64"
30 31 32 33
    GO_SUFFIX="linux-amd64"
    PROTOC_ZIP_SUFFIX="linux-x86_64"
    NODE_SUFFIX="linux-x64"
    GOLANGCILINT_SUFFIX="linux-amd64"
34
    CHROME_BIN="/usr/bin/chromium-browser"
35 36
  when "FreeBSD"
    OS="FreeBSD"
37
    # TODO: there are no swagger built packages for FreeBSD
38 39
    GOSWAGGER_BIN=""
    puts "WARNING: There are no FreeBSD packages for GOSWAGGER_BIN"
40
    GO_SUFFIX="freebsd-amd64"
41
    # TODO: there are no protoc built packages for FreeBSD (at least as of 3.10.0)
42
    PROTOC_ZIP_SUFFIX=""
43
    puts "WARNING: There are no protoc packages built for FreeBSD"
44 45
    NODE_SUFFIX="node-v10.16.3.tar.xz"
    GOLANGCILINT_SUFFIX="freebsd-amd64"
46
  else
47
    puts "ERROR: Unknown/unsupported OS: %s" % UNAME
48 49 50
    fail
  end

51
# Tool URLs
52
GOSWAGGER_URL = "https://github.com/go-swagger/go-swagger/releases/download/#{GOSWAGGER_VER}/#{GOSWAGGER_BIN}"
53 54 55
GOLANGCILINT_URL = "https://github.com/golangci/golangci-lint/releases/download/v#{GOLANGCILINT_VER}/golangci-lint-#{GOLANGCILINT_VER}-#{GOLANGCILINT_SUFFIX}.tar.gz"
GO_URL = "https://dl.google.com/go/go#{GO_VER}.#{GO_SUFFIX}.tar.gz"
PROTOC_URL = "https://github.com/protocolbuffers/protobuf/releases/download/v#{PROTOC_VER}/protoc-#{PROTOC_VER}-#{PROTOC_ZIP_SUFFIX}.zip"
56
PROTOC_GEN_GO_URL = 'github.com/golang/protobuf/protoc-gen-go'
57
SWAGGER_CODEGEN_URL = "https://oss.sonatype.org/content/repositories/releases/io/swagger/swagger-codegen-cli/#{SWAGGER_CODEGEN_VER}/swagger-codegen-cli-#{SWAGGER_CODEGEN_VER}.jar"
58
NODE_URL = "https://nodejs.org/dist/v#{NODE_VER}/node-v#{NODE_VER}-#{NODE_SUFFIX}.tar.xz"
59
MOCKERY_URL = 'github.com/vektra/mockery/.../@v1.0.0'
60 61 62 63
MOCKGEN_URL = 'github.com/golang/mock/mockgen'
RICHGO_URL = 'github.com/kyoh86/richgo'

# Tools and Other Paths
64
TOOLS_DIR = File.expand_path('tools')
65
NPX = "#{TOOLS_DIR}/node-v#{NODE_VER}-#{NODE_SUFFIX}/bin/npx"
66
SWAGGER_CODEGEN = "#{TOOLS_DIR}/swagger-codegen-cli-#{SWAGGER_CODEGEN_VER}.jar"
67 68
GOSWAGGER_DIR = "#{TOOLS_DIR}/#{GOSWAGGER_VER}"
GOSWAGGER = "#{GOSWAGGER_DIR}/#{GOSWAGGER_BIN}"
Michal Nowikowski's avatar
Michal Nowikowski committed
69
NG = File.expand_path('webui/node_modules/.bin/ng')
70 71 72 73 74
if ENV['GOPATH']
  GOHOME_DIR = ENV['GOPATH']
else
  GOHOME_DIR = File.expand_path('~/go')
end
75
GOBIN = "#{GOHOME_DIR}/bin"
76 77
GO_DIR = "#{TOOLS_DIR}/#{GO_VER}"
GO = "#{GO_DIR}/go/bin/go"
78
GOLANGCILINT = "#{TOOLS_DIR}/golangci-lint-#{GOLANGCILINT_VER}-#{GOLANGCILINT_SUFFIX}/golangci-lint"
79 80
PROTOC_DIR = "#{TOOLS_DIR}/#{PROTOC_VER}"
PROTOC = "#{PROTOC_DIR}/bin/protoc"
81
PROTOC_GEN_GO = "#{GOBIN}/protoc-gen-go-#{PROTOC_GEN_GO_VER}"
82 83 84 85
MOCKERY = "#{GOBIN}/mockery"
MOCKGEN = "#{GOBIN}/mockgen"
RICHGO = "#{GOBIN}/richgo"

Marcin Siodelski's avatar
Marcin Siodelski committed
86
# wget
87
if system("wget --version > /dev/null").nil?
Marcin Siodelski's avatar
Marcin Siodelski committed
88 89 90 91 92 93 94 95 96 97
  abort("wget is not installed on this system")
end
# extract wget version
WGET_VERSION = `wget --version | head -n 1 | sed -E 's/[^0-9]*([0-9]*\.[0-9]*)[^0-9]+.*/\1/g'`
# versions prior to 1.19 lack support for --retry-on-http-error
if !WGET_VERSION.empty? and WGET_VERSION < "1.19"
  WGET = 'wget --tries=inf --waitretry=3'
else
  WGET = 'wget --tries=inf --waitretry=3 --retry-on-http-error=429,500,503,504 '
end
98

99
# Patch PATH env
100
ENV['PATH'] = "#{TOOLS_DIR}/node-v#{NODE_VER}-#{NODE_SUFFIX}/bin:#{ENV['PATH']}"
101
ENV['PATH'] = "#{GO_DIR}/go/bin:#{ENV['PATH']}"
102 103
ENV['PATH'] = "#{GOBIN}:#{ENV['PATH']}"

104 105 106 107 108 109
# premium support
if ENV['cs_repo_access_token']
  ENV['premium'] = 'true'
end
if ENV['premium'] == 'true'
  DOCKER_COMPOSE_FILES='-f docker-compose.yaml -f docker-compose-premium.yaml'
110
  DOCKER_COMPOSE_PREMIUM_OPTS = "--build-arg CS_REPO_ACCESS_TOKEN=#{ENV['cs_repo_access_token']}"
111 112
else
  DOCKER_COMPOSE_FILES='-f docker-compose.yaml'
113
  DOCKER_COMPOSE_PREMIUM_OPTS = ''
114 115
end

116
# build date
117 118 119 120 121 122 123 124
now = Time.now
build_date = now.strftime("%Y-%m-%d %H:%M")
if ENV['STORK_BUILD_TIMESTAMP']
  TIMESTAMP = ENV['STORK_BUILD_TIMESTAMP']
else
  TIMESTAMP = now.strftime("%y%m%d%H%M%S")
end
puts "Stork build date: #{build_date} (timestamp: #{TIMESTAMP})"
125 126
go_build_date_opt = "-ldflags=\"-X 'isc.org/stork.BuildDate=#{build_date}'\""

127 128 129
# Documentation
SPHINXOPTS = "-v -E -a -W -j 2"

130 131
# Files
SWAGGER_FILE = File.expand_path('api/swagger.yaml')
132 133 134 135
SWAGGER_API_FILES = [
  'api/swagger.in.yaml',
  'api/services-defs.yaml', 'api/services-paths.yaml',
  'api/users-defs.yaml', 'api/users-paths.yaml',
136
  'api/dhcp-defs.yaml', 'api/dhcp-paths.yaml',
137
  'api/settings-defs.yaml', 'api/settings-paths.yaml',
138 139
  'api/search-defs.yaml', 'api/search-paths.yaml',
  'api/events-defs.yaml', 'api/events-paths.yaml'
140
]
141 142 143 144 145 146
AGENT_PROTO_FILE = File.expand_path('backend/api/agent.proto')
AGENT_PB_GO_FILE = File.expand_path('backend/api/agent.pb.go')

SERVER_GEN_FILES = Rake::FileList[
  File.expand_path('backend/server/gen/restapi/configure_stork.go'),
]
Michal Nowikowski's avatar
Michal Nowikowski committed
147

148 149 150 151 152 153 154 155 156 157 158 159 160
# locations for installing files
if ENV['DESTDIR']
  DESTDIR=ENV['DESTDIR']
else
  DESTDIR=File.join(File.dirname(__FILE__), 'root')
end
BIN_DIR=File.join(DESTDIR, 'usr/bin')
UNIT_DIR=File.join(DESTDIR, 'lib/systemd/system')
ETC_DIR=File.join(DESTDIR, 'etc/stork')
WWW_DIR=File.join(DESTDIR, 'usr/share/stork/www')
EXAMPLES_DIR=File.join(DESTDIR, 'usr/share/stork/examples')
MAN_DIR=File.join(DESTDIR, 'usr/share/man/man8')

161 162 163
# Directories
directory GOHOME_DIR
directory TOOLS_DIR
164 165 166 167 168 169 170 171 172
directory DESTDIR
directory BIN_DIR
directory UNIT_DIR
directory ETC_DIR
directory WWW_DIR
directory EXAMPLES_DIR
directory MAN_DIR

# establish Stork version
173
stork_version = '0.0.0'
174 175 176 177 178
version_file = 'backend/version.go'
text = File.open(version_file).read
text.each_line do |line|
  if line.start_with? 'const Version'
    parts = line.split('"')
179
    stork_version = parts[1]
180 181
  end
end
182
STORK_VERSION = stork_version
183

184 185 186 187 188 189 190
# CHROME_BIN is required for UI tests. If it is not set by the
# user, set the default value. It points to the location of the
# Chrome browser binary.
if !ENV['CHROME_BIN']
  ENV['CHROME_BIN'] = CHROME_BIN
end

191
### Backend Tasks #########################
192 193

file GO => [TOOLS_DIR, GOHOME_DIR] do
194
  sh "mkdir -p #{GO_DIR}"
195
  sh "#{WGET} #{GO_URL} -O #{GO_DIR}/go.tar.gz"
196
  Dir.chdir(GO_DIR) do
197 198 199 200
    sh 'tar -zxf go.tar.gz'
  end
end

201 202 203 204 205 206 207 208 209 210 211 212
YAMLINC = File.expand_path('webui/node_modules/.bin/yamlinc')

file YAMLINC do
  Rake::Task[NG].invoke()
end

file SWAGGER_FILE => [YAMLINC, *SWAGGER_API_FILES] do
  Dir.chdir('api') do
    sh "#{YAMLINC} -o swagger.yaml swagger.in.yaml"
  end
end

213
file SERVER_GEN_FILES => SWAGGER_FILE do
214
  Dir.chdir('backend') do
215
    sh "#{GOSWAGGER} generate server -s server/gen/restapi -m server/gen/models --name Stork --exclude-main --spec #{SWAGGER_FILE} --template stratoscale --regenerate-configureapi"
Michal Nowikowski's avatar
Michal Nowikowski committed
216 217 218
  end
end

219 220 221 222
desc 'Generate server part of REST API using goswagger based on swagger.yml'
task :gen_server => [GO, GOSWAGGER, SERVER_GEN_FILES]

file GOSWAGGER => TOOLS_DIR do
223
  sh "mkdir -p #{GOSWAGGER_DIR}"
224
  sh "#{WGET} #{GOSWAGGER_URL} -O #{GOSWAGGER}"
225
  sh "chmod a+x #{GOSWAGGER}"
Michal Nowikowski's avatar
Michal Nowikowski committed
226 227
end

228
desc 'Compile server part'
229
task :build_server => [GO, :gen_server, :gen_agent] do
230
  sh 'rm -f backend/server/agentcomm/api_mock.go'
231
  sh "cd backend/cmd/stork-server/ && #{GO} build #{go_build_date_opt}"
Michal Nowikowski's avatar
Michal Nowikowski committed
232 233
end

234
file PROTOC do
235
  sh "mkdir -p #{PROTOC_DIR}"
236
  sh "#{WGET} #{PROTOC_URL} -O #{PROTOC_DIR}/protoc.zip"
237
  Dir.chdir(PROTOC_DIR) do
238 239 240 241 242
    sh 'unzip protoc.zip'
  end
end

file PROTOC_GEN_GO do
243 244 245 246
  sh "#{GO} get -d -u #{PROTOC_GEN_GO_URL}"
  sh "git -C \"$(#{GO} env GOPATH)\"/src/github.com/golang/protobuf checkout #{PROTOC_GEN_GO_VER}"
  sh "#{GO} install github.com/golang/protobuf/protoc-gen-go"
  sh "cp #{GOBIN}/protoc-gen-go #{PROTOC_GEN_GO}"
247 248 249
end

file MOCKERY do
250 251 252
  Dir.chdir('backend') do
    sh "#{GO} get #{MOCKERY_URL}"
  end
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268
end

file MOCKGEN do
  sh "#{GO} get -u #{MOCKGEN_URL}"
end

file RICHGO do
  sh "#{GO} get -u #{RICHGO_URL}"
end

file AGENT_PB_GO_FILE => [GO, PROTOC, PROTOC_GEN_GO, AGENT_PROTO_FILE] do
  Dir.chdir('backend') do
    sh "#{PROTOC} -I api api/agent.proto --go_out=plugins=grpc:api"
  end
end

269 270 271 272 273 274 275 276 277 278 279
# prepare args for dlv debugger
headless = ''
if ENV['headless'] == 'true'
  headless = '--headless -l 0.0.0.0:45678'
end

desc 'Connect gdlv GUI Go debugger to waiting dlv debugger'
task :connect_dbg do
  sh 'gdlv connect 127.0.0.1:45678'
end

280 281 282 283 284
desc 'Generate API sources from agent.proto'
task :gen_agent => [AGENT_PB_GO_FILE]

desc 'Compile agent part'
file :build_agent => [GO, AGENT_PB_GO_FILE] do
285
  sh "cd backend/cmd/stork-agent/ && #{GO} build #{go_build_date_opt}"
286 287
end

288 289
desc 'Run agent'
task :run_agent => [:build_agent, GO] do
290 291 292 293 294
  if ENV['debug'] == 'true'
    sh "cd backend/cmd/stork-agent/ && dlv #{headless} debug"
  else
    sh "backend/cmd/stork-agent/stork-agent --port 8888"
  end
295 296
end

297
desc 'Run server'
298 299 300
task :run_server => [:build_server, GO] do |t, args|
  if ENV['debug'] == 'true'
    sh "cd backend/cmd/stork-server/ && dlv #{headless} debug"
301
  else
302 303
    cmd = 'backend/cmd/stork-server/stork-server'
    if ENV['dbtrace'] == 'true'
304
      cmd = "#{cmd} --db-trace-queries=run"
305 306
    end
    sh cmd
307 308 309 310
  end
end

desc 'Run server with local postgres docker container'
311
task :run_server_db do |t, args|
312 313 314 315 316 317
  ENV['STORK_DATABASE_NAME'] = "storkapp"
  ENV['STORK_DATABASE_USER_NAME'] = "storkapp"
  ENV['STORK_DATABASE_PASSWORD'] = "storkapp"
  ENV['STORK_DATABASE_HOST'] = "localhost"
  ENV['STORK_DATABASE_PORT'] = "5678"
  at_exit {
318
    sh "docker rm -f -v stork-app-pgsql"
319
  }
320
  sh 'docker run --name stork-app-pgsql -d -p 5678:5432 -e POSTGRES_DB=storkapp -e POSTGRES_USER=storkapp -e POSTGRES_PASSWORD=storkapp postgres:11 && sleep 5'
321
  Rake::Task["run_server"].invoke()
Michal Nowikowski's avatar
Michal Nowikowski committed
322 323
end

324

325 326
desc 'Compile database migrations tool'
task :build_migrations =>  [GO] do
327
  sh "cd backend/cmd/stork-db-migrate/ && #{GO} build #{go_build_date_opt}"
328 329
end

330 331 332 333
desc 'Compile whole backend: server, migrations and agent'
task :build_backend => [:build_agent, :build_server, :build_migrations]

file GOLANGCILINT => TOOLS_DIR do
334
  Dir.chdir(TOOLS_DIR) do
335
    sh "#{WGET} #{GOLANGCILINT_URL} -O golangci-lint.tar.gz"
336 337 338 339
    sh "tar -zxf golangci-lint.tar.gz"
  end
end

340 341 342 343 344
desc 'Build Stork backend continuously whenever source files change'
task :backend_live do
  sh 'find backend -name "*.go" | entr rake build_backend'
end

345 346
desc 'Check backend source code
arguments: fix=true - fixes some of the found issues'
347 348 349 350 351 352 353
task :lint_go => [GO, GOLANGCILINT, MOCKERY, MOCKGEN, :gen_agent, :gen_server] do
  at_exit {
    sh 'rm -f backend/server/agentcomm/api_mock.go'
  }
  sh 'rm -f backend/server/agentcomm/api_mock.go'
  Dir.chdir('backend') do
    sh "#{GO} generate -v ./..."
354 355 356 357 358 359

    opts = ''
    if ENV['fix'] == 'true'
      opts += ' --fix'
    end
    sh "#{GOLANGCILINT} run #{opts}"
360 361 362
  end
end

Matthijs Mekking's avatar
Matthijs Mekking committed
363 364 365 366 367 368 369
desc 'Format backend source code'
task :fmt_go => [GO, :gen_agent, :gen_server] do
  Dir.chdir('backend') do
    sh "#{GO} fmt ./..."
  end
end

370 371 372 373 374 375 376 377
def remove_remaining_databases(pgsql_host, pgsql_port)
    sh %{
       for db in $(psql -t -h #{pgsql_host} -p #{pgsql_port} -U storktest -c \"select datname from pg_database where datname ~ 'storktest.*'\"); do
         dropdb -h #{pgsql_host} -p #{pgsql_port} -U storktest $db
       done
    }
end

378
desc 'Run backend unit tests'
379
task :unittest_backend => [GO, RICHGO, MOCKERY, MOCKGEN, :build_server, :build_agent, :build_migrations] do
380 381 382 383
  at_exit {
    sh 'rm -f backend/server/agentcomm/api_mock.go'
  }
  sh 'rm -f backend/server/agentcomm/api_mock.go'
384 385 386

  cov_params = '-coverprofile=coverage.out'

387 388
  if ENV['scope']
    scope = ENV['scope']
389
    cov_params = ''
390 391 392
  else
    scope = './...'
  end
393

394
  if ENV['test']
395 396
    test_regex = "-run #{ENV['test']}"
    cov_params = ''
397
  else
398
    test_regex = ''
399 400
  end

401 402 403 404 405 406 407 408 409 410 411 412
  # establish location of postgresql database
  pgsql_host = 'localhost'
  pgsql_port = '5432'
  if ENV['POSTGRES_ADDR']
    pgsql_addr = ENV['POSTGRES_ADDR']
    if pgsql_addr.include? ':'
      pgsql_host, pgsql_port = pgsql_addr.split(':')
    else
      pgsql_host = pgsql_addr
    end
  end

413
  # prepare database for unit tests: clear any remainings from previous runs, prepare up-to-date template db
414
  if ENV['POSTGRES_IN_DOCKER'] != 'yes'
415 416
    remove_remaining_databases(pgsql_host, pgsql_port)
    sh "createdb -h #{pgsql_host} -p #{pgsql_port} -U storktest -O storktest storktest"
417 418 419
  end
  sh "STORK_DATABASE_PASSWORD=storktest ./backend/cmd/stork-db-migrate/stork-db-migrate -d storktest -u storktest --db-host #{pgsql_host} -p #{pgsql_port} up"

420 421 422
  if ENV['dbtrace'] == 'true'
    ENV['STORK_DATABASE_TRACE'] = 'true'
  end
423 424
  Dir.chdir('backend') do
    sh "#{GO} generate -v ./..."
425 426 427 428 429 430 431
    if ENV['debug'] == 'true'
      sh "dlv #{headless} test #{scope}"
    else
      gotool = RICHGO
      if ENV['richgo'] == 'false'
        gotool = GO
      end
432
      sh "#{gotool} test -race -v #{cov_params} #{test_regex} #{scope}"  # count=1 disables caching results
433
    end
434

435 436 437 438 439
    # drop test databases
    if ENV['POSTGRES_IN_DOCKER'] != 'yes'
      remove_remaining_databases(pgsql_host, pgsql_port)
    end

440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455
    # check coverage level (run it only for full tests scope)
    if not ENV['scope'] and not ENV['test']
      out = `#{GO} tool cover -func=coverage.out`
      puts out, ''
      problem = false
      out.each_line do |line|
        if line.start_with? 'total:' or line.include? 'api_mock.go'
          next
        end
        items = line.gsub(/\s+/m, ' ').strip.split(" ")
        file = items[0]
        func = items[1]
        cov = items[2].strip()[0..-2].to_f
        ignore_list = ['DetectServices', 'RestartKea', 'Serve', 'BeforeQuery', 'AfterQuery',
                       'Identity', 'LogoutHandler', 'NewDatabaseSettings', 'ConnectionParams',
                       'Password', 'loggingMiddleware', 'GlobalMiddleware', 'Authorizer',
456
                       'Listen', 'Shutdown', 'SetupLogging', 'UTCNow', 'detectApps',
457
                       'prepareTLS', 'handleRequest', 'pullerLoop', 'Output', 'Collect',
Tomek Mrugalski's avatar
Tomek Mrugalski committed
458 459 460 461 462 463
                       'collectTime', 'collectResolverStat', 'collectResolverLabelStat',

                       # Those two are tested in backend/server/server_test.go, in TestCommandLineSwitches*
                       # However, due to how it's executed (calling external binary), it's not detected
                       # by coverage.
                       'ParseArgs', 'NewStorkServer']
464 465 466 467
        if cov < 35 and not ignore_list.include? func
          puts "FAIL: %-80s %5s%% < 35%%" % ["#{file} #{func}", "#{cov}"]
          problem = true
        end
468
      end
469 470
      if problem
        fail("\nFAIL: Tests coverage is too low, add some tests\n\n")
471 472
      end
    end
473
  end
474 475
end

476 477 478
desc 'Run backend unit tests with local postgres docker container'
task :unittest_backend_db do
  at_exit {
479
    sh "docker rm -f -v stork-ut-pgsql"
480 481 482
  }
  sh "docker run --name stork-ut-pgsql -d -p 5678:5432 -e POSTGRES_DB=storktest -e POSTGRES_USER=storktest -e POSTGRES_PASSWORD=storktest postgres:11"
  ENV['POSTGRES_ADDR'] = "localhost:5678"
483
  ENV['POSTGRES_IN_DOCKER'] = 'yes'
484 485 486
  Rake::Task["unittest_backend"].invoke
end

487 488 489 490 491
desc 'Show backend coverage of unit tests in web browser'
task :show_cov do
  at_exit {
    sh 'rm -f backend/server/agentcomm/api_mock.go'
  }
492 493 494
  if not File.file?('backend/coverage.out')
    Rake::Task["unittest_backend_db"].invoke()
  end
495 496 497 498 499 500
  Dir.chdir('backend') do
    sh "#{GO} generate -v ./..."
    sh "#{GO} tool cover -html=coverage.out"
  end
end

501

502 503
### Web UI Tasks #########################

504
desc 'Generate client part of REST API using swagger_codegen based on swagger.yml'
505
task :gen_client => [SWAGGER_CODEGEN, SWAGGER_FILE] do
506
  Dir.chdir('webui') do
507
    sh "java -jar #{SWAGGER_CODEGEN} generate -l typescript-angular -i #{SWAGGER_FILE} -o src/app/backend --additional-properties snapshot=true,ngVersion=8.2.8"
Michal Nowikowski's avatar
Michal Nowikowski committed
508 509 510
  end
end

511
file SWAGGER_CODEGEN => TOOLS_DIR do
512
  sh "#{WGET} #{SWAGGER_CODEGEN_URL} -O #{SWAGGER_CODEGEN}"
Michal Nowikowski's avatar
Michal Nowikowski committed
513 514
end

515
file NPX => TOOLS_DIR do
516
  Dir.chdir(TOOLS_DIR) do
517
    sh "#{WGET} #{NODE_URL} -O #{TOOLS_DIR}/node.tar.xz"
518 519
    sh "tar -Jxf node.tar.xz"
  end
520 521
end

Michal Nowikowski's avatar
Michal Nowikowski committed
522 523
file NG => NPX do
  Dir.chdir('webui') do
524
    sh 'NG_CLI_ANALYTICS=false env -u DESTDIR npm install'
Michal Nowikowski's avatar
Michal Nowikowski committed
525
  end
526 527
end

528
desc 'Build angular application'
529
task :build_ui => [NG, :gen_client, :doc] do
530
  Dir.chdir('webui') do
Michal Nowikowski's avatar
Michal Nowikowski committed
531
    sh 'npx ng build --prod'
Michal Nowikowski's avatar
Michal Nowikowski committed
532 533 534
  end
end

535
desc 'Build Stork Angular app continuously whenever source files change'
536
task :serve_ui => [NG, :gen_client, :doc] do
537
  Dir.chdir('webui') do
538
    sh 'npx ng build --watch'
539 540 541
  end
end

542
desc 'Check frontend source code'
543 544 545
task :lint_ui => [NG, :gen_client] do
  Dir.chdir('webui') do
    sh 'npx ng lint'
546
    sh 'npx prettier --config .prettierrc --check \'**/*\''
547 548 549 550 551 552
  end
end

desc 'Make frontend source code prettier'
task :prettier_ui => [NG, :gen_client] do
  Dir.chdir('webui') do
553
    sh 'npx prettier --config .prettierrc --write \'**/*\''
554 555 556
  end
end

Michal Nowikowski's avatar
Michal Nowikowski committed
557
# internal task used in ci for running npm ci command with lint and tests together
558 559 560
task :ci_ui => [:gen_client] do
  Dir.chdir('webui') do
    sh 'npm ci'
561 562 563
  end

  Rake::Task["lint_ui"].invoke()
564
  Rake::Task["ng_test"].invoke()
565 566
end

567 568 569
# Common function for running ng test different progress, watch and browsers
# options.
def run_ng_test(progress, watch, browsers)
570 571 572 573 574 575 576
  if not File.file?(ENV['CHROME_BIN'])
    puts("Chrome binary not found in %s" % [ENV['CHROME_BIN']])
    puts("It is possible to override default Chrome location with CHROME_BIN")
    puts("environment variable. If this variable is set already it seems to")
    puts("point to a wrong location.")
    abort('Aborting tests because Chrome binary was not found.')
  end
577
   Dir.chdir('webui') do
578
     sh "npx ng test --progress #{progress} --watch #{watch} --browsers=#{browsers}"
579
#     sh 'npx ng e2e --progress false --watch false'
580 581 582
   end
end

583
desc 'Run unit tests for Angular with Chrome browser.'
584
task :ng_test => [NG] do
585 586 587 588 589
  if ENV['debug'] == "true"
    run_ng_test("true", "true", "Chrome")
  else
    run_ng_test("false", "false", "ChromeNoSandboxHeadless")
  end
590 591
end

592 593
### Docker Tasks #########################

594 595
desc 'Build containers with everything and statup all services using docker-compose
arguments: cache=false - forces rebuilding whole container'
596
task :docker_up => :build_all_in_container do
597
  at_exit {
598
    sh "docker-compose #{DOCKER_COMPOSE_FILES} down"
599
  }
600 601 602 603
  cache_opt = ''
  if ENV['cache'] == 'false'
    cache_opt = '--no-cache'
  end
604
  sh 'docker/gen-kea-config.py 7000 > docker/kea-dhcp4-many-subnets.conf'
605
  sh "docker-compose #{DOCKER_COMPOSE_FILES} build #{DOCKER_COMPOSE_PREMIUM_OPTS} #{cache_opt}"
606
  sh "docker-compose #{DOCKER_COMPOSE_FILES} up"
Michal Nowikowski's avatar
Michal Nowikowski committed
607 608
end

609
desc 'Shut down all containers'
Michal Nowikowski's avatar
Michal Nowikowski committed
610
task :docker_down do
611
  sh "docker-compose #{DOCKER_COMPOSE_FILES} down"
Michal Nowikowski's avatar
Michal Nowikowski committed
612
end
613

614 615
desc 'Build all in container'
task :build_all_in_container do
616 617 618 619 620
  # we increase the locked memory limit up to 512kb to work around the problem ocurring on Ubuntu 20.04.
  # for details, see: https://github.com/golang/go/issues/35777 and https://github.com/golang/go/issues/37436
  # The workaround added --ulimit memlock=512 to docker build and --privileged to docker run.
  sh "docker build --ulimit memlock=512 -f docker/docker-builder.txt -t stork-builder ."
  sh "docker run --privileged -v $PWD:/repo --rm stork-builder rake build_all_copy_in_subdir"
621 622 623 624 625
end

# internal task used by build_all_in_container
task :build_all_copy_in_subdir do
  sh 'mkdir -p ./build-root'
626
  sh 'rsync -av --exclude=webui/node_modules --exclude=webui/dist --exclude=webui/src/assets/arm --exclude=doc/_build --exclude=doc/doctrees --exclude=backend/server/gen --exclude=*~ --delete api backend doc etc webui Rakefile ./build-root'
627 628 629
  sh "cd ./build-root && GOPATH=/repo/build-root/go rake install_server install_agent"
end

630
desc 'Build container with Stork Agent and Kea DHCPv4 server'
631
task :build_kea_container do
632
  sh 'docker-compose build agent-kea'
633 634
end

635 636
desc 'Run container with Stork Agent and Kea and mount current Agent binary'
task :run_kea_container do
637 638 639
  at_exit {
    sh 'docker-compose down'
  }
640
  sh 'docker-compose up agent-kea'
641 642
end

643
desc 'Build container with Stork Agent and Kea DHCPv6 server'
644
task :build_kea6_container do
645
  sh 'docker-compose build agent-kea6'
646 647
end

648
desc 'Run container with Stork Agent and Kea DHCPv6 server and mount current Agent binary'
649
task :run_kea6_container do
650 651 652
  at_exit {
    sh 'docker-compose down'
  }
653
  sh 'docker-compose up agent-kea6'
654 655
end

656 657
desc 'Build two containers with Stork Agent and Kea HA pair
arguments: cache=false - forces rebuilding whole container'
658
task :build_kea_ha_containers do
659 660 661 662 663
  cache_opt = ''
  if ENV['cache'] == 'false'
    cache_opt = '--no-cache'
  end
  sh "docker-compose build #{cache_opt} agent-kea-ha1 agent-kea-ha2"
664 665
end

666
desc 'Run two containers with Stork Agent and Kea HA pair'
667
task :run_kea_ha_containers do
668 669 670 671
  at_exit {
    sh "docker-compose down"
  }
  sh 'docker-compose up agent-kea-ha1 agent-kea-ha2'
672 673
end

674
desc 'Build container with Stork Agent and Kea with host reseverations in db'
675
task :build_kea_hosts_container do
676
  sh "docker-compose #{DOCKER_COMPOSE_FILES} build #{DOCKER_COMPOSE_PREMIUM_OPTS} agent-kea-hosts"
677 678 679 680 681 682 683 684 685 686
end

desc 'Run container with Stork Agent and Kea with host reseverations in db'
task :run_kea_hosts_container do
  at_exit {
    sh "docker-compose #{DOCKER_COMPOSE_FILES} down"
  }
  sh "docker-compose #{DOCKER_COMPOSE_FILES} up agent-kea-hosts hosts-db"
end

687 688 689 690 691 692 693 694
desc 'Build container with Stork Agent and BIND 9'
task :build_bind9_container do
  sh 'docker build -f docker/docker-agent-bind9.txt -t agent-bind9 .'
end

desc 'Run container with Stork Agent and BIND 9 and mount current Agent binary'
task :run_bind9_container do
  # host[9999]->agent[8080]
695
  sh 'docker run --rm -ti -p 9999:8080 --name agent-bind9 -h agent-bind9 -v `pwd`/backend/cmd/stork-agent:/agent agent-bind9'
696 697
end

698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716
file 'tests/sim/venv/bin/activate' do
  Dir.chdir('tests/sim') do
    sh 'python3 -m venv venv'
    sh './venv/bin/pip install -U pip'
  end
end

file 'tests/sim/venv/bin/flask' => 'tests/sim/venv/bin/activate' do
  Dir.chdir('tests/sim') do
    sh './venv/bin/pip install -r requirements.txt'
  end
end

desc 'Run simulator for experimenting with Stork'
task :run_sim => 'tests/sim/venv/bin/flask' do
  Dir.chdir('tests/sim') do
    sh 'STORK_SERVER_URL=http://localhost:8080 FLASK_ENV=development FLASK_APP=sim.py LC_ALL=C.UTF-8 LANG=C.UTF-8 ./venv/bin/flask run --host 0.0.0.0 --port 5005'
  end
end
717

718 719
### Documentation Tasks #########################

720
desc 'Builds Stork documentation, using Sphinx'
Tomek Mrugalski's avatar
Tomek Mrugalski committed
721
task :doc do
722
  sh "sphinx-build -M html doc/ doc/_build #{SPHINXOPTS}"
723
  sh 'mkdir -p webui/src/assets/arm'
Tomek Mrugalski's avatar
Tomek Mrugalski committed
724
  sh 'cp -a doc/_build/html/* webui/src/assets/arm'
725
  sh "sphinx-build -M man doc/ doc/ #{SPHINXOPTS}"
726 727
end

728 729 730 731 732
desc 'Builds Stork documentation continuously whenever source files change'
task :doc_live do
  sh 'find doc -name "*.rst" | entr rake doc'
end

733

734 735 736
### Release Tasks #########################

desc 'Prepare release tarball with Stork sources'
737
task :tarball do
738 739 740 741 742
  sh "git archive --prefix=stork-#{STORK_VERSION}/ -o stork-#{STORK_VERSION}.tar.gz HEAD"
end

desc 'Build debs in Docker. It is used for developer purposes.'
task :build_debs_in_docker do
743
  sh "docker run -v $PWD:/repo --rm -ti registry.gitlab.isc.org/isc-projects/stork/pkgs-ubuntu-18-04:latest rake build_pkgs STORK_BUILD_TIMESTAMP=#{TIMESTAMP}"
744 745 746 747
end

desc 'Build RPMs in Docker. It is used for developer purposes.'
task :build_rpms_in_docker do
748
  sh "docker run -v $PWD:/repo --rm -ti registry.gitlab.isc.org/isc-projects/stork/pkgs-centos-8:latest rake build_pkgs STORK_BUILD_TIMESTAMP=#{TIMESTAMP}"
749 750
end

751 752
task :build_pkgs_in_docker => [:build_debs_in_docker, :build_rpms_in_docker]

753
# Internal task that copies sources and builds packages on a side. It is used by build_debs_in_docker and build_rpms_in_docker.
754
task :build_pkgs do
755
  sh 'rm -rf /build && mkdir /build'
756
  sh 'git ls-files | tar -czf /stork.tar.gz -T -'
757
  sh 'tar -C /build -zxvf /stork.tar.gz'
758 759 760 761
  cwd = Dir.pwd
  if Dir.exist?("#{cwd}/tools")
    sh "cp -a #{cwd}/tools /build"
  end
762 763 764 765
  if File.exist?('/etc/redhat-release')
    pkg_type = 'rpm'
  else
    pkg_type = 'deb'
766
  end
767 768
  sh "cd /build && rm -rf /build/root && rake #{pkg_type}_agent STORK_BUILD_TIMESTAMP=#{TIMESTAMP}"
  sh "cd /build && rm -rf /build/root && rake #{pkg_type}_server STORK_BUILD_TIMESTAMP=#{TIMESTAMP}"
769 770
  sh "cp /build/isc-stork* #{cwd}"
  sh "ls -al #{cwd}/isc-stork*"
771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
end

desc 'Build all. It builds backend and UI.'
task :build_all => [:build_backend, :build_ui] do
  sh 'echo DONE'
end

desc 'Install agent files to DESTDIR. It depends on building tasks.'
task :install_agent => [:build_agent, :doc, BIN_DIR, UNIT_DIR, ETC_DIR, MAN_DIR] do
  sh "cp -a backend/cmd/stork-agent/stork-agent #{BIN_DIR}"
  sh "cp -a etc/isc-stork-agent.service #{UNIT_DIR}"
  sh "cp -a etc/agent.env #{ETC_DIR}"
  sh "cp -a doc/man/stork-agent.8 #{MAN_DIR}"
end

desc 'Install server files to DESTDIR. It depends on building tasks.'
task :install_server => [:build_server, :build_migrations, :build_ui, :doc, BIN_DIR, UNIT_DIR, ETC_DIR, WWW_DIR, EXAMPLES_DIR, MAN_DIR] do
  sh "cp -a backend/cmd/stork-server/stork-server #{BIN_DIR}"
  sh "cp -a backend/cmd/stork-db-migrate/stork-db-migrate #{BIN_DIR}"
  sh "cp -a etc/isc-stork-server.service #{UNIT_DIR}"
  sh "cp -a etc/server.env #{ETC_DIR}"
  sh "cp -a etc/nginx-stork.conf #{EXAMPLES_DIR}"
  sh "cp -a webui/dist/stork/* #{WWW_DIR}"
  sh "cp -a doc/man/stork-server.8 #{MAN_DIR}"
795
  sh "cp -a doc/man/stork-db-migrate.8 #{MAN_DIR}"
796 797 798 799 800 801 802 803 804 805
end

# invoke fpm for building RPM or deb package
def fpm(pkg, fpm_target)
  cmd = "fpm -n isc-stork-#{pkg}"
  cmd += " -v #{STORK_VERSION}.#{TIMESTAMP}"
  cmd += " --license 'MPL 2.0'"
  cmd += " --vendor 'Internet Systems Consortium, Inc.'"
  cmd += " --url 'https://gitlab.isc.org/isc-projects/stork/'"
  cmd += " --description 'ISC Stork #{pkg.capitalize()}'"
806
  cmd += " --after-install etc/isc-stork-#{pkg}.postinst"
807
  cmd += " --before-remove etc/isc-stork-#{pkg}.prerm"
808
  cmd += " --after-remove etc/isc-stork-#{pkg}.postrm"
809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
  cmd += " -s dir"
  cmd += " -t #{fpm_target}"
  cmd += " -C #{DESTDIR} ."
  sh cmd
end

desc 'Build deb package with Stork agent. It depends on building and installing tasks.'
task :deb_agent => :install_agent do
  fpm('agent', 'deb')
end

desc 'Build RPM package with Stork agent. It depends on building and installing tasks.'
task :rpm_agent => :install_agent do
  fpm('agent', 'rpm')
end

desc 'Build deb package with Stork server. It depends on building and installing tasks.'
task :deb_server => :install_server do
  fpm('server', 'deb')
end

desc 'Build RPM package with Stork server. It depends on building and installing tasks.'
task :rpm_server => :install_server do
  fpm('server', 'rpm')
end

desc 'Prepare containers with FPM and other dependencies that are used for building RPM and deb packages'
task :build_fpm_containers do
837 838 839
#  sh 'docker build -f docker/pkgs/ubuntu-18-04.txt -t registry.gitlab.isc.org/isc-projects/stork/pkgs-ubuntu-18-04:latest docker/pkgs/'
#  sh 'docker build -f docker/pkgs/centos-8.txt -t registry.gitlab.isc.org/isc-projects/stork/pkgs-centos-8:latest docker/pkgs/'
  sh 'docker build -f docker/pkgs/cloudsmith.txt -t registry.gitlab.isc.org/isc-projects/stork/pkgs-cloudsmith:latest docker/pkgs/'
840 841
end

842 843 844

### System testing ######################

845 846 847 848 849 850 851 852 853 854 855 856 857
SELENIUM_DIR = "#{TOOLS_DIR}/selenium"
GECKO_DRV = "#{SELENIUM_DIR}/geckodriver"
CHROME_DRV = "#{SELENIUM_DIR}/chromedriver"
directory SELENIUM_DIR

if ENV['BROWSER'] == 'Chrome'
  selenium_driver_path = CHROME_DRV
else
  ENV['BROWSER'] = 'Firefox'
  selenium_driver_path = GECKO_DRV
end


858 859 860 861 862 863 864 865 866 867
file 'tests/system/venv/bin/activate' do
  Dir.chdir('tests/system') do
    sh 'python3 -m venv venv'
    sh './venv/bin/pip install -U pip'
  end
end

task :system_tests => 'tests/system/venv/bin/activate' do
  Dir.chdir('tests/system') do
    sh './venv/bin/pip install -r requirements.txt'
868
    sh './venv/bin/pytest --tb=long -l -r ap -s tests.py'
Wlodzimierz Wencel's avatar
Wlodzimierz Wencel committed
869 870 871
  end
end

872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892
file GECKO_DRV => SELENIUM_DIR do
  Dir.chdir(SELENIUM_DIR) do
    sh "#{WGET} https://github.com/mozilla/geckodriver/releases/download/v0.26.0/geckodriver-v0.26.0-linux32.tar.gz -O geckodriver.tar.gz"
    sh 'tar -xf geckodriver.tar.gz'
    sh 'rm geckodriver.tar.gz'
  end
end

file CHROME_DRV => SELENIUM_DIR do
  Dir.chdir(SELENIUM_DIR) do
  sh "#{WGET} https://chromedriver.storage.googleapis.com/85.0.4183.87/chromedriver_linux64.zip -O chromedriver_linux64.zip"
    sh "unzip chromedriver_linux64.zip"
    sh "rm chromedriver_linux64.zip"
  end
end

desc 'Run web UI system tests. By default Firefox is used.
It can be directly selected by BROWSER variable:
  rake system_tests_ui BROWSER=Firefox
  rake system_tests_ui BROWSER=Chrome'
task :system_tests_ui => ['tests/system/venv/bin/activate', selenium_driver_path] do
Wlodzimierz Wencel's avatar
Wlodzimierz Wencel committed
893 894
  Dir.chdir('tests/system') do
    sh './venv/bin/pip install -r requirements.txt'
895
    sh "./venv/bin/pytest --driver #{ENV['BROWSER']} --driver-path #{selenium_driver_path} -vv --full-trace -r ap -s ui/tests_ui_basic.py --headless"
Wlodzimierz Wencel's avatar
Wlodzimierz Wencel committed
896 897 898
  end
end

899 900


901
### Other Tasks #########################
902

903
desc 'Remove tools and other build or generated files'
904
task :clean do
905 906 907
  sh "rm -rf #{AGENT_PB_GO_FILE}"
  sh 'rm -rf backend/server/gen/*'
  sh 'rm -rf webui/src/app/backend/'
908 909 910
  sh 'rm -f backend/cmd/stork-agent/stork-agent'
  sh 'rm -f backend/cmd/stork-server/stork-server'
  sh 'rm -f backend/cmd/stork-db-migrate/stork-db-migrate'
911
end
912

913
desc 'Download all dependencies'
914 915 916 917
task :prepare_env => [GO, PROTOC, PROTOC_GEN_GO, GOSWAGGER, GOLANGCILINT, SWAGGER_CODEGEN, NPX] do
  Dir.chdir('backend') do
    sh "#{GO} mod download"
  end
918 919 920
  sh "#{GO} get -u github.com/go-delve/delve/cmd/dlv"
  sh "#{GO} get -u github.com/aarzilli/gdlv"
end
921 922 923

desc 'Generate ctags for Emacs'
task :ctags do
924
  sh 'etags.ctags -f TAGS -R --exclude=webui/node_modules --exclude=webui/dist --exclude=tools .'
925
end
926 927 928 929 930 931

desc 'Prepare containers that are using in GitLab CI processes'
task :build_ci_containers do
  sh 'docker build --no-cache -f docker/docker-ci-base.txt -t registry.gitlab.isc.org/isc-projects/stork/ci-base:latest docker/'
  #sh 'docker push registry.gitlab.isc.org/isc-projects/stork/ci-base:latest'
end