Commit de46e638 authored by Michal Nowikowski's avatar Michal Nowikowski

[#409] added detecting changed Kea CA address, added system test case for such situation

parent 3ac39792
......@@ -53,4 +53,9 @@ TAGS
/build-root
# generated files for docker containers
/docker/kea-dhcp4-many-subnets.conf
\ No newline at end of file
/docker/kea-dhcp4-many-subnets.conf
# tests venv and pycache
/tests/system/venv/
/tests/system/__pycache__/
/tests/system/test-results/
\ No newline at end of file
* 110 [bug] godfryd
Fixed handling situation when IP address to Kea Control Agent has changed.
Till now Stork was not able to detect this and was still communicating to
the old address. Now it checks if address has changed and updates it
in the database.
(Gitlab #409)
* 109 [doc] tomek
Updated Prerequisites section. We now have a single list of
......
......@@ -852,9 +852,14 @@ file 'tests/system/venv/bin/activate' do
end
task :system_tests => 'tests/system/venv/bin/activate' do
if ENV['test']
test = ENV['test']
else
test = 'tests.py'
end
Dir.chdir('tests/system') do
sh './venv/bin/pip install -r requirements.txt'
sh './venv/bin/pytest --tb=long -l -r ap -s tests.py'
sh "./venv/bin/pytest --tb=long -l -r ap -s #{test}"
end
end
......
......@@ -427,6 +427,7 @@ func TestCommitAppIntoDB(t *testing.T) {
require.NoError(t, err)
require.NotZero(t, machine.ID)
// add app with particular access point
var accessPoints []*dbmodel.AccessPoint
accessPoints = dbmodel.AppendAccessPoint(accessPoints, dbmodel.AccessPointControl, "", "", 1234)
app := &dbmodel.App{
......@@ -440,6 +441,7 @@ func TestCommitAppIntoDB(t *testing.T) {
err = CommitAppIntoDB(db, app, fec, nil)
require.NoError(t, err)
// now change access point (different port) and trigger updating app in database
accessPoints = []*dbmodel.AccessPoint{}
accessPoints = dbmodel.AppendAccessPoint(accessPoints, dbmodel.AccessPointControl, "", "", 2345)
app.AccessPoints = accessPoints
......
......@@ -2,7 +2,9 @@ package apps
import (
"context"
"time"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"isc.org/stork/server/agentcomm"
......@@ -40,46 +42,241 @@ func (puller *StatePuller) Shutdown() {
puller.PeriodicPuller.Shutdown()
}
// Gets the status of the Kea apps and stores useful information in the database.
// The High Availability status is stored in the database for those apps which
// have the HA enabled.
// Gets the status of machines and their apps and stores useful information in the database.
func (puller *StatePuller) pullData() (int, error) {
// get list of all apps from database
dbApps, err := dbmodel.GetAllApps(puller.Db)
// get list of all machines from database
dbMachines, err := dbmodel.GetAllMachines(puller.Db)
if err != nil {
return 0, err
}
// get ...TODO
// get state from machines and their apps
var lastErr error
appsOkCnt := 0
for _, dbApp := range dbApps {
dbApp2 := dbApp
err := puller.getAppState(&dbApp2)
if err != nil {
lastErr = err
log.Errorf("error occurred while getting stats from app %d: %+v", dbApp.ID, err)
okCnt := 0
for _, dbM := range dbMachines {
dbM2 := dbM
ctx := context.Background()
errStr := GetMachineAndAppsState(ctx, puller.Db, &dbM2, puller.Agents, puller.EventCenter)
if errStr != "" {
lastErr = errors.New(errStr)
log.Errorf("error occurred while getting info from machine %d: %s", dbM2.ID, errStr)
} else {
appsOkCnt++
okCnt++
}
}
log.Printf("completed pulling lease stats from Kea apps: %d/%d succeeded", appsOkCnt, len(dbApps))
return appsOkCnt, lastErr
log.Printf("completed pulling information from machines: %d/%d succeeded", okCnt, len(dbMachines))
return okCnt, lastErr
}
// Store updated machine fields in to database.
func updateMachineFields(db *dbops.PgDB, dbMachine *dbmodel.Machine, m *agentcomm.State) error {
// update state fields in machine
dbMachine.State.AgentVersion = m.AgentVersion
dbMachine.State.Cpus = m.Cpus
dbMachine.State.CpusLoad = m.CpusLoad
dbMachine.State.Memory = m.Memory
dbMachine.State.Hostname = m.Hostname
dbMachine.State.Uptime = m.Uptime
dbMachine.State.UsedMemory = m.UsedMemory
dbMachine.State.Os = m.Os
dbMachine.State.Platform = m.Platform
dbMachine.State.PlatformFamily = m.PlatformFamily
dbMachine.State.PlatformVersion = m.PlatformVersion
dbMachine.State.KernelVersion = m.KernelVersion
dbMachine.State.KernelArch = m.KernelArch
dbMachine.State.VirtualizationSystem = m.VirtualizationSystem
dbMachine.State.VirtualizationRole = m.VirtualizationRole
dbMachine.State.HostID = m.HostID
dbMachine.LastVisitedAt = m.LastVisitedAt
dbMachine.Error = m.Error
err := db.Update(dbMachine)
if err != nil {
return errors.Wrapf(err, "problem with updating machine %+v", dbMachine)
}
return nil
}
func (puller *StatePuller) getAppState(dbApp *dbmodel.App) error {
ctx := context.Background()
var err error
switch dbApp.Type {
case dbmodel.AppTypeKea:
events := kea.GetAppState(ctx, puller.Agents, dbApp, puller.EventCenter)
err = kea.CommitAppIntoDB(puller.Db, dbApp, puller.EventCenter, events)
case dbmodel.AppTypeBind9:
bind9.GetAppState(ctx, puller.Agents, dbApp, puller.EventCenter)
err = bind9.CommitAppIntoDB(puller.Db, dbApp, puller.EventCenter)
default:
err = nil
}
return err
// appCompare compares two apps on equality. Two apps are considered equal if
// their type matches and if they have the same control port. Return true if
// equal, false otherwise.
func appCompare(dbApp *dbmodel.App, app *agentcomm.App) bool {
if dbApp.Type != app.Type {
return false
}
var controlPortEqual bool
for _, pt1 := range dbApp.AccessPoints {
if pt1.Type != dbmodel.AccessPointControl {
continue
}
for _, pt2 := range app.AccessPoints {
if pt2.Type != dbmodel.AccessPointControl {
continue
}
if pt1.Port == pt2.Port {
controlPortEqual = true
break
}
}
// If a match is found, we can break.
if controlPortEqual {
break
}
}
return controlPortEqual
}
// Get old apps from the machine db object and new apps retrieved from the machine remotely
// and merge them into one list of all, unique apps.
func mergeNewAndOldApps(db *dbops.PgDB, dbMachine *dbmodel.Machine, discoveredApps []*agentcomm.App) ([]*dbmodel.App, string) {
// If there are any new apps then get their state and add to db.
// Old ones are just updated. Use GetAppsByMachine to retrieve
// machine's apps with their daemons.
oldAppsList, err := dbmodel.GetAppsByMachine(db, dbMachine.ID)
if err != nil {
log.Error(err)
return nil, "cannot get machine's apps from db"
}
// count old apps
oldKeaAppsCnt := 0
oldBind9AppsCnt := 0
for _, dbApp := range oldAppsList {
if dbApp.Type == dbmodel.AppTypeKea {
oldKeaAppsCnt++
} else if dbApp.Type == dbmodel.AppTypeBind9 {
oldBind9AppsCnt++
}
}
// count new apps
newKeaAppsCnt := 0
newBind9AppsCnt := 0
for _, app := range discoveredApps {
if app.Type == dbmodel.AppTypeKea {
newKeaAppsCnt++
} else if app.Type == dbmodel.AppTypeBind9 {
newBind9AppsCnt++
}
}
// new and old apps
allApps := []*dbmodel.App{}
// old apps found in new apps fetched from the machine
matchedApps := []*dbmodel.App{}
for _, app := range discoveredApps {
// try to find apps on machine with old apps from database
var dbApp *dbmodel.App = nil
for _, dbApp2 := range oldAppsList {
if (app.Type == dbmodel.AppTypeKea && dbApp2.Type == dbmodel.AppTypeKea && oldKeaAppsCnt == 1 && newKeaAppsCnt == 1) || (app.Type == dbmodel.AppTypeBind9 && dbApp2.Type == dbmodel.AppTypeBind9 && oldBind9AppsCnt == 1 && newBind9AppsCnt == 1) || appCompare(dbApp2, app) {
dbApp = dbApp2
matchedApps = append(matchedApps, dbApp)
break
}
}
// if no old app in db then prepare new record
if dbApp == nil {
dbApp = &dbmodel.App{
ID: 0,
MachineID: dbMachine.ID,
Machine: dbMachine,
Type: app.Type,
}
} else {
dbApp.Machine = dbMachine
}
allApps = append(allApps, dbApp)
// add or update access points
var accessPoints []*dbmodel.AccessPoint
for _, point := range app.AccessPoints {
accessPoints = append(accessPoints, &dbmodel.AccessPoint{
Type: point.Type,
Address: point.Address,
Port: point.Port,
Key: point.Key,
})
}
dbApp.AccessPoints = accessPoints
}
// add old, not matched apps to all apps
for _, dbApp := range oldAppsList {
toAdd := true
for _, app := range matchedApps {
if dbApp == app {
toAdd = false
break
}
}
if toAdd {
dbApp.Machine = dbMachine
allApps = append(allApps, dbApp)
}
}
return allApps, ""
}
// Retrieve remotely machine and its apps state, and store it in the database.
func GetMachineAndAppsState(ctx context.Context, db *dbops.PgDB, dbMachine *dbmodel.Machine, agents agentcomm.ConnectedAgents, eventCenter eventcenter.EventCenter) string {
ctx2, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
// get state of machine from agent
state, err := agents.GetState(ctx2, dbMachine.Address, dbMachine.AgentPort)
if err != nil {
log.Warn(err)
dbMachine.Error = "cannot get state of machine"
err = db.Update(dbMachine)
if err != nil {
log.Error(err)
return "problem with updating record in database"
}
return ""
}
// store machine's state in db
err = updateMachineFields(db, dbMachine, state)
if err != nil {
log.Error(err)
return "cannot update machine in db"
}
// take old apps from db and new apps fetched from the machine
// and match them and prepare a list of all apps
allApps, errStr := mergeNewAndOldApps(db, dbMachine, state.Apps)
if errStr != "" {
return errStr
}
// go through all apps and store their changes in database
for _, dbApp := range allApps {
// get app state from the machine
switch dbApp.Type {
case dbmodel.AppTypeKea:
events := kea.GetAppState(ctx2, agents, dbApp, eventCenter)
err = kea.CommitAppIntoDB(db, dbApp, eventCenter, events)
case dbmodel.AppTypeBind9:
bind9.GetAppState(ctx2, agents, dbApp, eventCenter)
err = bind9.CommitAppIntoDB(db, dbApp, eventCenter)
default:
err = nil
}
if err != nil {
log.Errorf("cannot store application state: %+v", err)
return "problem with storing application state in the database"
}
}
// add all apps to machine's apps list - it will be used in ReST API functions
// to return state of machine and its apps
dbMachine.Apps = allApps
return ""
}
......@@ -3,8 +3,6 @@ package dbmodel
import (
"github.com/go-pg/pg/v9"
"github.com/pkg/errors"
dbops "isc.org/stork/server/database"
)
// A structure reflecting the access_point SQL table.
......@@ -21,10 +19,10 @@ const AccessPointControl = "control"
const AccessPointStatistics = "statistics"
// GetAllAccessPointsByAppID returns all access points for an app with given ID.
func GetAllAccessPointsByAppID(db *dbops.PgDB, appID int64) ([]*AccessPoint, error) {
func GetAllAccessPointsByAppID(tx *pg.Tx, appID int64) ([]*AccessPoint, error) {
var accessPoints []*AccessPoint
err := db.Model(&accessPoints).
err := tx.Model(&accessPoints).
Where("app_id = ?", appID).
OrderExpr("access_point.type ASC").
Select()
......
......@@ -39,6 +39,7 @@ type App struct {
// updateAppAccessPoints updates the associated application access points into
// the database.
func updateAppAccessPoints(tx *pg.Tx, app *App, update bool) (err error) {
var oldAccessPoints []*AccessPoint
if update {
// First delete any access points previously associated with the app.
types := []string{}
......@@ -52,6 +53,8 @@ func updateAppAccessPoints(tx *pg.Tx, app *App, update bool) (err error) {
if err != nil {
return errors.Wrapf(err, "problem with removing access points from app %d", app.ID)
}
oldAccessPoints, err = GetAllAccessPointsByAppID(tx, app.ID)
}
// If there are any access points associated with the app,
......@@ -60,7 +63,19 @@ func updateAppAccessPoints(tx *pg.Tx, app *App, update bool) (err error) {
point.AppID = app.ID
point.MachineID = app.MachineID
if update {
_, err = tx.Model(point).OnConflict("(app_id, type) DO UPDATE").Insert()
// if new access point matches to old one then update it
oldPresent := false
for _, oldAP := range oldAccessPoints {
if point.Type == oldAP.Type {
_, err = tx.Model(point).WherePK().Update()
oldPresent = true
break
}
}
// otherwise create new one
if !oldPresent {
_, err = tx.Model(point).Insert()
}
} else {
_, err = tx.Model(point).Insert()
}
......@@ -265,7 +280,7 @@ func UpdateApp(dbIface interface{}, app *App) ([]*Daemon, []*Daemon, error) {
// Update access points.
err = updateAppAccessPoints(tx, app, true)
if err != nil {
return nil, nil, errors.Wrapf(err, "problem with updating access points to app: %+v", app)
return nil, nil, errors.WithMessagef(err, "problem with updating access points to app: %+v", app)
}
// Commit the changes if necessary.
......
......@@ -6,7 +6,6 @@ import (
"github.com/go-pg/pg/v9"
"github.com/go-pg/pg/v9/orm"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
// Part of machine table in database that describes state of machine. In DB it is stored as JSONB.
......@@ -42,7 +41,6 @@ type Machine struct {
}
func AddMachine(db *pg.DB, machine *Machine) error {
log.Infof("inserting machine %+v", machine)
err := db.Insert(machine)
if err != nil {
err = errors.Wrapf(err, "problem with inserting machine %+v", machine)
......@@ -147,6 +145,27 @@ func GetMachinesByPage(db *pg.DB, offset int64, limit int64, filterText *string,
return machines, int64(total), nil
}
// Get all machines from database.
func GetAllMachines(db *pg.DB) ([]Machine, error) {
var machines []Machine
// prepare query
q := db.Model(&machines)
q = q.Relation("Apps.AccessPoints")
q = q.Relation("Apps.Daemons.KeaDaemon.KeaDHCPDaemon")
q = q.Relation("Apps.Daemons.Bind9Daemon")
err := q.Select()
if err != nil {
if err == pg.ErrNoRows {
return []Machine{}, nil
}
return nil, errors.Wrapf(err, "problem with getting machines")
}
return machines, nil
}
func DeleteMachine(db *pg.DB, machine *Machine) error {
err := db.Delete(machine)
if err != nil {
......
......@@ -384,3 +384,34 @@ func TestRefreshMachineFromDb(t *testing.T) {
require.EqualValues(t, 4, m.State.Cpus)
require.Equal(t, "some error", m.Error)
}
func TestGetAllMachines(t *testing.T) {
db, _, teardown := dbtest.SetupDatabaseTestCase(t)
defer teardown()
// add 20 machines
for i := 1; i <= 20; i++ {
m := &Machine{
Address: "localhost",
AgentPort: 8080 + int64(i),
Error: "some error",
State: MachineState{
Hostname: "aaaa",
Cpus: 4,
},
}
err := AddMachine(db, m)
require.NoError(t, err)
}
// get all machines should return 20 machines
machines, err := GetAllMachines(db)
require.NoError(t, err)
require.Len(t, machines, 20)
// paged get should return indicated limit, not all
machines, total, err := GetMachinesByPage(db, 0, 10, nil, "", SortDirAny)
require.NoError(t, err)
require.Len(t, machines, 10)
require.EqualValues(t, 20, total)
}
......@@ -335,6 +335,7 @@ func GetDetailedServicesByAppID(db *dbops.PgDB, appID int64) ([]Service, error)
Relation("HAService").
Relation("Daemons.KeaDaemon.KeaDHCPDaemon").
Relation("Daemons.App").
Relation("Daemons.App.AccessPoints").
Where("app_id = ?", appID).
OrderExpr("service.id ASC").
Select()
......@@ -344,14 +345,6 @@ func GetDetailedServicesByAppID(db *dbops.PgDB, appID int64) ([]Service, error)
return services, err
}
// Retrieve the access points. This should be incorporated in the
// above query, ideally.
for _, service := range services {
for _, daemon := range service.Daemons {
daemon.App.AccessPoints, _ = GetAllAccessPointsByAppID(db, daemon.App.ID)
}
}
return services, nil
}
......
......@@ -385,6 +385,7 @@ func TestGetServicesByAppID(t *testing.T) {
appServices, err := GetDetailedServicesByAppID(db, services[0].Daemons[3].AppID)
require.NoError(t, err)
require.Len(t, appServices, 1)
require.Len(t, appServices[0].Daemons[0].App.AccessPoints, 1)
// Validate that the service returned is the service1.
service := appServices[0]
......
......@@ -15,11 +15,9 @@ import (
"isc.org/stork"
"isc.org/stork/server/agentcomm"
"isc.org/stork/server/apps/bind9"
"isc.org/stork/server/apps"
"isc.org/stork/server/apps/kea"
dbops "isc.org/stork/server/database"
dbmodel "isc.org/stork/server/database/model"
"isc.org/stork/server/eventcenter"
"isc.org/stork/server/gen/models"
dhcp "isc.org/stork/server/gen/restapi/operations/d_h_c_p"
"isc.org/stork/server/gen/restapi/operations/general"
......@@ -72,149 +70,6 @@ func (r *RestAPI) machineToRestAPI(dbMachine dbmodel.Machine) *models.Machine {
return &m
}
// appCompare compares two apps on equality. Two apps are considered equal if
// their type matches and if they have the same control port. Return true if
// equal, false otherwise.
func appCompare(dbApp *dbmodel.App, app *agentcomm.App) bool {
if dbApp.Type != app.Type {
return false
}
var controlPortEqual bool
for _, pt1 := range dbApp.AccessPoints {
if pt1.Type != dbmodel.AccessPointControl {
continue
}
for _, pt2 := range app.AccessPoints {
if pt2.Type != dbmodel.AccessPointControl {
continue
}
if pt1.Port == pt2.Port {
controlPortEqual = true
break
}
}
// If a match is found, we can break.
if controlPortEqual {
break
}
}
return controlPortEqual
}
func getMachineAndAppsState(ctx context.Context, db *dbops.PgDB, dbMachine *dbmodel.Machine, agents agentcomm.ConnectedAgents, eventCenter eventcenter.EventCenter) string {
ctx2, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
// get state of machine from agent
state, err := agents.GetState(ctx2, dbMachine.Address, dbMachine.AgentPort)
if err != nil {
log.Warn(err)
dbMachine.Error = "cannot get state of machine"
err = db.Update(dbMachine)
if err != nil {
log.Error(err)
return "problem with updating record in database"
}
return ""
}
// store machine's state in db
err = updateMachineFields(db, dbMachine, state)
if err != nil {
log.Error(err)
return "cannot update machine in db"
}
// If there are any new apps then get their state and add to db.
// Old ones are just updated. Use GetAppsByMachine to retrieve
// machine's apps with their daemons.
oldAppsList, err := dbmodel.GetAppsByMachine(db, dbMachine.ID)
if err != nil {
log.Error(err)
return "cannot get machine's apps from db"
}
dbMachine.Apps = []*dbmodel.App{}
for _, app := range state.Apps {
// look for old app
var dbApp *dbmodel.App = nil
for _, dbApp2 := range oldAppsList {
if appCompare(dbApp2, app) {
dbApp = dbApp2
break
}
}
// if no old app in db then prepare new record
if dbApp == nil {
var accessPoints []*dbmodel.AccessPoint
for _, point := range app.AccessPoints {
accessPoints = append(accessPoints, &dbmodel.AccessPoint{
Type: point.Type,
Address: point.Address,
Port: point.Port,
Key: point.Key,
})
}
dbApp = &dbmodel.App{
ID: 0,
MachineID: dbMachine.ID,
Machine: dbMachine,
Type: app.Type,
AccessPoints: accessPoints,
}
} else {