Commit d228bc2f authored by Marcin Siodelski's avatar Marcin Siodelski
Browse files

[5674] Implemented HA state machine configuration.

parent c407f5d3
......@@ -34,7 +34,7 @@ libha_la_SOURCES += ha_impl.cc ha_impl.h
libha_la_SOURCES += ha_log.cc ha_log.h
libha_la_SOURCES += ha_server_type.h
libha_la_SOURCES += ha_service.cc ha_service.h
libha_la_SOURCES += ha_service_states.h
libha_la_SOURCES += ha_service_states.cc ha_service_states.h
libha_la_SOURCES += query_filter.cc query_filter.h
libha_la_SOURCES += version.cc
......
......@@ -7,8 +7,24 @@
#include <exceptions/exceptions.h>
#include <util/strutil.h>
#include <ha_config.h>
#include <ha_service_states.h>
#include <sstream>
namespace {
/// @brief Creates default state configuration.
///
/// @param [out] state_map map of state configurations into which the
/// newly created configuration should be inserted.
/// @param state state for which new configuration is to be created.
void
createStateConfig(isc::ha::HAConfig::StateConfigMap& state_map, const int state) {
isc::ha::HAConfig::StateConfigPtr cfg(new isc::ha::HAConfig::StateConfig(state));
state_map[state] = cfg;
}
} // end of anonymous namespace
namespace isc {
namespace ha {
......@@ -76,11 +92,64 @@ HAConfig::PeerConfig::roleToString(const HAConfig::PeerConfig::Role& role) {
return ("");
}
HAConfig::StateConfig::StateConfig(const int state)
: state_(state), pausing_(HAConfig::StateConfig::PAUSE_NEVER) {
}
void
HAConfig::StateConfig::setPausing(const std::string& pausing) {
pausing_ = stringToPausing(pausing);
}
HAConfig::StateConfig::Pausing
HAConfig::StateConfig::stringToPausing(const std::string& pausing) {
if (pausing == "always") {
return (HAConfig::StateConfig::PAUSE_ALWAYS);
} else if (pausing == "never") {
return (HAConfig::StateConfig::PAUSE_NEVER);
} else if (pausing == "once") {
return (HAConfig::StateConfig::PAUSE_ONCE);
}
isc_throw(BadValue, "unsupported value " << pausing << " of 'pause' parameter");
}
std::string
HAConfig::StateConfig::pausingToString(const HAConfig::StateConfig::Pausing& pausing) {
switch (pausing) {
case HAConfig::StateConfig::PAUSE_ALWAYS:
return ("always");
case HAConfig::StateConfig::PAUSE_NEVER:
return ("never");
case HAConfig::StateConfig::PAUSE_ONCE:
return ("once");
default:
;
}
isc_throw(BadValue, "unsupported pause enumeration " << static_cast<int>(pausing));
}
HAConfig::HAConfig()
: this_server_name_(), ha_mode_(HOT_STANDBY), send_lease_updates_(true),
sync_leases_(true), sync_timeout_(60000), heartbeat_delay_(10000),
max_response_delay_(60000), max_ack_delay_(10000), max_unacked_clients_(10),
peers_() {
peers_(), state_machine_() {
// Create default state configurations.
createStateConfig(state_machine_, HA_BACKUP_ST);
createStateConfig(state_machine_, HA_HOT_STANDBY_ST);
createStateConfig(state_machine_, HA_LOAD_BALANCING_ST);
createStateConfig(state_machine_, HA_PARTNER_DOWN_ST);
createStateConfig(state_machine_, HA_READY_ST);
createStateConfig(state_machine_, HA_SYNCING_ST);
createStateConfig(state_machine_, HA_TERMINATED_ST);
createStateConfig(state_machine_, HA_WAITING_ST);
}
HAConfig::PeerConfigPtr
......@@ -178,6 +247,17 @@ HAConfig::getOtherServersConfig() const {
return (copy);
}
HAConfig::StateConfigPtr
HAConfig::getStateConfig(const int state) const {
auto state_config = state_machine_.find(state);
if (state_config == state_machine_.end()) {
isc_throw(BadValue, "no state machine configuration found for the "
<< "state identifier " << state);
}
return (state_config->second);
}
void
HAConfig::validate() const {
// Peers configurations must be provided.
......
......@@ -160,6 +160,70 @@ public:
/// @brief Map of the servers' configurations.
typedef std::map<std::string, PeerConfigPtr> PeerConfigMap;
/// @brief Configuration specific to a single HA state.
class StateConfig {
public:
/// @brief State machine pausing modes.
///
/// Supported modes are:
/// - always pause in the given state,
/// - never pause in the given state,
/// - pause upon first transition to the given state.
enum Pausing {
PAUSE_ALWAYS,
PAUSE_NEVER,
PAUSE_ONCE
};
/// @brief Constructor.
///
/// @param state state identifier.
explicit StateConfig(const int state);
/// @brief Returns identifier of the state.
int getState() const {
return (state_);
}
/// @brief Returns pausing mode for the given state.
Pausing getPausing() const {
return (pausing_);
}
/// @brief Sets pausing mode for the gievn state.
///
/// @param pausing new pausing mode in the textual form. Supported
/// values are: always, never, once.
void setPausing(const std::string& pausing);
/// @brief Converts pausing mode from the textual form.
///
/// @param pausing pausing mode in the textual form. Supported
/// values are: always, never, once.
static Pausing stringToPausing(const std::string& pausing);
/// @brief Returns pausing mode in the textual form.
///
/// @param pausing pausing mode.
static std::string pausingToString(const Pausing& pausing);
private:
/// @brief Idenitifier of state for which configuration is held.
int state_;
/// @brief Pausing mode in the given state.
Pausing pausing_;
};
/// @brief Pointer to the state configuration.
typedef boost::shared_ptr<StateConfig> StateConfigPtr;
/// @brief Map of configuration for supported states.
typedef std::map<int, StateConfigPtr> StateConfigMap;
/// @brief Constructor.
HAConfig();
......@@ -380,13 +444,27 @@ public:
return (peers_);
}
/// @brief Returns HA state configuration by state identifier.
///
/// @param state identifier of the state for which configuration should
/// be returned.
///
/// @return Pointer to the state configuration.
/// @throw BadValue if there is no configuration found for the given state.
StateConfigPtr getStateConfig(const int state) const;
/// @brief Returns state machine configuration.
///
/// @return Map of pointers to the configuration of all states.
StateConfigMap getStateMachineConfig() const {
return (state_machine_);
}
/// @brief Validates configuration.
///
/// @throw HAConfigValidationError if configuration is invalid.
void validate() const;
private:
std::string this_server_name_; ///< This server name.
HAMode ha_mode_; ///< Mode of operation.
bool send_lease_updates_; ///< Send lease updates to partner?
......@@ -397,6 +475,7 @@ private:
uint32_t max_ack_delay_; ///< Maximum DHCP message ack delay.
uint32_t max_unacked_clients_; ///< Maximum number of unacked clients.
PeerConfigMap peers_; ///< Map of peers' configurations.
StateConfigMap state_machine_; ///< Map of per states configurations.
};
/// @brief Pointer to the High Availability configuration structure.
......
......@@ -8,8 +8,10 @@
#include <ha_config_parser.h>
#include <ha_log.h>
#include <ha_service_states.h>
#include <cc/dhcp_config_error.h>
#include <limits>
#include <set>
using namespace isc::data;
using namespace isc::http;
......@@ -32,6 +34,11 @@ const SimpleDefaults HA_CONFIG_PEER_DEFAULTS = {
{ "auto-failover", Element::boolean, "true" }
};
/// @brief Default values for HA state configuration.
const SimpleDefaults HA_CONFIG_STATE_DEFAULTS = {
{ "pause", Element::string, "never" }
};
} // end of anonymous namespace
namespace isc {
......@@ -96,6 +103,12 @@ HAConfigParser::parseInternal(const HAConfigPtr& config_storage,
isc_throw(ConfigError, "'peers' parameter must be a list");
}
// State machine configuration must be a list of maps.
ConstElementPtr state_machine = c->get("state-machine");
if (state_machine && state_machine->getType() != Element::list) {
isc_throw(ConfigError, "'state-machine' parameter must be a list");
}
// We have made major sanity checks, so let's try to gather some values.
// Get 'this-server-name'.
......@@ -162,6 +175,37 @@ HAConfigParser::parseInternal(const HAConfigPtr& config_storage,
cfg->setAutoFailover(getBoolean(*p, "auto-failover"));
}
// Per state configuration is optional.
if (state_machine) {
const auto& state_machine_vec = state_machine->listValue();
std::set<int> configured_states;
// Go over per state configurations.
for (auto s = state_machine_vec.begin(); s != state_machine_vec.end(); ++s) {
// State configuration is held in map.
if ((*s)->getType() != Element::map) {
isc_throw(ConfigError, "state configuration must be a map");
}
setDefaults(*s, HA_CONFIG_STATE_DEFAULTS);
// Get state name and set per state configuration.
std::string state_name = getString(*s, "state");
int state = stringToState(state_name);
// Check that this configuration doesn't duplicate existing configuration.
if (configured_states.count(state) > 0) {
isc_throw(ConfigError, "duplicated configuration for the '"
<< state_name << "' state");
}
configured_states.insert(state);
config_storage->getStateConfig(state)->setPausing(getString(*s, "pause"));
}
}
// We have gone over the entire configuration and stored it in the configuration
// storage. However, we need to still validate it to detect errors like:
// duplicate secondary/primary servers, no configuration for this server etc.
......
......@@ -85,28 +85,28 @@ void
HAService::defineStates() {
StateModel::defineStates();
defineState(HA_BACKUP_ST, "backup",
defineState(HA_BACKUP_ST, stateToString(HA_BACKUP_ST),
boost::bind(&HAService::backupStateHandler, this));
defineState(HA_HOT_STANDBY_ST, "hot-standby",
defineState(HA_HOT_STANDBY_ST, stateToString(HA_HOT_STANDBY_ST),
boost::bind(&HAService::normalStateHandler, this));
defineState(HA_LOAD_BALANCING_ST, "load-balancing",
defineState(HA_LOAD_BALANCING_ST, stateToString(HA_LOAD_BALANCING_ST),
boost::bind(&HAService::normalStateHandler, this));
defineState(HA_PARTNER_DOWN_ST, "partner-down",
defineState(HA_PARTNER_DOWN_ST, stateToString(HA_PARTNER_DOWN_ST),
boost::bind(&HAService::partnerDownStateHandler, this));
defineState(HA_READY_ST, "ready",
defineState(HA_READY_ST, stateToString(HA_READY_ST),
boost::bind(&HAService::readyStateHandler, this));
defineState(HA_SYNCING_ST, "syncing",
defineState(HA_SYNCING_ST, stateToString(HA_SYNCING_ST),
boost::bind(&HAService::syncingStateHandler, this));
defineState(HA_TERMINATED_ST, "terminated",
defineState(HA_TERMINATED_ST, stateToString(HA_TERMINATED_ST),
boost::bind(&HAService::terminatedStateHandler, this));
defineState(HA_WAITING_ST, "waiting",
defineState(HA_WAITING_ST, stateToString(HA_WAITING_ST),
boost::bind(&HAService::waitingStateHandler, this));
}
......
// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
#include <ha_service_states.h>
namespace isc {
namespace ha {
std::string stateToString(int state) {
switch (state) {
case HA_BACKUP_ST:
return ("backup");
case HA_HOT_STANDBY_ST:
return ("hot-standby");
case HA_LOAD_BALANCING_ST:
return ("load-balancing");
case HA_PARTNER_DOWN_ST:
return ("partner-down");
case HA_READY_ST:
return ("ready");
case HA_SYNCING_ST:
return ("syncing");
case HA_TERMINATED_ST:
return ("terminated");
case HA_WAITING_ST:
return ("waiting");
case HA_UNAVAILABLE_ST:
return ("unavailable");
default:
;
}
isc_throw(BadValue, "unknown state identifier " << state);
}
int stringToState(const std::string& state_name) {
if (state_name == "backup") {
return (HA_BACKUP_ST);
} else if (state_name == "hot-standby") {
return (HA_HOT_STANDBY_ST);
} else if (state_name == "load-balancing") {
return (HA_LOAD_BALANCING_ST);
} else if (state_name == "partner-down") {
return (HA_PARTNER_DOWN_ST);
} else if (state_name == "ready") {
return (HA_READY_ST);
} else if (state_name == "syncing") {
return (HA_SYNCING_ST);
} else if (state_name == "terminated") {
return (HA_TERMINATED_ST);
} else if (state_name == "waiting") {
return (HA_WAITING_ST);
} else if (state_name == "unavailable") {
return (HA_UNAVAILABLE_ST);
}
isc_throw(BadValue, "unknown state " << state_name);
}
} // end of namespace isc::ha
} // end of namespace isc
......@@ -8,6 +8,7 @@
#define HA_SERVICE_STATES_H
#include <util/state_model.h>
#include <string>
namespace isc {
namespace ha {
......@@ -40,6 +41,20 @@ const int HA_WAITING_ST = util::StateModel::SM_DERIVED_STATE_MIN + 8;
/// the partner.
const int HA_UNAVAILABLE_ST = util::StateModel::SM_DERIVED_STATE_MIN + 1000;
/// @brief Returns state name.
///
/// @param state state identifier for which name should be returned.
///
/// @throw BadValue if the state identifier is unsupported.
std::string stateToString(int state);
/// @brief Returns state for a given name.
///
/// @param state_name name of the state to be returned.
///
/// @throw BadValue if the state name is unsupported.
int stringToState(const std::string& state_name);
} // end of namespace isc::ha
} // end of namespace isc
......
......@@ -7,6 +7,7 @@
#include <config.h>
#include <ha_impl.h>
#include <ha_service_states.h>
#include <ha_test.h>
#include <cc/command_interpreter.h>
#include <cc/data.h>
......@@ -43,6 +44,7 @@ public:
HAImplPtr impl(new HAImpl());
try {
impl->configure(Element::fromJSON(invalid_config));
ADD_FAILURE() << "expected ConfigError exception, thrown no exception";
} catch (const ConfigError& ex) {
EXPECT_EQ(expected_error, std::string(ex.what()));
......@@ -86,6 +88,20 @@ TEST_F(HAConfigTest, configureLoadBalancing) {
" \"role\": \"backup\","
" \"auto-failover\": false"
" }"
" ],"
" \"state-machine\": ["
" {"
" \"state\": \"waiting\","
" \"pause\": \"once\""
" },"
" {"
" \"state\": \"ready\","
" \"pause\": \"always\""
" },"
" {"
" \"state\": \"partner-down\","
" \"pause\": \"never\""
" }"
" ]"
" }"
"]";
......@@ -125,6 +141,37 @@ TEST_F(HAConfigTest, configureLoadBalancing) {
EXPECT_EQ(cfg->getLogLabel(), "server3 (http://127.0.0.1:8082/)");
EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole());
EXPECT_FALSE(cfg->isAutoFailover());
// Verify that per-state configuration is correct.
HAConfig::StateConfigPtr state_cfg;
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_BACKUP_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_LOAD_BALANCING_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_PARTNER_DOWN_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_READY_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_ALWAYS, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_SYNCING_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_TERMINATED_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_WAITING_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_ONCE, state_cfg->getPausing());
}
// Verifies that load balancing configuration is parsed correctly.
......@@ -188,6 +235,35 @@ TEST_F(HAConfigTest, configureHotStandby) {
EXPECT_EQ("http://127.0.0.1:8082/", cfg->getUrl().toText());
EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole());
EXPECT_FALSE(cfg->isAutoFailover());
HAConfig::StateConfigPtr state_cfg;
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_BACKUP_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_HOT_STANDBY_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_PARTNER_DOWN_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_READY_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_SYNCING_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_TERMINATED_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateConfig(HA_WAITING_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(HAConfig::StateConfig::PAUSE_NEVER, state_cfg->getPausing());
}
// This server name must not be empty.
......@@ -681,6 +757,110 @@ TEST_F(HAConfigTest, hotStandbySecondary) {
"secondary servers not allowed in the hot standby configuration");
}
// State name must be recognized.
TEST_F(HAConfigTest, invalidStateName) {
testInvalidConfig(
"["
" {"
" \"this-server-name\": \"server1\","
" \"mode\": \"hot-standby\","
" \"peers\": ["
" {"
" \"name\": \"server1\","
" \"url\": \"http://127.0.0.1:8080/\","
" \"role\": \"primary\","
" \"auto-failover\": false"
" },"
" {"
" \"name\": \"server2\","
" \"url\": \"http://127.0.0.1:8081/\","
" \"role\": \"standby\","
" \"auto-failover\": true"
" }"
" ],"
" \"state-machine\": ["
" {"
" \"state\": \"foo\","
" \"pause\": \"always\""
" }"
" ]"
" }"
"]",
"unknown state foo");
}
// Pause value must be recognized.
TEST_F(HAConfigTest, invalidPauseValue) {
testInvalidConfig(
"["
" {"
" \"this-server-name\": \"server1\","
" \"mode\": \"hot-standby\","
" \"peers\": ["
" {"
" \"name\": \"server1\","
" \"url\": \"http://127.0.0.1:8080/\","
" \"role\": \"primary\","
" \"auto-failover\": false"
" },"
" {"
" \"name\": \"server2\","
" \"url\": \"http://127.0.0.1:8081/\","