Commit b9f3f082 authored by Marcin Siodelski's avatar Marcin Siodelski

[master] Merge branch 'trac5674'

parents 5f53de38 a88eb83e
......@@ -34,7 +34,7 @@ libha_la_SOURCES += ha_impl.cc ha_impl.h
libha_la_SOURCES += ha_log.cc ha_log.h
libha_la_SOURCES += ha_server_type.h
libha_la_SOURCES += ha_service.cc ha_service.h
libha_la_SOURCES += ha_service_states.h
libha_la_SOURCES += ha_service_states.cc ha_service_states.h
libha_la_SOURCES += query_filter.cc query_filter.h
libha_la_SOURCES += version.cc
......
......@@ -7,8 +7,11 @@
#include <exceptions/exceptions.h>
#include <util/strutil.h>
#include <ha_config.h>
#include <ha_service_states.h>
#include <sstream>
using namespace isc::util;
namespace isc {
namespace ha {
......@@ -76,11 +79,69 @@ HAConfig::PeerConfig::roleToString(const HAConfig::PeerConfig::Role& role) {
return ("");
}
HAConfig::StateConfig::StateConfig(const int state)
: state_(state), pausing_(STATE_PAUSE_NEVER) {
}
void
HAConfig::StateConfig::setPausing(const std::string& pausing) {
pausing_ = stringToPausing(pausing);
}
StatePausing
HAConfig::StateConfig::stringToPausing(const std::string& pausing) {
if (pausing == "always") {
return (STATE_PAUSE_ALWAYS);
} else if (pausing == "never") {
return (STATE_PAUSE_NEVER);
} else if (pausing == "once") {
return (STATE_PAUSE_ONCE);
}
isc_throw(BadValue, "unsupported value " << pausing << " of 'pause' parameter");
}
std::string
HAConfig::StateConfig::pausingToString(const StatePausing& pausing) {
switch (pausing) {
case STATE_PAUSE_ALWAYS:
return ("always");
case STATE_PAUSE_NEVER:
return ("never");
case STATE_PAUSE_ONCE:
return ("once");
default:
;
}
isc_throw(BadValue, "unsupported pause enumeration " << static_cast<int>(pausing));
}
HAConfig::StateConfigPtr
HAConfig::StateMachineConfig::getStateConfig(const int state) {
// Return config for the state if it exists already.
auto state_config = states_.find(state);
if (state_config != states_.end()) {
return (state_config->second);
}
// Create config for the state and store its pointer.
StateConfigPtr new_state_config(new StateConfig(state));
states_[state] = new_state_config;
return (new_state_config);
}
HAConfig::HAConfig()
: this_server_name_(), ha_mode_(HOT_STANDBY), send_lease_updates_(true),
sync_leases_(true), sync_timeout_(60000), heartbeat_delay_(10000),
max_response_delay_(60000), max_ack_delay_(10000), max_unacked_clients_(10),
peers_() {
peers_(), state_machine_(new StateMachineConfig()) {
}
HAConfig::PeerConfigPtr
......
......@@ -9,6 +9,7 @@
#include <exceptions/exceptions.h>
#include <http/url.h>
#include <util/state_model.h>
#include <boost/shared_ptr.hpp>
#include <cstdint>
#include <map>
......@@ -160,6 +161,87 @@ public:
/// @brief Map of the servers' configurations.
typedef std::map<std::string, PeerConfigPtr> PeerConfigMap;
/// @brief Configuration specific to a single HA state.
class StateConfig {
public:
/// @brief Constructor.
///
/// @param state state identifier.
explicit StateConfig(const int state);
/// @brief Returns identifier of the state.
int getState() const {
return (state_);
}
/// @brief Returns pausing mode for the given state.
util::StatePausing getPausing() const {
return (pausing_);
}
/// @brief Sets pausing mode for the gievn state.
///
/// @param pausing new pausing mode in the textual form. Supported
/// values are: always, never, once.
void setPausing(const std::string& pausing);
/// @brief Converts pausing mode from the textual form.
///
/// @param pausing pausing mode in the textual form. Supported
/// values are: always, never, once.
static util::StatePausing stringToPausing(const std::string& pausing);
/// @brief Returns pausing mode in the textual form.
///
/// @param pausing pausing mode.
static std::string pausingToString(const util::StatePausing& pausing);
private:
/// @brief Idenitifier of state for which configuration is held.
int state_;
/// @brief Pausing mode in the given state.
util::StatePausing pausing_;
};
/// @brief Pointer to the state configuration.
typedef boost::shared_ptr<StateConfig> StateConfigPtr;
/// @brief State machine configuration information.
///
/// Currently it merely contains a collection of states specific
/// configurations. In the future it may also contain global
/// state machine configuration parameters.
class StateMachineConfig {
public:
/// @brief Constructor.
StateMachineConfig()
: states_() {
}
/// @brief Returns pointer to the state specific configuration.
///
/// If requested configuration doesn't exist yet, it is created.
///
/// @param state identifier of the state for which configuration
/// object should be returned.
///
/// @return Pointer to the state configuration.
StateConfigPtr getStateConfig(const int state);
private:
/// @brief Map of configuration for supported states.
std::map<int, StateConfigPtr> states_;
};
/// @brief Pointer to a state machine configuration.
typedef boost::shared_ptr<StateMachineConfig> StateMachineConfigPtr;
/// @brief Constructor.
HAConfig();
......@@ -380,23 +462,29 @@ public:
return (peers_);
}
/// @brief Returns state machine configuration.
///
/// @return Pointer to the state machine configuration.
StateMachineConfigPtr getStateMachineConfig() const {
return (state_machine_);
}
/// @brief Validates configuration.
///
/// @throw HAConfigValidationError if configuration is invalid.
void validate() const;
private:
std::string this_server_name_; ///< This server name.
HAMode ha_mode_; ///< Mode of operation.
bool send_lease_updates_; ///< Send lease updates to partner?
bool sync_leases_; ///< Synchronize databases on startup?
uint32_t sync_timeout_; ///< Timeout for syncing lease database (ms)
uint32_t heartbeat_delay_; ///< Heartbeat delay in milliseconds.
uint32_t max_response_delay_; ///< Max delay in response to heartbeats.
uint32_t max_ack_delay_; ///< Maximum DHCP message ack delay.
uint32_t max_unacked_clients_; ///< Maximum number of unacked clients.
PeerConfigMap peers_; ///< Map of peers' configurations.
std::string this_server_name_; ///< This server name.
HAMode ha_mode_; ///< Mode of operation.
bool send_lease_updates_; ///< Send lease updates to partner?
bool sync_leases_; ///< Synchronize databases on startup?
uint32_t sync_timeout_; ///< Timeout for syncing lease database (ms)
uint32_t heartbeat_delay_; ///< Heartbeat delay in milliseconds.
uint32_t max_response_delay_; ///< Max delay in response to heartbeats.
uint32_t max_ack_delay_; ///< Maximum DHCP message ack delay.
uint32_t max_unacked_clients_; ///< Maximum number of unacked clients.
PeerConfigMap peers_; ///< Map of peers' configurations.
StateMachineConfigPtr state_machine_; ///< State machine configuration.
};
/// @brief Pointer to the High Availability configuration structure.
......
......@@ -8,8 +8,10 @@
#include <ha_config_parser.h>
#include <ha_log.h>
#include <ha_service_states.h>
#include <cc/dhcp_config_error.h>
#include <limits>
#include <set>
using namespace isc::data;
using namespace isc::http;
......@@ -32,6 +34,11 @@ const SimpleDefaults HA_CONFIG_PEER_DEFAULTS = {
{ "auto-failover", Element::boolean, "true" }
};
/// @brief Default values for HA state configuration.
const SimpleDefaults HA_CONFIG_STATE_DEFAULTS = {
{ "pause", Element::string, "never" }
};
} // end of anonymous namespace
namespace isc {
......@@ -96,6 +103,21 @@ HAConfigParser::parseInternal(const HAConfigPtr& config_storage,
isc_throw(ConfigError, "'peers' parameter must be a list");
}
// State machine configuration must be a map.
ConstElementPtr state_machine = c->get("state-machine");
ConstElementPtr states_list;
if (state_machine) {
if (state_machine->getType() != Element::map) {
isc_throw(ConfigError, "'state-machine' parameter must be a map");
}
states_list = state_machine->get("states");
if (states_list && (states_list->getType() != Element::list)) {
isc_throw(ConfigError, "'states' parameter must be a list");
}
}
// We have made major sanity checks, so let's try to gather some values.
// Get 'this-server-name'.
......@@ -162,6 +184,38 @@ HAConfigParser::parseInternal(const HAConfigPtr& config_storage,
cfg->setAutoFailover(getBoolean(*p, "auto-failover"));
}
// Per state configuration is optional.
if (states_list) {
const auto& states_vec = states_list->listValue();
std::set<int> configured_states;
// Go over per state configurations.
for (auto s = states_vec.begin(); s != states_vec.end(); ++s) {
// State configuration is held in map.
if ((*s)->getType() != Element::map) {
isc_throw(ConfigError, "state configuration must be a map");
}
setDefaults(*s, HA_CONFIG_STATE_DEFAULTS);
// Get state name and set per state configuration.
std::string state_name = getString(*s, "state");
int state = stringToState(state_name);
// Check that this configuration doesn't duplicate existing configuration.
if (configured_states.count(state) > 0) {
isc_throw(ConfigError, "duplicated configuration for the '"
<< state_name << "' state");
}
configured_states.insert(state);
config_storage->getStateMachineConfig()->
getStateConfig(state)->setPausing(getString(*s, "pause"));
}
}
// We have gone over the entire configuration and stored it in the configuration
// storage. However, we need to still validate it to detect errors like:
// duplicate secondary/primary servers, no configuration for this server etc.
......
......@@ -296,6 +296,21 @@ of server startup or reconfiguration. The first argument provides the HA mode.
The second argument specifies the role of this server instance in this
configuration.
% HA_STATE_MACHINE_CONTINUED state machine is un-paused
This informational message is issued when the HA state machine is un-paused.
This unlocks the server from the current state. It may transition to any
other state if it needs to do so, e.g. 'partner-down' if its partner appears
to be offline. The server may also remain in the current state if the HA
setup state warrants such behavior.
% HA_STATE_MACHINE_PAUSED state machine paused in state %1
This informational message is issued when the HA state machine is paused.
HA state machine may be paused in certain states specified in the HA hooks library
confuguration. When the state machine is paused, the server remains in the given
state until it is explicitly unpaused (via ha-continue command). If the state
machine is paused, the server operates normally but can't transition to any
other state.
% HA_STATE_TRANSITION server transitions from %1 to %2 state, partner state is %3
This informational message is issued when the server transitions to a new
state as a result of some interaction (or lack of thereof) with its partner.
......
......@@ -85,29 +85,37 @@ void
HAService::defineStates() {
StateModel::defineStates();
defineState(HA_BACKUP_ST, "backup",
boost::bind(&HAService::backupStateHandler, this));
defineState(HA_BACKUP_ST, stateToString(HA_BACKUP_ST),
boost::bind(&HAService::backupStateHandler, this),
config_->getStateMachineConfig()->getStateConfig(HA_BACKUP_ST)->getPausing());
defineState(HA_HOT_STANDBY_ST, "hot-standby",
boost::bind(&HAService::normalStateHandler, this));
defineState(HA_HOT_STANDBY_ST, stateToString(HA_HOT_STANDBY_ST),
boost::bind(&HAService::normalStateHandler, this),
config_->getStateMachineConfig()->getStateConfig(HA_HOT_STANDBY_ST)->getPausing());
defineState(HA_LOAD_BALANCING_ST, "load-balancing",
boost::bind(&HAService::normalStateHandler, this));
defineState(HA_LOAD_BALANCING_ST, stateToString(HA_LOAD_BALANCING_ST),
boost::bind(&HAService::normalStateHandler, this),
config_->getStateMachineConfig()->getStateConfig(HA_LOAD_BALANCING_ST)->getPausing());
defineState(HA_PARTNER_DOWN_ST, "partner-down",
boost::bind(&HAService::partnerDownStateHandler, this));
defineState(HA_PARTNER_DOWN_ST, stateToString(HA_PARTNER_DOWN_ST),
boost::bind(&HAService::partnerDownStateHandler, this),
config_->getStateMachineConfig()->getStateConfig(HA_PARTNER_DOWN_ST)->getPausing());
defineState(HA_READY_ST, "ready",
boost::bind(&HAService::readyStateHandler, this));
defineState(HA_READY_ST, stateToString(HA_READY_ST),
boost::bind(&HAService::readyStateHandler, this),
config_->getStateMachineConfig()->getStateConfig(HA_READY_ST)->getPausing());
defineState(HA_SYNCING_ST, "syncing",
boost::bind(&HAService::syncingStateHandler, this));
defineState(HA_SYNCING_ST, stateToString(HA_SYNCING_ST),
boost::bind(&HAService::syncingStateHandler, this),
config_->getStateMachineConfig()->getStateConfig(HA_SYNCING_ST)->getPausing());
defineState(HA_TERMINATED_ST, "terminated",
boost::bind(&HAService::terminatedStateHandler, this));
defineState(HA_TERMINATED_ST, stateToString(HA_TERMINATED_ST),
boost::bind(&HAService::terminatedStateHandler, this),
config_->getStateMachineConfig()->getStateConfig(HA_TERMINATED_ST)->getPausing());
defineState(HA_WAITING_ST, "waiting",
boost::bind(&HAService::waitingStateHandler, this));
defineState(HA_WAITING_ST, stateToString(HA_WAITING_ST),
boost::bind(&HAService::waitingStateHandler, this),
config_->getStateMachineConfig()->getStateConfig(HA_WAITING_ST)->getPausing());
}
void
......@@ -134,6 +142,11 @@ HAService::normalStateHandler() {
scheduleHeartbeat();
if (isModelPaused()) {
postNextEvent(NOP_EVT);
return;
}
// Check if the clock skew is still acceptable. If not, transition to
// the terminated state.
if (shouldTerminate()) {
......@@ -184,6 +197,11 @@ HAService::partnerDownStateHandler() {
scheduleHeartbeat();
if (isModelPaused()) {
postNextEvent(NOP_EVT);
return;
}
// Check if the clock skew is still acceptable. If not, transition to
// the terminated state.
if (shouldTerminate()) {
......@@ -224,6 +242,11 @@ HAService::readyStateHandler() {
scheduleHeartbeat();
if (isModelPaused()) {
postNextEvent(NOP_EVT);
return;
}
// Check if the clock skew is still acceptable. If not, transition to
// the terminated state.
if (shouldTerminate()) {
......@@ -279,6 +302,11 @@ HAService::syncingStateHandler() {
adjustNetworkState();
}
if (isModelPaused()) {
postNextEvent(NOP_EVT);
return;
}
// Check if the clock skew is still acceptable. If not, transition to
// the terminated state.
if (shouldTerminate()) {
......@@ -363,14 +391,22 @@ HAService::waitingStateHandler() {
adjustNetworkState();
}
// Only schedule the heartbeat for non-backup servers.
if (config_->getThisServerConfig()->getRole() != HAConfig::PeerConfig::BACKUP) {
scheduleHeartbeat();
}
if (isModelPaused()) {
postNextEvent(NOP_EVT);
return;
}
// Backup server must remain in its own state.
if (config_->getThisServerConfig()->getRole() == HAConfig::PeerConfig::BACKUP) {
verboseTransition(HA_BACKUP_ST);
return;
}
scheduleHeartbeat();
// Check if the clock skew is still acceptable. If not, transition to
// the terminated state.
if (shouldTerminate()) {
......@@ -475,6 +511,24 @@ HAService::verboseTransition(const unsigned state) {
.arg(new_state_name);
}
}
// Inform the administrator if the state machine is paused.
if (isModelPaused()) {
std::string state_name = stateToString(state);
boost::to_upper(state_name);
LOG_INFO(ha_logger, HA_STATE_MACHINE_PAUSED)
.arg(state_name);
}
}
bool
HAService::unpause() {
if (isModelPaused()) {
LOG_INFO(ha_logger, HA_STATE_MACHINE_CONTINUED);
unpauseModel();
return (true);
}
return (false);
}
void
......
......@@ -246,6 +246,15 @@ protected:
public:
/// @brief Unpauses the HA state machine with logging.
///
/// It un-pauses the state machine if it is paused and logs an informational
/// message. It doesn't log the message if the state machine is not paused.
///
/// @return true if the state machine was unpaused, false if the state
/// machine was not paused when this method was invoked.
bool unpause();
/// @brief Instructs the HA service to serve default scopes.
///
/// This method is mostly useful for unit testing. The scopes need to be
......
// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
#include <ha_service_states.h>
namespace isc {
namespace ha {
std::string stateToString(int state) {
switch (state) {
case HA_BACKUP_ST:
return ("backup");
case HA_HOT_STANDBY_ST:
return ("hot-standby");
case HA_LOAD_BALANCING_ST:
return ("load-balancing");
case HA_PARTNER_DOWN_ST:
return ("partner-down");
case HA_READY_ST:
return ("ready");
case HA_SYNCING_ST:
return ("syncing");
case HA_TERMINATED_ST:
return ("terminated");
case HA_WAITING_ST:
return ("waiting");
case HA_UNAVAILABLE_ST:
return ("unavailable");
default:
;
}
isc_throw(BadValue, "unknown state identifier " << state);
}
int stringToState(const std::string& state_name) {
if (state_name == "backup") {
return (HA_BACKUP_ST);
} else if (state_name == "hot-standby") {
return (HA_HOT_STANDBY_ST);
} else if (state_name == "load-balancing") {
return (HA_LOAD_BALANCING_ST);
} else if (state_name == "partner-down") {
return (HA_PARTNER_DOWN_ST);
} else if (state_name == "ready") {
return (HA_READY_ST);
} else if (state_name == "syncing") {
return (HA_SYNCING_ST);
} else if (state_name == "terminated") {
return (HA_TERMINATED_ST);
} else if (state_name == "waiting") {
return (HA_WAITING_ST);
} else if (state_name == "unavailable") {
return (HA_UNAVAILABLE_ST);
}
isc_throw(BadValue, "unknown state " << state_name);
}
} // end of namespace isc::ha
} // end of namespace isc
......@@ -8,6 +8,7 @@
#define HA_SERVICE_STATES_H
#include <util/state_model.h>
#include <string>
namespace isc {
namespace ha {
......@@ -40,6 +41,20 @@ const int HA_WAITING_ST = util::StateModel::SM_DERIVED_STATE_MIN + 8;
/// the partner.
const int HA_UNAVAILABLE_ST = util::StateModel::SM_DERIVED_STATE_MIN + 1000;
/// @brief Returns state name.
///
/// @param state state identifier for which name should be returned.
///
/// @throw BadValue if the state identifier is unsupported.
std::string stateToString(int state);
/// @brief Returns state for a given name.
///
/// @param state_name name of the state to be returned.
///
/// @throw BadValue if the state name is unsupported.
int stringToState(const std::string& state_name);
} // end of namespace isc::ha
} // end of namespace isc
......
......@@ -7,11 +7,13 @@
#include <config.h>
#include <ha_impl.h>
#include <ha_service_states.h>
#include <ha_test.h>
#include <cc/command_interpreter.h>
#include <cc/data.h>
#include <cc/dhcp_config_error.h>
#include <config/command_mgr.h>
#include <util/state_model.h>
#include <string>
using namespace isc;
......@@ -20,6 +22,7 @@ using namespace isc::data;
using namespace isc::ha;
using namespace isc::hooks;
using namespace isc::ha::test;
using namespace isc::util;
namespace {
......@@ -43,6 +46,7 @@ public:
HAImplPtr impl(new HAImpl());
try {
impl->configure(Element::fromJSON(invalid_config));
ADD_FAILURE() << "expected ConfigError exception, thrown no exception";
} catch (const ConfigError& ex) {
EXPECT_EQ(expected_error, std::string(ex.what()));
......@@ -86,7 +90,23 @@ TEST_F(HAConfigTest, configureLoadBalancing) {
" \"role\": \"backup\","
" \"auto-failover\": false"
" }"
" ]"
" ],"
" \"state-machine\": {"
" \"states\": ["
" {"
" \"state\": \"waiting\","
" \"pause\": \"once\""
" },"
" {"
" \"state\": \"ready\","
" \"pause\": \"always\""
" },"
" {"
" \"state\": \"partner-down\","
" \"pause\": \"never\""
" }"
" ]"
" }"
" }"
"]";
......@@ -125,6 +145,44 @@ TEST_F(HAConfigTest, configureLoadBalancing) {
EXPECT_EQ(cfg->getLogLabel(), "server3 (http://127.0.0.1:8082/)");
EXPECT_EQ(HAConfig::PeerConfig::BACKUP, cfg->getRole());
EXPECT_FALSE(cfg->isAutoFailover());
// Verify that per-state configuration is correct.x
HAConfig::StateConfigPtr state_cfg;
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
getStateConfig(HA_BACKUP_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());
ASSERT_NO_THROW(state_cfg = impl->getConfig()->getStateMachineConfig()->
getStateConfig(HA_LOAD_BALANCING_ST));
ASSERT_TRUE(state_cfg);
EXPECT_EQ(STATE_PAUSE_NEVER, state_cfg->getPausing());