Commit ca4049e6 authored by Marcin Siodelski's avatar Marcin Siodelski

[5317] TimerMgr's handlers are now ran in the main thread.

parent 0ce848c0
......@@ -494,16 +494,6 @@ ControlledDhcpv4Srv::processConfig(isc::data::ConstElementPtr config) {
return (isc::config::createAnswer(1, err.str()));
}
// We're going to modify the timers configuration. This is not allowed
// when the thread is running.
try {
TimerMgr::instance()->stopThread();
} catch (const std::exception& ex) {
err << "Unable to stop worker thread running timers: "
<< ex.what() << ".";
return (isc::config::createAnswer(1, err.str()));
}
ConstElementPtr answer = configureDhcp4Server(*srv, config);
// Check that configuration was successful. If not, do not reopen sockets
......@@ -573,17 +563,6 @@ ControlledDhcpv4Srv::processConfig(isc::data::ConstElementPtr config) {
return (isc::config::createAnswer(1, err.str()));
}
// Start worker thread if there are any timers installed.
if (TimerMgr::instance()->timersCount() > 0) {
try {
TimerMgr::instance()->startThread();
} catch (const std::exception& ex) {
err << "Unable to start worker thread running timers: "
<< ex.what() << ".";
return (isc::config::createAnswer(1, err.str()));
}
}
return (answer);
}
......@@ -614,6 +593,8 @@ ControlledDhcpv4Srv::ControlledDhcpv4Srv(uint16_t port /*= DHCP4_SERVER_PORT*/)
}
server_ = this; // remember this instance for later use in handlers
TimerMgr::instance()->setIOService(getIOService());
// These are the commands always supported by the DHCPv4 server.
// Please keep the list in alphabetic order.
CommandMgr::instance().registerCommand("build-report",
......@@ -676,9 +657,6 @@ ControlledDhcpv4Srv::~ControlledDhcpv4Srv() {
try {
cleanup();
// Stop worker thread running timers, if it is running. Then
// unregister any timers.
timer_mgr_->stopThread();
timer_mgr_->unregisterTimers();
// Close the command socket (if it exists).
......
......@@ -5,7 +5,6 @@
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
#include <config.h>
#include <asiolink/io_address.h>
#include <dhcp/dhcp4.h>
#include <dhcp/duid.h>
#include <dhcp/hwaddr.h>
......@@ -418,7 +417,7 @@ const std::string Dhcpv4Srv::VENDOR_CLASS_PREFIX("VENDOR_CLASS_");
Dhcpv4Srv::Dhcpv4Srv(uint16_t port, const bool use_bcast,
const bool direct_response_desired)
: shutdown_(true), alloc_engine_(), port_(port),
: io_service_(new IOService()), shutdown_(true), alloc_engine_(), port_(port),
use_bcast_(use_bcast) {
LOG_DEBUG(dhcp4_logger, DBG_DHCP4_START, DHCP4_OPEN_SOCKET).arg(port);
......@@ -717,6 +716,7 @@ Dhcpv4Srv::run() {
while (!shutdown_) {
try {
run_one();
getIOService()->poll();
} catch (const std::exception& e) {
// General catch-all exception that are not caught by more specific
// catches. This one is for exceptions derived from std::exception.
......@@ -740,7 +740,10 @@ Dhcpv4Srv::run_one() {
Pkt4Ptr rsp;
try {
uint32_t timeout = 1000;
// Set select() timeout to 1s. This value should not be modified
// because it is important that the select() returns control
// frequently so as the IOService can be polled for ready handlers.
uint32_t timeout = 1;
LOG_DEBUG(packet4_logger, DBG_DHCP4_DETAIL, DHCP4_BUFFER_WAIT).arg(timeout);
query = receivePacket(timeout);
......
// Copyright (C) 2011-2016 Internet Systems Consortium, Inc. ("ISC")
// Copyright (C) 2011-2017 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
......@@ -7,6 +7,7 @@
#ifndef DHCPV4_SRV_H
#define DHCPV4_SRV_H
#include <asiolink/io_service.h>
#include <dhcp/dhcp4.h>
#include <dhcp/pkt4.h>
#include <dhcp/option.h>
......@@ -188,6 +189,10 @@ typedef boost::shared_ptr<Dhcpv4Exchange> Dhcpv4ExchangePtr;
/// See the derived \ref ControlledDhcpv4Srv class for support for
/// command and configuration updates over msgq.
class Dhcpv4Srv : public Daemon {
private:
/// @brief Pointer to IO service used by the server.
asiolink::IOServicePtr io_service_;
public:
......@@ -222,6 +227,11 @@ public:
/// @brief Destructor. Used during DHCPv4 service shutdown.
virtual ~Dhcpv4Srv();
/// @brief Returns pointer to the IO service used by the server.
asiolink::IOServicePtr& getIOService() {
return (io_service_);
}
/// @brief returns Kea version on stdout and exit.
/// redeclaration/redefinition. @ref Daemon::getVersion()
static std::string getVersion(bool extended);
......
......@@ -6,7 +6,9 @@
#include <config.h>
#include <asiolink/interval_timer.h>
#include <asiolink/io_address.h>
#include <asiolink/io_service.h>
#include <cc/command_interpreter.h>
#include <dhcp/dhcp4.h>
#include <dhcp/hwaddr.h>
......@@ -85,17 +87,15 @@ public:
/// @brief Runs timers for specified time.
///
/// Internally, this method calls @c IfaceMgr::receive4 to run the
/// callbacks for the installed timers.
///
/// @param io_service Pointer to the IO service to be ran.
/// @param timeout_ms Amount of time after which the method returns.
void runTimersWithTimeout(const long timeout_ms) {
isc::util::Stopwatch stopwatch;
while (stopwatch.getTotalMilliseconds() < timeout_ms) {
// Block for up to one millisecond waiting for the timers'
// callbacks to be executed.
IfaceMgr::instancePtr()->receive4(0, 1000);
}
void runTimersWithTimeout(const IOServicePtr& io_service, const long timeout_ms) {
IntervalTimer timer(*io_service);
timer.setup([this, &io_service]() {
io_service->stop();
}, timeout_ms, IntervalTimer::ONE_SHOT);
io_service->run();
io_service->get_io_service().reset();
}
/// Name of a config file used during tests
......@@ -571,7 +571,7 @@ TEST_F(JSONFileBackendTest, timers) {
// Poll the timers for a while to make sure that each of them is executed
// at least once.
ASSERT_NO_THROW(runTimersWithTimeout(5000));
ASSERT_NO_THROW(runTimersWithTimeout(srv->getIOService(), 5000));
// Verify that the leases in the database have been processed as expected.
......
......@@ -495,17 +495,6 @@ ControlledDhcpv6Srv::processConfig(isc::data::ConstElementPtr config) {
return (no_srv);
}
// We're going to modify the timers configuration. This is not allowed
// when the thread is running.
try {
TimerMgr::instance()->stopThread();
} catch (const std::exception& ex) {
std::ostringstream err;
err << "Unable to stop worker thread running timers: "
<< ex.what() << ".";
return (isc::config::createAnswer(1, err.str()));
}
ConstElementPtr answer = configureDhcp6Server(*srv, config);
// Check that configuration was successful. If not, do not reopen sockets
......@@ -594,18 +583,6 @@ ControlledDhcpv6Srv::processConfig(isc::data::ConstElementPtr config) {
return (isc::config::createAnswer(1, err.str()));
}
// Start worker thread if there are any timers installed.
if (TimerMgr::instance()->timersCount() > 0) {
try {
TimerMgr::instance()->startThread();
} catch (const std::exception& ex) {
std::ostringstream err;
err << "Unable to start worker thread running timers: "
<< ex.what() << ".";
return (isc::config::createAnswer(1, err.str()));
}
}
// Finally, we can commit runtime option definitions in libdhcp++. This is
// exception free.
LibDHCP::commitRuntimeOptionDefs();
......@@ -638,6 +615,8 @@ ControlledDhcpv6Srv::ControlledDhcpv6Srv(uint16_t port)
}
server_ = this; // remember this instance for use in callback
TimerMgr::instance()->setIOService(getIOService());
// These are the commands always supported by the DHCPv6 server.
// Please keep the list in alphabetic order.
CommandMgr::instance().registerCommand("build-report",
......@@ -699,9 +678,6 @@ ControlledDhcpv6Srv::~ControlledDhcpv6Srv() {
try {
cleanup();
// Stop worker thread running timers, if it is running. Then
// unregister any timers.
timer_mgr_->stopThread();
timer_mgr_->unregisterTimers();
// Close the command socket (if it exists).
......
......@@ -177,7 +177,8 @@ namespace dhcp {
const std::string Dhcpv6Srv::VENDOR_CLASS_PREFIX("VENDOR_CLASS_");
Dhcpv6Srv::Dhcpv6Srv(uint16_t port)
: port_(port), serverid_(), shutdown_(true), alloc_engine_()
: io_service_(new IOService()), port_(port), serverid_(), shutdown_(true),
alloc_engine_()
{
LOG_DEBUG(dhcp6_logger, DBG_DHCP6_START, DHCP6_OPEN_SOCKET).arg(port);
......@@ -374,6 +375,7 @@ bool Dhcpv6Srv::run() {
while (!shutdown_) {
try {
run_one();
getIOService()->poll();
} catch (const std::exception& e) {
// General catch-all standard exceptions that are not caught by more
// specific catches.
......@@ -395,7 +397,10 @@ void Dhcpv6Srv::run_one() {
Pkt6Ptr rsp;
try {
uint32_t timeout = 1000;
// Set select() timeout to 1s. This value should not be modified
// because it is important that the select() returns control
// frequently so as the IOService can be polled for ready handlers.
uint32_t timeout = 1;
LOG_DEBUG(packet6_logger, DBG_DHCP6_DETAIL, DHCP6_BUFFER_WAIT).arg(timeout);
query = receivePacket(timeout);
......
......@@ -7,6 +7,7 @@
#ifndef DHCPV6_SRV_H
#define DHCPV6_SRV_H
#include <asiolink/io_service.h>
#include <dhcp_ddns/ncr_msg.h>
#include <dhcp/dhcp6.h>
#include <dhcp/duid.h>
......@@ -53,6 +54,10 @@ public:
/// packets, processes them, manages leases assignment and generates
/// appropriate responses.
class Dhcpv6Srv : public Daemon {
private:
/// @brief Pointer to IO service used by the server.
asiolink::IOServicePtr io_service_;
public:
/// @brief defines if certain option may, must or must not appear
......@@ -78,6 +83,11 @@ public:
/// @brief Destructor. Used during DHCPv6 service shutdown.
virtual ~Dhcpv6Srv();
/// @brief Returns pointer to the IO service used by the server.
asiolink::IOServicePtr& getIOService() {
return (io_service_);
}
/// @brief returns Kea version on stdout and exit.
/// redeclaration/redefinition. @ref Daemon::getVersion()
static std::string getVersion(bool extended);
......
......@@ -74,17 +74,15 @@ public:
/// @brief Runs timers for specified time.
///
/// Internally, this method calls @c IfaceMgr::receive6 to run the
/// callbacks for the installed timers.
///
/// @param io_service Pointer to the IO service to be ran.
/// @param timeout_ms Amount of time after which the method returns.
void runTimersWithTimeout(const long timeout_ms) {
isc::util::Stopwatch stopwatch;
while (stopwatch.getTotalMilliseconds() < timeout_ms) {
// Block for up to one millisecond waiting for the timers'
// callbacks to be executed.
IfaceMgr::instancePtr()->receive6(0, 1000);
}
void runTimersWithTimeout(const IOServicePtr& io_service, const long timeout_ms) {
IntervalTimer timer(*io_service);
timer.setup([this, &io_service]() {
io_service->stop();
}, timeout_ms, IntervalTimer::ONE_SHOT);
io_service->run();
io_service->get_io_service().reset();
}
static const char* TEST_FILE;
......@@ -521,7 +519,7 @@ TEST_F(JSONFileBackendTest, timers) {
// Poll the timers for a while to make sure that each of them is executed
// at least once.
ASSERT_NO_THROW(runTimersWithTimeout(5000));
ASSERT_NO_THROW(runTimersWithTimeout(srv->getIOService(), 5000));
// Verify that the leases in the database have been processed as expected.
......
......@@ -76,8 +76,8 @@ IntervalTimerImpl::setup(const IntervalTimer::Callback& cbfunc,
const long interval,
const IntervalTimer::Mode& mode)
{
// Interval should not be less than or equal to 0.
if (interval <= 0) {
// Interval should not be less than 0.
if (interval < 0) {
isc_throw(isc::BadValue, "Interval should not be less than or "
"equal to 0");
}
......
......@@ -154,8 +154,7 @@ TEST_F(IntervalTimerTest, invalidArgumentToIntervalTimer) {
// expect throw if call back function is empty
EXPECT_THROW(itimer.setup(IntervalTimer::Callback(), 1),
isc::InvalidParameter);
// expect throw if interval is not greater than 0
EXPECT_THROW(itimer.setup(TimerCallBack(this), 0), isc::BadValue);
// expect throw if interval is negative.
EXPECT_THROW(itimer.setup(TimerCallBack(this), -1), isc::BadValue);
}
......
......@@ -878,45 +878,12 @@ An example of such operation is a periodic cleanup of
expired leases. The name of the timer is included in the
message.
% DHCPSRV_TIMERMGR_SOCKET_CLEAR_FAILED clearing watch socket for timer %1 failed: %2
An error message indicating that the specified timer elapsed,
the operation associated with the timer was executed but the
server was unable to signal this to the worker thread responsible
for dispatching timers. The thread will continue but it will
not be able to dispatch any operations for this timer. The
server reconfiguration or restart may solve the problem
but the situation may repeat.
% DHCPSRV_TIMERMGR_SOCKET_MARK_FAILED marking watch socket for timer %1 failed: %2
An error message indicating that the specified timer elapsed,
but the server was unable to flag that the handler function
should be executed for this timer. The callback will not
be executed this time and most likely the subsequent attempts
will not be successful too. This error is highly unlikely.
The name of the timer and the reason for failure is included
in the message.
% DHCPSRV_TIMERMGR_START_THREAD starting thread for timers
A debug message issued when the Timer Manager is starting a
worker thread to run started timers. The worker thread is
typically started right after all timers have been registered
and runs until timers need to be reconfigured, e.g. their
interval is changed, new timers are registered or existing
timers are unregistered.
% DHCPSRV_TIMERMGR_START_TIMER starting timer: %1
A debug message issued when the registered interval timer is
being started. If this operation is successful the timer will
periodically execute the operation associated with it. The
name of the started timer is included in the message.
% DHCPSRV_TIMERMGR_STOP_THREAD stopping thread for timers
A debug message issued when the Timer Manager is stopping
the worker thread which executes interval timers. When the
thread is stopped no timers will be executed. The thread is
typically stopped at the server reconfiguration or when the
server shuts down.
% DHCPSRV_TIMERMGR_STOP_TIMER stopping timer: %1
A debug message issued when the registered interval timer is
being stopped. The timer remains registered and can be restarted
......
......@@ -272,54 +272,28 @@ the server.
@section timerManager Timer Manager
The fundamental role of the DHCP server is to receive and process DHCP
messages received over the sockets opened on network interfaces. The
servers' include the main loops in which the servers passively wait
for the messages. This is done by calling the
@c isc::dhcp::IfaceMgr::receive4 and/or @c isc::dhcp::IfaceMgr::receive6
methods for DHCPv4 and DHCPv6 server respectively. Internally, these
methods call @c select() on open sockets, which blocks for a
specified amount of time.
The implication of using the @c select() is that the server has no
means to process any "events" while it is waiting for the @c select()
to return. An example of such an event is the expiration of the timer
which controls when the server should detect and process expired
leases.
The @c isc::dhcp::TimerMgr has been created to address the issue of
processing expired leases according to the the dedicated timer.
Nevertheless, this concept is universal and should be used for
all timers which need to be triggered asynchronously, i.e. independently
from processing the DHCP messages.
The @c TimerMgr allows for registering timers and associating them with
user callback functions, which are executed without waiting for the
call to the @c select() function to return as a result of the timeout.
When the particular timer elapses, the blocking call to select is
interrupted by sending data over a dedicated (for a timer)
@c isc::util::WatchSocket. Each timer has an instance of
@c isc::util::WatchSocket associated with it, and each such socket
is registered with the @c IfaceMgr using the @c IfaceMgr::addExternalSocket.
When the transmission of the data over the watch socket interrupts the
@c select() call, the user callback is executed by
@c isc::dhcp::IfaceMgr and the watch socket is cleared to accept
subsequent events for that particular timer.
The timers are implemented using the @c isc::asiolink::IntervalTimer class.
They are run in a dedicated thread which is owned (created and destroyed)
by @c isc::dhcp::TimerMgr. This worker thread runs an instance
of @c isc::asiolink::IOService object which is associated with all
registered timers. The thread uses a common callback function which
is executed when a timer elapses. This callback function receives
a name of the elapsed timer as an argument and, based on that, selects the
appropriate @c isc::util::WatchSocket to be marked as ready. In order to
overcome the race conditions with the main thread, the worker thread blocks
right after it marks the watch socket as ready, and waits for this
socket to be cleared by the main thread. This is the indication
that the timer specific callback function has been invoked and the
worker thread may continue monitoring registered timers and signal
their readiness when they elapse.
The @c isc::dhcp::TimerMgr is a singleton class used throughout the
server process to register and unregister timers triggering periodic
tasks such as lease file cleanup, reclamation of expired leases etc.
The Timer Manger is using ASIO deadline timers (wrapped in
@c isc::asiolink::IntervalTimer class) to execute tasks according to
the configured periods. Therefore, the server process must provide the
Timer Manager with the pointer to the @c isc::asiolink::IOService which
the server is using to run asynchronous tasks.
Current implementation of the DHCP servers uses synchronous calls to
@c select() function to check if any transmission has been received
on any socket. This poses a problem with running asynchronous calls
via @c IOService in the main server loop because the @c select()
blocks for a specified amount of time while asynchronous calls
are not triggered. In the future we should migrate from the synchronous
@c select() calls into asynchonous calls using ASIO. Currently,
we mitigate the problem by lowering the @c select() timeout to 1s,
and polling @c IOService for "ready" timers (handlers) after
@c select() returns. This may cause delays of "ready" handlers
execution by around 1s. However, this is acceptable for the current
applications of the periodic timers.
@section leaseReclamationRoutine Leases Reclamation Routine
......
......@@ -127,12 +127,6 @@ LFCSetup::LFCSetup(asiolink::IntervalTimer::Callback callback)
LFCSetup::~LFCSetup() {
try {
// If we're here it means that either the process is terminating
// or we're reconfiguring the server. In both cases the thread has
// probably been stopped already, but we make sure by calling
// stopThread explicitly here.
timer_mgr_->stopThread();
// Remove the timer. This will throw an exception if the timer does not
// exist. There are several possible reasons for this:
// a) It hasn't been registered (although if the LFC Setup instance
......
......@@ -5,20 +5,20 @@
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
#include <config.h>
#include <dhcp/iface_mgr.h>
#include <asiolink/asio_wrapper.h>
#include <asiolink/interval_timer.h>
#include <dhcpsrv/cfg_expiration.h>
#include <dhcpsrv/timer_mgr.h>
#include <exceptions/exceptions.h>
#include <testutils/test_to_element.h>
#include <util/stopwatch.h>
#include <boost/function.hpp>
#include <boost/shared_ptr.hpp>
#include <gtest/gtest.h>
#include <stdint.h>
using namespace isc;
using namespace isc::asiolink;
using namespace isc::dhcp;
using namespace isc::util;
namespace {
......@@ -297,7 +297,8 @@ public:
/// of the class members, it also stops the @c TimerMgr worker thread
/// and removes any registered timers.
CfgExpirationTimersTest()
: timer_mgr_(TimerMgr::instance()),
: io_service_(new IOService()),
timer_mgr_(TimerMgr::instance()),
stub_(new LeaseReclamationStub()),
cfg_(true) {
cleanupTimerMgr();
......@@ -313,8 +314,8 @@ public:
/// @brief Stop @c TimerMgr worker thread and remove the timers.
void cleanupTimerMgr() const {
timer_mgr_->stopThread();
timer_mgr_->unregisterTimers();
timer_mgr_->setIOService(io_service_);
}
/// @brief Runs timers for specified time.
......@@ -324,12 +325,13 @@ public:
///
/// @param timeout_ms Amount of time after which the method returns.
void runTimersWithTimeout(const long timeout_ms) {
Stopwatch stopwatch;
while (stopwatch.getTotalMilliseconds() < timeout_ms) {
// Block for up to one millisecond waiting for the timers'
// callbacks to be executed.
IfaceMgr::instancePtr()->receive6(0, 1000);
}
IntervalTimer timer(*io_service_);
timer.setup([this]() {
io_service_->stop();
}, timeout_ms, IntervalTimer::ONE_SHOT);
io_service_->run();
io_service_->get_io_service().reset();
}
/// @brief Setup timers according to the configuration and run them
......@@ -342,12 +344,13 @@ public:
stub_.get());
// Run timers.
ASSERT_NO_THROW({
timer_mgr_->startThread();
runTimersWithTimeout(timeout_ms);
timer_mgr_->stopThread();
});
}
/// @brief Pointer to the IO service used by the tests.
IOServicePtr io_service_;
/// @brief Pointer to the @c TimerMgr.
TimerMgrPtr timer_mgr_;
......
......@@ -6,6 +6,7 @@
#include <config.h>
#include <asiolink/asio_wrapper.h>
#include <asiolink/interval_timer.h>
#include <asiolink/io_address.h>
#include <dhcp/duid.h>
#include <dhcp/iface_mgr.h>
......@@ -102,8 +103,11 @@ public:
MemfileLeaseMgrTest() :
io4_(getLeaseFilePath("leasefile4_0.csv")),
io6_(getLeaseFilePath("leasefile6_0.csv")),
io_service_(new IOService()),
timer_mgr_(TimerMgr::instance()) {
timer_mgr_->setIOService(io_service_);
std::ostringstream s;
s << KEA_LFC_BUILD_DIR << "/kea-lfc";
setenv("KEA_LFC_EXECUTABLE", s.str().c_str(), 1);
......@@ -130,7 +134,6 @@ public:
/// destroys lease manager backend.
virtual ~MemfileLeaseMgrTest() {
// Stop TimerMgr worker thread if it is running.
timer_mgr_->stopThread();
// Make sure there are no timers registered.
timer_mgr_->unregisterTimers();
LeaseMgrFactory::destroy();
......@@ -207,12 +210,13 @@ public:
///
/// @param ms Duration in milliseconds.
void setTestTime(const uint32_t ms) {
// Measure test time and exit if timeout hit.
Stopwatch stopwatch;
while (stopwatch.getTotalMilliseconds() < ms) {
// Block for one 1 millisecond.
IfaceMgr::instance().receive6(0, 1000);
}
IntervalTimer timer(*io_service_);
timer.setup([this]() {
io_service_->stop();
}, ms, IntervalTimer::ONE_SHOT);
io_service_->run();
io_service_->get_io_service().reset();
}
/// @brief Waits for the specified process to finish.
......@@ -344,6 +348,9 @@ public:
/// @brief Object providing access to v6 lease IO.
LeaseFileIO io6_;
/// @brief Pointer to the IO service used by the tests.
IOServicePtr io_service_;
/// @brief Pointer to the instance of the @c TimerMgr.
TimerMgrPtr timer_mgr_;
};
......@@ -453,15 +460,9 @@ TEST_F(MemfileLeaseMgrTest, lfcTimer) {
boost::scoped_ptr<LFCMemfileLeaseMgr>
lease_mgr(new LFCMemfileLeaseMgr(pmap));