Commit 8d77665f authored by Francis Dupont's avatar Francis Dupont
Browse files

[master] Merge branch 'master' of ssh://git.kea.isc.org/git/kea

parents db57e8b0 797fada9
1046. [func] tmark
Upon startup Kea servers will now detect memfile lease files
that need upgrading, and will launch in instance of the LFC
to convert them to the most current memfile schema version.
(Trac #3601, git ce4b0e42e8a01bbf3b58fdb1f505bbd6e2fad134)
1045. [func] tmark
Added classes for storing client class definitions to libdhcpsrv.
(Trac #4095, git 1039a942450e2a45a1e1aa9924cae4fdbd1541fe)
......
......@@ -150,6 +150,27 @@
will create an empty lease file if one is not
present. Necessary disk write permission is required.
</para>
<section id="memfile-upgrade">
<title>Upgrading Memfile Lease Files from an Earlier Version of Kea</title>
<para>
There are no special steps required to upgrade memfile lease files
from an earlier version of Kea to a new version of Kea.
During startup the servers will check the schema version of the lease
files against their own. If there is a mismatch, the servers will
automatically launch the LFC process to convert the files to the
server's schema version. While this mechanism is primarily meant to
ease the process of upgrading to newer versions of Kea, it can also
be used for downgrading should the need arise. When upgrading, any
values not present in the original lease files will be assigned
appropriate default values. When downgrading, any data present in
the files but not in the server's schema will be dropped.
If you wish to convert the files manually, prior to starting the
servers you may do so by running the LFC process yourself.
See <xref linkend="kea-lfc"/> for more information.
</para>
</section>
<!-- @todo: document lease file upgrades once they are implemented in kea-admin -->
</section>
......
......@@ -22,14 +22,14 @@ namespace isc {
namespace dhcp {
CSVLeaseFile4::CSVLeaseFile4(const std::string& filename)
: CSVFile(filename) {
: VersionedCSVFile(filename) {
initColumns();
}
void
CSVLeaseFile4::open(const bool seek_to_end) {
// Call the base class to open the file
CSVFile::open(seek_to_end);
VersionedCSVFile::open(seek_to_end);
// and clear any statistics we may have
clearStatistics();
......@@ -62,7 +62,7 @@ CSVLeaseFile4::append(const Lease4& lease) {
row.writeAt(getColumnIndex("state"), lease.state_);
try {
CSVFile::append(row);
VersionedCSVFile::append(row);
} catch (const std::exception&) {
// Catch any errors so we can bump the error counter than rethrow it
++write_errs_;
......@@ -85,7 +85,7 @@ CSVLeaseFile4::next(Lease4Ptr& lease) {
try {
// Get the row of CSV values.
CSVRow row;
CSVFile::next(row);
VersionedCSVFile::next(row);
// The empty row signals EOF.
if (row == CSVFile::EMPTY_ROW()) {
lease.reset();
......@@ -137,16 +137,18 @@ CSVLeaseFile4::next(Lease4Ptr& lease) {
void
CSVLeaseFile4::initColumns() {
addColumn("address");
addColumn("hwaddr");
addColumn("client_id");
addColumn("valid_lifetime");
addColumn("expire");
addColumn("subnet_id");
addColumn("fqdn_fwd");
addColumn("fqdn_rev");
addColumn("hostname");
addColumn("state");
addColumn("address", "1.0");
addColumn("hwaddr", "1.0");
addColumn("client_id", "1.0");
addColumn("valid_lifetime", "1.0");
addColumn("expire", "1.0");
addColumn("subnet_id", "1.0");
addColumn("fqdn_fwd", "1.0");
addColumn("fqdn_rev", "1.0");
addColumn("hostname", "1.0");
addColumn("state", "2.0", "0");
// Any file with less than hostname is invalid
setMinimumValidColumns("hostname");
}
IOAddress
......
......@@ -20,7 +20,7 @@
#include <dhcpsrv/lease.h>
#include <dhcpsrv/subnet.h>
#include <dhcpsrv/lease_file_stats.h>
#include <util/csv_file.h>
#include <util/versioned_csv_file.h>
#include <stdint.h>
#include <string>
#include <time.h>
......@@ -39,7 +39,7 @@ namespace dhcp {
/// validation (see http://kea.isc.org/ticket/2405). However, when #2405
/// is implemented, the @c next function may need to be updated to use the
/// validation capablity of @c Lease4.
class CSVLeaseFile4 : public isc::util::CSVFile, public LeaseFileStats {
class CSVLeaseFile4 : public isc::util::VersionedCSVFile, public LeaseFileStats {
public:
/// @brief Constructor.
......
......@@ -23,14 +23,14 @@ namespace isc {
namespace dhcp {
CSVLeaseFile6::CSVLeaseFile6(const std::string& filename)
: CSVFile(filename) {
: VersionedCSVFile(filename) {
initColumns();
}
void
CSVLeaseFile6::open(const bool seek_to_end) {
// Call the base class to open the file
CSVFile::open(seek_to_end);
VersionedCSVFile::open(seek_to_end);
// and clear any statistics we may have
clearStatistics();
......@@ -61,7 +61,7 @@ CSVLeaseFile6::append(const Lease6& lease) {
}
row.writeAt(getColumnIndex("state"), lease.state_);
try {
CSVFile::append(row);
VersionedCSVFile::append(row);
} catch (const std::exception&) {
// Catch any errors so we can bump the error counter than rethrow it
++write_errs_;
......@@ -84,7 +84,7 @@ CSVLeaseFile6::next(Lease6Ptr& lease) {
try {
// Get the row of CSV values.
CSVRow row;
CSVFile::next(row);
VersionedCSVFile::next(row);
// The empty row signals EOF.
if (row == CSVFile::EMPTY_ROW()) {
lease.reset();
......@@ -122,20 +122,23 @@ CSVLeaseFile6::next(Lease6Ptr& lease) {
void
CSVLeaseFile6::initColumns() {
addColumn("address");
addColumn("duid");
addColumn("valid_lifetime");
addColumn("expire");
addColumn("subnet_id");
addColumn("pref_lifetime");
addColumn("lease_type");
addColumn("iaid");
addColumn("prefix_len");
addColumn("fqdn_fwd");
addColumn("fqdn_rev");
addColumn("hostname");
addColumn("hwaddr");
addColumn("state");
addColumn("address", "1.0");
addColumn("duid", "1.0");
addColumn("valid_lifetime", "1.0");
addColumn("expire", "1.0");
addColumn("subnet_id", "1.0");
addColumn("pref_lifetime", "1.0");
addColumn("lease_type", "1.0");
addColumn("iaid", "1.0");
addColumn("prefix_len", "1.0");
addColumn("fqdn_fwd", "1.0");
addColumn("fqdn_rev", "1.0");
addColumn("hostname", "1.0");
addColumn("hwaddr", "2.0");
addColumn("state", "3.0", "0");
// Any file with less than hostname is invalid
setMinimumValidColumns("hostname");
}
Lease::Type
......
......@@ -20,7 +20,7 @@
#include <dhcpsrv/lease.h>
#include <dhcpsrv/subnet.h>
#include <dhcpsrv/lease_file_stats.h>
#include <util/csv_file.h>
#include <util/versioned_csv_file.h>
#include <stdint.h>
#include <string>
......@@ -38,7 +38,7 @@ namespace dhcp {
/// validation (see http://kea.isc.org/ticket/2405). However, when #2405
/// is implemented, the @c next function may need to be updated to use the
/// validation capablity of @c Lease6.
class CSVLeaseFile6 : public isc::util::CSVFile, public LeaseFileStats {
class CSVLeaseFile6 : public isc::util::VersionedCSVFile, public LeaseFileStats {
public:
/// @brief Constructor.
......
......@@ -220,6 +220,13 @@ with the specified address to the memory file backend database.
The code has issued a commit call. For the memory file database, this is
a no-op.
% DHCPRSV_MEMFILE_CONVERTING_LEASE_FILES running LFC now to convert lease files to the current schema: %1.%2
A warning message issued when the server has detected lease files that need
to be either upgraded or downgraded to match the server's schema, and that
the server is automatically running the LFC process to perform the conversion.
This should only occur the first time the server is launched following a Kea
installation upgrade (or downgrade).
% DHCPSRV_MEMFILE_DB opening memory file lease database: %1
This informational message is logged when a DHCP server (either V4 or
V6) is about to open a memory file lease database. The parameters of
......@@ -352,6 +359,21 @@ timer used for lease file cleanup scheduling. This is highly unlikely
and indicates programming error. The message include the reason for this
error.
% DHCPSRV_MEMFILE_NEEDS_DOWNGRADING version of lease file: %1 schema is later than version %2
A warning message issued when the schema of the lease file loaded by the server
is newer than the memfile schema of the server. The server converts the lease
data from newer schemas to its schema as it is read, therefore the lease
information in use by the server will be correct. Note though, that any data
data stored in newer schema fields will be dropped. What remains is for the
file itself to be rewritten using the current schema.
% DHCPSRV_MEMFILE_NEEDS_UPGRADING version of lease file: %1 schema is earlier than version %2
A warning message issued when the schema of the lease file loaded by the server
pre-dates the memfile schema of the server. Note that the server converts the
lease data from older schemas to the current schema as it is read, therefore
the lease information in use by the server will be correct. What remains is
for the file itself to be rewritten using the current schema.
% DHCPSRV_MEMFILE_NO_STORAGE running in non-persistent mode, leases will be lost after restart
A warning message issued when writes of leases to disk have been disabled
in the configuration. This mode is useful for some kinds of performance
......
......@@ -17,7 +17,7 @@
#include <dhcpsrv/dhcpsrv_log.h>
#include <dhcpsrv/memfile_lease_storage.h>
#include <util/csv_file.h>
#include <util/versioned_csv_file.h>
#include <boost/shared_ptr.hpp>
......@@ -154,6 +154,16 @@ public:
}
}
if (lease_file.needsConversion()) {
LOG_WARN(dhcpsrv_logger,
(lease_file.getInputSchemaState()
== util::VersionedCSVFile::NEEDS_UPGRADE
? DHCPSRV_MEMFILE_NEEDS_UPGRADING
: DHCPSRV_MEMFILE_NEEDS_DOWNGRADING))
.arg(lease_file.getFilename())
.arg(lease_file.getSchemaVersion());
}
if (close_file_on_exit) {
lease_file.close();
}
......
......@@ -90,9 +90,13 @@ public:
/// or NULL. If this is NULL, the @c lease_file6 must be non-null.
/// @param lease_file6 A pointer to the DHCPv6 lease file to be cleaned up
/// or NULL. If this is NULL, the @c lease_file4 must be non-null.
/// @param run_once_now A flag that causes LFC to be invoked immediately,
/// regardless of the value of lfc_interval. This is primarily used to
/// cause lease file schema upgrades upon startup.
void setup(const uint32_t lfc_interval,
const boost::shared_ptr<CSVLeaseFile4>& lease_file4,
const boost::shared_ptr<CSVLeaseFile6>& lease_file6);
const boost::shared_ptr<CSVLeaseFile6>& lease_file6,
bool run_once_now = false);
/// @brief Spawns a new process.
void execute();
......@@ -155,58 +159,67 @@ LFCSetup::~LFCSetup() {
void
LFCSetup::setup(const uint32_t lfc_interval,
const boost::shared_ptr<CSVLeaseFile4>& lease_file4,
const boost::shared_ptr<CSVLeaseFile6>& lease_file6) {
const boost::shared_ptr<CSVLeaseFile6>& lease_file6,
bool run_once_now) {
// If LFC is enabled, we have to setup the interval timer and prepare for
// executing the kea-lfc process.
if (lfc_interval > 0) {
std::string executable;
char* c_executable = getenv(KEA_LFC_EXECUTABLE_ENV_NAME);
if (c_executable == NULL) {
executable = KEA_LFC_EXECUTABLE;
// If to nothing to do, punt
if (lfc_interval == 0 && !run_once_now) {
return;
}
} else {
executable = c_executable;
}
// Start preparing the command line for kea-lfc.
std::string executable;
char* c_executable = getenv(KEA_LFC_EXECUTABLE_ENV_NAME);
if (c_executable == NULL) {
executable = KEA_LFC_EXECUTABLE;
} else {
executable = c_executable;
}
// Start preparing the command line for kea-lfc.
// Gather the base file name.
std::string lease_file = lease_file4 ? lease_file4->getFilename() :
lease_file6->getFilename();
// Create the other names by appending suffixes to the base name.
util::ProcessArgs args;
// Universe: v4 or v6.
args.push_back(lease_file4 ? "-4" : "-6");
// Previous file.
args.push_back("-x");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_PREVIOUS));
// Input file.
args.push_back("-i");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_INPUT));
// Output file.
args.push_back("-o");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_OUTPUT));
// Finish file.
args.push_back("-f");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_FINISH));
// PID file.
args.push_back("-p");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_PID));
// The configuration file is currently unused.
args.push_back("-c");
args.push_back("ignored-path");
// Create the process (do not start it yet).
process_.reset(new util::ProcessSpawn(executable, args));
// Gather the base file name.
std::string lease_file = lease_file4 ? lease_file4->getFilename() :
lease_file6->getFilename();
// Create the other names by appending suffixes to the base name.
util::ProcessArgs args;
// Universe: v4 or v6.
args.push_back(lease_file4 ? "-4" : "-6");
// Previous file.
args.push_back("-x");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_PREVIOUS));
// Input file.
args.push_back("-i");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_INPUT));
// Output file.
args.push_back("-o");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_OUTPUT));
// Finish file.
args.push_back("-f");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_FINISH));
// PID file.
args.push_back("-p");
args.push_back(Memfile_LeaseMgr::appendSuffix(lease_file,
Memfile_LeaseMgr::FILE_PID));
// The configuration file is currently unused.
args.push_back("-c");
args.push_back("ignored-path");
// Create the process (do not start it yet).
process_.reset(new util::ProcessSpawn(executable, args));
// If we've been told to run it once now, invoke the callback directly.
if (run_once_now) {
callback_();
}
// If it's suposed to run periodically, setup that now.
if (lfc_interval > 0) {
// Set the timer to call callback function periodically.
LOG_INFO(dhcpsrv_logger, DHCPSRV_MEMFILE_LFC_SETUP).arg(lfc_interval);
......@@ -253,19 +266,25 @@ const int Memfile_LeaseMgr::MINOR_VERSION;
Memfile_LeaseMgr::Memfile_LeaseMgr(const DatabaseConnection::ParameterMap& parameters)
: LeaseMgr(), lfc_setup_(), conn_(parameters)
{
bool conversion_needed = false;
// Check the universe and use v4 file or v6 file.
std::string universe = conn_.getParameter("universe");
if (universe == "4") {
std::string file4 = initLeaseFilePath(V4);
if (!file4.empty()) {
loadLeasesFromFiles<Lease4, CSVLeaseFile4>(file4, lease_file4_,
storage4_);
conversion_needed = loadLeasesFromFiles<Lease4,
CSVLeaseFile4>(file4,
lease_file4_,
storage4_);
}
} else {
std::string file6 = initLeaseFilePath(V6);
if (!file6.empty()) {
loadLeasesFromFiles<Lease6, CSVLeaseFile6>(file6, lease_file6_,
storage6_);
conversion_needed = loadLeasesFromFiles<Lease6,
CSVLeaseFile6>(file6,
lease_file6_,
storage6_);
}
}
......@@ -275,9 +294,12 @@ Memfile_LeaseMgr::Memfile_LeaseMgr(const DatabaseConnection::ParameterMap& param
// operation.
if (!persistLeases(V4) && !persistLeases(V6)) {
LOG_WARN(dhcpsrv_logger, DHCPSRV_MEMFILE_NO_STORAGE);
} else {
lfcSetup();
if (conversion_needed) {
LOG_WARN(dhcpsrv_logger, DHCPRSV_MEMFILE_CONVERTING_LEASE_FILES)
.arg(MAJOR_VERSION).arg(MINOR_VERSION);
}
lfcSetup(conversion_needed);
}
}
......@@ -867,7 +889,7 @@ Memfile_LeaseMgr::initLeaseFilePath(Universe u) {
}
template<typename LeaseObjectType, typename LeaseFileType, typename StorageType>
void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
bool Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
boost::shared_ptr<LeaseFileType>& lease_file,
StorageType& storage) {
// Check if the instance of the LFC is running right now. If it is
......@@ -885,11 +907,12 @@ void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
storage.clear();
// Load the leasefile.completed, if exists.
bool conversion_needed = false;
lease_file.reset(new LeaseFileType(std::string(filename + ".completed")));
if (lease_file->exists()) {
LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
MAX_LEASE_ERRORS);
conversion_needed = conversion_needed || lease_file->needsConversion();
} else {
// If the leasefile.completed doesn't exist, let's load the leases
// from leasefile.2 and leasefile.1, if they exist.
......@@ -897,12 +920,14 @@ void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
if (lease_file->exists()) {
LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
MAX_LEASE_ERRORS);
conversion_needed = conversion_needed || lease_file->needsConversion();
}
lease_file.reset(new LeaseFileType(appendSuffix(filename, FILE_INPUT)));
if (lease_file->exists()) {
LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
MAX_LEASE_ERRORS);
conversion_needed = conversion_needed || lease_file->needsConversion();
}
}
......@@ -915,6 +940,9 @@ void Memfile_LeaseMgr::loadLeasesFromFiles(const std::string& filename,
lease_file.reset(new LeaseFileType(filename));
LeaseFileLoader::load<LeaseObjectType>(*lease_file, storage,
MAX_LEASE_ERRORS, false);
conversion_needed = conversion_needed || lease_file->needsConversion();
return (conversion_needed);
}
......@@ -942,7 +970,7 @@ Memfile_LeaseMgr::lfcCallback() {
}
void
Memfile_LeaseMgr::lfcSetup() {
Memfile_LeaseMgr::lfcSetup(bool conversion_needed) {
std::string lfc_interval_str = "0";
try {
lfc_interval_str = conn_.getParameter("lfc-interval");
......@@ -958,9 +986,9 @@ Memfile_LeaseMgr::lfcSetup() {
<< lfc_interval_str << " specified");
}
if (lfc_interval > 0) {
if (lfc_interval > 0 || conversion_needed) {
lfc_setup_.reset(new LFCSetup(boost::bind(&Memfile_LeaseMgr::lfcCallback, this)));
lfc_setup_->setup(lfc_interval, lease_file4_, lease_file6_);
lfc_setup_->setup(lfc_interval, lease_file4_, lease_file6_, conversion_needed);
}
}
......
......@@ -118,6 +118,18 @@ public:
/// @brief The sole lease manager constructor
///
/// This method:
/// - Initializes the new instance based on the parameters given
/// - Loads (or creates) the appropriate lease file(s)
/// - Initiates the periodic scheduling of the LFC (if enabled)
///
/// If any of the files loaded require conversion to the current schema
/// (upgrade or downgrade), @c lfcSetup() will be invoked with its
/// @c run_once_now parameter set to true. This causes lfcSetup() to
/// invoke the LFC process immediately regardless of whether LFC is
/// enabled. This ensures that any files which need conversion are
/// converted automatically.
///
/// dbconfig is a generic way of passing parameters. Parameters
/// are passed in the "name=value" format, separated by spaces.
/// Values may be enclosed in double quotes, if needed.
......@@ -549,11 +561,14 @@ private:
/// @tparam LeaseFileType @c CSVLeaseFile4 or @c CSVLeaseFile6.
/// @tparam StorageType @c Lease4Storage or @c Lease6Storage.
///
/// @return Returns true if any of the files loaded need conversion from
/// an older or newer schema.
///
/// @throw CSVFileError when parsing any of the lease files fails.
/// @throw DbOpenError when it is found that the LFC is in progress.
template<typename LeaseObjectType, typename LeaseFileType,
typename StorageType>
void loadLeasesFromFiles(const std::string& filename,
bool loadLeasesFromFiles(const std::string& filename,
boost::shared_ptr<LeaseFileType>& lease_file,
StorageType& storage);
......@@ -626,7 +641,11 @@ private:
/// Kea build directory, the @c KEA_LFC_EXECUTABLE environmental
/// variable should be set to hold an absolute path to the kea-lfc
/// excutable.
void lfcSetup();
/// @param conversion_needed flag that indicates input lease file(s) are
/// schema do not match the current schema (older or newer), and need
/// conversion. This value is passed through to LFCSetup::setup() via its
/// run_once_now parameter.
void lfcSetup(bool conversion_needed = false);
/// @brief Performs a lease file cleanup for DHCPv4 or DHCPv6.
///
......
......@@ -18,8 +18,6 @@
#include <dhcpsrv/csv_lease_file4.h>
#include <dhcpsrv/lease.h>
#include <dhcpsrv/tests/lease_file_io.h>
#include <boost/scoped_ptr.hpp>
#include <boost/shared_ptr.hpp>
#include <gtest/gtest.h>
#include <sstream>
......@@ -125,22 +123,22 @@ TEST_F(CSVLeaseFile4Test, parse) {
writeSampleFile();
// Open the lease file.
boost::scoped_ptr<CSVLeaseFile4> lf(new CSVLeaseFile4(filename_));
ASSERT_NO_THROW(lf->open());
CSVLeaseFile4 lf(filename_);
ASSERT_NO_THROW(lf.open());
// Verify the counters are cleared
{
SCOPED_TRACE("Check stats are empty");
checkStats(*lf, 0, 0, 0, 0, 0, 0);
checkStats(lf, 0, 0, 0, 0, 0, 0);
}
Lease4Ptr lease;
// Reading first read should be successful.
{
SCOPED_TRACE("First lease valid");
EXPECT_TRUE(lf->next(lease));
EXPECT_TRUE(lf.next(lease));
ASSERT_TRUE(lease);
checkStats(*lf, 1, 1, 0, 0, 0, 0);
checkStats(lf, 1, 1, 0, 0, 0, 0);
// Verify that the lease attributes are correct.
EXPECT_EQ("192.0.2.1", lease->addr_.toText());
......@@ -159,17 +157,17 @@ TEST_F(CSVLeaseFile4Test, parse) {
// Second lease is malformed - HW address is empty.
{
SCOPED_TRACE("Second lease malformed");
EXPECT_FALSE(lf->next(lease));
checkStats(*lf, 2, 1, 1, 0, 0, 0);
EXPECT_FALSE(lf.next(lease));
checkStats(lf, 2, 1, 1, 0, 0, 0);
}
// Even though parsing previous lease failed, reading the next lease should be
// successful.
{
SCOPED_TRACE("Third lease valid");
EXPECT_TRUE(lf->next(lease));
EXPECT_TRUE(lf.next(lease));