[3360] Memfile stores and reads leases from disk for v4 and v6.

src/lib/dhcpsrv/memfile_lease_mgr.cc

@@ -23,12 +23,30 @@ using namespace isc::dhcp;
 
 Memfile_LeaseMgr::Memfile_LeaseMgr(const ParameterMap& parameters)
     : LeaseMgr(parameters) {
-    // Get the lease files locations.
-    lease_file4_ = initLeaseFilePath(V4);
-    lease_file6_ = initLeaseFilePath(V6);
+    // Get the lease files locations and open for IO.
+    std::string file4 = initLeaseFilePath(V4);
+    if (!file4.empty()) {
+        lease_file4_.reset(new CSVLeaseFile4(file4));
+        lease_file4_->open();
+        load4();
+    }
+    std::string file6 = initLeaseFilePath(V6);
+    if (!file6.empty()) {
+        lease_file6_.reset(new CSVLeaseFile6(file6));
+        lease_file6_->open();
+        load6();
+    }
 }
 
 Memfile_LeaseMgr::~Memfile_LeaseMgr() {
+    if (lease_file4_) {
+        lease_file4_->close();
+        lease_file4_.reset();
+    }
+    if (lease_file6_) {
+        lease_file6_->close();
+        lease_file6_.reset();
+    }
 }
 
 bool
@@ -40,6 +58,14 @@ Memfile_LeaseMgr::addLease(const Lease4Ptr& lease) {
         // there is a lease with specified address already
         return (false);
     }
+
+    // Try to write a lease to disk first. If this fails, the lease will
+    // not be inserted to the memory and the disk and in-memory data will
+    // remain consistent.
+    if (lease_file4_) {
+        lease_file4_->append(*lease);
+    }
+
     storage4_.insert(lease);
     return (true);
 }
@@ -53,6 +79,14 @@ Memfile_LeaseMgr::addLease(const Lease6Ptr& lease) {
         // there is a lease with specified address already
         return (false);
     }
+
+    // Try to write a lease to disk first. If this fails, the lease will
+    // not be inserted to the memory and the disk and in-memory data will
+    // remain consistent.
+    if (lease_file6_) {
+        lease_file6_->append(*lease);
+    }
+
     storage6_.insert(lease);
     return (true);
 }
@@ -252,6 +286,14 @@ Memfile_LeaseMgr::updateLease4(const Lease4Ptr& lease) {
         isc_throw(NoSuchLease, "failed to update the lease with address "
                   << lease->addr_ << " - no such lease");
     }
+
+    // Try to write a lease to disk first. If this fails, the lease will
+    // not be inserted to the memory and the disk and in-memory data will
+    // remain consistent.
+    if (lease_file4_) {
+        lease_file4_->append(*lease);
+    }
+
     **lease_it = *lease;
 }
 
@@ -265,6 +307,14 @@ Memfile_LeaseMgr::updateLease6(const Lease6Ptr& lease) {
         isc_throw(NoSuchLease, "failed to update the lease with address "
                   << lease->addr_ << " - no such lease");
     }
+
+    // Try to write a lease to disk first. If this fails, the lease will
+    // not be inserted to the memory and the disk and in-memory data will
+    // remain consistent.
+    if (lease_file6_) {
+        lease_file6_->append(*lease);
+    }
+
     **lease_it = *lease;
 }
 
@@ -279,6 +329,15 @@ Memfile_LeaseMgr::deleteLease(const isc::asiolink::IOAddress& addr) {
             // No such lease
             return (false);
         } else {
+            if (lease_file4_) {
+                // Copy the lease. The valid lifetime needs to be modified and
+                // we don't modify the original lease.
+                Lease4 lease_copy = **l;
+                // Setting valid lifetime to 0 means that lease is being
+                // removed.
+                lease_copy.valid_lft_ = 0;
+                lease_file4_->append(lease_copy);
+            }
             storage4_.erase(l);
             return (true);
         }
@@ -290,6 +349,16 @@ Memfile_LeaseMgr::deleteLease(const isc::asiolink::IOAddress& addr) {
             // No such lease
             return (false);
         } else {
+            if (lease_file6_) {
+                // Copy the lease. The lifetimes need to be modified and we
+                // don't modify the original lease.
+                Lease6 lease_copy = **l;
+                // Setting lifetimes to 0 means that lease is being removed.
+                lease_copy.valid_lft_ = 0;
+                lease_copy.preferred_lft_ = 0;
+                lease_file6_->append(lease_copy);
+            }
+
             storage6_.erase(l);
             return (true);
         }
@@ -323,12 +392,25 @@ Memfile_LeaseMgr::getDefaultLeaseFilePath(Universe u) const {
     return (s.str());
 }
 
+std::string
+Memfile_LeaseMgr::getLeaseFilePath(Universe u) const {
+    if (u == V4) {
+        return (lease_file4_ ? lease_file4_->getFilename() : "");
+    }
+
+    return (lease_file6_ ? lease_file6_->getFilename() : "");
+}
+
 bool
 Memfile_LeaseMgr::persistLeases(Universe u) const {
-    // Currently, if the lease file is empty, it means that writes to disk have
-    // been explicitly disabled by the administrator. At some point, there may
-    // be a dedicated ON/OFF flag implemented to control this.
-    return (u == V4 ? !lease_file4_.empty() : !lease_file6_.empty());
+    // Currently, if the lease file IO is not created, it means that writes to
+    // disk have been explicitly disabled by the administrator. At some point,
+    // there may be a dedicated ON/OFF flag implemented to control this.
+    if (u == V4 && lease_file4_) {
+        return (true);
+    }
+
+    return (u == V6 && lease_file6_);
 }
 
 std::string
@@ -342,3 +424,113 @@ Memfile_LeaseMgr::initLeaseFilePath(Universe u) {
     }
     return (lease_file);
 }
+
+void
+Memfile_LeaseMgr::load4() {
+    // If lease file hasn't been opened, we are working in non-persistent mode.
+    // That's fine, just leave.
+    if (!lease_file4_) {
+        return;
+    }
+    // Remove existing leases (if any). We will recreate them based on the
+    // data on disk.
+    storage4_.clear();
+
+    Lease4Ptr lease;
+    do {
+        /// @todo Currently we stop parsing on first failure. It is possible
+        /// that only one (or a few) leases are bad, so in theory we could
+        /// continue parsing but that would require some error counters to
+        /// prevent endless loops. That is enhancement for later time.
+        if (!lease_file4_->next(lease)) {
+            isc_throw(DbOperationError, "Failed to parse the DHCPv6 lease in"
+                      " the lease file: " << lease_file4_->getReadMsg());
+        }
+        // If we got the lease, we update the internal container holding
+        // leases. Otherwise, we reached the end of file and we leave.
+        if (lease) {
+            loadLease4(lease);
+        }
+    } while (lease);
+}
+
+void
+Memfile_LeaseMgr::loadLease4(Lease4Ptr& lease) {
+    // Check if the lease already exists.
+    Lease4Storage::iterator lease_it = storage4_.find(lease->addr_);
+    // Lease doesn't exist.
+    if (lease_it == storage4_.end()) {
+        // Add the lease only if valid lifetime is greater than 0.
+        // We use valid lifetime of 0 to indicate that lease should
+        // be removed.
+        if (lease->valid_lft_ > 0) {
+           storage4_.insert(lease);
+       }
+    } else {
+        // We use valid lifetime of 0 to indicate that the lease is
+        // to be removed. In such case, erase the lease.
+        if (lease->valid_lft_ == 0) {
+            storage4_.erase(lease_it);
+
+        } else {
+            // Update existing lease.
+            **lease_it = *lease;
+        }
+    }
+}
+
+void
+Memfile_LeaseMgr::load6() {
+    // If lease file hasn't been opened, we are working in non-persistent mode.
+    // That's fine, just leave.
+    if (!lease_file6_) {
+        return;
+    }
+    // Remove existing leases (if any). We will recreate them based on the
+    // data on disk.
+    storage6_.clear();
+
+    Lease6Ptr lease;
+    do {
+        /// @todo Currently we stop parsing on first failure. It is possible
+        /// that only one (or a few) leases are bad, so in theory we could
+        /// continue parsing but that would require some error counters to
+        /// prevent endless loops. That is enhancement for later time.
+        if (!lease_file6_->next(lease)) {
+            isc_throw(DbOperationError, "Failed to parse the DHCPv6 lease in"
+                      " the lease file: " << lease_file6_->getReadMsg());
+        }
+        // If we got the lease, we update the internal container holding
+        // leases. Otherwise, we reached the end of file and we leave.
+        if (lease) {
+            loadLease6(lease);
+        }
+    } while (lease);
+}
+
+void
+Memfile_LeaseMgr::loadLease6(Lease6Ptr& lease) {
+    // Check if the lease already exists.
+    Lease6Storage::iterator lease_it = storage6_.find(lease->addr_);
+    // Lease doesn't exist.
+    if (lease_it == storage6_.end()) {
+        // Add the lease only if valid lifetime is greater than 0.
+        // We use valid lifetime of 0 to indicate that lease should
+        // be removed.
+        if (lease->valid_lft_ > 0) {
+            storage6_.insert(lease);
+       }
+    } else {
+        // We use valid lifetime of 0 to indicate that the lease is
+        // to be removed. In such case, erase the lease.
+        if (lease->valid_lft_ == 0) {
+            storage6_.erase(lease_it);
+
+        } else {
+            // Update existing lease.
+            **lease_it = *lease;
+        }
+    }
+
+}
+

src/lib/dhcpsrv/memfile_lease_mgr.h

@@ -16,6 +16,8 @@
 #define MEMFILE_LEASE_MGR_H
 
 #include <dhcp/hwaddr.h>
+#include <dhcpsrv/csv_lease_file4.h>
+#include <dhcpsrv/csv_lease_file6.h>
 #include <dhcpsrv/lease_mgr.h>
 
 #include <boost/multi_index/indexed_by.hpp>
@@ -27,14 +29,7 @@
 namespace isc {
 namespace dhcp {
 
-// This is a concrete implementation of a Lease database.
-//
-// It is for testing purposes only. It is NOT a production code.
-//
-// It does not do anything useful now, and is used for abstract LeaseMgr
-// class testing. It may later evolve into more useful backend if the
-// need arises. We can reuse code from memfile benchmark. See code in
-// tests/tools/dhcp-ubench/memfile_bench.{cc|h}
+/// @brief This is a concrete implementation of a Lease database.
 class Memfile_LeaseMgr : public LeaseMgr {
 public:
 
@@ -264,9 +259,10 @@ public:
     /// @brief Returns an absolute path to the lease file.
     ///
     /// @param u Universe (V4 or V6).
-    std::string getLeaseFilePath(Universe u) const {
-        return (u == V4 ? lease_file4_ : lease_file6_);
-    }
+    ///
+    /// @return Absolute path to the lease file or empty string if no lease
+    /// file is used.
+    std::string getLeaseFilePath(Universe u) const;
 
     /// @brief Specifies whether or not leases are written to disk.
     ///
@@ -283,6 +279,50 @@ public:
 
 protected:
 
+    /// @brief Load all DHCPv4 leases from the file.
+    ///
+    /// This method loads all DHCPv4 leases from a file to memory. It removes
+    /// existing leases before reading a file.
+    ///
+    /// @throw isc::DbOperationError If failed to read a lease from the lease
+    /// file.
+    void load4();
+
+    /// @brief Loads a single DHCPv4 lease from the file.
+    ///
+    /// This method reads a single lease record from the lease file. If the
+    /// corresponding record doesn't exist in the in-memory container, the
+    /// lease is added to the container (except for a lease which valid lifetime
+    /// is 0). If the corresponding lease exists, the lease being read updates
+    /// the existing lease. If the lease being read from the lease file has
+    /// valid lifetime of 0 and the corresponding lease exists in the in-memory
+    /// database, the existing lease is removed.
+    ///
+    /// @param lease Pointer to the lease read from the lease file.
+    void loadLease4(Lease4Ptr& lease);
+
+    /// @brief Load all DHCPv6 leases from the file.
+    ///
+    /// This method loads all DHCPv6 leases from a file to memory. It removes
+    /// existing leases before reading a file.
+    ///
+    /// @throw isc::DbOperationError If failed to read a lease from the lease
+    /// file.
+    void load6();
+
+    /// @brief Loads a single DHCPv6 lease from the file.
+    ///
+    /// This method reads a single lease record from the lease file. If the
+    /// corresponding record doesn't exist in the in-memory container, the
+    /// lease is added to the container (except for a lease which valid lifetime
+    /// is 0). If the corresponding lease exists, the lease being read updates
+    /// the existing lease. If the lease being read from the lease file has
+    /// valid lifetime of 0 and the corresponding lease exists in the in-memory
+    /// database, the existing lease is removed.
+    ///
+    /// @param lease Pointer to the lease read from the lease file.
+    void loadLease6(Lease6Ptr& lease);
+
     /// @brief Initialize the location of the lease file.
     ///
     /// This method uses the parameters passed as a map to the constructor to
@@ -368,7 +408,7 @@ protected:
             >,
 
             // Specification of the third index starts here.
-            boost::multi_index::ordered_unique<
+            boost::multi_index::ordered_non_unique<
                 // This is a composite index that uses two values to search for a
                 // lease: client id and subnet id.
                 boost::multi_index::composite_key<
@@ -383,7 +423,7 @@ protected:
             >,
 
             // Specification of the fourth index starts here.
-            boost::multi_index::ordered_unique<
+            boost::multi_index::ordered_non_unique<
                 // This is a composite index that uses two values to search for a
                 // lease: client id and subnet id.
                 boost::multi_index::composite_key<
@@ -409,11 +449,11 @@ protected:
     /// @brief stores IPv6 leases
     Lease6Storage storage6_;
 
-    /// @brief Holds the absolute path to the lease file for DHCPv4.
-    std::string lease_file4_;
+    /// @brief Holds the pointer to the DHCPv4 lease file IO.
+    boost::shared_ptr<CSVLeaseFile4> lease_file4_;
 
-    /// @brief Holds the absolute path to the lease file for DHCPv6.
-    std::string lease_file6_;
+    /// @brief Holds the pointer to the DHCPv6 lease file IO.
+    boost::shared_ptr<CSVLeaseFile6> lease_file6_;
 };
 
 }; // end of isc::dhcp namespace

src/lib/dhcpsrv/tests/alloc_engine_unittest.cc

@@ -101,7 +101,7 @@ public:
 
         initFqdn("", false, false);
 
-        factory_.create("type=memfile");
+        factory_.create("type=memfile leasefile4= leasefile6=");
     }
 
     /// @brief Configures a subnet and adds one pool to it.
@@ -424,7 +424,7 @@ public:
         subnet_->addPool(pool_);
         cfg_mgr.addSubnet4(subnet_);
 
-        factory_.create("type=memfile");
+        factory_.create("type=memfile leasefile4= leasefile6=");
     }
 
     /// @brief checks if Lease4 matches expected configuration

src/lib/dhcpsrv/tests/dbaccess_parser_unittest.cc

@@ -410,8 +410,8 @@ TEST_F(DbAccessParserTest, commit) {
             }, isc::dhcp::NoLeaseManager);
 
     // Set up the parser to open the memfile database.
-    const char* config[] = {"type", "memfile",
-                            NULL};
+    const char* config[] = {"type", "memfile", "leasefile4", "",
+                            "leasefile6", "", NULL};
     string json_config = toJson(config);
     ConstElementPtr json_elements = Element::fromJSON(json_config);
     EXPECT_TRUE(json_elements);

src/lib/dhcpsrv/tests/generic_lease_mgr_unittest.cc

@@ -717,6 +717,39 @@ GenericLeaseMgrTest::testBasicLease4() {
     l_returned = lmptr_->getLease4(ioaddress4_[2]);
     ASSERT_TRUE(l_returned);
     detailCompareLease(leases[2], l_returned);
+
+    reopen();
+
+    // The deleted lease should be still gone after we re-read leases from
+    // persistent storage.
+    l_returned = lmptr_->getLease4(ioaddress4_[1]);
+    EXPECT_FALSE(l_returned);
+
+    l_returned = lmptr_->getLease4(ioaddress4_[2]);
+    ASSERT_TRUE(l_returned);
+    detailCompareLease(leases[2], l_returned);
+
+    l_returned = lmptr_->getLease4(ioaddress4_[3]);
+    ASSERT_TRUE(l_returned);
+    detailCompareLease(leases[3], l_returned);
+
+    // Update some FQDN data, so as we can check that update in
+    // persistent storage works as expected.
+    leases[2]->hostname_ = "memfile.example.com.";
+    leases[2]->fqdn_rev_ = true;
+
+    ASSERT_NO_THROW(lmptr_->updateLease4(leases[2]));
+
+    reopen();
+
+    // The lease should be now updated in the storage.
+    l_returned = lmptr_->getLease4(ioaddress4_[2]);
+    ASSERT_TRUE(l_returned);
+    detailCompareLease(leases[2], l_returned);
+
+    l_returned = lmptr_->getLease4(ioaddress4_[3]);
+    ASSERT_TRUE(l_returned);
+    detailCompareLease(leases[3], l_returned);
 }
 
 
@@ -761,6 +794,32 @@ GenericLeaseMgrTest::testBasicLease6() {
     l_returned = lmptr_->getLease6(leasetype6_[2], ioaddress6_[2]);
     ASSERT_TRUE(l_returned);
     detailCompareLease(leases[2], l_returned);
+
+    reopen();
+
+    // The deleted lease should be still gone after we re-read leases from
+    // persistent storage.
+    l_returned = lmptr_->getLease6(leasetype6_[1], ioaddress6_[1]);
+    EXPECT_FALSE(l_returned);
+
+    // Check that the second address is still there.
+    l_returned = lmptr_->getLease6(leasetype6_[2], ioaddress6_[2]);
+    ASSERT_TRUE(l_returned);
+    detailCompareLease(leases[2], l_returned);
+
+    // Update some FQDN data, so as we can check that update in
+    // persistent storage works as expected.
+    leases[2]->hostname_ = "memfile.example.com.";
+    leases[2]->fqdn_rev_ = true;
+
+    ASSERT_NO_THROW(lmptr_->updateLease6(leases[2]));
+
+    reopen();
+
+    // The lease should be now updated in the storage.
+    l_returned = lmptr_->getLease6(leasetype6_[2], ioaddress6_[2]);
+    ASSERT_TRUE(l_returned);
+    detailCompareLease(leases[2], l_returned);
 }
 
 void

src/lib/dhcpsrv/tests/memfile_lease_mgr_unittest.cc

@@ -18,7 +18,9 @@
 #include <dhcp/duid.h>
 #include <dhcpsrv/cfgmgr.h>
 #include <dhcpsrv/lease_mgr.h>
+#include <dhcpsrv/lease_mgr_factory.h>
 #include <dhcpsrv/memfile_lease_mgr.h>
+#include <dhcpsrv/tests/lease_file_io.h>
 #include <dhcpsrv/tests/test_utils.h>
 #include <dhcpsrv/tests/generic_lease_mgr_unittest.h>
 #include <gtest/gtest.h>
@@ -33,6 +35,7 @@ using namespace isc::dhcp;
 using namespace isc::dhcp::test;
 
 namespace {
+
 // empty class for now, but may be extended once Addr6 becomes bigger
 class MemfileLeaseMgrTest : public GenericLeaseMgrTest {
 public:
@@ -40,21 +43,35 @@ public:
     /// @brief memfile lease mgr test constructor
     ///
     /// Creates memfile and stores it in lmptr_ pointer
-    MemfileLeaseMgrTest() {
-        const LeaseMgr::ParameterMap pmap;
-        lmptr_ = new Memfile_LeaseMgr(pmap);
+    MemfileLeaseMgrTest() :
+        io4_(getLeaseFilePath("leasefile4_0.csv")),
+        io6_(getLeaseFilePath("leasefile6_0.csv")) {
+
+        // Make sure there are no dangling files after previous tests.
+        io4_.removeFile();
+        io6_.removeFile();
+
+        try {
+            LeaseMgrFactory::create(getConfigString());
+        } catch (...) {
+            std::cerr << "*** ERROR: unable to create instance of the Memfile\n"
+                " lease database backend.\n";
+            throw;
+        }
+        lmptr_ = &(LeaseMgrFactory::instance());
     }
 
     virtual void reopen() {
-        /// @todo: write lease to disk, flush, read file from disk
+        LeaseMgrFactory::destroy();
+        LeaseMgrFactory::create(getConfigString());
+        lmptr_ = &(LeaseMgrFactory::instance());
     }
 
     /// @brief destructor
     ///
     /// destroys lease manager backend.
     virtual ~MemfileLeaseMgrTest() {
-        delete lmptr_;
-        lmptr_ = 0;
+        LeaseMgrFactory::destroy();
     }
 
     /// @brief Return path to the lease file used by unit tests.
@@ -63,19 +80,41 @@ public:
     /// directory where test data is held.
     ///
     /// @return Full path to the lease file.
-    std::string getLeaseFilePath(const std::string& filename) const {
+    static std::string getLeaseFilePath(const std::string& filename) {
         std::ostringstream s;
         s << TEST_DATA_BUILDDIR << "/" << filename;
         return (s.str());
     }
 
+    /// @brief Returns the configuration string for the backend.
+    ///
+    /// This string configures the @c LeaseMgrFactory to create the memfile
+    /// backend and use leasefile4_0.csv and leasefile6_0.csv files as
+    /// storage for leases.
+    ///
+    /// @return Configuration string for @c LeaseMgrFactory.
+    static std::string getConfigString() {
+        std::ostringstream s;
+        s << "type=memfile leasefile4=" << getLeaseFilePath("leasefile4_0.csv")
+          << " leasefile6=" << getLeaseFilePath("leasefile6_0.csv");
+        return (s.str());
+    }
+
+    /// @brief Object providing access to v4 lease IO.
+    LeaseFileIO io4_;
+
+    /// @brief Object providing access to v6 lease IO.
+    LeaseFileIO io6_;
+
 };
 
 // This test checks if the LeaseMgr can be instantiated and that it
 // parses parameters string properly.
 TEST_F(MemfileLeaseMgrTest, constructor) {
 
-    const LeaseMgr::ParameterMap pmap;  // Empty parameter map
+    LeaseMgr::ParameterMap pmap;
+    pmap["leasefile4"] = "";
+    pmap["leasefile6"] = "";
     boost::scoped_ptr<Memfile_LeaseMgr> lease_mgr;
 
     ASSERT_NO_THROW(lease_mgr.reset(new Memfile_LeaseMgr(pmap)));
@@ -89,53 +128,42 @@ TEST_F(MemfileLeaseMgrTest, getTypeAndName) {
 
 // Checks if the path to the lease files is initialized correctly.
 TEST_F(MemfileLeaseMgrTest, getLeaseFilePath) {
+    // Initialize IO objects, so as the test csv files get removed after the
+    // test (when destructors are called).
+    LeaseFileIO io4(getLeaseFilePath("leasefile4_1.csv"));
+    LeaseFileIO io6(getLeaseFilePath("leasefile6_1.csv"));
+
     LeaseMgr::ParameterMap pmap;
+    pmap["leasefile4"] = getLeaseFilePath("leasefile4_1.csv");
+    pmap["leasefile6"] = getLeaseFilePath("leasefile6_1.csv");
     boost::scoped_ptr<Memfile_LeaseMgr> lease_mgr(new Memfile_LeaseMgr(pmap));
 
-    std::ostringstream s4;
-    s4 << CfgMgr::instance().getDataDir() << "/" << "kea-leases4.csv";
-    std::ostringstream s6;
-    s6 << CfgMgr::instance().getDataDir() << "/" << "kea-leases6.csv";
-    EXPECT_EQ(s4.str(),
-              lease_mgr->getDefaultLeaseFilePath(Memfile_LeaseMgr::V4));
-    EXPECT_EQ(s6.str(),
-              lease_mgr->getDefaultLeaseFilePath(Memfile_LeaseMgr::V6));
-
-
-    EXPECT_EQ(lease_mgr->getDefaultLeaseFilePath(Memfile_LeaseMgr::V4),
-              lease_mgr->getLeaseFilePath(Memfile_LeaseMgr::V4));