Commit fb7c63f6 authored by Michal 'vorner' Vaner's avatar Michal 'vorner' Vaner
Browse files

Merge branch #1067

Conflicts:
	src/lib/datasrc/database.h
parents 58d6de47 6ea996c6
......@@ -21,7 +21,7 @@ libdatasrc_la_SOURCES += memory_datasrc.h memory_datasrc.cc
libdatasrc_la_SOURCES += zone.h
libdatasrc_la_SOURCES += result.h
libdatasrc_la_SOURCES += logger.h logger.cc
libdatasrc_la_SOURCES += client.h
libdatasrc_la_SOURCES += client.h iterator.h
libdatasrc_la_SOURCES += database.h database.cc
libdatasrc_la_SOURCES += sqlite3_accessor.h sqlite3_accessor.cc
nodist_libdatasrc_la_SOURCES = datasrc_messages.h datasrc_messages.cc
......
......@@ -16,12 +16,19 @@
#define __DATA_SOURCE_CLIENT_H 1
#include <boost/noncopyable.hpp>
#include <boost/shared_ptr.hpp>
#include <exceptions/exceptions.h>
#include <datasrc/zone.h>
namespace isc {
namespace datasrc {
// The iterator.h is not included on purpose, most application won't need it
class ZoneIterator;
typedef boost::shared_ptr<ZoneIterator> ZoneIteratorPtr;
/// \brief The base class of data source clients.
///
/// This is an abstract base class that defines the common interface for
......@@ -143,6 +150,36 @@ public:
/// \param name A domain name for which the search is performed.
/// \return A \c FindResult object enclosing the search result (see above).
virtual FindResult findZone(const isc::dns::Name& name) const = 0;
/// \brief Returns an iterator to the given zone
///
/// This allows for traversing the whole zone. The returned object can
/// provide the RRsets one by one.
///
/// This throws DataSourceError when the zone does not exist in the
/// datasource.
///
/// The default implementation throws isc::NotImplemented. This allows
/// for easy and fast deployment of minimal custom data sources, where
/// the user/implementator doesn't have to care about anything else but
/// the actual queries. Also, in some cases, it isn't possible to traverse
/// the zone from logic point of view (eg. dynamically generated zone
/// data).
///
/// It is not fixed if a concrete implementation of this method can throw
/// anything else.
///
/// \param name The name of zone apex to be traversed. It doesn't do
/// nearest match as findZone.
/// \return Pointer to the iterator.
virtual ZoneIteratorPtr getIterator(const isc::dns::Name& name) const {
// This is here to both document the parameter in doxygen (therefore it
// needs a name) and avoid unused parameter warning.
static_cast<void>(name);
isc_throw(isc::NotImplemented,
"Data source doesn't support iteration");
}
};
}
}
......
......@@ -15,10 +15,12 @@
#include <vector>
#include <datasrc/database.h>
#include <datasrc/data_source.h>
#include <datasrc/iterator.h>
#include <exceptions/exceptions.h>
#include <dns/name.h>
#include <dns/rrttl.h>
#include <dns/rrclass.h>
#include <dns/rdata.h>
#include <dns/rdataclass.h>
......@@ -27,7 +29,10 @@
#include <boost/foreach.hpp>
using isc::dns::Name;
#include <string>
using namespace isc::dns;
using std::string;
namespace isc {
namespace datasrc {
......@@ -109,7 +114,7 @@ void addOrCreate(isc::dns::RRsetPtr& rrset,
if (ttl < rrset->getTTL()) {
rrset->setTTL(ttl);
}
logger.info(DATASRC_DATABASE_FIND_TTL_MISMATCH)
logger.warn(DATASRC_DATABASE_FIND_TTL_MISMATCH)
.arg(db.getDBName()).arg(name).arg(cls)
.arg(type).arg(rrset->getTTL());
}
......@@ -401,5 +406,109 @@ DatabaseClient::Finder::getClass() const {
return isc::dns::RRClass::IN();
}
namespace {
/*
* This needs, beside of converting all data from textual representation, group
* together rdata of the same RRsets. To do this, we hold one row of data ahead
* of iteration. When we get a request to provide data, we create it from this
* data and load a new one. If it is to be put to the same rrset, we add it.
* Otherwise we just return what we have and keep the row as the one ahead
* for next time.
*/
class DatabaseIterator : public ZoneIterator {
public:
DatabaseIterator(const DatabaseAccessor::IteratorContextPtr& context,
const RRClass& rrclass) :
context_(context),
class_(rrclass),
ready_(true)
{
// Prepare data for the next time
getData();
}
virtual isc::dns::ConstRRsetPtr getNextRRset() {
if (!ready_) {
isc_throw(isc::Unexpected, "Iterating past the zone end");
}
if (!data_ready_) {
// At the end of zone
ready_ = false;
LOG_DEBUG(logger, DBG_TRACE_DETAILED,
DATASRC_DATABASE_ITERATE_END);
return (ConstRRsetPtr());
}
string name_str(name_), rtype_str(rtype_), ttl(ttl_);
Name name(name_str);
RRType rtype(rtype_str);
RRsetPtr rrset(new RRset(name, class_, rtype, RRTTL(ttl)));
while (data_ready_ && name_ == name_str && rtype_str == rtype_) {
if (ttl_ != ttl) {
if (ttl < ttl_) {
ttl_ = ttl;
rrset->setTTL(RRTTL(ttl));
}
LOG_WARN(logger, DATASRC_DATABASE_ITERATE_TTL_MISMATCH).
arg(name_).arg(class_).arg(rtype_).arg(rrset->getTTL());
}
rrset->addRdata(rdata::createRdata(rtype, class_, rdata_));
getData();
}
LOG_DEBUG(logger, DBG_TRACE_DETAILED, DATASRC_DATABASE_ITERATE_NEXT).
arg(rrset->getName()).arg(rrset->getType());
return (rrset);
}
private:
// Load next row of data
void getData() {
string data[DatabaseAccessor::COLUMN_COUNT];
data_ready_ = context_->getNext(data, DatabaseAccessor::COLUMN_COUNT);
name_ = data[DatabaseAccessor::NAME_COLUMN];
rtype_ = data[DatabaseAccessor::TYPE_COLUMN];
ttl_ = data[DatabaseAccessor::TTL_COLUMN];
rdata_ = data[DatabaseAccessor::RDATA_COLUMN];
}
// The context
const DatabaseAccessor::IteratorContextPtr context_;
// Class of the zone
RRClass class_;
// Status
bool ready_, data_ready_;
// Data of the next row
string name_, rtype_, rdata_, ttl_;
};
}
ZoneIteratorPtr
DatabaseClient::getIterator(const isc::dns::Name& name) const {
// Get the zone
std::pair<bool, int> zone(database_->getZone(name));
if (!zone.first) {
// No such zone, can't continue
isc_throw(DataSourceError, "Zone " + name.toText() +
" can not be iterated, because it doesn't exist "
"in this data source");
}
// Request the context
DatabaseAccessor::IteratorContextPtr
context(database_->getAllRecords(name, zone.second));
// It must not return NULL, that's a bug of the implementation
if (context == DatabaseAccessor::IteratorContextPtr()) {
isc_throw(isc::Unexpected, "Iterator context null at " +
name.toText());
}
// Create the iterator and return it
// TODO: Once #1062 is merged with this, we need to get the
// actual zone class from the connection, as the DatabaseClient
// doesn't know it and the iterator needs it (so it wouldn't query
// it each time)
LOG_DEBUG(logger, DBG_TRACE_DETAILED, DATASRC_DATABASE_ITERATE).
arg(name);
return (ZoneIteratorPtr(new DatabaseIterator(context, RRClass::IN())));
}
}
}
......@@ -18,6 +18,7 @@
#include <datasrc/client.h>
#include <dns/name.h>
#include <exceptions/exceptions.h>
namespace isc {
namespace datasrc {
......@@ -75,6 +76,86 @@ public:
*/
virtual std::pair<bool, int> getZone(const isc::dns::Name& name) const = 0;
/**
* \brief This holds the internal context of ZoneIterator for databases
*
* While the ZoneIterator implementation from DatabaseClient does all the
* translation from strings to DNS classes and validation, this class
* holds the pointer to where the database is at reading the data.
*
* It can either hold shared pointer to the connection which created it
* and have some kind of statement inside (in case single database
* connection can handle multiple concurrent SQL statements) or it can
* create a new connection (or, if it is more convenient, the connection
* itself can inherit both from DatabaseConnection and IteratorContext
* and just clone itself).
*/
class IteratorContext : public boost::noncopyable {
public:
/**
* \brief Destructor
*
* Virtual destructor, so any descendand class is destroyed correctly.
*/
virtual ~IteratorContext() { }
/**
* \brief Function to provide next resource record
*
* This function should provide data about the next resource record
* from the iterated zone. The data are not converted yet.
*
* \note The order of RRs is not strictly set, but the RRs for single
* RRset must not be interleaved with any other RRs (eg. RRsets must be
* "together").
*
* \param columns The data will be returned through here. The order
* is specified by the RecordColumns enum.
* \param Size of the columns array. Must be equal to COLUMN_COUNT,
* otherwise DataSourceError is thrown.
* \todo Do we consider databases where it is stored in binary blob
* format?
* \throw DataSourceError if there's database-related error. If the
* exception (or any other in case of derived class) is thrown,
* the iterator can't be safely used any more.
*/
virtual bool getNext(std::string columns[], size_t column_data) = 0;
};
typedef boost::shared_ptr<IteratorContext> IteratorContextPtr;
/**
* \brief Creates an iterator context for the whole zone.
*
* This should create a new iterator context to be used by
* DatabaseConnection's ZoneIterator. It can be created based on the name
* or the ID (returned from getZone()), what is more comfortable for the
* database implementation. Both are provided (and are guaranteed to match,
* the DatabaseClient first looks up the zone ID and then calls this).
*
* The default implementation throws isc::NotImplemented, to allow
* "minimal" implementations of the connection not supporting optional
* functionality.
*
* \param name The name of the zone.
* \param id The ID of the zone, returned from getZone().
* \return Newly created iterator context. Must not be NULL.
*/
virtual IteratorContextPtr getAllRecords(const isc::dns::Name& name,
int id) const
{
/*
* This is a compromise. We need to document the parameters in doxygen,
* so they need a name, but then it complains about unused parameter.
* This is a NOP that "uses" the parameters.
*/
static_cast<void>(name);
static_cast<void>(id);
isc_throw(isc::NotImplemented,
"This database datasource can't be iterated");
}
/**
* \brief Starts a new search for records of the given name in the given zone
*
......@@ -145,11 +226,12 @@ public:
SIGTYPE_COLUMN = 2, ///< For RRSIG records, this contains the RRTYPE
///< the RRSIG covers. In the current implementation,
///< this field is ignored.
RDATA_COLUMN = 3 ///< Full text representation of the record's RDATA
RDATA_COLUMN = 3, ///< Full text representation of the record's RDATA
NAME_COLUMN = 4 ///< The domain name of this RR
};
/// The number of fields the columns array passed to getNextRecord should have
static const size_t COLUMN_COUNT = 4;
static const size_t COLUMN_COUNT = 5;
/**
* \brief Returns a string identifying this dabase backend
......@@ -356,6 +438,25 @@ public:
*/
virtual FindResult findZone(const isc::dns::Name& name) const;
/**
* \brief Get the zone iterator
*
* The iterator allows going through the whole zone content. If the
* underlying DatabaseConnection is implemented correctly, it should
* be possible to have multiple ZoneIterators at once and query data
* at the same time.
*
* \exception DataSourceError if the zone doesn't exist.
* \exception isc::NotImplemented if the underlying DatabaseConnection
* doesn't implement iteration. But in case it is not implemented
* and the zone doesn't exist, DataSourceError is thrown.
* \exception Anything else the underlying DatabaseConnection might
* want to throw.
* \param name The origin of the zone to iterate.
* \return Shared pointer to the iterator (it will never be NULL)
*/
virtual ZoneIteratorPtr getIterator(const isc::dns::Name& name) const;
private:
/// \brief Our database.
const boost::shared_ptr<DatabaseAccessor> database_;
......
......@@ -75,8 +75,9 @@ name and type in the database.
% DATASRC_DATABASE_FIND_TTL_MISMATCH TTL values differ in %1 for elements of %2/%3/%4, setting to %5
The datasource backend provided resource records for the given RRset with
different TTL values. The TTL of the RRSET is set to the lowest value, which
is printed in the log message.
different TTL values. This isn't allowed on the wire and is considered
an error, so we set it to the lowest value we found (but we don't modify the
database). The data in database should be checked and fixed.
% DATASRC_DATABASE_FIND_UNCAUGHT_ERROR uncaught general error retrieving data from datasource %1: %2
There was an uncaught general exception while reading data from a datasource.
......@@ -117,6 +118,23 @@ The data returned by the database backend contained data for the given domain
name, and it either matches the type or has a relevant type. The RRset that is
returned is printed.
% DATASRC_DATABASE_ITERATE iterating zone %1
The program is reading the whole zone, eg. not searching for data, but going
through each of the RRsets there.
% DATASRC_DATABASE_ITERATE_END iterating zone finished
While iterating through the zone, the program reached end of the data.
% DATASRC_DATABASE_ITERATE_NEXT next RRset in zone is %1/%2
While iterating through the zone, the program extracted next RRset from it.
The name and RRtype of the RRset is indicated in the message.
% DATASRC_DATABASE_ITERATE_TTL_MISMATCH TTL values differ for RRs of %1/%2/%3, setting to %4
While iterating through the zone, the time to live for RRs of the given RRset
were found to be different. This isn't allowed on the wire and is considered
an error, so we set it to the lowest value we found (but we don't modify the
database). The data in database should be checked and fixed.
% DATASRC_DO_QUERY handling query for '%1/%2'
A debug message indicating that a query for the given name and RR type is being
processed.
......
// Copyright (C) 2011 Internet Systems Consortium, Inc. ("ISC")
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include <dns/rrset.h>
#include <boost/noncopyable.hpp>
namespace isc {
namespace datasrc {
/**
* \brief Read-only iterator to a zone.
*
* You can get an instance of (descendand of) ZoneIterator from
* DataSourceClient::getIterator() method. The actual concrete implementation
* will be different depending on the actual data source used. This is the
* abstract interface.
*
* There's no way to start iterating from the beginning again or return.
*/
class ZoneIterator : public boost::noncopyable {
public:
/**
* \brief Destructor
*
* Virtual destructor. It is empty, but ensures the right destructor from
* descendant is called.
*/
virtual ~ ZoneIterator() { }
/**
* \brief Get next RRset from the zone.
*
* This returns the next RRset in the zone as a shared pointer. The
* shared pointer is used to allow both accessing in-memory data and
* automatic memory management.
*
* Any special order is not guaranteed.
*
* While this can potentially throw anything (including standard allocation
* errors), it should be rare.
*
* \return Pointer to the next RRset or NULL pointer when the iteration
* gets to the end of the zone.
*/
virtual isc::dns::ConstRRsetPtr getNextRRset() = 0;
};
}
}
......@@ -25,6 +25,8 @@
#include <datasrc/memory_datasrc.h>
#include <datasrc/rbtree.h>
#include <datasrc/logger.h>
#include <datasrc/iterator.h>
#include <datasrc/data_source.h>
using namespace std;
using namespace isc::dns;
......@@ -32,6 +34,27 @@ using namespace isc::dns;
namespace isc {
namespace datasrc {
namespace {
// Some type aliases
/*
* Each domain consists of some RRsets. They will be looked up by the
* RRType.
*
* The use of map is questionable with regard to performance - there'll
* be usually only few RRsets in the domain, so the log n benefit isn't
* much and a vector/array might be faster due to its simplicity and
* continuous memory location. But this is unlikely to be a performance
* critical place and map has better interface for the lookups, so we use
* that.
*/
typedef map<RRType, ConstRRsetPtr> Domain;
typedef Domain::value_type DomainPair;
typedef boost::shared_ptr<Domain> DomainPtr;
// The tree stores domains
typedef RBTree<Domain> DomainTree;
typedef RBNode<Domain> DomainNode;
}
// Private data and hidden methods of InMemoryZoneFinder
struct InMemoryZoneFinder::InMemoryZoneFinderImpl {
// Constructor
......@@ -44,25 +67,6 @@ struct InMemoryZoneFinder::InMemoryZoneFinderImpl {
DomainPtr origin_domain(new Domain);
origin_data_->setData(origin_domain);
}
// Some type aliases
/*
* Each domain consists of some RRsets. They will be looked up by the
* RRType.
*
* The use of map is questionable with regard to performance - there'll
* be usually only few RRsets in the domain, so the log n benefit isn't
* much and a vector/array might be faster due to its simplicity and
* continuous memory location. But this is unlikely to be a performance
* critical place and map has better interface for the lookups, so we use
* that.
*/
typedef map<RRType, ConstRRsetPtr> Domain;
typedef Domain::value_type DomainPair;
typedef boost::shared_ptr<Domain> DomainPtr;
// The tree stores domains
typedef RBTree<Domain> DomainTree;
typedef RBNode<Domain> DomainNode;
static const DomainNode::Flags DOMAINFLAG_WILD = DomainNode::FLAG_USER1;
// Information about the zone
......@@ -634,7 +638,7 @@ InMemoryZoneFinder::load(const string& filename) {
LOG_DEBUG(logger, DBG_TRACE_BASIC, DATASRC_MEM_LOAD).arg(getOrigin()).
arg(filename);
// Load it into a temporary tree
InMemoryZoneFinderImpl::DomainTree tmp;
DomainTree tmp;
masterLoad(filename.c_str(), getOrigin(), getClass(),
boost::bind(&InMemoryZoneFinderImpl::addFromLoad, impl_, _1, &tmp));
// If it went well, put it inside
......@@ -700,8 +704,94 @@ InMemoryClient::addZone(ZoneFinderPtr zone_finder) {
InMemoryClient::FindResult
InMemoryClient::findZone(const isc::dns::Name& name) const {
LOG_DEBUG(logger, DBG_TRACE_DATA, DATASRC_MEM_FIND_ZONE).arg(name);
return (FindResult(impl_->zone_table.findZone(name).code,
impl_->zone_table.findZone(name).zone));
ZoneTable::FindResult result(impl_->zone_table.findZone(name));
return (FindResult(result.code, result.zone));
}
namespace {
class MemoryIterator : public ZoneIterator {
private:
RBTreeNodeChain<Domain> chain_;
Domain::const_iterator dom_iterator_;
const DomainTree& tree_;
const DomainNode* node_;
bool ready_;
public:
MemoryIterator(const DomainTree& tree, const Name& origin) :
tree_(tree),
ready_(true)
{
// Find the first node (origin) and preserve the node chain for future
// searches
DomainTree::Result result(tree_.find<void*>(origin, &node_, chain_,
NULL, NULL));
// It can't happen that the origin is not in there
if (result != DomainTree::EXACTMATCH) {
isc_throw(Unexpected,
"In-memory zone corrupted, missing origin node");
}
// Initialize the iterator if there's somewhere to point to
if (node_ != NULL && node_->getData() != DomainPtr()) {
dom_iterator_ = node_->getData()->begin();
}
}
virtual ConstRRsetPtr getNextRRset() {
if (!ready_) {
isc_throw(Unexpected, "Iterating past the zone end");
}
/*
* This cycle finds the first nonempty node with yet unused RRset.
* If it is NULL, we run out of nodes. If it is empty, it doesn't
* contain any RRsets. If we are at the end, just get to next one.
*/
while (node_ != NULL && (node_->getData() == DomainPtr() ||
dom_iterator_ == node_->getData()->end())) {
node_ = tree_.nextNode(chain_);
// If there's a node, initialize the iterator and check next time
// if the map is empty or not
if (node_ != NULL && node_->getData() != NULL) {
dom_iterator_ = node_->getData()->begin();
}
}
if (node_ == NULL) {
// That's all, folks
ready_ = false;
return (ConstRRsetPtr());
}
// The iterator points to the next yet unused RRset now
ConstRRsetPtr result(dom_iterator_->second);
// This one is used, move it to the next time for next call
++dom_iterator_;
return (result);
}
};
} // End of anonymous namespace
ZoneIteratorPtr
InMemoryClient::getIterator(const Name& name) const {
ZoneTable::FindResult result(impl_->zone_table.findZone(name));
if (result.code != result::SUCCESS) {
isc_throw(DataSourceError, "No such zone: " + name.toText());
}
const InMemoryZoneFinder*
zone(dynamic_cast<const InMemoryZoneFinder*>(result.zone.get()));
if (zone == NULL) {
/*
* TODO: This can happen only during some of the tests and only as
* a temporary solution. This should be fixed by #1159 and then
* this cast and check shouldn't be necessary. We don't have
* test for handling a "can not happen" condition.
*/
isc_throw(Unexpected, "The zone at " + name.toText() +
" is not InMemoryZoneFinder");
}
return (ZoneIteratorPtr(new MemoryIterator(zone->impl_->domains_, name)));
}
} // end of namespace datasrc
} // end of namespace dns
......@@ -182,6 +182,11 @@ private:
struct InMemoryZoneFinderImpl;
InMemoryZoneFinderImpl* impl_;
//@}
// The friend here is for InMemoryClient::getIterator. The iterator
// needs to access the data inside the zone, so the InMemoryClient
// extracts the pointer to data and puts it into the iterator.
// The access is read only.
friend class InMemoryClient;
};