Commit 6e500e57 authored by Michal 'vorner' Vaner's avatar Michal 'vorner' Vaner
Browse files

Merge #2096

Conflicts:
	src/lib/datasrc/memory/Makefile.am
parents ace90354 2497f548
......@@ -1166,6 +1166,7 @@ AC_CONFIG_FILES([Makefile
src/lib/datasrc/Makefile
src/lib/datasrc/memory/Makefile
src/lib/datasrc/memory/tests/Makefile
src/lib/datasrc/memory/benchmarks/Makefile
src/lib/datasrc/tests/Makefile
src/lib/datasrc/tests/testdata/Makefile
src/lib/xfr/Makefile
......
SUBDIRS = . tests
SUBDIRS = . tests benchmarks
AM_CPPFLAGS = -I$(top_srcdir)/src/lib -I$(top_builddir)/src/lib
AM_CPPFLAGS += -I$(top_srcdir)/src/lib/dns -I$(top_builddir)/src/lib/dns
......@@ -10,7 +10,9 @@ CLEANFILES = *.gcno *.gcda datasrc_messages.h datasrc_messages.cc
noinst_LTLIBRARIES = libdatasrc_memory.la
libdatasrc_memory_la_SOURCES = rdata_encoder.h rdata_encoder.cc
libdatasrc_memory_la_SOURCES += domaintree.h
libdatasrc_memory_la_SOURCES = \
rdata_serialization.h rdata_serialization.cc \
domaintree.h
libdatasrc_memory_la_SOURCES += zone_data.h
libdatasrc_memory_la_SOURCES += zone_table.h zone_table.cc
EXTRA_DIST = rdata_serialization_priv.cc
AM_CPPFLAGS = -I$(top_srcdir)/src/lib -I$(top_builddir)/src/lib
AM_CPPFLAGS += $(BOOST_INCLUDES)
AM_CXXFLAGS = $(B10_CXXFLAGS)
if USE_STATIC_LINK
AM_LDFLAGS = -static
endif
CLEANFILES = *.gcno *.gcda
noinst_PROGRAMS = rdata_reader_bench
rdata_reader_bench_SOURCES = rdata_reader_bench.cc
rdata_reader_bench_LDADD = $(top_builddir)/src/lib/datasrc/memory/libdatasrc_memory.la
rdata_reader_bench_LDADD += $(top_builddir)/src/lib/dns/libb10-dns++.la
// Copyright (C) 2012 Internet Systems Consortium, Inc. ("ISC")
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include <bench/benchmark.h>
#include <util/buffer.h>
#include <dns/name.h>
#include <dns/messagerenderer.h>
#include <dns/rrset.h>
#include <dns/rrclass.h>
#include <dns/masterload.h>
#include <datasrc/memory/rdata_serialization.h>
#include <boost/bind.hpp>
#include <vector>
#include <sstream>
#include <unistd.h>
using std::vector;
using namespace isc::bench;
using namespace isc::datasrc::memory;
using namespace isc::dns;
namespace {
struct EncodeParam {
EncodeParam(RdataEncoder& encoder, ConstRRsetPtr rrset,
ConstRRsetPtr sig_rrset = ConstRRsetPtr()) :
rrclass(rrset->getClass()), rrtype(rrset->getType()),
rdata_count(rrset->getRdataCount()),
sig_count(sig_rrset ? sig_rrset->getRdataCount() : 0)
{
encoder.start(rrclass, rrtype);
for (RdataIteratorPtr it = rrset->getRdataIterator();
!it->isLast();
it->next()) {
encoder.addRdata(it->getCurrent());
}
if (sig_rrset) {
for (RdataIteratorPtr it = sig_rrset->getRdataIterator();
!it->isLast();
it->next())
{
encoder.addSIGRdata(it->getCurrent());
}
}
const size_t data_len = encoder.getStorageLength();
data.resize(data_len);
encoder.encode(&data[0], data.size());
}
RRClass rrclass;
RRType rrtype;
size_t rdata_count;
size_t sig_count;
vector<uint8_t> data;
};
// Encapsulating parameters for a RdataReader. It extracts from the given
// RRset and its RRSIGs parameters that are necessary construct an RdataReader.
// RDATA data will be stored in the 'data' vector.
// members are defined as non const so we can use the object of this struct
// in a vector.
class ReaderBenchMark {
public:
ReaderBenchMark(const vector<EncodeParam>& encode_params,
MessageRenderer& renderer) :
encode_params_(encode_params), renderer_(renderer)
{}
unsigned int run() {
vector<EncodeParam>::const_iterator it;
const vector<EncodeParam>::const_iterator it_end =
encode_params_.end();
renderer_.clear();
for (it = encode_params_.begin(); it != it_end; ++it) {
RdataReader reader(it->rrclass, it->rrtype, &it->data[0],
it->rdata_count, it->sig_count,
boost::bind(&ReaderBenchMark::renderName,
this, _1, _2),
boost::bind(&ReaderBenchMark::renderData,
this, _1, _2));
reader.iterate();
reader.iterateAllSigs();
}
return (1);
}
void renderName(const LabelSequence& labels,
RdataNameAttributes attributes)
{
const bool compress =
(attributes & NAMEATTR_COMPRESSIBLE) != 0;
renderer_.writeName(labels, compress);
}
void renderData(const void* data, size_t data_len) {
renderer_.writeData(data, data_len);
}
private:
const vector<EncodeParam>& encode_params_;
MessageRenderer& renderer_;
};
// Builtin benchmark data. This is a list of RDATA (of RRs) in a response
// from a root server for the query for "www.example.com" (as of this
// implementation). We use a real world example to make the case practical.
const char* const rrsets_txt =
// AUTHORITY SECTION (NS)
"com. 172800 IN NS a.gtld-servers.net.\n"
"com. 172800 IN NS b.gtld-servers.net.\n"
"com. 172800 IN NS c.gtld-servers.net.\n"
"com. 172800 IN NS d.gtld-servers.net.\n"
"com. 172800 IN NS e.gtld-servers.net.\n"
"com. 172800 IN NS f.gtld-servers.net.\n"
"com. 172800 IN NS g.gtld-servers.net.\n"
"com. 172800 IN NS h.gtld-servers.net.\n"
"com. 172800 IN NS i.gtld-servers.net.\n"
"com. 172800 IN NS j.gtld-servers.net.\n"
"com. 172800 IN NS k.gtld-servers.net.\n"
"com. 172800 IN NS l.gtld-servers.net.\n"
"com. 172800 IN NS m.gtld-servers.net.\n"
// AUTHORITY SECTION (DS)
"com. 86400 IN DS 30909 8 2 "
"E2D3C916F6DEEAC73294E8268FB5885044A833FC5459588F4A9184CFC41A5766\n"
// AUTHORITY SECTION (RRSIG for DS)
"com. 86400 IN RRSIG DS 8 1 86400 20120822000000 20120814230000 50398 . "
"lcIpLRq4s91Fh1FihDXiDvVMMRqgy2jjlpiP4Y6sSjIrLue6Boi7xraj"
"Ouka34ubpl4KuIcopWe99LI/7Npvq0MYr9DaqfnX9dTW6Vc2C7/hKSsz"
"POYjraZZA3SCApgfNVzq+AscYlShi56f1vm7DQWw1eh1wHLdatidrQwNyDo=\n"
// ADDITIONAL SECTION
"a.gtld-servers.net. 172800 IN A 192.5.6.30\n"
"b.gtld-servers.net. 172800 IN A 192.33.14.30\n"
"c.gtld-servers.net. 172800 IN A 192.26.92.30\n"
"d.gtld-servers.net. 172800 IN A 192.31.80.30\n"
"e.gtld-servers.net. 172800 IN A 192.12.94.30\n"
"f.gtld-servers.net. 172800 IN A 192.35.51.30\n"
"g.gtld-servers.net. 172800 IN A 192.42.93.30\n"
"h.gtld-servers.net. 172800 IN A 192.54.112.30\n"
"i.gtld-servers.net. 172800 IN A 192.43.172.30\n"
"j.gtld-servers.net. 172800 IN A 192.48.79.30\n"
"k.gtld-servers.net. 172800 IN A 192.52.178.30\n"
"l.gtld-servers.net. 172800 IN A 192.41.162.30\n"
"m.gtld-servers.net. 172800 IN A 192.55.83.30\n"
"a.gtld-servers.net. 172800 IN AAAA 2001:503:a83e::2:30\n"
"b.gtld-servers.net. 172800 IN AAAA 2001:503:231d::2:30";
void
usage() {
std::cerr << "Usage: rdata_reader_bench [-n iterations]" << std::endl;
exit (1);
}
// Helper callback for masterLoad() used in main() to build test data.
void
setRRset(vector<ConstRRsetPtr>* rrsets, ConstRRsetPtr rrset) {
rrsets->push_back(rrset);
}
}
int
main(int argc, char* argv[]) {
int ch;
int iteration = 100000;
while ((ch = getopt(argc, argv, "n:")) != -1) {
switch (ch) {
case 'n':
iteration = atoi(optarg);
break;
case '?':
default:
usage();
}
}
argc -= optind;
argv += optind;
if (argc != 0) {
usage();
}
// Build test data. rrsets will consist of a list of RRsets corresponding
// to rrsets_txt defined above.
vector<ConstRRsetPtr> rrsets;
std::stringstream rrsets_stream(rrsets_txt);
masterLoad(rrsets_stream, Name::ROOT_NAME(), RRClass::IN(),
boost::bind(setRRset, &rrsets, _1));
// Create EncodeParam for each RRset (possibly with RRSIG) in rrsets,
// and store them in encode_param_list. It's the direct test input.
vector<EncodeParam> encode_param_list;
RdataEncoder encoder;
encode_param_list.push_back(EncodeParam(encoder, rrsets.at(0)));
encode_param_list.push_back(EncodeParam(encoder, rrsets.at(1),
rrsets.at(2)));
for (vector<ConstRRsetPtr>::const_iterator it = rrsets.begin() + 3;
it != rrsets.end();
++it) {
encode_param_list.push_back(EncodeParam(encoder, *it));
}
// The benchmark test uses a message renderer. Create it now and keep
// using it throughout the test.
isc::util::OutputBuffer buffer(4096); // 4096 should be sufficiently large
MessageRenderer renderer;
renderer.setBuffer(&buffer);
std::cout << "Benchmark for RdataReader" << std::endl;
BenchMark<ReaderBenchMark>(iteration,
ReaderBenchMark(encode_param_list, renderer));
return (0);
}
......@@ -12,6 +12,8 @@
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include "rdata_serialization.h"
#include <exceptions/exceptions.h>
#include <util/buffer.h>
......@@ -23,15 +25,10 @@
#include <dns/rrclass.h>
#include <dns/rrtype.h>
#include "rdata_encoder.h"
#include <boost/static_assert.hpp>
#include <cassert>
#include <cstring>
#include <vector>
#include <stdint.h>
#include <boost/static_assert.hpp>
using namespace isc::dns;
using std::vector;
......@@ -40,50 +37,9 @@ namespace isc {
namespace datasrc {
namespace memory {
namespace {
/// Specification of a single RDATA field in terms of internal encoding.
struct RdataFieldSpec {
enum FieldType {
FIXEDLEN_DATA = 0, // fixed-length data field
VARLEN_DATA, // variable-length data field
DOMAIN_NAME // domain name
};
const FieldType type; // field type
// The length of fixed-length data field. Only valid for FIXEDLEN_DATA.
// For type DOMAIN_NAME, set it to 0.
const uint16_t fixeddata_len;
// Attributes of the name. Only valid for DOMAIN_NAME.
// For type _DATA, set it to NAMEATTR_NONE.
const RdataNameAttributes name_attributes;
};
#include "rdata_serialization_priv.cc"
/// Specification of RDATA in terms of internal encoding.
///
/// The fields must be a sequence of:
/// <0 or 1 fixed/var-len data field>,
/// <1 or more domain name fields>,
/// <1 fixed/var-len data field>,
/// <1 or more domain name fields>,
/// <1 fixed/var-len data field>,
/// ...and so on.
/// There must not be more than one consecutive data fields (i.e., without
/// interleaved by a domain name); it would just be inefficient in terms of
/// memory footprint and iterating over the fields, and it would break
/// some assumption within the encoder implementation. For consecutive
/// data fields in the DNS protocol, if all fields have fixed lengths, they
/// should be combined into a single fixed-length field (like the last 20
/// bytes of SOA RDATA). If there's a variable length field, they should be
/// combined into a single variable-length field (such as DNSKEY, which has
/// 3 fixed-length fields followed by one variable-length field).
struct RdataEncodeSpec {
const uint16_t field_count; // total number of fields (# of fields member)
const uint16_t name_count; // number of domain name fields
const uint16_t varlen_count; // number of variable-length data fields
const RdataFieldSpec* const fields; // list of field specs
};
namespace {
// Many types of RDATA can be treated as a single-field, variable length
// field (in terms of our encoding). The following define such most general
......@@ -243,9 +199,11 @@ const size_t encode_spec_list_in_size =
sizeof(encode_spec_list_in) / sizeof(encode_spec_list_in[0]);
BOOST_STATIC_ASSERT(encode_spec_list_in_size == 48);
inline
}
/// \brief Get the spec for given class and type
const RdataEncodeSpec&
getRdataEncodeSpec(RRClass rrclass, RRType rrtype) {
getRdataEncodeSpec(const RRClass& rrclass, const RRType& rrtype) {
// Special case: for classes other than IN, we treat RDATA of RR types
// that are class-IN specific as generic opaque data.
if (rrclass != RRClass::IN() &&
......@@ -263,6 +221,8 @@ getRdataEncodeSpec(RRClass rrclass, RRType rrtype) {
return (generic_data_spec);
}
namespace {
// This class is a helper for RdataEncoder to divide the content of RDATA
// fields for encoding by "abusing" the message rendering logic.
// The idea is to identify domain name fields in the writeName() method,
......@@ -401,6 +361,7 @@ private:
// Placeholder to convert a name object to a label sequence.
uint8_t labels_placeholder_[LabelSequence::MAX_SERIALIZED_LENGTH];
};
} // end of unnamed namespace
struct RdataEncoder::RdataEncoderImpl {
......@@ -525,77 +486,149 @@ RdataEncoder::encode(void* buf, size_t buf_len) const {
assert(buf_len >= dp - dp_beg);
}
namespace testing {
RdataReader::RdataReader(const RRClass& rrclass, const RRType& rrtype,
const void* data,
size_t rdata_count, size_t sig_count,
const NameAction& name_action,
const DataAction& data_action) :
name_action_(name_action),
data_action_(data_action),
spec_(getRdataEncodeSpec(rrclass, rrtype)),
var_count_total_(spec_.varlen_count * rdata_count),
sig_count_(sig_count),
spec_count_(spec_.field_count * rdata_count),
// The lenghts are stored first
lengths_(reinterpret_cast<const uint16_t*>(data)),
// And the data just after all the lengths
data_(reinterpret_cast<const uint8_t*>(data) +
(var_count_total_ + sig_count_) * sizeof(uint16_t)),
sigs_(NULL)
{
rewind();
}
void
foreachRdataField(RRClass rrclass, RRType rrtype,
size_t rdata_count,
const vector<uint8_t>& encoded_data,
const vector<uint16_t>& varlen_list,
NameCallback name_callback, DataCallback data_callback)
RdataReader::rewind() {
data_pos_ = 0;
spec_pos_ = 0;
length_pos_ = 0;
sig_data_pos_ = 0;
sig_pos_ = 0;
}
RdataReader::Boundary
RdataReader::nextInternal(const NameAction& name_action,
const DataAction& data_action)
{
const RdataEncodeSpec& encode_spec = getRdataEncodeSpec(rrclass, rrtype);
size_t off = 0;
size_t varlen_count = 0;
size_t name_count = 0;
for (size_t count = 0; count < rdata_count; ++count) {
for (size_t i = 0; i < encode_spec.field_count; ++i) {
const RdataFieldSpec& field_spec = encode_spec.fields[i];
switch (field_spec.type) {
case RdataFieldSpec::FIXEDLEN_DATA:
if (data_callback) {
data_callback(&encoded_data.at(off),
field_spec.fixeddata_len);
}
off += field_spec.fixeddata_len;
break;
case RdataFieldSpec::VARLEN_DATA:
{
const size_t varlen = varlen_list.at(varlen_count);
if (data_callback && varlen > 0) {
data_callback(&encoded_data.at(off), varlen);
}
off += varlen;
++varlen_count;
break;
}
case RdataFieldSpec::DOMAIN_NAME:
{
++name_count;
const LabelSequence labels(&encoded_data.at(off));
if (name_callback) {
name_callback(labels, field_spec.name_attributes);
}
off += labels.getSerializedLength();
break;
}
}
if (spec_pos_ < spec_count_) {
const RdataFieldSpec& spec(spec_.fields[(spec_pos_++) %
spec_.field_count]);
if (spec.type == RdataFieldSpec::DOMAIN_NAME) {
const LabelSequence sequence(data_ + data_pos_);
data_pos_ += sequence.getSerializedLength();
name_action(sequence, spec.name_attributes);
} else {
const size_t length(spec.type == RdataFieldSpec::FIXEDLEN_DATA ?
spec.fixeddata_len : lengths_[length_pos_++]);
const uint8_t* const pos = data_ + data_pos_;
data_pos_ += length;
data_action(pos, length);
}
return (spec_pos_ % spec_.field_count == 0 ?
RDATA_BOUNDARY : NO_BOUNDARY);
} else {
sigs_ = data_ + data_pos_;
return (RRSET_BOUNDARY);
}
assert(name_count == encode_spec.name_count * rdata_count);
assert(varlen_count == encode_spec.varlen_count * rdata_count);
}
RdataReader::Boundary
RdataReader::next() {
return (nextInternal(name_action_, data_action_));
}
namespace {
void
foreachRRSig(const vector<uint8_t>& encoded_data,
const vector<uint16_t>& rrsiglen_list,
DataCallback data_callback)
{
size_t rrsig_totallen = 0;
for (vector<uint16_t>::const_iterator it = rrsiglen_list.begin();
it != rrsiglen_list.end();
++it) {
rrsig_totallen += *it;
emptyNameAction(const LabelSequence&, unsigned) {
// Do nothing here.
}
void
emptyDataAction(const void*, size_t) {
// Do nothing here.
}
}
RdataReader::Boundary
RdataReader::nextSig() {
if (sig_pos_ < sig_count_) {
if (sigs_ == NULL) {
// We didn't find where the signatures start yet. We do it
// by iterating the whole data and then returning the state
// back.
const size_t data_pos = data_pos_;
const size_t spec_pos = spec_pos_;
const size_t length_pos = length_pos_;
// When the next() gets to the last item, it sets the sigs_
while (nextInternal(emptyNameAction, emptyDataAction) !=
RRSET_BOUNDARY) {}
assert(sigs_ != NULL);
// Return the state
data_pos_ = data_pos;
spec_pos_ = spec_pos;
length_pos_ = length_pos;
}
// Extract the result
const size_t length = lengths_[var_count_total_ + sig_pos_];
const uint8_t* const pos = sigs_ + sig_data_pos_;
// Move the position of iterator.
sig_data_pos_ += lengths_[var_count_total_ + sig_pos_];
++sig_pos_;
// Call the callback
data_action_(pos, length);
return (RDATA_BOUNDARY);
} else {
return (RRSET_BOUNDARY);
}
assert(encoded_data.size() >= rrsig_totallen);
}
const uint8_t* dp = &encoded_data[encoded_data.size() - rrsig_totallen];
for (size_t i = 0; i < rrsiglen_list.size(); ++i) {
data_callback(dp, rrsiglen_list[i]);
dp += rrsiglen_list[i];
size_t
RdataReader::getSize() const {
size_t storage_size = 0; // this will be the end result
size_t data_pos = 0;
size_t length_pos = 0;
// Go over all data fields, adding their lengths to storage_size
for (size_t spec_pos = 0; spec_pos < spec_count_; ++spec_pos) {
const RdataFieldSpec& spec =
spec_.fields[spec_pos % spec_.field_count];
if (spec.type == RdataFieldSpec::DOMAIN_NAME) {
const size_t seq_len =
LabelSequence(data_ + data_pos).getSerializedLength();
data_pos += seq_len;
storage_size += seq_len;
} else {
const size_t data_len =
(spec.type == RdataFieldSpec::FIXEDLEN_DATA ?
spec.fixeddata_len : lengths_[length_pos++]);
data_pos += data_len;
storage_size += data_len;
}
}
// Same for all RRSIG data
for (size_t sig_pos = 0; sig_pos < sig_count_; ++sig_pos) {
const size_t sig_data_len = lengths_[length_pos++];
storage_size += sig_data_len;
}
// Finally, add the size for 16-bit length fields
storage_size += (var_count_total_ * sizeof(uint16_t) +
sig_count_ * sizeof(uint16_t));
return (storage_size);
}
} // namespace testing
} // namespace memory
} // namespace datasrc
......
......@@ -27,8 +27,7 @@
#include <vector>
/// \file rdata_encoder.h
/// \brief Set of utility classes for encoding RDATA in memory efficient way.
/// \file rdata_serialization.h
///
/// This file defines a set of interfaces (classes, types, constants) to
/// manipulate a given set of RDATA of the same type (normally associated with
......@@ -42,9 +41,9 @@
/// Two main classes are provided: one is
/// \c isc::datasrc::memory::RdataEncoder, which allows
/// the application to create encoded data for a set of RDATA;
/// the other (TBD) provides an interface to iterate over encoded set of
/// RDATA for purposes such as data lookups or rendering the data into the