Commit e9fa118c authored by JINMEI Tatuya's avatar JINMEI Tatuya
Browse files

[master] Merge branch 'trac2831'

parents bf06401c e825b22e
......@@ -884,6 +884,17 @@ if test "$BOOST_NUMERIC_CAST_WOULDFAIL" = "yes" -a X"$werror_ok" = X1 -a $CLANGP
AC_MSG_ERROR([Failed to compile a required header file. If you are using FreeBSD and Boost installed via ports, retry with specifying --without-werror. See the ChangeLog entry for Trac no. 1991 for more details.])
fi
use_shared_memory=yes
AC_ARG_WITH(shared-memory,
AC_HELP_STRING([--with-shared-memory],
[Build with Boost shared memory support; for large scale authoritative DNS servers]),
[use_shared_memory=$withval])
if test X$use_shared_memory = Xyes -a "$BOOST_MAPPED_FILE_WOULDFAIL" = "yes"; then
AC_MSG_ERROR([Boost shared memory does not compile on this system. If you don't need it (most normal users won't) build without it; using a different compiler or a different version of Boost may also help.])
fi
AM_CONDITIONAL([USE_SHARED_MEMORY], [test x$use_shared_memory = xyes])
AC_SUBST(BOOST_MAPPED_FILE_CXXFLAG)
# Add some default CPP flags needed for Boost, identified by the AX macro.
CPPFLAGS="$CPPFLAGS $CPPFLAGS_BOOST_THREADCONF"
......
......@@ -23,7 +23,11 @@ dnl BOOST_OFFSET_PTR_WOULDFAIL set to "yes" if offset_ptr would cause build
dnl error; otherwise set to "no"
dnl BOOST_NUMERIC_CAST_WOULDFAIL set to "yes" if numeric_cast would cause
dnl build error; otherwise set to "no"
dnl
dnl BOOST_MAPPED_FILE_WOULDFAIL set to "yes" if managed_mapped_file would
dnl cause build failure; otherwise set to "no"
dnl BOOST_MAPPED_FILE_CXXFLAG set to the compiler flag that would need to
dnl compile managed_mapped_file (can be empty).
dnl It is of no use if "WOULDFAIL" is yes.
AC_DEFUN([AX_BOOST_FOR_BIND10], [
AC_LANG_SAVE
......@@ -101,10 +105,49 @@ if test "X$GXX" = "Xyes"; then
CXXFLAGS="$CXXFLAGS_SAVED"
else
# This doesn't matter for non-g++
BOOST_NUMERIC_CAST_WOULDFAIL=no
# This doesn't matter for non-g++
BOOST_NUMERIC_CAST_WOULDFAIL=no
fi
# Boost interprocess::managed_mapped_file is highly system dependent and
# can cause many portability issues. We are going to check if it could
# compile at all, possibly with being lenient about compiler warnings.
BOOST_MAPPED_FILE_WOULDFAIL=yes
BOOST_MAPPED_FILE_CXXFLAG=
CXXFLAGS_SAVED="$CXXFLAGS"
try_flags="no"
if test "X$GXX" = "Xyes"; then
CXXFLAGS="$CXXFLAGS -Wall -Wextra -Werror"
try_flags="$try_flags -Wno-error"
fi
# clang can cause false positives with -Werror without -Qunused-arguments
AC_CHECK_DECL([__clang__], [CXXFLAGS="$CXXFLAGS -Qunused-arguments"], [])
AC_MSG_CHECKING([Boost managed_mapped_file compiles])
CXXFLAGS_SAVED2="$CXXFLAGS"
for flag in $try_flags; do
if test "$flag" != no; then
BOOST_MAPPED_FILE_CXXFLAG="$flag"
fi
CXXFLAGS="$CXXFLAGS $BOOST_MAPPED_FILE_CXXFLAG"
AC_TRY_COMPILE([
#include <boost/interprocess/managed_mapped_file.hpp>
],[
return (boost::interprocess::managed_mapped_file().all_memory_deallocated());
],[AC_MSG_RESULT([yes, with $flag flag])
BOOST_MAPPED_FILE_WOULDFAIL=no
break
],[])
CXXFLAGS="$CXXFLAGS_SAVED2"
done
if test $BOOST_MAPPED_FILE_WOULDFAIL = yes; then
AC_MSG_RESULT(no)
fi
CXXFLAGS="$CXXFLAGS_SAVED"
AC_SUBST(BOOST_INCLUDES)
CPPFLAGS="$CPPFLAGS_SAVED"
......
......@@ -6,6 +6,18 @@ AM_CPPFLAGS += -I$(top_srcdir)/src/lib/exceptions -I$(top_builddir)/src/lib/exce
AM_CPPFLAGS += $(BOOST_INCLUDES)
AM_CPPFLAGS += -DLOCKFILE_DIR=\"${localstatedir}/${PACKAGE_NAME}\"
AM_CXXFLAGS = $(B10_CXXFLAGS)
# If we use the shared-memory support, corresponding Boost library may
# cause build failures especially if it's strict about warnings. We've
# detected it in ./configure and set BOOST_MAPPED_FILE_CXXFLAG to be more
# lenient as necessary (specifically, when set it'd usually suppress -Werror).
# This is a module wide setting, and has a possible bad side effect of hiding
# issues in other files, but making it per-file seems to be too costly.
# So we begin with the wider setting. If the side effect turns out to be too
# harmful, we'll consider other measure, e.g, moving the related files into
# a subdirectory.
if USE_SHARED_MEMORY
AM_CXXFLAGS += $(BOOST_MAPPED_FILE_CXXFLAG)
endif
lib_LTLIBRARIES = libb10-util.la
libb10_util_la_SOURCES = filename.h filename.cc
......@@ -18,6 +30,9 @@ libb10_util_la_SOURCES += interprocess_sync_file.h interprocess_sync_file.cc
libb10_util_la_SOURCES += interprocess_sync_null.h interprocess_sync_null.cc
libb10_util_la_SOURCES += memory_segment.h
libb10_util_la_SOURCES += memory_segment_local.h memory_segment_local.cc
if USE_SHARED_MEMORY
libb10_util_la_SOURCES += memory_segment_mapped.h memory_segment_mapped.cc
endif
libb10_util_la_SOURCES += range_utilities.h
libb10_util_la_SOURCES += hash/sha1.h hash/sha1.cc
libb10_util_la_SOURCES += encode/base16_from_binary.h
......
......@@ -15,27 +15,107 @@
#ifndef MEMORY_SEGMENT_H
#define MEMORY_SEGMENT_H
#include <exceptions/exceptions.h>
#include <stdlib.h>
namespace isc {
namespace util {
/// \brief Exception that can be thrown when constructing a MemorySegment
/// object.
class MemorySegmentOpenError : public Exception {
public:
MemorySegmentOpenError(const char* file, size_t line, const char* what) :
isc::Exception(file, line, what) {}
};
/// \brief Exception that is thrown, when allocating space in a MemorySegment
/// results in growing the underlying segment.
///
/// See MemorySegment::allocate() for details.
class MemorySegmentGrown : public Exception {
public:
MemorySegmentGrown(const char* file, size_t line, const char* what) :
isc::Exception(file, line, what) {}
};
/// \brief General error that can be thrown by a MemorySegment
/// implementation.
class MemorySegmentError : public Exception {
public:
MemorySegmentError(const char* file, size_t line, const char* what) :
isc::Exception(file, line, what) {}
};
/// \brief Memory Segment Class
///
/// This class specifies an interface for allocating memory
/// segments. This is an abstract class and a real
/// implementation such as MemorySegmentLocal should be used
/// in code.
/// This class specifies an interface for allocating memory segments.
/// It's intended to provide a unified interface, whether the underlying
/// memory is local to a specific process or is sharable by multiple
/// processes.
///
/// This is an abstract class and a real implementation such as
/// MemorySegmentLocal should be used in code.
class MemorySegment {
public:
/// \brief Destructor
virtual ~MemorySegment() {}
/// \brief Allocate/acquire a segment of memory. The source of the
/// memory is dependent on the implementation used.
/// \brief Allocate/acquire a fragment of memory.
///
/// Throws <code>std::bad_alloc</code> if the implementation cannot
/// allocate the requested storage.
/// The source of the memory is dependent on the implementation used.
///
/// Depending on the implementation details, it may have to grow the
/// internal memory segment (again, in an implementation dependent way)
/// to allocate the required size of memory. In that case the
/// implementation must grow the internal segment sufficiently so the
/// next call to allocate() for the same size will succeed, and throw
/// a \c MemorySegmentGrown exception (not really allocating the memory
/// yet).
///
/// An application that uses this memory segment abstraction to allocate
/// memory should expect this exception, and should normally catch it
/// at an appropriate layer (which may be immediately after a call to
/// \c allocate() or a bit higher layer). It should interpret the
/// exception as any raw address that belongs to the segment may have
/// been remapped and must be re-fetched via an already established
/// named address using the \c getNamedAddress() method.
///
/// The intended use case of \c allocate() with the \c MemorySegmentGrown
/// exception is to build a complex object that would internally require
/// multiple calls to \c allocate():
///
/// \code
/// ComplicatedStuff* stuff = NULL;
/// while (!stuff) { // this must eventually succeed or result in bad_alloc
/// try {
/// // create() is a factory method that takes a memory segment
/// // and calls allocate() on it multiple times. create()
/// // provides an exception guarantee that any intermediately
/// // allocated memory will be properly deallocate()-ed on
/// // exception.
/// stuff = ComplicatedStuff::create(mem_segment);
/// } catch (const MemorySegmentGrown&) { /* just try again */ }
/// }
/// \endcode
///
/// This way, \c create() can be written as if each call to \c allocate()
/// always succeeds.
///
/// Alternatively, or in addition to this, we could introduce a "no throw"
/// version of this method with a way to tell the caller the reason of
/// any failure (whether it's really out of memory or just due to growing
/// the segment). That would be more convenient if the caller wants to
/// deal with the failures on a per-call basis rather than as a set
/// of calls like in the above example. At the moment, we don't expect
/// to have such use-cases, so we only provide the exception
/// version.
///
/// \throw std::bad_alloc The implementation cannot allocate the
/// requested storage.
/// \throw MemorySegmentGrown The memory segment doesn't have sufficient
/// space for the requested size and has grown internally.
///
/// \param size The size of the memory requested in bytes.
/// \return Returns pointer to the memory allocated.
......@@ -50,6 +130,18 @@ public:
/// use this argument in some implementations to test if all allocated
/// memory was deallocated properly.
///
/// Specific implementation may also throw \c MemorySegmentError if it
/// encounters violation of implementation specific restrictions.
///
/// In general, however, this method must succeed and exception free
/// as long as the caller passes valid parameters (\c ptr specifies
/// memory previously allocated and \c size is correct).
///
/// \throw OutOfRange The passed size doesn't match the allocated memory
/// size (when identifiable for the implementation).
/// \throw MemorySegmentError Failure of implementation specific
/// validation.
///
/// \param ptr Pointer to the block of memory to free/release. This
/// should be equal to a value returned by <code>allocate()</code>.
/// \param size The size of the memory to be freed in bytes. This
......@@ -58,12 +150,155 @@ public:
/// \brief Check if all allocated memory was deallocated.
///
/// \return Returns <code>true</code> if all allocated memory was
/// \return Returns <code>true</code> if all allocated memory (including
/// names associated by memory addresses by \c setNamedAddress()) was
/// deallocated, <code>false</code> otherwise.
virtual bool allMemoryDeallocated() const = 0;
/// \brief Associate specified address in the segment with a given name.
///
/// This method establishes an association between the given name and
/// the address in an implementation specific way. The stored address
/// is retrieved by the name later by calling \c getNamedAddress().
/// If the underlying memory segment is sharable by multiple processes,
/// the implementation must ensure the portability of the association;
/// if a process gives an address in the shared segment a name, another
/// process that shares the same segment should be able to retrieve the
/// corresponding address by that name (in such cases the real address
/// may be different between these two processes).
///
/// \c addr must be 0 (NULL) or an address that belongs to this segment.
/// The latter case means it must be the return value of a previous call
/// to \c allocate(). The actual implementation is encouraged to detect
/// violation of this restriction and signal it with an exception, but
/// it's not an API requirement. It's generally the caller's
/// responsibility to meet the restriction. Note that NULL is allowed
/// as \c addr even if it wouldn't be considered to "belong to" the
/// segment in its normal sense; it can be used to indicate that memory
/// has not been allocated for the specified name. A subsequent call
/// to \c getNamedAddress() will return NULL for that name.
///
/// \note Naming an address is intentionally separated from allocation
/// so that, for example, one module of a program can name a memory
/// region allocated by another module of the program.
///
/// There can be an existing association for the name; in that case the
/// association will be overridden with the newly given address.
///
/// While normally unexpected, it's possible that available space in the
/// segment is not sufficient to allocate a space (if not already exist)
/// for the specified name in the segment. In that case, if possible, the
/// implementation should try to grow the internal segment and retry
/// establishing the association. The implementation should throw
/// std::bad_alloc if even reasonable attempts of retry still fail.
///
/// This method should normally return false, but if the internal segment
/// had to grow to store the given name, it must return true. The
/// application should interpret it just like the case of
/// \c MemorySegmentGrown exception thrown from the \c allocate() method.
///
/// \note The behavior in case the internal segment grows is different
/// from that of \c allocate(). This is intentional. In intended use
/// cases (for the moment) this method will be called independently,
/// rather than as part of a set of allocations. It's also expected
/// that other raw memory addresses (which would have been invalidated
/// due to the change to the segment) won't be referenced directly
/// immediately after this call. So, the caller should normally be able
/// to call this method as mostly never-fail one (except in case of real
/// memory exhaustion) and ignore the return value.
///
/// \throw std::bad_alloc Allocation of a segment space for the given name
/// failed.
/// \throw InvalidParameter name is NULL.
/// \throw MemorySegmentError Failure of implementation specific
/// validation.
///
/// \param name A C string to be associated with \c addr. Must not be NULL.
/// \param addr A memory address returned by a prior call to \c allocate.
/// \return true if the internal segment has grown to allocate space for
/// the name; false otherwise (see above).
bool setNamedAddress(const char* name, void* addr) {
// This public method implements common validation. The actual
// work specific to the derived segment is delegated to the
// corresponding protected method.
if (!name) {
isc_throw(InvalidParameter,
"NULL name is given to setNamedAddress");
}
return (setNamedAddressImpl(name, addr));
}
/// \brief Return the address in the segment that has the given name.
///
/// This method returns the memory address in the segment corresponding
/// to the specified \c name. The name and address must have been
/// associated by a prior call to \c setNameAddress(). If no address
/// associated with the given name is found, it returns NULL.
///
/// This method should generally be considered exception free, but there
/// can be a small chance it throws, depending on the internal
/// implementation (e.g., if it converts the name to std::string), so the
/// API doesn't guarantee that property. In general, if this method
/// throws it should be considered a fatal condition.
///
/// \throw InvalidParameter name is NULL.
///
/// \param name A C string of which the segment memory address is to be
/// returned. Must not be NULL.
/// \return The address associated with the name, or NULL if not found.
void* getNamedAddress(const char* name) {
// This public method implements common validation. The actual
// work specific to the derived segment is delegated to the
// corresponding protected method.
if (!name) {
isc_throw(InvalidParameter,
"NULL name is given to getNamedAddress");
}
return (getNamedAddressImpl(name));
}
/// \brief Delete a name previously associated with a segment address.
///
/// This method deletes the association of the given \c name to
/// a corresponding segment address previously established by
/// \c setNamedAddress(). If there is no association for the given name
/// this method returns false; otherwise it returns true.
///
/// See \c getNamedAddress() about exception consideration.
///
/// \throw InvalidParameter name is NULL.
/// \throw MemorySegmentError Failure of implementation specific
/// validation.
///
/// \param name A C string of which the segment memory address is to be
/// deleted. Must not be NULL.
bool clearNamedAddress(const char* name) {
// This public method implements common validation. The actual
// work specific to the derived segment is delegated to the
// corresponding protected method.
if (!name) {
isc_throw(InvalidParameter,
"NULL name is given to clearNamedAddress");
}
return (clearNamedAddressImpl(name));
}
protected:
/// \brief Implementation of setNamedAddress beyond common validation.
virtual bool setNamedAddressImpl(const char* name, void* addr) = 0;
/// \brief Implementation of getNamedAddress beyond common validation.
virtual void* getNamedAddressImpl(const char* name) = 0;
/// \brief Implementation of clearNamedAddress beyond common validation.
virtual bool clearNamedAddressImpl(const char* name) = 0;
};
} // namespace util
} // namespace isc
#endif // MEMORY_SEGMENT_H
// Local Variables:
// mode: c++
// End:
......@@ -48,7 +48,28 @@ MemorySegmentLocal::deallocate(void* ptr, size_t size) {
bool
MemorySegmentLocal::allMemoryDeallocated() const {
return (allocated_size_ == 0);
return (allocated_size_ == 0 && named_addrs_.empty());
}
void*
MemorySegmentLocal::getNamedAddressImpl(const char* name) {
std::map<std::string, void*>::iterator found = named_addrs_.find(name);
if (found != named_addrs_.end()) {
return (found->second);
}
return (0);
}
bool
MemorySegmentLocal::setNamedAddressImpl(const char* name, void* addr) {
named_addrs_[name] = addr;
return (false);
}
bool
MemorySegmentLocal::clearNamedAddressImpl(const char* name) {
const size_t n_erased = named_addrs_.erase(name);
return (n_erased != 0);
}
} // namespace util
......
......@@ -17,6 +17,9 @@
#include <util/memory_segment.h>
#include <string>
#include <map>
namespace isc {
namespace util {
......@@ -63,14 +66,43 @@ public:
/// deallocated, <code>false</code> otherwise.
virtual bool allMemoryDeallocated() const;
/// \brief Local segment version of getNamedAddress.
///
/// There's a small chance this method could throw std::bad_alloc.
/// It should be considered a fatal error.
virtual void* getNamedAddressImpl(const char* name);
/// \brief Local segment version of setNamedAddress.
///
/// This version does not validate the given address to see whether it
/// belongs to this segment.
///
/// This implementation of this method always returns \c false (but the
/// application should expect a return value of \c true unless it knows
/// the memory segment class is \c MemorySegmentLocal and needs to
/// exploit the fact).
virtual bool setNamedAddressImpl(const char* name, void* addr);
/// \brief Local segment version of clearNamedAddress.
///
/// There's a small chance this method could throw std::bad_alloc.
/// It should be considered a fatal error.
virtual bool clearNamedAddressImpl(const char* name);
private:
// allocated_size_ can underflow, wrap around to max size_t (which
// is unsigned). But because we only do a check against 0 and not a
// relation comparison, this is okay.
size_t allocated_size_;
std::map<std::string, void*> named_addrs_;
};
} // namespace util
} // namespace isc
#endif // MEMORY_SEGMENT_LOCAL_H
// Local Variables:
// mode: c++
// End:
// Copyright (C) 2013 Internet Systems Consortium, Inc. ("ISC")
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.
#include <util/memory_segment_mapped.h>
#include <util/unittests/check_valgrind.h>
#include <exceptions/exceptions.h>
#include <boost/scoped_ptr.hpp>
#include <boost/interprocess/exceptions.hpp>
#include <boost/interprocess/managed_mapped_file.hpp>
#include <boost/interprocess/offset_ptr.hpp>
#include <boost/interprocess/mapped_region.hpp>
#include <boost/interprocess/sync/file_lock.hpp>
#include <cassert>
#include <string>
#include <new>
// boost::interprocess namespace is big and can cause unexpected import
// (e.g., it has "read_only"), so it's safer to be specific for shortcuts.
using boost::interprocess::basic_managed_mapped_file;
using boost::interprocess::rbtree_best_fit;
using boost::interprocess::null_mutex_family;
using boost::interprocess::iset_index;
using boost::interprocess::create_only_t;
using boost::interprocess::create_only;
using boost::interprocess::open_or_create_t;
using boost::interprocess::open_or_create;
using boost::interprocess::open_read_only;
using boost::interprocess::open_only;
using boost::interprocess::offset_ptr;
namespace isc {
namespace util {
// Definition of class static constant so it can be referenced by address
// or reference.
const size_t MemorySegmentMapped::INITIAL_SIZE;
// We customize managed_mapped_file to make it completely lock free. In our
// usage the application (or the system of applications) is expected to ensure
// there's at most one writer process or concurrent writing the shared memory
// segment is protected at a higher level. Using the null mutex is mainly for
// eliminating unnecessary dependency; the default version would require
// (probably depending on the system) Pthread library that is actually not
// needed and could cause various build time troubles.
typedef basic_managed_mapped_file<char,
rbtree_best_fit<null_mutex_family>,
iset_index> BaseSegment;
struct MemorySegmentMapped::Impl {
// Constructor for create-only (and read-write) mode. this case is
// tricky because we want to remove any existing file but we also want
// to detect possible conflict with other readers or writers using
// file lock.
Impl(const std::string& filename, create_only_t, size_t initial_size) :
read_only_(false), filename_(filename)
{
try {
// First, try opening it in boost create_only mode; it fails if
// the file exists (among other reasons).
base_sgmt_.reset(new BaseSegment(create_only, filename.c_str(),
initial_size));
} catch (const boost::interprocess::interprocess_exception& ex) {
// We assume this is because the file exists; otherwise creating
// file_lock would fail with interprocess_exception, and that's
// what we want here (we wouldn't be able to create a segment
// anyway).
lock_.reset(new boost::interprocess::file_lock(filename.c_str()));
// Confirm there's no other reader or writer, and then release
// the lock before we remove the file; there's a chance of race
// here, but this check doesn't intend to guarantee 100% safety
// and so it should be okay.
checkWriter();
lock_.reset();
// now remove the file (if it happens to have been delete, this
// will be no-op), then re-open it with create_only. this time
// it should succeed, and if it fails again, that's fatal for this
// constructor.
boost::interprocess::file_mapping::remove(filename.c_str());
base_sgmt_.reset(new BaseSegment(create_only, filename.c_str(),
initial_size));
}
// confirm there's no other user and there won't either.
lock_.reset(new boost::interprocess::file_lock(filename.c_str()));
checkWriter();
}
// Constructor for open-or-write (and read-write) mode
Impl(const std::string& filename, open_or_create_t, size_t initial_size) :
read_only_(false), filename_(filename),
base_sgmt_(new BaseSegment(open_or_create, filename.c_str(),
initial_size)),
lock_(new boost::interprocess::file_lock(filename.c_str()))
{
checkWriter();
}
// Constructor for existing segment, either read-only or read-write
Impl(const std::string& filename, bool read_only) :
read_only_(read_only), filename_(filename),
base_sgmt_(read_only_ ?
new BaseSegment(open_read_only, filename.c_str()) :
new BaseSegment(open_only, filename.c_str())),
lock_(new boost::interprocess::file_lock(filename.c_str()))
{
if (read_only_) {
checkReader();
} else {
checkWriter();
}
}
// Internal helper to grow the underlying mapped segment.
void growSegment() {
// We first need to unmap it before calling grow().
const size_t prev_size = base_sgmt_->get_size();
base_sgmt_.reset();
// Double the segment size. In theory, this process could repeat
// so many times, counting to "infinity", and new_size eventually
// overflows. That would cause a harsh disruption or unexpected