Commit 92701f4f authored by Marcin Siodelski's avatar Marcin Siodelski

[5189] Implemented JSONFeed class.

parent 4499b731
......@@ -8,9 +8,11 @@ lib_LTLIBRARIES = libkea-cc.la
libkea_cc_la_SOURCES = data.cc data.h
libkea_cc_la_SOURCES += cfg_to_element.h dhcp_config_error.h
libkea_cc_la_SOURCES += command_interpreter.cc command_interpreter.h
libkea_cc_la_SOURCES += json_feed.cc json_feed.h
libkea_cc_la_SOURCES += simple_parser.cc simple_parser.h
libkea_cc_la_LIBADD = $(top_builddir)/src/lib/exceptions/libkea-exceptions.la
libkea_cc_la_LIBADD = $(top_builddir)/src/lib/util/libkea-util.la
libkea_cc_la_LIBADD += $(top_builddir)/src/lib/exceptions/libkea-exceptions.la
libkea_cc_la_LIBADD += $(BOOST_LIBS)
libkea_cc_la_LDFLAGS = -no-undefined -version-info 2:0:0
......
// Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
#include <cc/data.h>
#include <cc/json_feed.h>
#include <boost/bind.hpp>
#include <iostream>
using namespace isc::data;
using namespace isc::util;
namespace isc {
namespace config {
const int JSONFeed::RECEIVE_START_ST;
const int JSONFeed::WHITESPACE_BEFORE_JSON_ST;
const int JSONFeed::JSON_START_ST;
const int JSONFeed::INNER_JSON_ST;
const int JSONFeed::JSON_END_ST;
const int JSONFeed::FEED_OK_ST;
const int JSONFeed::FEED_FAILED_ST;
const int JSONFeed::DATA_READ_OK_EVT;
const int JSONFeed::NEED_MORE_DATA_EVT;
const int JSONFeed::MORE_DATA_PROVIDED_EVT;
const int JSONFeed::FEED_OK_EVT;
const int JSONFeed::FEED_FAILED_EVT;
JSONFeed::JSONFeed()
: StateModel(), buffer_(), error_message_(), open_scopes_(0),
output_() {
}
void
JSONFeed::initModel() {
// Intialize dictionaries of events and states.
initDictionaries();
// Set the current state to starting state and enter the run loop.
setState(RECEIVE_START_ST);
// Parsing starts from here.
postNextEvent(START_EVT);
}
void
JSONFeed::poll() {
try {
// Process the input data until no more data is available or until
// JSON feed ends with matching closing brace.
do {
getState(getCurrState())->run();
} while (!isModelDone() && (getNextEvent() != NOP_EVT) &&
(getNextEvent() != NEED_MORE_DATA_EVT));
} catch (const std::exception& ex) {
abortModel(ex.what());
}
}
bool
JSONFeed::needData() const {
return ((getNextEvent() == NEED_MORE_DATA_EVT) ||
(getNextEvent() == START_EVT));
}
bool
JSONFeed::feedOk() const {
return ((getNextEvent() == END_EVT) &&
(getLastEvent() == FEED_OK_EVT));
}
ElementPtr
JSONFeed::toElement() const {
if (needData()) {
isc_throw(JSONFeedError, "unable to retrieve the data form the"
" JSON feed while parsing hasn't finished");
}
try {
return (Element::fromWire(output_));
} catch (const std::exception& ex) {
isc_throw(JSONFeedError, ex.what());
}
}
void
JSONFeed::postBuffer(const void* buf, const size_t buf_size) {
if (buf_size > 0) {
// The next event is NEED_MORE_DATA_EVT when the parser wants to
// signal that more data is needed. This method is called to supply
// more data and thus it should change the next event to
// MORE_DATA_PROVIDED_EVT.
if (getNextEvent() == NEED_MORE_DATA_EVT) {
transition(getCurrState(), MORE_DATA_PROVIDED_EVT);
}
buffer_.insert(buffer_.end(), static_cast<const char*>(buf),
static_cast<const char*>(buf) + buf_size);
}
}
void
JSONFeed::defineEvents() {
StateModel::defineEvents();
// Define JSONFeed specific events.
defineEvent(DATA_READ_OK_EVT, "DATA_READ_OK_EVT");
defineEvent(NEED_MORE_DATA_EVT, "NEED_MORE_DATA_EVT");
defineEvent(MORE_DATA_PROVIDED_EVT, "MORE_DATA_PROVIDED_EVT");
defineEvent(FEED_OK_EVT, "FEED_OK_EVT");
defineEvent(FEED_FAILED_EVT, "FEED_FAILED_EVT");
}
void
JSONFeed::verifyEvents() {
StateModel::verifyEvents();
getEvent(DATA_READ_OK_EVT);
getEvent(NEED_MORE_DATA_EVT);
getEvent(MORE_DATA_PROVIDED_EVT);
getEvent(FEED_OK_EVT);
getEvent(FEED_FAILED_EVT);
}
void
JSONFeed::defineStates() {
// Call parent class implementation first.
StateModel::defineStates();
defineState(RECEIVE_START_ST, "RECEIVE_START_ST",
boost::bind(&JSONFeed::receiveStartHandler, this));
defineState(WHITESPACE_BEFORE_JSON_ST, "WHITESPACE_BEFORE_JSON_ST",
boost::bind(&JSONFeed::whiteSpaceBeforeJSONHandler, this));
defineState(INNER_JSON_ST, "INNER_JSON_ST",
boost::bind(&JSONFeed::innerJSONHandler, this));
defineState(JSON_END_ST, "JSON_END_ST",
boost::bind(&JSONFeed::endJSONHandler, this));
}
void
JSONFeed::feedFailure(const std::string& error_msg) {
error_message_ = error_msg + " : " + getContextStr();
transition(FEED_FAILED_ST, FEED_FAILED_EVT);
}
void
JSONFeed::onModelFailure(const std::string& explanation) {
if (error_message_.empty()) {
error_message_ = explanation;
}
}
bool
JSONFeed::popNextFromBuffer(char& next) {
// If there are any characters in the buffer, pop next.
if (!buffer_.empty()) {
next = buffer_.front();
buffer_.pop_front();
return (true);
}
return (false);
}
char
JSONFeed::getNextFromBuffer() {
unsigned int ev = getNextEvent();
char c = '\0';
// The caller should always provide additional data when the
// NEED_MORE_DATA_EVT occurrs. If the next event is still
// NEED_MORE_DATA_EVT it indicates that the caller hasn't provided
// the data.
if (ev == NEED_MORE_DATA_EVT) {
isc_throw(JSONFeedError,
"JSONFeed requires new data to progress, but no data"
" have been provided. The transaction is aborted to avoid"
" a deadlock.");
} else {
// Try to pop next character from the buffer.
const bool data_exist = popNextFromBuffer(c);
if (!data_exist) {
// There is no more data so it is really not possible that we're
// at MORE_DATA_PROVIDED_EVT.
if (ev == MORE_DATA_PROVIDED_EVT) {
isc_throw(JSONFeedError,
"JSONFeed state indicates that new data have been"
" provided to be parsed, but the transaction buffer"
" contains no new data.");
} else {
// If there is no more data we should set NEED_MORE_DATA_EVT
// event to indicate that new data should be provided.
transition(getCurrState(), NEED_MORE_DATA_EVT);
}
}
}
return (c);
}
void
JSONFeed::invalidEventError(const std::string& handler_name,
const unsigned int event) {
isc_throw(JSONFeedError, handler_name << ": "
<< " invalid event " << getEventLabel(static_cast<int>(event)));
}
void
JSONFeed::receiveStartHandler() {
char c = getNextFromBuffer();
if (getNextEvent() != NEED_MORE_DATA_EVT) {
switch(getNextEvent()) {
case START_EVT:
switch (c) {
case '\t':
case '\n':
case '\r':
case ' ':
transition(WHITESPACE_BEFORE_JSON_ST, DATA_READ_OK_EVT);
return;
case '{':
case '[':
output_.push_back(c);
++open_scopes_;
transition(INNER_JSON_ST, DATA_READ_OK_EVT);
return;
default:
feedFailure("invalid first character " + std::string(1, c));
}
default:
invalidEventError("receiveStartHandler", getNextEvent());
}
}
}
void
JSONFeed::whiteSpaceBeforeJSONHandler() {
char c = getNextFromBuffer();
if (getNextEvent() != NEED_MORE_DATA_EVT) {
switch (c) {
case '\t':
case '\n':
case '\r':
case ' ':
transition(getCurrState(), DATA_READ_OK_EVT);
break;
case '{':
case '[':
output_.push_back(c);
++open_scopes_;
transition(INNER_JSON_ST, DATA_READ_OK_EVT);
break;
default:
feedFailure("invalid character " + std::string(1, c));
}
}
}
void
JSONFeed::innerJSONHandler() {
char c = getNextFromBuffer();
if (getNextEvent() != NEED_MORE_DATA_EVT) {
output_.push_back(c);
switch(c) {
case '{':
case '[':
transition(getCurrState(), DATA_READ_OK_EVT);
++open_scopes_;
break;
case '}':
case ']':
if (--open_scopes_ == 0) {
transition(JSON_END_ST, FEED_OK_EVT);
} else {
transition(getCurrState(), DATA_READ_OK_EVT);
}
break;
default:
transition(getCurrState(), DATA_READ_OK_EVT);
}
}
}
void
JSONFeed::endJSONHandler() {
switch (getNextEvent()) {
case FEED_OK_EVT:
transition(END_ST, END_EVT);
break;
case FEED_FAILED_EVT:
abortModel("reading into JSON feed failed");
break;
default:
invalidEventError("endJSONHandler", getNextEvent());
}
}
} // end of namespace config
} // end of namespace isc
// Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
#ifndef JSON_FEED_H
#define JSON_FEED_H
#include <exceptions/exceptions.h>
#include <util/state_model.h>
#include <list>
#include <stdint.h>
#include <string>
namespace isc {
namespace config {
/// @brief A generic exception thrown upon an error in the @ref JSONFeed.
class JSONFeedError : public Exception {
public:
JSONFeedError(const char* file, size_t line, const char* what) :
isc::Exception(file, line, what) { };
};
/// @brief State model for asynchronous read of data in JSON format.
///
/// Kea control channel uses stream sockets for forwarding commands received
/// by the Kea Control Agent to respective Kea services. The responses may
/// contain large amounts of data (e.g. lease queries may return thousands
/// of leases). Such responses rarely fit into a single data buffer and
/// require multiple calls to receive/read or asynchronous receive/read.
///
/// A receiver performing multiple reads from a socket must be able to
/// locate the boundaries of the command within the data stream. The
/// @ref JSONFeed state model solves this problem.
///
/// When the partial data is read from the stream socket it should be provided
/// to the @ref JSONFeed using @ref JSONFeed::postBuffer and then the
/// @ref JSONFeed::poll should be called to start processing the received
/// data. The actual JSON structure can be preceded by whitespaces. When first
/// occurrence of one of the '{' or '[' characters is found in the stream it is
/// considered a beginning of the JSON structure. The model includes an internal
/// counter of new '{' and '[' occurrences. The counter increases one of these
/// characters is found. When any of the '}' or ']' is found, the counter
/// is decreased. When the counter is decreased to 0 it indicates that the
/// entire JSON structure has been received and processed.
///
/// Note that this mechanism doesn't check if the JSON structure is well
/// formed. It merely detects the end of the JSON structure if this structure
/// is well formed. The structure is validated when @ref JSONFeed::toElement
/// is called to retrieve the data structures encapsulated with
/// @ref isc::data::Element objects.
class JSONFeed : public util::StateModel {
public:
/// @name States supported by the @ref JSONFeed
///
//@{
/// @brief State indicating a beginning of a feed.
static const int RECEIVE_START_ST = SM_DERIVED_STATE_MIN + 1;
/// @brief Skipping whitespaces before actual JSON.
static const int WHITESPACE_BEFORE_JSON_ST = SM_DERIVED_STATE_MIN + 2;
/// @brief Found first opening brace or square bracket.
static const int JSON_START_ST = SM_DERIVED_STATE_MIN + 3;
/// @brief Parsing JSON.
static const int INNER_JSON_ST = SM_DERIVED_STATE_MIN + 4;
/// @brief Found last closing brace or square bracket.
static const int JSON_END_ST = SM_DERIVED_STATE_MIN + 5;
/// @brief Found opening and closing brace or square bracket.
///
/// This doesn't however indicate that the JSON is well formed. It
/// only means that matching closing brace or square bracket was
/// found.
static const int FEED_OK_ST = SM_DERIVED_STATE_MIN + 100;
/// @brief Invalid syntax detected.
///
/// For example, non matching braces or invalid characters found.
static const int FEED_FAILED_ST = SM_DERIVED_STATE_MIN + 101;
//@}
/// @name Events used during data processing.
///
//@{
/// @brief Chunk of data successfully read and parsed.
static const int DATA_READ_OK_EVT = SM_DERIVED_EVENT_MIN + 1;
/// @brief Unable to proceed with parsing until new data is provided.
static const int NEED_MORE_DATA_EVT = SM_DERIVED_EVENT_MIN + 2;
/// @brief New data provided and parsing should continue.
static const int MORE_DATA_PROVIDED_EVT = SM_DERIVED_EVENT_MIN + 3;
/// @brief Found opening brace and the matching closing brace.
static const int FEED_OK_EVT = SM_DERIVED_EVENT_MIN + 100;
/// @brief Invalid syntax detected.
static const int FEED_FAILED_EVT = SM_DERIVED_EVENT_MIN + 101;
//@}
/// @brief Constructor.
JSONFeed();
/// @brief Initializes state model.
///
/// Initializes events and states. It sets the model to @c RECEIVE_START_ST
/// and the next event to @c START_EVT.
void initModel();
/// @brief Runs the model as long as data is available.
///
/// It processes the input data character by character until it reaches the
/// end of the input buffer, in which case it returns. The next event is set
/// to @c NEED_MORE_DATA_EVT to indicate the need for providing additional
/// data using @ref JSONFeed::postBuffer. This function also returns when
/// the end of the JSON structure has been detected or when an error has
/// occurred.
void poll();
/// @brief Checks if the model needs additional data to continue.
///
/// The caller can use this method to check if the model expects additional
/// data to be provided to finish processing input data.
///
/// @return true if more data is needed, false otherwise.
bool needData() const;
/// @brief Checks if the data have been successfully processed.
bool feedOk() const;
/// @brief Returns error string when data processing has failed.
std::string getErrorMessage() const;
/// @brief Returns processed data as a structure of @ref isc::data::Element
/// objects.
data::ElementPtr toElement() const;
/// @brief Receives additional data read from a data stream.
///
/// A caller invokes this method to pass additional chunk of data received
/// from the stream.
///
/// @param buf Pointer to a buffer holding additional input data.
/// @param buf_size Size of the data in the input buffer.
void postBuffer(const void* buf, const size_t buf_size);
private:
/// @brief Make @ref runModel private to make sure that the caller uses
/// @ref poll method instead.
using StateModel::runModel;
/// @brief Define events used by the feed.
virtual void defineEvents();
/// @brief Verifies events used by the feed.
virtual void verifyEvents();
/// @brief Defines states of the feed.
virtual void defineStates();
/// @brief Transition to failure state.
///
/// This method transitions the model to @ref FEED_FAILED_ST and
/// sets next event to FEED_FAILED_EVT.
///
/// @param error_msg Error message explaining the failure.
void feedFailure(const std::string& error_msg);
/// @brief A method called when state model fails.
///
/// @param explanation Error message explaining the reason for failure.
virtual void onModelFailure(const std::string& explanation);
/// @brief Retrieves next byte of data from the buffer.
///
/// During normal operation, when there is no more data in the buffer,
/// the NEED_MORE_DATA_EVT is set as next event to signal the need for
/// calling @ref JSONFeed::postBuffer.
///
/// @throw HttpRequestParserError If current event is already set to
/// NEED_MORE_DATA_EVT or MORE_DATA_PROVIDED_EVT. In the former case, it
/// indicates that the caller failed to provide new data using
/// @ref JSONFeed::postBuffer. The latter case is highly unlikely
/// as it indicates that no new data were provided but the state of the
/// parser was changed from NEED_MORE_DATA_EVT or the data were provided
/// but the data buffer is empty. In both cases, it is a programming
/// error.
char getNextFromBuffer();
/// @brief This method is called when invalid event occurred in a particular
/// state.
///
/// This method simply throws @ref JSONFeedError informing about invalid
/// event occurring for the particular state. The error message includes
/// the name of the handler in which the exception has been thrown.
/// It also includes the event which caused the exception.
///
/// @param handler_name Name of the handler in which the exception is
/// thrown.
/// @param event An event which caused the exception.
///
/// @throw JSONFeedError.
void invalidEventError(const std::string& handler_name,
const unsigned int event);
/// @brief Tries to read next byte from buffer.
///
/// @param [out] next A reference to the variable where read data should be
/// stored.
///
/// @return true if character was successfully read, false otherwise.
bool popNextFromBuffer(char& next);
/// @name State handlers.
///
//@{
/// @brief Handler for RECEIVE_START_ST.
void receiveStartHandler();
/// @brief Handler for WHITESPACE_BEFORE_JSON_ST.
void whiteSpaceBeforeJSONHandler();
/// @brief Handle for the FIRST_BRACE_ST.
void innerJSONHandler();
/// @brief Handler for the JSON_END_ST.
void endJSONHandler();
//@}
/// @brief Internal buffer from which the feed reads data.
std::list<char> buffer_;
/// @brief Error message set by @ref onModelFailure.
std::string error_message_;
/// @brief A counter increased when '{' or '[' is found and decreased when
/// '}' or ']' is found in the stream.
uint64_t open_scopes_;
/// @brief Holds processed data.
std::string output_;
};
} // end of namespace config
} // end of namespace isc
#endif // JSON_FEED_H
......@@ -15,7 +15,9 @@ TESTS =
if HAVE_GTEST
TESTS += run_unittests
run_unittests_SOURCES = command_interpreter_unittests.cc data_unittests.cc
run_unittests_SOURCES += data_file_unittests.cc run_unittests.cc
run_unittests_SOURCES += data_file_unittests.cc
run_unittests_SOURCES += json_feed_unittests.cc
run_unittests_SOURCES += run_unittests.cc
run_unittests_SOURCES += simple_parser_unittest.cc
run_unittests_CPPFLAGS = $(AM_CPPFLAGS) $(GTEST_INCLUDES)
run_unittests_LDFLAGS = $(AM_LDFLAGS) $(GTEST_LDFLAGS)
......
// Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
#include <config.h>
#include <cc/data.h>
#include <cc/json_feed.h>
#include <gtest/gtest.h>
#include <sstream>
#include <string>
using namespace isc::config;
using namespace isc::data;
namespace {
/// @brief Test fixture class for @ref JSONFeed class.
class JSONFeedTest : public ::testing::Test {
public:
/// @brief Constructor.
///
/// Initializes @ref json_map_ and @ref json_list_ which hold reference
/// JSON structures.
JSONFeedTest()
: json_map_(), json_list_() {
ElementPtr m = Element::fromJSON(createJSON());
ElementPtr l = Element::createList();
l->add(m);
json_map_ = m;
json_list_ = l;
}
/// @brief Creates a JSON map holding 20 elements.
///
/// Each map value is a list of 20 elements.
std::string createJSON() const {
// Create a list of 20 elements.
ElementPtr list_element = Element::createList();
for (unsigned i = 0; i < 20; ++i) {
std::ostringstream s;
s << "list_element" << i;
list_element->add(Element::create(s.str()));
}
// Create a map of 20 elements. Each map element holds a list
// of 20 elements.
ElementPtr map_element = Element::createMap();
for (unsigned i = 0; i < 20; ++i) {
std::ostringstream s;
s << "map_element" << i;
map_element->set(s.str(), list_element);
}
return (prettyPrint(map_element));
}