[master] Merge branch 'trac5451'

src/bin/agent/ca_response_creator.cc

-// Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2017-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -10,6 +10,7 @@
 #include <http/post_request_json.h>
 #include <http/response_json.h>
 #include <boost/pointer_cast.hpp>
+#include <iostream>
 
 using namespace isc::data;
 using namespace isc::http;
@@ -26,6 +27,15 @@ HttpResponsePtr
 CtrlAgentResponseCreator::
 createStockHttpResponse(const ConstHttpRequestPtr& request,
                         const HttpStatusCode& status_code) const {
+    HttpResponsePtr response = createStockHttpResponseInternal(request, status_code);
+    response->finalize();
+    return (response);
+}
+
+HttpResponsePtr
+CtrlAgentResponseCreator::
+createStockHttpResponseInternal(const ConstHttpRequestPtr& request,
+                                const HttpStatusCode& status_code) const {
     // The request hasn't been finalized so the request object
     // doesn't contain any information about the HTTP version number
     // used. But, the context should have this data (assuming the
@@ -73,8 +83,9 @@ createDynamicHttpResponse(const ConstHttpRequestPtr& request) {
     }
     // The response is ok, so let's create new HTTP response with the status OK.
     HttpResponseJsonPtr http_response = boost::dynamic_pointer_cast<
-        HttpResponseJson>(createStockHttpResponse(request, HttpStatusCode::OK));
+        HttpResponseJson>(createStockHttpResponseInternal(request, HttpStatusCode::OK));
     http_response->setBodyAsJson(response);
+    http_response->finalize();
 
     return (http_response);
 }

src/bin/agent/ca_response_creator.h

-// Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2017-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -57,6 +57,20 @@ public:
 
 private:
 
+    /// @brief Creates unfinalized stock HTTP response.
+    ///
+    /// The unfinilized response is the response that can't be sent over the
+    /// wire until @c finalize() is called, which commits the contents of the
+    /// message body.
+    ///
+    /// @param request Pointer to an object representing HTTP request.
+    /// @param status_code Status code of the response.
+    /// @return Pointer to an @ref isc::http::HttpResponseJson object
+    /// representing stock HTTP response.
+   http::HttpResponsePtr
+   createStockHttpResponseInternal(const http::ConstHttpRequestPtr& request,
+                                   const http::HttpStatusCode& status_code) const;
+
     /// @brief Creates implementation specific HTTP response.
     ///
     /// @param request Pointer to an object representing HTTP request.

src/lib/asiolink/tcp_socket.h

-// Copyright (C) 2011-2017 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2011-2018 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -94,6 +94,48 @@ public:
         return (false);
     }
 
+    /// \brief Checks if the connection is usable.
+    ///
+    /// The connection is usable if the socket is open and the peer has not
+    /// closed its connection.
+    ///
+    /// \return true if the connection is usable.
+    bool isUsable() const {
+        // If the socket is open it doesn't mean that it is still usable. The connection
+        // could have been closed on the other end. We have to check if we can still
+        // use this socket.
+        if (socket_.is_open()) {
+            // Remember the current non blocking setting.
+            const bool non_blocking_orig = socket_.non_blocking();
+            // Set the socket to non blocking mode. We're going to test if the socket
+            // returns would_block status on the attempt to read from it.
+            socket_.non_blocking(true);
+
+            boost::system::error_code ec;
+            char data[2];
+
+            // Use receive with message peek flag to avoid removing the data awaiting
+            // to be read.
+            socket_.receive(boost::asio::buffer(data, sizeof(data)),
+                            boost::asio::socket_base::message_peek,
+                            ec);
+
+            // Revert the original non_blocking flag on the socket.
+            socket_.non_blocking(non_blocking_orig);
+
+            // If the connection is alive we'd typically get would_block status code.
+            // If there are any data that haven't been read we may also get success
+            // status. We're guessing that try_again may also be returned by some
+            // implementations in some situations. Any other error code indicates a
+            // problem with the connection so we assume that the connection has been
+            // closed.
+            return (!ec || (ec.value() == boost::asio::error::try_again) ||
+                    (ec.value() == boost::asio::error::would_block));
+        }
+
+        return (false);
+    }
+
     /// \brief Open Socket
     ///
     /// Opens the TCP socket.  This is an asynchronous operation, completion of
@@ -231,7 +273,11 @@ TCPSocket<C>::~TCPSocket()
 
 template <typename C> void
 TCPSocket<C>::open(const IOEndpoint* endpoint, C& callback) {
-
+    // If socket is open on this end but has been closed by the peer,
+    // we need to reconnect.
+    if (socket_.is_open() && !isUsable()) {
+        close();
+    }
     // Ignore opens on already-open socket.  Don't throw a failure because
     // of uncertainties as to what precedes whan when using asynchronous I/O.
     // At also allows us a treat a passed-in socket as a self-managed socket.

src/lib/http/Makefile.am

@@ -22,13 +22,16 @@ EXTRA_DIST = http_messages.mes
 CLEANFILES = *.gcno *.gcda http_messages.h http_messages.cc s-messages
 
 lib_LTLIBRARIES = libkea-http.la
-libkea_http_la_SOURCES  = connection.cc connection.h
+libkea_http_la_SOURCES  = client.cc client.h
+libkea_http_la_SOURCES += connection.cc connection.h
 libkea_http_la_SOURCES += connection_pool.cc connection_pool.h
 libkea_http_la_SOURCES += date_time.cc date_time.h
 libkea_http_la_SOURCES += http_log.cc http_log.h
 libkea_http_la_SOURCES += header_context.h
 libkea_http_la_SOURCES += http_acceptor.h
 libkea_http_la_SOURCES += http_header.cc http_header.h
+libkea_http_la_SOURCES += http_message.cc http_message.h
+libkea_http_la_SOURCES += http_message_parser_base.cc http_message_parser_base.h
 libkea_http_la_SOURCES += http_types.h
 libkea_http_la_SOURCES += listener.cc listener.h
 libkea_http_la_SOURCES += post_request.cc post_request.h
@@ -37,9 +40,12 @@ libkea_http_la_SOURCES += request.cc request.h
 libkea_http_la_SOURCES += request_context.h
 libkea_http_la_SOURCES += request_parser.cc request_parser.h
 libkea_http_la_SOURCES += response.cc response.h
+libkea_http_la_SOURCES += response_parser.cc response_parser.h
+libkea_http_la_SOURCES += response_context.h
 libkea_http_la_SOURCES += response_creator.cc response_creator.h
 libkea_http_la_SOURCES += response_creator_factory.h
 libkea_http_la_SOURCES += response_json.cc response_json.h
+libkea_http_la_SOURCES += url.cc url.h
 
 nodist_libkea_http_la_SOURCES = http_messages.cc http_messages.h
 

src/lib/http/client.h

+// Copyright (C) 2018 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef HTTP_CLIENT_H
+#define HTTP_CLIENT_H
+
+#include <asiolink/io_service.h>
+#include <exceptions/exceptions.h>
+#include <http/url.h>
+#include <http/request.h>
+#include <http/response.h>
+#include <boost/shared_ptr.hpp>
+#include <functional>
+#include <string>
+
+namespace isc {
+namespace http {
+
+/// @brief A generic error raised by the @ref HttpClient class.
+class HttpClientError : public Exception {
+public:
+    HttpClientError(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) { };
+};
+
+class HttpClientImpl;
+
+/// @brief HTTP client class.
+///
+/// This class implements an asynchronous HTTP client. The caller can schedule
+/// transmission of the HTTP request using @ref HttpClient::asyncSendRequest
+/// method. The caller specifies target URL for each request. The caller also
+/// specifies a pointer to the @ref HttpRequest or derived class, holding a
+/// request that should be transmitted to the destination. Such request must
+/// be finalized, i.e. @ref HttpRequest::finalize method must be called prior
+/// to sending it. The caller must also provide a pointer to the
+/// @ref HttpResponse object or an object derived from it. The type of the
+/// response object must match the expected content type to be returned in the
+/// server's response. The last argument specified in this call is the pointer
+/// to the callback function, which should be launched when the response is
+/// received, an error occurs or when a timeout in the transmission is
+/// signalled.
+///
+/// The HTTP client supports multiple simultaneous and persistent connections
+/// with different destinations. The client determines if the connection is
+/// persistent by looking into the Connection header and HTTP version of the
+/// request. If the connection should be persistent the client doesn't
+/// close the connection after sending a request and receiving a response from
+/// the server. If the client is provided with the request to be sent to the
+/// particular destination, but there is an ongoing communication with this
+/// destination, e.g. as a result of sending previous request, the new
+/// request is queued in the FIFO queue. When the previous request completes,
+/// the next request in the queue for the particular URL will be initiated.
+///
+/// The client tests the persistent connection for usability before sending
+/// a request by trying to read from the socket (with message peeking). If
+/// the socket is usable the client uses it to transmit the request.
+///
+/// All errors are reported to the caller via the callback function supplied
+/// to the @ref HttpClient::asyncSendRequest. The IO errors are communicated
+/// via the @c boost::system::error code value. The response parsing errors
+/// are returned via the 3rd parameter of the callback.
+class HttpClient {
+public:
+
+    /// @brief HTTP request/response timeout value.
+    struct RequestTimeout {
+        /// @brief Constructor.
+        ///
+        /// @param value Reuqest/response timeout value in milliseconds.
+        explicit RequestTimeout(long value)
+            : value_(value) {
+        }
+        long value_; ///< Timeout value specified.
+    };
+
+    /// @brief Callback type used in call to @ref HttpClient::asyncSendRequest.
+    typedef std::function<void(const boost::system::error_code&,
+                               const HttpResponsePtr&,
+                               const std::string&)> RequestHandler;
+
+    /// @brief Constructor.
+    ///
+    /// @param io_service IO service to be used by the HTTP client.
+    explicit HttpClient(asiolink::IOService& io_service);
+
+    /// @brief Queues new asynchronous HTTP request.
+    ///
+    /// The client creates one connection for the specified URL. If the
+    /// connection with the particular destination already exists, it will be
+    /// re-used for the new transaction scheduled with this call. If another
+    /// transaction is still in progress, the new transaction is queued. The
+    /// queued transactions are started in the FIFO order one after another. If
+    /// the connection is idle or the connection doesn't exist, the new
+    /// transaction is started immediatelly.
+    ///
+    /// The existing connection is tested before it is used for the new
+    /// transaction by attempting to read (with message peeking) from the open
+    /// TCP socket. If the read attempt is successful, the client will transmit
+    /// the HTTP request to the server using this connection. It is possible
+    /// that the server closes the connection between the connection test and
+    /// sending the request. In such case, an error will be returned and the
+    /// caller will need to try re-sending the request.
+    ///
+    /// If the connection test fails, the client will close the socket and
+    /// reconnect to the server prior to sending the request.
+    ///
+    /// Pointers to the request and response objects are provided as arguments
+    /// of this method. These pointers should have appropriate types derived
+    /// from the @ref HttpRequest and @ref HttpResponse classes. For example,
+    /// if the request has content type "application/json", a pointer to the
+    /// @ref HttpResponseJson should be passed. In this case, the response type
+    /// should be @ref HttpResponseJson. These types are used to validate both
+    /// the request provided by the caller and the response received from the
+    /// server.
+    ///
+    /// The callback function provided by the caller is invoked when the
+    /// transaction terminates, i.e. when the server has responded or when an
+    /// error occured. The callback is expected to be exception safe, but the
+    /// client internally guards against exceptions thrown by the callback.
+    ///
+    /// The first argument of the callback indicates an IO error during
+    /// communication with the server. If the communication is successful the
+    /// error code of 0 is returned. However, in this case it is still possible
+    /// that the transaction is unsuccessful due to HTTP response parsing error,
+    /// e.g. invalid content type, malformed response etc. Such errors are
+    /// indicated via third argument.
+    ///
+    /// If message parsing was successful the second argument of the callback
+    /// contains a pointer to the parsed response (the same pointer as provided
+    ///by the caller as the argument). If parsing was unsuccessful, the null
+    /// pointer is returned.
+    ///
+    /// The default timeout for the transaction is set to 10 seconds
+    /// (10 000 ms). If the timeout occurs, the callback is invoked with the
+    //// error code of @c boost::asio::error::timed_out.
+    ///
+    /// @param url URL where the request should be send.
+    /// @param request Pointer to the object holding a request.
+    /// @param response Pointer to the object where response should be stored.
+    /// @param callback Pointer to the user callback function.
+    /// @param request_timeout Timeout for the transaction in milliseconds.
+    ///
+    /// @throw HttpClientError If invalid arguments were provided.
+    void asyncSendRequest(const Url& url,
+                          const HttpRequestPtr& request,
+                          const HttpResponsePtr& response,
+                          const RequestHandler& callback,
+                          const RequestTimeout& request_timeout =
+                          RequestTimeout(10000));
+
+    /// @brief Closes all connections.
+    void stop();
+
+private:
+
+    /// @brief Pointer to the HTTP client implementation.
+    boost::shared_ptr<HttpClientImpl> impl_;
+};
+
+} // end of namespace isc::http
+} // end of namespace isc
+
+#endif

src/lib/http/header_context.h

-// Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC")
+// Copyright (C) 2016-2017 Internet Systems Consortium, Inc. ("ISC")
 //
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -7,14 +7,40 @@
 #ifndef HTTP_HEADER_CONTEXT_H
 #define HTTP_HEADER_CONTEXT_H
 
+#include <boost/lexical_cast.hpp>
+#include <cstdint>
 #include <string>
 
 namespace isc {
 namespace http {
 
+/// @brief HTTP header context.
 struct HttpHeaderContext {
     std::string name_;
     std::string value_;
+
+    /// @brief Constructor.
+    ///
+    /// Sets header name and value to empty strings.
+    HttpHeaderContext()
+        : name_(), value_() {
+    }
+
+    /// @brief Constructor.
+    ///
+    /// @param name Header name.
+    /// @param value Header value.
+    HttpHeaderContext(const std::string& name, const std::string& value)
+        : name_(name), value_(value) {
+    }
+
+    /// @brief Constructor.
+    ///
+    /// @param name Header name.
+    /// @param value Numeric value for the header.
+    HttpHeaderContext(const std::string& name, const int64_t value)
+        : name_(name), value_(boost::lexical_cast<std::string>(value)) {
+    }
 };
 
 } // namespace http

src/lib/http/http_message.cc

+// Copyright (C) 2017-2018 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <http/http_message.h>
+
+namespace isc {
+namespace http {
+
+HttpMessage::HttpMessage(const HttpMessage::Direction& direction)
+    : direction_(direction), required_versions_(),
+      http_version_(HttpVersion::HTTP_10()), required_headers_(),
+      created_(false), finalized_(false), headers_() {
+}
+
+HttpMessage::~HttpMessage() {
+}
+
+void
+HttpMessage::requireHttpVersion(const HttpVersion& version) {
+    required_versions_.insert(version);
+}
+
+void
+HttpMessage::requireHeader(const std::string& header_name) {
+    // Empty value denotes that the header is required but no specific
+    // value is expected.
+    HttpHeaderPtr hdr(new HttpHeader(header_name));
+    required_headers_[hdr->getLowerCaseName()] = hdr;
+}
+
+void
+HttpMessage::requireHeaderValue(const std::string& header_name,
+                                const std::string& header_value) {
+    HttpHeaderPtr hdr(new HttpHeader(header_name, header_value));
+    required_headers_[hdr->getLowerCaseName()] = hdr;
+}
+
+bool
+HttpMessage::requiresBody() const {
+    // If Content-Length is required the body must exist too. There may
+    // be probably some cases when Content-Length is not provided but
+    // the body is provided. But, probably not in our use cases.
+    // Use lower case header name because this is how it is indexed in
+    // the storage.
+    return (required_headers_.find("content-length") != required_headers_.end());
+}
+
+HttpVersion
+HttpMessage::getHttpVersion() const {
+    checkCreated();
+    return (http_version_);
+}
+
+HttpHeaderPtr
+HttpMessage::getHeader(const std::string& header_name) const {
+    checkCreated();
+
+    HttpHeader hdr(header_name);
+    auto header_it = headers_.find(hdr.getLowerCaseName());
+    if (header_it != headers_.end()) {
+        return (header_it->second);
+    }
+
+    isc_throw(HttpMessageNonExistingHeader, header_name << " HTTP header"
+              " not found in the request");
+}
+
+std::string
+HttpMessage::getHeaderValue(const std::string& header_name) const {
+    return (getHeader(header_name)->getValue());
+}
+
+uint64_t
+HttpMessage::getHeaderValueAsUint64(const std::string& header_name) const {
+    try {
+        return (getHeader(header_name)->getUint64Value());
+
+    } catch (const std::exception& ex) {
+        // The specified header does exist, but the value is not a number.
+        isc_throw(HttpMessageError, ex.what());
+    }
+}
+
+void
+HttpMessage::checkCreated() const {
+    if (!created_) {
+        isc_throw(HttpMessageError, "unable to retrieve values of HTTP"
+                  " message because the HttpMessage::create() must be"
+                  " called first. This is a programmatic error");
+    }
+}
+
+void
+HttpMessage::checkFinalized() const {
+    if (!finalized_) {
+        isc_throw(HttpMessageError, "unable to retrieve body of HTTP"
+                  " message because the HttpMessage::finalize() must be"
+                  " called first. This is a programmatic error");
+    }
+}
+
+} // end of namespace isc::http
+} // end of namespace isc

src/lib/http/http_message.h

+// Copyright (C) 2017-2018 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef HTTP_MESSAGE_H
+#define HTTP_MESSAGE_H
+
+#include <exceptions/exceptions.h>
+#include <http/http_header.h>
+#include <http/http_types.h>
+#include <map>
+#include <set>
+#include <cstdint>
+#include <string>
+
+namespace isc {
+namespace http {
+
+/// @brief Generic exception thrown by @ref HttpMessage class.
+class HttpMessageError : public Exception {
+public:
+    HttpMessageError(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) { };
+};
+
+/// @brief Exception thrown when attempt is made to retrieve a
+/// non-existing header.
+class HttpMessageNonExistingHeader : public HttpMessageError {
+public:
+    HttpMessageNonExistingHeader(const char* file, size_t line,
+                                 const char* what) :
+        HttpMessageError(file, line, what) { };
+};
+
+
+/// @brief Base class for @ref HttpRequest and @ref HttpResponse.
+///
+/// This abstract class provides a common functionality for the HTTP
+/// requests and responses. Each such message can be marked as outbound
+/// or inbound. An HTTP inbound request is the one received by the server
+/// and HTTP inbound response is the response received by the client.
+/// Conversely, an HTTP outbound request is the request created by the
+/// client and HTTP outbound response is the response created by the
+/// server. There are differences in how the inbound and outbound
+/// messages are created. The inbound messages are received over the
+/// TCP sockets and parsed by the parsers. The parsed information is
+/// stored in a context, i.e. structure holding raw information and
+/// associated with the given @c HttpMessage instance. Once the message
+/// is parsed and all required information is stored in the context,
+/// the @c create method is called to validate and fetch information
+/// from the context into the message. The @c finalize method is called
+/// to commit the HTTP message body into the message.
+///
+/// The outbound message is created locally from the known data, e.g.
+/// HTTP version number, URI, method etc. The headers can be then
+/// appended to the message via the context. In order to use this message
+/// the @c finalize method must be called to commit this information.
+/// Them, @c toString method can be called to generate the message in
+/// the textual form, which can be transferred via TCP socket.
+class HttpMessage {
+public:
+
+    /// @brief Specifies the direction of the HTTP message.
+    enum Direction {
+        INBOUND,
+        OUTBOUND
+    };
+
+    /// @brief Constructor.
+    ///
+    /// @param direction Direction of the message (inbound or outbound).
+    explicit HttpMessage(const Direction& direction);
+
+    /// @brief Destructor.
+    virtual ~HttpMessage();
+
+    /// @brief Returns HTTP message direction.
+    Direction getDirection() const {
+        return (direction_);
+    }
+
+    /// @brief Sets direction for the HTTP message.
+    ///
+    /// This is mostly useful in unit testing.
+    ///
+    /// @param direction New direction of the HTTP message.
+    void setDirection(const Direction& direction) {
+        direction_ = direction;
+    }
+
+    /// @brief Specifies HTTP version allowed.
+    ///
+    /// Allowed HTTP versions must be specified prior to calling @ref create
+    /// method. If no version is specified, all versions are allowed.
+    ///
+    /// @param version Version number allowed for the request.
+    void requireHttpVersion(const HttpVersion& version);
+
+    /// @brief Specifies a required HTTP header for the HTTP message.
+    ///
+    /// Required headers must be specified prior to calling @ref create method.
+    /// The specified header must exist in the received HTTP request. This puts
+    /// no requirement on the header value.
+    ///
+    /// @param header_name Required header name.
+    void requireHeader(const std::string& header_name);
+
+    /// @brief Specifies a required value of a header in the message.
+    ///
+    /// Required header values must be specified prior to calling @ref create
+    /// method. The specified header must exist and its value must be equal to
+    /// the value specified as second parameter.
+    ///
+    /// @param header_name HTTP header name.
+    /// @param header_value HTTP header value.
+    void requireHeaderValue(const std::string& header_name,
+                            const std::string& header_value);
+
+    /// @brief Checks if the body is required for the HTTP message.
+    ///
+    /// Current implementation simply checks if the "Content-Length" header
+    /// is required.
+    ///
+    /// @return true if the body is required, false otherwise.
+    bool requiresBody() const;
+
+    /// @brief Reads parsed message from the context, validates the message and
+    /// stores parsed information.
+    ///
+    /// This method must be called before retrieving parsed data using accessors.
+    /// This method doesn't parse the HTTP request body.
+    virtual void create() = 0;
+
+    /// @brief Complete parsing HTTP message or creating an HTTP outbound message.
+    ///
+    /// This method is used in two situations: when a message has been received
+    /// into a context and may be fully parsed (including the body) or when the