[5017] Dhcp4 bison parser implemented.

doc/examples/kea4/classify.json

@@ -55,7 +55,7 @@
   {
       "name": "VoIP",
       "test": "substring(option[60].hex,0,6) == 'Aastra'"
-  },
+  }
 
   ],
 

doc/examples/kea4/dhcpv4-over-dhcpv6.json

@@ -19,13 +19,13 @@
 
   "subnet4": [
    {    "subnet": "10.10.10.0/24",
-        # Don't forget the "4o6-" before "interface" here!
+# Don't forget the "4o6-" before "interface" here!
         "4o6-interface": "eno33554984",
         "4o6-subnet": "2001:db8:1:1::/64",
         "pools": [ { "pool": "10.10.10.100 - 10.10.10.199" } ] }
   ],
 
-  # This enables DHCPv4-over-DHCPv6 support
+# This enables DHCPv4-over-DHCPv6 support
   "dhcp4o6-port": 6767
 
 },

src/bin/dhcp4/Makefile.am

@@ -63,6 +63,10 @@ libdhcp4_la_SOURCES += dhcp4_log.cc dhcp4_log.h
 libdhcp4_la_SOURCES += dhcp4_srv.cc dhcp4_srv.h
 libdhcp4_la_SOURCES += dhcp4to6_ipc.cc dhcp4to6_ipc.h
 
+libdhcp4_la_SOURCES += dhcp4_lexer.ll location.hh position.hh stack.hh
+libdhcp4_la_SOURCES += dhcp4_parser.cc dhcp4_parser.h
+libdhcp4_la_SOURCES += parser_context.cc parser_context.h
+
 libdhcp4_la_SOURCES += kea_controller.cc
 
 nodist_libdhcp4_la_SOURCES = dhcp4_messages.h dhcp4_messages.cc
@@ -104,3 +108,30 @@ endif
 
 kea_dhcp4dir = $(pkgdatadir)
 kea_dhcp4_DATA = dhcp4.spec
+
+if GENERATE_PARSER
+
+parser: dhcp4_lexer.cc location.hh position.hh stack.hh dhcp4_parser.cc dhcp4_parser.h
+	@echo "Flex/bison files regenerated"
+
+# --- Flex/Bison stuff below --------------------------------------------------
+# When debugging grammar issues, it's useful to add -v to bison parameters.
+# bison will generate parser.output file that explains the whole grammar.
+# It can be used to manually follow what's going on in the parser.
+# This is especially useful if yydebug_ is set to 1 as that variable
+# will cause parser to print out its internal state.
+# Call flex with -s to check that the default rule can be suppressed
+# Call bison with -W to get warnings like unmarked empty rules
+# Note C++11 deprecated register still used by flex < 2.6.0
+location.hh position.hh stack.hh dhcp4_parser.cc dhcp4_parser.h: dhcp4_parser.yy
+	$(YACC) --defines=dhcp4_parser.h --report=all --report-file=dhcp4_parser.report -o dhcp4_parser.cc dhcp4_parser.yy
+
+dhcp4_lexer.cc: dhcp4_lexer.ll
+	$(LEX) --prefix parser4_ -o dhcp4_lexer.cc dhcp4_lexer.ll
+
+else
+
+parser location.hh position.hh stack.hh dhcp4_parser.cc dhcp4_parser.h dhcp4_lexer.cc:
+	@echo Parser generation disabled. Configure with --enable-generate-parser to enable it.
+
+endif

src/bin/dhcp4/parser_context.cc

+// Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <dhcp4/parser_context.h>
+#include <dhcp4/dhcp4_parser.h>
+#include <exceptions/exceptions.h>
+#include <cc/data.h>
+#include <boost/lexical_cast.hpp>
+#include <fstream>
+#include <limits>
+
+namespace isc {
+namespace dhcp {
+
+Parser4Context::Parser4Context()
+  : trace_scanning_(false), trace_parsing_(false)
+{
+}
+
+Parser4Context::~Parser4Context()
+{
+}
+
+isc::data::ConstElementPtr
+Parser4Context::parseString(const std::string& str, ParserType parser_type)
+{
+    scanStringBegin(str, parser_type);
+    return (parseCommon());
+}
+
+isc::data::ConstElementPtr
+Parser4Context::parseFile(const std::string& filename, ParserType parser_type) {
+    FILE* f = fopen(filename.c_str(), "r");
+    if (!f) {
+        isc_throw(Dhcp4ParseError, "Unable to open file " << filename);
+    }
+    scanFileBegin(f, filename, parser_type);
+    return (parseCommon());
+}
+
+isc::data::ConstElementPtr
+Parser4Context::parseCommon() {
+    isc::dhcp::Dhcp4Parser parser(*this);
+    // Uncomment this to get detailed parser logs.
+    // trace_parsing_ = true;
+    parser.set_debug_level(trace_parsing_);
+    try {
+        int res = parser.parse();
+        if (res != 0) {
+            isc_throw(Dhcp4ParseError, "Parser abort");
+        }
+        scanEnd();
+    }
+    catch (...) {
+        scanEnd();
+        throw;
+    }
+    if (stack_.size() == 1) {
+        return (stack_[0]);
+    } else {
+        isc_throw(Dhcp4ParseError, "Expected exactly one terminal Element expected, found "
+                  << stack_.size());
+    }
+}
+
+
+void
+Parser4Context::error(const isc::dhcp::location& loc, const std::string& what)
+{
+    isc_throw(Dhcp4ParseError, loc << ": " << what);
+}
+
+void
+Parser4Context::error (const std::string& what)
+{
+    isc_throw(Dhcp4ParseError, what);
+}
+
+void
+Parser4Context::fatal (const std::string& what)
+{
+    isc_throw(Dhcp4ParseError, what);
+}
+
+isc::data::Element::Position
+Parser4Context::loc2pos(isc::dhcp::location& loc)
+{
+    const std::string& file = *loc.begin.filename;
+    const uint32_t line = loc.begin.line;
+    const uint32_t pos = loc.begin.column;
+    return (isc::data::Element::Position(file, line, pos));
+}
+
+void
+Parser4Context::enter(const ParserContext& ctx)
+{
+    cstack_.push_back(ctx_);
+    ctx_ = ctx;
+}
+
+void
+Parser4Context::leave()
+{
+#if 1
+    if (cstack_.empty()) {
+        fatal("unbalanced syntactic context");
+    }
+#endif
+    ctx_ = cstack_.back();
+    cstack_.pop_back();
+}
+
+const std::string
+Parser4Context::contextName()
+{
+    switch (ctx_) {
+    case NO_KEYWORD:
+        return ("__no keyword__");
+    case CONFIG:
+        return ("toplevel");
+    case DHCP4:
+        return ("Dhcp4");
+    case LOGGING:
+        return ("Logging");
+    case INTERFACES_CONFIG:
+        return ("interfaces-config");
+    case LEASE_DATABASE:
+        return ("lease-database");
+    case HOSTS_DATABASE:
+        return ("hosts-database");
+    case HOST_RESERVATION_IDENTIFIERS:
+        return ("host-reservation-identifiers");
+    case HOOKS_LIBRARIES:
+        return ("hooks-librairies");
+    case SUBNET4:
+        return ("subnet4");
+    case OPTION_DEF:
+        return ("option-def");
+    case OPTION_DATA:
+        return ("option-data");
+    case CLIENT_CLASSES:
+        return ("client-classes");
+    case SERVER_ID:
+        return ("server-id");
+    case CONTROL_SOCKET:
+        return ("control-socket");
+    case POOLS:
+        return ("pools");
+    case RESERVATIONS:
+        return ("reservations");
+    case RELAY:
+        return ("relay");
+    case CLIENT_CLASS:
+        return ("client-class");
+    case LOGGERS:
+        return ("loggers");
+    case OUTPUT_OPTIONS:
+        return ("output-options");
+    default:
+        return ("__unknown__");
+    }
+}
+
+};
+};

src/bin/dhcp4/parser_context.h

+// Copyright (C) 2015-2016 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef PARSER_CONTEXT_H
+#define PARSER_CONTEXT_H
+#include <string>
+#include <map>
+#include <vector>
+#include <dhcp4/dhcp4_parser.h>
+#include <dhcp4/parser_context_decl.h>
+#include <exceptions/exceptions.h>
+
+// Tell Flex the lexer's prototype ...
+#define YY_DECL isc::dhcp::Dhcp4Parser::symbol_type parser4_lex (Parser4Context& driver)
+
+// ... and declare it for the parser's sake.
+YY_DECL;
+
+namespace isc {
+namespace dhcp {
+
+/// @brief Evaluation error exception raised when trying to parse.
+///
+/// @todo: This probably should be common for Dhcp4 and Dhcp6.
+class Dhcp4ParseError : public isc::Exception {
+public:
+    Dhcp4ParseError(const char* file, size_t line, const char* what) :
+        isc::Exception(file, line, what) { };
+};
+
+
+/// @brief Evaluation context, an interface to the expression evaluation.
+class Parser4Context
+{
+public:
+
+    /// @brief Defines currently supported scopes
+    ///
+    /// Dhcp4Parser is able to parse several types of scope. Usually,
+    /// when it parses a config file, it expects the data to have a map
+    /// with Dhcp4 in it and all the parameters within that Dhcp4 map.
+    /// However, sometimes the parser is expected to parse only a subset
+    /// of that information. For example, it may be asked to parse
+    /// a structure that is host-reservation only, without the global
+    /// 'Dhcp4' or 'reservations' around it. In such case the parser
+    /// is being told to start parsing as PARSER_HOST_RESERVATION4.
+    typedef enum {
+        /// This parser will parse the content as generic JSON.
+        PARSER_JSON,
+
+        /// This parser will parse the content as Dhcp4 config wrapped in a map
+        /// (that's the regular config file)
+        PARSER_DHCP4,
+
+        /// This parser will parse the content of Dhcp4 (without outer { } and
+        /// without "Dhcp4"). It is mostly used in unit-tests as most of the
+        /// unit-tests do not define the outer map and Dhcp4 entity, just the
+        /// contents of it.
+        SUBPARSER_DHCP4,
+
+        /// This will parse the input as interfaces content.
+        PARSER_INTERFACES,
+
+        /// This will parse the input as Subnet6 content.
+        PARSER_SUBNET4,
+
+        /// This will parse the input as pool6 content.
+        PARSER_POOL4,
+
+        /// This will parse the input as host-reservation.
+        PARSER_HOST_RESERVATION,
+
+        /// This will parse the input as option definition.
+        PARSER_OPTION_DEF,
+
+        /// This will parse the input as option data.
+        PARSER_OPTION_DATA,
+
+        /// This will parse the input as hooks-library.
+        PARSER_HOOKS_LIBRARY
+    } ParserType;
+
+    /// @brief Default constructor.
+    Parser4Context();
+
+    /// @brief destructor
+    virtual ~Parser4Context();
+
+    /// @brief JSON elements being parsed.
+    std::vector<isc::data::ElementPtr> stack_;
+
+    /// @brief Method called before scanning starts on a string.
+    ///
+    /// @param str string to be parsed
+    /// @param parser_type specifies expected content
+    void scanStringBegin(const std::string& str, ParserType type);
+
+    /// @brief Method called before scanning starts on a file.
+    ///
+    /// @param f stdio FILE pointer
+    /// @param filename file to be parsed
+    /// @param parser_type specifies expected content
+    void scanFileBegin(FILE* f, const std::string& filename, ParserType type);
+
+    /// @brief Method called after the last tokens are scanned.
+    void scanEnd();
+
+    /// @brief Divert input to an include file.
+    ///
+    /// @param filename file to be included
+    void includeFile(const std::string& filename);
+
+    /// @brief Run the parser on the string specified.
+    ///
+    /// This method parses specified string. Depending on the value of
+    /// parser_type, parser may either check only that the input is valid
+    /// JSON, or may do more specific syntax checking. See @ref ParserType
+    /// for supported syntax checkers.
+    ///
+    /// @param str string to be parsed
+    /// @param parser_type specifies expected content (usually DHCP4 or generic JSON)
+    /// @return Element structure representing parsed text.
+    isc::data::ConstElementPtr parseString(const std::string& str,
+                                           ParserType parser_type);
+
+    /// @brief Run the parser on the file specified.
+    ///
+    /// This method parses specified file. Depending on the value of
+    /// parser_type, parser may either check only that the input is valid
+    /// JSON, or may do more specific syntax checking. See @ref ParserType
+    /// for supported syntax checkers.
+    ///
+    /// @param filename file to be parsed
+    /// @param parser_type specifies expected content (usually DHCP4 or generic JSON)
+    /// @return Element structure representing parsed text.
+    isc::data::ConstElementPtr parseFile(const std::string& filename,
+                                         ParserType parser_type);
+
+    /// @brief Error handler
+    ///
+    /// @param loc location within the parsed file when experienced a problem.
+    /// @param what string explaining the nature of the error.
+    /// @throw Dhcp4ParseError
+    void error(const isc::dhcp::location& loc, const std::string& what);
+
+    /// @brief Error handler
+    ///
+    /// This is a simplified error reporting tool for possible future
+    /// cases when the Dhcp4Parser is not able to handle the packet.
+    ///
+    /// @param what string explaining the nature of the error.
+    /// @throw Dhcp4ParseError
+    void error(const std::string& what);
+
+    /// @brief Fatal error handler
+    ///
+    /// This is for should not happen but fatal errors.
+    /// Used by YY_FATAL_ERROR macro so required to be static.
+    ///
+    /// @param what string explaining the nature of the error.
+    /// @throw Dhcp4ParseError
+    static void fatal(const std::string& what);
+
+    /// @brief Converts bison's position to one understandable by isc::data::Element
+    ///
+    /// Convert a bison location into an element position
+    /// (take the begin, the end is lost)
+    ///
+    /// @param loc location in bison format
+    /// @return Position in format accepted by Element
+    isc::data::Element::Position loc2pos(isc::dhcp::location& loc);
+
+    /// @brief Defines syntactic contexts for lexical tie-ins
+    typedef enum {
+        ///< This one is used in pure JSON mode.
+        NO_KEYWORD,
+
+        ///< Used while parsing top level (that contains Dhcp4, Logging and others)
+        CONFIG,
+
+        ///< Used while parsing content of Dhcp4.
+        DHCP4,
+
+        // not yet DHCP6,
+        // not yet DHCP_DDNS,
+
+        ///< Used while parsing content of Logging
+        LOGGING,
+
+        /// Used while parsing Dhcp4/interfaces structures.
+        INTERFACES_CONFIG,
+
+        /// Used while parsing Dhcp4/lease-database structures.
+        LEASE_DATABASE,
+
+        /// Used while parsing Dhcp4/hosts-database structures.
+        HOSTS_DATABASE,
+
+        /// Used while parsing Dhcp4/host-reservation-identifiers.
+        HOST_RESERVATION_IDENTIFIERS,
+
+        /// Used while parsing Dhcp4/hooks-libraries.
+        HOOKS_LIBRARIES,
+
+        /// Used while parsing Dhcp4/Subnet6 structures.
+        SUBNET4,
+
+        /// Used while parsing Dhcp4/option-def structures.
+        OPTION_DEF,
+
+        /// Used while parsing Dhcp4/option-data, Dhcp4/subnet6/option-data
+        /// or anywhere option-data is present (client classes, host
+        /// reservations and possibly others).
+        OPTION_DATA,
+
+        /// Used while parsing Dhcp4/client-classes structures.
+        CLIENT_CLASSES,
+
+        /// Used while parsing Dhcp4/server-id structures.
+        SERVER_ID,
+
+        /// Used while parsing Dhcp4/control-socket structures.
+        CONTROL_SOCKET,
+
+        /// Used while parsing Dhcp4/subnet6/pools structures.
+        POOLS,
+
+        /// Used while parsing Dhcp4/reservations structures.
+        RESERVATIONS,
+
+        /// Used while parsing Dhcp4/subnet6/relay structures.
+        RELAY,
+
+        /// Used while parsing Dhcp4/client-classes structures.
+        CLIENT_CLASS,
+
+        /// Used while parsing Logging/loggers structures.
+        LOGGERS,
+
+        /// Used while parsing Logging/loggers/output_options structures.
+        OUTPUT_OPTIONS
+    } ParserContext;
+
+    /// @brief File name
+    std::string file_;
+
+    /// @brief File name stack
+    std::vector<std::string> files_;
+
+    /// @brief Location of the current token
+    ///
+    /// The lexer will keep updating it. This variable will be useful
+    /// for logging errors.
+    isc::dhcp::location loc_;
+
+    /// @brief Location stack
+    std::vector<isc::dhcp::location> locs_;
+
+    /// @brief Lexer state stack
+    std::vector<struct yy_buffer_state*> states_;
+
+    /// @brief sFile (aka FILE)
+    FILE* sfile_;
+
+    /// @brief sFile (aka FILE) stack
+    ///
+    /// This is a stack of files. Typically there's only one file (the
+    /// one being currently parsed), but there may be more if one
+    /// file includes another.
+    std::vector<FILE*> sfiles_;
+
+    /// @brief Current syntactic context
+    ParserContext ctx_;
+
+    /// @brief Enter a new syntactic context
+    ///
+    /// Entering a new syntactic context is useful in several ways.
+    /// First, it allows the parser to avoid conflicts. Second, it
+    /// allows the lexer to return different tokens depending on
+    /// context (e.g. if "renew-timer" string is detected, the lexer
+    /// will return STRING token if in JSON mode or RENEW_TIMER if
+    /// in DHCP4 mode. Finally, the syntactic context allows the
+    /// error message to be more descriptive if the input string
+    /// does not parse properly.
+    ///
+    /// @param ctx the syntactic context to enter into
+    void enter(const ParserContext& ctx);
+
+    /// @brief Leave a syntactic context
+    ///
+    /// @throw isc::Unexpected if unbalanced
+    void leave();
+
+    /// @brief Get the syntactix context name
+    ///
+    /// @return printable name of the context.
+    const std::string contextName();
+
+ private:
+    /// @brief Flag determining scanner debugging.
+    bool trace_scanning_;
+
+    /// @brief Flag determing parser debugging.
+    bool trace_parsing_;
+
+    /// @brief Syntactic context stack
+    std::vector<ParserContext> cstack_;
+
+    /// @brief Common part of parseXXX
+    ///
+    /// @return Element structure representing parsed text.
+    isc::data::ConstElementPtr parseCommon();
+};
+
+}; // end of isc::eval namespace
+}; // end of isc namespace
+
+#endif

src/bin/dhcp4/parser_context_decl.h

+// Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#ifndef PARSER6_CONTEXT_DECL_H
+#define PARSER6_CONTEXT_DECL_H
+
+/// @file parser_context_decl.h Forward declaration of the ParserContext class
+
+namespace isc {
+namespace dhcp {
+
+class Parser4Context;
+
+}; // end of isc::dhcp namespace
+}; // end of isc namespace
+
+#endif

src/bin/dhcp4/tests/Makefile.am

@@ -23,6 +23,7 @@ AM_CPPFLAGS += -I$(top_srcdir)/src/bin
 AM_CPPFLAGS += $(BOOST_INCLUDES)
 AM_CPPFLAGS += -DTEST_DATA_BUILDDIR=\"$(abs_top_builddir)/src/bin/dhcp4/tests\"
 AM_CPPFLAGS += -DINSTALL_PROG=\"$(abs_top_srcdir)/install-sh\"
+AM_CPPFLAGS += -DCFG_EXAMPLES=\"$(abs_top_srcdir)/doc/examples/kea4\"
 
 CLEANFILES = $(builddir)/interfaces.txt $(builddir)/logger_lockfile
 CLEANFILES += $(builddir)/load_marker.txt $(builddir)/unload_marker.txt
@@ -88,6 +89,7 @@ dhcp4_unittests_SOURCES += inform_unittest.cc
 dhcp4_unittests_SOURCES += dora_unittest.cc
 dhcp4_unittests_SOURCES += host_options_unittest.cc
 dhcp4_unittests_SOURCES += release_unittest.cc
+dhcp4_unittests_SOURCES += parser_unittest.cc
 dhcp4_unittests_SOURCES += out_of_range_unittest.cc
 dhcp4_unittests_SOURCES += decline_unittest.cc
 dhcp4_unittests_SOURCES += kea_controller_unittest.cc

src/bin/dhcp4/tests/parser_unittest.cc

+// Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include <gtest/gtest.h>
+#include <cc/data.h>
+#include <dhcp4/parser_context.h>
+
+using namespace isc::data;
+using namespace std;
+
+namespace {
+
+void compareJSON(ConstElementPtr a, ConstElementPtr b, bool print = true) {
+    ASSERT_TRUE(a);
+    ASSERT_TRUE(b);
+    if (print) {
+        // std::cout << "JSON A: -----" << endl << a->str() << std::endl;
+        // std::cout << "JSON B: -----" << endl << b->str() << std::endl;
+        // cout << "---------" << endl << endl;
+    }
+    EXPECT_EQ(a->str(), b->str());
+}