[master] Merge branch 'trac5017' (dhcp4 bison parser)

# Conflicts:
#	doc/examples/kea4/classify.json

doc/Makefile.am

@@ -8,8 +8,10 @@ EXTRA_DIST += devel/unit-tests.dox
 
 nobase_dist_doc_DATA  = examples/ddns/sample1.json
 nobase_dist_doc_DATA += examples/ddns/template.json
+nobase_dist_doc_DATA += examples/kea4/advanced.json
 nobase_dist_doc_DATA += examples/kea4/backends.json
 nobase_dist_doc_DATA += examples/kea4/classify.json
+nobase_dist_doc_DATA += examples/kea4/dhcpv4-over-dhcpv6.json
 nobase_dist_doc_DATA += examples/kea4/hooks.json
 nobase_dist_doc_DATA += examples/kea4/leases-expiration.json
 nobase_dist_doc_DATA += examples/kea4/multiple-options.json
@@ -21,6 +23,8 @@ nobase_dist_doc_DATA += examples/kea4/single-subnet.json
 nobase_dist_doc_DATA += examples/kea6/advanced.json
 nobase_dist_doc_DATA += examples/kea6/backends.json
 nobase_dist_doc_DATA += examples/kea6/classify.json
+nobase_dist_doc_DATA += examples/kea6/dhcpv4-over-dhcpv6.json
+nobase_dist_doc_DATA += examples/kea6/duid.json
 nobase_dist_doc_DATA += examples/kea6/hooks.json
 nobase_dist_doc_DATA += examples/kea6/leases-expiration.json
 nobase_dist_doc_DATA += examples/kea6/multiple-options.json

doc/examples/kea4/advanced.json

+// This is an example configuration file for DHCPv4 server in Kea.
+// It covers some of the more advanced features. This file may not be coherent
+// as its main purpose is to demonstrate the features. They don't necessarily
+// have to make sense used together.
+
+// The new parser supports 3 comment styles:
+
+// This is C++ style.
+
+# This is a bash style.
+
+/* This is a C style comment. */
+
+/* C style comment
+   can span
+   multiple lines */
+
+{ "Dhcp4":
+
+{
+    // Kea is told to listen on ethX interface only.
+    "interfaces-config": {
+        "interfaces": [ "ethX" ],
+
+        // This specifies what type of socket Kea uses. Currently supported
+        // are 'raw' (which is the default) and 'udp'. Raw has the benefit
+        // of receiving all traffic every time and a downside of bypassing
+        // all firewall rules and having marginally bigger performance impact.
+        // 'udp' is generally better if you have only relayed traffic. Kea
+        // than opens up normal UDP socket and the kernel does all the
+        // Ethernet/IP stack processing.
+        "dhcp-socket-type": "udp"
+    },
+
+    // We need to specify the the database used to store leases. As of
+    // September 2016, four database backends are supported: MySQL,
+    // PostgreSQL, Cassandra, and the in-memory database, Memfile.
+    // We'll use memfile  because it doesn't require any prior set up.
+    // For memfile, it's important to always specify lfc-interval, so
+    // the lease file would not grow without bounds and be sanitized
+    // once per hour.
+    "lease-database": {
+        "type": "memfile",
+        "lfc-interval": 3600
+    },
+
+    // Addresses will be assigned with a lifetime of 4000 seconds.
+    // The client is told to start renewing after 1000 seconds. If the server
+    // does not respond within 2000 seconds of the lease being granted, client
+    // is supposed to start REBIND procedure (emergency renewal that allows
+    // switching to a different server).
+    "valid-lifetime": 4000,
+    "renew-timer": 1000,
+    "rebind-timer": 2000,
+
+    // RFC6842 says that the server is supposed to echo back client-id option.
+    // However, some older clients do not support this and are getting confused
+    // when they get their own client-id. Kea can disable RFC6842 support.
+    "echo-client-id": false,
+
+    // Some clients don't use stable client identifier, but rather generate them
+    // during each boot. This may cause a client that reboots frequently to get
+    // multiple leases, which may not be desirable. As such, sometimes admins
+    // prefer to tell their DHCPv4 server to ignore client-id value altogether
+    // and rely exclusively on MAC address. This is a parameter that is defined
+    // globally, but can be overridden on a subnet level.
+    "match-client-id": true,
+
+    // The following list defines subnets. Each subnet consists of at
+    // least subnet and pool entries.
+    "subnet4": [
+        {
+            "pools": [ { "pool":  "192.0.2.1 - 192.0.2.200" } ],
+            "subnet": "192.0.2.0/24"
+        },
+        {
+            // This particular subnet has match-client-id value changed.
+            // This causes Kea to ignore client-id values in this subnet
+            // and rely exclusively on MAC addresses.
+            "pools": [ { "pool": "192.0.3.100 - 192.0.3.200" } ],
+            "subnet": "192.0.3.0/24",
+            "match-client-id": false
+        },
+        {
+            "pools": [ { "pool": "192.0.4.1 - 192.0.4.254" } ],
+            "subnet": "192.0.4.0/24"
+        }
+    ]
+},
+
+  // The following configures logging. It assumes that messages with at least
+  // informational level (info, warn, error and fatal) should be logged to stdout.
+  "Logging": {
+      "loggers": [
+          {
+            "name": "kea-dhcp4",
+              "output_options": [
+                  {
+                      "output": "stdout"
+                  }
+              ],
+              "severity": "INFO"
+          }
+      ]
+  }
+
+}

doc/examples/kea4/classify.json

@@ -51,10 +51,16 @@
 
 # Let's pick VoIP phones. Those that send their class identifiers
 # as Aastra, should belong to VoIP class. For a list of all options,
-# see www.iana.org/assignments/bootp-dhcp-parameters/
+# see www.iana.org/assignments/bootp-dhcp-parameters/.
+# In this particular class, we want to set specific values
+# of certain DHCPv4 fields. If the incoming packet matches the
+# test, those fields will be set in outgoing responses.
   {
       "name": "VoIP",
-      "test": "substring(option[60].hex,0,6) == 'Aastra'"
+      "test": "substring(option[60].hex,0,6) == 'Aastra'",
+      "next-server": "192.0.2.254",
+      "server-hostname": "hal9000",
+      "boot-file-name": "/dev/null"
   }
 
   ],
@@ -65,7 +71,7 @@
 # to that class are allowed for that subnet.
   "subnet4": [
     {
-# This one is for VoIP devices only.          
+# This one is for VoIP devices only.
         "pools": [ { "pool":  "192.0.2.1 - 192.0.2.200" } ],
         "subnet": "192.0.2.0/24",
         "client-class": "VoIP",

doc/examples/kea4/hooks.json

@@ -37,7 +37,11 @@
         "library": "/opt/lib/security.so"
      },
      {
-        "library": "/opt/lib/charging.so"
+         "library": "/opt/lib/charging.so",
+         "parameters": {
+             "path": "/var/kea/var",
+             "base-name": "kea-forensic6"
+         }
      }
   ]
 }

doc/examples/kea4/reservations.json

@@ -107,6 +107,16 @@
                      "data": "10.1.1.202,10.1.1.203"
                  }
              ]
+         },
+# This reservation is for a client that needs specific DHCPv4 fields to be
+# set. Three supported fields are next-server, server-hostname and
+# boot-file-name
+         {
+             "client-id": "01:0a:0b:0c:0d:0e:of",
+             "ip-address": "192.0.2.205",
+             "next-server": "192.0.2.1",
+             "server-hostname": "hal9000",
+             "boot-file-name": "/dev/null"
          }
        ]
     }

doc/examples/kea4/several-subnets.json

@@ -27,15 +27,36 @@
   "renew-timer": 1000,
   "rebind-timer": 2000,
 
+# RFC6842 says that the server is supposed to echo back client-id option.
+# However, some older clients do not support this and are getting confused
+# when they get their own client-id. Kea can disable RFC6842 support.
+  "echo-client-id": false,
+
+# Some clients don't use stable client identifier, but rather generate them
+# during each boot. This may cause a client that reboots frequently to get
+# multiple leases, which may not be desirable. As such, sometimes admins
+# prefer to tell their DHCPv4 server to ignore client-id value altogether
+# and rely exclusively on MAC address. This is a parameter that is defined
+# globally, but can be overridden on a subnet level.
+  "match-client-id": true,
+
 # The following list defines subnets. Each subnet consists of at
 # least subnet and pool entries.
-  "subnet4": [ 
-  {    "pools": [ { "pool":  "192.0.2.1 - 192.0.2.200" } ],
-       "subnet": "192.0.2.0/24"  },
-  {    "pools": [ { "pool": "192.0.3.100 - 192.0.3.200" } ],
-       "subnet": "192.0.3.0/24"  },
-  {    "pools": [ { "pool": "192.0.4.1 - 192.0.4.254" } ],
-       "subnet": "192.0.4.0/24"  } ]
+  "subnet4": [
+  {
+      "pools": [ { "pool":  "192.0.2.1 - 192.0.2.200" } ],
+      "subnet": "192.0.2.0/24"
+  },
+  {
+# This particular subnet has match-client-id value changed.
+      "pools": [ { "pool": "192.0.3.100 - 192.0.3.200" } ],
+      "subnet": "192.0.3.0/24",
+      "match-client-id": false
+  },
+  {
+      "pools": [ { "pool": "192.0.4.1 - 192.0.4.254" } ],
+      "subnet": "192.0.4.0/24"
+  } ]
 },
 
 # The following configures logging. It assumes that messages with at least

doc/guide/config.xml

@@ -37,27 +37,23 @@
       <itemizedlist>
         <listitem>
           <simpara>shell comments: any text after the hash (#)
-          character is ignored. Dhcp6 allows # in any column, while
-          Dhcp4 and Ddns require hash to be in the first
-          column.</simpara>
+          character is ignored. Both Dhcp4 and Dhcp6 allow # in any column,
+          while Ddns requires hash to be in the first column.</simpara>
         </listitem>
         <listitem>
           <simpara>C comments: any text after the double slashes (//)
-          character is ignored. We're in a process of
-          migrating the configuation parsers and currently only Dhcp6
-          supports this feature.</simpara>
+          character is ignored. Both Dhcp4 and Dhcp6 supports this
+          feature.</simpara>
         </listitem>
         <listitem>
           <simpara>Multiline comments: any text between /* and */ is
-          ignored. This commenting can span multiple lines. We're in a
-          process of migrating the configuation parsers and currently
-          only Dhcp6 supports this feature.</simpara>
+          ignored. This commenting can span multiple lines. Both Dhcp4 and
+          Dhcp6 supports this feature.</simpara>
         </listitem>
         <listitem>
           <simpara>File inclusion: JSON files can include other JSON
           files. This can be done by using &lt;?include
-          "file.json"?&gt;. We're in a process of migrating the
-          configuation parsers and currently only Dhcp6 supports this
+          "file.json"?&gt;. Both Dhcp4 and Dhcp6 supports this
           feature.</simpara>
         </listitem>
       </itemizedlist>

src/bin/dhcp4/.gitignore

@@ -5,3 +5,4 @@
 /spec_config.h
 /spec_config.h.pre
 /s-messages
+/dhcp4_parser.report

src/bin/dhcp4/Makefile.am

@@ -63,6 +63,10 @@ libdhcp4_la_SOURCES += dhcp4_log.cc dhcp4_log.h
 libdhcp4_la_SOURCES += dhcp4_srv.cc dhcp4_srv.h
 libdhcp4_la_SOURCES += dhcp4to6_ipc.cc dhcp4to6_ipc.h
 
+libdhcp4_la_SOURCES += dhcp4_lexer.ll location.hh position.hh stack.hh
+libdhcp4_la_SOURCES += dhcp4_parser.cc dhcp4_parser.h
+libdhcp4_la_SOURCES += parser_context.cc parser_context.h parser_context_decl.h
+
 libdhcp4_la_SOURCES += kea_controller.cc
 
 nodist_libdhcp4_la_SOURCES = dhcp4_messages.h dhcp4_messages.cc
@@ -104,3 +108,30 @@ endif
 
 kea_dhcp4dir = $(pkgdatadir)
 kea_dhcp4_DATA = dhcp4.spec
+
+if GENERATE_PARSER
+
+parser: dhcp4_lexer.cc location.hh position.hh stack.hh dhcp4_parser.cc dhcp4_parser.h
+	@echo "Flex/bison files regenerated"
+
+# --- Flex/Bison stuff below --------------------------------------------------
+# When debugging grammar issues, it's useful to add -v to bison parameters.
+# bison will generate parser.output file that explains the whole grammar.
+# It can be used to manually follow what's going on in the parser.
+# This is especially useful if yydebug_ is set to 1 as that variable
+# will cause parser to print out its internal state.
+# Call flex with -s to check that the default rule can be suppressed
+# Call bison with -W to get warnings like unmarked empty rules
+# Note C++11 deprecated register still used by flex < 2.6.0
+location.hh position.hh stack.hh dhcp4_parser.cc dhcp4_parser.h: dhcp4_parser.yy
+	$(YACC) --defines=dhcp4_parser.h --report=all --report-file=dhcp4_parser.report -o dhcp4_parser.cc dhcp4_parser.yy
+
+dhcp4_lexer.cc: dhcp4_lexer.ll
+	$(LEX) --prefix parser4_ -o dhcp4_lexer.cc dhcp4_lexer.ll
+
+else
+
+parser location.hh position.hh stack.hh dhcp4_parser.cc dhcp4_parser.h dhcp4_lexer.cc:
+	@echo Parser generation disabled. Configure with --enable-generate-parser to enable it.
+
+endif

src/bin/dhcp4/dhcp4.dox

@@ -41,6 +41,43 @@ all configuration parsers. All DHCPv4 parsers deriving from this class
 directly have their entire implementation in the
 src/bin/dhcp4/json_config_parser.cc.
 
+@section dhcpv4ConfigParserBison Configuration Parser for DHCPv4 (bison)
+
+If you are here only to learn absolute minimum about the new parser, here's how you
+use it:
+
+@code
+ // The following code:
+ json = isc::data::Element::fromJSONFile(file_name, true);
+
+ // can be replaced with this:
+ Parser4Context parser;
+ json = parser.parseFile(file_name, Parser4Context::PARSER_DHCP4);
+@endcode
+
+For an introduction, rationale and issues the new parser tries to address,
+see @ref dhcpv6ConfigParserBison.
+
+The code change for 5017 introduces flex/bison based parser. It is
+essentially defined in two files: dhcp4_lexer.ll, which defines
+regular expressions that are used on the input (be it a file or a
+string in memory). In essence, this code is being called repeatedly
+and each time it returns a token. This repeats until either the
+parsing is complete or syntax error is encountered. For detailed
+discussion, how they operate see @ref dhcpv6ConfigParserBison.
+
+@section dhcpv4ConfigSubParser Parsing Partial Configuration in DHCPv4
+
+See @ref dhcpv6ConfigSubParser.
+
+@section dhcp4ParserIncludes Config File Includes
+
+See @ref dhcp6ParserIncludes.
+
+@section dhcp4ParserConflicts Avoiding syntactical conflicts in parsers
+
+See @ref dhcp6ParserConflicts.
+
 @section dhcpv4ConfigInherit DHCPv4 configuration inheritance
 
 One notable useful feature of DHCP configuration is its parameter inheritance.

src/bin/dhcp4/json_config_parser.cc

@@ -428,8 +428,7 @@ DhcpConfigParser* createGlobalDhcp4ConfigParser(const std::string& config_id,
         parser = new OptionDataListParser(config_id, CfgOptionPtr(), AF_INET);
     } else if (config_id.compare("option-def") == 0) {
         parser  = new OptionDefListParser(config_id, globalContext());
-    } else if ((config_id.compare("version") == 0) ||
-               (config_id.compare("next-server") == 0)) {
+    } else if ((config_id.compare("next-server") == 0)) {
         parser  = new StringParser(config_id,
                                     globalContext()->string_values_);
     } else if (config_id.compare("lease-database") == 0) {

src/bin/dhcp4/kea_controller.cc

@@ -9,6 +9,7 @@
 #include <dhcp4/json_config_parser.h>
 #include <dhcp4/ctrl_dhcp4_srv.h>
 #include <dhcp4/dhcp4_log.h>
+#include <dhcp4/parser_context.h>
 #include <dhcpsrv/cfgmgr.h>
 #include <exceptions/exceptions.h>
 
@@ -52,7 +53,8 @@ void configure(const std::string& file_name) {
         }
 
         // Read contents of the file and parse it as JSON
-        json = isc::data::Element::fromJSONFile(file_name, true);
+        Parser4Context parser;
+        json = parser.parseFile(file_name, Parser4Context::PARSER_DHCP4);
         if (!json) {
             isc_throw(isc::BadValue, "no configuration found");
         }

src/bin/dhcp4/location.hh

+// Generated 201612201711
+// A Bison parser, made by GNU Bison 3.0.4.
+
+// Locations for Bison parsers in C++
+
+// Copyright (C) 2002-2015 Free Software Foundation, Inc.
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+// As a special exception, you may create a larger work that contains
+// part or all of the Bison parser skeleton and distribute that work
+// under terms of your choice, so long as that work isn't itself a
+// parser generator using the skeleton or a modified version thereof
+// as a parser skeleton.  Alternatively, if you modify or redistribute
+// the parser skeleton itself, you may (at your option) remove this
+// special exception, which will cause the skeleton and the resulting
+// Bison output files to be licensed under the GNU General Public
+// License without this special exception.
+
+// This special exception was added by the Free Software Foundation in
+// version 2.2 of Bison.
+
+/**
+ ** \file location.hh
+ ** Define the isc::dhcp::location class.
+ */
+
+#ifndef YY_PARSER4_LOCATION_HH_INCLUDED
+# define YY_PARSER4_LOCATION_HH_INCLUDED
+
+# include "position.hh"
+
+#line 14 "dhcp4_parser.yy" // location.cc:296
+namespace isc { namespace dhcp {
+#line 46 "location.hh" // location.cc:296
+  /// Abstract a location.
+  class location
+  {
+  public:
+
+    /// Construct a location from \a b to \a e.
+    location (const position& b, const position& e)
+      : begin (b)
+      , end (e)
+    {
+    }
+
+    /// Construct a 0-width location in \a p.
+    explicit location (const position& p = position ())
+      : begin (p)
+      , end (p)
+    {
+    }
+
+    /// Construct a 0-width location in \a f, \a l, \a c.
+    explicit location (std::string* f,
+                       unsigned int l = 1u,
+                       unsigned int c = 1u)
+      : begin (f, l, c)
+      , end (f, l, c)
+    {
+    }
+
+
+    /// Initialization.
+    void initialize (std::string* f = YY_NULLPTR,
+                     unsigned int l = 1u,
+                     unsigned int c = 1u)
+    {
+      begin.initialize (f, l, c);
+      end = begin;
+    }
+
+    /** \name Line and Column related manipulators
+     ** \{ */
+  public:
+    /// Reset initial location to final location.
+    void step ()
+    {
+      begin = end;
+    }
+
+    /// Extend the current location to the COUNT next columns.
+    void columns (int count = 1)
+    {
+      end += count;
+    }
+
+    /// Extend the current location to the COUNT next lines.
+    void lines (int count = 1)
+    {
+      end.lines (count);
+    }
+    /** \} */
+
+
+  public:
+    /// Beginning of the located region.
+    position begin;
+    /// End of the located region.
+    position end;
+  };
+
+  /// Join two locations, in place.
+  inline location& operator+= (location& res, const location& end)
+  {
+    res.end = end.end;
+    return res;
+  }
+
+  /// Join two locations.
+  inline location operator+ (location res, const location& end)
+  {
+    return res += end;
+  }
+
+  /// Add \a width columns to the end position, in place.
+  inline location& operator+= (location& res, int width)
+  {
+    res.columns (width);
+    return res;
+  }
+
+  /// Add \a width columns to the end position.
+  inline location operator+ (location res, int width)
+  {
+    return res += width;
+  }
+
+  /// Subtract \a width columns to the end position, in place.
+  inline location& operator-= (location& res, int width)
+  {
+    return res += -width;
+  }
+
+  /// Subtract \a width columns to the end position.
+  inline location operator- (location res, int width)
+  {
+    return res -= width;
+  }
+
+  /// Compare two location objects.
+  inline bool
+  operator== (const location& loc1, const location& loc2)
+  {
+    return loc1.begin == loc2.begin && loc1.end == loc2.end;
+  }
+
+  /// Compare two location objects.
+  inline bool
+  operator!= (const location& loc1, const location& loc2)
+  {
+    return !(loc1 == loc2);
+  }
+
+  /** \brief Intercept output stream redirection.
+   ** \param ostr the destination output stream
+   ** \param loc a reference to the location to redirect
+   **
+   ** Avoid duplicate information.
+   */
+  template <typename YYChar>
+  inline std::basic_ostream<YYChar>&
+  operator<< (std::basic_ostream<YYChar>& ostr, const location& loc)
+  {
+    unsigned int end_col = 0 < loc.end.column ? loc.end.column - 1 : 0;
+    ostr << loc.begin;
+    if (loc.end.filename
+        && (!loc.begin.filename
+            || *loc.begin.filename != *loc.end.filename))
+      ostr << '-' << loc.end.filename << ':' << loc.end.line << '.' << end_col;
+    else if (loc.begin.line < loc.end.line)
+      ostr << '-' << loc.end.line << '.' << end_col;
+    else if (loc.begin.column < end_col)
+      ostr << '-' << end_col;
+    return ostr;
+  }
+
+#line 14 "dhcp4_parser.yy" // location.cc:296
+} } // isc::dhcp
+#line 192 "location.hh" // location.cc:296
+#endif // !YY_PARSER4_LOCATION_HH_INCLUDED

src/bin/dhcp4/parser_context.cc

+// Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC")
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+