[5137] A few fixes and many python style improvements

configure.ac

@@ -470,7 +470,10 @@ AC_ARG_ENABLE(shell, [AC_HELP_STRING([--enable-shell],
 
 if test "x$enable_shell" != xno ; then
 # If kea-shell is enabled, we really need python. 2.7 or anything newer will do.
-  AM_PATH_PYTHON([2.7])
+  AM_PATH_PYTHON([3], [found="yes"], [found="no"])
+  if test "x$found" = xno ; then
+    AM_PATH_PYTHON([2.7])
+  fi
 else
   PYTHON=no
 fi
@@ -1628,8 +1631,10 @@ AC_CONFIG_FILES([compatcheck/Makefile
                  src/bin/admin/tests/cql_tests.sh
                  src/bin/agent/tests/test_libraries.h
                  src/bin/shell/Makefile
+                 src/bin/shell/kea-shell
                  src/bin/shell/tests/Makefile
                  src/bin/shell/tests/shell_process_tests.sh
+                 src/bin/shell/tests/shell_unittest.py
                  src/hooks/Makefile
                  src/hooks/dhcp/Makefile
                  src/hooks/dhcp/user_chk/Makefile
@@ -1720,11 +1725,14 @@ AC_CONFIG_FILES([compatcheck/Makefile
 ])
 
  AC_CONFIG_COMMANDS([permissions], [
+           chmod +x src/bin/admin/kea-admin
            chmod +x src/bin/dhcp4/tests/dhcp4_process_tests.sh
            chmod +x src/bin/dhcp6/tests/dhcp6_process_tests.sh
            chmod +x src/bin/keactrl/keactrl
            chmod +x src/bin/keactrl/tests/keactrl_tests.sh
-           chmod +x src/bin/admin/kea-admin
+           chmod +x src/bin/shell/kea-shell
+           chmod +x src/bin/shell/tests/shell_process_tests.sh
+           chmod +x src/bin/shell/tests/shell_unittest.py
            chmod +x src/lib/dns/gen-rdatacode.py
            chmod +x src/lib/log/tests/console_test.sh
            chmod +x src/lib/log/tests/destination_test.sh
@@ -1800,6 +1808,7 @@ if test "$PYTHON" != "no" ; then
 cat >> config.report << END
 
 Python:
+  PYTHON:          ${PYTHON}
   PYTHON_VERSION:  ${PYTHON_VERSION}
 
 END

src/bin/agent/tests/Makefile.am

@@ -5,7 +5,6 @@ SHTESTS += ca_process_tests.sh
 
 noinst_SCRIPTS = ca_process_tests.sh
 EXTRA_DIST  = ca_process_tests.sh.in
-noinst_LTLIBRARIES = libbasic.la
 
 # test using command-line arguments, so use check-local target instead of TESTS
 check-local:
@@ -40,6 +39,8 @@ TESTS_ENVIRONMENT = \
 TESTS =
 if HAVE_GTEST
 
+noinst_LTLIBRARIES = libbasic.la
+
 TESTS += ca_unittests
 
 ca_unittests_SOURCES  = ca_cfg_mgr_unittests.cc

src/bin/shell/Makefile.am

@@ -5,7 +5,7 @@ EXTRA_DIST =
 if KEA_SHELL
 
 # Kea-shell is enabled, here are proper rules for it.
-kea_shell_PYTHON = kea-shell.py kea_conn.py kea_connector2.py kea_connector3.py
+kea_shell_PYTHON = kea_conn.py kea_connector2.py kea_connector3.py
 kea_shelldir = @localstatedir@/@PACKAGE@
 
 bin_SCRIPTS = kea-shell
@@ -13,12 +13,11 @@ bin_SCRIPTS = kea-shell
 else
 
 # Kea-shell is disabled, simply keep the files for make dist
-EXTRA_DIST += kea-shell.py kea_conn.py kea_connector2.py kea_connector3.py
+EXTRA_DIST += kea-shell kea_conn.py kea_connector2.py kea_connector3.py
 
 endif
 
-CLEANFILES = kea-shell kea-shell.pyc
-
+CLEANFILES = *.pyc
 
 man_MANS = kea-shell.8
 DISTCLEANFILES = $(man_MANS)
@@ -38,13 +37,6 @@ $(man_MANS):
 
 endif
 
-# This is done here since configure.ac AC_OUTPUT doesn't expand certain variables.
-kea-shell: kea-shell.py
-	$(SED) "s|@@PYTHONPATH@@|@pyexecdir@|" kea-shell.py > kea-shell.tmp
-	$(SED) "s|REPORTED_VERSION|@PACKAGE_VERSION@|" kea-shell.tmp >$@
-	rm -f kea-shell.tmp
-	chmod a+x $@
-
 install-data-local:
 	$(mkinstalldirs) $(DESTDIR)/@localstatedir@/@PACKAGE@
 

src/bin/shell/kea-shell.in

+#!@PYTHON@
+
+# Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Text client for Control Agent process
+"""
+
+# First, let's import the right kea_connector.
+# We have two versions: one for python 2.x and another for python 3.x.
+# Sadly, there's no unified way to handle http connections. The recommended
+# way is to use Requests (http://docs.python-requests.org/en/master/), but
+# that's a stand alone package that requires separate installation. One of
+# the design requirements was to not require any additional packages, so
+# the code uses standard libraries available in python. Hence two versions.
+import sys
+import signal
+import argparse
+
+from kea_conn import CARequest # CAResponse
+
+if sys.version_info[0] == 2:
+    # This is Python 2.x
+    import kea_connector2 as kea_connector
+elif sys.version_info[0] == 3:
+    # This is Python 3.x
+    import kea_connector3 as kea_connector
+else:
+    # This is... have no idea what it is.
+    raise SystemExit("Unknown python version:" + str(sys.version_info[0]))
+
+def timeout_handler(signum, frame):
+    """Connection timeoout handler"""
+    del signum, frame
+    print("Connection timeout")
+    sys.exit(1)
+
+VERSION = "@PACKAGE_VERSION@"
+
+def shell_body():
+    """
+    Second step: Need to parse command line parameters. We will use
+    argparse for that purpose. It does great job with having default
+    values, taking care of the help and sanity checking input
+    parameters.
+    """
+    parser = argparse.ArgumentParser(description='kea-shell is a simple text '
+                                     'client that uses REST interface to '
+                                     'connect to Kea Control Agent.')
+    parser.add_argument('--host', type=str, default='127.0.0.1',
+                        help='hostname of the CA to connect to '
+                        '(defaul:; 127.0.0.1)')
+    parser.add_argument('--port', type=int, default=8000,
+                        help='TCP port of the CA to connect to '
+                        '(default: 8000)')
+    parser.add_argument('--timeout', type=int, default='10',
+                        help='Timeout (in seconds) when attempting to '
+                        'connect to CA (default: 10)')
+    parser.add_argument('command', type=str, nargs="?",
+                        default='list-commands',
+                        help='command to be executed. If not specified, '
+                        '"list-commands" is used')
+    parser.add_argument('-v', action="store_true", help="Prints version")
+    cmd_args = parser.parse_args()
+
+    if cmd_args.v:
+        print(VERSION)
+        exit(0)
+
+    # Ok, now time to put the parameters parsed into the structure to be
+    # used by the connection.
+    params = CARequest()
+    params.command = cmd_args.command
+    params.http_host = cmd_args.host
+    params.http_port = cmd_args.port
+    params.timeout = cmd_args.timeout
+    params.version = VERSION
+
+    params.generate_body()
+    params.generate_headers()
+
+    # Load command processor
+    # @todo - command specific processing will be added as part of
+    # future work (either #5138 or #5139, whichever is implemented
+    # first)
+
+    # Read parameters from stdin (they're optional for some commands)
+    for line in sys.stdin:
+        params.params += line
+
+    # Set the timeout timer. If the connection takes too long,
+    # it will send a signal to us.
+    signal.signal(signal.SIGALRM, timeout_handler)
+    signal.alarm(params.timeout)
+
+    # Ok, everything is ready. Let's send the command and get a response.
+    try:
+        resp = kea_connector.send_to_control_agent(params)
+    except Exception as exc:
+        print("Failed to run: " + str(exc))
+        sys.exit(1)
+
+    resp.print_response()
+
+    sys.exit(0)
+
+if __name__ == "__main__":
+    shell_body()

src/bin/shell/kea-shell.py

-#!/usr/bin/python
-
-# This is going to be replaced with the actual version (see kea-shell target
-# in Makefile.am)
-VERSION = "REPORTED_VERSION"
-
-# First, let's import the right kea_connector.
-# We have two versions: one for python 2.x and another for python 3.x.
-# Sadly, there's no unified way to handle http connections. The recommended
-# way is to use Requests (http://docs.python-requests.org/en/master/), but
-# that's a stand alone package that requires separate installation. One of
-# the design requirements was to not require any additional packages, so
-# the code uses standard libraries available in python. Hence two versions.
-import sys
-import signal
-import argparse
-
-if (sys.version_info[0] == 2):
-    # This is Python 2.x
-    import kea_connector2 as kea_connector
-else:
-    if (sys.version_info[0] == 3):
-        # This is Python 3.x
-        import kea_connector3 as kea_connector
-    else:
-        # This is... have no idea what it is.
-        raise SystemExit("Unknown python version:" + str(sys.version_info[0]))
-
-from kea_conn import CARequest, CAResponse
-
-# Second step: Need to parse command line parameters. We will use argparse for
-# that purpose. It does great job with having default values, taking care of
-# the help and sanity checking input parameters.
-
-parser = argparse.ArgumentParser(description='kea-shell is a simple text client that uses REST '
-                                 'interface to connect to Kea Control Agent.')
-parser.add_argument('--host', type=str, default='127.0.0.1',
-                    help='hostname of the CA to connect to (default; 127.0.0.1)')
-parser.add_argument('--port', type=int, default=8000,
-                    help='TCP port of the CA to connect to (default: 8000)')
-parser.add_argument('--timeout', type=int, default='10',
-                    help='Timeout (in seconds) when attempting to connect to CA (default: 10)')
-parser.add_argument('command', type=str, nargs="?", default='list-commands',
-                    help='command to be executed. If not specified, "list-commands" is used')
-parser.add_argument('-v', action="store_true", help="Prints version")
-cmd_args = parser.parse_args()
-
-if (cmd_args.v):
-    print (VERSION)
-    exit(0)
-
-# Ok, now time to put the parameters parsed into the structure to be used by the
-# connection.
-params = CARequest()
-params.command = cmd_args.command
-params.http_host = cmd_args.host
-params.http_port = cmd_args.port
-params.timeout = cmd_args.timeout
-params.version = VERSION
-
-params.generateBody()
-params.generateHeaders()
-
-conn = kea_connector.KeaConnector()
-
-def timeout_handler(signum, frame):
-    print ("Connection timeout")
-    sys.exit(1)
-
-# Load command processor
-# @todo - command specific processing will be added as part of future work
-# (either #5138 or #5139, whichever is implemented first)
-
-# Read parameters from stdin (they're optional for some commands)
-for line in sys.stdin:
-    params.params += line
-
-# Set the timeout timer. If the connection takes too long,
-# it will send a signal to us.
-signal.signal(signal.SIGALRM, timeout_handler)
-signal.alarm(params.timeout)
-
-# Ok, everything is ready. Let's send the command and get a response.
-try:
-    resp = conn.sendCA(params)
-except Exception as e:
-    print("Failed to run: " + str(e))
-    sys.exit(1)
-
-resp.printResp()
-
-sys.exit(0)

src/bin/shell/kea-shell.xml

@@ -49,9 +49,10 @@
     <cmdsynopsis>
       <command>kea-shell</command>
       <arg><option>-h</option></arg>
+      <arg><option>-v</option></arg>
       <arg><option>--host</option></arg>
       <arg><option>--port</option></arg>
-      <arg><option>-v</option></arg>
+      <arg><option>--timeout</option></arg>
       <arg><option>command</option></arg>
     </cmdsynopsis>
   </refsynopsisdiv>
@@ -64,8 +65,8 @@
       Kea Control Agent (CA). It takes command as a command-line parameter
       that is being sent to CA with proper JSON
       encapsulation. Optional parameters may be specified on the
-      stdin. The request it sent of HTTP and a response is
-      retrieved. That response is printed out on stdout.
+      standard input. The request it sent of HTTP and a response is
+      retrieved. That response is displayed out on the standard output.
     </para>
 
   </refsect1>
@@ -78,16 +79,16 @@
     <variablelist>
 
       <varlistentry>
-        <term><option>-v</option></term>
+        <term><option>-h</option></term>
         <listitem><para>
-          Display the version.
+          Displays help regarding command line parameters.
         </para></listitem>
       </varlistentry>
 
       <varlistentry>
-        <term><option>-h</option></term>
+        <term><option>-v</option></term>
         <listitem><para>
-          Displays help regarding command line parameters.
+          Display the version.
         </para></listitem>
       </varlistentry>
 

src/bin/shell/kea_conn.py

-#!/usr/bin/python
+# Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file contains classes used for communication with Control Agent.
+"""
+This file contains classes used for communication with Control Agent.
+"""
 
-# This class defines the HTTP request to be sent.
-# The supported parameters listed are:
-# - path (specifies the path on the server, CA uses only /)
-# - http_host - hostname of the CA
-# - http-port - TCP port of the CA
-# - command - specifies the command to send (e.g. list-commands)
-# - timeout - timeout (in ms)
-# - headers - extra HTTP headers may be added here
-# - version - version to be reported in HTTP header
 class CARequest:
+    """
+    This class defines the HTTP request to be sent.
+    The supported parameters listed are:
+     - path (specifies the path on the server, CA uses only /)
+     - http_host - hostname of the CA
+     - http-port - TCP port of the CA
+     - command - specifies the command to send (e.g. list-commands)
+     - timeout - timeout (in ms)
+     - headers - extra HTTP headers may be added here
+     - version - version to be reported in HTTP header
+    """
     path = '/'
     http_host = ''
     http_port = 0
@@ -20,39 +28,47 @@ class CARequest:
     params = ''
     headers = {}
     version = ""
+    # This is a storage for generated command (input data to be sent over POST)
+    content = ''
 
-    # Generates the content, out of specified command line
-    # and optional content.
-    # @todo: Add support for parameters
-    # this stores the output in self.content
-    def generateBody(self):
+    def generate_body(self):
+        """
+        Generates the content, out of specified command line
+        and optional content.
+        @todo: Add support for parameters
+        this stores the output in self.content
+        """
         self.content = '{ "command": "' + self.command + '"'
-        if (len(self.params)):
+        if len(self.params):
             self.content += ', "parameters": { ' + self.params + ' }'
         self.content += ' }'
 
-    # Generate HTTP headers
-    #
-    # In particular, this method generates Content-Length and its value.
-    def generateHeaders(self):
+    def generate_headers(self):
+        """
+        Generate HTTP headers
+
+        In particular, this method generates Content-Length and its value.
+        """
         self.headers['Content-Type'] = 'application/json'
         self.headers['User-Agent'] = "Kea-shell/%s"%(self.version)
         self.headers['Accept'] = '*/*'
         self.headers['Content-Length'] = "%d"%(len(self.content))
 
-    # This is a storage for generated command (input data to be sent over POST)
-    content = ''
 
-# This class represents the HTTP response
 class CAResponse:
+    """
+    This class represents the HTTP response
+    """
 
-    # Constructor
-    #
-    # Three mandatory parameters are:
-    # status - numerical number the describe the status (e.g. 200 = OK)
-    # reason - textual explanation of what happened
-    # body - the actual body structure of the response
     def __init__(self, status, reason, body):
+        """
+        Constructor
+
+        Three mandatory parameters are:
+         status - numerical number the describe the status (e.g. 200 = OK)
+         reason - textual explanation of what happened
+         body - the actual body structure of the response
+        """
         self.status = status
         self.reason = reason
         self.body = body
@@ -61,11 +77,13 @@ class CAResponse:
     reason = ''
     body = ''
 
-    # Used for debugging
-    #
-    # if defug is true, this prints even more information
-    def printResp(self, debug = False):
-        if (debug):
+    def print_response(self, debug=False):
+        """
+        Used for debugging
+
+        if debug is true, this prints even more information
+        """
+        if debug:
             print(self.status)
             print(self.reason)
         print(self.body)

src/bin/shell/kea_connector2.py

-#!/usr/bin/python
+# Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This is PYTHON 2.x version of HTTP connection establishment
-
-from kea_conn import CARequest, CAResponse
+"""
+This is PYTHON 2.x version of HTTP connection establishment
+"""
 
 import httplib
 
-class KeaConnector:
-    def sendCA(self, params):
-        # Estalbish HTTP connection first.
-        conn = httplib.HTTPConnection(params.http_host, params.http_port)
-        conn.connect()
+from kea_conn import CAResponse # CARequest
+
+def send_to_control_agent(params):
+    """Establish HTTP connection first."""
+    conn = httplib.HTTPConnection(params.http_host, params.http_port)
+    conn.connect()
 
-        # Use POST to send it
-        request = conn.putrequest('POST', params.path)
+    # Use POST to send it
+    _ = conn.putrequest('POST', params.path)
 
-        # Send the headers first
-        for k in params.headers:
-            conn.putheader(k, params.headers[k])
-        conn.endheaders()
+    # Send the headers first
+    for k in params.headers:
+        conn.putheader(k, params.headers[k])
+    conn.endheaders()
 
-        # Send the content
-        conn.send(params.content)
+    # Send the content
+    conn.send(params.content)
 
-        # Now get the response
-        resp = conn.getresponse()
+    # Now get the response
+    resp = conn.getresponse()
 
-        # Now get the response details, put it in CAResponse and
-        # return it
-        x = CAResponse(resp.status, resp.reason, resp.read())
-        conn.close()
+    # Now get the response details, put it in CAResponse and
+    # return it
+    result = CAResponse(resp.status, resp.reason, resp.read())
+    conn.close()
 
-        return (x)
+    return result

src/bin/shell/kea_connector3.py

-#!/usr/bin/python
+# Copyright (C) 2017 Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from kea_conn import CARequest, CAResponse
+"""
+This is PYTHON 3.x version of HTTP connection establishment
+"""
 
 import urllib.request
 
-class KeaConnector:
-    def sendCA(self, params):
-        # Estalbish HTTP connection first.
-        url = "http://" + params.http_host + ":" + str(params.http_port) + str(params.path)
-        
-        req = urllib.request.Request(url = url, data = str.encode(params.content),
-                                     headers = params.headers)
-        resp = urllib.request.urlopen(req)
+from kea_conn import CAResponse # CARequest
 
-        # Now get the response details, put it in CAResponse and
-        # return it
-        x = CAResponse(resp.getcode(), resp.reason, resp.read().decode("utf-8"))
+def send_to_control_agent(params):
+    """Establish HTTP connection first."""
+    url = "http://" + params.http_host + ":"
+    url += str(params.http_port) + str(params.path)
 
-        return (x)
+    req = urllib.request.Request(url=url,
+                                 data=str.encode(params.content),
+                                 headers=params.headers)
+    resp = urllib.request.urlopen(req)
+
+    # Now get the response details, put it in CAResponse and return it
+    result = CAResponse(resp.getcode(), resp.reason,
+                        resp.read().decode("utf-8"))
+
+    return result

src/bin/shell/tests/shell_process_tests.sh.in

@@ -36,8 +36,8 @@ CONFIG="{
 }"
 
 # In these tests we need to use two binaries: Control Agent and Kea shell.
-# Using bin and bin_path would be confusing, so we omit defining bin and bin_path
-# on purpose.
+# Using bin and bin_path would be confusing, so we omit defining bin
+# and bin_path on purpose.
 ca_bin="kea-ctrl-agent"
 ca_bin_path=@abs_top_builddir@/src/bin/agent
 
@@ -83,8 +83,8 @@ shell_command_test() {
     # of configuration failure).
     get_pid ${ca_bin}
     if [ ${_GET_PIDS_NUM} -ne 1 ]; then
-        printf "ERROR: expected one Control Agent process to be started. Found %d processes\
- started.\n" ${_GET_PIDS_NUM}
+        printf "ERROR: expected one Control Agent process to be started.\
+ Found %d processes started.\n" ${_GET_PIDS_NUM}
         clean_exit 1
     fi
 
@@ -92,22 +92,26 @@ shell_command_test() {
     # It should be just once on startup.
     get_reconfigs
     if [ ${_GET_RECONFIGS} -ne 1 ]; then
-        printf "ERROR: server been configured ${_GET_RECONFIGS} time(s), but exactly 1 was expected.\n"
+        printf "ERROR: server been configured ${_GET_RECONFIGS} time(s),\
+ but exactly 1 was expected.\n"
         clean_exit 1
     else
         printf "Server successfully configured.\n"
     fi
 
     # Main test phase: send command, check response.
-    tmp="echo \"${params}\" | ${shell_bin_path}/${shell_bin} --host 127.0.0.1 --port 8081 ${cmd} > ${tmpfile_path}/shell-stdout.txt"
+    tmp="echo \"${params}\" | ${shell_bin_path}/${shell_bin} --host \
+ 127.0.0.1 --port 8081 ${cmd} > ${tmpfile_path}/shell-stdout.txt"
     echo "Executing kea-shell ($tmp)"
     
-    echo "${params}" | ${shell_bin_path}/${shell_bin} --host 127.0.0.1 --port 8081 ${cmd} > ${tmpfile_path}/shell-stdout.txt
+    echo "${params}" | ${shell_bin_path}/${shell_bin} --host 127.0.0.1 \
+ --port 8081 ${cmd} > ${tmpfile_path}/shell-stdout.txt
 
     # Check the exit code
     shell_exit_code=$?
     if [ ${shell_exit_code} -ne 0 ]; then
-        echo "ERROR: kea-shell returned ${shell_exit_code} exit code, expected 0."
+        echo "ERROR:" \
+	"kea-shell returned ${shell_exit_code} exit code,  expected 0."
     else
         echo "kea-shell returned ${shell_exit_code} exit code as expected."
     fi
@@ -118,7 +122,9 @@ shell_command_test() {
     diff ${tmpfile_path}/shell-stdout.txt ${tmpfile_path}/shell-expected.txt
     diff_code=$?