Added TCP high-water system tests

Note: ans6/ans6.py is a helper script that allows tests.sh to open/close
TCP connections to some BIND instance.

bin/tests/system/tcp/ans6/ans.py

+############################################################################
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+############################################################################
+
+############################################################################
+#
+# This tool allows an arbitrary number of TCP connections to be made to the
+# specified service and to keep them open until told otherwise.  It is
+# controlled by writing text commands to a TCP socket (default port: 5309).
+#
+# Currently supported commands:
+#
+#   - open <COUNT> <HOST> <PORT>
+#
+#     Opens <COUNT> TCP connections to <HOST>:<PORT> and keeps them open.
+#     <HOST> must be an IP address (IPv4 or IPv6).
+#
+#   - close <COUNT>
+#
+#     Close the oldest <COUNT> previously established connections.
+#
+############################################################################
+
+from __future__ import print_function
+
+import datetime
+import errno
+import os
+import select
+import signal
+import socket
+import sys
+import time
+
+
+# Timeout for establishing all connections requested by a single 'open' command.
+OPEN_TIMEOUT = 2
+
+
+def log(msg):
+    print(datetime.datetime.now().strftime('%d-%b-%Y %H:%M:%S.%f ') + msg)
+
+
+def open_connections(active_conns, count, host, port):
+    queued = []
+    errors = []
+
+    try:
+        socket.inet_aton(host)
+        family = socket.AF_INET
+    except socket.error:
+        family = socket.AF_INET6
+
+    log('Opening %d connections...' % count)
+
+    for _ in range(count):
+        sock = socket.socket(family, socket.SOCK_STREAM)
+        sock.setblocking(0)
+        err = sock.connect_ex((host, port))
+        if err not in (0, errno.EINPROGRESS):
+            log('%s on connect for socket %s' % (errno.errorcode[err], sock))
+            errors.append(sock)
+        else:
+            queued.append(sock)
+
+    start = time.time()
+    while queued:
+        now = time.time()
+        time_left = OPEN_TIMEOUT - (now - start)
+        if time_left <= 0:
+            break
+        _, wsocks, _ = select.select([], queued, [], time_left)
+        for sock in wsocks:
+            queued.remove(sock)
+            err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
+            if err:
+                log('%s for socket %s' % (errno.errorcode[err], sock))
+                errors.append(sock)
+            else:
+                active_conns.append(sock)
+
+    if errors:
+        log('result=FAIL: %d connection(s) failed' % len(errors))
+    elif queued:
+        log('result=FAIL: Timed out, aborting %d pending connections' % len(queued))
+        for sock in queued:
+            sock.close()
+    else:
+        log('result=OK: Successfully opened %d connections' % count)
+
+
+def close_connections(active_conns, count):
+    log('Closing %d connections...' % count)
+    for _ in range(count):
+        sock = active_conns.pop(0)
+        sock.close()
+    log('result=OK: Successfully closed %d connections' % count)
+
+
+def sigterm(*_):
+    log('SIGTERM received, shutting down')
+    os.remove('ans.pid')
+    sys.exit(0)
+
+
+def main():
+    active_conns = []
+
+    signal.signal(signal.SIGTERM, sigterm)
+
+    with open('ans.pid', 'w') as pidfile:
+        print(os.getpid(), file=pidfile)
+
+    listenip = '10.53.0.6'
+    try:
+        port = int(os.environ['CONTROLPORT'])
+    except KeyError:
+        port = 5309
+
+    log('Listening on %s:%d' % (listenip, port))
+
+    ctlsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    ctlsock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    ctlsock.bind((listenip, port))
+    ctlsock.listen(1)
+
+    while True:
+        (clientsock, _) = ctlsock.accept()
+        log('Accepted control connection from %s' % clientsock)
+        cmdline = clientsock.recv(512).decode('ascii').strip()
+        if cmdline:
+            log('Received command: %s' % cmdline)
+            cmd = cmdline.split()
+            if cmd[0] == 'open':
+                count, host, port = cmd[1:]
+                open_connections(active_conns, int(count), host, int(port))
+            elif cmd[0] == 'close':
+                (count, ) = cmd[1:]
+                close_connections(active_conns, int(count))
+            else:
+                log('result=FAIL: Unknown command')
+        clientsock.close()
+
+
+if __name__ == '__main__':
+    main()

bin/tests/system/tcp/clean.sh

@@ -13,6 +13,8 @@ rm -f */named.memstats
 rm -f */named.run
 rm -f */named.conf
 rm -f */named.stats
+rm -f ans6/ans.run*
 rm -f dig.out*
+rm -f rndc.out*
 rm -f ns*/named.lock
 rm -f ns*/managed-keys.bind*

bin/tests/system/tcp/ns5/named.conf.in

+/*
+ * Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * See the COPYRIGHT file distributed with this work for additional
+ * information regarding copyright ownership.
+ */
+
+// NS5
+
+key rndc_key {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+controls {
+	inet 10.53.0.5 port @CONTROLPORT@ allow { any; } keys { rndc_key; };
+};
+
+
+options {
+	query-source address 10.53.0.5;
+	notify-source 10.53.0.5;
+	transfer-source 10.53.0.5;
+	port @PORT@;
+	directory ".";
+	pid-file "named.pid";
+	listen-on { 10.53.0.5; };
+	listen-on-v6 { none; };
+        tcp-listen-queue 32;
+	recursion yes;
+	notify yes;
+        tcp-clients 17;
+	dnssec-validation no;
+};
+
+zone "." {
+	type hint;
+	file "../../common/root.hint";
+};

bin/tests/system/tcp/prereq.sh

+#!/bin/sh
+#
+# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# See the COPYRIGHT file distributed with this work for additional
+# information regarding copyright ownership.
+
+SYSTEMTESTTOP=..
+. $SYSTEMTESTTOP/conf.sh
+
+if ! test -n "$PYTHON"; then
+	echo_i "This test requires Python."
+	exit 1
+fi
+

bin/tests/system/tcp/setup.sh

@@ -18,3 +18,4 @@ copy_setports ns1/named.conf.in ns1/named.conf
 copy_setports ns2/named.conf.in ns2/named.conf
 copy_setports ns3/named.conf.in ns3/named.conf
 copy_setports ns4/named.conf.in ns4/named.conf
+copy_setports ns5/named.conf.in ns5/named.conf

bin/tests/system/tcp/tests.sh

@@ -14,6 +14,7 @@ SYSTEMTESTTOP=..
 
 DIGOPTS="-p ${PORT}"
 RNDCCMD="$RNDC -p ${CONTROLPORT} -c ../common/rndc.conf"
+SEND="$PERL $SYSTEMTESTTOP/send.pl 10.53.0.6 ${CONTROLPORT}"
 
 status=0
 
@@ -55,5 +56,94 @@ if [ "$ntcp21" -ge "$ntcp22" ];then ret=1; fi
 if [ $ret != 0 ]; then echo_i "failed"; fi
 status=`expr $status + $ret`
 
+# -------- TCP high-water tests ----------
+n=0
+
+refresh_tcp_stats() {
+	$RNDCCMD -s 10.53.0.5 status > rndc.out.$n || ret=1
+	TCP_CUR="$(sed -n "s/^tcp clients: \([0-9][0-9]*\).*/\1/p" rndc.out.$n)"
+	TCP_LIMIT="$(sed -n "s/^tcp clients: .*\/\([0-9][0-9]*\)/\1/p" rndc.out.$n)"
+	TCP_HIGH="$(sed -n "s/^TCP high-water: \([0-9][0-9]*\)/\1/p" rndc.out.$n)"
+}
+
+wait_for_log() {
+	msg=$1
+	file=$2
+	for i in 1 2 3 4 5 6 7 8 9 10; do
+		nextpart "$file" | grep "$msg" > /dev/null && return
+		sleep 1
+	done
+	echo_i "exceeded time limit waiting for '$msg' in $file"
+	ret=1
+}
+
+# Send a command to the tool script listening on 10.53.0.6.
+send_command() {
+	nextpart ans6/ans.run > /dev/null
+	echo "$*" | $SEND
+	wait_for_log "result=OK" ans6/ans.run
+}
+
+# Instructs ans6 to open $1 TCP connections to 10.53.0.5.
+open_connections() {
+	send_command "open" "${1}" 10.53.0.5 "${PORT}"
+}
+
+# Instructs ans6 to close $1 TCP connections to 10.53.0.5.
+close_connections() {
+	send_command "close" "${1}"
+}
+
+# Check TCP statistics after server startup before using them as a baseline for
+# subsequent checks.
+n=$((n + 1))
+echo_i "TCP high-water: check initial statistics ($n)"
+ret=0
+refresh_tcp_stats
+assert_int_equal "${TCP_CUR}" 1 "current TCP clients count"
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+# Ensure the TCP high-water statistic gets updated after some TCP connections
+# are established.
+n=$((n + 1))
+echo_i "TCP high-water: check value after some TCP connections are established ($n)"
+ret=0
+OLD_TCP_CUR="${TCP_CUR}"
+TCP_ADDED=9
+open_connections "${TCP_ADDED}"
+refresh_tcp_stats
+assert_int_equal "${TCP_CUR}" $((OLD_TCP_CUR + TCP_ADDED)) "current TCP clients count"
+assert_int_equal "${TCP_HIGH}" $((OLD_TCP_CUR + TCP_ADDED)) "TCP high-water value"
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+# Ensure the TCP high-water statistic remains unchanged after some TCP
+# connections are closed.
+n=$((n + 1))
+echo_i "TCP high-water: check value after some TCP connections are closed ($n)"
+ret=0
+OLD_TCP_CUR="${TCP_CUR}"
+OLD_TCP_HIGH="${TCP_HIGH}"
+TCP_REMOVED=5
+close_connections "${TCP_REMOVED}"
+refresh_tcp_stats
+assert_int_equal "${TCP_CUR}" $((OLD_TCP_CUR - TCP_REMOVED)) "current TCP clients count"
+assert_int_equal "${TCP_HIGH}" "${OLD_TCP_HIGH}" "TCP high-water value"
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
+# Ensure the TCP high-water statistic never exceeds the configured TCP clients
+# limit.
+n=$((n + 1))
+echo_i "TCP high-water: ensure tcp-clients is an upper bound ($n)"
+ret=0
+open_connections $((TCP_LIMIT + 1))
+refresh_tcp_stats
+assert_int_equal "${TCP_CUR}" "${TCP_LIMIT}" "current TCP clients count"
+assert_int_equal "${TCP_HIGH}" "${TCP_LIMIT}" "TCP high-water value"
+if [ $ret != 0 ]; then echo_i "failed"; fi
+status=`expr $status + $ret`
+
 echo_i "exit status: $status"
 [ $status -eq 0 ] || exit 1

util/copyrights

@@ -1077,7 +1077,9 @@
 ./bin/tests/system/synthfromdnssec/setup.sh	SH	2017,2018,2019
 ./bin/tests/system/synthfromdnssec/tests.sh	SH	2017,2018,2019
 ./bin/tests/system/system-test-driver.sh	X	2019
+./bin/tests/system/tcp/ans6/ans.py		PYTHON	2019
 ./bin/tests/system/tcp/clean.sh			SH	2014,2016,2018,2019
+./bin/tests/system/tcp/prereq.sh		SH	2019
 ./bin/tests/system/tcp/setup.sh			SH	2018,2019
 ./bin/tests/system/tcp/tests.sh			SH	2014,2016,2018,2019
 ./bin/tests/system/testcrypto.sh		SH	2014,2016,2017,2018,2019