Commit 4303fa38 authored by Ondřej Surý's avatar Ondřej Surý

Merge branch '1232-stats-channel-zone-timers' into 'master'

Resolve "[ISC-support #15166] expose zone timers (reload, refresh, expire)  via stats channel"

Closes #1232

See merge request !3308
parents b112bcd6 ce6cb628
Pipeline #41462 passed with stages
in 8 minutes and 42 seconds
5407. [func] The zone timers are now exported to the statistics
channel. Thanks to Paul Frieden, Verizon Media.
[GL #1232]
5406. [func] Added a new logging category "rpz-passthru". It allows
RPZ passthru actions to be logged into a separate
channel. [GL #54]
......
......@@ -775,7 +775,7 @@
<xsl:for-each select="views/view">
<h3>Zones for View <xsl:value-of select="@name"/></h3>
<table class="zones">
<thead><tr><th>Name</th><th>Class</th><th>Type</th><th>Serial</th></tr></thead>
<thead><tr><th>Name</th><th>Class</th><th>Type</th><th>Serial</th><th>Loaded</th><th>Expires</th><th>Refresh</th></tr></thead>
<tbody>
<xsl:for-each select="zones/zone">
<xsl:variable name="css-class15">
......@@ -788,7 +788,10 @@
<td><xsl:value-of select="@name"/></td>
<td><xsl:value-of select="@rdataclass"/></td>
<td><xsl:value-of select="type"/></td>
<td><xsl:value-of select="serial"/></td></tr>
<td><xsl:value-of select="serial"/></td>
<td><xsl:value-of select="loaded"/></td>
<td><xsl:value-of select="expires"/></td>
<td><xsl:value-of select="refresh"/></td></tr>
</xsl:for-each>
</tbody>
</table>
......
......@@ -1809,6 +1809,43 @@ zone_xmlrender(dns_zone_t *zone, void *arg) {
}
TRY0(xmlTextWriterEndElement(writer)); /* serial */
/*
* Export zone timers to the statistics channel in XML format. For
* master zones, only include the loaded time. For slave zones, also
* include the expires and refresh times.
*/
isc_time_t timestamp;
result = dns_zone_getloadtime(zone, &timestamp);
if (result != ISC_R_SUCCESS) {
goto error;
}
isc_time_formatISO8601(&timestamp, buf, 64);
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "loaded"));
TRY0(xmlTextWriterWriteString(writer, ISC_XMLCHAR buf));
TRY0(xmlTextWriterEndElement(writer));
if (dns_zone_gettype(zone) == dns_zone_slave) {
result = dns_zone_getexpiretime(zone, &timestamp);
if (result != ISC_R_SUCCESS) {
goto error;
}
isc_time_formatISO8601(&timestamp, buf, 64);
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "expires"));
TRY0(xmlTextWriterWriteString(writer, ISC_XMLCHAR buf));
TRY0(xmlTextWriterEndElement(writer));
result = dns_zone_getrefreshtime(zone, &timestamp);
if (result != ISC_R_SUCCESS) {
goto error;
}
isc_time_formatISO8601(&timestamp, buf, 64);
TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "refresh"));
TRY0(xmlTextWriterWriteString(writer, ISC_XMLCHAR buf));
TRY0(xmlTextWriterEndElement(writer));
}
if (statlevel == dns_zonestat_full) {
isc_stats_t *zonestats;
isc_stats_t *gluecachestats;
......@@ -2619,6 +2656,40 @@ zone_jsonrender(dns_zone_t *zone, void *arg) {
return (ISC_R_NOMEMORY);
}
/*
* Export zone timers to the statistics channel in JSON format. For
* master zones, only include the loaded time. For slave zones, also
* include the expires and refresh times.
*/
isc_time_t timestamp;
result = dns_zone_getloadtime(zone, &timestamp);
if (result != ISC_R_SUCCESS) {
goto error;
}
isc_time_formatISO8601(&timestamp, buf, 64);
json_object_object_add(zoneobj, "loaded", json_object_new_string(buf));
if (dns_zone_gettype(zone) == dns_zone_slave) {
result = dns_zone_getexpiretime(zone, &timestamp);
if (result != ISC_R_SUCCESS) {
goto error;
}
isc_time_formatISO8601(&timestamp, buf, 64);
json_object_object_add(zoneobj, "expires",
json_object_new_string(buf));
result = dns_zone_getrefreshtime(zone, &timestamp);
if (result != ISC_R_SUCCESS) {
goto error;
}
isc_time_formatISO8601(&timestamp, buf, 64);
json_object_object_add(zoneobj, "refresh",
json_object_new_string(buf));
}
if (statlevel == dns_zonestat_full) {
isc_stats_t *zonestats;
isc_stats_t *gluecachestats;
......
......@@ -185,10 +185,8 @@ fi
# Clean up files left from any potential previous runs
if test -f "$systest/clean.sh"
then
( cd "${systest}" && $SHELL clean.sh "$@" )
ret=$?
if [ $ret -ne 0 ]; then
echowarn "I:$systest:clean.sh script failed with $ret"
if ! ( cd "${systest}" && $SHELL clean.sh "$@" ); then
echowarn "I:$systest:clean.sh script failed"
fi
fi
......@@ -196,10 +194,8 @@ fi
# Set up any dynamically generated test data
if test -f "$systest/setup.sh"
then
( cd "${systest}" && $SHELL setup.sh "$@" )
ret=$?
if [ $ret -ne 0 ]; then
echowarn "I:$systest:clean.sh script failed with $ret"
if ! ( cd "${systest}" && $SHELL setup.sh "$@" ); then
echowarn "I:$systest:clean.sh script failed"
fi
fi
......
......@@ -12,9 +12,9 @@
rm -f traffic traffic.out.* traffic.json.* traffic.xml.*
rm -f zones zones.out.* zones.json.* zones.xml.* zones.expect.*
rm -f dig.out*
rm -f */named.memstats
rm -f */named.conf
rm -f */named.run*
rm -f ns*/named.memstats
rm -f ns*/named.conf
rm -f ns*/named.run*
rm -f ns*/named.lock
rm -f ns*/named.stats
rm -f xml.*stats json.*stats
......@@ -24,4 +24,6 @@ rm -f ns*/managed-keys.bind*
rm -f ns2/Kdnssec* ns2/dnssec.*.id
rm -f ns2/Kmanykeys* ns2/manykeys.*.id
rm -f ns2/*.db.signed* ns2/dsset-*. ns2/*.jbk
rm -f ns2/core
rm -f ns2/dnssec.db.signed* ns2/dsset-dnssec.
rm -f ns3/*.db
rm -rf ./.cache ./__pycache__
############################################################################
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
############################################################################
import os
import pytest
def pytest_configure(config):
config.addinivalue_line(
"markers", "requests: mark tests that need requests to function"
)
config.addinivalue_line(
"markers", "json: mark tests that need json to function"
)
config.addinivalue_line(
"markers", "xml: mark tests that need xml.etree to function"
)
config.addinivalue_line(
"markers", "dnspython: mark tests that need dnspython to function"
)
def pytest_collection_modifyitems(config, items):
# pylint: disable=unused-argument,unused-import,too-many-branches
# pylint: disable=import-outside-toplevel
# Test for requests module
skip_requests = pytest.mark.skip(
reason="need requests module to run")
try:
import requests # noqa: F401
except ModuleNotFoundError:
for item in items:
if "requests" in item.keywords:
item.add_marker(skip_requests)
# Test for json module
skip_json = pytest.mark.skip(
reason="need json module to run")
try:
import json # noqa: F401
except ModuleNotFoundError:
for item in items:
if "json" in item.keywords:
item.add_marker(skip_json)
# Test for xml module
skip_xml = pytest.mark.skip(
reason="need xml module to run")
try:
import xml.etree.ElementTree # noqa: F401
except ModuleNotFoundError:
for item in items:
if "xml" in item.keywords:
item.add_marker(skip_xml)
# Test if JSON statistics channel was enabled
no_jsonstats = pytest.mark.skip(
reason="need JSON statistics to be enabled")
if os.getenv("HAVEJSONSTATS") is None:
for item in items:
if "json" in item.keywords:
item.add_marker(no_jsonstats)
# Test if XML statistics channel was enabled
no_xmlstats = pytest.mark.skip(
reason="need XML statistics to be enabled")
if os.getenv("HAVEXMLSTATS") is None:
for item in items:
if "xml" in item.keywords:
item.add_marker(no_xmlstats)
# Test for dnspython module
skip_dnspython = pytest.mark.skip(
reason="need dnspython module to run")
try:
import dns.query # noqa: F401
except ModuleNotFoundError:
for item in items:
if "dnspython" in item.keywords:
item.add_marker(skip_dnspython)
@pytest.fixture
def statsport(request):
# pylint: disable=unused-argument
env_port = os.getenv("EXTRAPORT1")
if port is None:
env_port = 5301
else:
env_port = int(env_port)
return env_port
@pytest.fixture
def port(request):
# pylint: disable=unused-argument
env_port = os.getenv("PORT")
if port is None:
env_port = 5300
else:
env_port = int(env_port)
return env_port
############################################################################
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
############################################################################
import helper
def test_zone_timers_primary(fetch_zones, load_timers, **kwargs):
statsip = kwargs['statsip']
statsport = kwargs['statsport']
zonedir = kwargs['zonedir']
zones = fetch_zones(statsip, statsport)
for zone in zones:
(name, loaded, expires, refresh) = load_timers(zone, True)
mtime = helper.zone_mtime(zonedir, name)
helper.check_zone_timers(loaded, expires, refresh, mtime)
def test_zone_timers_secondary(fetch_zones, load_timers, **kwargs):
statsip = kwargs['statsip']
statsport = kwargs['statsport']
zonedir = kwargs['zonedir']
zones = fetch_zones(statsip, statsport)
for zone in zones:
(name, loaded, expires, refresh) = load_timers(zone, False)
mtime = helper.zone_mtime(zonedir, name)
helper.check_zone_timers(loaded, expires, refresh, mtime)
def test_zone_with_many_keys(fetch_zones, load_zone, **kwargs):
statsip = kwargs['statsip']
statsport = kwargs['statsport']
zones = fetch_zones(statsip, statsport)
for zone in zones:
name = load_zone(zone)
if name == 'manykeys':
helper.check_manykeys(name)
def test_traffic(fetch_traffic, **kwargs):
statsip = kwargs['statsip']
statsport = kwargs['statsport']
port = kwargs['port']
data = fetch_traffic(statsip, statsport)
exp = helper.create_expected(data)
msg = helper.create_msg("short.example.", "TXT")
helper.update_expected(exp, "dns-udp-requests-sizes-received-ipv4", msg)
ans = helper.udp_query(statsip, port, msg)
helper.update_expected(exp, "dns-udp-responses-sizes-sent-ipv4", ans)
data = fetch_traffic(statsip, statsport)
helper.check_traffic(data, exp)
msg = helper.create_msg("long.example.", "TXT")
helper.update_expected(exp, "dns-udp-requests-sizes-received-ipv4", msg)
ans = helper.udp_query(statsip, port, msg)
helper.update_expected(exp, "dns-udp-responses-sizes-sent-ipv4", ans)
data = fetch_traffic(statsip, statsport)
helper.check_traffic(data, exp)
msg = helper.create_msg("short.example.", "TXT")
helper.update_expected(exp, "dns-tcp-requests-sizes-received-ipv4", msg)
ans = helper.tcp_query(statsip, port, msg)
helper.update_expected(exp, "dns-tcp-responses-sizes-sent-ipv4", ans)
data = fetch_traffic(statsip, statsport)
helper.check_traffic(data, exp)
msg = helper.create_msg("long.example.", "TXT")
helper.update_expected(exp, "dns-tcp-requests-sizes-received-ipv4", msg)
ans = helper.tcp_query(statsip, port, msg)
helper.update_expected(exp, "dns-tcp-responses-sizes-sent-ipv4", ans)
data = fetch_traffic(statsip, statsport)
helper.check_traffic(data, exp)
############################################################################
# Copyright (C) Internet Systems Consortium, Inc. ("ISC")
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
#
# See the COPYRIGHT file distributed with this work for additional
# information regarding copyright ownership.
############################################################################
import os
import os.path
from collections import defaultdict
from datetime import datetime, timedelta
import dns.message
import dns.query
import dns.rcode
# ISO datetime format without msec
fmt = '%Y-%m-%dT%H:%M:%SZ'
# The constants were taken from BIND 9 source code (lib/dns/zone.c)
max_refresh = timedelta(seconds=2419200) # 4 weeks
max_expires = timedelta(seconds=14515200) # 24 weeks
now = datetime.utcnow().replace(microsecond=0)
dayzero = datetime.utcfromtimestamp(0).replace(microsecond=0)
TIMEOUT = 10
# Generic helper functions
def check_expires(expires, min_time, max_time):
assert expires >= min_time
assert expires <= max_time
def check_refresh(refresh, min_time, max_time):
assert refresh >= min_time
assert refresh <= max_time
def check_loaded(loaded, expected):
# Sanity check the zone timers values
assert loaded == expected
assert loaded < now
def check_zone_timers(loaded, expires, refresh, loaded_exp):
# Sanity checks the zone timers values
if expires is not None:
check_expires(expires, now, now + max_expires)
if refresh is not None:
check_refresh(refresh, now, now + max_refresh)
check_loaded(loaded, loaded_exp)
#
# The output is gibberish, but at least make sure it does not crash.
#
def check_manykeys(name, zone=None):
# pylint: disable=unused-argument
assert name == "manykeys"
def zone_mtime(zonedir, name):
try:
si = os.stat(os.path.join(zonedir, "{}.db".format(name)))
except FileNotFoundError:
return dayzero
mtime = datetime.utcfromtimestamp(si.st_mtime).replace(microsecond=0)
return mtime
def zone_keyid(nameserver, zone, key):
with open(f'{nameserver}/{zone}.{key}.id') as f:
keyid = f.read().strip()
print(f'{zone}-{key} ID: {keyid}')
return keyid
def create_msg(qname, qtype):
msg = dns.message.make_query(qname, qtype, want_dnssec=True,
use_edns=0, payload=4096)
return msg
def udp_query(ip, port, msg):
ans = dns.query.udp(msg, ip, TIMEOUT, port=port)
assert ans.rcode() == dns.rcode.NOERROR
return ans
def tcp_query(ip, port, msg):
ans = dns.query.tcp(msg, ip, TIMEOUT, port=port)
assert ans.rcode() == dns.rcode.NOERROR
return ans
def create_expected(data):
expected = {"dns-tcp-requests-sizes-received-ipv4": defaultdict(int),
"dns-tcp-responses-sizes-sent-ipv4": defaultdict(int),
"dns-tcp-requests-sizes-received-ipv6": defaultdict(int),
"dns-tcp-responses-sizes-sent-ipv6": defaultdict(int),
"dns-udp-requests-sizes-received-ipv4": defaultdict(int),
"dns-udp-requests-sizes-received-ipv6": defaultdict(int),
"dns-udp-responses-sizes-sent-ipv4": defaultdict(int),
"dns-udp-responses-sizes-sent-ipv6": defaultdict(int),
}
for k, v in data.items():
for kk, vv in v.items():
expected[k][kk] += vv
return expected
def update_expected(expected, key, msg):
msg_len = len(msg.to_wire())
bucket_num = (msg_len // 16) * 16
bucket = "{}-{}".format(bucket_num, bucket_num + 15)
expected[key][bucket] += 1
def check_traffic(data, expected):
def ordered(obj):
if isinstance(obj, dict):
return sorted((k, ordered(v)) for k, v in obj.items())
if isinstance(obj, list):
return sorted(ordered(x) for x in obj)
return obj
ordered_data = ordered(data)
ordered_expected = ordered(expected)
assert len(ordered_data) == 8
assert len(ordered_expected) == 8
assert len(data) == len(ordered_data)
assert len(expected) == len(ordered_expected)
assert ordered_data == ordered_expected
; Copyright (C) Internet Systems Consortium, Inc. ("ISC")
;
; This Source Code Form is subject to the terms of the Mozilla Public
; License, v. 2.0. If a copy of the MPL was not distributed with this
; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;
; See the COPYRIGHT file distributed with this work for additional
; information regarding copyright ownership.
$ORIGIN .
$TTL 300 ; 5 minutes
example IN SOA mname1. . (
1 ; serial
20 ; refresh (20 seconds)
20 ; retry (20 seconds)
1814400 ; expire (3 weeks)
3600 ; minimum (1 hour)
)
example. NS ns2.example.
ns2.example. A 10.53.0.2
$ORIGIN example.
a A 10.0.0.1
MX 10 mail.example.
short TXT "short text"
long TXT (
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
"longlonglonglonglonglonglonglonglonglong"
)
mail A 10.0.0.2
/*
* Copyright (C) Internet Systems Consortium, Inc. ("ISC")
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*
* See the COPYRIGHT file distributed with this work for additional
* information regarding copyright ownership.
*/
options {
query-source address 10.53.0.1;
notify-source 10.53.0.1;
transfer-source 10.53.0.1;
port @PORT@;
pid-file "named.pid";
listen-on { 10.53.0.1; };
listen-on-v6 { none; };
recursion no;
notify explicit;
minimal-responses no;
version none; // make statistics independent of the version number
};
statistics-channels { inet 10.53.0.1 port @EXTRAPORT1@ allow { localhost; }; };
key rndc_key {