9.16.6 exited with assertion failure after minor in-flight configuration change
Summary
10s after a minor configuration change, BIND exited with an assertion failure. After a restart, it has continued working as normal.
BIND version used
BIND 9.16.6 (Stable Release) <id:25846cf>
running on NetBSD amd64 9.0_RC1 NetBSD 9.0_RC1 (GENERIC) #0: Sat Dec 14 12:36:33 UTC 2019 mkrepro@mkrepro.NetBSD.org:/usr/src/sys/arch/amd64/compile/GENERIC
built by make with '--with-libxml2=yes' '--with-tuning=large' '--enable-dnstap' '--with-protobuf-c=/usr/pkg' '--with-libfstrm=/usr/pkg' '--sysconfdir=/etc' '--localstatedir=/var'
compiled by GCC 7.4.0
compiled with OpenSSL version: OpenSSL 1.1.1c 28 May 2019
linked to OpenSSL version: OpenSSL 1.1.1c 28 May 2019
compiled with libuv version: 1.38.0
linked to libuv version: 1.38.0
compiled with libxml2 version: 2.9.10
linked to libxml2 version: 20910
compiled with zlib version: 1.2.10
linked to zlib version: 1.2.10
compiled with protobuf-c version: 1.3.2
linked to protobuf-c version: 1.3.2
threads support is enabled
default paths:
named configuration: /etc/named.conf
rndc configuration: /etc/rndc.conf
DNSSEC root key: /etc/bind.keys
nsupdate session key: /var/run/named/session.key
named PID file: /var/run/named/named.pid
named lock file: /var/run/named/named.lock
Steps to reproduce
The configuration changes introduced were:
--- named.conf 2020/10/02 10:02:54 1.23
+++ named.conf 2020/10/02 10:06:18 1.24
@@ -1,7 +1,6 @@
options {
directory "/etc/namedb";
- dnssec-enable yes;
dnssec-validation yes;
managed-keys-directory "keys";
@@ -17,6 +16,10 @@
// minimization for now. May be related to forwarding...
//qname-minimization off;
+ // Be nice, conform to DNS flag day 2020
+ edns-udp-size 1232;
+ max-udp-size 1232;
+
// Force these in preparation for anycast addresses
// which we never want to use as query source
query-source address 158.37.2.68;
@@ -82,7 +85,7 @@
};
};
-managed-keys {
+trust-anchors {
"." initial-key 257 3 8
"AwEAAagAIKlVZrpC6Ia7gEzahOR+9W29euxhJhVVLOyQbSEW0O8gcCjF
FVQUTf6v58fLjwBd0YI0EzrAcQqBGCzh/RStIoO8g0NfnfL2MTJRkxoX
What is the current bug behavior?
Oct 2 12:06:07 tos-res named[6701]: reloading configuration succeeded
Oct 2 12:06:07 tos-res named[6701]: scheduled loading new zones
Oct 2 12:06:07 tos-res named[6701]: any newly configured zones are now loaded
Oct 2 12:06:07 tos-res named[6701]: running
Oct 2 12:06:17 tos-res named[6701]: resolver.c:10193: INSIST(((res->dbuckets[i].list).head == ((void *)0))) failed, back trace
Oct 2 12:06:17 tos-res named[6701]: #0 0x434978 in assertion_failed()+0x4d
Oct 2 12:06:17 tos-res named[6701]: #1 0x5edd88 in isc_assertion_failed()+0xa
Oct 2 12:06:17 tos-res named[6701]: #2 0x550e33 in dns_resolver_detach()+0x501
Oct 2 12:06:17 tos-res named[6701]: #3 0x5896cf in destroy()+0x129
Oct 2 12:06:17 tos-res named[6701]: #4 0x58a427 in adb_shutdown()+0x52
Oct 2 12:06:17 tos-res named[6701]: #5 0x610f77 in run()+0x6b2
Oct 2 12:06:17 tos-res named[6701]: #6 0x72753c20c1d8 in _fini()+0x72753bbc5778
Oct 2 12:06:17 tos-res named[6701]: #7 0x72753bc87af0 in _fini()+0x72753b641090
Oct 2 12:06:17 tos-res named[6701]: exiting (due to assertion failure)
What is the expected correct behavior?
BIND should have continued working as normal. I don't know, it may be coincidental, but 10s is "too close for comfort". Besides, that BIND continues working now after a full restart tends to indicate that the problem was the in-flight configuration change and not the configuration change itself.
Relevant configuration files
named-checkconf -px
output follows:
logging {
channel "normal" {
syslog "local2";
severity dynamic;
};
channel "trash" {
syslog "local3";
severity dynamic;
};
channel "security" {
syslog "local4";
severity dynamic;
};
channel "qerrs" {
syslog "local1";
severity dynamic;
};
channel "queries" {
syslog "local0";
severity dynamic;
};
channel "client_log" {
file "/var/log/client.log" versions 30 size 10485760;
severity dynamic;
print-time yes;
};
channel "rpzlog" {
file "/var/log/named.rpz" versions 50 size 10485760;
severity info;
print-time yes;
print-severity yes;
print-category yes;
};
channel "null" {
null ;
};
category "default" {
"normal";
"default_debug";
};
category "general" {
"normal";
"default_debug";
};
category "config" {
"normal";
"default_debug";
};
category "network" {
"normal";
"default_debug";
};
category "notify" {
"normal";
"default_debug";
};
category "xfer-in" {
"normal";
"default_debug";
};
category "xfer-out" {
"normal";
"default_debug";
};
category "dnssec" {
"security";
};
category "security" {
"security";
};
category "rpz" {
"rpzlog";
};
category "database" {
"null";
};
category "lame-servers" {
"null";
};
category "update-security" {
"null";
};
category "update" {
"null";
};
category "query-errors" {
"qerrs";
};
category "queries" {
"queries";
};
category "client" {
"client_log";
};
};
options {
datasize 8589934592;
directory "/etc/namedb";
dnstap-output unix"/var/run/named/dnstap.sock";
hostname "tos-res.uninett.no";
listen-on {
"any";
};
listen-on-v6 {
"any";
};
managed-keys-directory "keys";
querylog no;
server-id "tos-res.uninett.no";
dnssec-validation yes;
dnstap {
client query;
};
edns-udp-size 1232;
max-udp-size 1232;
qname-minimization relaxed;
query-source address 158.37.2.68 port 0;
query-source-v6 address 2001:700:0:804f::ca53 port 0;
recursion yes;
response-policy {
zone "dns-rpz.uninett.no";
zone "zone3.ph.rpz.switch.ch" policy disabled;
zone "zone3.mw.rpz.switch.ch" policy disabled;
zone "zone3.misc.rpz.switch.ch" policy disabled;
} break-dnssec yes;
allow-query {
"localnets";
78.91.0.0/16;
128.39.0.0/16;
129.177.0.0/16;
129.240.0.0/15;
129.242.0.0/16;
144.164.0.0/16;
151.157.0.0/16;
152.94.0.0/16;
156.116.0.0/16;
157.249.0.0/16;
158.36.0.0/14;
161.4.0.0/16;
192.111.33.0/24;
192.133.32.0/24;
192.146.238.0/23;
193.156.0.0/15;
2001:700::/32;
146.172.4.0/23;
148.122.20.52/31;
148.123.37.165/32;
2001:67c:29f4::/48;
44.141.124.0/24;
44.141.132.0/24;
193.35.52.0/22;
};
forward first;
forwarders {
158.38.0.168;
128.39.2.24;
};
};
statistics-channels {
inet 127.0.0.1 port 8053 allow {
127.0.0.1/32;
};
inet 158.37.2.68 port 8053 allow {
158.38.62.0/23;
158.38.10.0/24;
};
};
server 54.209.136.173/32 {
send-cookie no;
};
server 204.153.45.2/32 {
send-cookie no;
};
trust-anchors {
"." initial-key 257 3 8 "AwEAAagAIKlVZrpC6Ia7gEzahOR+9W29euxhJhVVLOyQbSEW0O8gcCjF
FVQUTf6v58fLjwBd0YI0EzrAcQqBGCzh/RStIoO8g0NfnfL2MTJRkxoX
bfDaUeVPQuYEhg37NZWAJQ9VnMVDxP/VHL496M/QZxkjf5/Efucp2gaD
X6RS6CXpoY68LsvPVjR0ZSwzz1apAzvN9dlzEheX7ICJBBtuA6G3LQpz
W5hOA2hzCTMjJPJ8LbqF6dsV6DoBQzgul0sGIcGOYl7OyQdXfZ57relS
Qageu+ipAdTTJ25AsRTAoub8ONGcLmqrAmRLKBP1dfwhYB4N7knNnulq
QxA+Uk1ihz0=";
"." initial-key 257 3 8 "AwEAAaz/tAm8yTn4Mfeh5eyI96WSVexTBAvkMgJzkKTOiW1vkIbzxeF3
+/4RgWOq7HrxRixHlFlExOLAJr5emLvN7SWXgnLh4+B5xQlNVz8Og8kv
ArMtNROxVQuCaSnIDdD5LKyWbRd2n9WGe2R8PzgCmr3EgVLrjyBxWezF
0jLHwVN8efS3rCj/EWgvIWgb9tarpVUDK/b58Da+sqqls3eNbuv7pr+e
oZG+SrDK6nWeL3c6H5Apxz7LjVc1uTIdsIXxuOLYA4/ilBmSVIzuDWfd
RUfhHdY6+cn8HFRm+2hM8AnXGXws9555KrUB5qihylGa8subX2Nn6UwN
R1AkUTV74bU=";
"7.4.nrenum.net" initial-key 257 3 8 "AwEAAdyLRICD7vMGdRG+uwF9176xm5u+E22zJehX7luBrY8LeUsw0aT9
WxBe2aKYSoBbAROVcuQJ/8EbbL+XhX5RKieRZFLDS1hQc+BpLY4Vse5G
2OeWYbH9lWEUM6/XErTsUikYfchXxWg6PkidN/howfNmo7iHDgeG/Xfz
E+i2MLZHCCnNND6v2DE8aP4qYzmU/jEc7n4814z2HR1dzpK/eXZwY3Tv
MjnTh3cqayi8b2B7+tedwV874plFOtMdTwywnMnXf1R3C3HBIZXHu55F
Ptd7cMbikW0lEc7BRRYL50knDMk7jcnsnA7MI1hOu3vI1cNAUWM+CmWX
DXShJKcLF0s=";
};
zone "." {
type hint;
file "root.cache";
};
zone "localhost" {
type master;
file "localhost";
};
zone "127.IN-ADDR.ARPA" {
type master;
file "127";
};
zone "1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" {
type master;
file "loopback.v6";
};
zone "dns-rpz.uninett.no" {
type slave;
file "sz/dns-rpz.uninett.no";
masters {
158.38.212.119;
};
};
zone "zone3.ph.rpz.switch.ch" {
type slave;
file "sz/zone3.ph.rpz.switch.ch";
masters {
158.38.212.119;
};
};
zone "zone3.mw.rpz.switch.ch" {
type slave;
file "sz/zone3.mw.rpz.switch.ch";
masters {
158.38.212.119;
};
};
zone "zone3.misc.rpz.switch.ch" {
type slave;
file "sz/zone3.misc.rpz.switch.ch";
masters {
158.38.212.119;
};
};
Relevant logs and/or screenshots
See above.
Possible fixes
Sorry, don't know.