Improve performance for delegation heavy answers and also general query performance (#44029)

CHANGES

+4605.	[performance]	Improve performance for delegation heavy answers
+			and also general query performance. Removes the
+			acache feature that didn't significantly improve
+			performance. Adds a glue cache. Removes
+			additional-from-cache and additional-from-auth
+			features. Enables minimal-responses by
+			default. Improves performance of compression
+			code, owner case restoration, hash function,
+			etc. Uses inline buffer implementation by
+			default. Many other performance changes and fixes.
+			[RT #44029]
+
 4604.	[bug]		Don't use ERR_load_crypto_strings() when building
 			with OpenSSL 1.1.0. [RT #45117]
 

bin/named/config.c

@@ -94,7 +94,6 @@ options {\n\
 "\
 	recursive-clients 1000;\n\
 	resolver-query-timeout 10;\n\
-	rrset-order { order random; };\n\
 #	serial-queries <obsolete>;\n\
 	serial-query-rate 20;\n\
 	server-id none;\n\
@@ -140,15 +139,13 @@ options {\n\
 #	topology <none>\n\
 	auth-nxdomain false;\n\
 	minimal-any false;\n\
-	minimal-responses false;\n\
+	minimal-responses true;\n\
 	recursion true;\n\
 	provide-ixfr true;\n\
 	request-ixfr true;\n\
 	request-expire true;\n\
 #	fetch-glue <obsolete>;\n\
 #	rfc2308-type1 <obsolete>;\n\
-	additional-from-auth true;\n\
-	additional-from-cache true;\n\
 	query-source address *;\n\
 	query-source-v6 address *;\n\
 	notify-source *;\n\
@@ -167,9 +164,6 @@ options {\n\
 	check-dup-records warn;\n\
 	check-mx warn;\n\
 	check-spf warn;\n\
-	acache-enable no;\n\
-	acache-cleaning-interval 60;\n\
-	max-acache-size 16M;\n\
 	dnssec-enable yes;\n\
 	dnssec-validation yes; \n\
 	dnssec-accept-expired no;\n\

bin/named/include/named/server.h

@@ -108,8 +108,6 @@ struct ns_server {
 	unsigned int		dispatchgen;
 	ns_dispatchlist_t	dispatches;
 
-	dns_acache_t		*acache;
-
 	ns_statschannellist_t	statschannels;
 
 	dns_tsigkey_t		*sessionkey;

bin/named/server.c

@@ -52,7 +52,6 @@
 
 #include <bind9/check.h>
 
-#include <dns/acache.h>
 #include <dns/adb.h>
 #include <dns/badcache.h>
 #include <dns/cache.h>
@@ -1279,12 +1278,14 @@ configure_order(dns_order_t *order, const cfg_obj_t *ent) {
 #if DNS_RDATASET_FIXED
 		mode = DNS_RDATASETATTR_FIXEDORDER;
 #else
-		mode = 0;
+		mode = DNS_RDATASETATTR_CYCLIC;
 #endif /* DNS_RDATASET_FIXED */
 	else if (!strcasecmp(str, "random"))
 		mode = DNS_RDATASETATTR_RANDOMIZE;
 	else if (!strcasecmp(str, "cyclic"))
-		mode = 0;
+		mode = DNS_RDATASETATTR_CYCLIC;
+	else if (!strcasecmp(str, "none"))
+		mode = DNS_RDATASETATTR_NONE;
 	else
 		INSIST(0);
 
@@ -2517,8 +2518,6 @@ configure_catz_zone(dns_view_t *view, const cfg_obj_t *config,
 			RUNTIME_CHECK(tresult == ISC_R_SUCCESS);
 
 			dns_zone_setview(dnszone, view);
-			if (view->acache != NULL)
-				dns_zone_setacache(dnszone, view->acache);
 			dns_view_addzone(view, dnszone);
 		}
 
@@ -3310,7 +3309,6 @@ configure_view(dns_view_t *view, dns_viewlist_t *viewlist,
 	unsigned int cleaning_interval;
 	size_t max_cache_size;
 	isc_uint32_t max_cache_size_percent = 0;
-	size_t max_acache_size;
 	size_t max_adb_size;
 	isc_uint32_t lame_ttl, fail_ttl;
 	dns_tsig_keyring_t *ring = NULL;
@@ -3377,53 +3375,6 @@ configure_view(dns_view_t *view, dns_viewlist_t *viewlist,
 	CHECKM(ns_config_getport(config, &port), "port");
 	dns_view_setdstport(view, port);
 
-	/*
-	 * Create additional cache for this view and zones under the view
-	 * if explicitly enabled.
-	 * XXX950 default to on.
-	 */
-	obj = NULL;
-	(void)ns_config_get(maps, "acache-enable", &obj);
-	if (obj != NULL && cfg_obj_asboolean(obj)) {
-		cmctx = NULL;
-		CHECK(isc_mem_create(0, 0, &cmctx));
-		CHECK(dns_acache_create(&view->acache, cmctx, ns_g_taskmgr,
-					ns_g_timermgr));
-		isc_mem_setname(cmctx, "acache", NULL);
-		isc_mem_detach(&cmctx);
-	}
-	if (view->acache != NULL) {
-		obj = NULL;
-		result = ns_config_get(maps, "acache-cleaning-interval", &obj);
-		INSIST(result == ISC_R_SUCCESS);
-		dns_acache_setcleaninginterval(view->acache,
-					       cfg_obj_asuint32(obj) * 60);
-
-		obj = NULL;
-		result = ns_config_get(maps, "max-acache-size", &obj);
-		INSIST(result == ISC_R_SUCCESS);
-		if (cfg_obj_isstring(obj)) {
-			str = cfg_obj_asstring(obj);
-			INSIST(strcasecmp(str, "unlimited") == 0);
-			max_acache_size = 0;
-		} else {
-			isc_resourcevalue_t value;
-			value = cfg_obj_asuint64(obj);
-			if (value > SIZE_MAX) {
-				cfg_obj_log(obj, ns_g_lctx,
-					    ISC_LOG_WARNING,
-					    "'max-acache-size "
-					    "%" ISC_PRINT_QUADFORMAT "u' "
-					    "is too large for this "
-					    "system; reducing to %lu",
-					    value, (unsigned long)SIZE_MAX);
-				value = SIZE_MAX;
-			}
-			max_acache_size = (size_t) value;
-		}
-		dns_acache_setcachesize(view->acache, max_acache_size);
-	}
-
 	CHECK(configure_view_acl(vconfig, config, "allow-query", NULL, actx,
 				 ns_g_mctx, &view->queryacl));
 	if (view->queryacl == NULL) {
@@ -4305,32 +4256,6 @@ configure_view(dns_view_t *view, dns_viewlist_t *viewlist,
 	INSIST(result == ISC_R_SUCCESS);
 	view->trust_anchor_telemetry = cfg_obj_asboolean(obj);
 
-	/*
-	 * Set sources where additional data and CNAME/DNAME
-	 * targets for authoritative answers may be found.
-	 */
-	obj = NULL;
-	result = ns_config_get(maps, "additional-from-auth", &obj);
-	INSIST(result == ISC_R_SUCCESS);
-	view->additionalfromauth = cfg_obj_asboolean(obj);
-	if (view->recursion && ! view->additionalfromauth) {
-		cfg_obj_log(obj, ns_g_lctx, ISC_LOG_WARNING,
-			    "'additional-from-auth no' is only supported "
-			    "with 'recursion no'");
-		view->additionalfromauth = ISC_TRUE;
-	}
-
-	obj = NULL;
-	result = ns_config_get(maps, "additional-from-cache", &obj);
-	INSIST(result == ISC_R_SUCCESS);
-	view->additionalfromcache = cfg_obj_asboolean(obj);
-	if (view->recursion && ! view->additionalfromcache) {
-		cfg_obj_log(obj, ns_g_lctx, ISC_LOG_WARNING,
-			    "'additional-from-cache no' is only supported "
-			    "with 'recursion no'");
-		view->additionalfromcache = ISC_TRUE;
-	}
-
 	/*
 	 * Set "allow-query-cache", "allow-query-cache-on",
 	 * "allow-recursion", and "allow-recursion-on" acls if
@@ -5600,8 +5525,6 @@ configure_zone(const cfg_obj_t *config, const cfg_obj_t *zconfig,
 		 * new view.
 		 */
 		dns_zone_setview(zone, view);
-		if (view->acache != NULL)
-			dns_zone_setacache(zone, view->acache);
 	} else {
 		/*
 		 * We cannot reuse an existing zone, we have
@@ -5610,8 +5533,6 @@ configure_zone(const cfg_obj_t *config, const cfg_obj_t *zconfig,
 		CHECK(dns_zonemgr_createzone(ns_g_server->zonemgr, &zone));
 		CHECK(dns_zone_setorigin(zone, origin));
 		dns_zone_setview(zone, view);
-		if (view->acache != NULL)
-			dns_zone_setacache(zone, view->acache);
 		CHECK(dns_zonemgr_managezone(ns_g_server->zonemgr, zone));
 		dns_zone_setstats(zone, ns_g_server->zonestats);
 	}
@@ -5670,8 +5591,6 @@ configure_zone(const cfg_obj_t *config, const cfg_obj_t *zconfig,
 			CHECK(dns_zone_create(&raw, mctx));
 			CHECK(dns_zone_setorigin(raw, origin));
 			dns_zone_setview(raw, view);
-			if (view->acache != NULL)
-				dns_zone_setacache(raw, view->acache);
 			dns_zone_setstats(raw, ns_g_server->zonestats);
 			CHECK(dns_zone_link(zone, raw));
 		}
@@ -5768,9 +5687,6 @@ add_keydata_zone(dns_view_t *view, const char *directory, isc_mem_t *mctx) {
 
 	CHECK(dns_zonemgr_managezone(ns_g_server->zonemgr, zone));
 
-	if (view->acache != NULL)
-		dns_zone_setacache(zone, view->acache);
-
 	CHECK(dns_acl_none(mctx, &none));
 	dns_zone_setqueryacl(zone, none);
 	dns_zone_setqueryonacl(zone, none);

bin/tests/system/additional/ns1/named1.conf

@@ -13,7 +13,6 @@ options {
 	notify-source 10.53.0.1;
 	transfer-source 10.53.0.1;
 	recursion no;
-	additional-from-auth no;
 	port 5300;
 	pid-file "named.pid";
 	listen-on { 10.53.0.1; };

bin/tests/system/additional/ns1/named2.conf

@@ -13,7 +13,6 @@ options {
 	notify-source 10.53.0.1;
 	transfer-source 10.53.0.1;
 	recursion no;
-	additional-from-auth no;
 	port 5300;
 	pid-file "named.pid";
 	listen-on { 10.53.0.1; };

bin/tests/system/additional/ns1/named3.conf

@@ -11,7 +11,6 @@ options {
 	notify-source 10.53.0.1;
 	transfer-source 10.53.0.1;
 	recursion no;
-	additional-from-auth no;
 	port 5300;
 	pid-file "named.pid";
 	listen-on { 10.53.0.1; };

bin/tests/system/additional/ns1/named4.conf

@@ -11,7 +11,6 @@ options {
 	notify-source 10.53.0.1;
 	transfer-source 10.53.0.1;
 	recursion no;
-	additional-from-auth no;
 	port 5300;
 	pid-file "named.pid";
 	listen-on { 10.53.0.1; };

bin/tests/system/autosign/clean.sh

@@ -6,7 +6,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-rm -f */K* */dsset-* */*.signed */trusted.conf */tmp* */*.jnl */*.bk
+rm -f */K* */dsset-* */*.signed */tmp* */*.jnl */*.bk
+rm -f */trusted.conf */private.conf
 rm -f */core
 rm -f */example.bk
 rm -f */named.memstats

bin/tests/system/autosign/ns2/keygen.sh

@@ -31,9 +31,19 @@ $DSFROMKEY $kskname.key > dsset-${zone}$TP
 zone=private.secure.example
 zonefile="${zone}.db"
 infile="${zonefile}.in"
-cp $infile $zonefile
-$KEYGEN -3 -q -r $RANDFILE -fk $zone > /dev/null
+ksk=`$KEYGEN -3 -q -r $RANDFILE -fk $zone`
 $KEYGEN -3 -q -r $RANDFILE $zone > /dev/null
+cat $ksk.key | grep -v '^; ' | $PERL -n -e '
+local ($dn, $class, $type, $flags, $proto, $alg, @rest) = split;
+local $key = join("", @rest);
+print <<EOF
+trusted-keys {
+    "$dn" $flags $proto $alg "$key";
+};
+EOF
+' > private.conf
+cp private.conf ../ns4/private.conf
+$SIGNER -S -3 beef -A -o $zone -f $zonefile $infile > /dev/null 2>&1
 
 # Extract saved keys for the revoke-to-duplicate-key test
 zone=bar

bin/tests/system/autosign/ns4/named.conf

@@ -6,8 +6,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-/* $Id: named.conf,v 1.3 2009/11/30 23:48:02 tbox Exp $ */
-
 // NS4
 
 controls { /* empty */ };
@@ -21,7 +19,6 @@ options {
 	listen-on { 10.53.0.4; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	dnssec-enable yes;
 	dnssec-validation yes;
 	dnssec-must-be-secure mustbesecure.example yes;
@@ -33,3 +30,4 @@ zone "." {
 };
 
 include "trusted.conf";
+include "private.conf";

bin/tests/system/autosign/ns5/named.conf

@@ -21,7 +21,6 @@ options {
 	listen-on { 10.53.0.5; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	dnssec-enable yes;
 	dnssec-validation yes;
 };

bin/tests/system/autosign/tests.sh

@@ -692,8 +692,7 @@ $DIG $DIGOPTS +noauth a.private.secure.example. a @10.53.0.4 \
 	> dig.out.ns4.test$n || ret=1
 $PERL ../digcomp.pl dig.out.ns2.test$n dig.out.ns4.test$n || ret=1
 grep "NOERROR" dig.out.ns4.test$n > /dev/null || ret=1
-# Note - this is looking for failure, hence the &&
-grep "flags:.*ad.*QUERY" dig.out.ns4.test$n > /dev/null && ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns4.test$n > /dev/null || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo "I:failed"; fi
 status=`expr $status + $ret`
@@ -714,13 +713,9 @@ status=`expr $status + $ret`
 
 echo "I:checking privately secure to nxdomain works ($n)"
 ret=0
-$DIG $DIGOPTS +noauth private2secure-nxdomain.private.secure.example. SOA @10.53.0.2 \
-	> dig.out.ns2.test$n || ret=1
-$DIG $DIGOPTS +noauth private2secure-nxdomain.private.secure.example. SOA @10.53.0.4 \
-	> dig.out.ns4.test$n || ret=1
-$PERL ../digcomp.pl dig.out.ns2.test$n dig.out.ns4.test$n || ret=1
-# Note - this is looking for failure, hence the &&
-grep "flags:.*ad.*QUERY" dig.out.ns4.test$n > /dev/null && ret=1
+$DIG $DIGOPTS +noauth private2secure-nxdomain.private.secure.example. SOA @10.53.0.4 > dig.out.ns4.test$n || ret=1
+grep "NXDOMAIN" dig.out.ns4.test$n > /dev/null || ret=1
+grep "flags:.*ad.*QUERY" dig.out.ns4.test$n > /dev/null || ret=1
 n=`expr $n + 1`
 if [ $ret != 0 ]; then echo "I:failed"; fi
 status=`expr $status + $ret`

bin/tests/system/cacheclean/ns1/named.conf

@@ -6,8 +6,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-/* $Id: named.conf,v 1.11 2011/08/02 23:47:52 tbox Exp $ */
-
 controls { /* empty */ };
 
 options {
@@ -21,6 +19,7 @@ options {
 	recursion no;
 	notify yes;
 	check-integrity no;
+	minimal-responses no;
 };
 
 zone "." {

bin/tests/system/case/ns1/named.conf

@@ -20,6 +20,7 @@ options {
 	notify yes;
 	ixfr-from-differences yes;
 	check-integrity no;
+	minimal-responses no;
 };
 
 zone "example" {

bin/tests/system/case/ns2/named.conf

@@ -21,6 +21,7 @@ options {
 	ixfr-from-differences yes;
 	check-integrity no;
 	no-case-compress { 10.53.0.2; };
+	minimal-responses no;
 };
 
 zone "example" {

bin/tests/system/checknames/ns2/named.conf

@@ -19,7 +19,6 @@ options {
 	listen-on { 10.53.0.2; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	check-names response warn;
 	notify yes;
 };

bin/tests/system/checknames/ns3/named.conf

@@ -19,7 +19,6 @@ options {
 	listen-on { 10.53.0.3; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	check-names response fail;
 	notify yes;
 };

bin/tests/system/checknames/ns4/named.conf

@@ -19,7 +19,6 @@ options {
 	listen-on { 10.53.0.4; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	check-names master ignore;
 	notify yes;
 };

bin/tests/system/cookie/ns1/named.conf

@@ -24,7 +24,6 @@ options {
 	listen-on { 10.53.0.1; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	deny-answer-addresses { 192.0.2.0/24; 2001:db8:beef::/48; }
 		 except-from { "example.org"; };
 	deny-answer-aliases { "example.org"; }

bin/tests/system/cookie/ns2/named.conf

@@ -17,7 +17,6 @@ options {
 	listen-on { 10.53.0.2; };
 	listen-on-v6 { none; };
 	recursion no;
-	acache-enable yes;
 	send-cookie yes;
 	nocookie-udp-size 512;
 };

bin/tests/system/cookie/ns3/named.conf

@@ -24,7 +24,6 @@ options {
 	listen-on { 10.53.0.3; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	deny-answer-addresses { 192.0.2.0/24; 2001:db8:beef::/48; }
 		 except-from { "example.org"; };
 	deny-answer-aliases { "example.org"; }

bin/tests/system/digdelv/ns3/named.conf

@@ -15,7 +15,6 @@ options {
 	listen-on { 10.53.0.3; };
 	listen-on-v6 { fd92:7065:b8e:ffff::3; };
 	recursion yes;
-	acache-enable yes;
 	dnssec-enable no;
 	dnssec-validation no;
 	server-id "ns3";

bin/tests/system/dlv/ns5/named.conf

@@ -49,7 +49,6 @@ options {
 	listen-on { 10.53.0.5; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	notify yes;
 	dnssec-enable yes;
 	dnssec-validation yes;

bin/tests/system/dnssec/ns2/named.conf

@@ -6,8 +6,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-/* $Id: named.conf,v 1.36 2011/03/21 23:47:21 tbox Exp $ */
-
 // NS2
 
 controls { /* empty */ };
@@ -25,6 +23,7 @@ options {
 	dnssec-enable yes;
 	dnssec-validation yes;
 	notify-delay 1;
+	minimal-responses no;
 };
 
 zone "." {

bin/tests/system/dnssec/ns3/named.conf

@@ -6,8 +6,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-/* $Id: named.conf,v 1.49 2011/10/28 06:20:05 each Exp $ */
-
 // NS3
 
 controls { /* empty */ };
@@ -25,6 +23,7 @@ options {
 	dnssec-enable yes;
 	dnssec-validation yes;
 	session-keyfile "session.key";
+	minimal-responses no;
 };
 
 key rndc_key {

bin/tests/system/dnssec/ns4/named1.conf

@@ -6,8 +6,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-/* $Id: named1.conf,v 1.3 2011/01/04 23:47:13 tbox Exp $ */
-
 // NS4
 
 controls { /* empty */ };
@@ -21,10 +19,10 @@ options {
 	listen-on { 10.53.0.4; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	dnssec-enable yes;
 	dnssec-validation yes;
 	dnssec-must-be-secure mustbesecure.example yes;
+	minimal-responses no;
 
 	nta-lifetime 10s;
 	nta-recheck 7s;

bin/tests/system/dnssec/ns4/named2.conf

@@ -6,8 +6,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-/* $Id: named2.conf,v 1.3 2011/01/04 23:47:13 tbox Exp $ */
-
 // NS4
 
 controls { /* empty */ };
@@ -22,10 +20,10 @@ options {
 	listen-on { 10.53.0.4; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	dnssec-enable yes;
 	dnssec-validation auto;
 	bindkeys-file "managed.conf";
+	minimal-responses no;
 };
 
 key rndc_key {

bin/tests/system/dnssec/ns4/named3.conf

@@ -6,8 +6,6 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  */
 
-/* $Id: named2.conf,v 1.3 2011/01/04 23:47:13 tbox Exp $ */
-
 // NS4
 
 controls { /* empty */ };
@@ -21,11 +19,11 @@ options {
 	listen-on { 10.53.0.4; };
 	listen-on-v6 { none; };
 	recursion yes;
-	acache-enable yes;
 	dnssec-enable yes;
 	dnssec-validation auto;
 	bindkeys-file "managed.conf";
 	dnssec-accept-expired yes;
+	minimal-responses no;
 };
 
 key rndc_key {

bin/tests/system/dnssec/ns4/named4.conf

@@ -38,4 +38,48 @@ controls {
 zone "." {
 	type hint;
 	file "../../common/root.hint";
+}
+
+key auth {
+	secret "1234abcd8765";
+	algorithm hmac-sha256;
+};
+
+include "trusted.conf";
+
+view rec {
+	match-recursive-only yes;
+	recursion yes;
+	dnssec-validation yes;
+	dnssec-accept-expired yes;
+
+	zone "." {
+		type hint;
+		file "../../common/root.hint";
+	};
+
+	zone secure.example {
+		type static-stub;
+		server-addresses { 10.53.0.4; };
+	};
+
+	zone insecure.secure.example {
+		type static-stub;
+		server-addresses { 10.53.0.4; };
+	};
+};
+
+view auth {
+	recursion no;
+	allow-recursion { none; };
+
+	zone secure.example {
+		type slave;
+		masters { 10.53.0.3; };
+	};
+
+	zone insecure.secure.example {
+		type slave;
+		masters { 10.53.0.2; };