Commit d39ab744 authored by Evan Hunt's avatar Evan Hunt

[master] automatically tune max-journal-size

4613.	[func]		By default, the maximum size of a zone journal file
			is now twice the size of the zone's contents (there
			is little benefit to a journal larger than this).
			This can be overridden by setting "max-journal-size"
			to "unlimited" or to an explicit value up to 2G.
			Thanks to Tony Finch. [RT #38324]
parent 4c97cb13
4613. [func] By default, the maximum size of a zone journal file
is now twice the size of the zone's contents (there
is little benefit to a journal larger than this).
This can be overridden by setting "max-journal-size"
to "unlimited" or to an explicit value up to 2G.
Thanks to Tony Finch. [RT #38324]
4612. [bug] Silence 'may be use uninitalised' warning and simplify
the code in lwres/getaddinfo:process_answer.
[RT #45158]
......
......@@ -233,7 +233,7 @@ options {\n\
sig-signing-type 65534;\n\
inline-signing no;\n\
zone-statistics terse;\n\
max-journal-size unlimited;\n\
max-journal-size default;\n\
ixfr-from-differences false;\n\
check-wildcard yes;\n\
check-sibling yes;\n\
......
......@@ -20,6 +20,7 @@
#include <dns/db.h>
#include <dns/ipkeylist.h>
#include <dns/fixedname.h>
#include <dns/journal.h>
#include <dns/log.h>
#include <dns/name.h>
#include <dns/masterdump.h>
......@@ -1211,12 +1212,16 @@ ns_zone_configure(const cfg_obj_t *config, const cfg_obj_t *vconfig,
dns_zone_setjournalsize(zone, -1);
if (cfg_obj_isstring(obj)) {
const char *str = cfg_obj_asstring(obj);
INSIST(strcasecmp(str, "unlimited") == 0);
journal_size = ISC_UINT32_MAX / 2;
if (strcasecmp(str, "unlimited") == 0) {
journal_size = DNS_JOURNAL_SIZE_MAX;
} else {
INSIST(strcasecmp(str, "default") == 0);
journal_size = -1;
}
} else {
isc_resourcevalue_t value;
value = cfg_obj_asuint64(obj);
if (value > ISC_UINT32_MAX / 2) {
if (value > DNS_JOURNAL_SIZE_MAX) {
cfg_obj_log(obj, ns_g_lctx,
ISC_LOG_ERROR,
"'max-journal-size "
......@@ -1331,12 +1336,16 @@ ns_zone_configure(const cfg_obj_t *config, const cfg_obj_t *vconfig,
dns_zone_setjournalsize(zone, -1);
if (cfg_obj_isstring(obj)) {
const char *str = cfg_obj_asstring(obj);
INSIST(strcasecmp(str, "unlimited") == 0);
journal_size = ISC_UINT32_MAX / 2;
if (strcasecmp(str, "unlimited") == 0) {
journal_size = DNS_JOURNAL_SIZE_MAX;
} else {
INSIST(strcasecmp(str, "default") == 0);
journal_size = -1;
}
} else {
isc_resourcevalue_t value;
value = cfg_obj_asuint64(obj);
if (value > ISC_UINT32_MAX / 2) {
if (value > DNS_JOURNAL_SIZE_MAX) {
cfg_obj_log(obj, ns_g_lctx,
ISC_LOG_ERROR,
"'max-journal-size "
......
......@@ -19,6 +19,7 @@ rm -f ns*/named.lock
rm -f ns1/*.jnl ns2/*.jnl ns3/*.jnl
rm -f ns1/example.db ns1/unixtime.db ns1/yyyymmddvv.db ns1/update.db ns1/other.db ns1/keytests.db
rm -f ns1/many.test.db
rm -f ns1/maxjournal.db
rm -f ns1/md5.key ns1/sha1.key ns1/sha224.key ns1/sha256.key ns1/sha384.key
rm -f ns1/sha512.key ns1/ddns.key
rm -f ns2/example.bk
......
......@@ -4,14 +4,13 @@
; License, v. 2.0. If a copy of the MPL was not distributed with this
; file, You can obtain one at http://mozilla.org/MPL/2.0/.
$ORIGIN .
$TTL 300 ; 5 minutes
many.test IN SOA ns1.example.nil. hostmaster.example.nil. (
@ IN SOA ns1.example.nil. hostmaster.example.nil. (
1 ; serial
2000 ; refresh (2000 seconds)
2000 ; retry (2000 seconds)
1814400 ; expire (3 weeks)
3600 ; minimum (1 hour)
)
many.test NS ns1.example.nil.
many.test NS ns2.example.nil.
@ NS ns1.example.nil.
NS ns2.example.nil.
; Copyright (C) 2014, 2016 Internet Systems Consortium, Inc. ("ISC")
;
; This Source Code Form is subject to the terms of the Mozilla Public
; License, v. 2.0. If a copy of the MPL was not distributed with this
; file, You can obtain one at http://mozilla.org/MPL/2.0/.
$TTL 300 ; 5 minutes
@ IN SOA ns1.example.nil. hostmaster.example.nil. (
1 ; serial
2000 ; refresh (2000 seconds)
2000 ; retry (2000 seconds)
1814400 ; expire (3 weeks)
3600 ; minimum (1 hour)
)
@ NS ns1.example.nil.
......@@ -129,3 +129,10 @@ zone "sample" {
allow-update { any; };
file "sample.db";
};
zone "maxjournal.test" {
type master;
allow-update { any; };
file "maxjournal.db";
max-journal-size default;
};
......@@ -61,3 +61,6 @@ rm -f ns1/many.test.db.jnl
cp ns1/sample.db.in ns1/sample.db
cp ns2/sample.db.in ns2/sample.db
cp -f ns1/maxjournal.db.in ns1/maxjournal.db
rm -f ns1/maxjournal.db.jnl
......@@ -6,8 +6,6 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# $Id: tests.sh,v 1.42 2011/12/16 23:01:17 each Exp $
SYSTEMTESTTOP=..
. $SYSTEMTESTTOP/conf.sh
......@@ -583,6 +581,7 @@ if [ $ret -ne 0 ]; then
fi
n=`expr $n + 1`
ret=0
echo "I:check that yyyymmddvv serial number is correctly generated ($n)"
oldserial=`$DIG +short yyyymmddvv.nil. soa @10.53.0.1 -p 5300 | awk '{print $3}'` || ret=1
$NSUPDATE <<END > /dev/null 2>&1 || ret=1
......@@ -630,6 +629,46 @@ test ${lines:-0} -eq 64 || ret=1
[ $ret = 0 ] || { echo I:failed; status=1; }
fi
n=`expr $n + 1`
echo "I:check max-journal-size limits ($n)"
ret=0
rm -f nsupdate.out1-$n
# add one record
$NSUPDATE << EOF >> nsupdate.out1-$n 2>&1
server 10.53.0.1 5300
zone maxjournal.test
update add z.maxjournal.test 300 IN A 10.20.30.40
send
EOF
for i in 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20; do
# repeatedly add and remove the same set of records to fill up
# the journal file without changing the zone content
$NSUPDATE << EOF >> nsupdate.out1-$n 2>&1
server 10.53.0.1 5300
zone maxjournal.test
update add a.maxjournal.test 300 IN A 1.2.3.4
update add b.maxjournal.test 300 IN A 1.2.3.4
update add c.maxjournal.test 300 IN A 1.2.3.4
update add d.maxjournal.test 300 IN A 1.2.3.4
send
update del a.maxjournal.test
update del b.maxjournal.test
update del c.maxjournal.test
update del d.maxjournal.test
send
EOF
done
# check that the journal is big enough to require truncation.
size=`$PERL -e 'use File::stat; my $sb = stat(@ARGV[0]); printf("%s\n", $sb->size);' ns1/maxjournal.db.jnl`
[ "$size" -gt 6000 ] || ret=1
sleep 1
$RNDC -c ../common/rndc.conf -s 10.53.0.1 -p 9953 sync maxjournal.test
sleep 1
size=`$PERL -e 'use File::stat; my $sb = stat(@ARGV[0]); printf("%s\n", $sb->size);' ns1/maxjournal.db.jnl`
[ "$size" -lt 5000 ] || ret=1
[ $ret = 0 ] || { echo I:failed; status=1; }
n=`expr $n + 1`
echo "I:check check-names processing ($n)"
ret=0
......
......@@ -8492,16 +8492,24 @@ avoid-v6-udp-ports { 40000; range 50000 60000; };
<term><command>max-journal-size</command></term>
<listitem>
<para>
Sets a maximum size for each journal file
(see <xref linkend="journal"/>). When the journal file
approaches
the specified size, some of the oldest transactions in the
journal
will be automatically removed. The largest permitted
value is 2 gigabytes. The default is
<literal>unlimited</literal>, which also
means 2 gigabytes.
This may also be set on a per-zone basis.
Sets a maximum size for each journal file (see
<xref linkend="journal"/>), expressed in bytes
or, if followed by an optional unit suffix ('k',
'm', or 'g'), in kilobytes, megabytes, or gigabytes.
When the journal file approaches the specified size,
some of the oldest transactions in the journal
will be automatically removed. The largest
permitted value is 2 gigabytes. Very small
values are rounded up to 4096 bytes. You
can specify <literal>unlimited</literal>, which
also means 2 gigabytes. If you set the limit to
<literal>default</literal> or leave it unset, the
journal is allowed to grow up to twice as large as
the zone. (There is little benefit in storing
larger journals.)
</para>
<para>
This option may also be set on a per-zone basis.
</para>
</listitem>
</varlistentry>
......
......@@ -151,6 +151,16 @@
<section xml:id="relnotes_features"><info><title>New Features</title></info>
<itemizedlist>
<listitem>
<para>
Setting <command>max-journal-size</command> to
<literal>default</command> limits journal sizes to twice the
size of the zone contents. This can be overridden by setting
<command>max-journal-size</command> to <literal>unlimited</literal>
or to an explicit value up to 2G. Thanks to Tony Finch for
the contribution. [RT #38324]
</para>
</listitem>
<listitem>
<para>
The <command>new-zones-directory</command> option allows
......
......@@ -41,6 +41,9 @@
#define DNS_JOURNAL_CREATE 0x00000001 /* ISC_TRUE */
#define DNS_JOURNAL_WRITE 0x00000002
#define DNS_JOURNAL_SIZE_MAX ISC_INT32_MAX
#define DNS_JOURNAL_SIZE_MIN 4096
/***
*** Types
***/
......@@ -169,6 +172,12 @@ dns_journal_write_transaction(dns_journal_t *j, dns_diff_t *diff);
* Reading transactions from journals.
*/
isc_boolean_t
dns_journal_empty(dns_journal_t *j);
/*<
* Find out if a journal is empty.
*/
isc_uint32_t
dns_journal_first_serial(dns_journal_t *j);
isc_uint32_t
......
......@@ -6,8 +6,6 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
/* $Id: journal.c,v 1.120 2011/12/22 07:32:41 each Exp $ */
#include <config.h>
#include <stdlib.h>
......@@ -1536,6 +1534,11 @@ dns_journal_print(isc_mem_t *mctx, const char *filename, FILE *file) {
/*
* Miscellaneous accessors.
*/
isc_boolean_t
dns_journal_empty(dns_journal_t *j) {
return (JOURNAL_EMPTY(&j->header));
}
isc_uint32_t
dns_journal_first_serial(dns_journal_t *j) {
return (j->header.begin.serial);
......@@ -2139,6 +2142,8 @@ dns_journal_compact(isc_mem_t *mctx, char *filename, isc_uint32_t serial,
*/
indexend = sizeof(journal_rawheader_t) +
j->header.index_size * sizeof(journal_rawpos_t);
if (target_size < DNS_JOURNAL_SIZE_MIN)
target_size = DNS_JOURNAL_SIZE_MIN;
if (target_size < indexend * 2)
target_size = target_size/2 + indexend;
......
......@@ -405,6 +405,7 @@ dns_journal_commit
dns_journal_compact
dns_journal_current_rr
dns_journal_destroy
dns_journal_empty
dns_journal_first_rr
dns_journal_first_serial
dns_journal_get_sourceserial
......
......@@ -4385,11 +4385,13 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
! DNS_ZONE_OPTION(zone, DNS_ZONEOPT_IXFRFROMDIFFS)) {
isc_uint32_t jserial;
dns_journal_t *journal = NULL;
isc_boolean_t empty = ISC_FALSE;
result = dns_journal_open(zone->mctx, zone->journal,
DNS_JOURNAL_READ, &journal);
if (result == ISC_R_SUCCESS) {
jserial = dns_journal_last_serial(journal);
empty = dns_journal_empty(journal);
dns_journal_destroy(&journal);
} else {
jserial = serial;
......@@ -4397,9 +4399,10 @@ zone_postload(dns_zone_t *zone, dns_db_t *db, isc_time_t loadtime,
}
if (jserial != serial) {
dns_zone_log(zone, ISC_LOG_INFO,
"journal file is out of date: "
"removing journal file");
if (!empty)
dns_zone_log(zone, ISC_LOG_INFO,
"journal file is out of date: "
"removing journal file");
if (remove(zone->journal) < 0 && errno != ENOENT) {
char strbuf[ISC_STRERRORSIZE];
isc__strerror(errno, strbuf, sizeof(strbuf));
......@@ -9756,6 +9759,52 @@ dns_zone_refresh(dns_zone_t *zone) {
UNLOCK_ZONE(zone);
}
static void
zone_journal_compact(dns_zone_t *zone, dns_db_t *db, isc_uint32_t serial) {
isc_result_t result;
isc_int32_t journalsize;
dns_dbversion_t *ver = NULL;
isc_uint64_t dbsize;
INSIST(LOCKED_ZONE(zone));
if (inline_raw(zone))
INSIST(LOCKED_ZONE(zone->secure));
journalsize = zone->journalsize;
if (journalsize == -1) {
journalsize = DNS_JOURNAL_SIZE_MAX;
dns_db_currentversion(db, &ver);
result = dns_db_getsize(db, ver, NULL, &dbsize);
dns_db_closeversion(db, &ver, ISC_FALSE);
if (result != ISC_R_SUCCESS) {
dns_zone_log(zone, ISC_LOG_ERROR,
"zone_journal_compact: "
"could not get zone size: %s",
isc_result_totext(result));
} else if (dbsize < DNS_JOURNAL_SIZE_MAX / 2) {
journalsize = (isc_int32_t)dbsize * 2;
}
}
zone_debuglog(zone, "zone_journal_compact", 1,
"target journal size %d", journalsize);
result = dns_journal_compact(zone->mctx, zone->journal,
serial, journalsize);
switch (result) {
case ISC_R_SUCCESS:
case ISC_R_NOSPACE:
case ISC_R_NOTFOUND:
dns_zone_log(zone, ISC_LOG_DEBUG(3),
"dns_journal_compact: %s",
dns_result_totext(result));
break;
default:
dns_zone_log(zone, ISC_LOG_ERROR,
"dns_journal_compact failed: %s",
dns_result_totext(result));
break;
}
}
isc_result_t
dns_zone_flush(dns_zone_t *zone) {
isc_result_t result = ISC_R_SUCCESS;
......@@ -9773,7 +9822,7 @@ dns_zone_flush(dns_zone_t *zone) {
dumping = ISC_TRUE;
UNLOCK_ZONE(zone);
if (!dumping)
result = zone_dump(zone, ISC_FALSE); /* Unknown task. */
result = zone_dump(zone, ISC_TRUE); /* Unknown task. */
return (result);
}
......@@ -9841,8 +9890,7 @@ dump_done(void *arg, isc_result_t result) {
ENTER;
if (result == ISC_R_SUCCESS && zone->journal != NULL &&
zone->journalsize != -1) {
if (result == ISC_R_SUCCESS && zone->journal != NULL) {
/*
* We don't own these, zone->dctx must stay valid.
*/
......@@ -9887,37 +9935,19 @@ dump_done(void *arg, isc_result_t result) {
}
ZONEDB_UNLOCK(&secure->dblock, isc_rwlocktype_read);
}
if (secure != NULL)
UNLOCK_ZONE(secure);
UNLOCK_ZONE(zone);
/*
* Note: we are task locked here so we can test
* zone->xfr safely.
*/
if (tresult == ISC_R_SUCCESS && zone->xfr == NULL) {
tresult = dns_journal_compact(zone->mctx,
zone->journal,
serial,
zone->journalsize);
switch (tresult) {
case ISC_R_SUCCESS:
case ISC_R_NOSPACE:
case ISC_R_NOTFOUND:
dns_zone_log(zone, ISC_LOG_DEBUG(3),
"dns_journal_compact: %s",
dns_result_totext(tresult));
break;
default:
dns_zone_log(zone, ISC_LOG_ERROR,
"dns_journal_compact failed: %s",
dns_result_totext(tresult));
break;
dns_db_t *zdb = NULL;
if (dns_zone_getdb(zone, &zdb) == ISC_R_SUCCESS) {
zone_journal_compact(zone, zdb, serial);
dns_db_detach(&db);
}
} else if (tresult == ISC_R_SUCCESS) {
compact = ISC_TRUE;
zone->compact_serial = serial;
}
if (secure != NULL)
UNLOCK_ZONE(secure);
UNLOCK_ZONE(zone);
}
LOCK_ZONE(zone);
......@@ -13112,7 +13142,6 @@ dns_zone_setzeronosoattl(dns_zone_t *zone, isc_boolean_t state) {
void
dns_zone_setchecknames(dns_zone_t *zone, dns_severity_t severity) {
REQUIRE(DNS_ZONE_VALID(zone));
zone->check_names = severity;
......@@ -13120,7 +13149,6 @@ dns_zone_setchecknames(dns_zone_t *zone, dns_severity_t severity) {
dns_severity_t
dns_zone_getchecknames(dns_zone_t *zone) {
REQUIRE(DNS_ZONE_VALID(zone));
return (zone->check_names);
......@@ -13128,7 +13156,6 @@ dns_zone_getchecknames(dns_zone_t *zone) {
void
dns_zone_setjournalsize(dns_zone_t *zone, isc_int32_t size) {
REQUIRE(DNS_ZONE_VALID(zone));
zone->journalsize = size;
......@@ -13136,7 +13163,6 @@ dns_zone_setjournalsize(dns_zone_t *zone, isc_int32_t size) {
isc_int32_t
dns_zone_getjournalsize(dns_zone_t *zone) {
REQUIRE(DNS_ZONE_VALID(zone));
return (zone->journalsize);
......@@ -14445,7 +14471,7 @@ zone_replacedb(dns_zone_t *zone, dns_db_t *db, isc_boolean_t dump) {
unsigned int nscount = 0;
/*
* 'zone' and 'zonedb' locked by caller.
* 'zone' and 'zone->db' locked by caller.
*/
REQUIRE(DNS_ZONE_VALID(zone));
REQUIRE(LOCKED_ZONE(zone));
......@@ -14530,24 +14556,8 @@ zone_replacedb(dns_zone_t *zone, dns_db_t *db, isc_boolean_t dump) {
goto fail;
if (dump)
zone_needdump(zone, DNS_DUMP_DELAY);
else if (zone->journalsize != -1) {
result = dns_journal_compact(zone->mctx, zone->journal,
serial, zone->journalsize);
switch (result) {
case ISC_R_SUCCESS:
case ISC_R_NOSPACE:
case ISC_R_NOTFOUND:
dns_zone_log(zone, ISC_LOG_DEBUG(3),
"dns_journal_compact: %s",
dns_result_totext(result));
break;
default:
dns_zone_log(zone, ISC_LOG_ERROR,
"dns_journal_compact failed: %s",
dns_result_totext(result));
break;
}
}
else
zone_journal_compact(zone, zone->db, serial);
if (zone->type == dns_zone_master && inline_raw(zone))
zone_send_secureserial(zone, serial);
} else {
......@@ -14860,24 +14870,12 @@ zone_xfrdone(dns_zone_t *zone, isc_result_t result) {
* Handle any deferred journal compaction.
*/
if (DNS_ZONE_FLAG(zone, DNS_ZONEFLG_NEEDCOMPACT)) {
result = dns_journal_compact(zone->mctx, zone->journal,
zone->compact_serial,
zone->journalsize);
switch (result) {
case ISC_R_SUCCESS:
case ISC_R_NOSPACE:
case ISC_R_NOTFOUND:
dns_zone_log(zone, ISC_LOG_DEBUG(3),
"dns_journal_compact: %s",
dns_result_totext(result));
break;
default:
dns_zone_log(zone, ISC_LOG_ERROR,
"dns_journal_compact failed: %s",
dns_result_totext(result));
break;
dns_db_t *db = NULL;
if (dns_zone_getdb(zone, &db) == ISC_R_SUCCESS) {
zone_journal_compact(zone, db, zone->compact_serial);
dns_db_detach(&db);
DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT);
}
DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_NEEDCOMPACT);
}
if (secure != NULL)
......
......@@ -2008,7 +2008,7 @@ zone_clauses[] = {
{ "masterfile-format", &cfg_type_masterformat, 0 },
{ "masterfile-style", &cfg_type_masterstyle, 0 },
{ "max-ixfr-log-size", &cfg_type_size, CFG_CLAUSEFLAG_OBSOLETE },
{ "max-journal-size", &cfg_type_sizenodefault, 0 },
{ "max-journal-size", &cfg_type_size, 0 },
{ "max-records", &cfg_type_uint32, 0 },
{ "max-refresh-time", &cfg_type_uint32, 0 },
{ "max-retry-time", &cfg_type_uint32, 0 },
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment