Commit 612b2e2c authored by Evan Hunt's avatar Evan Hunt

[master] timestamp suffixes for log files

4579.	[func]		Logging channels and dnstap output files can now
			be configured with a "suffix" option, set to
			either "increment" or "timestamp", indicating
			whether to use incrementing numbers or timestamps
			as the file suffix when rolling over a log file.
			[RT #42838]
parent aa00b31b
4579. [func] Logging channels and dnstap output files can now
be configured with a "suffix" option, set to
either "increment" or "timestamp", indicating
whether to use incrementing numbers or timestamps
as the file suffix when rolling over a log file.
[RT #42838]
4578. [security] Some chaining (CNAME or DNAME) responses to upstream
queries could trigger assertion failures.
(CVE-2017-3137) [RT #44734]
......
......@@ -132,7 +132,10 @@ channel_fromconf(const cfg_obj_t *channel, isc_logconfig_t *logconfig) {
const cfg_obj_t *sizeobj = cfg_tuple_get(fileobj, "size");
const cfg_obj_t *versionsobj =
cfg_tuple_get(fileobj, "versions");
const cfg_obj_t *suffixobj =
cfg_tuple_get(fileobj, "suffix");
isc_int32_t versions = ISC_LOG_ROLLNEVER;
isc_log_rollsuffix_t suffix = isc_log_rollsuffix_increment;
isc_offset_t size = 0;
isc_uint64_t maxoffset;
......@@ -155,16 +158,21 @@ channel_fromconf(const cfg_obj_t *channel, isc_logconfig_t *logconfig) {
if (versionsobj != NULL && cfg_obj_isuint32(versionsobj))
versions = cfg_obj_asuint32(versionsobj);
if (versionsobj != NULL && cfg_obj_isstring(versionsobj) &&
else if (versionsobj != NULL && cfg_obj_isstring(versionsobj) &&
strcasecmp(cfg_obj_asstring(versionsobj), "unlimited") == 0)
versions = ISC_LOG_ROLLINFINITE;
if (sizeobj != NULL &&
cfg_obj_isuint64(sizeobj) &&
cfg_obj_asuint64(sizeobj) < maxoffset)
size = (isc_offset_t)cfg_obj_asuint64(sizeobj);
if (suffixobj != NULL && cfg_obj_isstring(suffixobj) &&
strcasecmp(cfg_obj_asstring(suffixobj), "timestamp") == 0)
suffix = isc_log_rollsuffix_timestamp;
dest.file.stream = NULL;
dest.file.name = cfg_obj_asstring(pathobj);
dest.file.versions = versions;
dest.file.suffix = suffix;
dest.file.maximum_size = size;
} else if (syslogobj != NULL) {
int facility = LOG_DAEMON;
......@@ -182,6 +190,7 @@ channel_fromconf(const cfg_obj_t *channel, isc_logconfig_t *logconfig) {
dest.file.stream = stderr;
dest.file.name = NULL;
dest.file.versions = ISC_LOG_ROLLNEVER;
dest.file.suffix = isc_log_rollsuffix_increment;
dest.file.maximum_size = 0;
}
......
......@@ -3090,6 +3090,8 @@ configure_dnstap(const cfg_obj_t **maps, dns_view_t *view) {
dns_dtmode_t dmode;
isc_uint64_t max_size = 0;
isc_uint32_t rolls = 0;
isc_log_rollsuffix_t suffix = isc_log_rollsuffix_increment;
obj = NULL;
CHECKM(ns_config_get(maps, "dnstap-output", &obj),
"'dnstap-output' must be set if 'dnstap' is set");
......@@ -3126,12 +3128,15 @@ configure_dnstap(const cfg_obj_t **maps, dns_view_t *view) {
obj2 = cfg_tuple_get(obj, "versions");
if (obj2 != NULL && cfg_obj_isuint32(obj2)) {
rolls = cfg_obj_asuint32(obj2);
} else if (obj2 != NULL && cfg_obj_isstring(obj2) &&
strcasecmp(cfg_obj_asstring(obj2), "unlimited") == 0)
{
rolls = ISC_LOG_ROLLINFINITE;
} else {
rolls = ISC_LOG_ROLLNEVER;
rolls = ISC_LOG_ROLLINFINITE;
}
obj2 = cfg_tuple_get(obj, "suffix");
if (obj2 != NULL && cfg_obj_isstring(obj2) &&
strcasecmp(cfg_obj_asstring(obj2), "timestamp") == 0)
{
suffix = isc_log_rollsuffix_timestamp;
}
fopt = fstrm_iothr_options_init();
......@@ -3202,7 +3207,8 @@ configure_dnstap(const cfg_obj_t **maps, dns_view_t *view) {
&fopt, &ns_g_server->dtenv),
"unable to create dnstap environment");
CHECKM(dns_dt_setupfile(ns_g_server->dtenv, max_size, rolls),
CHECKM(dns_dt_setupfile(ns_g_server->dtenv,
max_size, rolls, suffix),
"unable to set up dnstap logfile");
}
......@@ -13730,9 +13736,12 @@ ns_server_dnstap(ns_server_t *server, isc_lex_t *lex, isc_buffer_t **text) {
n = sscanf(ptr, "%u", &backups);
if (n != 1U)
return (ISC_R_BADNUMBER);
} else {
backups = ISC_LOG_ROLLINFINITE;
}
} else
} else {
return (DNS_R_SYNTAX);
}
result = isc_task_beginexclusive(server->task);
RUNTIME_CHECK(result == ISC_R_SUCCESS);
......
......@@ -7,5 +7,6 @@
*/
options {
dnstap-output file "/tmp/dnstap.log" size unlimited versions unlimited;
dnstap-output file "/tmp/dnstap.log"
size unlimited versions unlimited suffix increment;
};
......@@ -7,5 +7,6 @@
*/
options {
dnstap-output file "/tmp/dnstap.log" size 10k versions 3;
dnstap-output file "/tmp/dnstap.log"
size 10k versions 3 suffix timestamp;
};
......@@ -23,5 +23,7 @@ rm -f ns1/rndc.out.test*
rm -f ns1/dig.out.test*
rm -f ns1/named_vers
rm -f ns1/named_vers.*
rm -f ns1/named_ts
rm -f ns1/named_ts.*
rm -f ns1/named_unlimited
rm -f ns1/named_unlimited.*
/*
* Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC")
*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
options {
query-source address 10.53.0.1;
notify-source 10.53.0.1;
transfer-source 10.53.0.1;
port 5300;
pid-file "named.pid";
listen-on port 5300 {
10.53.0.1;
};
listen-on-v6 { none; };
recursion no;
notify yes;
};
logging {
channel default_log {
buffered no;
file "named_ts" versions 10 size 1000 suffix timestamp; # small size
severity debug 100;
print-time yes;
};
category default { default_log; default_debug; };
category lame-servers { null; };
channel query_log {
file "query_log";
print-time yes;
buffered yes;
};
category queries { query_log; };
};
controls {
inet 127.0.0.1 port 9593 allow {
127.0.0.1/32; ::1/128; }
keys { "rndc-key"; };
};
key "rndc-key" {
algorithm hmac-sha256;
secret "Am9vCg==";
};
zone "." {
type master;
file "root.db";
};
......@@ -19,6 +19,7 @@ PLAINCONF="${THISDIR}/${CONFDIR}/named.plainconf"
ISOCONF="${THISDIR}/${CONFDIR}/named.iso8601"
ISOCONFUTC="${THISDIR}/${CONFDIR}/named.iso8601-utc"
VERSCONF="${THISDIR}/${CONFDIR}/named.versconf"
TSCONF="${THISDIR}/${CONFDIR}/named.tsconf"
UNLIMITEDCONF="${THISDIR}/${CONFDIR}/named.unlimited"
PLAINFILE="named_log"
DIRFILE="named_dir"
......@@ -28,6 +29,7 @@ DLFILE="named_deflog"
ISOFILE="named_iso8601"
ISOUTCFILE="named_iso8601_utc"
VERSFILE="named_vers"
TSFILE="named_ts"
UNLIMITEDFILE="named_unlimited"
PIDFILE="${THISDIR}/${CONFDIR}/named.pid"
myRNDC="$RNDC -c ${THISDIR}/${CONFDIR}/rndc.conf"
......@@ -335,6 +337,32 @@ then
status=`expr $status + 1`
fi
n=`expr $n + 1`
echo "I: testing timestamped versions ($n)"
cp $TSCONF named.conf
# a seconds since epoch version number
touch $TSFILE.2015010112000012
t1=`$PERL -e 'print time()."\n";'`
$myRNDC reconfig > rndc.out.test$n 2>&1
$DIG version.bind txt ch @10.53.0.1 -p 5300 > dig.out.test$n
t2=`$PERL -e 'print time()."\n";'`
t=`expr ${t2:-0} - ${t1:-0}`
if test ${t:-1000} -gt 5
then
echo "I: testing timestamped versions failed cleanup of old entries took too long ($t secs)"
status=`expr $status + 1`
fi
if ! grep "status: NOERROR" dig.out.test$n > /dev/null
then
echo "I: testing timestamped versions failed DiG lookup failed"
status=`expr $status + 1`
fi
if test -f $TSFILE.1480039317
then
echo "I: testing timestamped versions failed $TSFILE.1480039317 not removed"
status=`expr $status + 1`
fi
n=`expr $n + 1`
echo "I: testing unlimited versions ($n)"
cp $UNLIMITEDCONF named.conf
......
......@@ -3638,7 +3638,8 @@ $ORIGIN 0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa.
[ <command>channel</command> <replaceable>channel_name</replaceable> <command>{</command>
( ( <command>file</command> <replaceable>path_name</replaceable>
[ <command>versions</command> ( <replaceable>number</replaceable> | <option>unlimited</option> ) ]
[ <command>size</command> <replaceable>size_spec</replaceable> ] )
[ <command>size</command> <replaceable>size_spec</replaceable> ]
[ <command>suffix</command> ( <option>increment</option> | <option>timestamp</option> ) )
| <command>syslog</command> <replaceable>syslog_facility</replaceable>
| <command>stderr</command>
| <command>null</command> ) <command>;</command>
......@@ -3733,61 +3734,68 @@ $ORIGIN 0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa.
<para>
The <command>file</command> destination clause directs
the channel
to a disk file. It can include limitations
both on how large the file is allowed to become, and how many
versions
of the file will be saved each time the file is opened.
</para>
<para>
If you use the <command>versions</command> log file
option, then
<command>named</command> will retain that many backup
versions of the file by
renaming them when opening. For example, if you choose to keep
three old versions
of the file <filename>lamers.log</filename>, then just
before it is opened
<filename>lamers.log.1</filename> is renamed to
<filename>lamers.log.2</filename>, <filename>lamers.log.0</filename> is renamed
to <filename>lamers.log.1</filename>, and <filename>lamers.log</filename> is
renamed to <filename>lamers.log.0</filename>.
You can say <command>versions unlimited</command> to
not limit
the number of versions.
If a <command>size</command> option is associated with
the log file,
then renaming is only done when the file being opened exceeds the
indicated size. No backup versions are kept by default; any
existing
log file is simply appended.
</para>
<para>
The <command>size</command> option for files is used
to limit log
growth. If the file ever exceeds the size, then <command>named</command> will
stop writing to the file unless it has a <command>versions</command> option
associated with it. If backup versions are kept, the files are
rolled as
described above and a new one begun. If there is no
the channel to a disk file. It can include additional
arguments to specify how large the file is allowed to
become before it is rolled to a backup file
(<command>size</command>), how many backup versions of
the file will be saved each time this happens
(<command>versions</command>), and the format to use
for naming backup versions (<command>suffix</command>).
</para>
<para>
The <command>size</command> option is used to limit
log file growth. If the file ever exceeds the specified
size, then <command>named</command> will stop writing to the
file unless it has a <command>versions</command> option
associated with it. If backup versions are kept, the files
are rolled as described below. If there is no
<command>versions</command> option, no more data will
be written to the log
until some out-of-band mechanism removes or truncates the log to
less than the
maximum size. The default behavior is not to limit the size of
the
file.
</para>
<para>
Example usage of the <command>size</command> and
<command>versions</command> options:
be written to the log until some out-of-band mechanism
removes or truncates the log to less than the maximum size.
The default behavior is not to limit the size of the file.
</para>
<para>
File rolling only occurs when the file exceeds the size
specified with the <command>size</command> option. No
backup versions are kept by default; any existing
log file is simply appended. The
<command>versions</command> option specifies
how many backup versions of the file should be kept.
If set to <literal>unlimited</literal>, there is no limit.
</para>
<para>
The <command>suffix</command> option can be set to
either <literal>increment</literal> or
<literal>timestamp</literal>. If set to
<literal>timestamp</literal>, then when a log file is
rolled, it is saved with the current timestamp as a
file suffix. If set to <literal>increment</literal>,
then backup files are saved with incrementing numbers
as suffixes; older files are renamed when rolling.
For example, if <command>versions</command>
is set to 3 and <command>suffix</command> to
<literal>increment</literal>, then when
<filename>filename.log</filename> reaches the size
specified by <command>size</command>,
<filename>filename.log.1</filename> is renamed to
<filename>filename.log.2</filename>,
<filename>filename.log.0</filename> is renamed
to <filename>filename.log.1</filename>,
and <filename>filename.log</filename> is
renamed to <filename>filename.log.0</filename>,
whereupon a new <filename>filename.log</filename> is
opened.
</para>
<para>
Example usage of the <command>size</command>,
<command>versions</command>, and <command>suffix</command>
options:
</para>
<programlisting>channel an_example_channel {
file "example.log" versions 3 size 20m;
file "example.log" versions 3 size 20m suffix increment;
print-time yes;
print-category yes;
};
......@@ -5004,16 +5012,20 @@ badresp:1,adberr:0,findfail:0,valfail:0]
</para>
<para>
If the first argument is <literal>file</literal>, then
two additional options can be added:
up to three additional options can be added:
<command>size</command> indicates the size to which a
<command>dnstap</command> log file can grow before being
rolled to a new file, and <command>versions</command>
specifies the number of rolled log files to retain. These
are similar to the <command>size</command> and
<command>versions</command> options in a
<command>logging</command> channel. The default
is to allow <command>dnstap</command> log files to grow to
any size without rolling.
rolled to a new file; <command>versions</command>
specifies the number of rolled log files to retain; and
<command>suffix</command> indicates whether to retain
rolled log files with an incrementing counter as the
suffix (<literal>increment</literal>) or with the
current timestamp (<literal>timestamp</literal>).
These are similar to the <command>size</command>,
<command>versions</command>, and <command>suffix</command>
options in a <command>logging</comamnd> channel.
The default is to allow <command>dnstap</command> log
files to grow to any size without rolling.
</para>
<para>
<command>dnstap-output</command> can only be set globally
......
......@@ -154,7 +154,7 @@
<command>dnstap</command> logfiles can now be configured to
automatically roll when they reach a specified size. If
<command>dnstap-output</command> is configured with mode
<literal>file</literal>, then it can take optional
<literal>file</command>, then it can take optional
<command>size</command> and <command>versions</command>
key-value arguments to set the logfile rolling parameters.
(These have the same semantics as the corresponding
......@@ -162,6 +162,19 @@
[RT #44502]
</para>
</listitem>
<listitem>
<para>
Logging channels and <command>dnstap-output</command> files can
now be configured with a <command>suffix</command> option,
set to either <literal>increment</literal> or
<literal>timestamp</literal>, indicating whether log files
should be given incrementing suffixes when they roll
over (e.g., <filename>logfile.0</filename>,
<filename>.1</filename>, <filename>.2</filename>, etc)
or suffixes indicating the time of the roll. The default
is <literal>increment</literal>. [RT #42838]
</para>
</listitem>
<listitem>
<para>
<command>dig +ednsopt</command> now accepts the names
......
......@@ -33,8 +33,8 @@ logging {
category <string> { <string>; ... }; // may occur multiple times
channel <string> {
buffered <boolean>;
file <quoted_string> [ versions ( "unlimited" | <integer> )
] [ size <size> ];
file <quoted_string> [ versions ( unlimited | <integer> ) ]
[ size <size> ] [ suffix ( increment | timestamp ) ];
null;
print-category <boolean>;
print-severity <boolean>;
......@@ -155,7 +155,8 @@ options {
hostname ); // not configured
dnstap-output ( file | unix ) <quoted_string> [
size ( unlimited | <size> ) ] [ versions (
unlimited | <integer> ) ]; // not configured
unlimited | <integer> ) ] [ suffix ( increment
| timestamp ) ]; // not configured
dnstap-version ( <quoted_string> | none ); // not configured
dscp <integer>;
dual-stack-servers [ port <integer> ] { ( <quoted_string> [ port
......
......@@ -1476,6 +1476,16 @@ check_options(const cfg_obj_t *options, isc_log_t *logctx, isc_mem_t *mctx,
"cannot be set with mode unix");
return (ISC_R_FAILURE);
}
obj2 = cfg_tuple_get(obj, "suffix");
if (obj2 != NULL && !cfg_obj_isvoid(obj2) &&
dmode == dns_dtmode_unix)
{
cfg_obj_log(obj, logctx, ISC_LOG_ERROR,
"dnstap-output suffix "
"cannot be set with mode unix");
return (ISC_R_FAILURE);
}
}
#endif
......
......@@ -50,6 +50,7 @@
#include <isc/buffer.h>
#include <isc/file.h>
#include <isc/log.h>
#include <isc/mem.h>
#include <isc/once.h>
#include <isc/print.h>
......@@ -104,8 +105,9 @@ struct dns_dtenv {
isc_region_t version;
char *path;
dns_dtmode_t mode;
isc_uint64_t max_size;
isc_uint32_t rolls;
isc_offset_t max_size;
int rolls;
isc_log_rollsuffix_t suffix;
isc_stats_t *stats;
};
......@@ -185,7 +187,7 @@ dns_dt_create(isc_mem_t *mctx, dns_dtmode_t mode, const char *path,
REQUIRE(path != NULL);
REQUIRE(envp != NULL && *envp == NULL);
REQUIRE(foptp!= NULL && *foptp != NULL);
REQUIRE(foptp != NULL && *foptp != NULL);
isc_log_write(dns_lctx, DNS_LOGCATEGORY_DNSTAP,
DNS_LOGMODULE_DNSTAP, ISC_LOG_INFO,
......@@ -244,7 +246,7 @@ dns_dt_create(isc_mem_t *mctx, dns_dtmode_t mode, const char *path,
}
env->mode = mode;
env->max_size = 0;
env->rolls = ISC_LOG_ROLLNEVER;
env->rolls = ISC_LOG_ROLLINFINITE;
env->fopt = *foptp;
*foptp = NULL;
......@@ -279,17 +281,28 @@ dns_dt_create(isc_mem_t *mctx, dns_dtmode_t mode, const char *path,
}
isc_result_t
dns_dt_setupfile(dns_dtenv_t *env, isc_uint64_t max_size, int rolls) {
dns_dt_setupfile(dns_dtenv_t *env, isc_uint64_t max_size, int rolls,
isc_log_rollsuffix_t suffix)
{
REQUIRE(VALID_DTENV(env));
if (max_size != 0 && rolls != ISC_LOG_ROLLNEVER &&
env->mode != dns_dtmode_file)
{
return (ISC_R_INVALIDFILE);
/*
* If we're using unix domain socket mode, then any
* change from the default values is invalid.
*/
if (env->mode == dns_dtmode_unix) {
if (max_size == 0 && rolls == ISC_LOG_ROLLINFINITE &&
suffix == isc_log_rollsuffix_increment)
{
return (ISC_R_SUCCESS);
} else {
return (ISC_R_INVALIDFILE);
}
}
env->max_size = max_size;
env->rolls = rolls;
env->suffix = suffix;
return (ISC_R_SUCCESS);
}
......@@ -310,14 +323,16 @@ dns_dt_reopen(dns_dtenv_t *env, int roll) {
* Check that we can create a new fw object.
*/
fwopt = fstrm_writer_options_init();
if (fwopt == NULL)
if (fwopt == NULL) {
return (ISC_R_NOMEMORY);
}
res = fstrm_writer_options_add_content_type(fwopt,
DNSTAP_CONTENT_TYPE,
sizeof(DNSTAP_CONTENT_TYPE) - 1);
if (res != fstrm_res_success)
if (res != fstrm_res_success) {
CHECK(ISC_R_FAILURE);
}
if (env->mode == dns_dtmode_file) {
ffwopt = fstrm_file_options_init();
......@@ -332,11 +347,13 @@ dns_dt_reopen(dns_dtenv_t *env, int roll) {
env->path);
fw = fstrm_unix_writer_init(fuwopt, fwopt);
}
} else
} else {
CHECK(ISC_R_NOTIMPLEMENTED);
}
if (fw == NULL)
if (fw == NULL) {
CHECK(ISC_R_FAILURE);
}
/*
* We are committed here.
......@@ -349,10 +366,15 @@ dns_dt_reopen(dns_dtenv_t *env, int roll) {
generation++;
if (env->iothr != NULL)
if (env->iothr != NULL) {
fstrm_iothr_destroy(&env->iothr);
}
if (roll != 0) {
roll = env->rolls;
}
if (env->mode == dns_dtmode_file && roll >= 0) {
if (env->mode == dns_dtmode_file && roll != 0) {
/*
* Create a temporary isc_logfile_t structure so we can
* take advantage of the logfile rolling facility.
......@@ -360,9 +382,10 @@ dns_dt_reopen(dns_dtenv_t *env, int roll) {
char *filename = isc_mem_strdup(env->mctx, env->path);
file.name = filename;
file.stream = NULL;
file.versions = roll != 0 ? roll : ISC_LOG_ROLLINFINITE;
file.versions = roll != 0 ? roll : env->rolls;
file.maximum_size = 0;
file.maximum_reached = ISC_FALSE;
file.suffix = env->suffix;
result = isc_logfile_roll(&file);
isc_mem_free(env->mctx, filename);
CHECK(result);
......@@ -714,7 +737,8 @@ dns_dt_send(dns_view_t *view, dns_dtmsgtype_t msgtype,
if (view->dtenv->max_size != 0) {
struct stat statbuf;
if (stat(view->dtenv->path, &statbuf) >= 0 &&
(size_t) statbuf.st_size > view->dtenv->max_size) {
statbuf.st_size > view->dtenv->max_size)
{
dns_dt_reopen(view->dtenv, view->dtenv->rolls);
}
}
......
......@@ -28,6 +28,7 @@
struct fstrm_iothr_options;
#endif /* HAVE_DNSTAP */
#include <isc/log.h>
#include <isc/refcount.h>
#include <isc/region.h>
#include <isc/sockaddr.h>
......@@ -153,7 +154,8 @@ dns_dt_create(isc_mem_t *mctx, dns_dtmode_t mode, const char *path,
*/
isc_result_t
dns_dt_setupfile(dns_dtenv_t *env, isc_uint64_t max_size, int rolls);
dns_dt_setupfile(dns_dtenv_t *env, isc_uint64_t max_size, int rolls,
isc_log_rollsuffix_t suffix);
/*%<
* Sets up the dnstap logfile limits.
*
......@@ -161,6 +163,8 @@ dns_dt_setupfile(dns_dtenv_t *env, isc_uint64_t max_size, int rolls);
*
* 'rolls' is the number of rolled files to retain.
*
* 'suffix' is the logfile suffix setting, increment or timestamp.
*
* Requires:
*
*\li 'env' is a valid dnstap environment.
......
......@@ -78,6 +78,17 @@
*/
#define ISC_LOG_ROLLINFINITE (-1)
#define ISC_LOG_ROLLNEVER (-2)
#define ISC_LOG_MAX_VERSIONS 256
/*@}*/
/*@{*/
/*!
* \brief Type of suffix used on rolled log files.
*/
typedef enum {
isc_log_rollsuffix_increment,
isc_log_rollsuffix_timestamp
} isc_log_rollsuffix_t;
/*@}*/
/*!
......@@ -115,6 +126,7 @@ typedef struct isc_logfile {
FILE *stream; /*%< Initialized to NULL for #ISC_LOG_TOFILE. */
const char *name; /*%< NULL for #ISC_LOG_TOFILEDESC. */
int versions; /* >= 0, #ISC_LOG_ROLLNEVER, #ISC_LOG_ROLLINFINITE. */
isc_log_rollsuffix_t suffix;
/*%
* stdio's ftell is standardized to return a long, which may well not
* be big enough for the largest file supportable by the operating
......