From: Evan Hunt Date: Tue, 7 Feb 2017 00:03:37 +0000 (-0800) Subject: [master] dnstap size and versions options X-Git-Tag: v9.12.0a1~459 X-Git-Url: http://git.ipfire.org/gitweb/index.cgi?a=commitdiff_plain;h=c4e4bd6a09e802fb5f3d6a3c4d02403d405b1c0b;p=thirdparty%2Fbind9.git [master] dnstap size and versions options 4572. [func] The "dnstap-output" option can now take "size" and "versions" parameters to indicate the maximum size a dnstap log file can grow before rolling to a new file, and how many old files to retain. [RT #44502] --- diff --git a/CHANGES b/CHANGES index 990b2452700..404f3856f43 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,8 @@ +4572. [func] The "dnstap-output" option can now take "size" and + "versions" parameters to indicate the maximum size + a dnstap log file can grow before rolling to a new + file, and how many old files to retain. [RT #44502] + 4571. [bug] Out-of-tree builds of backtrace_test failed. 4570. [cleanup] named did not correctly fall back to the built-in diff --git a/bin/named/server.c b/bin/named/server.c index 521f745b9ec..f7c8cc19fdd 100644 --- a/bin/named/server.c +++ b/bin/named/server.c @@ -3036,7 +3036,6 @@ configure_dnstap(const cfg_obj_t **maps, dns_view_t *view) { const char *dpath = ns_g_defaultdnstap; const cfg_obj_t *dlist = NULL; dns_dtmsgtype_t dttypes = 0; - dns_dtmode_t dmode; unsigned int i; struct fstrm_iothr_options *fopt = NULL; @@ -3086,6 +3085,9 @@ configure_dnstap(const cfg_obj_t **maps, dns_view_t *view) { } if (ns_g_server->dtenv == NULL && dttypes != 0) { + dns_dtmode_t dmode; + isc_uint64_t max_size = 0; + isc_uint32_t rolls = 0; obj = NULL; CHECKM(ns_config_get(maps, "dnstap-output", &obj), "'dnstap-output' must be set if 'dnstap' is set"); @@ -3104,6 +3106,32 @@ configure_dnstap(const cfg_obj_t **maps, dns_view_t *view) { dpath = cfg_obj_asstring(obj2); + obj2 = cfg_tuple_get(obj, "size"); + if (obj2 != NULL && cfg_obj_isuint64(obj2)) { + max_size = cfg_obj_asuint64(obj2); + if (max_size > SIZE_MAX) { + cfg_obj_log(obj2, ns_g_lctx, + ISC_LOG_WARNING, + "'dnstap-output size " + "%" ISC_PRINT_QUADFORMAT "u' " + "is too large for this " + "system; reducing to %lu", + max_size, (unsigned long)SIZE_MAX); + max_size = SIZE_MAX; + } + } + + obj2 = cfg_tuple_get(obj, "versions"); + if (obj2 != NULL && cfg_obj_isuint32(obj2)) { + rolls = cfg_obj_asuint32(obj2); + } else if (obj2 != NULL && cfg_obj_isstring(obj2) && + strcasecmp(cfg_obj_asstring(obj2), "unlimited") == 0) + { + rolls = ISC_LOG_ROLLINFINITE; + } else { + rolls = ISC_LOG_ROLLNEVER; + } + fopt = fstrm_iothr_options_init(); fstrm_iothr_options_set_num_input_queues(fopt, ns_g_cpus); fstrm_iothr_options_set_queue_model(fopt, @@ -3168,9 +3196,12 @@ configure_dnstap(const cfg_obj_t **maps, dns_view_t *view) { fstrm_iothr_options_set_reopen_interval(fopt, i); } - CHECKM(dns_dt_create(ns_g_mctx, dmode, dpath, &fopt, - &ns_g_server->dtenv), + CHECKM(dns_dt_create(ns_g_mctx, dmode, dpath, + &fopt, &ns_g_server->dtenv), "unable to create dnstap environment"); + + CHECKM(dns_dt_setupfile(ns_g_server->dtenv, max_size, rolls), + "unable to set up dnstap logfile"); } if (ns_g_server->dtenv == NULL) @@ -13689,7 +13720,7 @@ ns_server_dnstap(ns_server_t *server, isc_lex_t *lex, isc_buffer_t **text) { } if (reopen || strcasecmp(ptr, "-reopen") == 0) { - backups = -1; + backups = ISC_LOG_ROLLNEVER; } else if ((strcasecmp(ptr, "-roll") == 0)) { unsigned int n; ptr = next_token(lex, text); diff --git a/bin/tests/system/dnstap/bad-size-version.conf b/bin/tests/system/dnstap/bad-size-version.conf new file mode 100644 index 00000000000..3fb6bc506a3 --- /dev/null +++ b/bin/tests/system/dnstap/bad-size-version.conf @@ -0,0 +1,11 @@ +/* + * Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +options { + dnstap-output unix "/tmp/dnstap.sock" size 10k versions 3; +}; diff --git a/bin/tests/system/dnstap/good-size-version.conf b/bin/tests/system/dnstap/good-size-version.conf new file mode 100644 index 00000000000..417f600044a --- /dev/null +++ b/bin/tests/system/dnstap/good-size-version.conf @@ -0,0 +1,11 @@ +/* + * Copyright (C) 2016 Internet Systems Consortium, Inc. ("ISC") + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + */ + +options { + dnstap-output file "/tmp/dnstap.log" size 10k versions 3; +}; diff --git a/bin/tests/system/dnstap/ns1/named.conf b/bin/tests/system/dnstap/ns1/named.conf index cadf23e5d10..8d7b27cfa4d 100644 --- a/bin/tests/system/dnstap/ns1/named.conf +++ b/bin/tests/system/dnstap/ns1/named.conf @@ -21,7 +21,7 @@ options { statistics-file "named.stats"; dnstap-identity "ns1"; dnstap-version "xxx"; - dnstap-output file "dnstap.out"; + dnstap-output file "dnstap.out" size 30k versions 10; dnstap { all; }; send-cookie no; require-server-cookie no; diff --git a/bin/tests/system/dnstap/tests.sh b/bin/tests/system/dnstap/tests.sh index 06fc24a9420..1a6ba6066c9 100644 --- a/bin/tests/system/dnstap/tests.sh +++ b/bin/tests/system/dnstap/tests.sh @@ -349,7 +349,7 @@ if [ -n "$PYTHON" ] ; then $PYTHON -c "import yaml" && HAS_PYYAML=1 fi -if [ $HAS_PYYAML ] ; then +if [ $HAS_PYYAML -ne 0 ] ; then echo "I:checking dnstap-read YAML output" ret=0 $PYTHON ydump.py "$DNSTAPREAD" "ns3/dnstap.out.save" > /dev/null || ret=1 diff --git a/doc/arm/Bv9ARM-book.xml b/doc/arm/Bv9ARM-book.xml index f37b2cb9bda..0f889ba37a7 100644 --- a/doc/arm/Bv9ARM-book.xml +++ b/doc/arm/Bv9ARM-book.xml @@ -4416,7 +4416,7 @@ badresp:1,adberr:0,findfail:0,valfail:0] [ server-id server_id_string ; ] [ directory path_name ; ] [ dnstap { message_type ; ... } ; ] - [ dnstap-output ( | ) path_name ; ] + [ dnstap-output ( | ) path_name [ size size_spec ] [ versions ( number | ) ] ; ] [ dnstap-identity ( string | | ) ; ] [ dnstap-version ( string | ) ; ] [ fstrm-set-buffer-hint number ; ] @@ -5000,6 +5000,19 @@ badresp:1,adberr:0,findfail:0,valfail:0] (provided with libfstrm) is listening on the socket.) + + If the first argument is file, then + two additional options can be added: + size indicates the size to which a + dnstap log file can grow before being + rolled to a new file, and versions + specifies the number of rolled log files to retain. These + are similar to the size and + versions options in a + logging channel. The default + is to allow dnstap log files to grow to + any size without rolling. + dnstap-output can only be set globally in options. Currently, it can only be diff --git a/doc/arm/notes.xml b/doc/arm/notes.xml index 4121401aa7a..68357837f1d 100644 --- a/doc/arm/notes.xml +++ b/doc/arm/notes.xml @@ -128,6 +128,19 @@
New Features + + + dnstap logfiles can now be configured to + automatically roll when they reach a specified size. If + dnstap-output is configured with mode + file, then it can take optional + size and versions + key-value arguments to set the logfile rolling parameters. + (These have the same semantics as the corresponding + options in a logging channel statement.) + [RT #44502] + + dig +ednsopt now accepts the names diff --git a/lib/bind9/check.c b/lib/bind9/check.c index c13b947eed9..34ce342adec 100644 --- a/lib/bind9/check.c +++ b/lib/bind9/check.c @@ -1437,6 +1437,46 @@ check_options(const cfg_obj_t *options, isc_log_t *logctx, isc_mem_t *mctx, } } } + + /* Check that dnstap-ouput values are consistent */ + obj = NULL; + (void) cfg_map_get(options, "dnstap-output", &obj); + if (obj != NULL) { + const cfg_obj_t *obj2; + dns_dtmode_t dmode; + + obj2 = cfg_tuple_get(obj, "mode"); + if (obj2 == NULL) { + cfg_obj_log(obj, logctx, ISC_LOG_ERROR, + "dnstap-output mode not found"); + return (ISC_R_FAILURE); + } + + if (strcasecmp(cfg_obj_asstring(obj2), "file") == 0) + dmode = dns_dtmode_file; + else + dmode = dns_dtmode_unix; + + obj2 = cfg_tuple_get(obj, "size"); + if (obj2 != NULL && !cfg_obj_isvoid(obj2) && + dmode == dns_dtmode_unix) + { + cfg_obj_log(obj, logctx, ISC_LOG_ERROR, + "dnstap-output size " + "cannot be set with mode unix"); + return (ISC_R_FAILURE); + } + + obj2 = cfg_tuple_get(obj, "versions"); + if (obj2 != NULL && !cfg_obj_isvoid(obj2) && + dmode == dns_dtmode_unix) + { + cfg_obj_log(obj, logctx, ISC_LOG_ERROR, + "dnstap-output versions " + "cannot be set with mode unix"); + return (ISC_R_FAILURE); + } + } #endif return (result); diff --git a/lib/dns/dnstap.c b/lib/dns/dnstap.c index 11a97b80ec0..e8b4564f64f 100644 --- a/lib/dns/dnstap.c +++ b/lib/dns/dnstap.c @@ -104,6 +104,8 @@ struct dns_dtenv { isc_region_t version; char *path; dns_dtmode_t mode; + isc_uint64_t max_size; + isc_uint32_t rolls; isc_stats_t *stats; }; @@ -241,6 +243,8 @@ dns_dt_create(isc_mem_t *mctx, dns_dtmode_t mode, const char *path, CHECK(ISC_R_FAILURE); } env->mode = mode; + env->max_size = 0; + env->rolls = ISC_LOG_ROLLNEVER; env->fopt = *foptp; *foptp = NULL; @@ -274,6 +278,22 @@ dns_dt_create(isc_mem_t *mctx, dns_dtmode_t mode, const char *path, return (result); } +isc_result_t +dns_dt_setupfile(dns_dtenv_t *env, isc_uint64_t max_size, int rolls) { + REQUIRE(VALID_DTENV(env)); + + if (max_size != 0 && rolls != ISC_LOG_ROLLNEVER && + env->mode != dns_dtmode_file) + { + return (ISC_R_INVALIDFILE); + } + + env->max_size = max_size; + env->rolls = rolls; + + return (ISC_R_SUCCESS); +} + isc_result_t dns_dt_reopen(dns_dtenv_t *env, int roll) { isc_result_t result = ISC_R_SUCCESS; @@ -691,6 +711,14 @@ dns_dt_send(dns_view_t *view, dns_dtmsgtype_t msgtype, REQUIRE(VALID_DTENV(view->dtenv)); + if (view->dtenv->max_size != 0) { + struct stat statbuf; + if (stat(view->dtenv->path, &statbuf) >= 0 && + (size_t) statbuf.st_size > view->dtenv->max_size) { + dns_dt_reopen(view->dtenv, view->dtenv->rolls); + } + } + TIME_NOW(&now); t = &now; diff --git a/lib/dns/include/dns/dnstap.h b/lib/dns/include/dns/dnstap.h index 7f6793b22ae..81f27067757 100644 --- a/lib/dns/include/dns/dnstap.h +++ b/lib/dns/include/dns/dnstap.h @@ -152,6 +152,24 @@ dns_dt_create(isc_mem_t *mctx, dns_dtmode_t mode, const char *path, *\li Other errors are possible. */ +isc_result_t +dns_dt_setupfile(dns_dtenv_t *env, isc_uint64_t max_size, int rolls); +/*%< + * Sets up the dnstap logfile limits. + * + * 'max_size' is the size a log file may grow before it is rolled + * + * 'rolls' is the number of rolled files to retain. + * + * Requires: + * + *\li 'env' is a valid dnstap environment. + * + * Returns: + *\li #ISC_R_SUCCESS on success + *\li #ISC_R_INVALIDFILE if dnstap is set to use a UNIX domain socket + */ + isc_result_t dns_dt_reopen(dns_dtenv_t *env, int roll); /*%< diff --git a/lib/isccfg/namedconf.c b/lib/isccfg/namedconf.c index af9a57e9777..db37d616f2b 100644 --- a/lib/isccfg/namedconf.c +++ b/lib/isccfg/namedconf.c @@ -107,6 +107,7 @@ static cfg_type_t cfg_type_key; static cfg_type_t cfg_type_logfile; static cfg_type_t cfg_type_logging; static cfg_type_t cfg_type_logseverity; +static cfg_type_t cfg_type_logversions; static cfg_type_t cfg_type_lwres; static cfg_type_t cfg_type_masterselement; static cfg_type_t cfg_type_maxttl; @@ -1297,6 +1298,18 @@ static cfg_type_t cfg_type_dnstap = { /*% * dnstap-output */ +static keyword_type_t dtsize_kw = { "size", &cfg_type_sizenodefault}; +static cfg_type_t cfg_type_dnstap_size = { + "dnstap_size", parse_optional_keyvalue, print_keyvalue, + doc_optional_keyvalue, &cfg_rep_uint64, &dtsize_kw +}; + +static keyword_type_t dtversions_kw = { "versions", &cfg_type_logversions}; +static cfg_type_t cfg_type_dnstap_versions = { + "dnstap_versions", parse_optional_keyvalue, print_keyvalue, + doc_optional_keyvalue, &cfg_rep_uint32, &dtversions_kw +}; + static const char *dtoutmode_enums[] = { "file", "unix", NULL }; static cfg_type_t cfg_type_dtmode = { "dtmode", cfg_parse_enum, cfg_print_ustring, cfg_doc_enum, @@ -1306,6 +1319,8 @@ static cfg_type_t cfg_type_dtmode = { static cfg_tuplefielddef_t dtout_fields[] = { { "mode", &cfg_type_dtmode, 0 }, { "path", &cfg_type_qstring, 0 }, + { "size", &cfg_type_dnstap_size, 0 }, + { "versions", &cfg_type_dnstap_versions, 0 }, { NULL, NULL, 0 } }; @@ -2384,7 +2399,6 @@ static cfg_type_t cfg_type_sizenodefault = { /*% * A size in absolute values or percents. */ - static cfg_type_t cfg_type_sizeval_percent = { "sizeval_percent", parse_sizeval_percent, cfg_print_ustring, doc_sizeval_percent, &cfg_rep_string, NULL