]> git.ipfire.org Git - thirdparty/bind9.git/commitdiff
Set a LMDB mapsize and also provide a config option to control it (#44954)
authorMukund Sivaraman <muks@isc.org>
Wed, 26 Apr 2017 18:13:35 +0000 (23:43 +0530)
committerMukund Sivaraman <muks@isc.org>
Wed, 26 Apr 2017 18:21:26 +0000 (23:51 +0530)
18 files changed:
CHANGES
bin/named/config.c
bin/named/named.conf.docbook
bin/named/server.c
bin/tests/system/checkconf/bad-lmdb-mapsize-bogus.conf [new file with mode: 0644]
bin/tests/system/checkconf/bad-lmdb-mapsize-toolarge.conf [new file with mode: 0644]
bin/tests/system/checkconf/bad-lmdb-mapsize-toosmall.conf [new file with mode: 0644]
bin/tests/system/checkconf/bad-lmdb-mapsize-unlimited.conf [new file with mode: 0644]
bin/tests/system/checkconf/good-lmdb-mapsize-largest.conf [new file with mode: 0644]
bin/tests/system/checkconf/good-lmdb-mapsize-smallest.conf [new file with mode: 0644]
doc/arm/Bv9ARM-book.xml
doc/misc/options
lib/bind9/check.c
lib/dns/include/dns/view.h
lib/dns/view.c
lib/isccfg/include/isccfg/grammar.h
lib/isccfg/namedconf.c
lib/isccfg/parser.c

diff --git a/CHANGES b/CHANGES
index 7e2a5f8c871dee2770825d48ca7e511d0c74ae81..acaf41e29c08647a4babe1e7014622ce54992b9a 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,10 @@
+4611.  [bug]           The default LMDB mapsize was too low and caused
+                       errors after few thousand zones were added using
+                       rndc addzone. A new config option "lmdb-mapsize"
+                       has been introduced to configure the LMDB
+                       mapsize depending on operational needs.
+                       [RT #44954]
+
 4610.  [func]          The "new-zones-directory" option specifies the
                        location of NZF or NZD files for storing
                        configuration of zones added by "rndc addzone".
index 585528fc84bfdf6f8ce65d624ce9d68994216220..cd5488bc74fdd3b354eed8fc30aeeb44f1dac396 100644 (file)
@@ -176,6 +176,7 @@ options {\n\
        zero-no-soa-ttl-cache no;\n\
        nsec3-test-zone no;\n\
        allow-new-zones no;\n\
+       lmdb-mapsize 32M;\n\
        fetches-per-server 0;\n\
        require-server-cookie no;\n\
        v6-bias 50;\n\
index 80481c811bf90e888fb0583e2b39c4af2b9fa262..3f99c779b6bec36f7e7ad371aaf038ce703eec0e 100644 (file)
@@ -1,5 +1,5 @@
 <!--
- - Copyright (C) 2004-2017  Internet Systems Consortium, Inc. ("ISC")
+ - Copyright (C) 2004-2017 Internet Systems Consortium, Inc. ("ISC")
  -
  - This Source Code Form is subject to the terms of the Mozilla Public
  - License, v. 2.0. If a copy of the MPL was not distributed with this
@@ -10,7 +10,7 @@
 
 <refentry xmlns:db="http://docbook.org/ns/docbook" version="5.0" xml:id="man.named.conf">
   <info>
-    <date>2017-03-08</date>
+    <date>2017-04-25</date>
   </info>
   <refentryinfo>
     <corpname>ISC</corpname>
@@ -320,6 +320,7 @@ options {
        listen-on-v6 <optional> port <replaceable>integer</replaceable> </optional> <optional> dscp
            <replaceable>integer</replaceable> </optional> {
            <replaceable>address_match_element</replaceable>; ... };
+       lmdb-mapsize <replaceable>sizeval</replaceable>;
        lock-file ( <replaceable>quoted_string</replaceable> | none );
        managed-keys-directory <replaceable>quoted_string</replaceable>;
        masterfile-format ( map | raw | text );
@@ -636,6 +637,7 @@ view <replaceable>string</replaceable> <optional> <replaceable>class</replaceabl
        };
        key-directory <replaceable>quoted_string</replaceable>;
        lame-ttl <replaceable>ttlval</replaceable>;
+       lmdb-mapsize <replaceable>sizeval</replaceable>;
        managed-keys { <replaceable>string</replaceable> <replaceable>string</replaceable>
            <replaceable>integer</replaceable> <replaceable>integer</replaceable> <replaceable>integer</replaceable>
            <replaceable>quoted_string</replaceable>; ... };
index d11baaaaf00d28a9c2b37f35b6f11cf968b1c945..f8467ffa5fd37d3e1a9d7a2083bd1764fd9da318 100644 (file)
@@ -6631,7 +6631,9 @@ setup_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
        const cfg_obj_t *nz = NULL;
        const cfg_obj_t *nzdir = NULL;
        const char *dir = NULL;
+       const cfg_obj_t *obj = NULL;
        int i = 0;
+       isc_uint64_t mapsize = 0ULL;
 
        REQUIRE (config != NULL);
 
@@ -6665,6 +6667,32 @@ setup_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
                dns_view_setnewzonedir(view, dir);
        }
 
+#ifdef HAVE_LMDB
+       result = ns_config_get(maps, "lmdb-mapsize", &obj);
+       if (result == ISC_R_SUCCESS && obj != NULL) {
+               mapsize = cfg_obj_asuint64(obj);
+               if (mapsize < (1ULL << 20)) { /* 1 megabyte */
+                       cfg_obj_log(obj, ns_g_lctx,
+                                   ISC_LOG_ERROR,
+                                   "'lmdb-mapsize "
+                                   "%" ISC_PRINT_QUADFORMAT "d' "
+                                   "is too small",
+                                   mapsize);
+                       return (ISC_R_FAILURE);
+               } else if (mapsize > (1ULL << 40)) { /* 1 terabyte */
+                       cfg_obj_log(obj, ns_g_lctx,
+                                   ISC_LOG_ERROR,
+                                   "'lmdb-mapsize "
+                                   "%" ISC_PRINT_QUADFORMAT "d' "
+                                   "is too large",
+                                   mapsize);
+                       return (ISC_R_FAILURE);
+               }
+       }
+#else
+       UNUSED(obj);
+#endif /* HAVE_LMDB */
+
        /*
         * A non-empty catalog-zones statement implies allow-new-zones
         */
@@ -6680,7 +6708,7 @@ setup_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
        }
 
        if (!allow) {
-               dns_view_setnewzones(view, ISC_FALSE, NULL, NULL);
+               dns_view_setnewzones(view, ISC_FALSE, NULL, NULL, 0ULL);
                if (num_zones != NULL)
                        *num_zones = 0;
                return (ISC_R_SUCCESS);
@@ -6688,14 +6716,14 @@ setup_newzones(dns_view_t *view, cfg_obj_t *config, cfg_obj_t *vconfig,
 
        nzcfg = isc_mem_get(view->mctx, sizeof(*nzcfg));
        if (nzcfg == NULL) {
-               dns_view_setnewzones(view, ISC_FALSE, NULL, NULL);
+               dns_view_setnewzones(view, ISC_FALSE, NULL, NULL, 0ULL);
                return (ISC_R_NOMEMORY);
        }
 
        memset(nzcfg, 0, sizeof(*nzcfg));
 
        result = dns_view_setnewzones(view, ISC_TRUE, nzcfg,
-                                     newzone_cfgctx_destroy);
+                                     newzone_cfgctx_destroy, mapsize);
        if (result != ISC_R_SUCCESS) {
                isc_mem_free(view->mctx, nzcfg);
                return (result);
@@ -11140,11 +11168,25 @@ nzd_save(MDB_txn **txnp, MDB_dbi dbi, dns_zone_t *zone,
                const cfg_obj_t *zoptions;
 
                result = isc_buffer_allocate(view->mctx, &text, 256);
-               if (result != ISC_R_SUCCESS)
+               if (result != ISC_R_SUCCESS) {
+                       isc_log_write(ns_g_lctx,
+                                     NS_LOGCATEGORY_GENERAL,
+                                     NS_LOGMODULE_SERVER,
+                                     ISC_LOG_ERROR,
+                                     "Unable to allocate buffer in "
+                                     "nzd_save(): %s",
+                                     isc_result_totext(result));
                        goto cleanup;
+               }
 
                zoptions = cfg_tuple_get(zconfig, "options");
                if (zoptions == NULL) {
+                       isc_log_write(ns_g_lctx,
+                                     NS_LOGCATEGORY_GENERAL,
+                                     NS_LOGMODULE_SERVER,
+                                     ISC_LOG_ERROR,
+                                     "Unable to get options from config in "
+                                     "nzd_save()");
                        result = ISC_R_FAILURE;
                        goto cleanup;
                }
@@ -11177,8 +11219,16 @@ nzd_save(MDB_txn **txnp, MDB_dbi dbi, dns_zone_t *zone,
                (void) mdb_txn_abort(*txnp);
        else {
                status = mdb_txn_commit(*txnp);
-               if (status != 0)
+               if (status != 0) {
+                       isc_log_write(ns_g_lctx,
+                                     NS_LOGCATEGORY_GENERAL,
+                                     NS_LOGMODULE_SERVER,
+                                     ISC_LOG_ERROR,
+                                     "Error committing "
+                                     "NZD database: %s",
+                                     mdb_strerror(status));
                        result = ISC_R_FAILURE;
+               }
        }
        *txnp = NULL;
 
@@ -11694,8 +11744,18 @@ do_addzone(ns_server_t *server, ns_cfgctx_t *cfg, dns_view_t *view,
                if (view->redirect == NULL)
                        CHECK(ISC_R_NOTFOUND);
                dns_zone_attach(view->redirect, &zone);
-       } else
-               CHECK(dns_zt_find(view->zonetable, name, 0, NULL, &zone));
+       } else {
+               result = dns_zt_find(view->zonetable, name, 0, NULL, &zone);
+               if (result != ISC_R_SUCCESS) {
+                       isc_log_write(ns_g_lctx,
+                                     NS_LOGCATEGORY_GENERAL,
+                                     NS_LOGMODULE_SERVER,
+                                     ISC_LOG_ERROR,
+                                     "added new zone was not found: %s",
+                                     isc_result_totext(result));
+                       goto cleanup;
+               }
+       }
 
 #ifndef HAVE_LMDB
        /*
diff --git a/bin/tests/system/checkconf/bad-lmdb-mapsize-bogus.conf b/bin/tests/system/checkconf/bad-lmdb-mapsize-bogus.conf
new file mode 100644 (file)
index 0000000..f27e87f
--- /dev/null
@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+       lmdb-mapsize bogusvalue;
+};
diff --git a/bin/tests/system/checkconf/bad-lmdb-mapsize-toolarge.conf b/bin/tests/system/checkconf/bad-lmdb-mapsize-toolarge.conf
new file mode 100644 (file)
index 0000000..3315091
--- /dev/null
@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+       lmdb-mapsize 2048G;
+};
diff --git a/bin/tests/system/checkconf/bad-lmdb-mapsize-toosmall.conf b/bin/tests/system/checkconf/bad-lmdb-mapsize-toosmall.conf
new file mode 100644 (file)
index 0000000..803ea6b
--- /dev/null
@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+       lmdb-mapsize 1;
+};
diff --git a/bin/tests/system/checkconf/bad-lmdb-mapsize-unlimited.conf b/bin/tests/system/checkconf/bad-lmdb-mapsize-unlimited.conf
new file mode 100644 (file)
index 0000000..b1b8066
--- /dev/null
@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+       lmdb-mapsize unlimited;
+};
diff --git a/bin/tests/system/checkconf/good-lmdb-mapsize-largest.conf b/bin/tests/system/checkconf/good-lmdb-mapsize-largest.conf
new file mode 100644 (file)
index 0000000..9625e57
--- /dev/null
@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+       lmdb-mapsize 1024G;
+};
diff --git a/bin/tests/system/checkconf/good-lmdb-mapsize-smallest.conf b/bin/tests/system/checkconf/good-lmdb-mapsize-smallest.conf
new file mode 100644 (file)
index 0000000..0f6fa79
--- /dev/null
@@ -0,0 +1,11 @@
+/*
+ * Copyright (C) 2016  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+options {
+       lmdb-mapsize 1M;
+};
index 30468bf70f5cdb5672e60232e10aa70276e8168c..2facfe03e592113ed765fbd9aeb543b6a7abd5e9 100644 (file)
@@ -5085,6 +5085,31 @@ badresp:1,adberr:0,findfail:0,valfail:0]
            </listitem>
          </varlistentry>
 
+         <varlistentry>
+           <term><command>lmdb-mapsize</command></term>
+           <listitem>
+             <para>
+               When <command>named</command> is built with liblmdb,
+               this option sets a maximum size for the memory map of
+               the new-zone database (NZD) in LMDB database format.
+               This database is used to store configuration information
+               for zones added using <command>rndc addzone</command>.
+               Note that this is not the NZD database file size, but
+               the largest size that the database may grow to.
+             </para>
+             <para>
+               Because the database file is memory mapped, its size is
+               limited by the address space of the named process.  The
+               default of 32 megabytes was chosen to be usable with
+               32-bit <command>named</command> builds.  The largest
+               permitted value is 1 terabyte. Given typical zone
+               configurations without elaborate ACLs, a 32 MB NZD file
+               ought to be able to hold configurations of about 100,000
+               zones.
+             </para>
+           </listitem>
+         </varlistentry>
+
          <varlistentry>
            <term><command>managed-keys-directory</command></term>
            <listitem>
@@ -5954,6 +5979,15 @@ options {
                  incompatible with use as a file name, in which case a
                  cryptographic hash of the view name is used instead.
                </para>
+               <para>
+                 Zones added at runtime will have their configuration
+                 stored either in a new-zone file (NZF) or a new-zone
+                 database (NZD) depending on whether
+                 <command>named</command> was linked with
+                 liblmdb at compile time.
+                 See <xref linkend="man.rndc"/> for further details
+                 about <command>rndc addzone</command>.
+               </para>
              </listitem>
            </varlistentry>
 
index 306970f64ea3c5c0aa1da62db93d018af8554d1d..64663969e758b76821cc75f03d6764be211ffb7e 100644 (file)
@@ -207,6 +207,7 @@ options {
         listen-on-v6 [ port <integer> ] [ dscp
             <integer> ] {
             <address_match_element>; ... }; // may occur multiple times
+        lmdb-mapsize <sizeval>;
         lock-file ( <quoted_string> | none );
         maintain-ixfr-base <boolean>; // obsolete
         managed-keys-directory <quoted_string>;
@@ -527,6 +528,7 @@ view <string> [ <class> ] {
         }; // may occur multiple times
         key-directory <quoted_string>;
         lame-ttl <ttlval>;
+        lmdb-mapsize <sizeval>;
         maintain-ixfr-base <boolean>; // obsolete
         managed-keys { <string> <string>
             <integer> <integer> <integer>
index e691d4b07e5c89d827eebdbc2583fe37e200190e..097dd96a0be7793623c5769db43e7c9856f2374b 100644 (file)
@@ -22,6 +22,7 @@
 #include <isc/netaddr.h>
 #include <isc/parseint.h>
 #include <isc/platform.h>
+#include <isc/print.h>
 #include <isc/region.h>
 #include <isc/result.h>
 #include <isc/sha1.h>
@@ -1490,6 +1491,30 @@ check_options(const cfg_obj_t *options, isc_log_t *logctx, isc_mem_t *mctx,
        }
 #endif
 
+       obj = NULL;
+       (void)cfg_map_get(options, "lmdb-mapsize", &obj);
+       if (obj != NULL) {
+               isc_uint64_t mapsize = cfg_obj_asuint64(obj);
+
+               if (mapsize < (1ULL << 20)) { /* 1 megabyte */
+                       cfg_obj_log(obj, logctx,
+                                   ISC_LOG_ERROR,
+                                   "'lmdb-mapsize "
+                                   "%" ISC_PRINT_QUADFORMAT "d' "
+                                   "is too small",
+                                   mapsize);
+                       return (ISC_R_RANGE);
+               } else if (mapsize > (1ULL << 40)) { /* 1 terabyte */
+                       cfg_obj_log(obj, logctx,
+                                   ISC_LOG_ERROR,
+                                   "'lmdb-mapsize "
+                                   "%" ISC_PRINT_QUADFORMAT "d' "
+                                   "is too large",
+                                   mapsize);
+                       return (ISC_R_RANGE);
+               }
+       }
+
        return (result);
 }
 
index 846eee6ebc785795609acfe78c68f4c997bcc7db..e70b3f6f76cb8392a743a8057055d5b37f61b2b1 100644 (file)
@@ -1216,7 +1216,7 @@ dns_view_untrust(dns_view_t *view, const dns_name_t *keyname,
 
 isc_result_t
 dns_view_setnewzones(dns_view_t *view, isc_boolean_t allow, void *cfgctx,
-                    void (*cfg_destroy)(void **));
+                    void (*cfg_destroy)(void **), isc_uint64_t mapsize);
 /*%<
  * Set whether or not to allow zones to be created or deleted at runtime.
  *
index 8f465b66587536baffc9d7b408d6e618ddd636a1..4a8a55d52ed38f20b7b0fbbef176c23e66afe269 100644 (file)
@@ -503,7 +503,7 @@ destroy(dns_view_t *view) {
        if (view->dtenv != NULL)
                dns_dt_detach(&view->dtenv);
 #endif /* HAVE_DNSTAP */
-       dns_view_setnewzones(view, ISC_FALSE, NULL, NULL);
+       dns_view_setnewzones(view, ISC_FALSE, NULL, NULL, 0ULL);
        if (view->new_zone_file != NULL) {
                isc_mem_free(view->mctx, view->new_zone_file);
                view->new_zone_file = NULL;
@@ -2031,7 +2031,7 @@ nz_legacy(const char *directory, const char *viewname,
 
 isc_result_t
 dns_view_setnewzones(dns_view_t *view, isc_boolean_t allow, void *cfgctx,
-                    void (*cfg_destroy)(void **))
+                    void (*cfg_destroy)(void **), isc_uint64_t mapsize)
 {
        isc_result_t result;
        char buffer[1024];
@@ -2088,6 +2088,14 @@ dns_view_setnewzones(dns_view_t *view, isc_boolean_t allow, void *cfgctx,
                goto out;
        }
 
+       if (mapsize != 0ULL) {
+               status = mdb_env_set_mapsize(env, mapsize);
+               if (status != 0) {
+                       result = ISC_R_FAILURE;
+                       goto out;
+               }
+       }
+
        status = mdb_env_open(env, view->new_zone_db,
                              MDB_NOSUBDIR|MDB_CREATE, 0600);
        if (status != 0) {
index 087db23a10dcad3f49c61a6205c9ed8ac4c8b62b..e5fb79f1ae5d9bbe4bbe3752d4311568b6734737 100644 (file)
@@ -48,6 +48,9 @@
 #define CFG_CLAUSEFLAG_NOTCONFIGURED   0x00000080
 /*% A option for a experimental feature. */
 #define CFG_CLAUSEFLAG_EXPERIMENTAL    0x00000100
+/*% A configuration option that is ineffective due to
+ * compile time options, but is harmless. */
+#define CFG_CLAUSEFLAG_NOOP            0x00000200
 
 typedef struct cfg_clausedef cfg_clausedef_t;
 typedef struct cfg_tuplefielddef cfg_tuplefielddef_t;
index 2045190a26c23ab8a5356698c11d2d5bf6a5de69..f13178f5af5178342dbf0972e6bba87d9bf00f39 100644 (file)
@@ -134,6 +134,7 @@ static cfg_type_t cfg_type_server_key_kludge;
 static cfg_type_t cfg_type_size;
 static cfg_type_t cfg_type_sizenodefault;
 static cfg_type_t cfg_type_sizeorpercent;
+static cfg_type_t cfg_type_sizeval;
 static cfg_type_t cfg_type_sockaddr4wild;
 static cfg_type_t cfg_type_sockaddr6wild;
 static cfg_type_t cfg_type_statschannels;
@@ -1882,6 +1883,11 @@ view_clauses[] = {
 #endif
        { "ixfr-from-differences", &cfg_type_ixfrdifftype, 0 },
        { "lame-ttl", &cfg_type_ttlval, 0 },
+#ifdef HAVE_LMDB
+       { "lmdb-mapsize", &cfg_type_sizeval, 0 },
+#else
+       { "lmdb-mapsize", &cfg_type_sizeval, CFG_CLAUSEFLAG_NOOP },
+#endif
        { "nocookie-udp-size", &cfg_type_uint32, 0 },
        { "nosit-udp-size", &cfg_type_uint32, CFG_CLAUSEFLAG_OBSOLETE },
        { "max-acache-size", &cfg_type_sizenodefault,
index c77cbefbceae07d82d45da7c8e6430e6e5467423..13c5151ad48defdcb5fcbbbb6a7f8114dc00868b 100644 (file)
@@ -1717,6 +1717,12 @@ cfg_parse_mapbody(cfg_parser_t *pctx, const cfg_type_t *type, cfg_obj_t **ret)
                        cfg_parser_warning(pctx, 0, "option '%s' is "
                                       "not implemented", clause->name);
 
+               if ((clause->flags & CFG_CLAUSEFLAG_NOOP) != 0) {
+                       cfg_parser_warning(pctx, 0, "option '%s' was not "
+                                          "enabled at compile time "
+                                          "(ignored)", clause->name);
+               }
+
                if ((clause->flags & CFG_CLAUSEFLAG_NOTCONFIGURED) != 0) {
                        cfg_parser_warning(pctx, 0, "option '%s' was not "
                                           "enabled at compile time",
@@ -1973,6 +1979,7 @@ static struct flagtext {
        { CFG_CLAUSEFLAG_NOTCONFIGURED, "not configured" },
        { CFG_CLAUSEFLAG_MULTI, "may occur multiple times" },
        { CFG_CLAUSEFLAG_EXPERIMENTAL, "experimental" },
+       { CFG_CLAUSEFLAG_NOOP, "non-operational" },
        { 0, NULL }
 };