]> git.ipfire.org Git - thirdparty/bind9.git/commitdiff
4562. [func] Add additional memory statistics currently malloced
authorMark Andrews <marka@isc.org>
Thu, 2 Feb 2017 04:36:38 +0000 (15:36 +1100)
committerMark Andrews <marka@isc.org>
Thu, 2 Feb 2017 04:36:38 +0000 (15:36 +1100)
                        and maxmalloced per memory context. [RT #43593]

CHANGES
bin/named/bind9.xsl
bin/named/bind9.xsl.h
bin/named/statschannel.c
bin/tests/system/statschannel/clean.sh
bin/tests/system/statschannel/mem-xml.pl [new file with mode: 0644]
bin/tests/system/statschannel/tests.sh
lib/isc/mem.c

diff --git a/CHANGES b/CHANGES
index f6307b66d94503dd38fb56a5d15c62084b587a5e..2a0a3cee4c4395b1d897ff7de6c0d0d432edf913 100644 (file)
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,6 @@
+4562.  [func]          Add additional memory statistics currently malloced
+                       and maxmalloced per memory context. [RT #43593]
+
 4561.  [port]          Silence a warning in strict C99 compilers. [RT #44414]
 
 4560.  [bug]           mdig: add -m option to enable memory debugging rather
index 09aa18d956f08b8380afab74c03f476ea5807ae4..563894eda7025ec173c6004d4532ff376d82dd49 100644 (file)
@@ -9,7 +9,7 @@
 
 <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns="http://www.w3.org/1999/xhtml" version="1.0">
   <xsl:output method="html" indent="yes" version="4.0"/>
-  <xsl:template match="statistics[@version=&quot;3.8&quot;]">
+  <xsl:template match="statistics[@version=&quot;3.9&quot;]">
     <html>
       <head>
         <xsl:if test="system-property('xsl:vendor')!='Transformiix'">
               <th>TotalUse</th>
               <th>InUse</th>
               <th>MaxUse</th>
+              <th>Malloced</th>
+              <th>MaxMalloced</th>
               <th>BlockSize</th>
               <th>Pools</th>
               <th>HiWater</th>
                 <td>
                   <xsl:value-of select="maxinuse"/>
                 </td>
+                <td>
+                  <xsl:value-of select="malloced"/>
+                </td>
+                <td>
+                  <xsl:value-of select="maxmalloced"/>
+                </td>
                 <td>
                   <xsl:value-of select="blocksize"/>
                 </td>
index 52dc218d038a15b06ce4a0daf35f0180e0f7fb4e..e12366a8296186fc151586df701103120019b926 100644 (file)
@@ -14,7 +14,7 @@ static char xslmsg[] =
        "\n"
        "<xsl:stylesheet xmlns:xsl=\"http://www.w3.org/1999/XSL/Transform\" xmlns=\"http://www.w3.org/1999/xhtml\" version=\"1.0\">\n"
        " <xsl:output method=\"html\" indent=\"yes\" version=\"4.0\"/>\n"
-       " <xsl:template match=\"statistics[@version=&quot;3.8&quot;]\">\n"
+       " <xsl:template match=\"statistics[@version=&quot;3.9&quot;]\">\n"
        " <html>\n"
        " <head>\n"
        " <xsl:if test=\"system-property('xsl:vendor')!='Transformiix'\">\n"
@@ -965,6 +965,8 @@ static char xslmsg[] =
        " <th>TotalUse</th>\n"
        " <th>InUse</th>\n"
        " <th>MaxUse</th>\n"
+       " <th>Malloced</th>\n"
+       " <th>MaxMalloced</th>\n"
        " <th>BlockSize</th>\n"
        " <th>Pools</th>\n"
        " <th>HiWater</th>\n"
@@ -998,6 +1000,12 @@ static char xslmsg[] =
        " <xsl:value-of select=\"maxinuse\"/>\n"
        " </td>\n"
        " <td>\n"
+       " <xsl:value-of select=\"malloced\"/>\n"
+       " </td>\n"
+       " <td>\n"
+       " <xsl:value-of select=\"maxmalloced\"/>\n"
+       " </td>\n"
+       " <td>\n"
        " <xsl:value-of select=\"blocksize\"/>\n"
        " </td>\n"
        " <td>\n"
index f07ab555b25e179edf59a6bac0275fa456fffa02..3283190ff1607bbea3128cd034be1d51f74a41f2 100644 (file)
@@ -1530,7 +1530,7 @@ generatexml(ns_server_t *server, isc_uint32_t flags,
                        ISC_XMLCHAR "type=\"text/xsl\" href=\"/bind9.xsl\""));
        TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "statistics"));
        TRY0(xmlTextWriterWriteAttribute(writer, ISC_XMLCHAR "version",
-                                        ISC_XMLCHAR "3.8"));
+                                        ISC_XMLCHAR "3.9"));
 
        /* Set common fields for statistics dump */
        dumparg.type = isc_statsformat_xml;
@@ -2286,7 +2286,7 @@ generatejson(ns_server_t *server, size_t *msglen,
        /*
         * These statistics are included no matter which URL we use.
         */
-       obj = json_object_new_string("1.2");
+       obj = json_object_new_string("1.3");
        CHECKMEM(obj);
        json_object_object_add(bindstats, "json-stats-version", obj);
 
index 8f25c51147ec6847526e1dee7d7ad4162e818a1b..0c1c5320b8633897169798b40d9b21f7e0f48c8b 100644 (file)
@@ -13,4 +13,5 @@ rm -f */named.run
 rm -f ns*/named.lock
 rm -f ns*/named.stats
 rm -f xml.*stats json.*stats
+rm -f xml.*mem json.*mem
 rm -f compressed.headers regular.headers compressed.out regular.out
diff --git a/bin/tests/system/statschannel/mem-xml.pl b/bin/tests/system/statschannel/mem-xml.pl
new file mode 100644 (file)
index 0000000..b452896
--- /dev/null
@@ -0,0 +1,16 @@
+#!/usr/bin/perl
+#
+# Copyright (C) 2015, 2016  Internet Systems Consortium, Inc. ("ISC")
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# server-xml.pl:
+# Parses the XML version of the server stats into a normalized format.
+
+use XML::Simple;
+use Data::Dumper;
+
+my $ref = XMLin("xml.mem");
+print Dumper($ref);
index b37f40e14d6b9ba8040287d5819a4bd63a690588..a55322683b520612f5a2d9429ac52e5f930e6790 100644 (file)
@@ -172,6 +172,26 @@ if [ $ret != 0 ]; then echo "I: failed"; fi
 status=`expr $status + $ret`
 n=`expr $n + 1`
 
+ret=0
+echo "I:checking malloced memory statistics xml/json ($n)"
+if [ $PERL_XML ]; then
+    file=`$PERL fetch.pl xml/v3/mem`
+    mv $file xml.mem
+    $PERL mem-xml.pl $file > xml.fmtmem
+    grep "'Malloced' => '[0-9][0-9]*'" xml.fmtmem > /dev/null || ret=1
+    grep "'malloced' => '[0-9][0-9]*'" xml.fmtmem > /dev/null || ret=1
+    grep "'maxmalloced' => '[0-9][0-9]*'" xml.fmtmem > /dev/null || ret=1
+fi
+if [ $PERL_JSON ]; then
+    file=`$PERL fetch.pl json/v1/mem`
+    mv $file json.mem
+    grep '"malloced":[0-9][0-9]*,' json.mem > /dev/null || ret=1
+    grep '"maxmalloced":[0-9][0-9]*,' json.mem > /dev/null || ret=1
+    grep '"Malloced":[0-9][0-9]*,' json.mem > /dev/null || ret=1
+fi
+if [ $ret != 0 ]; then echo "I: failed"; fi
+status=`expr $status + $ret`
+n=`expr $n + 1`
 
 ret=0
 echo "I:checking consistency between regular and compressed output ($n)"
index 6781a9146e954ca5baf33100cf398e03f19ed5aa..1a289ab125014dba15314894d8f98e51cc60e408 100644 (file)
@@ -135,6 +135,8 @@ struct isc__mem {
        size_t                  total;
        size_t                  inuse;
        size_t                  maxinuse;
+       size_t                  malloced;
+       size_t                  maxmalloced;
        size_t                  hi_water;
        size_t                  lo_water;
        isc_boolean_t           hi_called;
@@ -414,6 +416,9 @@ add_trace_entry(isc__mem_t *mctx, const void *ptr, size_t size FLARG) {
 
        dl = malloc(sizeof(debuglink_t));
        INSIST(dl != NULL);
+       mctx->malloced += sizeof(debuglink_t);
+       if (mctx->malloced > mctx->maxmalloced)
+               mctx->maxmalloced = mctx->malloced;
 
        ISC_LINK_INIT(dl, link);
        for (i = 1; i < DEBUGLIST_COUNT; i++) {
@@ -468,6 +473,7 @@ delete_trace_entry(isc__mem_t *mctx, const void *ptr, size_t size,
                                        ISC_LIST_UNLINK(mctx->debuglist[size],
                                                        dl, link);
                                        free(dl);
+                                       mctx->malloced -= sizeof(*dl);
                                }
                                return;
                        }
@@ -532,11 +538,16 @@ more_basic_blocks(isc__mem_t *ctx) {
                        ctx->memalloc_failures++;
                        return (ISC_FALSE);
                }
+               ctx->malloced += table_size * sizeof(unsigned char *);
+               if (ctx->malloced > ctx->maxmalloced)
+                       ctx->maxmalloced = ctx->malloced;
                if (ctx->basic_table_size != 0) {
                        memmove(table, ctx->basic_table,
                                ctx->basic_table_size *
                                  sizeof(unsigned char *));
                        (ctx->memfree)(ctx->arg, ctx->basic_table);
+                       ctx->malloced -= ctx->basic_table_size *
+                                        sizeof(unsigned char *);
                }
                ctx->basic_table = table;
                ctx->basic_table_size = table_size;
@@ -550,6 +561,9 @@ more_basic_blocks(isc__mem_t *ctx) {
        ctx->total += increment;
        ctx->basic_table[ctx->basic_table_count] = new;
        ctx->basic_table_count++;
+       ctx->malloced += NUM_BASIC_BLOCKS * ctx->mem_target;
+       if (ctx->malloced > ctx->maxmalloced)
+                       ctx->maxmalloced = ctx->malloced;
 
        curr = new;
        next = curr + ctx->mem_target;
@@ -658,6 +672,9 @@ mem_getunlocked(isc__mem_t *ctx, size_t size) {
                ctx->inuse += size;
                ctx->stats[ctx->max_size].gets++;
                ctx->stats[ctx->max_size].totalgets++;
+               ctx->malloced += size;
+               if (ctx->malloced > ctx->maxmalloced)
+                       ctx->maxmalloced = ctx->malloced;
                /*
                 * If we don't set new_size to size, then the
                 * ISC_MEM_FILL code might write over bytes we
@@ -735,6 +752,7 @@ mem_putunlocked(isc__mem_t *ctx, void *mem, size_t size) {
                ctx->stats[ctx->max_size].gets--;
                INSIST(size <= ctx->inuse);
                ctx->inuse -= size;
+               ctx->malloced -= size;
                return;
        }
 
@@ -787,6 +805,11 @@ mem_get(isc__mem_t *ctx, size_t size) {
                ret[size-1] = 0xbe;
 #  endif
 #endif
+       if (ret != NULL) {
+               ctx->malloced += size;
+               if (ctx->malloced > ctx->maxmalloced)
+                       ctx->maxmalloced = ctx->malloced;
+       }
 
        return (ret);
 }
@@ -802,10 +825,9 @@ mem_put(isc__mem_t *ctx, void *mem, size_t size) {
 #endif
 #if ISC_MEM_FILL
        memset(mem, 0xde, size); /* Mnemonic for "dead". */
-#else
-       UNUSED(size);
 #endif
        (ctx->memfree)(ctx->arg, mem);
+       ctx->malloced -= size;
 }
 
 /*!
@@ -924,6 +946,8 @@ isc_mem_createx2(size_t init_max_size, size_t target_size,
        ctx->total = 0;
        ctx->inuse = 0;
        ctx->maxinuse = 0;
+       ctx->malloced = sizeof(*ctx);
+       ctx->maxmalloced = sizeof(*ctx);
        ctx->hi_water = 0;
        ctx->lo_water = 0;
        ctx->hi_called = ISC_FALSE;
@@ -960,6 +984,8 @@ isc_mem_createx2(size_t init_max_size, size_t target_size,
                goto error;
        }
        memset(ctx->stats, 0, (ctx->max_size + 1) * sizeof(struct stats));
+       ctx->malloced += (ctx->max_size+1) * sizeof(struct stats);
+       ctx->maxmalloced += (ctx->max_size+1) * sizeof(struct stats);
 
        if ((flags & ISC_MEMFLAG_INTERNAL) != 0) {
                if (target_size == 0U)
@@ -974,6 +1000,8 @@ isc_mem_createx2(size_t init_max_size, size_t target_size,
                }
                memset(ctx->freelists, 0,
                       ctx->max_size * sizeof(element *));
+               ctx->malloced += ctx->max_size * sizeof(element *);
+               ctx->maxmalloced += ctx->max_size * sizeof(element *);
        }
 
 #if ISC_MEM_TRACKLINES
@@ -988,6 +1016,8 @@ isc_mem_createx2(size_t init_max_size, size_t target_size,
                }
                for (i = 0; i <= ctx->max_size; i++)
                        ISC_LIST_INIT(ctx->debuglist[i]);
+               ctx->malloced += (ctx->max_size+1) * sizeof(debuglist_t);
+               ctx->maxmalloced += (ctx->max_size+1) * sizeof(debuglist_t);
        }
 #endif
 
@@ -1051,9 +1081,11 @@ destroy(isc__mem_t *ctx) {
                                        ISC_LIST_UNLINK(ctx->debuglist[i],
                                                        dl, link);
                                        free(dl);
+                                       ctx->malloced -= sizeof(*dl);
                                }
                }
                (ctx->memfree)(ctx->arg, ctx->debuglist);
+               ctx->malloced -= (ctx->max_size+1) * sizeof(debuglist_t);
        }
 #endif
        INSIST(ctx->references == 0);
@@ -1075,19 +1107,29 @@ destroy(isc__mem_t *ctx) {
        }
 
        (ctx->memfree)(ctx->arg, ctx->stats);
+       ctx->malloced -= (ctx->max_size+1) * sizeof(struct stats);
 
        if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) {
-               for (i = 0; i < ctx->basic_table_count; i++)
+               for (i = 0; i < ctx->basic_table_count; i++) {
                        (ctx->memfree)(ctx->arg, ctx->basic_table[i]);
+                       ctx->malloced -= NUM_BASIC_BLOCKS * ctx->mem_target;
+               }
                (ctx->memfree)(ctx->arg, ctx->freelists);
-               if (ctx->basic_table != NULL)
+               ctx->malloced -= ctx->max_size * sizeof(element *);
+               if (ctx->basic_table != NULL) {
                        (ctx->memfree)(ctx->arg, ctx->basic_table);
+                       ctx->malloced -= ctx->basic_table_size *
+                                         sizeof(unsigned char *);
+               }
        }
 
        ondest = ctx->ondestroy;
 
        if ((ctx->flags & ISC_MEMFLAG_NOLOCK) == 0)
                DESTROYLOCK(&ctx->lock);
+       ctx->malloced -= sizeof(*ctx);
+       if (ctx->checkfree)
+               INSIST(ctx->malloced == 0);
        (ctx->memfree)(ctx->arg, ctx);
 
        isc_ondestroy_notify(&ondest, ctx);
@@ -2323,6 +2365,7 @@ isc_mem_references(isc_mem_t *ctx0) {
 typedef struct summarystat {
        isc_uint64_t    total;
        isc_uint64_t    inuse;
+       isc_uint64_t    malloced;
        isc_uint64_t    blocksize;
        isc_uint64_t    contextsize;
 } summarystat_t;
@@ -2387,6 +2430,19 @@ xml_renderctx(isc__mem_t *ctx, summarystat_t *summary,
                                            (isc_uint64_t)ctx->maxinuse));
        TRY0(xmlTextWriterEndElement(writer)); /* maxinuse */
 
+       summary->malloced += ctx->malloced;
+       TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "malloced"));
+       TRY0(xmlTextWriterWriteFormatString(writer,
+                                           "%" ISC_PRINT_QUADFORMAT "u",
+                                           (isc_uint64_t)ctx->malloced));
+       TRY0(xmlTextWriterEndElement(writer)); /* malloced */
+
+       TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "maxmalloced"));
+       TRY0(xmlTextWriterWriteFormatString(writer,
+                                           "%" ISC_PRINT_QUADFORMAT "u",
+                                           (isc_uint64_t)ctx->maxmalloced));
+       TRY0(xmlTextWriterEndElement(writer)); /* maxmalloced */
+
        TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "blocksize"));
        if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) {
                summary->blocksize += ctx->basic_table_count *
@@ -2468,6 +2524,12 @@ isc_mem_renderxml(xmlTextWriterPtr writer) {
                                            summary.inuse));
        TRY0(xmlTextWriterEndElement(writer)); /* InUse */
 
+       TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "Malloced"));
+       TRY0(xmlTextWriterWriteFormatString(writer,
+                                           "%" ISC_PRINT_QUADFORMAT "u",
+                                           summary.malloced));
+       TRY0(xmlTextWriterEndElement(writer)); /* InUse */
+
        TRY0(xmlTextWriterStartElement(writer, ISC_XMLCHAR "BlockSize"));
        TRY0(xmlTextWriterWriteFormatString(writer,
                                            "%" ISC_PRINT_QUADFORMAT "u",
@@ -2519,6 +2581,7 @@ json_renderctx(isc__mem_t *ctx, summarystat_t *summary, json_object *array) {
                ctx->basic_table_count * sizeof(char *);
        summary->total += ctx->total;
        summary->inuse += ctx->inuse;
+       summary->malloced += ctx->malloced;
        if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0)
                summary->blocksize += ctx->basic_table_count *
                        NUM_BASIC_BLOCKS * ctx->mem_target;
@@ -2560,6 +2623,14 @@ json_renderctx(isc__mem_t *ctx, summarystat_t *summary, json_object *array) {
        CHECKMEM(obj);
        json_object_object_add(ctxobj, "maxinuse", obj);
 
+       obj = json_object_new_int64(ctx->malloced);
+       CHECKMEM(obj);
+       json_object_object_add(ctxobj, "malloced", obj);
+
+       obj = json_object_new_int64(ctx->maxmalloced);
+       CHECKMEM(obj);
+       json_object_object_add(ctxobj, "maxmalloced", obj);
+
        if ((ctx->flags & ISC_MEMFLAG_INTERNAL) != 0) {
                isc_uint64_t blocksize;
                blocksize = ctx->basic_table_count * NUM_BASIC_BLOCKS *
@@ -2629,6 +2700,10 @@ isc_mem_renderjson(json_object *memobj) {
        CHECKMEM(obj);
        json_object_object_add(memobj, "InUse", obj);
 
+       obj = json_object_new_int64(summary.malloced);
+       CHECKMEM(obj);
+       json_object_object_add(memobj, "Malloced", obj);
+
        obj = json_object_new_int64(summary.blocksize);
        CHECKMEM(obj);
        json_object_object_add(memobj, "BlockSize", obj);