Implement SBuf and MemBlob size distribution stats

diff --git a/src/Makefile.am b/src/Makefile.am
index 97a94c1d2d42b527c7cb75e7b001ce46c926a227..930e7dbf5cbaab3b69d3beb9c3a7d17586de8c5f 100644 (file)
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -481,6 +481,8 @@ squid_SOURCES = \
        send-announce.h \
        send-announce.cc \
        $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       SBufDetailedStats.cc \
        SBufStatsAction.h \
        SBufStatsAction.cc \
        $(SNMP_SOURCE) \
@@ -1154,6 +1156,9 @@ tests_testHttpReply_SOURCES=\
        Packer.h \
        SquidString.h \
        SquidTime.h \
+       $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        String.cc \
        StrList.h \
        StrList.cc \
@@ -1258,6 +1263,9 @@ tests_testACLMaxUserIP_SOURCES= \
        StrList.cc \
        tests/stub_StatHist.cc \
        stmem.cc \
+       $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        String.cc \
        store_dir.cc \
        StoreIOState.cc \
@@ -1509,6 +1517,8 @@ tests_testCacheManager_SOURCES = \
        RemovalPolicy.cc \
        Server.cc \
        $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        $(SNMP_SOURCE) \
        SquidMath.h \
        SquidMath.cc \
@@ -1674,6 +1684,9 @@ tests_testDiskIO_SOURCES = \
        StatHist.h \
        tests/stub_StatHist.cc \
        stmem.cc \
+       $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        StoreFileSystem.cc \
        StoreIOState.cc \
        tests/stub_StoreMeta.cc \
@@ -1924,6 +1937,8 @@ tests_testEvent_SOURCES = \
        StrList.h \
        StrList.cc \
        $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        $(SNMP_SOURCE) \
        SquidMath.cc \
        SquidMath.h \
@@ -2166,6 +2181,8 @@ tests_testEventLoop_SOURCES = \
        refresh.cc \
        Server.cc \
        $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        $(SNMP_SOURCE) \
        SquidMath.h \
        SquidMath.cc \
@@ -2406,6 +2423,8 @@ tests_test_http_range_SOURCES = \
        RemovalPolicy.cc \
        Server.cc \
        $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        $(SNMP_SOURCE) \
        SquidMath.h \
        SquidMath.cc \
@@ -2691,6 +2710,8 @@ tests_testHttpRequest_SOURCES = \
        RemovalPolicy.cc \
        Server.cc \
        $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        $(SNMP_SOURCE) \
        SquidMath.h \
        SquidMath.cc \
@@ -2856,6 +2877,9 @@ tests_testStore_SOURCES= \
        StoreSwapLogData.cc \
        store_key_md5.h \
        store_key_md5.cc \
+       $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        String.cc \
        StrList.h \
        StrList.cc \
@@ -3085,6 +3109,9 @@ tests_testUfs_SOURCES = \
        tests/stub_cache_cf.cc \
        tests/stub_helper.cc \
        cbdata.cc \
+       $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        String.cc \
        tests/stub_debug.cc \
        tests/stub_client_side_request.cc \
@@ -3263,6 +3290,9 @@ tests_testRock_SOURCES = \
        store_key_md5.cc \
        store_swapmeta.cc \
        store_swapout.cc \
+       $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        String.cc \
        StrList.h \
        StrList.cc \
@@ -3484,6 +3514,8 @@ tests_testURL_SOURCES = \
        RemovalPolicy.cc \
        Server.cc \
        $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        $(SNMP_SOURCE) \
        SquidMath.h \
        SquidMath.cc \
@@ -3599,6 +3631,8 @@ tests_testSBuf_SOURCES= \
        tests/SBufFindTest.h \
        tests/SBufFindTest.cc \
        $(SBUF_SOURCE) \
+       SBufDetailedStats.h \
+       tests/stub_SBufDetailedStats.cc \
        SBufStream.h \
        tests/stub_time.cc \
        mem.cc \

diff --git a/src/MemBlob.cc b/src/MemBlob.cc
index c57d9c57501bac90c4f92e78b655109656da7ebf..3234215695a87d14be3fd5baf5194001f824e323 100644 (file)
--- a/src/MemBlob.cc
+++ b/src/MemBlob.cc
@@ -31,6 +31,7 @@
 #include "Debug.h"
 #include "Mem.h"
 #include "MemBlob.h"
+#include "SBufDetailedStats.h"
 
 #if HAVE_IOSTREAM
 #include <iostream>
@@ -96,6 +97,7 @@ MemBlob::~MemBlob()
         memFreeString(capacity,mem);
     Stats.liveBytes -= capacity;
     --Stats.live;
+    recordMemBlobSizeAtDestruct(size);
 
     debugs(MEMBLOB_DEBUGSECTION,9, HERE << "destructed, this="
            << static_cast<void*>(this) << " id=" << id

diff --git a/src/SBuf.cc b/src/SBuf.cc
index db745a1672f64238a6a9af1fb3a52aa3f13cb4c4..26480b8dcdcedb001e6912422c7ae66d37683fd6 100644 (file)
--- a/src/SBuf.cc
+++ b/src/SBuf.cc
@@ -31,6 +31,7 @@
 #include "Debug.h"
 #include "OutOfBoundsException.h"
 #include "SBuf.h"
+#include "SBufDetailedStats.h"
 #include "SBufExceptions.h"
 #include "util.h"
 
@@ -148,6 +149,7 @@ SBuf::~SBuf()
 {
     debugs(24, 8, id << " destructed");
     --stats.live;
+    recordSBufSizeAtDestruct(len_);
 }
 
 MemBlob::Pointer

diff --git a/src/SBufDetailedStats.cc b/src/SBufDetailedStats.cc
new file mode 100644 (file)
index 0000000..52dec38
--- /dev/null
+++ b/src/SBufDetailedStats.cc
@@ -0,0 +1,47 @@
+#include "squid.h"
+#include "SBufDetailedStats.h"
+#include "StatHist.h"
+
+/*
+ * Implementation note: the purpose of this construct is to avoid adding
+ * external dependencies to the SBuf code
+ */
+
+static StatHist sbufDestructTimeStats;
+static StatHist memblobDestructTimeStats;
+
+namespace SBufDetailedStatsHistInitializer {
+    // run the post-instantiation initialization methods for StatHist objects
+    struct Initializer
+    {
+        Initializer() {
+            sbufDestructTimeStats.logInit(300,30.0,128000.0);
+            memblobDestructTimeStats.logInit(300,30.0,128000.0);
+        }
+    };
+    Initializer initializer;
+}
+
+void
+recordSBufSizeAtDestruct(SBuf::size_type sz)
+{
+    sbufDestructTimeStats.count(static_cast<double>(sz));
+}
+
+const StatHist *
+collectSBufDestructTimeStats()
+{
+    return &sbufDestructTimeStats;
+}
+
+void
+recordMemBlobSizeAtDestruct(SBuf::size_type sz)
+{
+    memblobDestructTimeStats.count(static_cast<double>(sz));
+}
+
+const StatHist *
+collectMemBlobDestructTimeStats()
+{
+    return &memblobDestructTimeStats;
+}

diff --git a/src/SBufDetailedStats.h b/src/SBufDetailedStats.h
new file mode 100644 (file)
index 0000000..8f8c6a1
--- /dev/null
+++ b/src/SBufDetailedStats.h
@@ -0,0 +1,27 @@
+
+#ifndef SQUID_SBUFDETAILEDSTATS_H
+#define SQUID_SBUFDETAILEDSTATS_H
+
+#include "SBuf.h"
+
+class StatHist;
+
+/// Record the size a SBuf had when it was destructed
+void recordSBufSizeAtDestruct(SBuf::size_type sz);
+
+/** Collect the SBuf size-at-destruct-time histogram
+ *
+ * \note the returned StatHist object must not be freed
+ */
+const StatHist * collectSBufDestructTimeStats();
+
+/// Record the size a MemBlob had when it was destructed
+void recordMemBlobSizeAtDestruct(MemBlob::size_type sz);
+
+/** Collect the MemBlob size-at-destruct-time histogram
+ *
+ * \note the returned StatHist object must not be freed
+ */
+const StatHist * collectMemBlobDestructTimeStats();
+
+#endif /* SQUID_SBUFDETAILEDSTATS_H */

diff --git a/src/SBufStatsAction.cc b/src/SBufStatsAction.cc
index 7fb34d47e53e6cbd927c2f29158261a04ab9ca3f..2484dd67a28af619d0e4c224f1a421cd988f7b0e 100644 (file)
--- a/src/SBufStatsAction.cc
+++ b/src/SBufStatsAction.cc
@@ -30,6 +30,7 @@
 #include "ipc/Messages.h"
 #include "ipc/TypedMsgHdr.h"
 #include "mgr/Registration.h"
+#include "SBufDetailedStats.h"
 #include "SBufStatsAction.h"
 #include "StoreEntryStream.h"
 
@@ -48,6 +49,8 @@ SBufStatsAction::add(const Mgr::Action& action)
 {
     sbdata += dynamic_cast<const SBufStatsAction&>(action).sbdata;
     mbdata += dynamic_cast<const SBufStatsAction&>(action).mbdata;
+    sbsizesatdestruct += dynamic_cast<const SBufStatsAction&>(action).sbsizesatdestruct;
+    mbsizesatdestruct += dynamic_cast<const SBufStatsAction&>(action).mbsizesatdestruct;
 }
 
 void
@@ -55,17 +58,24 @@ SBufStatsAction::collect()
 {
     sbdata = SBuf::GetStats();
     mbdata = MemBlob::GetStats();
+    sbsizesatdestruct = *collectSBufDestructTimeStats();
+    mbsizesatdestruct = *collectMemBlobDestructTimeStats();
 }
 
 void
 SBufStatsAction::dump(StoreEntry* entry)
 {
     StoreEntryStream ses(entry);
+    ses << "\n\n\nThese statistics are experimental; their format and contents "
+        "should not be relied upon, they are bound to change as "
+        "the SBuf feature is evolved\n";
     sbdata.dump(ses);
     mbdata.dump(ses);
-    ses << "\n\n\nThese statistics are experimental; their format and contents "
-    "should not be relied upon, they are bound to change as "
-    "the SBuf feature is evolved";
+    ses << "\n";
+    ses << "SBuf size distribution at destruct time:\n";
+    sbsizesatdestruct.dump(entry,NULL);
+    ses << "MemBlob size distribution at destruct time:\n";
+    mbsizesatdestruct.dump(entry,NULL);
 }
 
 void

diff --git a/src/SBufStatsAction.h b/src/SBufStatsAction.h
index 629277305d8e8bdb493ef6397686337d9fec2036..f7feae6bbe08b01e7786f00de9c20d07ca28e7b3 100644 (file)
--- a/src/SBufStatsAction.h
+++ b/src/SBufStatsAction.h
@@ -31,6 +31,7 @@
 
 #include "mgr/Action.h"
 #include "SBuf.h"
+#include "StatHist.h"
 
 class StoreEntry;
 
@@ -55,6 +56,8 @@ private:
 
     SBufStats sbdata;
     MemBlobStats mbdata;
+    StatHist sbsizesatdestruct;
+    StatHist mbsizesatdestruct;
 };
 
 #endif /* SQUID_SBUFSTATSACTION_H */

diff --git a/src/StatHist.cc b/src/StatHist.cc
index 6523d393d06a958946630396efc6f167d53b7006..f22573b623600cb2f91f2914164b60c969226aa6 100644 (file)
--- a/src/StatHist.cc
+++ b/src/StatHist.cc
@@ -205,6 +205,26 @@ StatHist::dump(StoreEntry * sentry, StatHistBinDumper * bd) const
     }
 }
 
+StatHist &
+StatHist::operator += (const StatHist &B)
+{
+    Must(capacity_ == B.capacity_);
+    Must(min_ == B.min_);
+    Must(max_ == B.max_);
+
+    if (B.bins == NULL) { // B was not yet initializted
+        return *this;
+    }
+    if (bins == NULL) { // this histogram was not yet initialized
+        *this = B;
+        return *this;
+    }
+    for (unsigned int i = 0; i < capacity_; ++i) {
+        bins[i] += B.bins[i];
+    }
+    return *this;
+}
+
 /* log based histogram */
 double
 Math::Log(double x)

diff --git a/src/StatHist.h b/src/StatHist.h
index 8f28ae4496cb574f7ae334c54af164ea44744bba..014d745b5ede434d0748cb4c32277e8acb55c0c6 100644 (file)
--- a/src/StatHist.h
+++ b/src/StatHist.h
@@ -73,23 +73,36 @@ public:
      *  this and the supplied histogram.
      */
     double deltaPctile(const StatHist &B, double pctile) const;
+
     /** obtain the output-transformed value from the specified bin
      *
      */
     double val(unsigned int bin) const;
+
     /** increment the counter for the histogram entry
      * associated to the supplied value
      */
     void count(double val);
+
     /** iterate the supplied bd function over the histogram values
      */
     void dump(StoreEntry *sentry, StatHistBinDumper * bd) const;
+
     /** Initialize the Histogram using a logarithmic values distribution
      */
     void logInit(unsigned int capacity, double min, double max);
+
     /** initialize the histogram to count occurrences in an enum-represented set
      */
     void enumInit(unsigned int last_enum);
+
+    /** Import values from another histogram
+     *
+     * \note: the two histograms MUST have the same capicity, min and max or
+     *      an exception will be raised
+     */
+    StatHist &operator += (const StatHist &B);
+
 protected:
     /** low-level initialize function. called by *Init high-level functions
      * \note Important restrictions on val_in and val_out functions:
@@ -103,16 +116,21 @@ protected:
      *  See log and linear based histograms for examples
      */
     void init(unsigned int capacity, hbase_f * val_in, hbase_f * val_out, double min, double max);
+
     /// find what entry in the histogram corresponds to v, by applying
     /// the preset input transformation function
     unsigned int findBin(double v);
+
     /// the histogram counters
     bins_type *bins;
     unsigned int capacity_;
+
     /// minimum value to be stored, corresponding to the first bin
     double min_;
+
     /// value of the maximum counter in the histogram
     double max_;
+
     /// scaling factor when looking for a bin
     double scale_;
     hbase_f *val_in;        /* e.g., log() for log-based histogram */

diff --git a/src/tests/stub_SBufDetailedStats.cc b/src/tests/stub_SBufDetailedStats.cc
new file mode 100644 (file)
index 0000000..8233ac2
--- /dev/null
+++ b/src/tests/stub_SBufDetailedStats.cc
@@ -0,0 +1,12 @@
+#include "squid.h"
+#include "SBuf.h"
+
+#define STUB_API "SBufDetailedStats.cc"
+#include "tests/STUB.h"
+
+class StatHist;
+
+void recordSBufSizeAtDestruct(SBuf::size_type) STUB_NOP
+const StatHist * collectSBufDestructTimeStats() STUB_RETVAL(NULL)
+void recordMemBlobSizeAtDestruct(SBuf::size_type) STUB_NOP
+const StatHist * collectMemBlobDestructTimeStats() STUB_RETVAL(NULL)

diff --git a/src/tests/testSBuf.cc b/src/tests/testSBuf.cc
index 479a9c21e0d9734d77aa2c85114cf8ff0925d917..b424444eb213cbe45dd3c329f08c2dedf06c691b 100644 (file)
--- a/src/tests/testSBuf.cc
+++ b/src/tests/testSBuf.cc
@@ -674,7 +674,8 @@ testSBuf::testScanf()
     CPPUNIT_ASSERT_EQUAL(static_cast<float>(123.5),f);
 }
 
-void testSBuf::testCopy()
+void
+testSBuf::testCopy()
 {
     char buf[40]; //shorter than literal()
     SBuf s(fox1),s2;
@@ -687,7 +688,8 @@ void testSBuf::testCopy()
     CPPUNIT_ASSERT_EQUAL(s2,s);
 }
 
-void testSBuf::testStringOps()
+void
+testSBuf::testStringOps()
 {
     SBuf sng(literal.toLower()),
     ref("the quick brown fox jumped over the lazy dog");
@@ -700,7 +702,8 @@ void testSBuf::testStringOps()
     CPPUNIT_ASSERT_EQUAL(0,SBuf("the").compare(SBuf("THE"),caseInsensitive,6));
 }
 
-void testSBuf::testGrow()
+void
+testSBuf::testGrow()
 {
     SBuf t;
     t.assign("foo");
@@ -714,7 +717,8 @@ void testSBuf::testGrow()
     CPPUNIT_ASSERT_EQUAL(ref,match);
 }
 
-void testSBuf::testStartsWith()
+void
+testSBuf::testStartsWith()
 {
     static SBuf casebuf("THE QUICK");
     CPPUNIT_ASSERT(literal.startsWith(SBuf(fox1)));
@@ -730,7 +734,8 @@ void testSBuf::testStartsWith()
     CPPUNIT_ASSERT_EQUAL(false,literal.startsWith(casebuf,caseInsensitive));
 }
 
-void testSBuf::testSBufStream()
+void
+testSBuf::testSBufStream()
 {
     SBuf b("const.string, int 10 and a float 10.5");
     SBufStream ss;
@@ -747,7 +752,8 @@ void testSBuf::testSBufStream()
     CPPUNIT_ASSERT_EQUAL(f1,SBuf(fox1));
 }
 
-void testSBuf::testFindFirstOf()
+void
+testSBuf::testFindFirstOf()
 {
     SBuf haystack(literal);
     SBuf::size_type idx;
@@ -769,13 +775,15 @@ void testSBuf::testFindFirstOf()
     CPPUNIT_ASSERT_EQUAL(4U,idx);
 }
 
-void testSBuf::testAutoFind()
+void
+testSBuf::testAutoFind()
 {
     SBufFindTest test;
     test.run();
 }
 
-void testSBuf::testStdStringOps()
+void
+testSBuf::testStdStringOps()
 {
     const char *alphabet="abcdefghijklmnopqrstuvwxyz";
     std::string astr(alphabet);

diff --git a/src/tests/testStatHist.cc b/src/tests/testStatHist.cc
index 545c7046dfb83ca0ef41c20f6fba521e1634d56e..1bb824f175ae6e536ba408700a9f0ce6eaf78105 100644 (file)
--- a/src/tests/testStatHist.cc
+++ b/src/tests/testStatHist.cc
@@ -73,3 +73,25 @@ testStatHist::testStatHistLog()
     test.count(max);
     //CPPUNIT_ASSERT(test.val(capacity-1)==1); //FIXME: val() returns a density
 }
+
+void
+testStatHist::testStatHistSum()
+{
+    InspectingStatHist s1, s2;
+    s1.logInit(30,1.0,100.0);
+    s2.logInit(30,1.0,100.0);
+    s1.count(3);
+    s2.count(30);
+    InspectingStatHist ts1, ts2;
+    ts1=s1;
+    ts1+=s2;
+    ts2=s2;
+    ts2+=s1;
+    CPPUNIT_ASSERT(ts1 == ts2);
+    InspectingStatHist ts3;
+    ts3.logInit(30,1.0,100.0);
+    ts3.count(3);
+    ts3.count(30);
+    CPPUNIT_ASSERT(ts3 == ts1);
+
+}

diff --git a/src/tests/testStatHist.h b/src/tests/testStatHist.h
index 9572c8dd316aa832398d06207844057c8e5cb217..c76c0c54b8b30ad0465472db290d5082e9f9511b 100644 (file)
--- a/src/tests/testStatHist.h
+++ b/src/tests/testStatHist.h
@@ -13,6 +13,7 @@ class testStatHist : public CPPUNIT_NS::TestFixture
     CPPUNIT_TEST( testStatHistBaseEquality );
     CPPUNIT_TEST( testStatHistBaseAssignment );
     CPPUNIT_TEST( testStatHistLog );
+    CPPUNIT_TEST( testStatHistSum );
     CPPUNIT_TEST_SUITE_END();
 
 public:
@@ -21,6 +22,7 @@ protected:
     void testStatHistBaseEquality();
     void testStatHistBaseAssignment();
     void testStatHistLog();
+    void testStatHistSum();
 };
 
 #endif /* TESTSTATHIST_H_ */