From: Francesco Chemolli Date: Wed, 27 Nov 2013 15:52:03 +0000 (+0100) Subject: Implement SBuf and MemBlob size distribution stats X-Git-Tag: SQUID_3_5_0_1~494 X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=ff1eb053559c22063f2335e169d7152c42474ebe;p=thirdparty%2Fsquid.git Implement SBuf and MemBlob size distribution stats --- diff --git a/src/Makefile.am b/src/Makefile.am index 97a94c1d2d..930e7dbf5c 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -481,6 +481,8 @@ squid_SOURCES = \ send-announce.h \ send-announce.cc \ $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + SBufDetailedStats.cc \ SBufStatsAction.h \ SBufStatsAction.cc \ $(SNMP_SOURCE) \ @@ -1154,6 +1156,9 @@ tests_testHttpReply_SOURCES=\ Packer.h \ SquidString.h \ SquidTime.h \ + $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ String.cc \ StrList.h \ StrList.cc \ @@ -1258,6 +1263,9 @@ tests_testACLMaxUserIP_SOURCES= \ StrList.cc \ tests/stub_StatHist.cc \ stmem.cc \ + $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ String.cc \ store_dir.cc \ StoreIOState.cc \ @@ -1509,6 +1517,8 @@ tests_testCacheManager_SOURCES = \ RemovalPolicy.cc \ Server.cc \ $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ $(SNMP_SOURCE) \ SquidMath.h \ SquidMath.cc \ @@ -1674,6 +1684,9 @@ tests_testDiskIO_SOURCES = \ StatHist.h \ tests/stub_StatHist.cc \ stmem.cc \ + $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ StoreFileSystem.cc \ StoreIOState.cc \ tests/stub_StoreMeta.cc \ @@ -1924,6 +1937,8 @@ tests_testEvent_SOURCES = \ StrList.h \ StrList.cc \ $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ $(SNMP_SOURCE) \ SquidMath.cc \ SquidMath.h \ @@ -2166,6 +2181,8 @@ tests_testEventLoop_SOURCES = \ refresh.cc \ Server.cc \ $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ $(SNMP_SOURCE) \ SquidMath.h \ SquidMath.cc \ @@ -2406,6 +2423,8 @@ tests_test_http_range_SOURCES = \ RemovalPolicy.cc \ Server.cc \ $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ $(SNMP_SOURCE) \ SquidMath.h \ SquidMath.cc \ @@ -2691,6 +2710,8 @@ tests_testHttpRequest_SOURCES = \ RemovalPolicy.cc \ Server.cc \ $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ $(SNMP_SOURCE) \ SquidMath.h \ SquidMath.cc \ @@ -2856,6 +2877,9 @@ tests_testStore_SOURCES= \ StoreSwapLogData.cc \ store_key_md5.h \ store_key_md5.cc \ + $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ String.cc \ StrList.h \ StrList.cc \ @@ -3085,6 +3109,9 @@ tests_testUfs_SOURCES = \ tests/stub_cache_cf.cc \ tests/stub_helper.cc \ cbdata.cc \ + $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ String.cc \ tests/stub_debug.cc \ tests/stub_client_side_request.cc \ @@ -3263,6 +3290,9 @@ tests_testRock_SOURCES = \ store_key_md5.cc \ store_swapmeta.cc \ store_swapout.cc \ + $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ String.cc \ StrList.h \ StrList.cc \ @@ -3484,6 +3514,8 @@ tests_testURL_SOURCES = \ RemovalPolicy.cc \ Server.cc \ $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ $(SNMP_SOURCE) \ SquidMath.h \ SquidMath.cc \ @@ -3599,6 +3631,8 @@ tests_testSBuf_SOURCES= \ tests/SBufFindTest.h \ tests/SBufFindTest.cc \ $(SBUF_SOURCE) \ + SBufDetailedStats.h \ + tests/stub_SBufDetailedStats.cc \ SBufStream.h \ tests/stub_time.cc \ mem.cc \ diff --git a/src/MemBlob.cc b/src/MemBlob.cc index c57d9c5750..3234215695 100644 --- a/src/MemBlob.cc +++ b/src/MemBlob.cc @@ -31,6 +31,7 @@ #include "Debug.h" #include "Mem.h" #include "MemBlob.h" +#include "SBufDetailedStats.h" #if HAVE_IOSTREAM #include @@ -96,6 +97,7 @@ MemBlob::~MemBlob() memFreeString(capacity,mem); Stats.liveBytes -= capacity; --Stats.live; + recordMemBlobSizeAtDestruct(size); debugs(MEMBLOB_DEBUGSECTION,9, HERE << "destructed, this=" << static_cast(this) << " id=" << id diff --git a/src/SBuf.cc b/src/SBuf.cc index db745a1672..26480b8dcd 100644 --- a/src/SBuf.cc +++ b/src/SBuf.cc @@ -31,6 +31,7 @@ #include "Debug.h" #include "OutOfBoundsException.h" #include "SBuf.h" +#include "SBufDetailedStats.h" #include "SBufExceptions.h" #include "util.h" @@ -148,6 +149,7 @@ SBuf::~SBuf() { debugs(24, 8, id << " destructed"); --stats.live; + recordSBufSizeAtDestruct(len_); } MemBlob::Pointer diff --git a/src/SBufDetailedStats.cc b/src/SBufDetailedStats.cc new file mode 100644 index 0000000000..52dec38828 --- /dev/null +++ b/src/SBufDetailedStats.cc @@ -0,0 +1,47 @@ +#include "squid.h" +#include "SBufDetailedStats.h" +#include "StatHist.h" + +/* + * Implementation note: the purpose of this construct is to avoid adding + * external dependencies to the SBuf code + */ + +static StatHist sbufDestructTimeStats; +static StatHist memblobDestructTimeStats; + +namespace SBufDetailedStatsHistInitializer { + // run the post-instantiation initialization methods for StatHist objects + struct Initializer + { + Initializer() { + sbufDestructTimeStats.logInit(300,30.0,128000.0); + memblobDestructTimeStats.logInit(300,30.0,128000.0); + } + }; + Initializer initializer; +} + +void +recordSBufSizeAtDestruct(SBuf::size_type sz) +{ + sbufDestructTimeStats.count(static_cast(sz)); +} + +const StatHist * +collectSBufDestructTimeStats() +{ + return &sbufDestructTimeStats; +} + +void +recordMemBlobSizeAtDestruct(SBuf::size_type sz) +{ + memblobDestructTimeStats.count(static_cast(sz)); +} + +const StatHist * +collectMemBlobDestructTimeStats() +{ + return &memblobDestructTimeStats; +} diff --git a/src/SBufDetailedStats.h b/src/SBufDetailedStats.h new file mode 100644 index 0000000000..8f8c6a1bc1 --- /dev/null +++ b/src/SBufDetailedStats.h @@ -0,0 +1,27 @@ + +#ifndef SQUID_SBUFDETAILEDSTATS_H +#define SQUID_SBUFDETAILEDSTATS_H + +#include "SBuf.h" + +class StatHist; + +/// Record the size a SBuf had when it was destructed +void recordSBufSizeAtDestruct(SBuf::size_type sz); + +/** Collect the SBuf size-at-destruct-time histogram + * + * \note the returned StatHist object must not be freed + */ +const StatHist * collectSBufDestructTimeStats(); + +/// Record the size a MemBlob had when it was destructed +void recordMemBlobSizeAtDestruct(MemBlob::size_type sz); + +/** Collect the MemBlob size-at-destruct-time histogram + * + * \note the returned StatHist object must not be freed + */ +const StatHist * collectMemBlobDestructTimeStats(); + +#endif /* SQUID_SBUFDETAILEDSTATS_H */ diff --git a/src/SBufStatsAction.cc b/src/SBufStatsAction.cc index 7fb34d47e5..2484dd67a2 100644 --- a/src/SBufStatsAction.cc +++ b/src/SBufStatsAction.cc @@ -30,6 +30,7 @@ #include "ipc/Messages.h" #include "ipc/TypedMsgHdr.h" #include "mgr/Registration.h" +#include "SBufDetailedStats.h" #include "SBufStatsAction.h" #include "StoreEntryStream.h" @@ -48,6 +49,8 @@ SBufStatsAction::add(const Mgr::Action& action) { sbdata += dynamic_cast(action).sbdata; mbdata += dynamic_cast(action).mbdata; + sbsizesatdestruct += dynamic_cast(action).sbsizesatdestruct; + mbsizesatdestruct += dynamic_cast(action).mbsizesatdestruct; } void @@ -55,17 +58,24 @@ SBufStatsAction::collect() { sbdata = SBuf::GetStats(); mbdata = MemBlob::GetStats(); + sbsizesatdestruct = *collectSBufDestructTimeStats(); + mbsizesatdestruct = *collectMemBlobDestructTimeStats(); } void SBufStatsAction::dump(StoreEntry* entry) { StoreEntryStream ses(entry); + ses << "\n\n\nThese statistics are experimental; their format and contents " + "should not be relied upon, they are bound to change as " + "the SBuf feature is evolved\n"; sbdata.dump(ses); mbdata.dump(ses); - ses << "\n\n\nThese statistics are experimental; their format and contents " - "should not be relied upon, they are bound to change as " - "the SBuf feature is evolved"; + ses << "\n"; + ses << "SBuf size distribution at destruct time:\n"; + sbsizesatdestruct.dump(entry,NULL); + ses << "MemBlob size distribution at destruct time:\n"; + mbsizesatdestruct.dump(entry,NULL); } void diff --git a/src/SBufStatsAction.h b/src/SBufStatsAction.h index 629277305d..f7feae6bbe 100644 --- a/src/SBufStatsAction.h +++ b/src/SBufStatsAction.h @@ -31,6 +31,7 @@ #include "mgr/Action.h" #include "SBuf.h" +#include "StatHist.h" class StoreEntry; @@ -55,6 +56,8 @@ private: SBufStats sbdata; MemBlobStats mbdata; + StatHist sbsizesatdestruct; + StatHist mbsizesatdestruct; }; #endif /* SQUID_SBUFSTATSACTION_H */ diff --git a/src/StatHist.cc b/src/StatHist.cc index 6523d393d0..f22573b623 100644 --- a/src/StatHist.cc +++ b/src/StatHist.cc @@ -205,6 +205,26 @@ StatHist::dump(StoreEntry * sentry, StatHistBinDumper * bd) const } } +StatHist & +StatHist::operator += (const StatHist &B) +{ + Must(capacity_ == B.capacity_); + Must(min_ == B.min_); + Must(max_ == B.max_); + + if (B.bins == NULL) { // B was not yet initializted + return *this; + } + if (bins == NULL) { // this histogram was not yet initialized + *this = B; + return *this; + } + for (unsigned int i = 0; i < capacity_; ++i) { + bins[i] += B.bins[i]; + } + return *this; +} + /* log based histogram */ double Math::Log(double x) diff --git a/src/StatHist.h b/src/StatHist.h index 8f28ae4496..014d745b5e 100644 --- a/src/StatHist.h +++ b/src/StatHist.h @@ -73,23 +73,36 @@ public: * this and the supplied histogram. */ double deltaPctile(const StatHist &B, double pctile) const; + /** obtain the output-transformed value from the specified bin * */ double val(unsigned int bin) const; + /** increment the counter for the histogram entry * associated to the supplied value */ void count(double val); + /** iterate the supplied bd function over the histogram values */ void dump(StoreEntry *sentry, StatHistBinDumper * bd) const; + /** Initialize the Histogram using a logarithmic values distribution */ void logInit(unsigned int capacity, double min, double max); + /** initialize the histogram to count occurrences in an enum-represented set */ void enumInit(unsigned int last_enum); + + /** Import values from another histogram + * + * \note: the two histograms MUST have the same capicity, min and max or + * an exception will be raised + */ + StatHist &operator += (const StatHist &B); + protected: /** low-level initialize function. called by *Init high-level functions * \note Important restrictions on val_in and val_out functions: @@ -103,16 +116,21 @@ protected: * See log and linear based histograms for examples */ void init(unsigned int capacity, hbase_f * val_in, hbase_f * val_out, double min, double max); + /// find what entry in the histogram corresponds to v, by applying /// the preset input transformation function unsigned int findBin(double v); + /// the histogram counters bins_type *bins; unsigned int capacity_; + /// minimum value to be stored, corresponding to the first bin double min_; + /// value of the maximum counter in the histogram double max_; + /// scaling factor when looking for a bin double scale_; hbase_f *val_in; /* e.g., log() for log-based histogram */ diff --git a/src/tests/stub_SBufDetailedStats.cc b/src/tests/stub_SBufDetailedStats.cc new file mode 100644 index 0000000000..8233ac2c38 --- /dev/null +++ b/src/tests/stub_SBufDetailedStats.cc @@ -0,0 +1,12 @@ +#include "squid.h" +#include "SBuf.h" + +#define STUB_API "SBufDetailedStats.cc" +#include "tests/STUB.h" + +class StatHist; + +void recordSBufSizeAtDestruct(SBuf::size_type) STUB_NOP +const StatHist * collectSBufDestructTimeStats() STUB_RETVAL(NULL) +void recordMemBlobSizeAtDestruct(SBuf::size_type) STUB_NOP +const StatHist * collectMemBlobDestructTimeStats() STUB_RETVAL(NULL) diff --git a/src/tests/testSBuf.cc b/src/tests/testSBuf.cc index 479a9c21e0..b424444eb2 100644 --- a/src/tests/testSBuf.cc +++ b/src/tests/testSBuf.cc @@ -674,7 +674,8 @@ testSBuf::testScanf() CPPUNIT_ASSERT_EQUAL(static_cast(123.5),f); } -void testSBuf::testCopy() +void +testSBuf::testCopy() { char buf[40]; //shorter than literal() SBuf s(fox1),s2; @@ -687,7 +688,8 @@ void testSBuf::testCopy() CPPUNIT_ASSERT_EQUAL(s2,s); } -void testSBuf::testStringOps() +void +testSBuf::testStringOps() { SBuf sng(literal.toLower()), ref("the quick brown fox jumped over the lazy dog"); @@ -700,7 +702,8 @@ void testSBuf::testStringOps() CPPUNIT_ASSERT_EQUAL(0,SBuf("the").compare(SBuf("THE"),caseInsensitive,6)); } -void testSBuf::testGrow() +void +testSBuf::testGrow() { SBuf t; t.assign("foo"); @@ -714,7 +717,8 @@ void testSBuf::testGrow() CPPUNIT_ASSERT_EQUAL(ref,match); } -void testSBuf::testStartsWith() +void +testSBuf::testStartsWith() { static SBuf casebuf("THE QUICK"); CPPUNIT_ASSERT(literal.startsWith(SBuf(fox1))); @@ -730,7 +734,8 @@ void testSBuf::testStartsWith() CPPUNIT_ASSERT_EQUAL(false,literal.startsWith(casebuf,caseInsensitive)); } -void testSBuf::testSBufStream() +void +testSBuf::testSBufStream() { SBuf b("const.string, int 10 and a float 10.5"); SBufStream ss; @@ -747,7 +752,8 @@ void testSBuf::testSBufStream() CPPUNIT_ASSERT_EQUAL(f1,SBuf(fox1)); } -void testSBuf::testFindFirstOf() +void +testSBuf::testFindFirstOf() { SBuf haystack(literal); SBuf::size_type idx; @@ -769,13 +775,15 @@ void testSBuf::testFindFirstOf() CPPUNIT_ASSERT_EQUAL(4U,idx); } -void testSBuf::testAutoFind() +void +testSBuf::testAutoFind() { SBufFindTest test; test.run(); } -void testSBuf::testStdStringOps() +void +testSBuf::testStdStringOps() { const char *alphabet="abcdefghijklmnopqrstuvwxyz"; std::string astr(alphabet); diff --git a/src/tests/testStatHist.cc b/src/tests/testStatHist.cc index 545c7046df..1bb824f175 100644 --- a/src/tests/testStatHist.cc +++ b/src/tests/testStatHist.cc @@ -73,3 +73,25 @@ testStatHist::testStatHistLog() test.count(max); //CPPUNIT_ASSERT(test.val(capacity-1)==1); //FIXME: val() returns a density } + +void +testStatHist::testStatHistSum() +{ + InspectingStatHist s1, s2; + s1.logInit(30,1.0,100.0); + s2.logInit(30,1.0,100.0); + s1.count(3); + s2.count(30); + InspectingStatHist ts1, ts2; + ts1=s1; + ts1+=s2; + ts2=s2; + ts2+=s1; + CPPUNIT_ASSERT(ts1 == ts2); + InspectingStatHist ts3; + ts3.logInit(30,1.0,100.0); + ts3.count(3); + ts3.count(30); + CPPUNIT_ASSERT(ts3 == ts1); + +} diff --git a/src/tests/testStatHist.h b/src/tests/testStatHist.h index 9572c8dd31..c76c0c54b8 100644 --- a/src/tests/testStatHist.h +++ b/src/tests/testStatHist.h @@ -13,6 +13,7 @@ class testStatHist : public CPPUNIT_NS::TestFixture CPPUNIT_TEST( testStatHistBaseEquality ); CPPUNIT_TEST( testStatHistBaseAssignment ); CPPUNIT_TEST( testStatHistLog ); + CPPUNIT_TEST( testStatHistSum ); CPPUNIT_TEST_SUITE_END(); public: @@ -21,6 +22,7 @@ protected: void testStatHistBaseEquality(); void testStatHistBaseAssignment(); void testStatHistLog(); + void testStatHistSum(); }; #endif /* TESTSTATHIST_H_ */