From: Francesco Chemolli Date: Wed, 27 Oct 2010 21:26:26 +0000 (+0200) Subject: First part of StringNg merge: MemBlob. X-Git-Tag: take1~134 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=43d1bbe4718cc022ccbe2d58ba52f54189f855c9;p=thirdparty%2Fsquid.git First part of StringNg merge: MemBlob. --- diff --git a/configure.in b/configure.in index 5d737f0bf4..62a7b02da7 100644 --- a/configure.in +++ b/configure.in @@ -2198,6 +2198,7 @@ AC_CHECK_HEADERS( \ iosfwd \ iomanip \ iostream \ + climits \ ip_compat.h \ ip_fil_compat.h \ ip_fil.h \ diff --git a/doc/debug-sections.txt b/doc/debug-sections.txt index 70254fdb05..5966a87046 100644 --- a/doc/debug-sections.txt +++ b/doc/debug-sections.txt @@ -50,6 +50,7 @@ section 21 Time Functions section 22 Refresh Calculation section 23 URL Parsing section 23 URL Scheme parsing +section 24 Low-level String-Buffer functions section 25 MIME Parsing and Internal Icons section 25 MiME Header Parsing section 26 Secure Sockets Layer Proxy diff --git a/include/util.h b/include/util.h index cae47dea16..5211069157 100644 --- a/include/util.h +++ b/include/util.h @@ -141,6 +141,8 @@ extern void gb_flush(gb_t *); /* internal, do not use this */ */ int statMemoryAccounted(void); +SQUIDCEXTERN unsigned int RoundTo(const unsigned int num, const unsigned int what); + /* Windows Port */ /* win32lib.c */ #ifdef _SQUID_MSWIN_ diff --git a/lib/util.c b/lib/util.c index 0e017f5ae9..6685dff34a 100644 --- a/lib/util.c +++ b/lib/util.c @@ -989,3 +989,11 @@ gb_to_str(const gb_t * g) return buf; } + +/** + * rounds num to the next upper integer multiple of what + */ +unsigned int RoundTo(const unsigned int num, const unsigned int what) +{ + return what * ((num + what -1)/what); +} diff --git a/src/Makefile.am b/src/Makefile.am index f9478910e1..aecf549632 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -19,6 +19,11 @@ DNSSOURCE += \ DnsLookupDetails.h \ DnsLookupDetails.cc +SBUF_SOURCE= \ + base/InstanceId.h \ + MemBlob.h \ + MemBlob.cc + SNMP_ALL_SOURCE = \ snmp_core.cc \ snmp_agent.cc @@ -418,6 +423,7 @@ squid_SOURCES = \ RemovalPolicy.cc \ RemovalPolicy.h \ send-announce.cc \ + $(SBUF_SOURCE) \ $(SNMP_SOURCE) \ squid.h \ SquidMath.h \ diff --git a/src/Mem.h b/src/Mem.h index d1401f1ae4..951db4e709 100644 --- a/src/Mem.h +++ b/src/Mem.h @@ -59,4 +59,6 @@ protected: static void RegisterWithCacheManager(void); }; +extern const size_t squidSystemPageSize; + #endif /* SQUID_MEM */ diff --git a/src/MemBlob.cc b/src/MemBlob.cc new file mode 100644 index 0000000000..40b9345822 --- /dev/null +++ b/src/MemBlob.cc @@ -0,0 +1,189 @@ +/* + * MemBlob.cc (C) 2009 Francesco Chemolli + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + */ + + +#include "config.h" +#include "base/TextException.h" +#include "Debug.h" +#include "MemBlob.h" +#if HAVE_IOSTREAM +#include +#endif + +#define MEMBLOB_USES_MEM_POOLS 0 + +#if MEMBLOB_USES_MEM_POOLS +#include "protos.h" +#endif + +MemBlobStats MemBlob::Stats; +InstanceIdDefinitions(MemBlob, "blob"); + + +/* MemBlobStats */ + +MemBlobStats::MemBlobStats(): alloc(0), live(0), append(0) +{} + +std::ostream& +MemBlobStats::dump(std::ostream &os) const +{ + os << + "MemBlob created: " << alloc << + "\nMemBlob alive: " << live << + "\nMemBlob append calls: " << append << + "\nMemBlob currently allocated size: " << liveBytes << + "\nlive MemBlob mean current allocation size: " << + (static_cast(liveBytes)/(live?live:1)) << std::endl; + return os; +} + + +/* MemBlob */ + +MemBlob::MemBlob(const MemBlob::size_type reserveSize) : + mem(NULL), capacity(0), size(0) // will be set by memAlloc +{ + debugs(MEMBLOB_DEBUGSECTION,9, HERE << "constructed, this=" + << static_cast(this) << " id=" << id + << " reserveSize=" << reserveSize); + memAlloc(reserveSize); +} + +MemBlob::MemBlob(const char *buffer, const MemBlob::size_type bufSize) : + mem(NULL), capacity(0), size(0) // will be set by memAlloc +{ + debugs(MEMBLOB_DEBUGSECTION,9, HERE << "constructed, this=" + << static_cast(this) << " id=" << id + << " buffer=" << static_cast(buffer) + << " bufSize=" << bufSize); + memAlloc(bufSize); + append(buffer, bufSize); +} + +MemBlob::~MemBlob() +{ +#if MEMBLOB_USES_MEM_POOLS + //no mempools for now + // \todo reinstate mempools use + memFreeString(capacity,mem); +#else + xfree(mem); +#endif + Stats.liveBytes -= capacity; + --Stats.live; + + debugs(MEMBLOB_DEBUGSECTION,9, HERE << "destructed, this=" + << static_cast(this) << " id=" << id + << " capacity=" << capacity + << " size=" << size); +} + +/** + * Given the requested minimum size, return a rounded allocation size + * for the backing store. + * This is a stopgap call, this job is eventually expected to be handled + * by MemPools via memAllocString. + */ +MemBlob::size_type +MemBlob::calcAllocSize(const size_type size) const +{ + if (size <= 36) return 36; + if (size <= 128) return 128; + if (size <= 512) return 512; + if (size <= 4096) return RoundTo(size, 512); + // XXX: recover squidSystemPageSize functionality. It's easy for + // the main squid, harder for tests +#if 0 + return RoundTo(size, squidSystemPageSize); +#else + return RoundTo(size, 4096); +#endif +} + +/** Allocate an available space area of at least minSize bytes in size. + * Must be called by constructors and only by constructors. + */ +void +MemBlob::memAlloc(const size_type minSize) +{ + size_t actualAlloc = calcAllocSize(minSize); + + Must(!mem); +#if MEMBLOB_USES_MEM_POOLS + // XXX: for now, do without mempools. In order to do it, MemPools + // need to be singletons so that initialization order can be enforced + mem = static_cast(memAllocString(minSize, &actualAlloc)); +#else + // \todo reinstate mempools use + mem = static_cast(xmalloc(actualAlloc)); +#endif + Must(mem); + + capacity = actualAlloc; + size = 0; + debugs(MEMBLOB_DEBUGSECTION, 8, + id << " memAlloc: requested=" << minSize << + ", received=" << capacity); + ++Stats.live; + ++Stats.alloc; + Stats.liveBytes += capacity; +} + +void +MemBlob::append(const char *source, const size_type n) +{ + if (n > 0) { // appending zero bytes is allowed but only affects the stats + Must(willFit(n)); + Must(source); + /// \note memcpy() is safe because we copy to an unused area + memcpy(mem + size, source, n); + size += n; + } + ++Stats.append; +} + + +const MemBlobStats& +MemBlob::GetStats() +{ + return Stats; +} + +std::ostream& +MemBlob::dump(std::ostream &os) const +{ + os << "id @" << (void *)this + << "mem:" << static_cast(mem) + << ",capacity:" << capacity + << ",size:" << size + << ",refs:" << RefCountCount() << "; "; + return os; +} diff --git a/src/MemBlob.h b/src/MemBlob.h new file mode 100644 index 0000000000..f4ea42decd --- /dev/null +++ b/src/MemBlob.h @@ -0,0 +1,141 @@ +/* + * MemBlob.h (C) 2009 Francesco Chemolli + * + * SQUID Web Proxy Cache http://www.squid-cache.org/ + * ---------------------------------------------------------- + * + * Squid is the result of efforts by numerous individuals from + * the Internet community; see the CONTRIBUTORS file for full + * details. Many organizations have provided support for Squid's + * development; see the SPONSORS file for full details. Squid is + * Copyrighted (C) 2001 by the Regents of the University of + * California; see the COPYRIGHT file for full details. Squid + * incorporates software developed and/or copyrighted by other + * sources; see the CREDITS file for full details. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. + */ + +#ifndef SQUID_MEMBLOB_H_ +#define SQUID_MEMBLOB_H_ + +#define MEMBLOB_DEBUGSECTION 24 + +#include "base/InstanceId.h" +#include "MemPool.h" +#include "RefCount.h" + +/// Various MemBlob class-wide statistics. +class MemBlobStats +{ +public: + MemBlobStats(); + + /// dumps class-wide statistics + std::ostream& dump(std::ostream& os) const; + +public: + u_int64_t alloc; ///< number of MemBlob instances created so far + u_int64_t live; ///< number of MemBlob instances currently alive + u_int64_t append; ///< number of MemBlob::append() calls + u_int64_t liveBytes; ///< the total size of currently allocated storage +}; + + +/** Refcountable, fixed-size, content-agnostic memory buffer. + * + * Allocated memory block is divided into two sequential areas: + * "used memory" and "available space". The used area can be filled during + * construction, grows via the append() call, and can be clear()ed. + * + * MemBlob users can cooperate to safely share the used area. However, MemBlob + * provides weak use accounting and no sharing protections besides refcounting. + */ +class MemBlob: public RefCountable +{ +public: + typedef RefCount Pointer; + typedef int32_t size_type; + + MEMPROXY_CLASS(MemBlob); + + /// obtain a const view of class-wide statistics + static const MemBlobStats& GetStats(); + + /// create a new MemBlob with at least reserveSize capacity + explicit MemBlob(const size_type reserveSize); + + /// create a MemBlob containing a copy of the buffer of a given size + MemBlob(const char *buffer, const size_type bufferSize); + + virtual ~MemBlob(); + + /// the number unused bytes at the end of the allocated blob + size_type spaceSize() const { return capacity - size; } + + /** check whether the caller can successfully append() n bytes + * + * \return true the caller may append() n bytes to this blob now + * \param off the end of the blob area currently used by the caller + * \param n the total number of bytes the caller wants to append + */ + bool canAppend(const size_type off, const size_type n) const + { + // TODO: ignore offset (and adjust size) when the blob is not shared? + return isAppendOffset(off) && willFit(n); + } + + /** copies exactly n bytes from the source to the available space area, + * enlarging the used area by n bytes + * + * \throw TextException if there is not enough space in the blob + * \param source raw buffer to be copied + * \param n the number of bytes to copy from the source buffer + */ + void append(const char *source, const size_type n); + + /// extends the available space to the entire allocated blob + void clear() { size = 0; } + + /// dump debugging information + std::ostream & dump(std::ostream &os) const; + +public: + /* MemBlob users should avoid these and must treat them as read-only */ + char *mem; ///< raw allocated memory block + size_type capacity; ///< size of the raw allocated memory block + size_type size; ///< maximum allocated memory in use by callers + const InstanceId id; ///< blob identifier + +private: + static MemBlobStats Stats; ///< class-wide statistics + + void memAlloc(const size_type memSize); + size_type calcAllocSize(const size_type minSize) const; + + /// whether the offset points to the end of the used area + bool isAppendOffset(const size_type off) const { return off == size; } + + /// whether n more bytes can be appended + bool willFit(const size_type n) const { return n <= spaceSize(); } + + /* copying is not implemented */ + MemBlob(const MemBlob &); + MemBlob& operator =(const MemBlob &); +}; + +MEMPROXY_CLASS_INLINE(MemBlob); + +#endif /* SQUID_MEMBLOB_H_ */ diff --git a/src/mem.cc b/src/mem.cc index 3b8e71ed2b..00ab2d2551 100644 --- a/src/mem.cc +++ b/src/mem.cc @@ -51,6 +51,7 @@ #endif /* module globals */ +const size_t squidSystemPageSize=getpagesize(); /* local prototypes */ static void memStringStats(std::ostream &);