src/MemBuf.cc

   1 /*
   2  * $Id$
   3  *
   4  * DEBUG: section 59    auto-growing Memory Buffer with printf
   5  * AUTHOR: Alex Rousskov
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 /**
  36  \todo use memory pools for .buf recycling @?@ @?@
  37  */
  38
  39 /**
  40  \verbatim
  41  * Rationale:
  42  * ----------
  43  *
  44  * Here is how one would Comm::Write an object without MemBuffer:
  45  *
  46  * {
  47  * -- allocate:
  48  * buf = malloc(big_enough);
  49  *
  50  * -- "pack":
  51  * snprintf object(s) piece-by-piece constantly checking for overflows
  52  * and maintaining (buf+offset);
  53  * ...
  54  *
  55  * -- write
  56  * Comm::Write(buf, free, ...);
  57  * }
  58  *
  59  * The whole "packing" idea is quite messy: We are given a buffer of fixed
  60  * size and we have to check all the time that we still fit. Sounds logical.
  61  *
  62  * However, what happens if we have more data? If we are lucky to stop before
  63  * we overrun any buffers, we still may have garbage (e.g. half of ETag) in
  64  * the buffer.
  65  *
  66  * MemBuffer:
  67  * ----------
  68  *
  69  * MemBuffer is a memory-resident buffer with printf()-like interface. It
  70  * hides all offest handling and overflow checking. Moreover, it has a
  71  * build-in control that no partial data has been written.
  72  *
  73  * MemBuffer is designed to handle relatively small data. It starts with a
  74  * small buffer of configurable size to avoid allocating huge buffers all the
  75  * time.  MemBuffer doubles the buffer when needed. It assert()s that it will
  76  * not grow larger than a configurable limit. MemBuffer has virtually no
  77  * overhead (and can even reduce memory consumption) compared to old
  78  * "packing" approach.
  79  *
  80  * MemBuffer eliminates both "packing" mess and truncated data:
  81  *
  82  * {
  83  * -- setup
  84  * MemBuf buf;
  85  *
  86  * -- required init with optional size tuning (see #defines for defaults)
  87  * buf.init(initial-size, absolute-maximum);
  88  *
  89  * -- "pack" (no need to handle offsets or check for overflows)
  90  * buf.Printf(...);
  91  * ...
  92  *
  93  * -- write
  94  * Comm::Write(fd, buf, callback);
  95  *
  96  * -- *iff* you did not give the buffer away, free it yourself
  97  * -- buf.clean();
  98  * }
  99  \endverbatim
 100  */
 101
 102 #include "squid.h"
 103 #include "MemBuf.h"
 104 #include "profiler/Profiler.h"
 105 #include "protos.h"
 106
 107 #ifdef VA_COPY
 108 #undef VA_COPY
 109 #endif
 110 #if defined HAVE_VA_COPY
 111 #define VA_COPY va_copy
 112 #elif defined HAVE___VA_COPY
 113 #define VA_COPY __va_copy
 114 #endif
 115
 116 /* local constants */
 117
 118 /* default values for buffer sizes, used by memBufDefInit */
 119 #define MEM_BUF_INIT_SIZE   (2*1024)
 120 #define MEM_BUF_MAX_SIZE    (2*1000*1024*1024)
 121
 122 CBDATA_CLASS_INIT(MemBuf);
 123
 124 /** init with defaults */
 125 void
 126 MemBuf::init()
 127 {
 128     init(MEM_BUF_INIT_SIZE, MEM_BUF_MAX_SIZE);
 129 }
 130
 131 /** init with specific sizes */
 132 void
 133 MemBuf::init(mb_size_t szInit, mb_size_t szMax)
 134 {
 135     assert(szInit > 0 && szMax > 0);
 136     buf = NULL;
 137     size = 0;
 138     max_capacity = szMax;
 139     capacity = 0;
 140     stolen = 0;
 141     grow(szInit);
 142 }
 143
 144 /**
 145  * cleans the mb; last function to call if you do not give .buf away with
 146  * memBufFreeFunc
 147  */
 148 void
 149 MemBuf::clean()
 150 {
 151     if (isNull()) {
 152         // nothing to do
 153     } else {
 154         assert(buf);
 155         assert(!stolen);        /* not frozen */
 156
 157         memFreeBuf(capacity, buf);
 158         buf = NULL;
 159         size = capacity = max_capacity = 0;
 160     }
 161 }
 162
 163 /**
 164  * Cleans the buffer without changing its capacity
 165  * if called with a Null buffer, calls memBufDefInit()
 166  */
 167 void
 168 MemBuf::reset()
 169 {
 170     if (isNull()) {
 171         init();
 172     } else {
 173         assert(!stolen);        /* not frozen */
 174         /* reset */
 175         memset(buf, 0, capacity);
 176         size = 0;
 177     }
 178 }
 179
 180 /**
 181  * Unfortunate hack to test if the buffer has been Init()ialized
 182  */
 183 int
 184 MemBuf::isNull()
 185 {
 186     if (!buf && !max_capacity && !capacity && !size)
 187         return 1;               /* is null (not initialized) */
 188
 189     assert(buf && max_capacity && capacity);    /* paranoid */
 190
 191     return 0;
 192 }
 193
 194 mb_size_t MemBuf::spaceSize() const
 195 {
 196     const mb_size_t terminatedSize = size + 1;
 197     return (terminatedSize < capacity) ? capacity - terminatedSize : 0;
 198 }
 199
 200 mb_size_t MemBuf::potentialSpaceSize() const
 201 {
 202     const mb_size_t terminatedSize = size + 1;
 203     return (terminatedSize < max_capacity) ? max_capacity - terminatedSize : 0;
 204 }
 205
 206 /// removes sz bytes and "packs" by moving content left
 207 void MemBuf::consume(mb_size_t shiftSize)
 208 {
 209     const mb_size_t cSize = contentSize();
 210     assert(0 <= shiftSize && shiftSize <= cSize);
 211     assert(!stolen); /* not frozen */
 212
 213     PROF_start(MemBuf_consume);
 214     if (shiftSize > 0) {
 215         if (shiftSize < cSize)
 216             memmove(buf, buf + shiftSize, cSize - shiftSize);
 217
 218         size -= shiftSize;
 219
 220         terminate();
 221     }
 222     PROF_stop(MemBuf_consume);
 223 }
 224
 225 // removes last tailSize bytes
 226 void MemBuf::truncate(mb_size_t tailSize)
 227 {
 228     const mb_size_t cSize = contentSize();
 229     assert(0 <= tailSize && tailSize <= cSize);
 230     assert(!stolen); /* not frozen */
 231     size -= tailSize;
 232 }
 233
 234 /**
 235  * calls memcpy, appends exactly size bytes,
 236  * extends buffer or creates buffer if needed.
 237  */
 238 void MemBuf::append(const char *newContent, mb_size_t sz)
 239 {
 240     assert(sz >= 0);
 241     assert(buf || (0==capacity && 0==size));
 242     assert(!stolen); /* not frozen */
 243
 244     PROF_start(MemBuf_append);
 245     if (sz > 0) {
 246         if (size + sz + 1 > capacity)
 247             grow(size + sz + 1);
 248
 249         assert(size + sz <= capacity); /* paranoid */
 250         memcpy(space(), newContent, sz);
 251         appended(sz);
 252     }
 253     PROF_stop(MemBuf_append);
 254 }
 255
 256 /// updates content size after external append
 257 void MemBuf::appended(mb_size_t sz)
 258 {
 259     assert(size + sz <= capacity);
 260     size += sz;
 261     terminate();
 262 }
 263
 264 /**
 265  * Null-terminate in case we are used as a string.
 266  * Extra octet is not counted in the content size (or space size)
 267  *
 268  \note XXX: but the extra octet is counted when growth decisions are made!
 269  *     This will cause the buffer to grow when spaceSize() == 1 on append,
 270  *     which will assert() if the buffer cannot grow any more.
 271  */
 272 void MemBuf::terminate()
 273 {
 274     assert(size < capacity);
 275     *space() = '\0';
 276 }
 277
 278 /* calls memBufVPrintf */
 279 void
 280 MemBuf::Printf(const char *fmt,...)
 281 {
 282     va_list args;
 283     va_start(args, fmt);
 284     vPrintf(fmt, args);
 285     va_end(args);
 286 }
 287
 288 /**
 289  * vPrintf for other printf()'s to use; calls vsnprintf, extends buf if needed
 290  */
 291 void
 292 MemBuf::vPrintf(const char *fmt, va_list vargs)
 293 {
 294 #ifdef VA_COPY
 295     va_list ap;
 296 #endif
 297
 298     int sz = 0;
 299     assert(fmt);
 300     assert(buf);
 301     assert(!stolen);    /* not frozen */
 302     /* assert in Grow should quit first, but we do not want to have a scary infinite loop */
 303
 304     while (capacity <= max_capacity) {
 305         mb_size_t free_space = capacity - size;
 306         /* put as much as we can */
 307
 308 #ifdef VA_COPY
 309         /* Fix of bug 753r. The value of vargs is undefined
 310          * after vsnprintf() returns. Make a copy of vargs
 311          * incase we loop around and call vsnprintf() again.
 312          */
 313         VA_COPY(ap,vargs);
 314         sz = vsnprintf(buf + size, free_space, fmt, ap);
 315         va_end(ap);
 316 #else /* VA_COPY */
 317
 318         sz = vsnprintf(buf + size, free_space, fmt, vargs);
 319 #endif /*VA_COPY*/
 320         /* check for possible overflow */
 321         /* snprintf on Linuz returns -1 on overflows */
 322         /* snprintf on FreeBSD returns at least free_space on overflows */
 323
 324         if (sz < 0 || sz >= free_space)
 325             grow(capacity + 1);
 326         else
 327             break;
 328     }
 329
 330     size += sz;
 331     /* on Linux and FreeBSD, '\0' is not counted in return value */
 332     /* on XXX it might be counted */
 333     /* check that '\0' is appended and not counted */
 334
 335     if (!size || buf[size - 1]) {
 336         assert(!buf[size]);
 337     } else {
 338         --size;
 339     }
 340 }
 341
 342 /**
 343  * Important:
 344  *   calling this function "freezes" mb,
 345  *   do not _update_ mb after that in any way
 346  *   (you still can read-access .buf and .size)
 347  *
 348  \retval free() function to be used.
 349  */
 350 FREE *
 351 MemBuf::freeFunc()
 352 {
 353     FREE *ff;
 354     assert(buf);
 355     assert(!stolen);    /* not frozen */
 356
 357     ff = memFreeBufFunc((size_t) capacity);
 358     stolen = 1;         /* freeze */
 359     return ff;
 360 }
 361
 362 /**
 363  * Grows (doubles) internal buffer to satisfy required minimal capacity
 364  */
 365 void
 366 MemBuf::grow(mb_size_t min_cap)
 367 {
 368     size_t new_cap;
 369     size_t buf_cap;
 370
 371     assert(!stolen);
 372     assert(capacity < min_cap);
 373
 374     PROF_start(MemBuf_grow);
 375
 376     /* determine next capacity */
 377
 378     if (min_cap > 64 * 1024) {
 379         new_cap = 64 * 1024;
 380
 381         while (new_cap < (size_t) min_cap)
 382             new_cap += 64 * 1024;       /* increase in reasonable steps */
 383     } else {
 384         new_cap = (size_t) min_cap;
 385     }
 386
 387     /* last chance to fit before we assert(!overflow) */
 388     if (new_cap > (size_t) max_capacity)
 389         new_cap = (size_t) max_capacity;
 390
 391     assert(new_cap <= (size_t) max_capacity);   /* no overflow */
 392
 393     assert(new_cap > (size_t) capacity);        /* progress */
 394
 395     buf_cap = (size_t) capacity;
 396
 397     buf = (char *)memReallocBuf(buf, new_cap, &buf_cap);
 398
 399     /* done */
 400     capacity = (mb_size_t) buf_cap;
 401     PROF_stop(MemBuf_grow);
 402 }
 403
 404 /* Reports */
 405
 406 /**
 407  * Puts report on MemBuf _module_ usage into mb
 408  */
 409 void
 410 memBufReport(MemBuf * mb)
 411 {
 412     assert(mb);
 413     mb->Printf("memBufReport is not yet implemented @?@\n");
 414 }
 415
 416 #if !_USE_INLINE_
 417 #include "MemBuf.cci"
 418 #endif