src/MemBuf.cc

   1 /*
   2  * $Id: MemBuf.cc,v 1.43 2008/02/26 21:49:34 amosjeffries Exp $
   3  *
   4  * DEBUG: section 59    auto-growing Memory Buffer with printf
   5  * AUTHOR: Alex Rousskov
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 /**
  36  \todo use memory pools for .buf recycling @?@ @?@
  37  */
  38
  39 /**
  40  \verbatim
  41  * Rationale:
  42  * ----------
  43  *
  44  * Here is how one would comm_write an object without MemBuffer:
  45  *
  46  * {
  47  * -- allocate:
  48  * buf = malloc(big_enough);
  49  *
  50  * -- "pack":
  51  * snprintf object(s) piece-by-piece constantly checking for overflows
  52  * and maintaining (buf+offset);
  53  * ...
  54  *
  55  * -- write
  56  * comm_write(buf, free, ...);
  57  * }
  58  *
  59  * The whole "packing" idea is quite messy: We are given a buffer of fixed
  60  * size and we have to check all the time that we still fit. Sounds logical.
  61  *
  62  * However, what happens if we have more data? If we are lucky to stop before
  63  * we overrun any buffers, we still may have garbage (e.g. half of ETag) in
  64  * the buffer.
  65  *
  66  * MemBuffer:
  67  * ----------
  68  *
  69  * MemBuffer is a memory-resident buffer with printf()-like interface. It
  70  * hides all offest handling and overflow checking. Moreover, it has a
  71  * build-in control that no partial data has been written.
  72  *
  73  * MemBuffer is designed to handle relatively small data. It starts with a
  74  * small buffer of configurable size to avoid allocating huge buffers all the
  75  * time.  MemBuffer doubles the buffer when needed. It assert()s that it will
  76  * not grow larger than a configurable limit. MemBuffer has virtually no
  77  * overhead (and can even reduce memory consumption) compared to old
  78  * "packing" approach.
  79  *
  80  * MemBuffer eliminates both "packing" mess and truncated data:
  81  *
  82  * {
  83  * -- setup
  84  * MemBuf buf;
  85  *
  86  * -- required init with optional size tuning (see #defines for defaults)
  87  * buf.init(initial-size, absolute-maximum);
  88  *
  89  * -- "pack" (no need to handle offsets or check for overflows)
  90  * buf.Printf(...);
  91  * ...
  92  *
  93  * -- write
  94  * comm_write_mbuf(fd, buf, handler, data);
  95  *
  96  * -- *iff* you did not give the buffer away, free it yourself
  97  * -- buf.clean();
  98  * }
  99  \endverbatim
 100  */
 101
 102 /* if you have configure you can use this */
 103 #if defined(HAVE_CONFIG_H)
 104 #include "config.h"
 105 #endif
 106
 107 #ifdef VA_COPY
 108 #undef VA_COPY
 109 #endif
 110 #if defined HAVE_VA_COPY
 111 #define VA_COPY va_copy
 112 #elif defined HAVE___VA_COPY
 113 #define VA_COPY __va_copy
 114 #endif
 115
 116 #include "squid.h"
 117 #include "MemBuf.h"
 118
 119 /* local constants */
 120
 121 /* default values for buffer sizes, used by memBufDefInit */
 122 #define MEM_BUF_INIT_SIZE   (2*1024)
 123 #define MEM_BUF_MAX_SIZE    (2*1000*1024*1024)
 124
 125 CBDATA_CLASS_INIT(MemBuf);
 126
 127 /** init with defaults */
 128 void
 129 MemBuf::init()
 130 {
 131     init(MEM_BUF_INIT_SIZE, MEM_BUF_MAX_SIZE);
 132 }
 133
 134
 135 /** init with specific sizes */
 136 void
 137 MemBuf::init(mb_size_t szInit, mb_size_t szMax)
 138 {
 139     assert(szInit > 0 && szMax > 0);
 140     buf = NULL;
 141     size = 0;
 142     max_capacity = szMax;
 143     capacity = 0;
 144     stolen = 0;
 145     grow(szInit);
 146 }
 147
 148 /**
 149  * cleans the mb; last function to call if you do not give .buf away with
 150  * memBufFreeFunc
 151  */
 152 void
 153 MemBuf::clean()
 154 {
 155     if (isNull()) {
 156         // nothing to do
 157     } else {
 158         assert(buf);
 159         assert(!stolen);        /* not frozen */
 160
 161         memFreeBuf(capacity, buf);
 162         buf = NULL;
 163         size = capacity = max_capacity = 0;
 164     }
 165 }
 166
 167 /**
 168  * Cleans the buffer without changing its capacity
 169  * if called with a Null buffer, calls memBufDefInit()
 170  */
 171 void
 172 MemBuf::reset()
 173 {
 174     if (isNull()) {
 175         init();
 176     } else {
 177         assert(!stolen);        /* not frozen */
 178         /* reset */
 179         memset(buf, 0, capacity);
 180         size = 0;
 181     }
 182 }
 183
 184 /**
 185  * Unfortunate hack to test if the buffer has been Init()ialized
 186  */
 187 int
 188 MemBuf::isNull()
 189 {
 190     if (!buf && !max_capacity && !capacity && !size)
 191         return 1;               /* is null (not initialized) */
 192
 193     assert(buf && max_capacity && capacity);    /* paranoid */
 194
 195     return 0;
 196 }
 197
 198 mb_size_t MemBuf::spaceSize() const
 199 {
 200     const mb_size_t terminatedSize = size + 1;
 201     return (terminatedSize < capacity) ? capacity - terminatedSize : 0;
 202 }
 203
 204 mb_size_t MemBuf::potentialSpaceSize() const
 205 {
 206     const mb_size_t terminatedSize = size + 1;
 207     return (terminatedSize < max_capacity) ? max_capacity - terminatedSize : 0;
 208 }
 209
 210 /// removes sz bytes and "packs" by moving content left
 211 void MemBuf::consume(mb_size_t shiftSize)
 212 {
 213     const mb_size_t cSize = contentSize();
 214     assert(0 <= shiftSize && shiftSize <= cSize);
 215     assert(!stolen); /* not frozen */
 216
 217     PROF_start(MemBuf_consume);
 218     if (shiftSize > 0) {
 219         if (shiftSize < cSize)
 220             xmemmove(buf, buf + shiftSize, cSize - shiftSize);
 221
 222         size -= shiftSize;
 223
 224         terminate();
 225     }
 226     PROF_stop(MemBuf_consume);
 227 }
 228
 229 /**
 230  * calls memcpy, appends exactly size bytes,
 231  * extends buffer or creates buffer if needed.
 232  */
 233 void MemBuf::append(const char *newContent, mb_size_t sz)
 234 {
 235     assert(sz >= 0);
 236     assert(buf || (0==capacity && 0==size));
 237     assert(!stolen); /* not frozen */
 238
 239     PROF_start(MemBuf_append);
 240     if (sz > 0) {
 241         if (size + sz + 1 > capacity)
 242             grow(size + sz + 1);
 243
 244         assert(size + sz <= capacity); /* paranoid */
 245
 246         xmemcpy(space(), newContent, sz);
 247
 248         appended(sz);
 249     }
 250     PROF_stop(MemBuf_append);
 251 }
 252
 253 /// updates content size after external append
 254 void MemBuf::appended(mb_size_t sz)
 255 {
 256     assert(size + sz <= capacity);
 257     size += sz;
 258     terminate();
 259 }
 260
 261 /**
 262  * Null-terminate in case we are used as a string.
 263  * Extra octet is not counted in the content size (or space size)
 264  *
 265  \note XXX: but the extra octet is counted when growth decisions are made!
 266  *     This will cause the buffer to grow when spaceSize() == 1 on append,
 267  *     which will assert() if the buffer cannot grow any more.
 268  */
 269 void MemBuf::terminate()
 270 {
 271     assert(size < capacity);
 272     *space() = '\0';
 273 }
 274
 275 /* calls memBufVPrintf */
 276 #if STDC_HEADERS
 277 void
 278 MemBuf::Printf(const char *fmt,...)
 279 {
 280     va_list args;
 281     va_start(args, fmt);
 282 #else
 283 void
 284 MemBuf::Printf(va_alist)
 285 va_dcl {
 286     va_list args;
 287     mb_size_t sz = 0;
 288     va_start(args);
 289     const char *fmt = va_arg(args, char *);
 290 #endif
 291
 292     vPrintf(fmt, args);
 293     va_end(args);
 294 }
 295
 296
 297 /**
 298  * vPrintf for other printf()'s to use; calls vsnprintf, extends buf if needed
 299  */
 300 void
 301 MemBuf::vPrintf(const char *fmt, va_list vargs) {
 302 #ifdef VA_COPY
 303     va_list ap;
 304 #endif
 305
 306     int sz = 0;
 307     assert(fmt);
 308     assert(buf);
 309     assert(!stolen);    /* not frozen */
 310     /* assert in Grow should quit first, but we do not want to have a scary infinite loop */
 311
 312     while (capacity <= max_capacity) {
 313         mb_size_t free_space = capacity - size;
 314         /* put as much as we can */
 315
 316 #ifdef VA_COPY
 317         /* Fix of bug 753r. The value of vargs is undefined
 318          * after vsnprintf() returns. Make a copy of vargs
 319          * incase we loop around and call vsnprintf() again.
 320          */
 321         VA_COPY(ap,vargs);
 322         sz = vsnprintf(buf + size, free_space, fmt, ap);
 323         va_end(ap);
 324 #else /* VA_COPY */
 325
 326         sz = vsnprintf(buf + size, free_space, fmt, vargs);
 327 #endif /*VA_COPY*/
 328         /* check for possible overflow */
 329         /* snprintf on Linuz returns -1 on overflows */
 330         /* snprintf on FreeBSD returns at least free_space on overflows */
 331
 332         if (sz < 0 || sz >= free_space)
 333             grow(capacity + 1);
 334         else
 335             break;
 336     }
 337
 338     size += sz;
 339     /* on Linux and FreeBSD, '\0' is not counted in return value */
 340     /* on XXX it might be counted */
 341     /* check that '\0' is appended and not counted */
 342
 343     if (!size || buf[size - 1]) {
 344         assert(!buf[size]);
 345     } else {
 346         size--;
 347     }
 348 }
 349
 350 /**
 351  * Important:
 352  *   calling this function "freezes" mb,
 353  *   do not _update_ mb after that in any way
 354  *   (you still can read-access .buf and .size)
 355  *
 356  \retval free() function to be used.
 357  */
 358 FREE *
 359 MemBuf::freeFunc() {
 360     FREE *ff;
 361     assert(buf);
 362     assert(!stolen);    /* not frozen */
 363
 364     ff = memFreeBufFunc((size_t) capacity);
 365     stolen = 1;         /* freeze */
 366     return ff;
 367 }
 368
 369 /**
 370  * Grows (doubles) internal buffer to satisfy required minimal capacity
 371  */
 372 void
 373 MemBuf::grow(mb_size_t min_cap) {
 374     size_t new_cap;
 375     size_t buf_cap;
 376
 377     assert(!stolen);
 378     assert(capacity < min_cap);
 379
 380     PROF_start(MemBuf_grow);
 381
 382     /* determine next capacity */
 383
 384     if (min_cap > 64 * 1024) {
 385         new_cap = 64 * 1024;
 386
 387         while (new_cap < (size_t) min_cap)
 388             new_cap += 64 * 1024;       /* increase in reasonable steps */
 389     } else {
 390         new_cap = (size_t) min_cap;
 391     }
 392
 393     /* last chance to fit before we assert(!overflow) */
 394     if (new_cap > (size_t) max_capacity)
 395         new_cap = (size_t) max_capacity;
 396
 397     assert(new_cap <= (size_t) max_capacity);   /* no overflow */
 398
 399     assert(new_cap > (size_t) capacity);        /* progress */
 400
 401     buf_cap = (size_t) capacity;
 402
 403     buf = (char *)memReallocBuf(buf, new_cap, &buf_cap);
 404
 405     /* done */
 406     capacity = (mb_size_t) buf_cap;
 407     PROF_stop(MemBuf_grow);
 408 }
 409
 410
 411 /* Reports */
 412
 413 /**
 414  * Puts report on MemBuf _module_ usage into mb
 415  */
 416 void
 417 memBufReport(MemBuf * mb) {
 418     assert(mb);
 419     mb->Printf("memBufReport is not yet implemented @?@\n");
 420 }
 421
 422 #ifndef _USE_INLINE_
 423 #include "MemBuf.cci"
 424 #endif