src/MemBuf.cc

   1 /*
   2  * $Id$
   3  *
   4  * DEBUG: section 59    auto-growing Memory Buffer with printf
   5  * AUTHOR: Alex Rousskov
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 /**
  36  \todo use memory pools for .buf recycling @?@ @?@
  37  */
  38
  39 /**
  40  \verbatim
  41  * Rationale:
  42  * ----------
  43  *
  44  * Here is how one would comm_write an object without MemBuffer:
  45  *
  46  * {
  47  * -- allocate:
  48  * buf = malloc(big_enough);
  49  *
  50  * -- "pack":
  51  * snprintf object(s) piece-by-piece constantly checking for overflows
  52  * and maintaining (buf+offset);
  53  * ...
  54  *
  55  * -- write
  56  * comm_write(buf, free, ...);
  57  * }
  58  *
  59  * The whole "packing" idea is quite messy: We are given a buffer of fixed
  60  * size and we have to check all the time that we still fit. Sounds logical.
  61  *
  62  * However, what happens if we have more data? If we are lucky to stop before
  63  * we overrun any buffers, we still may have garbage (e.g. half of ETag) in
  64  * the buffer.
  65  *
  66  * MemBuffer:
  67  * ----------
  68  *
  69  * MemBuffer is a memory-resident buffer with printf()-like interface. It
  70  * hides all offest handling and overflow checking. Moreover, it has a
  71  * build-in control that no partial data has been written.
  72  *
  73  * MemBuffer is designed to handle relatively small data. It starts with a
  74  * small buffer of configurable size to avoid allocating huge buffers all the
  75  * time.  MemBuffer doubles the buffer when needed. It assert()s that it will
  76  * not grow larger than a configurable limit. MemBuffer has virtually no
  77  * overhead (and can even reduce memory consumption) compared to old
  78  * "packing" approach.
  79  *
  80  * MemBuffer eliminates both "packing" mess and truncated data:
  81  *
  82  * {
  83  * -- setup
  84  * MemBuf buf;
  85  *
  86  * -- required init with optional size tuning (see #defines for defaults)
  87  * buf.init(initial-size, absolute-maximum);
  88  *
  89  * -- "pack" (no need to handle offsets or check for overflows)
  90  * buf.Printf(...);
  91  * ...
  92  *
  93  * -- write
  94  * comm_write_mbuf(fd, buf, handler, data);
  95  *
  96  * -- *iff* you did not give the buffer away, free it yourself
  97  * -- buf.clean();
  98  * }
  99  \endverbatim
 100  */
 101
 102 /* if you have configure you can use this */
 103 #if defined(HAVE_CONFIG_H)
 104 #include "config.h"
 105 #endif
 106
 107 #ifdef VA_COPY
 108 #undef VA_COPY
 109 #endif
 110 #if defined HAVE_VA_COPY
 111 #define VA_COPY va_copy
 112 #elif defined HAVE___VA_COPY
 113 #define VA_COPY __va_copy
 114 #endif
 115
 116 #include "squid.h"
 117 #include "MemBuf.h"
 118
 119 /* local constants */
 120
 121 /* default values for buffer sizes, used by memBufDefInit */
 122 #define MEM_BUF_INIT_SIZE   (2*1024)
 123 #define MEM_BUF_MAX_SIZE    (2*1000*1024*1024)
 124
 125 CBDATA_CLASS_INIT(MemBuf);
 126
 127 /** init with defaults */
 128 void
 129 MemBuf::init()
 130 {
 131     init(MEM_BUF_INIT_SIZE, MEM_BUF_MAX_SIZE);
 132 }
 133
 134
 135 /** init with specific sizes */
 136 void
 137 MemBuf::init(mb_size_t szInit, mb_size_t szMax)
 138 {
 139     assert(szInit > 0 && szMax > 0);
 140     buf = NULL;
 141     size = 0;
 142     max_capacity = szMax;
 143     capacity = 0;
 144     stolen = 0;
 145     grow(szInit);
 146 }
 147
 148 /**
 149  * cleans the mb; last function to call if you do not give .buf away with
 150  * memBufFreeFunc
 151  */
 152 void
 153 MemBuf::clean()
 154 {
 155     if (isNull()) {
 156         // nothing to do
 157     } else {
 158         assert(buf);
 159         assert(!stolen);        /* not frozen */
 160
 161         memFreeBuf(capacity, buf);
 162         buf = NULL;
 163         size = capacity = max_capacity = 0;
 164     }
 165 }
 166
 167 /**
 168  * Cleans the buffer without changing its capacity
 169  * if called with a Null buffer, calls memBufDefInit()
 170  */
 171 void
 172 MemBuf::reset()
 173 {
 174     if (isNull()) {
 175         init();
 176     } else {
 177         assert(!stolen);        /* not frozen */
 178         /* reset */
 179         memset(buf, 0, capacity);
 180         size = 0;
 181     }
 182 }
 183
 184 /**
 185  * Unfortunate hack to test if the buffer has been Init()ialized
 186  */
 187 int
 188 MemBuf::isNull()
 189 {
 190     if (!buf && !max_capacity && !capacity && !size)
 191         return 1;               /* is null (not initialized) */
 192
 193     assert(buf && max_capacity && capacity);    /* paranoid */
 194
 195     return 0;
 196 }
 197
 198 mb_size_t MemBuf::spaceSize() const
 199 {
 200     const mb_size_t terminatedSize = size + 1;
 201     return (terminatedSize < capacity) ? capacity - terminatedSize : 0;
 202 }
 203
 204 mb_size_t MemBuf::potentialSpaceSize() const
 205 {
 206     const mb_size_t terminatedSize = size + 1;
 207     return (terminatedSize < max_capacity) ? max_capacity - terminatedSize : 0;
 208 }
 209
 210 /// removes sz bytes and "packs" by moving content left
 211 void MemBuf::consume(mb_size_t shiftSize)
 212 {
 213     const mb_size_t cSize = contentSize();
 214     assert(0 <= shiftSize && shiftSize <= cSize);
 215     assert(!stolen); /* not frozen */
 216
 217     PROF_start(MemBuf_consume);
 218     if (shiftSize > 0) {
 219         if (shiftSize < cSize)
 220             xmemmove(buf, buf + shiftSize, cSize - shiftSize);
 221
 222         size -= shiftSize;
 223
 224         terminate();
 225     }
 226     PROF_stop(MemBuf_consume);
 227 }
 228
 229 /**
 230  * calls memcpy, appends exactly size bytes,
 231  * extends buffer or creates buffer if needed.
 232  */
 233 void MemBuf::append(const char *newContent, mb_size_t sz)
 234 {
 235     assert(sz >= 0);
 236     assert(buf || (0==capacity && 0==size));
 237     assert(!stolen); /* not frozen */
 238
 239     PROF_start(MemBuf_append);
 240     if (sz > 0) {
 241         if (size + sz + 1 > capacity)
 242             grow(size + sz + 1);
 243
 244         assert(size + sz <= capacity); /* paranoid */
 245
 246         xmemcpy(space(), newContent, sz);
 247
 248         appended(sz);
 249     }
 250     PROF_stop(MemBuf_append);
 251 }
 252
 253 /// updates content size after external append
 254 void MemBuf::appended(mb_size_t sz)
 255 {
 256     assert(size + sz <= capacity);
 257     size += sz;
 258     terminate();
 259 }
 260
 261 /**
 262  * Null-terminate in case we are used as a string.
 263  * Extra octet is not counted in the content size (or space size)
 264  *
 265  \note XXX: but the extra octet is counted when growth decisions are made!
 266  *     This will cause the buffer to grow when spaceSize() == 1 on append,
 267  *     which will assert() if the buffer cannot grow any more.
 268  */
 269 void MemBuf::terminate()
 270 {
 271     assert(size < capacity);
 272     *space() = '\0';
 273 }
 274
 275 /* calls memBufVPrintf */
 276 void
 277 MemBuf::Printf(const char *fmt,...)
 278 {
 279     va_list args;
 280     va_start(args, fmt);
 281     vPrintf(fmt, args);
 282     va_end(args);
 283 }
 284
 285
 286 /**
 287  * vPrintf for other printf()'s to use; calls vsnprintf, extends buf if needed
 288  */
 289 void
 290 MemBuf::vPrintf(const char *fmt, va_list vargs) {
 291 #ifdef VA_COPY
 292     va_list ap;
 293 #endif
 294
 295     int sz = 0;
 296     assert(fmt);
 297     assert(buf);
 298     assert(!stolen);    /* not frozen */
 299     /* assert in Grow should quit first, but we do not want to have a scary infinite loop */
 300
 301     while (capacity <= max_capacity) {
 302         mb_size_t free_space = capacity - size;
 303         /* put as much as we can */
 304
 305 #ifdef VA_COPY
 306         /* Fix of bug 753r. The value of vargs is undefined
 307          * after vsnprintf() returns. Make a copy of vargs
 308          * incase we loop around and call vsnprintf() again.
 309          */
 310         VA_COPY(ap,vargs);
 311         sz = vsnprintf(buf + size, free_space, fmt, ap);
 312         va_end(ap);
 313 #else /* VA_COPY */
 314
 315         sz = vsnprintf(buf + size, free_space, fmt, vargs);
 316 #endif /*VA_COPY*/
 317         /* check for possible overflow */
 318         /* snprintf on Linuz returns -1 on overflows */
 319         /* snprintf on FreeBSD returns at least free_space on overflows */
 320
 321         if (sz < 0 || sz >= free_space)
 322             grow(capacity + 1);
 323         else
 324             break;
 325     }
 326
 327     size += sz;
 328     /* on Linux and FreeBSD, '\0' is not counted in return value */
 329     /* on XXX it might be counted */
 330     /* check that '\0' is appended and not counted */
 331
 332     if (!size || buf[size - 1]) {
 333         assert(!buf[size]);
 334     } else {
 335         size--;
 336     }
 337 }
 338
 339 /**
 340  * Important:
 341  *   calling this function "freezes" mb,
 342  *   do not _update_ mb after that in any way
 343  *   (you still can read-access .buf and .size)
 344  *
 345  \retval free() function to be used.
 346  */
 347 FREE *
 348 MemBuf::freeFunc() {
 349     FREE *ff;
 350     assert(buf);
 351     assert(!stolen);    /* not frozen */
 352
 353     ff = memFreeBufFunc((size_t) capacity);
 354     stolen = 1;         /* freeze */
 355     return ff;
 356 }
 357
 358 /**
 359  * Grows (doubles) internal buffer to satisfy required minimal capacity
 360  */
 361 void
 362 MemBuf::grow(mb_size_t min_cap) {
 363     size_t new_cap;
 364     size_t buf_cap;
 365
 366     assert(!stolen);
 367     assert(capacity < min_cap);
 368
 369     PROF_start(MemBuf_grow);
 370
 371     /* determine next capacity */
 372
 373     if (min_cap > 64 * 1024) {
 374         new_cap = 64 * 1024;
 375
 376         while (new_cap < (size_t) min_cap)
 377             new_cap += 64 * 1024;       /* increase in reasonable steps */
 378     } else {
 379         new_cap = (size_t) min_cap;
 380     }
 381
 382     /* last chance to fit before we assert(!overflow) */
 383     if (new_cap > (size_t) max_capacity)
 384         new_cap = (size_t) max_capacity;
 385
 386     assert(new_cap <= (size_t) max_capacity);   /* no overflow */
 387
 388     assert(new_cap > (size_t) capacity);        /* progress */
 389
 390     buf_cap = (size_t) capacity;
 391
 392     buf = (char *)memReallocBuf(buf, new_cap, &buf_cap);
 393
 394     /* done */
 395     capacity = (mb_size_t) buf_cap;
 396     PROF_stop(MemBuf_grow);
 397 }
 398
 399
 400 /* Reports */
 401
 402 /**
 403  * Puts report on MemBuf _module_ usage into mb
 404  */
 405 void
 406 memBufReport(MemBuf * mb) {
 407     assert(mb);
 408     mb->Printf("memBufReport is not yet implemented @?@\n");
 409 }
 410
 411 #ifndef _USE_INLINE_
 412 #include "MemBuf.cci"
 413 #endif