src/MemBuf.cc

   1 /*
   2  * Copyright (C) 1996-2014 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 59    auto-growing Memory Buffer with printf */
  10
  11 /**
  12  \verbatim
  13  * Rationale:
  14  * ----------
  15  *
  16  * Here is how one would Comm::Write an object without MemBuffer:
  17  *
  18  * {
  19  * -- allocate:
  20  * buf = malloc(big_enough);
  21  *
  22  * -- "pack":
  23  * snprintf object(s) piece-by-piece constantly checking for overflows
  24  * and maintaining (buf+offset);
  25  * ...
  26  *
  27  * -- write
  28  * Comm::Write(buf, free, ...);
  29  * }
  30  *
  31  * The whole "packing" idea is quite messy: We are given a buffer of fixed
  32  * size and we have to check all the time that we still fit. Sounds logical.
  33  *
  34  * However, what happens if we have more data? If we are lucky to stop before
  35  * we overrun any buffers, we still may have garbage (e.g. half of ETag) in
  36  * the buffer.
  37  *
  38  * MemBuffer:
  39  * ----------
  40  *
  41  * MemBuffer is a memory-resident buffer with printf()-like interface. It
  42  * hides all offest handling and overflow checking. Moreover, it has a
  43  * build-in control that no partial data has been written.
  44  *
  45  * MemBuffer is designed to handle relatively small data. It starts with a
  46  * small buffer of configurable size to avoid allocating huge buffers all the
  47  * time.  MemBuffer doubles the buffer when needed. It assert()s that it will
  48  * not grow larger than a configurable limit. MemBuffer has virtually no
  49  * overhead (and can even reduce memory consumption) compared to old
  50  * "packing" approach.
  51  *
  52  * MemBuffer eliminates both "packing" mess and truncated data:
  53  *
  54  * {
  55  * -- setup
  56  * MemBuf buf;
  57  *
  58  * -- required init with optional size tuning (see #defines for defaults)
  59  * buf.init(initial-size, absolute-maximum);
  60  *
  61  * -- "pack" (no need to handle offsets or check for overflows)
  62  * buf.Printf(...);
  63  * ...
  64  *
  65  * -- write
  66  * Comm::Write(fd, buf, callback);
  67  *
  68  * -- *iff* you did not give the buffer away, free it yourself
  69  * -- buf.clean();
  70  * }
  71  \endverbatim
  72  */
  73
  74 #include "squid.h"
  75 #include "mem/forward.h"
  76 #include "MemBuf.h"
  77 #include "profiler/Profiler.h"
  78
  79 #ifdef VA_COPY
  80 #undef VA_COPY
  81 #endif
  82 #if defined HAVE_VA_COPY
  83 #define VA_COPY va_copy
  84 #elif defined HAVE___VA_COPY
  85 #define VA_COPY __va_copy
  86 #endif
  87
  88 /* local constants */
  89
  90 /* default values for buffer sizes, used by memBufDefInit */
  91 #define MEM_BUF_INIT_SIZE   (2*1024)
  92 #define MEM_BUF_MAX_SIZE    (2*1000*1024*1024)
  93
  94 CBDATA_CLASS_INIT(MemBuf);
  95
  96 /** init with defaults */
  97 void
  98 MemBuf::init()
  99 {
 100     init(MEM_BUF_INIT_SIZE, MEM_BUF_MAX_SIZE);
 101 }
 102
 103 /** init with specific sizes */
 104 void
 105 MemBuf::init(mb_size_t szInit, mb_size_t szMax)
 106 {
 107     assert(szInit > 0 && szMax > 0);
 108     buf = NULL;
 109     size = 0;
 110     max_capacity = szMax;
 111     capacity = 0;
 112     stolen = 0;
 113     grow(szInit);
 114     terminate();
 115 }
 116
 117 /**
 118  * cleans the mb; last function to call if you do not give .buf away with
 119  * memBufFreeFunc
 120  */
 121 void
 122 MemBuf::clean()
 123 {
 124     if (isNull()) {
 125         // nothing to do
 126     } else {
 127         assert(buf);
 128         assert(!stolen);    /* not frozen */
 129
 130         memFreeBuf(capacity, buf);
 131         buf = NULL;
 132         size = capacity = max_capacity = 0;
 133     }
 134 }
 135
 136 /**
 137  * Cleans the buffer without changing its capacity
 138  * if called with a Null buffer, calls memBufDefInit()
 139  */
 140 void
 141 MemBuf::reset()
 142 {
 143     if (isNull()) {
 144         init();
 145     } else {
 146         assert(!stolen);    /* not frozen */
 147         /* reset */
 148         memset(buf, 0, capacity);
 149         size = 0;
 150     }
 151 }
 152
 153 /**
 154  * Unfortunate hack to test if the buffer has been Init()ialized
 155  */
 156 int
 157 MemBuf::isNull()
 158 {
 159     if (!buf && !max_capacity && !capacity && !size)
 160         return 1;       /* is null (not initialized) */
 161
 162     assert(buf && max_capacity && capacity);    /* paranoid */
 163
 164     return 0;
 165 }
 166
 167 mb_size_t MemBuf::spaceSize() const
 168 {
 169     const mb_size_t terminatedSize = size + 1;
 170     return (terminatedSize < capacity) ? capacity - terminatedSize : 0;
 171 }
 172
 173 mb_size_t MemBuf::potentialSpaceSize() const
 174 {
 175     const mb_size_t terminatedSize = size + 1;
 176     return (terminatedSize < max_capacity) ? max_capacity - terminatedSize : 0;
 177 }
 178
 179 /// removes sz bytes and "packs" by moving content left
 180 void MemBuf::consume(mb_size_t shiftSize)
 181 {
 182     const mb_size_t cSize = contentSize();
 183     assert(0 <= shiftSize && shiftSize <= cSize);
 184     assert(!stolen); /* not frozen */
 185
 186     PROF_start(MemBuf_consume);
 187     if (shiftSize > 0) {
 188         if (shiftSize < cSize)
 189             memmove(buf, buf + shiftSize, cSize - shiftSize);
 190
 191         size -= shiftSize;
 192
 193         terminate();
 194     }
 195     PROF_stop(MemBuf_consume);
 196 }
 197
 198 /// removes all whitespace prefix bytes and "packs" by moving content left
 199 void MemBuf::consumeWhitespacePrefix()
 200 {
 201     PROF_start(MemBuf_consumeWhitespace);
 202     if (contentSize() > 0) {
 203         const char *end = buf + contentSize();
 204         const char *p = buf;
 205         for (; p<end && xisspace(*p); ++p);
 206         if (p-buf > 0)
 207             consume(p-buf);
 208     }
 209     PROF_stop(MemBuf_consumeWhitespace);
 210 }
 211
 212 // removes last tailSize bytes
 213 void MemBuf::truncate(mb_size_t tailSize)
 214 {
 215     const mb_size_t cSize = contentSize();
 216     assert(0 <= tailSize && tailSize <= cSize);
 217     assert(!stolen); /* not frozen */
 218     size -= tailSize;
 219 }
 220
 221 /**
 222  * calls memcpy, appends exactly size bytes,
 223  * extends buffer or creates buffer if needed.
 224  */
 225 void MemBuf::append(const char *newContent, mb_size_t sz)
 226 {
 227     assert(sz >= 0);
 228     assert(buf || (0==capacity && 0==size));
 229     assert(!stolen); /* not frozen */
 230
 231     PROF_start(MemBuf_append);
 232     if (sz > 0) {
 233         if (size + sz + 1 > capacity)
 234             grow(size + sz + 1);
 235
 236         assert(size + sz <= capacity); /* paranoid */
 237         memcpy(space(), newContent, sz);
 238         appended(sz);
 239     }
 240     PROF_stop(MemBuf_append);
 241 }
 242
 243 /// updates content size after external append
 244 void MemBuf::appended(mb_size_t sz)
 245 {
 246     assert(size + sz <= capacity);
 247     size += sz;
 248     terminate();
 249 }
 250
 251 /**
 252  * Null-terminate in case we are used as a string.
 253  * Extra octet is not counted in the content size (or space size)
 254  *
 255  \note XXX: but the extra octet is counted when growth decisions are made!
 256  *     This will cause the buffer to grow when spaceSize() == 1 on append,
 257  *     which will assert() if the buffer cannot grow any more.
 258  */
 259 void MemBuf::terminate()
 260 {
 261     assert(size < capacity);
 262     *space() = '\0';
 263 }
 264
 265 /* calls memBufVPrintf */
 266 void
 267 MemBuf::Printf(const char *fmt,...)
 268 {
 269     va_list args;
 270     va_start(args, fmt);
 271     vPrintf(fmt, args);
 272     va_end(args);
 273 }
 274
 275 /**
 276  * vPrintf for other printf()'s to use; calls vsnprintf, extends buf if needed
 277  */
 278 void
 279 MemBuf::vPrintf(const char *fmt, va_list vargs)
 280 {
 281 #ifdef VA_COPY
 282     va_list ap;
 283 #endif
 284
 285     int sz = 0;
 286     assert(fmt);
 287     assert(buf);
 288     assert(!stolen);    /* not frozen */
 289     /* assert in Grow should quit first, but we do not want to have a scary infinite loop */
 290
 291     while (capacity <= max_capacity) {
 292         mb_size_t free_space = capacity - size;
 293         /* put as much as we can */
 294
 295 #ifdef VA_COPY
 296         /* Fix of bug 753r. The value of vargs is undefined
 297          * after vsnprintf() returns. Make a copy of vargs
 298          * incase we loop around and call vsnprintf() again.
 299          */
 300         VA_COPY(ap,vargs);
 301         sz = vsnprintf(buf + size, free_space, fmt, ap);
 302         va_end(ap);
 303 #else /* VA_COPY */
 304
 305         sz = vsnprintf(buf + size, free_space, fmt, vargs);
 306 #endif /*VA_COPY*/
 307         /* check for possible overflow */
 308         /* snprintf on Linuz returns -1 on overflows */
 309         /* snprintf on FreeBSD returns at least free_space on overflows */
 310
 311         if (sz < 0 || sz >= free_space)
 312             grow(capacity + 1);
 313         else
 314             break;
 315     }
 316
 317     size += sz;
 318     /* on Linux and FreeBSD, '\0' is not counted in return value */
 319     /* on XXX it might be counted */
 320     /* check that '\0' is appended and not counted */
 321
 322     if (!size || buf[size - 1]) {
 323         assert(!buf[size]);
 324     } else {
 325         --size;
 326     }
 327 }
 328
 329 /**
 330  * Important:
 331  *   calling this function "freezes" mb,
 332  *   do not _update_ mb after that in any way
 333  *   (you still can read-access .buf and .size)
 334  *
 335  \retval free() function to be used.
 336  */
 337 FREE *
 338 MemBuf::freeFunc()
 339 {
 340     FREE *ff;
 341     assert(buf);
 342     assert(!stolen);    /* not frozen */
 343
 344     ff = memFreeBufFunc((size_t) capacity);
 345     stolen = 1;     /* freeze */
 346     return ff;
 347 }
 348
 349 /**
 350  * Grows (doubles) internal buffer to satisfy required minimal capacity
 351  */
 352 void
 353 MemBuf::grow(mb_size_t min_cap)
 354 {
 355     size_t new_cap;
 356     size_t buf_cap;
 357
 358     assert(!stolen);
 359     assert(capacity < min_cap);
 360
 361     PROF_start(MemBuf_grow);
 362
 363     /* determine next capacity */
 364
 365     if (min_cap > 64 * 1024) {
 366         new_cap = 64 * 1024;
 367
 368         while (new_cap < (size_t) min_cap)
 369             new_cap += 64 * 1024;   /* increase in reasonable steps */
 370     } else {
 371         new_cap = (size_t) min_cap;
 372     }
 373
 374     /* last chance to fit before we assert(!overflow) */
 375     if (new_cap > (size_t) max_capacity)
 376         new_cap = (size_t) max_capacity;
 377
 378     assert(new_cap <= (size_t) max_capacity);   /* no overflow */
 379
 380     assert(new_cap > (size_t) capacity);    /* progress */
 381
 382     buf_cap = (size_t) capacity;
 383
 384     buf = (char *)memReallocBuf(buf, new_cap, &buf_cap);
 385
 386     /* done */
 387     capacity = (mb_size_t) buf_cap;
 388     PROF_stop(MemBuf_grow);
 389 }
 390
 391 /* Reports */
 392
 393 /**
 394  * Puts report on MemBuf _module_ usage into mb
 395  */
 396 void
 397 memBufReport(MemBuf * mb)
 398 {
 399     assert(mb);
 400     mb->Printf("memBufReport is not yet implemented @?@\n");
 401 }
 402