src/patches/squid/squid-3.5-14067.patch

   1 ------------------------------------------------------------
   2 revno: 14067
   3 revision-id: squid3@treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
   4 parent: squid3@treenet.co.nz-20160701113616-vpjak1pq4uecadd2
   5 fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4534
   6 committer: Amos Jeffries <squid3@treenet.co.nz>
   7 branch nick: 3.5
   8 timestamp: Sat 2016-07-23 19:16:20 +1200
   9 message:
  10   Bug 4534: assertion failure in xcalloc when using many cache_dir
  11 ------------------------------------------------------------
  12 # Bazaar merge directive format 2 (Bazaar 0.90)
  13 # revision_id: squid3@treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
  14 # target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
  15 # testament_sha1: fcd663f0fd4a24d505f81eb94ef95d627a4ca363
  16 # timestamp: 2016-07-23 07:24:01 +0000
  17 # source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
  18 # base_revision_id: squid3@treenet.co.nz-20160701113616-\
  19 #   vpjak1pq4uecadd2
  20 #
  21 # Begin patch
  22 === modified file 'src/CacheDigest.cc'
  23 --- src/CacheDigest.cc  2016-01-01 00:14:27 +0000
  24 +++ src/CacheDigest.cc  2016-07-23 07:16:20 +0000
  25 @@ -35,12 +35,12 @@
  26  static uint32_t hashed_keys[4];
  27
  28  static void
  29 -cacheDigestInit(CacheDigest * cd, int capacity, int bpe)
  30 +cacheDigestInit(CacheDigest * cd, uint64_t capacity, uint8_t bpe)
  31  {
  32 -    const size_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
  33 +    const uint32_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
  34      assert(cd);
  35      assert(capacity > 0 && bpe > 0);
  36 -    assert(mask_size > 0);
  37 +    assert(mask_size != 0);
  38      cd->capacity = capacity;
  39      cd->bits_per_entry = bpe;
  40      cd->mask_size = mask_size;
  41 @@ -50,7 +50,7 @@
  42  }
  43
  44  CacheDigest *
  45 -cacheDigestCreate(int capacity, int bpe)
  46 +cacheDigestCreate(uint64_t capacity, uint8_t bpe)
  47  {
  48      CacheDigest *cd = (CacheDigest *)memAllocate(MEM_CACHE_DIGEST);
  49      assert(SQUID_MD5_DIGEST_LENGTH == 16);  /* our hash functions rely on 16 byte keys */
  50 @@ -97,7 +97,7 @@
  51
  52  /* changes mask size, resets bits to 0, preserves "cd" pointer */
  53  void
  54 -cacheDigestChangeCap(CacheDigest * cd, int new_cap)
  55 +cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap)
  56  {
  57      assert(cd);
  58      cacheDigestClean(cd);
  59 @@ -278,12 +278,12 @@
  60      storeAppendPrintf(e, "%s digest: size: %d bytes\n",
  61                        label ? label : "", stats.bit_count / 8
  62                       );
  63 -    storeAppendPrintf(e, "\t entries: count: %d capacity: %d util: %d%%\n",
  64 +    storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %" PRIu64 " util: %d%%\n",
  65                        cd->count,
  66                        cd->capacity,
  67                        xpercentInt(cd->count, cd->capacity)
  68                       );
  69 -    storeAppendPrintf(e, "\t deletion attempts: %d\n",
  70 +    storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n",
  71                        cd->del_count
  72                       );
  73      storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d util: %d%%\n",
  74 @@ -297,16 +297,18 @@
  75                       );
  76  }
  77
  78 -size_t
  79 -cacheDigestCalcMaskSize(int cap, int bpe)
  80 +uint32_t
  81 +cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe)
  82  {
  83 -    return (size_t) (cap * bpe + 7) / 8;
  84 +    uint64_t bitCount = (cap * bpe) + 7;
  85 +    assert(bitCount < INT_MAX); // dont 31-bit overflow later
  86 +    return static_cast<uint32_t>(bitCount / 8);
  87  }
  88
  89  static void
  90  cacheDigestHashKey(const CacheDigest * cd, const cache_key * key)
  91  {
  92 -    const unsigned int bit_count = cd->mask_size * 8;
  93 +    const uint32_t bit_count = cd->mask_size * 8;
  94      unsigned int tmp_keys[4];
  95      /* we must memcpy to ensure alignment */
  96      memcpy(tmp_keys, key, sizeof(tmp_keys));
  97
  98 === modified file 'src/CacheDigest.h'
  99 --- src/CacheDigest.h   2016-01-01 00:14:27 +0000
 100 +++ src/CacheDigest.h   2016-07-23 07:16:20 +0000
 101 @@ -22,23 +22,23 @@
 102  {
 103  public:
 104      /* public, read-only */
 105 -    char *mask;         /* bit mask */
 106 -    int mask_size;      /* mask size in bytes */
 107 -    int capacity;       /* expected maximum for .count, not a hard limit */
 108 -    int bits_per_entry;     /* number of bits allocated for each entry from capacity */
 109 -    int count;          /* number of digested entries */
 110 -    int del_count;      /* number of deletions performed so far */
 111 +    uint64_t count;          /* number of digested entries */
 112 +    uint64_t del_count;      /* number of deletions performed so far */
 113 +    uint64_t capacity;       /* expected maximum for .count, not a hard limit */
 114 +    char *mask;              /* bit mask */
 115 +    uint32_t mask_size;      /* mask size in bytes */
 116 +    int8_t bits_per_entry;   /* number of bits allocated for each entry from capacity */
 117  };
 118
 119 -CacheDigest *cacheDigestCreate(int capacity, int bpe);
 120 +CacheDigest *cacheDigestCreate(uint64_t capacity, uint8_t bpe);
 121  void cacheDigestDestroy(CacheDigest * cd);
 122  CacheDigest *cacheDigestClone(const CacheDigest * cd);
 123  void cacheDigestClear(CacheDigest * cd);
 124 -void cacheDigestChangeCap(CacheDigest * cd, int new_cap);
 125 +void cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap);
 126  int cacheDigestTest(const CacheDigest * cd, const cache_key * key);
 127  void cacheDigestAdd(CacheDigest * cd, const cache_key * key);
 128  void cacheDigestDel(CacheDigest * cd, const cache_key * key);
 129 -size_t cacheDigestCalcMaskSize(int cap, int bpe);
 130 +uint32_t cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe);
 131  int cacheDigestBitUtil(const CacheDigest * cd);
 132  void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int real_hit, int guess_hit);
 133  void cacheDigestGuessStatsReport(const CacheDigestGuessStats * stats, StoreEntry * sentry, const char *label);
 134
 135 === modified file 'src/PeerDigest.h'
 136 --- src/PeerDigest.h    2016-01-01 00:14:27 +0000
 137 +++ src/PeerDigest.h    2016-07-23 07:16:20 +0000
 138 @@ -52,7 +52,7 @@
 139      store_client *old_sc;
 140      HttpRequest *request;
 141      int offset;
 142 -    int mask_offset;
 143 +    uint32_t mask_offset;
 144      time_t start_time;
 145      time_t resp_time;
 146      time_t expires;
 147
 148 === modified file 'src/peer_digest.cc'
 149 --- src/peer_digest.cc  2016-01-01 00:14:27 +0000
 150 +++ src/peer_digest.cc  2016-07-23 07:16:20 +0000
 151 @@ -754,7 +754,7 @@
 152      if (!reason && !size) {
 153          if (!pd->cd)
 154              reason = "null digest?!";
 155 -        else if (fetch->mask_offset != (int)pd->cd->mask_size)
 156 +        else if (fetch->mask_offset != pd->cd->mask_size)
 157              reason = "premature end of digest?!";
 158          else if (!peerDigestUseful(pd))
 159              reason = "useless digest";
 160
 161 === modified file 'src/store_digest.cc'
 162 --- src/store_digest.cc 2016-01-01 00:14:27 +0000
 163 +++ src/store_digest.cc 2016-07-23 07:16:20 +0000
 164 @@ -76,36 +76,63 @@
 165  static void storeDigestRewriteFinish(StoreEntry * e);
 166  static EVH storeDigestSwapOutStep;
 167  static void storeDigestCBlockSwapOut(StoreEntry * e);
 168 -static int storeDigestCalcCap(void);
 169 -static int storeDigestResize(void);
 170  static void storeDigestAdd(const StoreEntry *);
 171
 172 +/// calculates digest capacity
 173 +static uint64_t
 174 +storeDigestCalcCap()
 175 +{
 176 +    /*
 177 +     * To-Do: Bloom proved that the optimal filter utilization is 50% (half of
 178 +     * the bits are off). However, we do not have a formula to calculate the
 179 +     * number of _entries_ we want to pre-allocate for.
 180 +     */
 181 +    const uint64_t hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
 182 +    const uint64_t lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize;
 183 +    const uint64_t e_count = StoreEntry::inUseCount();
 184 +    uint64_t cap = e_count ? e_count : hi_cap;
 185 +    debugs(71, 2, "have: " << e_count << ", want " << cap <<
 186 +           " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
 187 +
 188 +    if (cap < lo_cap)
 189 +        cap = lo_cap;
 190 +
 191 +    /* do not enforce hi_cap limit, average-based estimation may be wrong
 192 +     *if (cap > hi_cap)
 193 +     *  cap = hi_cap;
 194 +     */
 195 +
 196 +    // Bug 4534: we still have to set an upper-limit at some reasonable value though.
 197 +    // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX
 198 +    const uint64_t absolute_max = (INT_MAX -8) / Config.digest.bits_per_entry;
 199 +    if (cap > absolute_max) {
 200 +        static time_t last_loud = 0;
 201 +        if (last_loud < squid_curtime - 86400) {
 202 +            debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max);
 203 +            last_loud = squid_curtime;
 204 +        } else {
 205 +            debugs(71, 3, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max);
 206 +        }
 207 +        cap = absolute_max;
 208 +    }
 209 +
 210 +    return cap;
 211 +}
 212  #endif /* USE_CACHE_DIGESTS */
 213
 214 -static void
 215 -storeDigestRegisterWithCacheManager(void)
 216 +void
 217 +storeDigestInit(void)
 218  {
 219      Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport, 0, 1);
 220 -}
 221 -
 222 -/*
 223 - * PUBLIC FUNCTIONS
 224 - */
 225 -
 226 -void
 227 -storeDigestInit(void)
 228 -{
 229 -    storeDigestRegisterWithCacheManager();
 230
 231  #if USE_CACHE_DIGESTS
 232 -    const int cap = storeDigestCalcCap();
 233 -
 234      if (!Config.onoff.digest_generation) {
 235          store_digest = NULL;
 236          debugs(71, 3, "Local cache digest generation disabled");
 237          return;
 238      }
 239
 240 +    const uint64_t cap = storeDigestCalcCap();
 241      store_digest = cacheDigestCreate(cap, Config.digest.bits_per_entry);
 242      debugs(71, DBG_IMPORTANT, "Local cache digest enabled; rebuild/rewrite every " <<
 243             (int) Config.digest.rebuild_period << "/" <<
 244 @@ -290,6 +317,31 @@
 245      storeDigestRebuildResume();
 246  }
 247
 248 +/// \returns true if we actually resized the digest
 249 +static bool
 250 +storeDigestResize()
 251 +{
 252 +    const uint64_t cap = storeDigestCalcCap();
 253 +    assert(store_digest);
 254 +    uint64_t diff;
 255 +    if (cap > store_digest->capacity)
 256 +        diff = cap - store_digest->capacity;
 257 +    else
 258 +        diff = store_digest->capacity - cap;
 259 +    debugs(71, 2, store_digest->capacity << " -> " << cap << "; change: " <<
 260 +           diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" );
 261 +    /* avoid minor adjustments */
 262 +
 263 +    if (diff <= store_digest->capacity / 10) {
 264 +        debugs(71, 2, "small change, will not resize.");
 265 +        return false;
 266 +    } else {
 267 +        debugs(71, 2, "big change, resizing.");
 268 +        cacheDigestChangeCap(store_digest, cap);
 269 +    }
 270 +    return true;
 271 +}
 272 +
 273  /* called be Rewrite to push Rebuild forward */
 274  static void
 275  storeDigestRebuildResume(void)
 276 @@ -439,7 +491,7 @@
 277      assert(e);
 278      /* _add_ check that nothing bad happened while we were waiting @?@ @?@ */
 279
 280 -    if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size)
 281 +    if (static_cast<uint32_t>(sd_state.rewrite_offset + chunk_size) > store_digest->mask_size)
 282          chunk_size = store_digest->mask_size - sd_state.rewrite_offset;
 283
 284      e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size);
 285 @@ -451,7 +503,7 @@
 286      sd_state.rewrite_offset += chunk_size;
 287
 288      /* are we done ? */
 289 -    if (sd_state.rewrite_offset >= store_digest->mask_size)
 290 +    if (static_cast<uint32_t>(sd_state.rewrite_offset) >= store_digest->mask_size)
 291          storeDigestRewriteFinish(e);
 292      else
 293          eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data, 0.0, 1, false);
 294 @@ -467,60 +519,10 @@
 295      sd_state.cblock.count = htonl(store_digest->count);
 296      sd_state.cblock.del_count = htonl(store_digest->del_count);
 297      sd_state.cblock.mask_size = htonl(store_digest->mask_size);
 298 -    sd_state.cblock.bits_per_entry = (unsigned char)
 299 -                                     Config.digest.bits_per_entry;
 300 +    sd_state.cblock.bits_per_entry = Config.digest.bits_per_entry;
 301      sd_state.cblock.hash_func_count = (unsigned char) CacheDigestHashFuncCount;
 302      e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock));
 303  }
 304
 305 -/* calculates digest capacity */
 306 -static int
 307 -storeDigestCalcCap(void)
 308 -{
 309 -    /*
 310 -     * To-Do: Bloom proved that the optimal filter utilization is 50% (half of
 311 -     * the bits are off). However, we do not have a formula to calculate the
 312 -     * number of _entries_ we want to pre-allocate for.
 313 -     */
 314 -    const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
 315 -    const int lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize;
 316 -    const int e_count = StoreEntry::inUseCount();
 317 -    int cap = e_count ? e_count :hi_cap;
 318 -    debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " << cap <<
 319 -           " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
 320 -
 321 -    if (cap < lo_cap)
 322 -        cap = lo_cap;
 323 -
 324 -    /* do not enforce hi_cap limit, average-based estimation may be wrong
 325 -     *if (cap > hi_cap)
 326 -     *  cap = hi_cap;
 327 -     */
 328 -    return cap;
 329 -}
 330 -
 331 -/* returns true if we actually resized the digest */
 332 -static int
 333 -storeDigestResize(void)
 334 -{
 335 -    const int cap = storeDigestCalcCap();
 336 -    int diff;
 337 -    assert(store_digest);
 338 -    diff = abs(cap - store_digest->capacity);
 339 -    debugs(71, 2, "storeDigestResize: " <<
 340 -           store_digest->capacity << " -> " << cap << "; change: " <<
 341 -           diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" );
 342 -    /* avoid minor adjustments */
 343 -
 344 -    if (diff <= store_digest->capacity / 10) {
 345 -        debugs(71, 2, "storeDigestResize: small change, will not resize.");
 346 -        return 0;
 347 -    } else {
 348 -        debugs(71, 2, "storeDigestResize: big change, resizing.");
 349 -        cacheDigestChangeCap(store_digest, cap);
 350 -        return 1;
 351 -    }
 352 -}
 353 -
 354  #endif /* USE_CACHE_DIGESTS */
 355
 356
 357 === modified file 'src/tests/stub_CacheDigest.cc'
 358 --- src/tests/stub_CacheDigest.cc       2016-01-01 00:14:27 +0000
 359 +++ src/tests/stub_CacheDigest.cc       2016-07-23 07:16:20 +0000
 360 @@ -16,11 +16,11 @@
 361  class CacheDigestGuessStats;
 362  class StoreEntry;
 363
 364 -CacheDigest * cacheDigestCreate(int, int) STUB_RETVAL(NULL)
 365 +CacheDigest * cacheDigestCreate(uint64_t, uint8_t) STUB_RETVAL(NULL)
 366  void cacheDigestDestroy(CacheDigest *) STUB
 367  CacheDigest * cacheDigestClone(const CacheDigest *) STUB_RETVAL(NULL)
 368  void cacheDigestClear(CacheDigest * ) STUB
 369 -void cacheDigestChangeCap(CacheDigest *,int) STUB
 370 +void cacheDigestChangeCap(CacheDigest *,uint64_t) STUB
 371  int cacheDigestTest(const CacheDigest *, const cache_key *) STUB_RETVAL(1)
 372  void cacheDigestAdd(CacheDigest *, const cache_key *) STUB
 373  void cacheDigestDel(CacheDigest *, const cache_key *) STUB
 374 @@ -28,5 +28,4 @@
 375  void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB
 376  void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry *, const char *) STUB
 377  void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
 378 -size_t cacheDigestCalcMaskSize(int, int) STUB_RETVAL(1)
 379 -
 380 +uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)
 381