1 ------------------------------------------------------------
3 revision-id: squid3@treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
4 parent: squid3@treenet.co.nz-20160701113616-vpjak1pq4uecadd2
5 fixes bug: http://bugs.squid-cache.org/show_bug.cgi?id=4534
6 committer: Amos Jeffries <squid3@treenet.co.nz>
8 timestamp: Sat 2016-07-23 19:16:20 +1200
10 Bug 4534: assertion failure in xcalloc when using many cache_dir
11 ------------------------------------------------------------
12 # Bazaar merge directive format 2 (Bazaar 0.90)
13 # revision_id: squid3@treenet.co.nz-20160723071620-1wzqpbyi1rk5w6vg
14 # target_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
15 # testament_sha1: fcd663f0fd4a24d505f81eb94ef95d627a4ca363
16 # timestamp: 2016-07-23 07:24:01 +0000
17 # source_branch: http://bzr.squid-cache.org/bzr/squid3/3.5
18 # base_revision_id: squid3@treenet.co.nz-20160701113616-\
22 === modified file 'src/CacheDigest.cc'
23 --- src/CacheDigest.cc 2016-01-01 00:14:27 +0000
24 +++ src/CacheDigest.cc 2016-07-23 07:16:20 +0000
26 static uint32_t hashed_keys[4];
29 -cacheDigestInit(CacheDigest * cd, int capacity, int bpe)
30 +cacheDigestInit(CacheDigest * cd, uint64_t capacity, uint8_t bpe)
32 - const size_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
33 + const uint32_t mask_size = cacheDigestCalcMaskSize(capacity, bpe);
35 assert(capacity > 0 && bpe > 0);
36 - assert(mask_size > 0);
37 + assert(mask_size != 0);
38 cd->capacity = capacity;
39 cd->bits_per_entry = bpe;
40 cd->mask_size = mask_size;
45 -cacheDigestCreate(int capacity, int bpe)
46 +cacheDigestCreate(uint64_t capacity, uint8_t bpe)
48 CacheDigest *cd = (CacheDigest *)memAllocate(MEM_CACHE_DIGEST);
49 assert(SQUID_MD5_DIGEST_LENGTH == 16); /* our hash functions rely on 16 byte keys */
52 /* changes mask size, resets bits to 0, preserves "cd" pointer */
54 -cacheDigestChangeCap(CacheDigest * cd, int new_cap)
55 +cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap)
60 storeAppendPrintf(e, "%s digest: size: %d bytes\n",
61 label ? label : "", stats.bit_count / 8
63 - storeAppendPrintf(e, "\t entries: count: %d capacity: %d util: %d%%\n",
64 + storeAppendPrintf(e, "\t entries: count: %" PRIu64 " capacity: %" PRIu64 " util: %d%%\n",
67 xpercentInt(cd->count, cd->capacity)
69 - storeAppendPrintf(e, "\t deletion attempts: %d\n",
70 + storeAppendPrintf(e, "\t deletion attempts: %" PRIu64 "\n",
73 storeAppendPrintf(e, "\t bits: per entry: %d on: %d capacity: %d util: %d%%\n",
79 -cacheDigestCalcMaskSize(int cap, int bpe)
81 +cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe)
83 - return (size_t) (cap * bpe + 7) / 8;
84 + uint64_t bitCount = (cap * bpe) + 7;
85 + assert(bitCount < INT_MAX); // dont 31-bit overflow later
86 + return static_cast<uint32_t>(bitCount / 8);
90 cacheDigestHashKey(const CacheDigest * cd, const cache_key * key)
92 - const unsigned int bit_count = cd->mask_size * 8;
93 + const uint32_t bit_count = cd->mask_size * 8;
94 unsigned int tmp_keys[4];
95 /* we must memcpy to ensure alignment */
96 memcpy(tmp_keys, key, sizeof(tmp_keys));
98 === modified file 'src/CacheDigest.h'
99 --- src/CacheDigest.h 2016-01-01 00:14:27 +0000
100 +++ src/CacheDigest.h 2016-07-23 07:16:20 +0000
104 /* public, read-only */
105 - char *mask; /* bit mask */
106 - int mask_size; /* mask size in bytes */
107 - int capacity; /* expected maximum for .count, not a hard limit */
108 - int bits_per_entry; /* number of bits allocated for each entry from capacity */
109 - int count; /* number of digested entries */
110 - int del_count; /* number of deletions performed so far */
111 + uint64_t count; /* number of digested entries */
112 + uint64_t del_count; /* number of deletions performed so far */
113 + uint64_t capacity; /* expected maximum for .count, not a hard limit */
114 + char *mask; /* bit mask */
115 + uint32_t mask_size; /* mask size in bytes */
116 + int8_t bits_per_entry; /* number of bits allocated for each entry from capacity */
119 -CacheDigest *cacheDigestCreate(int capacity, int bpe);
120 +CacheDigest *cacheDigestCreate(uint64_t capacity, uint8_t bpe);
121 void cacheDigestDestroy(CacheDigest * cd);
122 CacheDigest *cacheDigestClone(const CacheDigest * cd);
123 void cacheDigestClear(CacheDigest * cd);
124 -void cacheDigestChangeCap(CacheDigest * cd, int new_cap);
125 +void cacheDigestChangeCap(CacheDigest * cd, uint64_t new_cap);
126 int cacheDigestTest(const CacheDigest * cd, const cache_key * key);
127 void cacheDigestAdd(CacheDigest * cd, const cache_key * key);
128 void cacheDigestDel(CacheDigest * cd, const cache_key * key);
129 -size_t cacheDigestCalcMaskSize(int cap, int bpe);
130 +uint32_t cacheDigestCalcMaskSize(uint64_t cap, uint8_t bpe);
131 int cacheDigestBitUtil(const CacheDigest * cd);
132 void cacheDigestGuessStatsUpdate(CacheDigestGuessStats * stats, int real_hit, int guess_hit);
133 void cacheDigestGuessStatsReport(const CacheDigestGuessStats * stats, StoreEntry * sentry, const char *label);
135 === modified file 'src/PeerDigest.h'
136 --- src/PeerDigest.h 2016-01-01 00:14:27 +0000
137 +++ src/PeerDigest.h 2016-07-23 07:16:20 +0000
139 store_client *old_sc;
140 HttpRequest *request;
143 + uint32_t mask_offset;
148 === modified file 'src/peer_digest.cc'
149 --- src/peer_digest.cc 2016-01-01 00:14:27 +0000
150 +++ src/peer_digest.cc 2016-07-23 07:16:20 +0000
152 if (!reason && !size) {
154 reason = "null digest?!";
155 - else if (fetch->mask_offset != (int)pd->cd->mask_size)
156 + else if (fetch->mask_offset != pd->cd->mask_size)
157 reason = "premature end of digest?!";
158 else if (!peerDigestUseful(pd))
159 reason = "useless digest";
161 === modified file 'src/store_digest.cc'
162 --- src/store_digest.cc 2016-01-01 00:14:27 +0000
163 +++ src/store_digest.cc 2016-07-23 07:16:20 +0000
165 static void storeDigestRewriteFinish(StoreEntry * e);
166 static EVH storeDigestSwapOutStep;
167 static void storeDigestCBlockSwapOut(StoreEntry * e);
168 -static int storeDigestCalcCap(void);
169 -static int storeDigestResize(void);
170 static void storeDigestAdd(const StoreEntry *);
172 +/// calculates digest capacity
174 +storeDigestCalcCap()
177 + * To-Do: Bloom proved that the optimal filter utilization is 50% (half of
178 + * the bits are off). However, we do not have a formula to calculate the
179 + * number of _entries_ we want to pre-allocate for.
181 + const uint64_t hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
182 + const uint64_t lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize;
183 + const uint64_t e_count = StoreEntry::inUseCount();
184 + uint64_t cap = e_count ? e_count : hi_cap;
185 + debugs(71, 2, "have: " << e_count << ", want " << cap <<
186 + " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
191 + /* do not enforce hi_cap limit, average-based estimation may be wrong
196 + // Bug 4534: we still have to set an upper-limit at some reasonable value though.
197 + // this matches cacheDigestCalcMaskSize doing (cap*bpe)+7 < INT_MAX
198 + const uint64_t absolute_max = (INT_MAX -8) / Config.digest.bits_per_entry;
199 + if (cap > absolute_max) {
200 + static time_t last_loud = 0;
201 + if (last_loud < squid_curtime - 86400) {
202 + debugs(71, DBG_IMPORTANT, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max);
203 + last_loud = squid_curtime;
205 + debugs(71, 3, "WARNING: Cache Digest cannot store " << cap << " entries. Limiting to " << absolute_max);
207 + cap = absolute_max;
212 #endif /* USE_CACHE_DIGESTS */
215 -storeDigestRegisterWithCacheManager(void)
217 +storeDigestInit(void)
219 Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport, 0, 1);
227 -storeDigestInit(void)
229 - storeDigestRegisterWithCacheManager();
231 #if USE_CACHE_DIGESTS
232 - const int cap = storeDigestCalcCap();
234 if (!Config.onoff.digest_generation) {
236 debugs(71, 3, "Local cache digest generation disabled");
240 + const uint64_t cap = storeDigestCalcCap();
241 store_digest = cacheDigestCreate(cap, Config.digest.bits_per_entry);
242 debugs(71, DBG_IMPORTANT, "Local cache digest enabled; rebuild/rewrite every " <<
243 (int) Config.digest.rebuild_period << "/" <<
245 storeDigestRebuildResume();
248 +/// \returns true if we actually resized the digest
252 + const uint64_t cap = storeDigestCalcCap();
253 + assert(store_digest);
255 + if (cap > store_digest->capacity)
256 + diff = cap - store_digest->capacity;
258 + diff = store_digest->capacity - cap;
259 + debugs(71, 2, store_digest->capacity << " -> " << cap << "; change: " <<
260 + diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" );
261 + /* avoid minor adjustments */
263 + if (diff <= store_digest->capacity / 10) {
264 + debugs(71, 2, "small change, will not resize.");
267 + debugs(71, 2, "big change, resizing.");
268 + cacheDigestChangeCap(store_digest, cap);
273 /* called be Rewrite to push Rebuild forward */
275 storeDigestRebuildResume(void)
278 /* _add_ check that nothing bad happened while we were waiting @?@ @?@ */
280 - if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size)
281 + if (static_cast<uint32_t>(sd_state.rewrite_offset + chunk_size) > store_digest->mask_size)
282 chunk_size = store_digest->mask_size - sd_state.rewrite_offset;
284 e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size);
286 sd_state.rewrite_offset += chunk_size;
289 - if (sd_state.rewrite_offset >= store_digest->mask_size)
290 + if (static_cast<uint32_t>(sd_state.rewrite_offset) >= store_digest->mask_size)
291 storeDigestRewriteFinish(e);
293 eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data, 0.0, 1, false);
294 @@ -467,60 +519,10 @@
295 sd_state.cblock.count = htonl(store_digest->count);
296 sd_state.cblock.del_count = htonl(store_digest->del_count);
297 sd_state.cblock.mask_size = htonl(store_digest->mask_size);
298 - sd_state.cblock.bits_per_entry = (unsigned char)
299 - Config.digest.bits_per_entry;
300 + sd_state.cblock.bits_per_entry = Config.digest.bits_per_entry;
301 sd_state.cblock.hash_func_count = (unsigned char) CacheDigestHashFuncCount;
302 e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock));
305 -/* calculates digest capacity */
307 -storeDigestCalcCap(void)
310 - * To-Do: Bloom proved that the optimal filter utilization is 50% (half of
311 - * the bits are off). However, we do not have a formula to calculate the
312 - * number of _entries_ we want to pre-allocate for.
314 - const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize;
315 - const int lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize;
316 - const int e_count = StoreEntry::inUseCount();
317 - int cap = e_count ? e_count :hi_cap;
318 - debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " << cap <<
319 - " entries; limits: [" << lo_cap << ", " << hi_cap << "]");
324 - /* do not enforce hi_cap limit, average-based estimation may be wrong
331 -/* returns true if we actually resized the digest */
333 -storeDigestResize(void)
335 - const int cap = storeDigestCalcCap();
337 - assert(store_digest);
338 - diff = abs(cap - store_digest->capacity);
339 - debugs(71, 2, "storeDigestResize: " <<
340 - store_digest->capacity << " -> " << cap << "; change: " <<
341 - diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" );
342 - /* avoid minor adjustments */
344 - if (diff <= store_digest->capacity / 10) {
345 - debugs(71, 2, "storeDigestResize: small change, will not resize.");
348 - debugs(71, 2, "storeDigestResize: big change, resizing.");
349 - cacheDigestChangeCap(store_digest, cap);
354 #endif /* USE_CACHE_DIGESTS */
357 === modified file 'src/tests/stub_CacheDigest.cc'
358 --- src/tests/stub_CacheDigest.cc 2016-01-01 00:14:27 +0000
359 +++ src/tests/stub_CacheDigest.cc 2016-07-23 07:16:20 +0000
361 class CacheDigestGuessStats;
364 -CacheDigest * cacheDigestCreate(int, int) STUB_RETVAL(NULL)
365 +CacheDigest * cacheDigestCreate(uint64_t, uint8_t) STUB_RETVAL(NULL)
366 void cacheDigestDestroy(CacheDigest *) STUB
367 CacheDigest * cacheDigestClone(const CacheDigest *) STUB_RETVAL(NULL)
368 void cacheDigestClear(CacheDigest * ) STUB
369 -void cacheDigestChangeCap(CacheDigest *,int) STUB
370 +void cacheDigestChangeCap(CacheDigest *,uint64_t) STUB
371 int cacheDigestTest(const CacheDigest *, const cache_key *) STUB_RETVAL(1)
372 void cacheDigestAdd(CacheDigest *, const cache_key *) STUB
373 void cacheDigestDel(CacheDigest *, const cache_key *) STUB
375 void cacheDigestGuessStatsUpdate(CacheDigestGuessStats *, int, int) STUB
376 void cacheDigestGuessStatsReport(const CacheDigestGuessStats *, StoreEntry *, const char *) STUB
377 void cacheDigestReport(CacheDigest *, const char *, StoreEntry *) STUB
378 -size_t cacheDigestCalcMaskSize(int, int) STUB_RETVAL(1)
380 +uint32_t cacheDigestCalcMaskSize(uint64_t, uint8_t) STUB_RETVAL(1)