]>
Commit | Line | Data |
---|---|---|
8638fc66 | 1 | /* |
8638fc66 | 2 | * DEBUG: section 71 Store Digest Manager |
3 | * AUTHOR: Alex Rousskov | |
4 | * | |
2b6662ba | 5 | * SQUID Web Proxy Cache http://www.squid-cache.org/ |
e25c139f | 6 | * ---------------------------------------------------------- |
8638fc66 | 7 | * |
2b6662ba | 8 | * Squid is the result of efforts by numerous individuals from |
9 | * the Internet community; see the CONTRIBUTORS file for full | |
10 | * details. Many organizations have provided support for Squid's | |
11 | * development; see the SPONSORS file for full details. Squid is | |
12 | * Copyrighted (C) 2001 by the Regents of the University of | |
13 | * California; see the COPYRIGHT file for full details. Squid | |
14 | * incorporates software developed and/or copyrighted by other | |
15 | * sources; see the CREDITS file for full details. | |
8638fc66 | 16 | * |
17 | * This program is free software; you can redistribute it and/or modify | |
18 | * it under the terms of the GNU General Public License as published by | |
19 | * the Free Software Foundation; either version 2 of the License, or | |
20 | * (at your option) any later version. | |
26ac0430 | 21 | * |
8638fc66 | 22 | * This program is distributed in the hope that it will be useful, |
23 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
24 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
25 | * GNU General Public License for more details. | |
26ac0430 | 26 | * |
8638fc66 | 27 | * You should have received a copy of the GNU General Public License |
28 | * along with this program; if not, write to the Free Software | |
cbdec147 | 29 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. |
e25c139f | 30 | * |
8638fc66 | 31 | */ |
32 | ||
6168bccd | 33 | /* |
34 | * TODO: We probably do not track all the cases when | |
35 | * storeDigestNoteStoreReady() must be called; this may prevent | |
36 | * storeDigestRebuild/write schedule to be activated | |
37 | */ | |
38 | ||
582c2af2 FC |
39 | #include "squid.h" |
40 | #include "Debug.h" | |
a553a5a3 | 41 | #include "event.h" |
582c2af2 | 42 | #include "globals.h" |
8822ebee | 43 | #include "mgr/Registration.h" |
35a28a37 | 44 | #include "store_digest.h" |
d6fd3381 | 45 | |
b814e8d4 FC |
46 | #if USE_CACHE_DIGESTS |
47 | #include "CacheDigest.h" | |
528b2c61 | 48 | #include "HttpReply.h" |
582c2af2 | 49 | #include "HttpRequest.h" |
308e60be | 50 | #include "internal.h" |
528b2c61 | 51 | #include "MemObject.h" |
aa839030 | 52 | #include "PeerDigest.h" |
c6f15d40 | 53 | #include "refresh.h" |
4d5904f7 | 54 | #include "SquidConfig.h" |
985c86bc | 55 | #include "SquidTime.h" |
582c2af2 | 56 | #include "Store.h" |
c8f4eac4 | 57 | #include "StoreSearch.h" |
528b2c61 | 58 | |
582c2af2 FC |
59 | #if HAVE_MATH_H |
60 | #include <math.h> | |
61 | #endif | |
62 | ||
d6fd3381 | 63 | /* |
64 | * local types | |
65 | */ | |
12784378 | 66 | |
c8f4eac4 | 67 | class StoreDigestState |
62e76326 | 68 | { |
c8f4eac4 | 69 | |
70 | public: | |
12784378 | 71 | StoreDigestCBlock cblock; |
b644367b | 72 | int rebuild_lock; /* bucket number */ |
aa839030 | 73 | StoreEntry * rewrite_lock; /* points to store entry with the digest */ |
c8f4eac4 | 74 | StoreSearchPointer theSearch; |
12784378 | 75 | int rewrite_offset; |
76 | int rebuild_count; | |
b644367b | 77 | int rewrite_count; |
c8f4eac4 | 78 | }; |
12784378 | 79 | |
26ac0430 | 80 | typedef struct { |
4b4cd312 | 81 | int del_count; /* #store entries deleted from store_digest */ |
82 | int del_lost_count; /* #store entries not found in store_digest on delete */ | |
83 | int add_count; /* #store entries accepted to store_digest */ | |
84 | int add_coll_count; /* #accepted entries that collided with existing ones */ | |
85 | int rej_count; /* #store entries not accepted to store_digest */ | |
86 | int rej_coll_count; /* #not accepted entries that collided with existing ones */ | |
2fadd50d | 87 | } StoreDigestStats; |
6168bccd | 88 | |
462f66d2 | 89 | /* local vars */ |
12784378 | 90 | static StoreDigestState sd_state; |
6168bccd | 91 | static StoreDigestStats sd_stats; |
12784378 | 92 | |
93 | /* local prototypes */ | |
6168bccd | 94 | static void storeDigestRebuildStart(void *datanotused); |
d6fd3381 | 95 | static void storeDigestRebuildResume(void); |
96 | static void storeDigestRebuildFinish(void); | |
12784378 | 97 | static void storeDigestRebuildStep(void *datanotused); |
d6fd3381 | 98 | static void storeDigestRewriteStart(void *); |
99 | static void storeDigestRewriteResume(void); | |
b644367b | 100 | static void storeDigestRewriteFinish(StoreEntry * e); |
d6f51e3c | 101 | static EVH storeDigestSwapOutStep; |
b644367b | 102 | static void storeDigestCBlockSwapOut(StoreEntry * e); |
d6fd3381 | 103 | static int storeDigestCalcCap(void); |
104 | static int storeDigestResize(void); | |
105 | static void storeDigestAdd(const StoreEntry *); | |
12784378 | 106 | |
d6fd3381 | 107 | #endif /* USE_CACHE_DIGESTS */ |
108 | ||
5f5e883f FC |
109 | static void |
110 | storeDigestRegisterWithCacheManager(void) | |
111 | { | |
8822ebee | 112 | Mgr::RegisterAction("store_digest", "Store Digest", storeDigestReport, 0, 1); |
5f5e883f FC |
113 | } |
114 | ||
d6fd3381 | 115 | /* |
116 | * PUBLIC FUNCTIONS | |
117 | */ | |
12784378 | 118 | |
8638fc66 | 119 | void |
d6fd3381 | 120 | storeDigestInit(void) |
8638fc66 | 121 | { |
d120ed12 FC |
122 | storeDigestRegisterWithCacheManager(); |
123 | ||
6cfa8966 | 124 | #if USE_CACHE_DIGESTS |
304b267e | 125 | const int cap = storeDigestCalcCap(); |
7e3ce7b9 | 126 | |
127 | if (!Config.onoff.digest_generation) { | |
62e76326 | 128 | store_digest = NULL; |
bf8fe701 | 129 | debugs(71, 3, "Local cache digest generation disabled"); |
62e76326 | 130 | return; |
7e3ce7b9 | 131 | } |
62e76326 | 132 | |
7e3ce7b9 | 133 | store_digest = cacheDigestCreate(cap, Config.digest.bits_per_entry); |
e0236918 | 134 | debugs(71, DBG_IMPORTANT, "Local cache digest enabled; rebuild/rewrite every " << |
bf8fe701 | 135 | (int) Config.digest.rebuild_period << "/" << |
136 | (int) Config.digest.rewrite_period << " sec"); | |
137 | ||
12784378 | 138 | memset(&sd_state, 0, sizeof(sd_state)); |
d6fd3381 | 139 | #else |
140 | store_digest = NULL; | |
bf8fe701 | 141 | debugs(71, 3, "Local cache digest is 'off'"); |
d6fd3381 | 142 | #endif |
8638fc66 | 143 | } |
144 | ||
6168bccd | 145 | /* called when store_rebuild completes */ |
8638fc66 | 146 | void |
d6fd3381 | 147 | storeDigestNoteStoreReady(void) |
12784378 | 148 | { |
d6fd3381 | 149 | #if USE_CACHE_DIGESTS |
62e76326 | 150 | |
7e3ce7b9 | 151 | if (Config.onoff.digest_generation) { |
62e76326 | 152 | storeDigestRebuildStart(NULL); |
153 | storeDigestRewriteStart(NULL); | |
7e3ce7b9 | 154 | } |
62e76326 | 155 | |
d6fd3381 | 156 | #endif |
157 | } | |
158 | ||
35a28a37 | 159 | //TODO: this seems to be dead code. Is it needed? |
d6fd3381 | 160 | void |
161 | storeDigestDel(const StoreEntry * entry) | |
162 | { | |
163 | #if USE_CACHE_DIGESTS | |
62e76326 | 164 | |
7e3ce7b9 | 165 | if (!Config.onoff.digest_generation) { |
62e76326 | 166 | return; |
7e3ce7b9 | 167 | } |
62e76326 | 168 | |
d6fd3381 | 169 | assert(entry && store_digest); |
bf8fe701 | 170 | debugs(71, 6, "storeDigestDel: checking entry, key: " << entry->getMD5Text()); |
62e76326 | 171 | |
d46a87a8 | 172 | if (!EBIT_TEST(entry->flags, KEY_PRIVATE)) { |
62e76326 | 173 | if (!cacheDigestTest(store_digest, (const cache_key *)entry->key)) { |
5db6bf73 | 174 | ++sd_stats.del_lost_count; |
bf8fe701 | 175 | debugs(71, 6, "storeDigestDel: lost entry, key: " << entry->getMD5Text() << " url: " << entry->url() ); |
62e76326 | 176 | } else { |
5db6bf73 | 177 | ++sd_stats.del_count; |
62e76326 | 178 | cacheDigestDel(store_digest, (const cache_key *)entry->key); |
bf8fe701 | 179 | debugs(71, 6, "storeDigestDel: deled entry, key: " << entry->getMD5Text()); |
62e76326 | 180 | } |
d6fd3381 | 181 | } |
35a28a37 | 182 | #endif //USE_CACHE_DIGESTS |
12784378 | 183 | } |
184 | ||
12784378 | 185 | void |
d6fd3381 | 186 | storeDigestReport(StoreEntry * e) |
187 | { | |
188 | #if USE_CACHE_DIGESTS | |
62e76326 | 189 | |
7e3ce7b9 | 190 | if (!Config.onoff.digest_generation) { |
62e76326 | 191 | return; |
7e3ce7b9 | 192 | } |
62e76326 | 193 | |
d6fd3381 | 194 | if (store_digest) { |
62e76326 | 195 | cacheDigestReport(store_digest, "store", e); |
196 | storeAppendPrintf(e, "\t added: %d rejected: %d ( %.2f %%) del-ed: %d\n", | |
197 | sd_stats.add_count, | |
198 | sd_stats.rej_count, | |
199 | xpercent(sd_stats.rej_count, sd_stats.rej_count + sd_stats.add_count), | |
200 | sd_stats.del_count); | |
201 | storeAppendPrintf(e, "\t collisions: on add: %.2f %% on rej: %.2f %%\n", | |
202 | xpercent(sd_stats.add_coll_count, sd_stats.add_count), | |
203 | xpercent(sd_stats.rej_coll_count, sd_stats.rej_count)); | |
d6fd3381 | 204 | } else { |
62e76326 | 205 | storeAppendPrintf(e, "store digest: disabled.\n"); |
d6fd3381 | 206 | } |
62e76326 | 207 | |
35a28a37 | 208 | #endif //USE_CACHE_DIGESTS |
d6fd3381 | 209 | } |
210 | ||
211 | /* | |
212 | * LOCAL FUNCTIONS | |
213 | */ | |
214 | ||
215 | #if USE_CACHE_DIGESTS | |
216 | ||
c68e9c6b | 217 | /* should we digest this entry? used by storeDigestAdd() */ |
218 | static int | |
219 | storeDigestAddable(const StoreEntry * e) | |
220 | { | |
221 | /* add some stats! XXX */ | |
222 | ||
bf8fe701 | 223 | debugs(71, 6, "storeDigestAddable: checking entry, key: " << e->getMD5Text()); |
c68e9c6b | 224 | |
3900307b | 225 | /* check various entry flags (mimics StoreEntry::checkCachable XXX) */ |
62e76326 | 226 | |
c68e9c6b | 227 | if (!EBIT_TEST(e->flags, ENTRY_CACHABLE)) { |
bf8fe701 | 228 | debugs(71, 6, "storeDigestAddable: NO: not cachable"); |
62e76326 | 229 | return 0; |
c68e9c6b | 230 | } |
62e76326 | 231 | |
c68e9c6b | 232 | if (EBIT_TEST(e->flags, KEY_PRIVATE)) { |
bf8fe701 | 233 | debugs(71, 6, "storeDigestAddable: NO: private key"); |
62e76326 | 234 | return 0; |
c68e9c6b | 235 | } |
62e76326 | 236 | |
c68e9c6b | 237 | if (EBIT_TEST(e->flags, ENTRY_NEGCACHED)) { |
bf8fe701 | 238 | debugs(71, 6, "storeDigestAddable: NO: negative cached"); |
62e76326 | 239 | return 0; |
c68e9c6b | 240 | } |
62e76326 | 241 | |
c68e9c6b | 242 | if (EBIT_TEST(e->flags, RELEASE_REQUEST)) { |
bf8fe701 | 243 | debugs(71, 6, "storeDigestAddable: NO: release requested"); |
62e76326 | 244 | return 0; |
c68e9c6b | 245 | } |
62e76326 | 246 | |
c68e9c6b | 247 | if (e->store_status == STORE_OK && EBIT_TEST(e->flags, ENTRY_BAD_LENGTH)) { |
bf8fe701 | 248 | debugs(71, 6, "storeDigestAddable: NO: wrong content-length"); |
62e76326 | 249 | return 0; |
c68e9c6b | 250 | } |
62e76326 | 251 | |
c68e9c6b | 252 | /* do not digest huge objects */ |
47f6e231 | 253 | if (e->swap_file_sz > (uint64_t )Config.Store.maxObjectSize) { |
bf8fe701 | 254 | debugs(71, 6, "storeDigestAddable: NO: too big"); |
62e76326 | 255 | return 0; |
c68e9c6b | 256 | } |
62e76326 | 257 | |
c68e9c6b | 258 | /* still here? check staleness */ |
259 | /* Note: We should use the time of the next rebuild, not (cur_time+period) */ | |
7e3ce7b9 | 260 | if (refreshCheckDigest(e, Config.digest.rebuild_period)) { |
4a7a3d56 | 261 | debugs(71, 6, "storeDigestAdd: entry expires within " << Config.digest.rebuild_period << " secs, ignoring"); |
62e76326 | 262 | return 0; |
c68e9c6b | 263 | } |
62e76326 | 264 | |
17a80fc2 | 265 | /* |
266 | * idea: how about also skipping very fresh (thus, potentially | |
267 | * unstable) entries? Should be configurable through | |
268 | * cd_refresh_pattern, of course. | |
269 | */ | |
270 | /* | |
271 | * idea: skip objects that are going to be purged before the next | |
272 | * update. | |
273 | */ | |
c68e9c6b | 274 | return 1; |
275 | } | |
276 | ||
d6fd3381 | 277 | static void |
6168bccd | 278 | storeDigestAdd(const StoreEntry * entry) |
279 | { | |
6168bccd | 280 | assert(entry && store_digest); |
c68e9c6b | 281 | |
282 | if (storeDigestAddable(entry)) { | |
5db6bf73 | 283 | ++sd_stats.add_count; |
62e76326 | 284 | |
285 | if (cacheDigestTest(store_digest, (const cache_key *)entry->key)) | |
5db6bf73 | 286 | ++sd_stats.add_coll_count; |
62e76326 | 287 | |
288 | cacheDigestAdd(store_digest, (const cache_key *)entry->key); | |
289 | ||
bf8fe701 | 290 | debugs(71, 6, "storeDigestAdd: added entry, key: " << entry->getMD5Text()); |
6168bccd | 291 | } else { |
5db6bf73 | 292 | ++sd_stats.rej_count; |
62e76326 | 293 | |
294 | if (cacheDigestTest(store_digest, (const cache_key *)entry->key)) | |
5db6bf73 | 295 | ++sd_stats.rej_coll_count; |
6168bccd | 296 | } |
12784378 | 297 | } |
298 | ||
12784378 | 299 | /* rebuilds digest from scratch */ |
300 | static void | |
6168bccd | 301 | storeDigestRebuildStart(void *datanotused) |
8638fc66 | 302 | { |
303 | assert(store_digest); | |
12784378 | 304 | /* prevent overlapping if rebuild schedule is too tight */ |
62e76326 | 305 | |
12784378 | 306 | if (sd_state.rebuild_lock) { |
e0236918 | 307 | debugs(71, DBG_IMPORTANT, "storeDigestRebuildStart: overlap detected, consider increasing rebuild period"); |
62e76326 | 308 | return; |
12784378 | 309 | } |
62e76326 | 310 | |
12784378 | 311 | sd_state.rebuild_lock = 1; |
bf8fe701 | 312 | debugs(71, 2, "storeDigestRebuildStart: rebuild #" << sd_state.rebuild_count + 1); |
62e76326 | 313 | |
6168bccd | 314 | if (sd_state.rewrite_lock) { |
bf8fe701 | 315 | debugs(71, 2, "storeDigestRebuildStart: waiting for Rewrite to finish."); |
62e76326 | 316 | return; |
6168bccd | 317 | } |
62e76326 | 318 | |
6168bccd | 319 | storeDigestRebuildResume(); |
320 | } | |
321 | ||
322 | /* called be Rewrite to push Rebuild forward */ | |
323 | static void | |
d6fd3381 | 324 | storeDigestRebuildResume(void) |
6168bccd | 325 | { |
326 | assert(sd_state.rebuild_lock); | |
327 | assert(!sd_state.rewrite_lock); | |
c8f4eac4 | 328 | sd_state.theSearch = Store::Root().search(NULL, NULL); |
304b267e | 329 | /* resize or clear */ |
62e76326 | 330 | |
304b267e | 331 | if (!storeDigestResize()) |
62e76326 | 332 | cacheDigestClear(store_digest); /* not clean()! */ |
333 | ||
6168bccd | 334 | memset(&sd_stats, 0, sizeof(sd_stats)); |
62e76326 | 335 | |
c43f5247 | 336 | eventAdd("storeDigestRebuildStep", storeDigestRebuildStep, NULL, 0.0, 1); |
12784378 | 337 | } |
338 | ||
339 | /* finishes swap out sequence for the digest; schedules next rebuild */ | |
340 | static void | |
d6fd3381 | 341 | storeDigestRebuildFinish(void) |
12784378 | 342 | { |
343 | assert(sd_state.rebuild_lock); | |
344 | sd_state.rebuild_lock = 0; | |
5db6bf73 | 345 | ++sd_state.rebuild_count; |
bf8fe701 | 346 | debugs(71, 2, "storeDigestRebuildFinish: done."); |
7e3ce7b9 | 347 | eventAdd("storeDigestRebuildStart", storeDigestRebuildStart, NULL, (double) |
62e76326 | 348 | Config.digest.rebuild_period, 1); |
6168bccd | 349 | /* resume pending Rewrite if any */ |
62e76326 | 350 | |
12784378 | 351 | if (sd_state.rewrite_lock) |
62e76326 | 352 | storeDigestRewriteResume(); |
12784378 | 353 | } |
354 | ||
355 | /* recalculate a few hash buckets per invocation; schedules next step */ | |
356 | static void | |
357 | storeDigestRebuildStep(void *datanotused) | |
358 | { | |
c8f4eac4 | 359 | /* TODO: call Store::Root().size() to determine this.. */ |
360 | int count = Config.Store.objectsPerBucket * (int) ceil((double) store_hash_buckets * | |
361 | (double) Config.digest.rebuild_chunk_percentage / 100.0); | |
12784378 | 362 | assert(sd_state.rebuild_lock); |
62e76326 | 363 | |
bf8fe701 | 364 | debugs(71, 3, "storeDigestRebuildStep: buckets: " << store_hash_buckets << " entries to check: " << count); |
62e76326 | 365 | |
c8f4eac4 | 366 | while (count-- && !sd_state.theSearch->isDone() && sd_state.theSearch->next()) |
367 | storeDigestAdd(sd_state.theSearch->currentItem()); | |
62e76326 | 368 | |
12784378 | 369 | /* are we done ? */ |
c8f4eac4 | 370 | if (sd_state.theSearch->isDone()) |
62e76326 | 371 | storeDigestRebuildFinish(); |
12784378 | 372 | else |
62e76326 | 373 | eventAdd("storeDigestRebuildStep", storeDigestRebuildStep, NULL, 0.0, 1); |
12784378 | 374 | } |
375 | ||
12784378 | 376 | /* starts swap out sequence for the digest */ |
377 | static void | |
6168bccd | 378 | storeDigestRewriteStart(void *datanotused) |
12784378 | 379 | { |
f206b652 | 380 | RequestFlags flags; |
462f66d2 | 381 | char *url; |
6168bccd | 382 | StoreEntry *e; |
12784378 | 383 | |
384 | assert(store_digest); | |
385 | /* prevent overlapping if rewrite schedule is too tight */ | |
62e76326 | 386 | |
12784378 | 387 | if (sd_state.rewrite_lock) { |
e0236918 | 388 | debugs(71, DBG_IMPORTANT, "storeDigestRewrite: overlap detected, consider increasing rewrite period"); |
62e76326 | 389 | return; |
12784378 | 390 | } |
62e76326 | 391 | |
bf8fe701 | 392 | debugs(71, 2, "storeDigestRewrite: start rewrite #" << sd_state.rewrite_count + 1); |
12784378 | 393 | /* make new store entry */ |
e13ee7ad | 394 | url = internalLocalUri("/squid-internal-periodic/", StoreDigestFileName); |
45e5102d | 395 | flags.cachable = 1; |
c2a7cefd | 396 | e = storeCreateEntry(url, url, flags, Http::METHOD_GET); |
28c60158 | 397 | assert(e); |
aa839030 | 398 | sd_state.rewrite_lock = e; |
bf8fe701 | 399 | debugs(71, 3, "storeDigestRewrite: url: " << url << " key: " << e->getMD5Text()); |
c21ad0f5 | 400 | HttpRequest *req = HttpRequest::CreateFromUrl(url); |
6dd9f4bd | 401 | e->mem_obj->request = HTTPMSGLOCK(req); |
6168bccd | 402 | /* wait for rebuild (if any) to finish */ |
62e76326 | 403 | |
6168bccd | 404 | if (sd_state.rebuild_lock) { |
bf8fe701 | 405 | debugs(71, 2, "storeDigestRewriteStart: waiting for rebuild to finish."); |
62e76326 | 406 | return; |
6168bccd | 407 | } |
62e76326 | 408 | |
6168bccd | 409 | storeDigestRewriteResume(); |
410 | } | |
411 | ||
412 | static void | |
d6fd3381 | 413 | storeDigestRewriteResume(void) |
6168bccd | 414 | { |
28c60158 | 415 | StoreEntry *e; |
6168bccd | 416 | |
417 | assert(sd_state.rewrite_lock); | |
418 | assert(!sd_state.rebuild_lock); | |
aa839030 | 419 | e = sd_state.rewrite_lock; |
12784378 | 420 | sd_state.rewrite_offset = 0; |
d46a87a8 | 421 | EBIT_SET(e->flags, ENTRY_SPECIAL); |
462f66d2 | 422 | /* setting public key will purge old digest entry if any */ |
d88e3c49 | 423 | e->setPublicKey(); |
462f66d2 | 424 | /* fake reply */ |
06a5ae20 | 425 | HttpReply *rep = new HttpReply; |
11992b6f AJ |
426 | rep->setHeaders(HTTP_OK, "Cache Digest OK", |
427 | "application/cache-digest", (store_digest->mask_size + sizeof(sd_state.cblock)), | |
428 | squid_curtime, (squid_curtime + Config.digest.rewrite_period) ); | |
26ac0430 | 429 | debugs(71, 3, "storeDigestRewrite: entry expires on " << rep->expires << |
bf8fe701 | 430 | " (" << std::showpos << (int) (rep->expires - squid_curtime) << ")"); |
3900307b | 431 | e->buffer(); |
db237875 | 432 | e->replaceHttpReply(rep); |
12784378 | 433 | storeDigestCBlockSwapOut(e); |
3900307b | 434 | e->flush(); |
aa839030 | 435 | eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, sd_state.rewrite_lock, 0.0, 1, false); |
12784378 | 436 | } |
437 | ||
438 | /* finishes swap out sequence for the digest; schedules next rewrite */ | |
439 | static void | |
b644367b | 440 | storeDigestRewriteFinish(StoreEntry * e) |
12784378 | 441 | { |
aa839030 | 442 | assert(e == sd_state.rewrite_lock); |
528b2c61 | 443 | e->complete(); |
3900307b | 444 | e->timestampsSet(); |
26ac0430 | 445 | debugs(71, 2, "storeDigestRewriteFinish: digest expires at " << e->expires << |
bf8fe701 | 446 | " (" << std::showpos << (int) (e->expires - squid_curtime) << ")"); |
6168bccd | 447 | /* is this the write order? @?@ */ |
528b2c61 | 448 | e->mem_obj->unlinkRequest(); |
97b5e68f | 449 | e->unlock(); |
feefc1d9 | 450 | sd_state.rewrite_lock = NULL; |
5db6bf73 | 451 | ++sd_state.rewrite_count; |
7e3ce7b9 | 452 | eventAdd("storeDigestRewriteStart", storeDigestRewriteStart, NULL, (double) |
62e76326 | 453 | Config.digest.rewrite_period, 1); |
6168bccd | 454 | /* resume pending Rebuild if any */ |
62e76326 | 455 | |
6168bccd | 456 | if (sd_state.rebuild_lock) |
62e76326 | 457 | storeDigestRebuildResume(); |
12784378 | 458 | } |
459 | ||
460 | /* swaps out one digest "chunk" per invocation; schedules next swap out */ | |
461 | static void | |
52040193 | 462 | storeDigestSwapOutStep(void *data) |
12784378 | 463 | { |
aa839030 | 464 | StoreEntry *e = static_cast<StoreEntry *>(data); |
7e3ce7b9 | 465 | int chunk_size = Config.digest.swapout_chunk_size; |
aa839030 | 466 | assert(e == sd_state.rewrite_lock); |
12784378 | 467 | assert(e); |
12784378 | 468 | /* _add_ check that nothing bad happened while we were waiting @?@ @?@ */ |
62e76326 | 469 | |
57d55dfa | 470 | if (sd_state.rewrite_offset + chunk_size > store_digest->mask_size) |
62e76326 | 471 | chunk_size = store_digest->mask_size - sd_state.rewrite_offset; |
472 | ||
3900307b | 473 | e->append(store_digest->mask + sd_state.rewrite_offset, chunk_size); |
62e76326 | 474 | |
e4049756 | 475 | debugs(71, 3, "storeDigestSwapOutStep: size: " << store_digest->mask_size << |
476 | " offset: " << sd_state.rewrite_offset << " chunk: " << | |
477 | chunk_size << " bytes"); | |
62e76326 | 478 | |
12784378 | 479 | sd_state.rewrite_offset += chunk_size; |
62e76326 | 480 | |
12784378 | 481 | /* are we done ? */ |
57d55dfa | 482 | if (sd_state.rewrite_offset >= store_digest->mask_size) |
62e76326 | 483 | storeDigestRewriteFinish(e); |
12784378 | 484 | else |
aa839030 | 485 | eventAdd("storeDigestSwapOutStep", storeDigestSwapOutStep, data, 0.0, 1, false); |
12784378 | 486 | } |
487 | ||
488 | static void | |
b644367b | 489 | storeDigestCBlockSwapOut(StoreEntry * e) |
12784378 | 490 | { |
12784378 | 491 | memset(&sd_state.cblock, 0, sizeof(sd_state.cblock)); |
462f66d2 | 492 | sd_state.cblock.ver.current = htons(CacheDigestVer.current); |
493 | sd_state.cblock.ver.required = htons(CacheDigestVer.required); | |
494 | sd_state.cblock.capacity = htonl(store_digest->capacity); | |
495 | sd_state.cblock.count = htonl(store_digest->count); | |
496 | sd_state.cblock.del_count = htonl(store_digest->del_count); | |
497 | sd_state.cblock.mask_size = htonl(store_digest->mask_size); | |
7e3ce7b9 | 498 | sd_state.cblock.bits_per_entry = (unsigned char) |
62e76326 | 499 | Config.digest.bits_per_entry; |
6168bccd | 500 | sd_state.cblock.hash_func_count = (unsigned char) CacheDigestHashFuncCount; |
3900307b | 501 | e->append((char *) &sd_state.cblock, sizeof(sd_state.cblock)); |
8638fc66 | 502 | } |
503 | ||
304b267e | 504 | /* calculates digest capacity */ |
505 | static int | |
d6fd3381 | 506 | storeDigestCalcCap(void) |
304b267e | 507 | { |
508 | /* | |
509 | * To-Do: Bloom proved that the optimal filter utilization is 50% (half of | |
26ac0430 | 510 | * the bits are off). However, we do not have a formula to calculate the |
304b267e | 511 | * number of _entries_ we want to pre-allocate for. |
512 | */ | |
58d5c5dd DK |
513 | const int hi_cap = Store::Root().maxSize() / Config.Store.avgObjectSize; |
514 | const int lo_cap = 1 + Store::Root().currentSize() / Config.Store.avgObjectSize; | |
528b2c61 | 515 | const int e_count = StoreEntry::inUseCount(); |
26ac0430 | 516 | int cap = e_count ? e_count :hi_cap; |
bf8fe701 | 517 | debugs(71, 2, "storeDigestCalcCap: have: " << e_count << ", want " << cap << |
518 | " entries; limits: [" << lo_cap << ", " << hi_cap << "]"); | |
62e76326 | 519 | |
304b267e | 520 | if (cap < lo_cap) |
62e76326 | 521 | cap = lo_cap; |
522 | ||
6168bccd | 523 | /* do not enforce hi_cap limit, average-based estimation may be wrong |
524 | *if (cap > hi_cap) | |
26ac0430 | 525 | * cap = hi_cap; |
6168bccd | 526 | */ |
527 | return cap; | |
304b267e | 528 | } |
529 | ||
530 | /* returns true if we actually resized the digest */ | |
531 | static int | |
d6fd3381 | 532 | storeDigestResize(void) |
304b267e | 533 | { |
534 | const int cap = storeDigestCalcCap(); | |
535 | int diff; | |
536 | assert(store_digest); | |
537 | diff = abs(cap - store_digest->capacity); | |
26ac0430 AJ |
538 | debugs(71, 2, "storeDigestResize: " << |
539 | store_digest->capacity << " -> " << cap << "; change: " << | |
bf8fe701 | 540 | diff << " (" << xpercentInt(diff, store_digest->capacity) << "%)" ); |
304b267e | 541 | /* avoid minor adjustments */ |
62e76326 | 542 | |
4b4cd312 | 543 | if (diff <= store_digest->capacity / 10) { |
bf8fe701 | 544 | debugs(71, 2, "storeDigestResize: small change, will not resize."); |
62e76326 | 545 | return 0; |
304b267e | 546 | } else { |
bf8fe701 | 547 | debugs(71, 2, "storeDigestResize: big change, resizing."); |
62e76326 | 548 | cacheDigestChangeCap(store_digest, cap); |
549 | return 1; | |
304b267e | 550 | } |
551 | } | |
552 | ||
d6fd3381 | 553 | #endif /* USE_CACHE_DIGESTS */ |