]>
Commit | Line | Data |
---|---|---|
528b2c61 | 1 | /* |
4ac4a490 | 2 | * Copyright (C) 1996-2017 The Squid Software Foundation and contributors |
528b2c61 | 3 | * |
bbc27441 AJ |
4 | * Squid software is distributed under GPLv2+ license and includes |
5 | * contributions from numerous individuals and organizations. | |
6 | * Please see the COPYING and CONTRIBUTORS files for details. | |
528b2c61 | 7 | */ |
8 | ||
bbc27441 AJ |
9 | /* DEBUG: section 19 Store Memory Primitives */ |
10 | ||
582c2af2 | 11 | #include "squid.h" |
3e4bebf8 | 12 | #include "comm/Connection.h" |
582c2af2 | 13 | #include "Generic.h" |
af69c635 | 14 | #include "globals.h" |
528b2c61 | 15 | #include "HttpReply.h" |
582c2af2 FC |
16 | #include "HttpRequest.h" |
17 | #include "MemBuf.h" | |
18 | #include "MemObject.h" | |
19 | #include "profiler/Profiler.h" | |
4d5904f7 | 20 | #include "SquidConfig.h" |
528b2c61 | 21 | #include "Store.h" |
22 | #include "StoreClient.h" | |
582c2af2 | 23 | |
9a0a18de | 24 | #if USE_DELAY_POOLS |
b67e2c8c | 25 | #include "DelayPools.h" |
26 | #endif | |
528b2c61 | 27 | |
28 | /* TODO: make this global or private */ | |
29 | #if URL_CHECKSUM_DEBUG | |
30 | static unsigned int url_checksum(const char *url); | |
31 | unsigned int | |
32 | url_checksum(const char *url) | |
33 | { | |
34 | unsigned int ck; | |
c3031d67 | 35 | SquidMD5_CTX M; |
528b2c61 | 36 | static unsigned char digest[16]; |
c3031d67 | 37 | SquidMD5Init(&M); |
38 | SquidMD5Update(&M, (unsigned char *) url, strlen(url)); | |
39 | SquidMD5Final(digest, &M); | |
41d00cd3 | 40 | memcpy(&ck, digest, sizeof(ck)); |
528b2c61 | 41 | return ck; |
42 | } | |
62e76326 | 43 | |
528b2c61 | 44 | #endif |
45 | ||
aa839030 | 46 | RemovalPolicy * mem_policy = NULL; |
47 | ||
528b2c61 | 48 | size_t |
49 | MemObject::inUseCount() | |
50 | { | |
9f9e06f3 | 51 | return Pool().inUseCount(); |
528b2c61 | 52 | } |
53 | ||
c877c0bc | 54 | const char * |
9d4e9cfb AR |
55 | MemObject::storeId() const |
56 | { | |
dcd84f80 | 57 | if (!storeId_.size()) { |
c877c0bc AR |
58 | debugs(20, DBG_IMPORTANT, "Bug: Missing MemObject::storeId value"); |
59 | dump(); | |
60 | storeId_ = "[unknown_URI]"; | |
61 | } | |
62 | return storeId_.termedBuf(); | |
9487bae9 AR |
63 | } |
64 | ||
c877c0bc | 65 | const char * |
9d4e9cfb AR |
66 | MemObject::logUri() const |
67 | { | |
dcd84f80 | 68 | return logUri_.size() ? logUri_.termedBuf() : storeId(); |
c877c0bc | 69 | } |
4a56ee8d | 70 | |
c877c0bc | 71 | bool |
9d4e9cfb AR |
72 | MemObject::hasUris() const |
73 | { | |
dcd84f80 | 74 | return storeId_.size(); |
c877c0bc | 75 | } |
4a56ee8d | 76 | |
c877c0bc AR |
77 | void |
78 | MemObject::setUris(char const *aStoreId, char const *aLogUri, const HttpRequestMethod &aMethod) | |
79 | { | |
80 | storeId_ = aStoreId; | |
81 | ||
82 | // fast pointer comparison for a common storeCreateEntry(url,url,...) case | |
9d4e9cfb | 83 | if (!aLogUri || aLogUri == aStoreId) |
c877c0bc AR |
84 | logUri_.clean(); // use storeId_ by default to minimize copying |
85 | else | |
86 | logUri_ = aLogUri; | |
62e76326 | 87 | |
c877c0bc | 88 | method = aMethod; |
4a56ee8d | 89 | |
c877c0bc AR |
90 | #if URL_CHECKSUM_DEBUG |
91 | chksum = url_checksum(urlXXX()); | |
528b2c61 | 92 | #endif |
c877c0bc | 93 | } |
62e76326 | 94 | |
cc8c4af2 AJ |
95 | MemObject::MemObject() : |
96 | inmem_lo(0), | |
97 | nclients(0), | |
98 | smpCollapsed(false), | |
d59e4742 FC |
99 | request(nullptr), |
100 | ping_reply_callback(nullptr), | |
101 | ircb_data(nullptr), | |
cc8c4af2 AJ |
102 | id(0), |
103 | object_sz(-1), | |
104 | swap_hdr_sz(0), | |
105 | #if URL_CHECKSUM_DEBUG | |
106 | chksum(0), | |
107 | #endif | |
d59e4742 | 108 | vary_headers(nullptr) |
c877c0bc | 109 | { |
cc8c4af2 AJ |
110 | debugs(20, 3, "new MemObject " << this); |
111 | memset(&start_ping, 0, sizeof(start_ping)); | |
112 | memset(&abort, 0, sizeof(abort)); | |
c877c0bc AR |
113 | _reply = new HttpReply; |
114 | HTTPMSGLOCK(_reply); | |
528b2c61 | 115 | } |
116 | ||
117 | MemObject::~MemObject() | |
118 | { | |
cc8c4af2 | 119 | debugs(20, 3, "del MemObject " << this); |
0df20c61 | 120 | const Ctx ctx = ctx_enter(hasUris() ? urlXXX() : "[unknown_ctx]"); |
62e76326 | 121 | |
c877c0bc AR |
122 | #if URL_CHECKSUM_DEBUG |
123 | checkUrlChecksum(); | |
528b2c61 | 124 | #endif |
62e76326 | 125 | |
4475555f | 126 | if (!shutting_down) { // Store::Root() is FATALly missing during shutdown |
4475555f AR |
127 | assert(xitTable.index < 0); |
128 | assert(memCache.index < 0); | |
528b2c61 | 129 | assert(swapout.sio == NULL); |
4475555f | 130 | } |
62e76326 | 131 | |
528b2c61 | 132 | data_hdr.freeContent(); |
62e76326 | 133 | |
9cdee68d | 134 | #if 0 |
528b2c61 | 135 | /* |
136 | * There is no way to abort FD-less clients, so they might | |
9cdee68d | 137 | * still have mem->clients set. |
528b2c61 | 138 | */ |
9cdee68d | 139 | assert(clients.head == NULL); |
140 | ||
141 | #endif | |
62e76326 | 142 | |
6dd9f4bd | 143 | HTTPMSGUNLOCK(_reply); |
62e76326 | 144 | |
6dd9f4bd | 145 | HTTPMSGUNLOCK(request); |
62e76326 | 146 | |
528b2c61 | 147 | ctx_exit(ctx); /* must exit before we free mem->url */ |
528b2c61 | 148 | } |
149 | ||
150 | void | |
151 | MemObject::unlinkRequest() | |
152 | { | |
6dd9f4bd | 153 | HTTPMSGUNLOCK(request); |
528b2c61 | 154 | } |
155 | ||
156 | void | |
55759ffb | 157 | MemObject::write(const StoreIOBuffer &writeBuffer) |
528b2c61 | 158 | { |
1d5161bd | 159 | PROF_start(MemObject_write); |
4a7a3d56 | 160 | debugs(19, 6, "memWrite: offset " << writeBuffer.offset << " len " << writeBuffer.length); |
528b2c61 | 161 | |
528b2c61 | 162 | /* We don't separate out mime headers yet, so ensure that the first |
26ac0430 | 163 | * write is at offset 0 - where they start |
528b2c61 | 164 | */ |
165 | assert (data_hdr.endOffset() || writeBuffer.offset == 0); | |
166 | ||
167 | assert (data_hdr.write (writeBuffer)); | |
1d5161bd | 168 | PROF_stop(MemObject_write); |
528b2c61 | 169 | } |
170 | ||
171 | void | |
172 | MemObject::dump() const | |
173 | { | |
42a503bd | 174 | data_hdr.dump(); |
528b2c61 | 175 | #if 0 |
176 | /* do we want this one? */ | |
e0236918 | 177 | debugs(20, DBG_IMPORTANT, "MemObject->data.origin_offset: " << (data_hdr.head ? data_hdr.head->nodeBuffer.offset : 0)); |
528b2c61 | 178 | #endif |
62e76326 | 179 | |
e0236918 FC |
180 | debugs(20, DBG_IMPORTANT, "MemObject->start_ping: " << start_ping.tv_sec << "."<< std::setfill('0') << std::setw(6) << start_ping.tv_usec); |
181 | debugs(20, DBG_IMPORTANT, "MemObject->inmem_hi: " << data_hdr.endOffset()); | |
182 | debugs(20, DBG_IMPORTANT, "MemObject->inmem_lo: " << inmem_lo); | |
183 | debugs(20, DBG_IMPORTANT, "MemObject->nclients: " << nclients); | |
184 | debugs(20, DBG_IMPORTANT, "MemObject->reply: " << _reply); | |
185 | debugs(20, DBG_IMPORTANT, "MemObject->request: " << request); | |
c877c0bc AR |
186 | debugs(20, DBG_IMPORTANT, "MemObject->logUri: " << logUri_); |
187 | debugs(20, DBG_IMPORTANT, "MemObject->storeId: " << storeId_); | |
528b2c61 | 188 | } |
189 | ||
190 | HttpReply const * | |
191 | MemObject::getReply() const | |
192 | { | |
193 | return _reply; | |
194 | } | |
195 | ||
4a56ee8d | 196 | void |
197 | MemObject::replaceHttpReply(HttpReply *newrep) | |
198 | { | |
6dd9f4bd | 199 | HTTPMSGUNLOCK(_reply); |
b248c2a3 AJ |
200 | _reply = newrep; |
201 | HTTPMSGLOCK(_reply); | |
4a56ee8d | 202 | } |
203 | ||
26ac0430 AJ |
204 | struct LowestMemReader : public unary_function<store_client, void> { |
205 | LowestMemReader(int64_t seed):current(seed) {} | |
62e76326 | 206 | |
26ac0430 | 207 | void operator() (store_client const &x) { |
62e76326 | 208 | if (x.memReaderHasLowerOffset(current)) |
209 | current = x.copyInto.offset; | |
210 | } | |
211 | ||
47f6e231 | 212 | int64_t current; |
528b2c61 | 213 | }; |
214 | ||
26ac0430 AJ |
215 | struct StoreClientStats : public unary_function<store_client, void> { |
216 | StoreClientStats(MemBuf *anEntry):where(anEntry),index(0) {} | |
62e76326 | 217 | |
26ac0430 | 218 | void operator()(store_client const &x) { |
aec55359 FC |
219 | x.dumpStats(where, index); |
220 | ++index; | |
528b2c61 | 221 | } |
62e76326 | 222 | |
fcc35180 | 223 | MemBuf *where; |
528b2c61 | 224 | size_t index; |
225 | }; | |
226 | ||
227 | void | |
83af6fa2 | 228 | MemObject::stat(MemBuf * mb) const |
528b2c61 | 229 | { |
4391cd15 | 230 | mb->appendf("\t" SQUIDSBUFPH " %s\n", SQUIDSBUFPRINT(method.image()), logUri()); |
90ab8f20 AJ |
231 | if (!vary_headers.isEmpty()) |
232 | mb->appendf("\tvary_headers: " SQUIDSBUFPH "\n", SQUIDSBUFPRINT(vary_headers)); | |
4391cd15 AJ |
233 | mb->appendf("\tinmem_lo: %" PRId64 "\n", inmem_lo); |
234 | mb->appendf("\tinmem_hi: %" PRId64 "\n", data_hdr.endOffset()); | |
235 | mb->appendf("\tswapout: %" PRId64 " bytes queued\n", swapout.queue_offset); | |
62e76326 | 236 | |
528b2c61 | 237 | if (swapout.sio.getRaw()) |
4391cd15 | 238 | mb->appendf("\tswapout: %" PRId64 " bytes written\n", (int64_t) swapout.sio->offset()); |
62e76326 | 239 | |
752fd8d2 | 240 | if (xitTable.index >= 0) |
4391cd15 | 241 | mb->appendf("\ttransient index: %d state: %d\n", xitTable.index, xitTable.io); |
752fd8d2 | 242 | if (memCache.index >= 0) |
4391cd15 | 243 | mb->appendf("\tmem-cache index: %d state: %d offset: %" PRId64 "\n", memCache.index, memCache.io, memCache.offset); |
752fd8d2 | 244 | if (object_sz >= 0) |
4391cd15 | 245 | mb->appendf("\tobject_sz: %" PRId64 "\n", object_sz); |
752fd8d2 | 246 | if (smpCollapsed) |
4391cd15 | 247 | mb->appendf("\tsmp-collapsed\n"); |
752fd8d2 | 248 | |
fcc35180 | 249 | StoreClientStats statsVisitor(mb); |
62e76326 | 250 | |
4cbb7fa8 | 251 | for_each<StoreClientStats>(clients, statsVisitor); |
528b2c61 | 252 | } |
253 | ||
47f6e231 | 254 | int64_t |
528b2c61 | 255 | MemObject::endOffset () const |
256 | { | |
257 | return data_hdr.endOffset(); | |
258 | } | |
259 | ||
3756e5c0 AR |
260 | void |
261 | MemObject::markEndOfReplyHeaders() | |
262 | { | |
263 | const int hdr_sz = endOffset(); | |
264 | assert(hdr_sz >= 0); | |
265 | assert(_reply); | |
266 | _reply->hdr_sz = hdr_sz; | |
267 | } | |
268 | ||
47f6e231 | 269 | int64_t |
528b2c61 | 270 | MemObject::size() const |
271 | { | |
62e76326 | 272 | if (object_sz < 0) |
273 | return endOffset(); | |
274 | ||
528b2c61 | 275 | return object_sz; |
276 | } | |
277 | ||
aa1a691e | 278 | int64_t |
9199139f AR |
279 | MemObject::expectedReplySize() const |
280 | { | |
aa1a691e AR |
281 | debugs(20, 7, HERE << "object_sz: " << object_sz); |
282 | if (object_sz >= 0) // complete() has been called; we know the exact answer | |
283 | return object_sz; | |
284 | ||
285 | if (_reply) { | |
286 | const int64_t clen = _reply->bodySize(method); | |
287 | debugs(20, 7, HERE << "clen: " << clen); | |
288 | if (clen >= 0 && _reply->hdr_sz > 0) // yuck: HttpMsg sets hdr_sz to 0 | |
289 | return clen + _reply->hdr_sz; | |
290 | } | |
291 | ||
292 | return -1; // not enough information to predict | |
293 | } | |
294 | ||
528b2c61 | 295 | void |
296 | MemObject::reset() | |
297 | { | |
298 | assert(swapout.sio == NULL); | |
299 | data_hdr.freeContent(); | |
300 | inmem_lo = 0; | |
301 | /* Should we check for clients? */ | |
302 | } | |
303 | ||
47f6e231 | 304 | int64_t |
528b2c61 | 305 | MemObject::lowestMemReaderOffset() const |
306 | { | |
307 | LowestMemReader lowest (endOffset() + 1); | |
308 | ||
4cbb7fa8 | 309 | for_each <LowestMemReader>(clients, lowest); |
62e76326 | 310 | |
528b2c61 | 311 | return lowest.current; |
312 | } | |
313 | ||
314 | /* XXX: This is wrong. It breaks *badly* on range combining */ | |
315 | bool | |
316 | MemObject::readAheadPolicyCanRead() const | |
317 | { | |
f54986ad | 318 | const bool canRead = endOffset() - getReply()->hdr_sz < |
9d4e9cfb | 319 | lowestMemReaderOffset() + Config.readAheadGap; |
f54986ad AR |
320 | |
321 | if (!canRead) { | |
322 | debugs(19, 9, "no: " << endOffset() << '-' << getReply()->hdr_sz << | |
323 | " < " << lowestMemReaderOffset() << '+' << Config.readAheadGap); | |
324 | } | |
325 | ||
326 | return canRead; | |
528b2c61 | 327 | } |
328 | ||
329 | void | |
330 | MemObject::addClient(store_client *aClient) | |
331 | { | |
332 | ++nclients; | |
333 | dlinkAdd(aClient, &aClient->node, &clients); | |
334 | } | |
335 | ||
336 | #if URL_CHECKSUM_DEBUG | |
337 | void | |
338 | MemObject::checkUrlChecksum () const | |
339 | { | |
c877c0bc | 340 | assert(chksum == url_checksum(urlXXX())); |
528b2c61 | 341 | } |
62e76326 | 342 | |
528b2c61 | 343 | #endif |
344 | ||
345 | /* | |
346 | * How much of the object data is on the disk? | |
347 | */ | |
47f6e231 | 348 | int64_t |
528b2c61 | 349 | MemObject::objectBytesOnDisk() const |
350 | { | |
351 | /* | |
352 | * NOTE: storeOffset() represents the disk file size, | |
353 | * not the amount of object data on disk. | |
26ac0430 | 354 | * |
528b2c61 | 355 | * If we don't have at least 'swap_hdr_sz' bytes |
356 | * then none of the object data is on disk. | |
357 | * | |
358 | * This should still be safe if swap_hdr_sz == 0, | |
359 | * meaning we haven't even opened the swapout file | |
360 | * yet. | |
361 | */ | |
62e76326 | 362 | |
528b2c61 | 363 | if (swapout.sio.getRaw() == NULL) |
62e76326 | 364 | return 0; |
365 | ||
47f6e231 | 366 | int64_t nwritten = swapout.sio->offset(); |
62e76326 | 367 | |
ed013b6c | 368 | if (nwritten <= (int64_t)swap_hdr_sz) |
62e76326 | 369 | return 0; |
370 | ||
47f6e231 | 371 | return (nwritten - swap_hdr_sz); |
528b2c61 | 372 | } |
373 | ||
47f6e231 | 374 | int64_t |
10aeba1d | 375 | MemObject::policyLowestOffsetToKeep(bool swap) const |
528b2c61 | 376 | { |
377 | /* | |
378 | * Careful. lowest_offset can be greater than endOffset(), such | |
379 | * as in the case of a range request. | |
380 | */ | |
47f6e231 | 381 | int64_t lowest_offset = lowestMemReaderOffset(); |
62e76326 | 382 | |
528b2c61 | 383 | if (endOffset() < lowest_offset || |
ff4b33f4 | 384 | endOffset() - inmem_lo > (int64_t)Config.Store.maxInMemObjSize || |
10aeba1d | 385 | (swap && !Config.onoff.memory_cache_first)) |
62e76326 | 386 | return lowest_offset; |
387 | ||
528b2c61 | 388 | return inmem_lo; |
389 | } | |
390 | ||
391 | void | |
392 | MemObject::trimSwappable() | |
393 | { | |
10aeba1d | 394 | int64_t new_mem_lo = policyLowestOffsetToKeep(1); |
528b2c61 | 395 | /* |
396 | * We should only free up to what we know has been written | |
397 | * to disk, not what has been queued for writing. Otherwise | |
398 | * there will be a chunk of the data which is not in memory | |
399 | * and is not yet on disk. | |
400 | * The -1 makes sure the page isn't freed until storeSwapOut has | |
aa1a691e | 401 | * walked to the next page. |
528b2c61 | 402 | */ |
47f6e231 | 403 | int64_t on_disk; |
62e76326 | 404 | |
528b2c61 | 405 | if ((on_disk = objectBytesOnDisk()) - 1 < new_mem_lo) |
62e76326 | 406 | new_mem_lo = on_disk - 1; |
407 | ||
528b2c61 | 408 | if (new_mem_lo == -1) |
f53969cc | 409 | new_mem_lo = 0; /* the above might become -1 */ |
62e76326 | 410 | |
528b2c61 | 411 | data_hdr.freeDataUpto(new_mem_lo); |
62e76326 | 412 | |
528b2c61 | 413 | inmem_lo = new_mem_lo; |
414 | } | |
415 | ||
416 | void | |
417 | MemObject::trimUnSwappable() | |
418 | { | |
99921d9d AR |
419 | if (const int64_t new_mem_lo = policyLowestOffsetToKeep(false)) { |
420 | assert (new_mem_lo > 0); | |
421 | data_hdr.freeDataUpto(new_mem_lo); | |
422 | inmem_lo = new_mem_lo; | |
423 | } // else we should not trim anything at this time | |
528b2c61 | 424 | } |
425 | ||
528b2c61 | 426 | bool |
427 | MemObject::isContiguous() const | |
428 | { | |
47f6e231 | 429 | bool result = data_hdr.hasContigousContentRange (Range<int64_t>(inmem_lo, endOffset())); |
528b2c61 | 430 | /* XXX : make this higher level */ |
bf8fe701 | 431 | debugs (19, result ? 4 :3, "MemObject::isContiguous: Returning " << (result ? "true" : "false")); |
528b2c61 | 432 | return result; |
433 | } | |
b67e2c8c | 434 | |
435 | int | |
384a7590 | 436 | MemObject::mostBytesWanted(int max, bool ignoreDelayPools) const |
b67e2c8c | 437 | { |
9a0a18de | 438 | #if USE_DELAY_POOLS |
384a7590 JP |
439 | if (!ignoreDelayPools) { |
440 | /* identify delay id with largest allowance */ | |
441 | DelayId largestAllowance = mostBytesAllowed (); | |
442 | return largestAllowance.bytesWanted(0, max); | |
443 | } | |
444 | #endif | |
62e76326 | 445 | |
b67e2c8c | 446 | return max; |
b67e2c8c | 447 | } |
448 | ||
a46d2c0e | 449 | void |
450 | MemObject::setNoDelay(bool const newValue) | |
451 | { | |
9a0a18de | 452 | #if USE_DELAY_POOLS |
a46d2c0e | 453 | |
454 | for (dlink_node *node = clients.head; node; node = node->next) { | |
455 | store_client *sc = (store_client *) node->data; | |
456 | sc->delayId.setNoDelay(newValue); | |
457 | } | |
458 | ||
459 | #endif | |
460 | } | |
461 | ||
462 | void | |
463 | MemObject::delayRead(DeferredRead const &aRead) | |
464 | { | |
465 | deferredReads.delayRead(aRead); | |
466 | } | |
467 | ||
468 | void | |
469 | MemObject::kickReads() | |
470 | { | |
471 | deferredReads.kickReads(-1); | |
472 | } | |
473 | ||
9a0a18de | 474 | #if USE_DELAY_POOLS |
b67e2c8c | 475 | DelayId |
476 | MemObject::mostBytesAllowed() const | |
477 | { | |
478 | int j; | |
479 | int jmax = -1; | |
480 | DelayId result; | |
62e76326 | 481 | |
b67e2c8c | 482 | for (dlink_node *node = clients.head; node; node = node->next) { |
62e76326 | 483 | store_client *sc = (store_client *) node->data; |
d576a6a6 | 484 | #if 0 |
62e76326 | 485 | /* This test is invalid because the client may be writing data |
486 | * and thus will want data immediately. | |
487 | * If we include the test, there is a race condition when too much | |
488 | * data is read - if all sc's are writing when a read is scheduled. | |
489 | * XXX: fixme. | |
490 | */ | |
491 | ||
492 | if (!sc->callbackPending()) | |
493 | /* not waiting for more data */ | |
494 | continue; | |
495 | ||
d576a6a6 | 496 | #endif |
62e76326 | 497 | |
62e76326 | 498 | j = sc->delayId.bytesWanted(0, sc->copyInto.length); |
499 | ||
500 | if (j > jmax) { | |
501 | jmax = j; | |
502 | result = sc->delayId; | |
503 | } | |
b67e2c8c | 504 | } |
62e76326 | 505 | |
b67e2c8c | 506 | return result; |
507 | } | |
62e76326 | 508 | |
b67e2c8c | 509 | #endif |
5b55f1f1 CT |
510 | |
511 | int64_t | |
512 | MemObject::availableForSwapOut() const | |
513 | { | |
514 | return endOffset() - swapout.queue_offset; | |
515 | } | |
f53969cc | 516 |