5 * DEBUG: section 19 Store Memory Primitives
6 * AUTHOR: Robert Collins
8 * SQUID Web Proxy Cache http://www.squid-cache.org/
9 * ----------------------------------------------------------
11 * Squid is the result of efforts by numerous individuals from
12 * the Internet community; see the CONTRIBUTORS file for full
13 * details. Many organizations have provided support for Squid's
14 * development; see the SPONSORS file for full details. Squid is
15 * Copyrighted (C) 2001 by the Regents of the University of
16 * California; see the COPYRIGHT file for full details. Squid
17 * incorporates software developed and/or copyrighted by other
18 * sources; see the CREDITS file for full details.
20 * This program is free software; you can redistribute it and/or modify
21 * it under the terms of the GNU General Public License as published by
22 * the Free Software Foundation; either version 2 of the License, or
23 * (at your option) any later version.
25 * This program is distributed in the hope that it will be useful,
26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
28 * GNU General Public License for more details.
30 * You should have received a copy of the GNU General Public License
31 * along with this program; if not, write to the Free Software
32 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
37 #include "comm/Connection.h"
38 #include "MemObject.h"
39 #include "HttpRequest.h"
40 #include "HttpReply.h"
42 #include "StoreClient.h"
45 #include "DelayPools.h"
49 /* TODO: make this global or private */
50 #if URL_CHECKSUM_DEBUG
51 static unsigned int url_checksum(const char *url
);
53 url_checksum(const char *url
)
57 static unsigned char digest
[16];
59 SquidMD5Update(&M
, (unsigned char *) url
, strlen(url
));
60 SquidMD5Final(digest
, &M
);
61 memcpy(&ck
, digest
, sizeof(ck
));
67 RemovalPolicy
* mem_policy
= NULL
;
70 MemObject::inUseCount()
72 return Pool().inUseCount();
76 MemObject::resetUrls(char const *aUrl
, char const *aLog_url
)
79 safe_free(log_url
); /* XXX account log_url */
80 log_url
= xstrdup(aLog_url
);
84 MemObject::MemObject(char const *aUrl
, char const *aLog_url
)
86 debugs(20, 3, HERE
<< "new MemObject " << this);
87 HttpReply
*rep
= new HttpReply
;
89 _reply
= HTTPMSGLOCK(rep
);
92 #if URL_CHECKSUM_DEBUG
94 chksum
= url_checksum(url
);
98 log_url
= xstrdup(aLog_url
);
102 /* XXX account log_url */
104 swapout
.decision
= SwapOut::swNeedsCheck
;
107 MemObject::~MemObject()
109 debugs(20, 3, HERE
<< "del MemObject " << this);
110 const Ctx ctx
= ctx_enter(url
);
111 #if URL_CHECKSUM_DEBUG
113 assert(chksum
== url_checksum(url
));
117 assert(swapout
.sio
== NULL
);
119 data_hdr
.freeContent();
123 * There is no way to abort FD-less clients, so they might
124 * still have mem->clients set.
126 assert(clients
.head
== NULL
);
130 HTTPMSGUNLOCK(_reply
);
132 HTTPMSGUNLOCK(request
);
134 ctx_exit(ctx
); /* must exit before we free mem->url */
138 safe_free(log_url
); /* XXX account log_url */
140 safe_free(vary_headers
);
144 MemObject::unlinkRequest()
146 HTTPMSGUNLOCK(request
);
150 MemObject::write ( StoreIOBuffer writeBuffer
, STMCB
*callback
, void *callbackData
)
152 PROF_start(MemObject_write
);
153 debugs(19, 6, "memWrite: offset " << writeBuffer
.offset
<< " len " << writeBuffer
.length
);
155 /* the offset is into the content, not the headers */
156 writeBuffer
.offset
+= (_reply
? _reply
->hdr_sz
: 0);
158 /* We don't separate out mime headers yet, so ensure that the first
159 * write is at offset 0 - where they start
161 assert (data_hdr
.endOffset() || writeBuffer
.offset
== 0);
163 assert (data_hdr
.write (writeBuffer
));
164 callback (callbackData
, writeBuffer
);
165 PROF_stop(MemObject_write
);
169 MemObject::dump() const
173 /* do we want this one? */
174 debugs(20, 1, "MemObject->data.origin_offset: " << (data_hdr
.head
? data_hdr
.head
->nodeBuffer
.offset
: 0));
177 debugs(20, 1, "MemObject->start_ping: " << start_ping
.tv_sec
<< "."<< std::setfill('0') << std::setw(6) << start_ping
.tv_usec
);
178 debugs(20, 1, "MemObject->inmem_hi: " << data_hdr
.endOffset());
179 debugs(20, 1, "MemObject->inmem_lo: " << inmem_lo
);
180 debugs(20, 1, "MemObject->nclients: " << nclients
);
181 debugs(20, 1, "MemObject->reply: " << _reply
);
182 debugs(20, 1, "MemObject->request: " << request
);
183 debugs(20, 1, "MemObject->log_url: " << log_url
<< " " << checkNullString(log_url
));
187 MemObject::getReply() const
193 MemObject::replaceHttpReply(HttpReply
*newrep
)
195 HTTPMSGUNLOCK(_reply
);
196 _reply
= HTTPMSGLOCK(newrep
);
199 struct LowestMemReader
: public unary_function
<store_client
, void> {
200 LowestMemReader(int64_t seed
):current(seed
) {}
202 void operator() (store_client
const &x
) {
203 if (x
.memReaderHasLowerOffset(current
))
204 current
= x
.copyInto
.offset
;
210 struct StoreClientStats
: public unary_function
<store_client
, void> {
211 StoreClientStats(MemBuf
*anEntry
):where(anEntry
),index(0) {}
213 void operator()(store_client
const &x
) {
214 x
.dumpStats(where
, index
++);
222 MemObject::stat(MemBuf
* mb
) const
224 mb
->Printf("\t%s %s\n",
225 RequestMethodStr(method
), log_url
);
227 mb
->Printf("\tvary_headers: %s\n", vary_headers
);
228 mb
->Printf("\tinmem_lo: %"PRId64
"\n", inmem_lo
);
229 mb
->Printf("\tinmem_hi: %"PRId64
"\n", data_hdr
.endOffset());
230 mb
->Printf("\tswapout: %"PRId64
" bytes queued\n",
231 swapout
.queue_offset
);
233 if (swapout
.sio
.getRaw())
234 mb
->Printf("\tswapout: %"PRId64
" bytes written\n",
235 (int64_t) swapout
.sio
->offset());
237 StoreClientStats
statsVisitor(mb
);
239 for_each
<StoreClientStats
>(clients
, statsVisitor
);
243 MemObject::endOffset () const
245 return data_hdr
.endOffset();
249 MemObject::markEndOfReplyHeaders()
251 const int hdr_sz
= endOffset();
254 _reply
->hdr_sz
= hdr_sz
;
258 MemObject::size() const
267 MemObject::expectedReplySize() const
269 debugs(20, 7, HERE
<< "object_sz: " << object_sz
);
270 if (object_sz
>= 0) // complete() has been called; we know the exact answer
274 const int64_t clen
= _reply
->bodySize(method
);
275 debugs(20, 7, HERE
<< "clen: " << clen
);
276 if (clen
>= 0 && _reply
->hdr_sz
> 0) // yuck: HttpMsg sets hdr_sz to 0
277 return clen
+ _reply
->hdr_sz
;
280 return -1; // not enough information to predict
286 assert(swapout
.sio
== NULL
);
287 data_hdr
.freeContent();
289 /* Should we check for clients? */
294 MemObject::lowestMemReaderOffset() const
296 LowestMemReader
lowest (endOffset() + 1);
298 for_each
<LowestMemReader
>(clients
, lowest
);
300 return lowest
.current
;
303 /* XXX: This is wrong. It breaks *badly* on range combining */
305 MemObject::readAheadPolicyCanRead() const
307 return endOffset() - getReply()->hdr_sz
< lowestMemReaderOffset() + Config
.readAheadGap
;
311 MemObject::addClient(store_client
*aClient
)
314 dlinkAdd(aClient
, &aClient
->node
, &clients
);
317 #if URL_CHECKSUM_DEBUG
319 MemObject::checkUrlChecksum () const
321 assert(chksum
== url_checksum(url
));
327 * How much of the object data is on the disk?
330 MemObject::objectBytesOnDisk() const
333 * NOTE: storeOffset() represents the disk file size,
334 * not the amount of object data on disk.
336 * If we don't have at least 'swap_hdr_sz' bytes
337 * then none of the object data is on disk.
339 * This should still be safe if swap_hdr_sz == 0,
340 * meaning we haven't even opened the swapout file
344 if (swapout
.sio
.getRaw() == NULL
)
347 int64_t nwritten
= swapout
.sio
->offset();
349 if (nwritten
<= (int64_t)swap_hdr_sz
)
352 return (nwritten
- swap_hdr_sz
);
356 MemObject::policyLowestOffsetToKeep(bool swap
) const
359 * Careful. lowest_offset can be greater than endOffset(), such
360 * as in the case of a range request.
362 int64_t lowest_offset
= lowestMemReaderOffset();
364 if (endOffset() < lowest_offset
||
365 endOffset() - inmem_lo
> (int64_t)Config
.Store
.maxInMemObjSize
||
366 (swap
&& !Config
.onoff
.memory_cache_first
))
367 return lowest_offset
;
373 MemObject::trimSwappable()
375 int64_t new_mem_lo
= policyLowestOffsetToKeep(1);
377 * We should only free up to what we know has been written
378 * to disk, not what has been queued for writing. Otherwise
379 * there will be a chunk of the data which is not in memory
380 * and is not yet on disk.
381 * The -1 makes sure the page isn't freed until storeSwapOut has
382 * walked to the next page.
386 if ((on_disk
= objectBytesOnDisk()) - 1 < new_mem_lo
)
387 new_mem_lo
= on_disk
- 1;
389 if (new_mem_lo
== -1)
390 new_mem_lo
= 0; /* the above might become -1 */
392 data_hdr
.freeDataUpto(new_mem_lo
);
394 inmem_lo
= new_mem_lo
;
398 MemObject::trimUnSwappable()
400 int64_t new_mem_lo
= policyLowestOffsetToKeep(0);
401 assert (new_mem_lo
> 0);
403 data_hdr
.freeDataUpto(new_mem_lo
);
404 inmem_lo
= new_mem_lo
;
409 MemObject::isContiguous() const
411 bool result
= data_hdr
.hasContigousContentRange (Range
<int64_t>(inmem_lo
, endOffset()));
412 /* XXX : make this higher level */
413 debugs (19, result
? 4 :3, "MemObject::isContiguous: Returning " << (result
? "true" : "false"));
418 MemObject::mostBytesWanted(int max
) const
421 /* identify delay id with largest allowance */
422 DelayId largestAllowance
= mostBytesAllowed ();
423 return largestAllowance
.bytesWanted(0, max
);
431 MemObject::setNoDelay(bool const newValue
)
435 for (dlink_node
*node
= clients
.head
; node
; node
= node
->next
) {
436 store_client
*sc
= (store_client
*) node
->data
;
437 sc
->delayId
.setNoDelay(newValue
);
444 MemObject::delayRead(DeferredRead
const &aRead
)
446 deferredReads
.delayRead(aRead
);
450 MemObject::kickReads()
452 deferredReads
.kickReads(-1);
457 MemObject::mostBytesAllowed() const
463 for (dlink_node
*node
= clients
.head
; node
; node
= node
->next
) {
464 store_client
*sc
= (store_client
*) node
->data
;
466 /* This test is invalid because the client may be writing data
467 * and thus will want data immediately.
468 * If we include the test, there is a race condition when too much
469 * data is read - if all sc's are writing when a read is scheduled.
473 if (!sc
->callbackPending())
474 /* not waiting for more data */
479 if (sc
->getType() != STORE_MEM_CLIENT
)
480 /* reading off disk */
483 j
= sc
->delayId
.bytesWanted(0, sc
->copyInto
.length
);
487 result
= sc
->delayId
;