]> git.ipfire.org Git - thirdparty/squid.git/blame - src/store.cc
projects / thirdparty / squid.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update
[thirdparty/squid.git] / src / store.cc
CommitLineData
30a4f2a8 1/*
620da955 2 * $Id: store.cc,v 1.95 1996/09/04 22:03:31 wessels Exp $
30a4f2a8 3 *
4 * DEBUG: section 20 Storeage Manager
5 * AUTHOR: Harvest Derived
6 *
7 * SQUID Internet Object Cache http://www.nlanr.net/Squid/
8 * --------------------------------------------------------
9 *
10 * Squid is the result of efforts by numerous individuals from the
11 * Internet community. Development is led by Duane Wessels of the
12 * National Laboratory for Applied Network Research and funded by
13 * the National Science Foundation.
14 *
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
19 *
20 * This program is distributed in the hope that it will be useful,
21 * but WITHOUT ANY WARRANTY; without even the implied warranty of
22 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23 * GNU General Public License for more details.
24 *
25 * You should have received a copy of the GNU General Public License
26 * along with this program; if not, write to the Free Software
27 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
28 *
29 */
c943f331 30
31/*
30a4f2a8 32 * Copyright (c) 1994, 1995. All rights reserved.
33 *
34 * The Harvest software was developed by the Internet Research Task
35 * Force Research Group on Resource Discovery (IRTF-RD):
36 *
37 * Mic Bowman of Transarc Corporation.
38 * Peter Danzig of the University of Southern California.
39 * Darren R. Hardy of the University of Colorado at Boulder.
40 * Udi Manber of the University of Arizona.
41 * Michael F. Schwartz of the University of Colorado at Boulder.
42 * Duane Wessels of the University of Colorado at Boulder.
43 *
44 * This copyright notice applies to software in the Harvest
45 * ``src/'' directory only. Users should consult the individual
46 * copyright notices in the ``components/'' subdirectories for
47 * copyright information about other software bundled with the
48 * Harvest source code distribution.
49 *
50 * TERMS OF USE
51 *
52 * The Harvest software may be used and re-distributed without
53 * charge, provided that the software origin and research team are
54 * cited in any use of the system. Most commonly this is
55 * accomplished by including a link to the Harvest Home Page
56 * (http://harvest.cs.colorado.edu/) from the query page of any
57 * Broker you deploy, as well as in the query result pages. These
58 * links are generated automatically by the standard Broker
59 * software distribution.
60 *
61 * The Harvest software is provided ``as is'', without express or
62 * implied warranty, and with no support nor obligation to assist
63 * in its use, correction, modification or enhancement. We assume
64 * no liability with respect to the infringement of copyrights,
65 * trade secrets, or any patents, and are not responsible for
66 * consequential damages. Proper use of the Harvest software is
67 * entirely the responsibility of the user.
68 *
69 * DERIVATIVE WORKS
70 *
71 * Users may make derivative works from the Harvest software, subject
72 * to the following constraints:
73 *
74 * - You must include the above copyright notice and these
75 * accompanying paragraphs in all forms of derivative works,
76 * and any documentation and other materials related to such
77 * distribution and use acknowledge that the software was
78 * developed at the above institutions.
79 *
80 * - You must notify IRTF-RD regarding your distribution of
81 * the derivative work.
82 *
83 * - You must clearly notify users that your are distributing
84 * a modified version and not the original Harvest software.
85 *
86 * - Any derivative product is also subject to these copyright
87 * and use restrictions.
88 *
89 * Note that the Harvest software is NOT in the public domain. We
90 * retain copyright, as specified above.
91 *
92 * HISTORY OF FREE SOFTWARE STATUS
93 *
94 * Originally we required sites to license the software in cases
95 * where they were going to build commercial products/services
96 * around Harvest. In June 1995 we changed this policy. We now
97 * allow people to use the core Harvest software (the code found in
98 * the Harvest ``src/'' directory) for free. We made this change
99 * in the interest of encouraging the widest possible deployment of
100 * the technology. The Harvest software is really a reference
101 * implementation of a set of protocols and formats, some of which
102 * we intend to standardize. We encourage commercial
103 * re-implementations of code complying to this set of standards.
c943f331 104 */
090089c4 105
106/*
107 * Here is a summary of the routines which change mem_status and swap_status:
108 * Added 11/18/95
109 *
110 * Routine mem_status swap_status status
111 * ---------------------------------------------------------------------------
6eb42cae 112 * storeCreateEntry NOT_IN_MEMORY NO_SWAP
090089c4 113 * storeComplete IN_MEMORY NO_SWAP
114 * storeSwapOutStart SWAPPING_OUT
115 * storeSwapOutHandle(fail) NO_SWAP
116 * storeSwapOutHandle(ok) SWAP_OK
117 * ---------------------------------------------------------------------------
118 * storeAddDiskRestore NOT_IN_MEMORY SWAP_OK
119 * storeSwapInStart SWAPPING_IN
120 * storeSwapInHandle(fail) NOT_IN_MEMORY
121 * storeSwapInHandle(ok) IN_MEMORY
122 * ---------------------------------------------------------------------------
123 * storeAbort IN_MEMORY NO_SWAP
124 * storePurgeMem NOT_IN_MEMORY
125 * ---------------------------------------------------------------------------
126 * You can reclaim an object's space if it's:
127 * storeGetSwapSpace !SWAPPING_IN !SWAPPING_OUT !STORE_PENDING
128 *
129 */
130
44a47c6e 131#include "squid.h" /* goes first */
090089c4 132
133#define REBUILD_TIMESTAMP_DELTA_MAX 2
134#define MAX_SWAP_FILE (1<<21)
135#define SWAP_BUF DISK_PAGE_SIZE
090089c4 136
227fbb74 137#define WITH_MEMOBJ 1
138#define WITHOUT_MEMOBJ 0
139
090089c4 140/* rate of checking expired objects in main loop */
30a4f2a8 141#define STORE_MAINTAIN_RATE (10)
090089c4 142
143#define STORE_BUCKETS (7921)
fc07a0e4 144#define STORE_IN_MEM_BUCKETS (143)
090089c4 145
2285407f 146#define STORE_LOG_CREATE 0
147#define STORE_LOG_SWAPIN 1
148#define STORE_LOG_SWAPOUT 2
149#define STORE_LOG_RELEASE 3
150
147d3115 151static char *storeLogTags[] =
152{
153 "CREATE",
154 "SWAPIN",
155 "SWAPOUT",
156 "RELEASE"
2285407f 157};
158
e62d2dea 159char *memStatusStr[] =
160{
9dfb6c1c 161 "NOT_IN_MEMORY",
162 "SWAPPING_IN",
163 "IN_MEMORY"
164};
165
e62d2dea 166char *pingStatusStr[] =
167{
9dfb6c1c 168 "PING_WAITING",
169 "PING_TIMEOUT",
170 "PING_DONE",
171 "PING_NONE"
172};
173
e62d2dea 174char *storeStatusStr[] =
175{
9dfb6c1c 176 "STORE_OK",
177 "STORE_PENDING",
178 "STORE_ABORTED"
179};
180
e62d2dea 181char *swapStatusStr[] =
182{
9dfb6c1c 183 "NO_SWAP",
184 "SWAPPING_OUT",
185 "SWAP_OK"
186};
187
3facc31e 188struct storeRebuild_data {
189 FILE *log;
190 int objcount; /* # objects successfully reloaded */
191 int expcount; /* # objects expired */
192 int linecount; /* # lines parsed from cache logfile */
193 int clashcount; /* # swapfile clashes avoided */
194 int dupcount; /* # duplicates purged */
195 time_t start, stop;
30a4f2a8 196 int speed; /* # Objects per run */
3facc31e 197 char line_in[4096];
198};
199
30a4f2a8 200/* initializtion flag */
201int store_rebuilding = STORE_REBUILDING_SLOW;
202
203/* Static Functions */
204static int storeSwapInStart _PARAMS((StoreEntry *, SIH, void *));
205static void destroy_MemObject _PARAMS((MemObject *));
206static void destroy_MemObjectData _PARAMS((MemObject *));
207static void destroy_StoreEntry _PARAMS((StoreEntry *));
208static MemObject *new_MemObject _PARAMS((void));
209static mem_ptr new_MemObjectData _PARAMS((void));
210static StoreEntry *new_StoreEntry _PARAMS((int mem_obj_flag));
211static int storeCheckPurgeMem _PARAMS((StoreEntry * e));
620da955 212static int storeCheckExpired _PARAMS((StoreEntry * e));
f5469bce 213static void storeSwapLog _PARAMS((StoreEntry *));
f7a5493b 214static int storeHashDelete _PARAMS((StoreEntry *));
a1e47288 215static char *storeDescribeStatus _PARAMS((StoreEntry *));
6e40f263 216static int compareLastRef _PARAMS((StoreEntry ** e1, StoreEntry ** e2));
217static int compareSize _PARAMS((StoreEntry ** e1, StoreEntry ** e2));
218static int storeClientListSearch _PARAMS((MemObject *, int fd));
b32508fb 219static void storeHashMemInsert _PARAMS((StoreEntry *));
220static void storeHashMemDelete _PARAMS((StoreEntry *));
30a4f2a8 221
090089c4 222/* Now, this table is inaccessible to outsider. They have to use a method
223 * to access a value in internal storage data structure. */
ec74cf5f 224HashID store_table = 0;
090089c4 225/* hash table for in-memory-only objects */
226HashID in_mem_table = 0;
227
090089c4 228/* current memory storage size */
229static unsigned long store_mem_size = 0;
230static unsigned long store_mem_high = 0;
231static unsigned long store_mem_low = 0;
232
233/* current hotvm object */
234/* defaults for 16M cache and 12.5 cache_hot_vm_factor */
235static int store_hotobj_high = 180;
236static int store_hotobj_low = 120;
237
238
239/* current file name, swap file, use number as a filename */
240static unsigned long swapfileno = 0;
241static int store_swap_size = 0; /* kilobytes !! */
242static unsigned long store_swap_high = 0;
243static unsigned long store_swap_low = 0;
2285407f 244static int swaplog_fd = -1;
245static int swaplog_lock = 0;
2285407f 246static int storelog_fd = -1;
090089c4 247
090089c4 248/* key temp buffer */
ec74cf5f 249static char key_temp_buffer[MAX_URL + 100];
c943f331 250static char swaplog_file[MAX_FILE_NAME_LEN];
3facc31e 251static char tmp_filename[MAX_FILE_NAME_LEN];
090089c4 252
253/* patch cache_dir to accomodate multiple disk storage */
254dynamic_array *cache_dirs = NULL;
255int ncache_dirs = 0;
256
227fbb74 257static MemObject *new_MemObject()
258{
30a4f2a8 259 MemObject *mem = get_free_mem_obj();
260 mem->reply = xcalloc(1, sizeof(struct _http_reply));
227fbb74 261 meta_data.store_in_mem_objects++;
30a4f2a8 262 meta_data.misc += sizeof(struct _http_reply);
263 debug(20, 3, "new_MemObject: returning %p\n", mem);
264 return mem;
227fbb74 265}
266
267static StoreEntry *new_StoreEntry(mem_obj_flag)
fc07a0e4 268 int mem_obj_flag;
090089c4 269{
270 StoreEntry *e = NULL;
271
30a4f2a8 272 e = xcalloc(1, sizeof(StoreEntry));
227fbb74 273 meta_data.store_entries++;
274 if (mem_obj_flag)
275 e->mem_obj = new_MemObject();
276 debug(20, 3, "new_StoreEntry: returning %p\n", e);
277 return e;
090089c4 278}
279
30a4f2a8 280static void destroy_MemObject(mem)
281 MemObject *mem;
090089c4 282{
30a4f2a8 283 int i;
284 debug(20, 3, "destroy_MemObject: destroying %p\n", mem);
285 destroy_MemObjectData(mem);
286 safe_free(mem->pending);
287 if (mem->client_list) {
6e40f263 288 for (i = 0; i < mem->client_list_size; i++)
289 safe_free(mem->client_list[i]);
30a4f2a8 290 safe_free(mem->client_list);
291 }
292 safe_free(mem->mime_hdr);
293 safe_free(mem->reply);
294 safe_free(mem->e_abort_msg);
295 requestUnlink(mem->request);
296 mem->request = NULL;
1061b406 297 memset(mem, '\0', sizeof(MemObject));
30a4f2a8 298 put_free_mem_obj(mem);
227fbb74 299 meta_data.store_in_mem_objects--;
30a4f2a8 300 meta_data.misc -= sizeof(struct _http_reply);
090089c4 301}
302
227fbb74 303static void destroy_StoreEntry(e)
090089c4 304 StoreEntry *e;
305{
227fbb74 306 debug(20, 3, "destroy_StoreEntry: destroying %p\n", e);
307 if (!e)
6eb42cae 308 fatal_dump("destroy_StoreEntry: NULL Entry");
227fbb74 309 if (e->mem_obj)
310 destroy_MemObject(e->mem_obj);
30a4f2a8 311 if (e->url) {
312 meta_data.url_strings -= strlen(e->url);
313 safe_free(e->url);
314 } else {
315 debug(20, 3, "destroy_StoreEntry: WARNING! Entry without URL string!\n");
316 }
317 if (BIT_TEST(e->flag, KEY_URL))
318 e->key = NULL;
319 else
320 safe_free(e->key);
1061b406 321 memset(e, '\0', sizeof(StoreEntry));
227fbb74 322 xfree(e);
323 meta_data.store_entries--;
090089c4 324}
325
227fbb74 326static mem_ptr new_MemObjectData()
327{
328 debug(20, 3, "new_MemObjectData: calling memInit()\n");
b32508fb 329 /* meta_data.hot_vm++; */
227fbb74 330 return memInit();
331}
090089c4 332
30a4f2a8 333static void destroy_MemObjectData(mem)
334 MemObject *mem;
090089c4 335{
30a4f2a8 336 debug(20, 3, "destroy_MemObjectData: destroying %p\n", mem->data);
337 store_mem_size -= mem->e_current_len - mem->e_lowest_offset;
227fbb74 338 debug(20, 8, "destroy_MemObjectData: Freeing %d in-memory bytes\n",
30a4f2a8 339 mem->e_current_len);
227fbb74 340 debug(20, 8, "destroy_MemObjectData: store_mem_size = %d\n",
341 store_mem_size);
30a4f2a8 342 if (mem->data) {
343 mem->data->mem_free(mem->data);
344 mem->data = NULL;
b32508fb 345 /* meta_data.hot_vm--; */
090089c4 346 }
30a4f2a8 347 mem->e_current_len = 0;
090089c4 348}
349
090089c4 350/* ----- INTERFACE BETWEEN STORAGE MANAGER AND HASH TABLE FUNCTIONS --------- */
351
352/*
353 * Create 2 hash tables, "table" has all objects, "in_mem_table" has only
354 * objects in the memory.
355 */
356
357HashID storeCreateHashTable(cmp_func)
358 int (*cmp_func) (char *, char *);
359{
ec74cf5f 360 store_table = hash_create(cmp_func, STORE_BUCKETS, hash_url);
361 in_mem_table = hash_create(cmp_func, STORE_IN_MEM_BUCKETS, hash_url);
362 return store_table;
090089c4 363}
364
b32508fb 365static void storeHashMemInsert(e)
366 StoreEntry *e;
367{
368 hash_insert(in_mem_table, e->key, e);
369 meta_data.hot_vm++;
370}
371
372static void storeHashMemDelete(e)
373 StoreEntry *e;
374{
375 hash_link *hptr = hash_lookup(in_mem_table, e->key);
376 if (hptr == NULL)
377 fatal_dump("storeHashMemDelete: key not found");
378 hash_delete_link(in_mem_table, hptr);
379 meta_data.hot_vm--;
380}
090089c4 381
6602e70e 382static int storeHashInsert(e)
090089c4 383 StoreEntry *e;
384{
227fbb74 385 debug(20, 3, "storeHashInsert: Inserting Entry %p key '%s'\n",
386 e, e->key);
090089c4 387 if (e->mem_status == IN_MEMORY)
b32508fb 388 storeHashMemInsert(e);
389 return hash_join(store_table, (hash_link *) e);
090089c4 390}
391
f7a5493b 392static int storeHashDelete(e)
b6f794d6 393 StoreEntry *e;
090089c4 394{
b32508fb 395 if (e->mem_status == IN_MEMORY)
396 storeHashMemDelete(e);
397 return hash_remove_link(store_table, (hash_link *) e);
090089c4 398}
399
400/*
401 * maintain the in-mem hash table according to the changes of mem_status
234967c9 402 * This routine replaces the instruction "e->store_status = status;"
090089c4 403 */
090089c4 404void storeSetMemStatus(e, status)
405 StoreEntry *e;
30a4f2a8 406 mem_status_t status;
090089c4 407{
6eb42cae 408 if (e->key == NULL)
409 fatal_dump("storeSetMemStatus: NULL key");
b32508fb 410 if (status != IN_MEMORY && e->mem_status == IN_MEMORY)
411 storeHashMemDelete(e);
412 else if (status == IN_MEMORY && e->mem_status != IN_MEMORY)
413 storeHashMemInsert(e);
090089c4 414 e->mem_status = status;
415}
416
417/* -------------------------------------------------------------------------- */
418
147d3115 419static char *time_describe(t)
420 time_t t;
2285407f 421{
95d659f0 422 LOCAL_ARRAY(char, buf, 128);
2285407f 423
147d3115 424 if (t < 60) {
425 sprintf(buf, "%ds", (int) t);
426 } else if (t < 3600) {
427 sprintf(buf, "%dm", (int) t / 60);
428 } else if (t < 86400) {
429 sprintf(buf, "%dh", (int) t / 3600);
430 } else if (t < 604800) {
431 sprintf(buf, "%dD", (int) t / 86400);
432 } else if (t < 2592000) {
433 sprintf(buf, "%dW", (int) t / 604800);
434 } else if (t < 31536000) {
435 sprintf(buf, "%dM", (int) t / 2592000);
436 } else {
437 sprintf(buf, "%dY", (int) t / 31536000);
438 }
439 return buf;
2285407f 440}
441
442static void storeLog(tag, e)
443 int tag;
444 StoreEntry *e;
445{
0be3ec60 446 LOCAL_ARRAY(char, logmsg, MAX_URL << 1);
2285407f 447 time_t t;
448 int expect_len = 0;
449 int actual_len = 0;
450 int code = 0;
2285407f 451 if (storelog_fd < 0)
452 return;
b8de7ebe 453 t = e->expires - squid_curtime;
2285407f 454 if (e->mem_obj) {
455 code = e->mem_obj->reply->code;
456 expect_len = (int) e->mem_obj->reply->content_length;
457 actual_len = (int) e->mem_obj->e_current_len - e->mem_obj->reply->hdr_sz;
458 }
459 sprintf(logmsg, "%9d.%03d %-7s %4d %9d [%3s] %d/%d %s\n",
460 (int) current_time.tv_sec,
461 (int) current_time.tv_usec / 1000,
462 storeLogTags[tag],
463 code,
464 (int) t,
465 time_describe(t),
466 expect_len,
467 actual_len,
468 e->key);
469 file_write(storelog_fd,
470 xstrdup(logmsg),
471 strlen(logmsg),
472 0,
473 NULL,
e47afcdb 474 NULL,
475 xfree);
2285407f 476}
477
090089c4 478
479/* get rid of memory copy of the object */
620da955 480/* Only call this if storeCheckPurgeMem(e) returns 1 */
090089c4 481void storePurgeMem(e)
482 StoreEntry *e;
483{
2285407f 484 debug(20, 3, "storePurgeMem: Freeing memory-copy of %s\n", e->key);
2aca8433 485 if (e->mem_obj == NULL)
090089c4 486 return;
090089c4 487 storeSetMemStatus(e, NOT_IN_MEMORY);
227fbb74 488 destroy_MemObject(e->mem_obj);
489 e->mem_obj = NULL;
090089c4 490}
491
492/* lock the object for reading, start swapping in if necessary */
30a4f2a8 493/* Called by:
620da955 494 * icpProcessRequest()
30a4f2a8 495 * storeAbort()
496 * {http,ftp,gopher,wais}Start()
497 */
498int storeLockObject(e, handler, data)
090089c4 499 StoreEntry *e;
30a4f2a8 500 SIH handler;
501 void *data;
090089c4 502{
090089c4 503 int status = 0;
090089c4 504 e->lock_count++;
148a2921 505 debug(20, 3, "storeLockObject: key '%s' count=%d\n",
506 e->key, (int) e->lock_count);
a1e47288 507 if (e->mem_status != NOT_IN_MEMORY)
508 /* ok, its either IN_MEMORY or SWAPPING_IN */
6e40f263 509 debug(20, 5, "storeLockObject: OK: mem_status is %s\n", memStatusStr[e->mem_status]);
a1e47288 510 else if (e->swap_status == SWAP_OK)
511 /* ok, its NOT_IN_MEMORY, but its swapped out */
6e40f263 512 debug(20, 5, "storeLockObject: OK: swap_status is %s\n", swapStatusStr[e->swap_status]);
a1e47288 513 else if (e->store_status == STORE_PENDING)
514 /* ok, we're reading it in right now */
6e40f263 515 debug(20, 5, "storeLockObject: OK: store_status is %s\n", storeStatusStr[e->store_status]);
516 else
a1e47288 517 fatal_dump(storeDescribeStatus(e));
b8de7ebe 518 e->lastref = squid_curtime;
30a4f2a8 519 /* If the object is NOT_IN_MEMORY, fault it in. */
090089c4 520 if ((e->mem_status == NOT_IN_MEMORY) && (e->swap_status == SWAP_OK)) {
521 /* object is in disk and no swapping daemon running. Bring it in. */
e6f18b4a 522 if ((status = storeSwapInStart(e, handler, data)) < 0) {
523 /* We couldn't find or couldn't open object's swapfile.
090089c4 524 * So, return a -1 here, indicating that we will treat
525 * the reference like a MISS_TTL, force a keychange and
526 storeRelease. */
527 e->lock_count--;
528 }
30a4f2a8 529 } else if (e->mem_status == IN_MEMORY && handler) {
530 /* its already in memory, so call the handler */
e6f18b4a 531 handler(0, data);
30a4f2a8 532 } else if (handler) {
533 /* The object is probably in state SWAPPING_IN, not much we can do.
534 * Instead of returning failure here, we should have a list of complete
535 * handlers which we could append to... */
e6f18b4a 536 handler(1, data);
090089c4 537 }
538 return status;
539}
540
2daae136 541void storeReleaseRequest(e)
147d3115 542 StoreEntry *e;
2285407f 543{
58bcd31b 544 if (e->flag & RELEASE_REQUEST)
545 return;
30a4f2a8 546 if (!storeEntryLocked(e))
547 fatal_dump("Somebody called storeReleaseRequest on an unlocked entry");
2daae136 548 debug(20, 3, "storeReleaseRequest: FOR '%s'\n", e->key ? e->key : e->url);
d3c1a245 549 e->flag |= RELEASE_REQUEST;
2285407f 550}
551
090089c4 552/* unlock object, return -1 if object get released after unlock
553 * otherwise lock_count */
090089c4 554int storeUnlockObject(e)
555 StoreEntry *e;
556{
30a4f2a8 557 int lock_count;
6602e70e 558
090089c4 559 if ((int) e->lock_count > 0)
560 e->lock_count--;
561 else if (e->lock_count == 0) {
c943f331 562 debug(20, 0, "Entry lock count %d is out-of-whack\n", e->lock_count);
090089c4 563 }
983061ed 564 debug(20, 3, "storeUnlockObject: key '%s' count=%d\n", e->key, e->lock_count);
565
30a4f2a8 566 if (e->lock_count)
a1e47288 567 return (int) e->lock_count;
30a4f2a8 568
090089c4 569 /* Prevent UMR if we end up freeing the entry */
30a4f2a8 570 lock_count = (int) e->lock_count;
090089c4 571
30a4f2a8 572 if (e->flag & RELEASE_REQUEST) {
573 storeRelease(e);
574 } else if (e->flag & ABORT_MSG_PENDING) {
575 /* This is where the negative cache gets storeAppended */
576 /* Briefly lock to replace content with abort message */
577 e->lock_count++;
578 destroy_MemObjectData(e->mem_obj);
579 e->object_len = 0;
580 e->mem_obj->data = new_MemObjectData();
581 storeAppend(e, e->mem_obj->e_abort_msg, strlen(e->mem_obj->e_abort_msg));
582 e->object_len = e->mem_obj->e_current_len
583 = strlen(e->mem_obj->e_abort_msg);
584 BIT_RESET(e->flag, ABORT_MSG_PENDING);
585 e->lock_count--;
586 } else if (storeCheckPurgeMem(e)) {
587 storePurgeMem(e);
090089c4 588 }
30a4f2a8 589 return lock_count;
090089c4 590}
591
592/* Lookup an object in the cache.
593 * return just a reference to object, don't start swapping in yet. */
594StoreEntry *storeGet(url)
595 char *url;
596{
597 hash_link *hptr = NULL;
598
6602e70e 599 debug(20, 3, "storeGet: looking up %s\n", url);
090089c4 600
ec74cf5f 601 if ((hptr = hash_lookup(store_table, url)) != NULL)
227fbb74 602 return (StoreEntry *) hptr;
090089c4 603 return NULL;
604}
605
04e8dbaa 606unsigned int getKeyCounter()
607{
2d25dc1a 608 static unsigned int key_counter = 0;
609 if (++key_counter == 0)
610 ++key_counter;
611 return key_counter;
04e8dbaa 612}
613
73ddce89 614char *storeGeneratePrivateKey(url, method, num)
6eb42cae 615 char *url;
30a4f2a8 616 method_t method;
6eb42cae 617 int num;
618{
6eb42cae 619 if (num == 0)
04e8dbaa 620 num = getKeyCounter();
6eb42cae 621 debug(20, 3, "storeGeneratePrivateKey: '%s'\n", url);
622 key_temp_buffer[0] = '\0';
623 sprintf(key_temp_buffer, "%d/%s/%s",
624 num,
73ddce89 625 RequestMethodStr[method],
6eb42cae 626 url);
627 return key_temp_buffer;
628}
629
73ddce89 630char *storeGeneratePublicKey(url, method)
090089c4 631 char *url;
30a4f2a8 632 method_t method;
090089c4 633{
73ddce89 634 debug(20, 3, "storeGeneratePublicKey: type=%d %s\n", method, url);
635 switch (method) {
6eb42cae 636 case METHOD_GET:
227fbb74 637 return url;
983061ed 638 /* NOTREACHED */
227fbb74 639 break;
6eb42cae 640 case METHOD_POST:
090089c4 641 sprintf(key_temp_buffer, "/post/%s", url);
642 return key_temp_buffer;
983061ed 643 /* NOTREACHED */
227fbb74 644 break;
30a4f2a8 645 case METHOD_PUT:
646 sprintf(key_temp_buffer, "/put/%s", url);
647 return key_temp_buffer;
648 /* NOTREACHED */
649 break;
6eb42cae 650 case METHOD_HEAD:
090089c4 651 sprintf(key_temp_buffer, "/head/%s", url);
652 return key_temp_buffer;
983061ed 653 /* NOTREACHED */
227fbb74 654 break;
3facc31e 655 case METHOD_CONNECT:
656 sprintf(key_temp_buffer, "/connect/%s", url);
657 return key_temp_buffer;
983061ed 658 /* NOTREACHED */
3facc31e 659 break;
227fbb74 660 default:
6eb42cae 661 fatal_dump("storeGeneratePublicKey: Unsupported request method");
227fbb74 662 break;
663 }
664 return NULL;
665}
666
6eb42cae 667void storeSetPrivateKey(e)
668 StoreEntry *e;
227fbb74 669{
6eb42cae 670 StoreEntry *e2 = NULL;
671 hash_link *table_entry = NULL;
07622ccd 672 char *newkey = NULL;
227fbb74 673
6eb42cae 674 if (e->key && BIT_TEST(e->flag, KEY_PRIVATE))
675 return; /* is already private */
090089c4 676
73ddce89 677 newkey = storeGeneratePrivateKey(e->url, e->method, 0);
ec74cf5f 678 if ((table_entry = hash_lookup(store_table, newkey))) {
07622ccd 679 e2 = (StoreEntry *) table_entry;
680 debug(20, 0, "storeSetPrivateKey: Entry already exists with key '%s'\n",
681 newkey);
682 debug(20, 0, "storeSetPrivateKey: Entry Dump:\n%s\n", storeToString(e2));
683 fatal_dump("Private key already exists.");
227fbb74 684 }
07622ccd 685 if (e->key)
edae7bc0 686 storeHashDelete(e);
07622ccd 687 if (e->key && !BIT_TEST(e->flag, KEY_URL))
688 safe_free(e->key);
689 e->key = xstrdup(newkey);
6eb42cae 690 storeHashInsert(e);
227fbb74 691 BIT_RESET(e->flag, KEY_URL);
6eb42cae 692 BIT_SET(e->flag, KEY_CHANGE);
693 BIT_SET(e->flag, KEY_PRIVATE);
227fbb74 694}
695
6eb42cae 696void storeSetPublicKey(e)
227fbb74 697 StoreEntry *e;
698{
6eb42cae 699 StoreEntry *e2 = NULL;
700 hash_link *table_entry = NULL;
07622ccd 701 char *newkey = NULL;
234967c9 702 int loop_detect = 0;
6eb42cae 703
704 if (e->key && !BIT_TEST(e->flag, KEY_PRIVATE))
705 return; /* is already public */
706
73ddce89 707 newkey = storeGeneratePublicKey(e->url, e->method);
ec74cf5f 708 while ((table_entry = hash_lookup(store_table, newkey))) {
30a4f2a8 709 debug(20, 3, "storeSetPublicKey: Making old '%s' private.\n", newkey);
07622ccd 710 e2 = (StoreEntry *) table_entry;
711 storeSetPrivateKey(e2);
30a4f2a8 712 storeRelease(e2);
234967c9 713 if (loop_detect++ == 10)
714 fatal_dump("storeSetPublicKey() is looping!!");
715 newkey = storeGeneratePublicKey(e->url, e->method);
6eb42cae 716 }
07622ccd 717 if (e->key)
edae7bc0 718 storeHashDelete(e);
07622ccd 719 if (e->key && !BIT_TEST(e->flag, KEY_URL))
720 safe_free(e->key);
73ddce89 721 if (e->method == METHOD_GET) {
6eb42cae 722 e->key = e->url;
723 BIT_SET(e->flag, KEY_URL);
724 BIT_RESET(e->flag, KEY_CHANGE);
725 } else {
07622ccd 726 e->key = xstrdup(newkey);
6eb42cae 727 BIT_RESET(e->flag, KEY_URL);
728 BIT_SET(e->flag, KEY_CHANGE);
227fbb74 729 }
07622ccd 730 BIT_RESET(e->flag, KEY_PRIVATE);
227fbb74 731 storeHashInsert(e);
732}
733
6eb42cae 734StoreEntry *storeCreateEntry(url, req_hdr, flags, method)
090089c4 735 char *url;
6eb42cae 736 char *req_hdr;
737 int flags;
30a4f2a8 738 method_t method;
090089c4 739{
090089c4 740 StoreEntry *e = NULL;
30a4f2a8 741 MemObject *mem = NULL;
6602e70e 742 debug(20, 3, "storeCreateEntry: '%s' icp flags=%x\n", url, flags);
090089c4 743
6eb42cae 744 e = new_StoreEntry(WITH_MEMOBJ);
30a4f2a8 745 e->lock_count = 1; /* Note lock here w/o calling storeLock() */
746 mem = e->mem_obj;
090089c4 747 e->url = xstrdup(url);
6eb42cae 748 meta_data.url_strings += strlen(url);
73ddce89 749 e->method = method;
6eb42cae 750 if (req_hdr)
30a4f2a8 751 mem->mime_hdr = xstrdup(req_hdr);
6eb42cae 752 if (BIT_TEST(flags, REQ_NOCACHE))
753 BIT_SET(e->flag, REFRESH_REQUEST);
234967c9 754 if (BIT_TEST(flags, REQ_CACHABLE)) {
1c481e00 755 BIT_SET(e->flag, ENTRY_CACHABLE);
090089c4 756 BIT_RESET(e->flag, RELEASE_REQUEST);
757 } else {
1c481e00 758 BIT_RESET(e->flag, ENTRY_CACHABLE);
2daae136 759 storeReleaseRequest(e);
090089c4 760 }
234967c9 761 if (BIT_TEST(flags, REQ_HIERARCHICAL))
762 BIT_SET(e->flag, HIERARCHICAL);
763 else
764 BIT_RESET(e->flag, HIERARCHICAL);
765 if (neighbors_do_private_keys || !BIT_TEST(flags, REQ_HIERARCHICAL))
6eb42cae 766 storeSetPrivateKey(e);
090089c4 767 else
6eb42cae 768 storeSetPublicKey(e);
769 if (BIT_TEST(flags, REQ_HTML))
770 BIT_SET(e->flag, ENTRY_HTML);
090089c4 771
234967c9 772 e->store_status = STORE_PENDING;
090089c4 773 storeSetMemStatus(e, NOT_IN_MEMORY);
774 e->swap_status = NO_SWAP;
775 e->swap_file_number = -1;
30a4f2a8 776 mem->data = new_MemObjectData();
090089c4 777 e->refcount = 0;
b8de7ebe 778 e->lastref = squid_curtime;
090089c4 779 e->timestamp = 0; /* set in storeSwapOutHandle() */
30a4f2a8 780 e->ping_status = PING_NONE;
227fbb74 781
090089c4 782 /* allocate pending list */
30a4f2a8 783 mem->pending_list_size = MIN_PENDING;
784 mem->pending = (struct pentry **)
785 xcalloc(mem->pending_list_size, sizeof(struct pentry *));
090089c4 786
787 /* allocate client list */
30a4f2a8 788 mem->client_list_size = MIN_CLIENT;
6e40f263 789 mem->client_list =
30a4f2a8 790 xcalloc(mem->client_list_size, sizeof(ClientStatusEntry *));
2285407f 791 /* storeLog(STORE_LOG_CREATE, e); */
090089c4 792 return e;
793}
794
795/* Add a new object to the cache with empty memory copy and pointer to disk
796 * use to rebuild store from disk. */
f5469bce 797StoreEntry *storeAddDiskRestore(url, file_number, size, expires, timestamp, lastmod)
090089c4 798 char *url;
799 int file_number;
800 int size;
801 time_t expires;
802 time_t timestamp;
f5469bce 803 time_t lastmod;
090089c4 804{
805 StoreEntry *e = NULL;
806
c943f331 807 debug(20, 5, "StoreAddDiskRestore: <URL:%s>: size %d: expires %d: file_number %d\n",
090089c4 808 url, size, expires, file_number);
809
3facc31e 810 /* if you call this you'd better be sure file_number is not
811 * already in use! */
66a91d2d 812
090089c4 813 meta_data.url_strings += strlen(url);
814
227fbb74 815 e = new_StoreEntry(WITHOUT_MEMOBJ);
090089c4 816 e->url = xstrdup(url);
73ddce89 817 e->method = METHOD_GET;
6eb42cae 818 storeSetPublicKey(e);
1c481e00 819 BIT_SET(e->flag, ENTRY_CACHABLE);
090089c4 820 BIT_RESET(e->flag, RELEASE_REQUEST);
6eb42cae 821 BIT_SET(e->flag, ENTRY_HTML);
234967c9 822 e->store_status = STORE_OK;
090089c4 823 storeSetMemStatus(e, NOT_IN_MEMORY);
824 e->swap_status = SWAP_OK;
825 e->swap_file_number = file_number;
826 file_map_bit_set(file_number);
827 e->object_len = size;
828 e->lock_count = 0;
090089c4 829 BIT_RESET(e->flag, CLIENT_ABORT_REQUEST);
830 e->refcount = 0;
b8de7ebe 831 e->lastref = squid_curtime;
f5469bce 832 e->timestamp = timestamp;
833 e->expires = expires;
834 e->lastmod = lastmod;
30a4f2a8 835 e->ping_status = PING_NONE;
090089c4 836 return e;
837}
838
839/* Register interest in an object currently being retrieved. */
840int storeRegister(e, fd, handler, data)
841 StoreEntry *e;
842 int fd;
843 PIF handler;
2318883b 844 void *data;
090089c4 845{
73ddce89 846 PendingEntry *pe = NULL;
847 int old_size;
848 int i;
9dfb6c1c 849 int j;
850 MemObject *mem = e->mem_obj;
090089c4 851
2285407f 852 debug(20, 3, "storeRegister: FD %d '%s'\n", fd, e->key);
090089c4 853
30a4f2a8 854 pe = xcalloc(1, sizeof(PendingEntry));
090089c4 855 pe->fd = fd;
856 pe->handler = handler;
857 pe->data = data;
858
859 /*
860 * I've rewritten all this pendings stuff so that num_pending goes
861 * away, and to fix all of the 'array bounds' problems we were having.
862 * It's now a very simple array, with any NULL slot empty/avail.
863 * If something needs to be added and there are no empty slots,
864 * it'll grow the array.
865 */
866 /* find an empty slot */
9dfb6c1c 867 for (i = 0; i < (int) mem->pending_list_size; i++) {
868 if (mem->pending[i] == NULL)
090089c4 869 break;
9dfb6c1c 870 }
090089c4 871
9dfb6c1c 872 if (i == mem->pending_list_size) {
090089c4 873 /* grow the array */
874 struct pentry **tmp = NULL;
875
9dfb6c1c 876 old_size = mem->pending_list_size;
090089c4 877
878 /* set list_size to an appropriate amount */
9dfb6c1c 879 mem->pending_list_size += MIN_PENDING;
090089c4 880
881 /* allocate, and copy old pending list over to the new one */
9dfb6c1c 882 tmp = xcalloc(mem->pending_list_size, sizeof(struct pentry *));
e62d2dea 883 for (j = 0; j < old_size; j++)
884 tmp[j] = mem->pending[j];
090089c4 885
886 /* free the old list and set the new one */
9dfb6c1c 887 safe_free(mem->pending);
888 mem->pending = tmp;
090089c4 889
9dfb6c1c 890 debug(20, 9, "storeRegister: grew pending list to %d for slot %d.\n",
891 mem->pending_list_size, i);
090089c4 892 }
9dfb6c1c 893 mem->pending[i] = pe;
090089c4 894 return 0;
895}
896
897/* remove handler assoicate to that fd from store pending list */
898/* Also remove entry from client_list if exist. */
899/* return number of successfully free pending entries */
900int storeUnregister(e, fd)
901 StoreEntry *e;
902 int fd;
903{
904 int i;
905 int freed = 0;
6e40f263 906 MemObject *mem = e->mem_obj;
9dfb6c1c 907 debug(20, 9, "storeUnregister: called for FD %d '%s'\n", fd, e->key);
090089c4 908 /* look for entry in client_list */
6e40f263 909 if ((i = storeClientListSearch(mem, fd)) > -1) {
910 safe_free(mem->client_list[i]);
911 mem->client_list[i] = NULL;
090089c4 912 }
913 /* walk the entire list looking for matched fd */
6e40f263 914 for (i = 0; i < (int) mem->pending_list_size; i++) {
915 if (mem->pending[i] == NULL)
916 continue;
917 if (mem->pending[i]->fd != fd)
918 continue;
919 /* found the match fd */
920 safe_free(mem->pending[i]);
921 mem->pending[i] = NULL;
922 freed++;
090089c4 923 }
148a2921 924 if (mem->fd_of_first_client == fd)
925 mem->fd_of_first_client = -1;
9dfb6c1c 926 debug(20, 9, "storeUnregister: returning %d\n", freed);
090089c4 927 return freed;
928}
929
234967c9 930int storeGetLowestReaderOffset(entry)
931 StoreEntry *entry;
932{
30a4f2a8 933 MemObject *mem = entry->mem_obj;
934 int lowest = mem->e_current_len;
234967c9 935 int i;
30a4f2a8 936 for (i = 0; i < mem->client_list_size; i++) {
937 if (mem->client_list[i] == NULL)
234967c9 938 continue;
30a4f2a8 939 if (mem->client_list[i]->last_offset < lowest)
940 lowest = mem->client_list[i]->last_offset;
234967c9 941 }
942 return lowest;
943}
944
090089c4 945/* Call to delete behind upto "target lowest offset"
234967c9 946 * also, update e_lowest_offset */
090089c4 947void storeDeleteBehind(e)
948 StoreEntry *e;
949{
30a4f2a8 950 MemObject *mem = e->mem_obj;
090089c4 951 int free_up_to;
952 int target_offset;
090089c4 953
c943f331 954 debug(20, 3, "storeDeleteBehind: Object: %s\n", e->key);
234967c9 955 debug(20, 3, "storeDeleteBehind: Original Lowest Offset: %d\n",
30a4f2a8 956 mem->e_lowest_offset);
090089c4 957
30a4f2a8 958 free_up_to = mem->e_lowest_offset;
234967c9 959 target_offset = storeGetLowestReaderOffset(e);
090089c4 960
234967c9 961 debug(20, 3, "storeDeleteBehind: target offset: %d\n", target_offset);
090089c4 962 if (target_offset) {
30a4f2a8 963 free_up_to = (int) mem->data->mem_free_data_upto(mem->data, target_offset);
234967c9 964 debug(20, 3, "--> Object is freed upto : %d\n", free_up_to);
30a4f2a8 965 store_mem_size -= free_up_to - mem->e_lowest_offset;
090089c4 966 }
234967c9 967 debug(20, 3, "storeDeleteBehind: New lowest offset: %d\n", free_up_to);
30a4f2a8 968 mem->e_lowest_offset = free_up_to;
090089c4 969}
970
971/* Call handlers waiting for data to be appended to E. */
972static void InvokeHandlers(e)
973 StoreEntry *e;
974{
975 int i;
9dfb6c1c 976 MemObject *mem = e->mem_obj;
090089c4 977
978 /* walk the entire list looking for valid handlers */
9dfb6c1c 979 for (i = 0; i < (int) mem->pending_list_size; i++) {
980 if (mem->pending[i] && mem->pending[i]->handler) {
090089c4 981 /*
982 * Once we call the handler, it is no longer needed
983 * until the write process sends all available data
984 * from the object entry.
985 */
9dfb6c1c 986 (mem->pending[i]->handler)
987 (mem->pending[i]->fd, e, mem->pending[i]->data);
988 safe_free(mem->pending[i]);
989 mem->pending[i] = NULL;
090089c4 990 }
991 }
992
993}
994
6eb42cae 995/* Mark object as expired */
9174e204 996void storeExpireNow(e)
997 StoreEntry *e;
998{
2285407f 999 debug(20, 3, "storeExpireNow: '%s'\n", e->key);
b8de7ebe 1000 e->expires = squid_curtime;
9174e204 1001}
1002
6eb42cae 1003/* switch object to deleting behind mode call by
1004 * retrieval module when object gets too big. */
090089c4 1005void storeStartDeleteBehind(e)
1006 StoreEntry *e;
1007{
c943f331 1008 debug(20, 2, "storeStartDeleteBehind: Object: %s\n", e->key);
3bbba102 1009 if (e->flag & DELETE_BEHIND)
090089c4 1010 return;
3bbba102 1011 debug(20, 2, "storeStartDeleteBehind:is now in delete behind mode.\n");
1012 /* change its key, so it can't be found by another client */
6eb42cae 1013 storeSetPrivateKey(e);
090089c4 1014 BIT_SET(e->flag, DELETE_BEHIND);
2daae136 1015 storeReleaseRequest(e);
1c481e00 1016 BIT_RESET(e->flag, ENTRY_CACHABLE);
9174e204 1017 storeExpireNow(e);
090089c4 1018}
1019
1020/* Append incoming data from a primary server to an entry. */
2aca8433 1021void storeAppend(e, data, len)
090089c4 1022 StoreEntry *e;
1023 char *data;
1024 int len;
1025{
1026 /* validity check -- sometimes it's called with bogus values */
2aca8433 1027 if (e == NULL)
1028 fatal_dump("storeAppend: NULL entry.");
1029 if (e->mem_obj == NULL)
1030 fatal_dump("storeAppend: NULL entry->mem_obj");
1031 if (e->mem_obj->data == NULL)
1032 fatal_dump("storeAppend: NULL entry->mem_obj->data");
1033
090089c4 1034 if (len) {
2285407f 1035 debug(20, 5, "storeAppend: appending %d bytes for '%s'\n", len, e->key);
090089c4 1036 (void) storeGetMemSpace(len, 0);
1037 store_mem_size += len;
e47afcdb 1038 (void) e->mem_obj->data->mem_append(e->mem_obj->data, data, len);
22e4fa85 1039 e->mem_obj->e_current_len += len;
090089c4 1040 }
e47afcdb 1041 if (e->store_status != STORE_ABORTED && !(e->flag & DELAY_SENDING))
090089c4 1042 InvokeHandlers(e);
090089c4 1043}
1044
c30c5a73 1045#if defined(__STRICT_ANSI__)
15c05bb0 1046void storeAppendPrintf(StoreEntry * e, char *fmt,...)
c30c5a73 1047{
15c05bb0 1048 va_list args;
95d659f0 1049 LOCAL_ARRAY(char, buf, 4096);
15c05bb0 1050 va_start(args, fmt);
c30c5a73 1051#else
1052void storeAppendPrintf(va_alist)
15c05bb0 1053 va_dcl
1054{
1055 va_list args;
1056 StoreEntry *e = NULL;
1057 char *fmt = NULL;
95d659f0 1058 LOCAL_ARRAY(char, buf, 4096);
15c05bb0 1059 va_start(args);
1060 e = va_arg(args, StoreEntry *);
1061 fmt = va_arg(args, char *);
c30c5a73 1062#endif
15c05bb0 1063 buf[0] = '\0';
1064 vsprintf(buf, fmt, args);
1065 storeAppend(e, buf, strlen(buf));
1066 va_end(args);
c30c5a73 1067}
1068
090089c4 1069/* add directory to swap disk */
1070int storeAddSwapDisk(path)
1071 char *path;
1072{
1073 if (cache_dirs == NULL)
1074 cache_dirs = create_dynamic_array(5, 5);
0ffd22bc 1075 /* XXX note xstrdup here prob means we
d68adf86 1076 * can't use destroy_dynamic_array() */
0ffd22bc 1077 insert_dynamic_array(cache_dirs, xstrdup(path));
090089c4 1078 return ++ncache_dirs;
1079}
1080
1081/* return the nth swap directory */
1082char *swappath(n)
1083 int n;
1084{
f7a5493b 1085 return (char *) cache_dirs->collection[n % ncache_dirs];
090089c4 1086}
1087
1088
1089/* return full name to swapfile */
1090char *storeSwapFullPath(fn, fullpath)
1091 int fn;
1092 char *fullpath;
1093{
95d659f0 1094 LOCAL_ARRAY(char, fullfilename, MAX_FILE_NAME_LEN);
f5469bce 1095 if (!fullpath)
1096 fullpath = fullfilename;
1097 fullpath[0] = '\0';
1098 sprintf(fullpath, "%s/%02X/%02X/%08X",
090089c4 1099 swappath(fn),
f5469bce 1100 (fn / ncache_dirs) % SWAP_DIRECTORIES_L1,
1101 (fn / ncache_dirs) / SWAP_DIRECTORIES_L1 % SWAP_DIRECTORIES_L2,
090089c4 1102 fn);
f5469bce 1103 return fullpath;
090089c4 1104}
1105
1106/* swapping in handle */
1107int storeSwapInHandle(fd_notused, buf, len, flag, e, offset_notused)
1108 int fd_notused;
1109 char *buf;
1110 int len;
1111 int flag;
1112 StoreEntry *e;
1113 int offset_notused;
1114{
30a4f2a8 1115 MemObject *mem = e->mem_obj;
1c481e00 1116 SIH handler = NULL;
1117 void *data = NULL;
2285407f 1118 debug(20, 2, "storeSwapInHandle: '%s'\n", e->key);
090089c4 1119
6e40f263 1120 if (mem == NULL) /* XXX remove later */
1121 fatal_dump(NULL);
1122
090089c4 1123 if ((flag < 0) && (flag != DISK_EOF)) {
c943f331 1124 debug(20, 0, "storeSwapInHandle: SwapIn failure (err code = %d).\n", flag);
30a4f2a8 1125 put_free_8k_page(mem->e_swap_buf);
090089c4 1126 storeSetMemStatus(e, NOT_IN_MEMORY);
e47afcdb 1127 file_close(mem->swapin_fd);
090089c4 1128 swapInError(-1, e); /* Invokes storeAbort() and completes the I/O */
1c481e00 1129 if ((handler = mem->swapin_complete_handler)) {
1130 data = mem->swapin_complete_data;
30a4f2a8 1131 mem->swapin_complete_handler = NULL;
1132 mem->swapin_complete_data = NULL;
1c481e00 1133 handler(2, data);
30a4f2a8 1134 }
090089c4 1135 return -1;
1136 }
30a4f2a8 1137 debug(20, 5, "storeSwapInHandle: e->swap_offset = %d\n", mem->swap_offset);
1138 debug(20, 5, "storeSwapInHandle: len = %d\n", len);
1139 debug(20, 5, "storeSwapInHandle: e->e_current_len = %d\n", mem->e_current_len);
1140 debug(20, 5, "storeSwapInHandle: e->object_len = %d\n", e->object_len);
090089c4 1141
6e40f263 1142 if (len < 0) /* XXX remove later */
1143 fatal_dump(NULL);
1144
090089c4 1145 /* always call these, even if len == 0 */
30a4f2a8 1146 mem->swap_offset += len;
090089c4 1147 storeAppend(e, buf, len);
1148
30a4f2a8 1149 if (mem->e_current_len < e->object_len && flag != DISK_EOF) {
090089c4 1150 /* some more data to swap in, reschedule */
e47afcdb 1151 file_read(mem->swapin_fd,
30a4f2a8 1152 mem->e_swap_buf,
090089c4 1153 SWAP_BUF,
30a4f2a8 1154 mem->swap_offset,
090089c4 1155 (FILE_READ_HD) storeSwapInHandle,
51496678 1156 (void *) e);
090089c4 1157 } else {
1158 /* complete swapping in */
1159 storeSetMemStatus(e, IN_MEMORY);
30a4f2a8 1160 put_free_8k_page(mem->e_swap_buf);
e47afcdb 1161 file_close(mem->swapin_fd);
2285407f 1162 storeLog(STORE_LOG_SWAPIN, e);
c943f331 1163 debug(20, 5, "storeSwapInHandle: SwapIn complete: <URL:%s> from %s.\n",
090089c4 1164 e->url, storeSwapFullPath(e->swap_file_number, NULL));
30a4f2a8 1165 if (mem->e_current_len != e->object_len) {
c943f331 1166 debug(20, 0, "storeSwapInHandle: WARNING! Object size mismatch.\n");
1167 debug(20, 0, " --> <URL:%s>\n", e->url);
1168 debug(20, 0, " --> Expecting %d bytes from file: %s\n", e->object_len,
090089c4 1169 storeSwapFullPath(e->swap_file_number, NULL));
c943f331 1170 debug(20, 0, " --> Only read %d bytes\n",
30a4f2a8 1171 mem->e_current_len);
1172 }
1c481e00 1173 if ((handler = mem->swapin_complete_handler)) {
1174 data = mem->swapin_complete_data;
30a4f2a8 1175 mem->swapin_complete_handler = NULL;
1176 mem->swapin_complete_data = NULL;
1c481e00 1177 handler(0, data);
090089c4 1178 }
1179 if (e->flag & RELEASE_REQUEST)
1180 storeRelease(e);
1181 }
1182 return 0;
1183}
1184
1185/* start swapping in */
30a4f2a8 1186static int storeSwapInStart(e, swapin_complete_handler, swapin_complete_data)
090089c4 1187 StoreEntry *e;
30a4f2a8 1188 SIH swapin_complete_handler;
1189 void *swapin_complete_data;
090089c4 1190{
1191 int fd;
fc07a0e4 1192 char *path = NULL;
e47afcdb 1193 MemObject *mem = NULL;
090089c4 1194
1195 /* sanity check! */
6e40f263 1196 if (e->swap_status != SWAP_OK)
1197 fatal_dump("storeSwapInStart: bad swap_status");
1198 if (e->swap_file_number < 0)
1199 fatal_dump("storeSwapInStart: bad swap_file_number");
1200 if (e->mem_obj)
1201 fatal_dump("storeSwapInStart: mem_obj already present");
1202
e47afcdb 1203 e->mem_obj = mem = new_MemObject();
090089c4 1204
fc07a0e4 1205 path = storeSwapFullPath(e->swap_file_number, NULL);
0ffd22bc 1206 if ((fd = file_open(path, NULL, O_RDONLY)) < 0) {
30a4f2a8 1207 debug(20, 0, "storeSwapInStart: Failed for '%s'\n", e->url);
090089c4 1208 storeSetMemStatus(e, NOT_IN_MEMORY);
fc07a0e4 1209 /* Invoke a store abort that should free the memory object */
090089c4 1210 return -1;
1211 }
e47afcdb 1212 mem->swapin_fd = (short) fd;
c943f331 1213 debug(20, 5, "storeSwapInStart: initialized swap file '%s' for <URL:%s>\n",
fc07a0e4 1214 path, e->url);
090089c4 1215 storeSetMemStatus(e, SWAPPING_IN);
e47afcdb 1216 mem->data = new_MemObjectData();
1217 mem->swap_offset = 0;
1218 mem->e_swap_buf = get_free_8k_page();
090089c4 1219
1220 /* start swapping daemon */
fc07a0e4 1221 file_read(fd,
e47afcdb 1222 mem->e_swap_buf,
090089c4 1223 SWAP_BUF,
e47afcdb 1224 mem->swap_offset,
090089c4 1225 (FILE_READ_HD) storeSwapInHandle,
51496678 1226 (void *) e);
e47afcdb 1227 mem->swapin_complete_handler = swapin_complete_handler;
1228 mem->swapin_complete_data = swapin_complete_data;
090089c4 1229 return 0;
1230}
1231
f5469bce 1232static void storeSwapLog(e)
1233 StoreEntry *e;
1234{
0be3ec60 1235 LOCAL_ARRAY(char, logmsg, MAX_URL << 1);
f5469bce 1236 /* Note this printf format appears in storeWriteCleanLog() too */
1237 sprintf(logmsg, "%08x %08x %08x %08x %9d %s\n",
1238 (int) e->swap_file_number,
1239 (int) e->timestamp,
1240 (int) e->expires,
1241 (int) e->lastmod,
1242 e->object_len,
1243 e->url);
1244 file_write(swaplog_fd,
1245 xstrdup(logmsg),
1246 strlen(logmsg),
1247 swaplog_lock,
1248 NULL,
1249 NULL,
1250 xfree);
1251}
1252
090089c4 1253void storeSwapOutHandle(fd, flag, e)
1254 int fd;
1255 int flag;
1256 StoreEntry *e;
1257{
95d659f0 1258 LOCAL_ARRAY(char, filename, MAX_FILE_NAME_LEN);
e47afcdb 1259 MemObject *mem = e->mem_obj;
090089c4 1260
2285407f 1261 debug(20, 3, "storeSwapOutHandle: '%s'\n", e->key);
3e98df20 1262 if (mem == NULL) {
a1e47288 1263 debug(20, 0, "HELP! Someone is swapping out a bad entry:\n");
1264 debug(20, 0, "%s\n", storeToString(e));
1265 fatal_dump(NULL);
3e98df20 1266 return;
1267 }
b8de7ebe 1268 e->timestamp = squid_curtime;
090089c4 1269 storeSwapFullPath(e->swap_file_number, filename);
090089c4 1270
1271 if (flag < 0) {
c943f331 1272 debug(20, 1, "storeSwapOutHandle: SwapOut failure (err code = %d).\n",
090089c4 1273 flag);
1274 e->swap_status = NO_SWAP;
e47afcdb 1275 put_free_8k_page(mem->e_swap_buf);
090089c4 1276 file_close(fd);
30a4f2a8 1277 storeRelease(e);
090089c4 1278 if (e->swap_file_number != -1) {
1279 file_map_bit_reset(e->swap_file_number);
1280 safeunlink(filename, 0); /* remove it */
1281 e->swap_file_number = -1;
1282 }
1283 if (flag == DISK_NO_SPACE_LEFT) {
1284 /* reduce the swap_size limit to the current size. */
1285 setCacheSwapMax(store_swap_size);
b6f794d6 1286 store_swap_high = (long) (((float) Config.Swap.maxSize *
1287 (float) Config.Swap.highWaterMark) / (float) 100);
1288 store_swap_low = (long) (((float) Config.Swap.maxSize *
1289 (float) Config.Swap.lowWaterMark) / (float) 100);
090089c4 1290 }
1291 return;
1292 }
3e98df20 1293 debug(20, 6, "storeSwapOutHandle: e->swap_offset = %d\n", mem->swap_offset);
1294 debug(20, 6, "storeSwapOutHandle: e->e_swap_buf_len = %d\n", mem->e_swap_buf_len);
1295 debug(20, 6, "storeSwapOutHandle: e->object_len = %d\n", e->object_len);
1296 debug(20, 6, "storeSwapOutHandle: store_swap_size = %dk\n", store_swap_size);
090089c4 1297
e47afcdb 1298 mem->swap_offset += mem->e_swap_buf_len;
090089c4 1299 /* round up */
e47afcdb 1300 store_swap_size += ((mem->e_swap_buf_len + 1023) >> 10);
1301 if (mem->swap_offset >= e->object_len) {
090089c4 1302 /* swapping complete */
1303 e->swap_status = SWAP_OK;
e47afcdb 1304 file_close(mem->swapout_fd);
2285407f 1305 storeLog(STORE_LOG_SWAPOUT, e);
c943f331 1306 debug(20, 5, "storeSwapOutHandle: SwapOut complete: <URL:%s> to %s.\n",
090089c4 1307 e->url, storeSwapFullPath(e->swap_file_number, NULL));
e47afcdb 1308 put_free_8k_page(mem->e_swap_buf);
f5469bce 1309 storeSwapLog(e);
30a4f2a8 1310 CacheInfo->proto_newobject(CacheInfo,
e47afcdb 1311 mem->request->protocol,
30a4f2a8 1312 e->object_len,
1313 FALSE);
090089c4 1314 /* check if it's request to be released. */
1315 if (e->flag & RELEASE_REQUEST)
1316 storeRelease(e);
30a4f2a8 1317 else if (storeCheckPurgeMem(e))
1318 storePurgeMem(e);
090089c4 1319 return;
1320 }
1321 /* write some more data, reschedule itself. */
3facc31e 1322 storeCopy(e,
e47afcdb 1323 mem->swap_offset,
3facc31e 1324 SWAP_BUF,
e47afcdb 1325 mem->e_swap_buf,
1326 &(mem->e_swap_buf_len));
1327 file_write(mem->swapout_fd,
1328 mem->e_swap_buf,
1329 mem->e_swap_buf_len,
1330 mem->e_swap_access,
3facc31e 1331 storeSwapOutHandle,
e47afcdb 1332 e,
1333 NULL);
090089c4 1334 return;
090089c4 1335}
1336
1337
1338/* start swapping object to disk */
07622ccd 1339static int storeSwapOutStart(e)
090089c4 1340 StoreEntry *e;
1341{
1342 int fd;
e47afcdb 1343 int x;
6e40f263 1344 LOCAL_ARRAY(char, swapfilename, MAX_FILE_NAME_LEN);
e47afcdb 1345 MemObject *mem = e->mem_obj;
090089c4 1346 /* Suggest a new swap file number */
1347 swapfileno = (swapfileno + 1) % (MAX_SWAP_FILE);
1348 /* Record the number returned */
1349 swapfileno = file_map_allocate(swapfileno);
1350 storeSwapFullPath(swapfileno, swapfilename);
e47afcdb 1351 fd = file_open(swapfilename, NULL, O_WRONLY | O_CREAT | O_TRUNC);
090089c4 1352 if (fd < 0) {
c943f331 1353 debug(20, 0, "storeSwapOutStart: Unable to open swapfile: %s\n",
090089c4 1354 swapfilename);
1355 file_map_bit_reset(swapfileno);
1356 e->swap_file_number = -1;
1357 return -1;
1358 }
e47afcdb 1359 mem->swapout_fd = (short) fd;
c943f331 1360 debug(20, 5, "storeSwapOutStart: Begin SwapOut <URL:%s> to FD %d FILE %s.\n",
090089c4 1361 e->url, fd, swapfilename);
090089c4 1362 e->swap_file_number = swapfileno;
e47afcdb 1363 if ((mem->e_swap_access = file_write_lock(mem->swapout_fd)) < 0) {
c943f331 1364 debug(20, 0, "storeSwapOutStart: Unable to lock swapfile: %s\n",
090089c4 1365 swapfilename);
1366 file_map_bit_reset(e->swap_file_number);
1367 e->swap_file_number = -1;
1368 return -1;
1369 }
1370 e->swap_status = SWAPPING_OUT;
e47afcdb 1371 mem->swap_offset = 0;
1372 mem->e_swap_buf = get_free_8k_page();
1373 mem->e_swap_buf_len = 0;
1374 storeCopy(e,
1375 0,
1376 SWAP_BUF,
1377 mem->e_swap_buf,
a1e47288 1378 &mem->e_swap_buf_len);
090089c4 1379 /* start swapping daemon */
e47afcdb 1380 x = file_write(mem->swapout_fd,
1381 mem->e_swap_buf,
1382 mem->e_swap_buf_len,
1383 mem->e_swap_access,
1384 storeSwapOutHandle,
1385 e,
1386 NULL);
1387 if (x != DISK_OK)
1388 fatal_dump(NULL); /* This shouldn't happen */
090089c4 1389 return 0;
1390}
1391
1392/* recreate meta data from disk image in swap directory */
3facc31e 1393
1394/* Add one swap file at a time from disk storage */
1395static int storeDoRebuildFromDisk(data)
1396 struct storeRebuild_data *data;
1397{
95d659f0 1398 LOCAL_ARRAY(char, swapfile, MAXPATHLEN);
1399 LOCAL_ARRAY(char, url, MAX_URL + 1);
090089c4 1400 StoreEntry *e = NULL;
090089c4 1401 time_t expires;
1402 time_t timestamp;
f5469bce 1403 time_t lastmod;
1404 int scan1;
1405 int scan2;
1406 int scan3;
1407 int scan4;
3facc31e 1408 struct stat sb;
1409 off_t size;
090089c4 1410 int delta;
090089c4 1411 int sfileno = 0;
30a4f2a8 1412 int count;
f5469bce 1413 int x;
30a4f2a8 1414
1415 /* load a number of objects per invocation */
1416 for (count = 0; count < data->speed; count++) {
1417 if (!fgets(data->line_in, 4095, data->log))
e47afcdb 1418 return !diskWriteIsComplete(swaplog_fd); /* We are done */
30a4f2a8 1419
1420 if ((++data->linecount & 0xFFF) == 0)
1421 debug(20, 1, " %7d Lines read so far.\n", data->linecount);
1422
f5469bce 1423 debug(20, 9, "line_in: %s", data->line_in);
30a4f2a8 1424 if ((data->line_in[0] == '\0') || (data->line_in[0] == '\n') ||
1425 (data->line_in[0] == '#'))
1426 continue; /* skip bad lines */
1427
f5469bce 1428 url[0] = '\0';
1429 swapfile[0] = '\0';
1430 sfileno = 0;
1431 scan1 = 0;
1432 scan2 = 0;
30a4f2a8 1433 scan3 = 0;
f5469bce 1434 scan4 = 0;
1435 x = sscanf(data->line_in, "%x %x %x %x %d %s",
0be3ec60 1436 &sfileno, /* swap_file_number */
1437 &scan1, /* timestamp */
1438 &scan2, /* expires */
1439 &scan3, /* last modified */
1440 &scan4, /* size */
1441 url); /* url */
f5469bce 1442 if (x > 0)
0be3ec60 1443 storeSwapFullPath(sfileno, swapfile);
f5469bce 1444 if (x != 6) {
1445 if (opt_unlink_on_reload && swapfile[0])
1446 safeunlink(swapfile, 0);
30a4f2a8 1447 continue;
1448 }
f5469bce 1449 timestamp = (time_t) scan1;
1450 expires = (time_t) scan2;
1451 lastmod = (time_t) scan3;
1452 size = (off_t) scan4;
30a4f2a8 1453
1454 if (store_rebuilding != STORE_REBUILDING_FAST) {
1455 if (stat(swapfile, &sb) < 0) {
1456 if (expires < squid_curtime) {
1457 debug(20, 3, "storeRebuildFromDisk: Expired: <URL:%s>\n", url);
1458 if (opt_unlink_on_reload)
1459 safeunlink(swapfile, 1);
1460 data->expcount++;
1461 } else {
1462 debug(20, 3, "storeRebuildFromDisk: Swap file missing: <URL:%s>: %s: %s.\n", url, swapfile, xstrerror());
1463 if (opt_unlink_on_reload)
f5469bce 1464 safeunlink(swapfile, 1);
30a4f2a8 1465 }
1466 continue;
1467 }
1468 /* Empty swap file? */
1469 if (sb.st_size == 0) {
30731487 1470 if (opt_unlink_on_reload)
f5469bce 1471 safeunlink(swapfile, 1);
30a4f2a8 1472 continue;
090089c4 1473 }
30a4f2a8 1474 /* timestamp might be a little bigger than sb.st_mtime */
1475 delta = (int) (timestamp - sb.st_mtime);
1476 if (delta > REBUILD_TIMESTAMP_DELTA_MAX || delta < 0) {
1477 /* this log entry doesn't correspond to this file */
1478 data->clashcount++;
1479 continue;
1480 }
1481 /* Wrong size? */
1482 if (sb.st_size != size) {
1483 /* this log entry doesn't correspond to this file */
1484 data->clashcount++;
1485 continue;
1486 }
1487 timestamp = sb.st_mtime;
9dfb6c1c 1488 debug(20, 9, "storeRebuildFromDisk: swap file exists: <URL:%s>: %s\n",
30a4f2a8 1489 url, swapfile);
3facc31e 1490 }
30a4f2a8 1491 if ((e = storeGet(url))) {
1492 if (e->timestamp > timestamp) {
1493 /* already have a newer object in memory, throw old one away */
1494 debug(20, 3, "storeRebuildFromDisk: Replaced: %s\n", url);
1495 if (opt_unlink_on_reload)
1496 safeunlink(swapfile, 1);
1497 data->dupcount++;
1498 continue;
1499 }
1500 debug(20, 6, "storeRebuildFromDisk: Duplicate: <URL:%s>\n", url);
1501 storeRelease(e);
1502 data->objcount--;
1503 data->dupcount++;
090089c4 1504 }
30a4f2a8 1505 if (expires < squid_curtime) {
1506 debug(20, 3, "storeRebuildFromDisk: Expired: <URL:%s>\n", url);
30731487 1507 if (opt_unlink_on_reload)
d68adf86 1508 safeunlink(swapfile, 1);
30a4f2a8 1509 data->expcount++;
1510 continue;
090089c4 1511 }
30a4f2a8 1512 /* Is the swap file number already taken? */
1513 if (file_map_bit_test(sfileno)) {
1514 /* Yes it is, we can't use this swapfile */
1515 debug(20, 2, "storeRebuildFromDisk: Line %d Active clash: file #%d\n",
1516 data->linecount,
1517 sfileno);
1518 debug(20, 3, "storeRebuildFromDisk: --> <URL:%s>\n", url);
1519 /* don't unlink the file! just skip this log entry */
1520 data->clashcount++;
1521 continue;
1522 }
1523 /* update store_swap_size */
1524 store_swap_size += (int) ((size + 1023) >> 10);
1525 data->objcount++;
f5469bce 1526 e = storeAddDiskRestore(url,
30a4f2a8 1527 sfileno,
1528 (int) size,
1529 expires,
f5469bce 1530 timestamp,
1531 lastmod);
1532 storeSwapLog(e);
30a4f2a8 1533 CacheInfo->proto_newobject(CacheInfo,
1ac9bc38 1534 urlParseProtocol(url),
30a4f2a8 1535 (int) size,
1536 TRUE);
c943f331 1537 }
3facc31e 1538 return 1;
1539}
1540
1541/* meta data recreated from disk image in swap directory */
1542static void storeRebuiltFromDisk(data)
1543 struct storeRebuild_data *data;
1544{
1545 time_t r;
1546 time_t stop;
c943f331 1547
b37a37f9 1548 stop = getCurrentTime();
3facc31e 1549 r = stop - data->start;
c943f331 1550 debug(20, 1, "Finished rebuilding storage from disk image.\n");
3facc31e 1551 debug(20, 1, " %7d Lines read from previous logfile.\n", data->linecount);
1552 debug(20, 1, " %7d Objects loaded.\n", data->objcount);
1553 debug(20, 1, " %7d Objects expired.\n", data->expcount);
1554 debug(20, 1, " %7d Duplicate URLs purged.\n", data->dupcount);
1555 debug(20, 1, " %7d Swapfile clashes avoided.\n", data->clashcount);
c943f331 1556 debug(20, 1, " Took %d seconds (%6.1lf objects/sec).\n",
3facc31e 1557 r > 0 ? r : 0, (double) data->objcount / (r > 0 ? r : 1));
c943f331 1558 debug(20, 1, " store_swap_size = %dk\n", store_swap_size);
090089c4 1559
30a4f2a8 1560 store_rebuilding = STORE_NOT_REBUILDING;
3facc31e 1561
1562 fclose(data->log);
1563 safe_free(data);
1564 sprintf(tmp_filename, "%s.new", swaplog_file);
1565 if (rename(tmp_filename, swaplog_file) < 0) {
9a70f206 1566 debug(20, 0, "storeRebuiltFromDisk: %s,%s: %s\n",
9ff15015 1567 tmp_filename, swaplog_file, xstrerror());
9a70f206 1568 fatal_dump("storeRebuiltFromDisk: rename failed");
3facc31e 1569 }
9a70f206 1570 if (file_write_unlock(swaplog_fd, swaplog_lock) != DISK_OK)
1571 fatal_dump("storeRebuiltFromDisk: swaplog unlock failed");
2c0a39aa 1572 file_close(swaplog_fd);
e47afcdb 1573 if ((swaplog_fd = file_open(swaplog_file, NULL, O_WRONLY | O_CREAT)) < 0)
9a70f206 1574 fatal_dump("storeRebuiltFromDisk: file_open(swaplog_file) failed");
e83421f8 1575 swaplog_lock = file_write_lock(swaplog_fd);
090089c4 1576}
1577
3facc31e 1578void storeStartRebuildFromDisk()
1579{
1580 struct stat sb;
1581 int i;
1582 struct storeRebuild_data *data;
1583 time_t last_clean;
1584
1585 if (stat(swaplog_file, &sb) < 0) {
1586 debug(20, 1, "storeRebuildFromDisk: No log file\n");
30a4f2a8 1587 store_rebuilding = STORE_NOT_REBUILDING;
3facc31e 1588 return;
1589 }
1590 data = xcalloc(1, sizeof(*data));
1591
6e40f263 1592 for (i = 0; i < ncache_dirs; i++)
3facc31e 1593 debug(20, 1, "Rebuilding storage from disk image in %s\n", swappath(i));
1594 data->start = getCurrentTime();
1595
1596 /* Check if log is clean */
1597 sprintf(tmp_filename, "%s/log-last-clean", swappath(0));
1598 if (stat(tmp_filename, &sb) >= 0) {
1599 last_clean = sb.st_mtime;
9a70f206 1600 if (stat(swaplog_file, &sb) >= 0)
30a4f2a8 1601 store_rebuilding = (sb.st_mtime <= last_clean) ?
1602 STORE_REBUILDING_FAST : STORE_REBUILDING_SLOW;
3facc31e 1603 }
30a4f2a8 1604 /* Remove timestamp in case we crash during rebuild */
1605 safeunlink(tmp_filename, 1);
3facc31e 1606 /* close the existing write-only swaplog, and open a temporary
1607 * write-only swaplog */
1608 if (file_write_unlock(swaplog_fd, swaplog_lock) != DISK_OK)
1609 fatal_dump("storeStartRebuildFromDisk: swaplog unlock failed");
1610 if (swaplog_fd > -1)
1611 file_close(swaplog_fd);
1612 sprintf(tmp_filename, "%s.new", swaplog_file);
e47afcdb 1613 swaplog_fd = file_open(tmp_filename, NULL, O_WRONLY | O_CREAT | O_TRUNC);
66a91d2d 1614 debug(20, 3, "swaplog_fd %d is now '%s'\n", swaplog_fd, tmp_filename);
3facc31e 1615 if (swaplog_fd < 0) {
1616 debug(20, 0, "storeStartRebuildFromDisk: %s: %s\n",
1617 tmp_filename, xstrerror());
1618 fatal("storeStartRebuildFromDisk: Can't open tmp swaplog");
1619 }
1620 swaplog_lock = file_write_lock(swaplog_fd);
3facc31e 1621 /* Open the existing swap log for reading */
1622 if ((data->log = fopen(swaplog_file, "r")) == (FILE *) NULL) {
1623 sprintf(tmp_error_buf, "storeRebuildFromDisk: %s: %s",
1624 swaplog_file, xstrerror());
1625 fatal(tmp_error_buf);
1626 }
66a91d2d 1627 debug(20, 3, "data->log %d is now '%s'\n", fileno(data->log), swaplog_file);
30a4f2a8 1628 if (store_rebuilding == STORE_REBUILDING_FAST)
3facc31e 1629 debug(20, 1, "Rebuilding in FAST MODE.\n");
1630
1631 memset(data->line_in, '\0', 4096);
30a4f2a8 1632 data->speed = store_rebuilding == STORE_REBUILDING_FAST ? 50 : 5;
3facc31e 1633
1634 /* Start reading the log file */
30a4f2a8 1635 if (opt_foreground_rebuild) {
1636 while (storeDoRebuildFromDisk(data));
1637 storeRebuiltFromDisk(data);
1638 } else {
1639 runInBackground("storeRebuild",
1640 (int (*)(void *)) storeDoRebuildFromDisk,
1641 data,
1642 (void (*)(void *)) storeRebuiltFromDisk);
1643 }
3facc31e 1644}
090089c4 1645
1646/* return current swap size in kilo-bytes */
1647int storeGetSwapSize()
1648{
1649 return store_swap_size;
1650}
1651
1652/* return current swap size in bytes */
1653int storeGetMemSize()
1654{
1655 return store_mem_size;
1656}
1657
6602e70e 1658static int storeCheckSwapable(e)
1659 StoreEntry *e;
1660{
1661
234967c9 1662 if (e->expires <= squid_curtime) {
3e98df20 1663 debug(20, 2, "storeCheckSwapable: NO: expires now\n");
73ddce89 1664 } else if (e->method != METHOD_GET) {
2285407f 1665 debug(20, 2, "storeCheckSwapable: NO: non-GET method\n");
1c481e00 1666 } else if (!BIT_TEST(e->flag, ENTRY_CACHABLE)) {
2285407f 1667 debug(20, 2, "storeCheckSwapable: NO: not cachable\n");
6602e70e 1668 } else if (BIT_TEST(e->flag, RELEASE_REQUEST)) {
2285407f 1669 debug(20, 2, "storeCheckSwapable: NO: release requested\n");
6602e70e 1670 } else if (!storeEntryValidLength(e)) {
2285407f 1671 debug(20, 2, "storeCheckSwapable: NO: wrong content-length\n");
3e98df20 1672 } else if (e->expires <= squid_curtime + Config.negativeTtl) {
1673 debug(20, 2, "storeCheckSwapable: NO: expires soon\n");
1674 return 0; /* avoid release call below */
6602e70e 1675 } else
1676 return 1;
1677
2daae136 1678 storeReleaseRequest(e);
1c481e00 1679 BIT_RESET(e->flag, ENTRY_CACHABLE);
6602e70e 1680 return 0;
1681}
1682
1683
090089c4 1684
1685/* Complete transfer into the local cache. */
1686void storeComplete(e)
1687 StoreEntry *e;
1688{
2285407f 1689 debug(20, 3, "storeComplete: '%s'\n", e->key);
22e4fa85 1690 e->object_len = e->mem_obj->e_current_len;
090089c4 1691 InvokeHandlers(e);
b8de7ebe 1692 e->lastref = squid_curtime;
234967c9 1693 e->store_status = STORE_OK;
090089c4 1694 storeSetMemStatus(e, IN_MEMORY);
1695 e->swap_status = NO_SWAP;
6602e70e 1696 if (storeCheckSwapable(e))
090089c4 1697 storeSwapOutStart(e);
090089c4 1698 /* free up incoming MIME */
22e4fa85 1699 safe_free(e->mem_obj->mime_hdr);
090089c4 1700 if (e->flag & RELEASE_REQUEST)
1701 storeRelease(e);
1702}
1703
1704/*
1705 * Fetch aborted. Tell all clients to go home. Negatively cache
1706 * abort message, freeing the data for this object
1707 */
1708int storeAbort(e, msg)
1709 StoreEntry *e;
1710 char *msg;
1711{
95d659f0 1712 LOCAL_ARRAY(char, mime_hdr, 300);
1713 LOCAL_ARRAY(char, abort_msg, 2000);
3e98df20 1714 MemObject *mem = e->mem_obj;
090089c4 1715
b32508fb 1716 if (e->store_status != STORE_PENDING) /* XXX remove later */
1717 fatal_dump("storeAbort: bad store_status");
6e40f263 1718 if (mem == NULL) /* XXX remove later */
1719 fatal_dump("storeAbort: null mem");
1720
2285407f 1721 debug(20, 6, "storeAbort: '%s'\n", e->key);
b6f794d6 1722 e->expires = squid_curtime + Config.negativeTtl;
234967c9 1723 e->store_status = STORE_ABORTED;
090089c4 1724 storeSetMemStatus(e, IN_MEMORY);
1725 /* No DISK swap for negative cached object */
1726 e->swap_status = NO_SWAP;
b8de7ebe 1727 e->lastref = squid_curtime;
090089c4 1728 /* In case some parent responds late and
1729 * tries to restart the fetch, say that it's been
1730 * dispatched already.
1731 */
6eb42cae 1732 BIT_SET(e->flag, ENTRY_DISPATCHED);
090089c4 1733
30a4f2a8 1734 storeLockObject(e, NULL, NULL);
090089c4 1735
1736 /* Count bytes faulted through cache but not moved to disk */
1ac9bc38 1737 CacheInfo->proto_touchobject(CacheInfo,
3e98df20 1738 mem->request ? mem->request->protocol : PROTO_NONE,
1739 mem->e_current_len);
090089c4 1740 mk_mime_hdr(mime_hdr,
b6f794d6 1741 (time_t) Config.negativeTtl,
090089c4 1742 6 + strlen(msg),
b8de7ebe 1743 squid_curtime,
090089c4 1744 "text/html");
1745 if (msg) {
1746 /* This can run off the end here. Be careful */
1747 if ((int) (strlen(msg) + strlen(mime_hdr) + 50) < 2000) {
1748 sprintf(abort_msg, "HTTP/1.0 400 Cache Detected Error\r\n%s\r\n\r\n%s", mime_hdr, msg);
1749 } else {
a8f7d3ee 1750 debug(20, 0, "storeAbort: WARNING: Must increase msg length!\n");
090089c4 1751 }
1752 storeAppend(e, abort_msg, strlen(abort_msg));
a1e47288 1753 safe_free(mem->e_abort_msg);
3e98df20 1754 mem->e_abort_msg = xstrdup(abort_msg);
090089c4 1755 /* Set up object for negative caching */
1756 BIT_SET(e->flag, ABORT_MSG_PENDING);
1757 }
1758 /* We assign an object length here--The only other place we assign the
1759 * object length is in storeComplete() */
3e98df20 1760 e->object_len = mem->e_current_len;
090089c4 1761
1762 /* Call handlers so they can report error. */
1763 InvokeHandlers(e);
1764
1765 storeUnlockObject(e);
1766 return 0;
1767}
1768
1769/* get the first in memory object entry in the storage */
1770hash_link *storeFindFirst(id)
1771 HashID id;
1772{
1773 if (id == (HashID) 0)
1774 return NULL;
1775 return (hash_first(id));
1776}
1777
1778/* get the next in memory object entry in the storage for a given
1779 * search pointer */
1780hash_link *storeFindNext(id)
1781 HashID id;
1782{
1783 if (id == (HashID) 0)
1784 return NULL;
1785 return (hash_next(id));
1786}
1787
1788/* get the first in memory object entry in the storage */
1789StoreEntry *storeGetInMemFirst()
1790{
1791 hash_link *first = NULL;
1792 first = storeFindFirst(in_mem_table);
1793 return (first ? ((StoreEntry *) first->item) : NULL);
1794}
1795
1796
1797/* get the next in memory object entry in the storage for a given
1798 * search pointer */
1799StoreEntry *storeGetInMemNext()
1800{
1801 hash_link *next = NULL;
1802 next = storeFindNext(in_mem_table);
1803 return (next ? ((StoreEntry *) next->item) : NULL);
1804}
1805
1806/* get the first entry in the storage */
1807StoreEntry *storeGetFirst()
1808{
ec74cf5f 1809 return ((StoreEntry *) storeFindFirst(store_table));
090089c4 1810}
1811
1812
1813/* get the next entry in the storage for a given search pointer */
1814StoreEntry *storeGetNext()
1815{
ec74cf5f 1816 return ((StoreEntry *) storeFindNext(store_table));
090089c4 1817}
1818
620da955 1819/* free up all ttl-expired objects */
1820int storePurgeOld()
090089c4 1821{
1822 StoreEntry *e = NULL;
090089c4 1823 int n = 0;
620da955 1824 int count = 0;
090089c4 1825 for (e = storeGetFirst(); e; e = storeGetNext()) {
0a5b9b32 1826 if ((++n & 0xFF) == 0) {
b37a37f9 1827 getCurrentTime();
0a5b9b32 1828 if (shutdown_pending || reread_pending)
1829 break;
1830 }
090089c4 1831 if ((n & 0xFFF) == 0)
c943f331 1832 debug(20, 2, "storeWalkThrough: %7d objects so far.\n", n);
620da955 1833 if (storeCheckExpired(e))
1834 if (storeRelease(e) == 0)
1835 count++;
090089c4 1836 }
1837 return count;
1838}
1839
1840
090089c4 1841#define MEM_LRUSCAN_BLOCK 16
1842#define MEM_MAX_HELP 5
1843/* Clear Memory storage to accommodate the given object len */
1844int storeGetMemSpace(size, check_vm_number)
1845 int size;
1846 int check_vm_number;
1847{
b32508fb 1848 StoreEntry *e = NULL;
1849 StoreEntry **list = NULL;
1850 int list_count = 0;
090089c4 1851 int n_expired = 0;
66c2848f 1852 int n_purged = 0;
3bbba102 1853 int n_locked = 0;
090089c4 1854 int i;
b32508fb 1855 static time_t last_warning = 0;
1856 static time_t last_check = 0;
090089c4 1857
b32508fb 1858 if (squid_curtime == last_check)
090089c4 1859 return 0;
b32508fb 1860 last_check = squid_curtime;
090089c4 1861
b32508fb 1862 if (meta_data.hot_vm < store_hotobj_high && ((store_mem_size + size) < store_mem_high))
1863 return 0;
c943f331 1864 debug(20, 2, "storeGetMemSpace: Starting...\n");
090089c4 1865
b32508fb 1866 list = xcalloc(meta_data.store_in_mem_objects, sizeof(ipcache_entry *));
090089c4 1867 for (e = storeGetInMemFirst(); e; e = storeGetInMemNext()) {
b32508fb 1868 if (list_count == meta_data.store_in_mem_objects)
1869 break;
620da955 1870 if (!storeCheckPurgeMem(e)) {
3bbba102 1871 n_locked++;
090089c4 1872 continue;
3bbba102 1873 }
620da955 1874 if (storeCheckExpired(e)) {
c943f331 1875 debug(20, 2, "storeGetMemSpace: Expired: %s\n", e->url);
090089c4 1876 n_expired++;
090089c4 1877 storeRelease(e);
1878 continue;
1879 }
b32508fb 1880 debug(20, 3, "storeGetMemSpace: Adding '%s'\n", e->url);
1881 *(list + list_count) = e;
1882 list_count++;
090089c4 1883 }
f5469bce 1884#ifdef EXTRA_DEBUGGING
6602e70e 1885 debug(20, 5, "storeGetMemSpace: Current size: %7d bytes\n", store_mem_size);
1886 debug(20, 5, "storeGetMemSpace: High W Mark: %7d bytes\n", store_mem_high);
1887 debug(20, 5, "storeGetMemSpace: Low W Mark: %7d bytes\n", store_mem_low);
1888 debug(20, 5, "storeGetMemSpace: Entry count: %7d items\n", meta_data.store_entries);
f5469bce 1889#endif
b32508fb 1890 debug(20, 5, "storeGetMemSpace: Sorting LRU_list: %7d items\n", list_count);
1891 qsort((char *) list,
1892 list_count,
6e40f263 1893 sizeof(StoreEntry *),
b32508fb 1894 (QS) compareSize);
090089c4 1895
1896 /* Kick LRU out until we have enough memory space */
b32508fb 1897 for (i = 0; i < list_count; i++) {
1898 if (meta_data.hot_vm > store_hotobj_low) {
66c2848f 1899 storePurgeMem(*(list + i));
1900 n_purged++;
b32508fb 1901 } else if ((store_mem_size + size) > store_mem_low) {
66c2848f 1902 storePurgeMem(*(list + i));
1903 n_purged++;
b32508fb 1904 } else
1905 break;
090089c4 1906 }
1907
c943f331 1908 debug(20, 2, "storeGetMemSpace: After freeing size: %7d bytes\n", store_mem_size);
66c2848f 1909 debug(20, 2, "storeGetMemSpace: Purged: %7d items\n", n_purged);
b32508fb 1910 if (meta_data.hot_vm > store_hotobj_high) {
3bbba102 1911 if (squid_curtime - last_warning > 600) {
b32508fb 1912 debug(20, 0, "WARNING: Over hot_vm object count (%d > %d)\n",
1913 meta_data.hot_vm, store_hotobj_high);
3bbba102 1914 debug(20, 0, " : %d objects locked in memory\n", n_locked);
66c2848f 1915 debug(20, 0, " : purged %d of %d candidates\n",
1916 n_purged,
3bbba102 1917 list_count);
b32508fb 1918 last_warning = squid_curtime;
090089c4 1919 }
b32508fb 1920 } else if ((store_mem_size + size) > store_mem_high) {
3bbba102 1921 if (squid_curtime - last_warning > 600) {
b32508fb 1922 debug(20, 0, "WARNING: Over store_mem high-water mark (%d > %d)\n",
1923 store_mem_size + size, store_mem_high);
3bbba102 1924 debug(20, 0, " : %d objects locked in memory\n", n_locked);
1925 debug(20, 0, " : released %d of %d candidates\n",
66c2848f 1926 n_purged,
3bbba102 1927 list_count);
b32508fb 1928 last_warning = squid_curtime;
090089c4 1929 }
090089c4 1930 }
c943f331 1931 debug(20, 2, "storeGetMemSpace: Done.\n");
3d700ed6 1932 xfree(list);
090089c4 1933 return 0;
1934}
1935
6e40f263 1936static int compareSize(e1, e2)
090089c4 1937 StoreEntry **e1, **e2;
1938{
6e40f263 1939 if (!e1 || !e2)
1940 fatal_dump(NULL);
22e4fa85 1941 if ((*e1)->mem_obj->e_current_len > (*e2)->mem_obj->e_current_len)
090089c4 1942 return (1);
22e4fa85 1943 if ((*e1)->mem_obj->e_current_len < (*e2)->mem_obj->e_current_len)
090089c4 1944 return (-1);
090089c4 1945 return (0);
1946}
1947
6e40f263 1948static int compareLastRef(e1, e2)
090089c4 1949 StoreEntry **e1, **e2;
1950{
1951 if (!e1 || !e2)
1952 fatal_dump(NULL);
090089c4 1953 if ((*e1)->lastref > (*e2)->lastref)
1954 return (1);
090089c4 1955 if ((*e1)->lastref < (*e2)->lastref)
1956 return (-1);
090089c4 1957 return (0);
1958}
1959
1960/* returns the bucket number to work on,
1961 * pointer to next bucket after each calling
1962 */
1963unsigned int storeGetBucketNum()
1964{
1965 static unsigned int bucket = 0;
090089c4 1966 if (bucket >= STORE_BUCKETS)
1967 bucket = 0;
1968 return (bucket++);
1969}
1970
1971#define SWAP_LRUSCAN_BLOCK 16
1972#define SWAP_MAX_HELP STORE_BUCKETS/2
1973
1974/* The maximum objects to scan for maintain storage space */
1975#define SWAP_LRUSCAN_COUNT (256)
1976
1977/* Removes at most 30 LRU objects for one loop */
1978#define SWAP_LRU_REMOVE_COUNT (8)
1979
1980/* Clear Swap storage to accommodate the given object len */
1981int storeGetSwapSpace(size)
1982 int size;
1983{
1984 static int fReduceSwap = 0;
1985 static int swap_help = 0;
1986 StoreEntry *LRU = NULL, *e = NULL;
1987 int scanned = 0;
1988 int removed = 0;
1989 int expired = 0;
1990 int locked = 0;
1991 int locked_size = 0;
1992 int scan_in_objs = 0;
1993 int i;
1994 int LRU_cur_size = meta_data.store_entries;
1995 dynamic_array *LRU_list;
1996 hash_link *link_ptr = NULL, *next = NULL;
1997 unsigned int kb_size = ((size + 1023) >> 10);
1998
1999 if (store_swap_size + kb_size <= store_swap_low)
2000 fReduceSwap = 0;
2001 if (!fReduceSwap && (store_swap_size + kb_size <= store_swap_high)) {
2002 return 0;
2003 }
c943f331 2004 debug(20, 2, "storeGetSwapSpace: Starting...\n");
090089c4 2005
2006 /* Set flag if swap size over high-water-mark */
2007 if (store_swap_size + kb_size > store_swap_high)
2008 fReduceSwap = 1;
2009
c943f331 2010 debug(20, 2, "storeGetSwapSpace: Need %d bytes...\n", size);
090089c4 2011
2012 LRU_list = create_dynamic_array(LRU_cur_size, LRU_cur_size);
2013 /* remove expired objects until recover enough space or no expired objects */
6e40f263 2014 for (i = 0; i < STORE_BUCKETS; i++) {
090089c4 2015 int expired_in_one_bucket = 0;
2016
ec74cf5f 2017 link_ptr = hash_get_bucket(store_table, storeGetBucketNum());
090089c4 2018 if (link_ptr == NULL)
2019 continue;
2020 /* this while loop handles one bucket of hash table */
2021 expired_in_one_bucket = 0;
2022 while (link_ptr) {
2023 scanned++;
2024 next = link_ptr->next;
2025 e = (StoreEntry *) link_ptr;
2026
2027 /* Identify objects that aren't locked, for replacement */
a1e47288 2028 if (!storeEntryLocked(e)) {
b8de7ebe 2029 if (squid_curtime > e->expires) {
2285407f 2030 debug(20, 2, "storeGetSwapSpace: Expired: <URL:%s>\n", e->url);
090089c4 2031 /* just call release. don't have to check for lock status.
2032 * storeRelease will take care of that and set a pending flag
2033 * if it's still locked. */
2034 ++expired_in_one_bucket;
2035 storeRelease(e);
2036 } else {
2037 /* Prepare to do LRU replacement */
2038 insert_dynamic_array(LRU_list, e);
2039 ++scan_in_objs;
2040 }
2041 } else {
090089c4 2042 locked++;
22e4fa85 2043 locked_size += e->mem_obj->e_current_len;
090089c4 2044 }
2045 link_ptr = next;
2046 } /* while, end of one bucket of hash table */
2047
2048 expired += expired_in_one_bucket;
2049 if (expired_in_one_bucket &&
2050 ((!fReduceSwap && (store_swap_size + kb_size <= store_swap_high)) ||
2051 (fReduceSwap && (store_swap_size + kb_size <= store_swap_low)))
2052 ) {
2053 fReduceSwap = 0;
2054 destroy_dynamic_array(LRU_list);
c943f331 2055 debug(20, 2, "storeGetSwapSpace: Finished, %d objects expired.\n",
090089c4 2056 expired);
2057 return 0;
2058 }
6e40f263 2059 qsort((char *) LRU_list->collection,
2060 LRU_list->index,
2061 sizeof(StoreEntry *),
2062 (QS) compareLastRef);
090089c4 2063 /* keep the first n LRU objects only */
2064 cut_dynamic_array(LRU_list, SWAP_LRU_REMOVE_COUNT);
2065
2066 /* Scan in about SWAP_LRU_COUNT for one call */
2067 if (scan_in_objs >= SWAP_LRUSCAN_COUNT)
2068 break;
2069 } /* for */
2070
30a4f2a8 2071#ifdef LOTSA_DEBUGGING
090089c4 2072 /* end of candidate selection */
30a4f2a8 2073 debug(20, 2, "storeGetSwapSpace: Current Size: %7d kbytes\n",
2074 store_swap_size);
2075 debug(20, 2, "storeGetSwapSpace: High W Mark: %7d kbytes\n",
2076 store_swap_high);
2077 debug(20, 2, "storeGetSwapSpace: Low W Mark: %7d kbytes\n",
2078 store_swap_low);
2079 debug(20, 2, "storeGetSwapSpace: Entry count: %7d items\n",
2080 meta_data.store_entries);
2081 debug(20, 2, "storeGetSwapSpace: Visited: %7d buckets\n",
2082 i + 1);
2083 debug(20, 2, "storeGetSwapSpace: Scanned: %7d items\n",
2084 scanned);
2085 debug(20, 2, "storeGetSwapSpace: Expired: %7d items\n",
2086 expired);
2087 debug(20, 2, "storeGetSwapSpace: Locked: %7d items\n",
2088 locked);
2089 debug(20, 2, "storeGetSwapSpace: Locked Space: %7d bytes\n",
2090 locked_size);
2091 debug(20, 2, "storeGetSwapSpace: Scan in array: %7d bytes\n",
2092 scan_in_objs);
2093 debug(20, 2, "storeGetSwapSpace: LRU candidate: %7d items\n",
2094 LRU_list->index);
2095#endif /* LOTSA_DEBUGGING */
090089c4 2096
2097 /* Although all expired objects removed, still didn't recover enough */
2098 /* space. Kick LRU out until we have enough swap space */
6e40f263 2099 for (i = 0; i < LRU_list->index; i++) {
090089c4 2100 if (store_swap_size + kb_size <= store_swap_low) {
2101 fReduceSwap = 0;
2102 break;
2103 }
2104 if ((LRU = LRU_list->collection[i]) != NULL) {
2105 if (storeRelease(LRU) == 0) {
2106 removed++;
2107 } else {
c943f331 2108 debug(20, 2, "storeGetSwapSpace: Help! Can't remove objects. <%s>\n",
090089c4 2109 LRU->url);
2110 }
2111 }
2112 }
c943f331 2113 debug(20, 2, "storeGetSwapSpace: After Freeing Size: %7d kbytes\n", store_swap_size);
090089c4 2114
2115 /* free the list */
2116 destroy_dynamic_array(LRU_list);
2117
2118 if ((store_swap_size + kb_size > store_swap_high)) {
2119 if (++swap_help > SWAP_MAX_HELP) {
c943f331 2120 debug(20, 0, "storeGetSwapSpace: Nothing to free with %d Kbytes in use.\n",
090089c4 2121 store_swap_size);
c943f331 2122 debug(20, 0, "--> Asking for %d bytes\n", size);
2123 debug(20, 0, "WARNING! Repeated failures to allocate swap space!\n");
2124 debug(20, 0, "WARNING! Please check your disk space.\n");
090089c4 2125 swap_help = 0;
2126 } else {
c943f331 2127 debug(20, 2, "storeGetSwapSpace: Nothing to free with %d Kbytes in use.\n",
090089c4 2128 store_swap_size);
c943f331 2129 debug(20, 2, "--> Asking for %d bytes\n", size);
090089c4 2130 }
2131 } else {
2132 swap_help = 0;
2133 }
2134
30a4f2a8 2135 getCurrentTime(); /* we may have taken more than one second */
2136 debug(20, 2, "Removed %d objects\n", removed);
090089c4 2137 return 0;
2138}
2139
2140
2141/* release an object from a cache */
2142/* return 0 when success. */
2143int storeRelease(e)
2144 StoreEntry *e;
2145{
2146 StoreEntry *result = NULL;
2147 StoreEntry *head_result = NULL;
2148 hash_link *hptr = NULL;
227fbb74 2149 hash_link *head_table_entry = NULL;
090089c4 2150
6602e70e 2151 debug(20, 3, "storeRelease: Releasing: '%s'\n", e->key);
2152
090089c4 2153 /* If, for any reason we can't discard this object because of an
2154 * outstanding request, mark it for pending release */
2155 if (storeEntryLocked(e)) {
9174e204 2156 storeExpireNow(e);
6602e70e 2157 debug(20, 3, "storeRelease: Only setting RELEASE_REQUEST bit\n");
2daae136 2158 storeReleaseRequest(e);
090089c4 2159 return -1;
2160 }
227fbb74 2161 if (e->key != NULL) {
ec74cf5f 2162 if ((hptr = hash_lookup(store_table, e->key)) == NULL) {
2285407f 2163 debug(20, 0, "storeRelease: Not Found: '%s'\n", e->key);
227fbb74 2164 debug(20, 0, "Dump of Entry 'e':\n %s\n", storeToString(e));
2165 fatal_dump(NULL);
2166 }
2167 result = (StoreEntry *) hptr;
2168 if (result != e) {
2169 debug(20, 0, "storeRelease: Duplicated entry? <URL:%s>\n",
2170 result->url ? result->url : "NULL");
2171 debug(20, 0, "Dump of Entry 'e':\n%s", storeToString(e));
2172 debug(20, 0, "Dump of Entry 'result':\n%s", storeToString(result));
2173 fatal_dump(NULL);
2174 }
090089c4 2175 }
73ddce89 2176 if (e->method == METHOD_GET) {
090089c4 2177 /* check if coresponding HEAD object exists. */
ec74cf5f 2178 head_table_entry = hash_lookup(store_table,
6eb42cae 2179 storeGeneratePublicKey(e->url, METHOD_HEAD));
090089c4 2180 if (head_table_entry) {
2181 head_result = (StoreEntry *) head_table_entry;
2182 if (head_result) {
2183 /* recursive call here to free up /head/ */
2184 storeRelease(head_result);
2185 }
2186 }
2187 }
30a4f2a8 2188 if (store_rebuilding == STORE_REBUILDING_FAST) {
2189 debug(20, 2, "storeRelease: Delaying release until store is rebuilt: '%s'\n",
2190 e->key ? e->key : e->url ? e->url : "NO URL");
2191 storeExpireNow(e);
2192 storeSetPrivateKey(e);
2193 return -1;
2194 }
227fbb74 2195 if (e->key)
2196 debug(20, 5, "storeRelease: Release object key: %s\n", e->key);
2197 else
2198 debug(20, 5, "storeRelease: Release anonymous object\n");
090089c4 2199
2200 if (e->swap_status == SWAP_OK && (e->swap_file_number > -1)) {
30a4f2a8 2201 (void) safeunlink(storeSwapFullPath(e->swap_file_number, NULL), 1);
090089c4 2202 file_map_bit_reset(e->swap_file_number);
2203 e->swap_file_number = -1;
2204 store_swap_size -= (e->object_len + 1023) >> 10;
30a4f2a8 2205 CacheInfo->proto_purgeobject(CacheInfo,
1ac9bc38 2206 urlParseProtocol(e->url),
30a4f2a8 2207 e->object_len);
090089c4 2208 }
f7a5493b 2209 storeHashDelete(e);
147d3115 2210 storeLog(STORE_LOG_RELEASE, e);
30a4f2a8 2211 destroy_StoreEntry(e);
090089c4 2212 return 0;
2213}
2214
2215
090089c4 2216/* return if the current key is the original one. */
2217int storeOriginalKey(e)
2218 StoreEntry *e;
2219{
2220 if (!e)
2221 return 1;
090089c4 2222 return !(e->flag & KEY_CHANGE);
2223}
2224
2225/* return 1 if a store entry is locked */
2226int storeEntryLocked(e)
2227 StoreEntry *e;
2228{
2229 if (!e) {
c943f331 2230 debug(20, 0, "This entry should be valid.\n");
2231 debug(20, 0, "%s", storeToString(e));
090089c4 2232 fatal_dump(NULL);
2233 }
30a4f2a8 2234 if (e->lock_count)
2235 return 1;
2236 if (e->swap_status == SWAPPING_OUT)
2237 return 1;
2238 if (e->mem_status == SWAPPING_IN)
2239 return 1;
a1e47288 2240 if (e->store_status == STORE_PENDING)
2241 return 1;
30a4f2a8 2242 return 0;
090089c4 2243}
2244
2245/* use this for internal call only */
2246int storeCopy(e, stateoffset, maxSize, buf, size)
2247 StoreEntry *e;
2248 int stateoffset;
2249 int maxSize;
2250 char *buf;
2251 int *size;
2252{
2253 int available_to_write = 0;
2254
22e4fa85 2255 available_to_write = e->mem_obj->e_current_len - stateoffset;
090089c4 2256
22e4fa85 2257 if (stateoffset < e->mem_obj->e_lowest_offset) {
090089c4 2258 /* this should not happen. Logic race !!! */
c943f331 2259 debug(20, 1, "storeCopy: Client Request a chunk of data in area lower than the lowest_offset\n");
2260 debug(20, 1, " Current Lowest offset : %d\n", e->mem_obj->e_lowest_offset);
2261 debug(20, 1, " Requested offset : %d\n", stateoffset);
090089c4 2262 /* can't really do anything here. Client may hang until lifetime runout. */
2263 return 0;
2264 }
2265 *size = (available_to_write >= maxSize) ?
2266 maxSize : available_to_write;
2267
c943f331 2268 debug(20, 6, "storeCopy: avail_to_write=%d, store_offset=%d\n",
090089c4 2269 *size, stateoffset);
2270
2271 if (*size > 0)
22e4fa85 2272 (void) e->mem_obj->data->mem_copy(e->mem_obj->data, stateoffset, buf, *size);
090089c4 2273
2274 return *size;
2275}
2276
2277/* check if there is any client waiting for this object at all */
2278/* return 1 if there is at least one client */
2279int storeClientWaiting(e)
2280 StoreEntry *e;
2281{
2282 int i;
6e40f263 2283 MemObject *mem = e->mem_obj;
2284 if (mem->client_list) {
2285 for (i = 0; i < mem->client_list_size; i++) {
2286 if (mem->client_list[i])
090089c4 2287 return 1;
2288 }
2289 }
6e40f263 2290 if (mem->pending) {
2291 for (i = 0; i < (int) mem->pending_list_size; i++) {
2292 if (mem->pending[i])
090089c4 2293 return 1;
2294 }
2295 }
2296 return 0;
2297}
2298
2299/* return index to matched clientstatus in client_list, -1 on NOT_FOUND */
6e40f263 2300static int storeClientListSearch(mem, fd)
2301 MemObject *mem;
090089c4 2302 int fd;
2303{
2304 int i;
6e40f263 2305 if (mem->client_list) {
2306 for (i = 0; i < mem->client_list_size; i++) {
2307 if (mem->client_list[i] == NULL)
2308 continue;
2309 if (mem->client_list[i]->fd != fd)
2310 continue;
090089c4 2311 return i;
6e40f263 2312 }
090089c4 2313 }
2314 return -1;
2315}
2316
2317/* add client with fd to client list */
2318void storeClientListAdd(e, fd, last_offset)
2319 StoreEntry *e;
2320 int fd;
2321 int last_offset;
2322{
2323 int i;
6e40f263 2324 MemObject *mem = e->mem_obj;
2325 ClientStatusEntry **oldlist = NULL;
2326 int oldsize;
090089c4 2327 /* look for empty slot */
6e40f263 2328 if (mem->client_list == NULL) {
2329 mem->client_list_size = MIN_CLIENT;
2330 mem->client_list =
2331 xcalloc(mem->client_list_size, sizeof(ClientStatusEntry *));
090089c4 2332 }
6e40f263 2333 for (i = 0; i < mem->client_list_size; i++) {
2334 if (mem->client_list[i] == NULL)
2335 break;
2336 }
2337 if (i == mem->client_list_size) {
2338 debug(20, 3, "storeClientListAdd: FD %d Growing client_list for '%s'\n",
2339 fd, e->url);
2340 oldlist = mem->client_list;
2341 oldsize = mem->client_list_size;
2342 mem->client_list_size <<= 1;
2343 mem->client_list =
2344 xcalloc(mem->client_list_size, sizeof(ClientStatusEntry *));
2345 for (i = 0; i < oldsize; i++)
2346 mem->client_list[i] = oldlist[i];
2347 safe_free(oldlist);
2348 }
2349 mem->client_list[i] = xcalloc(1, sizeof(ClientStatusEntry));
2350 mem->client_list[i]->fd = fd;
2351 mem->client_list[i]->last_offset = last_offset;
090089c4 2352}
2353
2354/* same to storeCopy but also register client fd and last requested offset
2355 * for each client */
2356int storeClientCopy(e, stateoffset, maxSize, buf, size, fd)
2357 StoreEntry *e;
2358 int stateoffset;
2359 int maxSize;
2360 char *buf;
2361 int *size;
2362 int fd;
2363{
2364 int next_offset;
2365 int client_list_index;
22e4fa85 2366 int available_to_write = e->mem_obj->e_current_len - stateoffset;
6e40f263 2367 MemObject *mem = e->mem_obj;
090089c4 2368
6e40f263 2369 if (stateoffset < mem->e_lowest_offset) {
090089c4 2370 /* this should not happen. Logic race !!! */
c943f331 2371 debug(20, 1, "storeClientCopy: Client Request a chunk of data in area lower than the lowest_offset\n");
2372 debug(20, 1, " fd : %d\n", fd);
6e40f263 2373 debug(20, 1, " Current Lowest offset : %d\n", mem->e_lowest_offset);
c943f331 2374 debug(20, 1, " Requested offset : %d\n", stateoffset);
090089c4 2375 /* can't really do anything here. Client may hang until lifetime runout. */
2376 return 0;
2377 }
6e40f263 2378 *size = (available_to_write >= maxSize) ? maxSize : available_to_write;
090089c4 2379
c943f331 2380 debug(20, 6, "storeCopy: avail_to_write=%d, store_offset=%d\n",
090089c4 2381 *size, stateoffset);
2382
2383 /* update the lowest requested offset */
2384 next_offset = (stateoffset + *size);
6e40f263 2385 if ((client_list_index = storeClientListSearch(mem, fd)) >= 0) {
2386 mem->client_list[client_list_index]->last_offset = next_offset;
090089c4 2387 } else {
2388 storeClientListAdd(e, fd, next_offset);
2389 }
2390
2391 if (*size > 0)
6e40f263 2392 (void) mem->data->mem_copy(mem->data, stateoffset, buf, *size);
090089c4 2393
2394 /* see if we can get rid of some data if we are in "delete behind" mode . */
2395 if (e->flag & DELETE_BEHIND) {
2396 /* call the handler to delete behind the lowest offset */
2397 storeDeleteBehind(e);
2398 }
2399 return *size;
2400}
2401
090089c4 2402
2403int storeEntryValidToSend(e)
2404 StoreEntry *e;
2405{
b8de7ebe 2406 if (squid_curtime < e->expires)
9174e204 2407 return 1;
1c481e00 2408 if (e->expires != 0)
2409 return 0; /* Expired! */
2410 if (e->store_status != STORE_PENDING)
2411 return 0;
2412 if (e->mem_status != IN_MEMORY)
2413 return 0;
2414 return 1; /* STORE_PENDING, IN_MEMORY, exp=0 */
090089c4 2415}
2416
6602e70e 2417int storeEntryValidLength(e)
2418 StoreEntry *e;
2419{
ffe4a367 2420 int diff;
2285407f 2421 int hdr_sz;
2422 int content_length;
ffe4a367 2423
2285407f 2424 if (e->mem_obj == NULL)
ffe4a367 2425 fatal_dump("storeEntryValidLength: NULL mem_obj");
2426
2285407f 2427 hdr_sz = e->mem_obj->reply->hdr_sz;
147d3115 2428 content_length = e->mem_obj->reply->content_length;
2285407f 2429
ffe4a367 2430 debug(20, 3, "storeEntryValidLength: Checking '%s'\n", e->key);
60bf30cb 2431 debug(20, 5, "storeEntryValidLength: object_len = %d\n", e->object_len);
2285407f 2432 debug(20, 5, "storeEntryValidLength: hdr_sz = %d\n", hdr_sz);
2433 debug(20, 5, "storeEntryValidLength: content_length = %d\n", content_length);
ffe4a367 2434
2285407f 2435 if (content_length == 0) {
60bf30cb 2436 debug(20, 5, "storeEntryValidLength: Zero content length; assume valid; '%s'\n",
ffe4a367 2437 e->key);
2438 return 1;
2439 }
2285407f 2440 if (hdr_sz == 0) {
60bf30cb 2441 debug(20, 5, "storeEntryValidLength: Zero header size; assume valid; '%s'\n",
ffe4a367 2442 e->key);
2443 return 1;
2444 }
2285407f 2445 diff = hdr_sz + content_length - e->object_len;
ffe4a367 2446 if (diff != 0) {
60bf30cb 2447 debug(20, 3, "storeEntryValidLength: %d bytes too %s; '%s'\n",
ffe4a367 2448 diff < 0 ? -diff : diff,
2449 diff < 0 ? "small" : "big",
2450 e->key);
2451 return 0;
2452 }
2453 return 1;
2454}
6602e70e 2455
c943f331 2456static int storeVerifySwapDirs(clean)
2457 int clean;
090089c4 2458{
090089c4 2459 int inx;
2460 char *path = NULL;
c943f331 2461 struct stat sb;
2462 int directory_created = 0;
2463 char *cmdbuf = NULL;
090089c4 2464
6e40f263 2465 for (inx = 0; inx < ncache_dirs; inx++) {
090089c4 2466 path = swappath(inx);
9dfb6c1c 2467 debug(20, 9, "storeVerifySwapDirs: Creating swap space in %s\n", path);
090089c4 2468 if (stat(path, &sb) < 0) {
2469 /* we need to create a directory for swap file here. */
2470 if (mkdir(path, 0777) < 0) {
c943f331 2471 if (errno != EEXIST) {
2472 sprintf(tmp_error_buf, "Failed to create swap directory %s: %s",
2473 path,
2474 xstrerror());
2475 fatal(tmp_error_buf);
2476 }
090089c4 2477 }
2478 if (stat(path, &sb) < 0) {
c943f331 2479 sprintf(tmp_error_buf,
2480 "Failed to verify swap directory %s: %s",
090089c4 2481 path, xstrerror());
c943f331 2482 fatal(tmp_error_buf);
090089c4 2483 }
b37a37f9 2484 debug(20, 1, "storeVerifySwapDirs: Created swap directory %s\n", path);
090089c4 2485 directory_created = 1;
2486 }
234967c9 2487 if (clean && opt_unlink_on_reload) {
b37a37f9 2488 debug(20, 1, "storeVerifySwapDirs: Zapping all objects on disk storage.\n");
c943f331 2489 /* This could be dangerous, second copy of cache can destroy
2490 * the existing swap files of the previous cache. We may
2491 * use rc file do it. */
2492 cmdbuf = xcalloc(1, BUFSIZ);
2493 sprintf(cmdbuf, "cd %s; /bin/rm -rf log [0-9][0-9]", path);
b37a37f9 2494 debug(20, 1, "storeVerifySwapDirs: Running '%s'\n", cmdbuf);
c943f331 2495 system(cmdbuf); /* XXX should avoid system(3) */
2496 xfree(cmdbuf);
090089c4 2497 }
2498 }
c943f331 2499 return directory_created;
2500}
090089c4 2501
c943f331 2502static void storeCreateSwapSubDirs()
2503{
f5469bce 2504 int i, j, k;
95d659f0 2505 LOCAL_ARRAY(char, name, MAXPATHLEN);
c943f331 2506 for (j = 0; j < ncache_dirs; j++) {
f5469bce 2507 for (i = 0; i < SWAP_DIRECTORIES_L1; i++) {
2508 sprintf(name, "%s/%02X", swappath(j), i);
e6f18b4a 2509 debug(20, 1, "Making directories in %s\n", name);
c943f331 2510 if (mkdir(name, 0755) < 0) {
2511 if (errno != EEXIST) {
2512 sprintf(tmp_error_buf,
2513 "Failed to make swap directory %s: %s",
2514 name, xstrerror());
2515 fatal(tmp_error_buf);
2516 }
2517 }
f5469bce 2518 for (k = 0; k < SWAP_DIRECTORIES_L2; k++) {
2519 sprintf(name, "%s/%02X/%02X", swappath(j), i, k);
2520 if (mkdir(name, 0755) < 0) {
2521 if (errno != EEXIST) {
2522 sprintf(tmp_error_buf,
2523 "Failed to make swap directory %s: %s",
2524 name, xstrerror());
2525 fatal(tmp_error_buf);
2526 }
2527 }
2528 }
c943f331 2529 }
090089c4 2530 }
c943f331 2531}
090089c4 2532
c943f331 2533int storeInit()
2534{
2535 int dir_created;
0ffd22bc 2536 wordlist *w = NULL;
234967c9 2537 char *fname = NULL;
c943f331 2538
b6f794d6 2539 if (strcmp((fname = Config.Log.store), "none") == 0)
234967c9 2540 storelog_fd = -1;
2541 else
e47afcdb 2542 storelog_fd = file_open(fname, NULL, O_WRONLY | O_CREAT);
234967c9 2543 if (storelog_fd < 0)
2544 debug(20, 1, "Store logging disabled\n");
2285407f 2545
b6f794d6 2546 for (w = Config.cache_dirs; w; w = w->next)
0ffd22bc 2547 storeAddSwapDisk(w->key);
a26bdc75 2548 storeSanityCheck();
c943f331 2549 file_map_create(MAX_SWAP_FILE);
b6f794d6 2550 dir_created = storeVerifySwapDirs(opt_zap_disk_store);
b37a37f9 2551 storeCreateHashTable(urlcmp);
c943f331 2552
2553 sprintf(swaplog_file, "%s/log", swappath(0));
090089c4 2554
e47afcdb 2555 swaplog_fd = file_open(swaplog_file, NULL, O_WRONLY | O_CREAT);
66a91d2d 2556 debug(20, 3, "swaplog_fd %d is now '%s'\n", swaplog_fd, swaplog_file);
c943f331 2557 if (swaplog_fd < 0) {
2558 sprintf(tmp_error_buf, "Cannot open swap logfile: %s", swaplog_file);
2559 fatal(tmp_error_buf);
2560 }
c943f331 2561 swaplog_lock = file_write_lock(swaplog_fd);
3facc31e 2562
b6f794d6 2563 if (!opt_zap_disk_store)
3facc31e 2564 storeStartRebuildFromDisk();
30a4f2a8 2565 else
2566 store_rebuilding = STORE_NOT_REBUILDING;
c943f331 2567
b6f794d6 2568 if (dir_created || opt_zap_disk_store)
c943f331 2569 storeCreateSwapSubDirs();
2570
b6f794d6 2571 store_mem_high = (long) (Config.Mem.maxSize / 100) *
2572 Config.Mem.highWaterMark;
2573 store_mem_low = (long) (Config.Mem.maxSize / 100) *
2574 Config.Mem.lowWaterMark;
090089c4 2575
b6f794d6 2576 store_hotobj_high = (int) (Config.hotVmFactor *
090089c4 2577 store_mem_high / (1 << 20));
b6f794d6 2578 store_hotobj_low = (int) (Config.hotVmFactor *
090089c4 2579 store_mem_low / (1 << 20));
2580
b32508fb 2581 /* Note, there will always be a few 'hot_vm' objects as the
2582 * make the transition from STORE_PENDING to SWAPPING_OUT.
2583 Set some minimum values to prevent thrasing and warning
2584 messages. */
2585 if (store_hotobj_high < 32) {
2586 store_hotobj_high = 32;
2587 store_hotobj_low = 16;
2588 }
090089c4 2589 /* check for validity */
2590 if (store_hotobj_low > store_hotobj_high)
2591 store_hotobj_low = store_hotobj_high;
2592
b6f794d6 2593 store_swap_high = (long) (Config.Swap.maxSize / 100) *
2594 Config.Swap.highWaterMark;
2595 store_swap_low = (long) (Config.Swap.maxSize / 100) *
2596 Config.Swap.lowWaterMark;
090089c4 2597
2598 return 0;
2599}
2600
2601/*
2602 * storeSanityCheck - verify that all swap storage areas exist, and
2603 * are writable; otherwise, force -z.
2604 */
2605void storeSanityCheck()
2606{
95d659f0 2607 LOCAL_ARRAY(char, name, 4096);
090089c4 2608 int i;
2609
2610 if (ncache_dirs < 1)
f133a434 2611 storeAddSwapDisk(DefaultSwapDir);
090089c4 2612
0be3ec60 2613 for (i = 0; i < SWAP_DIRECTORIES_L1; i++) {
2614 sprintf(name, "%s/%02X", swappath(i), i);
090089c4 2615 errno = 0;
2616 if (access(name, W_OK)) {
2617 /* A very annoying problem occurs when access() fails because
b8de7ebe 2618 * the system file table is full. To prevent squid from
090089c4 2619 * deleting your entire disk cache on a whim, insist that the
2620 * errno indicates that the directory doesn't exist */
2621 if (errno != ENOENT)
2622 continue;
b8de7ebe 2623 debug(20, 0, "WARNING: Cannot write to swap directory '%s'\n",
2624 name);
2625 debug(20, 0, "Forcing a *full restart* (e.g., %s -z)...\n",
2626 appname);
b6f794d6 2627 opt_zap_disk_store = 1;
090089c4 2628 return;
2629 }
2630 }
2631}
2632
2633int urlcmp(url1, url2)
2634 char *url1, *url2;
2635{
2636 if (!url1 || !url2)
6eb42cae 2637 fatal_dump("urlcmp: Got a NULL url pointer.");
090089c4 2638 return (strcmp(url1, url2));
2639}
2640
090089c4 2641/*
2642 * This routine is to be called by main loop in main.c.
2643 * It removes expired objects on only one bucket for each time called.
2644 * returns the number of objects removed
2645 *
2646 * This should get called 1/s from main().
2647 */
2648int storeMaintainSwapSpace()
2649{
30a4f2a8 2650 static time_t last_time = 0;
090089c4 2651 static unsigned int bucket = 0;
2652 hash_link *link_ptr = NULL, *next = NULL;
2653 StoreEntry *e = NULL;
2654 int rm_obj = 0;
2655
30a4f2a8 2656 /* We can't delete objects while rebuilding swap */
2657 if (store_rebuilding == STORE_REBUILDING_FAST)
2658 return -1;
2659
090089c4 2660 /* Scan row of hash table each second and free storage if we're
2661 * over the high-water mark */
2662 storeGetSwapSpace(0);
2663
2664 /* Purges expired objects, check one bucket on each calling */
30a4f2a8 2665 if (squid_curtime - last_time >= STORE_MAINTAIN_RATE) {
2666 last_time = squid_curtime;
090089c4 2667 if (bucket >= STORE_BUCKETS)
2668 bucket = 0;
ec74cf5f 2669 link_ptr = hash_get_bucket(store_table, bucket++);
090089c4 2670 while (link_ptr) {
2671 next = link_ptr->next;
2672 e = (StoreEntry *) link_ptr;
b8de7ebe 2673 if ((squid_curtime > e->expires) &&
090089c4 2674 (e->swap_status == SWAP_OK)) {
c943f331 2675 debug(20, 2, "storeMaintainSwapSpace: Expired: <TTL:%d> <URL:%s>\n",
b8de7ebe 2676 e->expires - squid_curtime, e->url);
090089c4 2677 /* just call release. don't have to check for lock status.
2678 * storeRelease will take care of that and set a pending flag
2679 * if it's still locked. */
2680 storeRelease(e);
2681 ++rm_obj;
2682 }
2683 link_ptr = next;
2684 }
2685 }
30a4f2a8 2686 debug(20, rm_obj ? 2 : 3, "Removed %d expired objects\n", rm_obj);
090089c4 2687 return rm_obj;
2688}
2689
090089c4 2690
2691/*
2692 * storeWriteCleanLog
2693 *
2694 * Writes a "clean" swap log file from in-memory metadata.
2695 */
2696int storeWriteCleanLog()
2697{
2698 StoreEntry *e = NULL;
95d659f0 2699 LOCAL_ARRAY(char, swapfilename, MAX_FILE_NAME_LEN);
090089c4 2700 FILE *fp = NULL;
2701 int n = 0;
30a4f2a8 2702 int x = 0;
090089c4 2703 time_t start, stop, r;
2704
30a4f2a8 2705 if (store_rebuilding) {
c943f331 2706 debug(20, 1, "storeWriteCleanLog: Not currently OK to rewrite swap log.\n");
2707 debug(20, 1, "storeWriteCleanLog: Operation aborted.\n");
090089c4 2708 return 0;
2709 }
c943f331 2710 debug(20, 1, "storeWriteCleanLog: Starting...\n");
b37a37f9 2711 start = getCurrentTime();
3facc31e 2712 sprintf(tmp_filename, "%s/log_clean", swappath(0));
2713 if ((fp = fopen(tmp_filename, "a+")) == NULL) {
a8f7d3ee 2714 debug(20, 0, "storeWriteCleanLog: %s: %s\n", tmp_filename, xstrerror());
090089c4 2715 return 0;
2716 }
2717 for (e = storeGetFirst(); e; e = storeGetNext()) {
c943f331 2718 debug(20, 5, "storeWriteCleanLog: <URL:%s>\n", e->url);
090089c4 2719 if (e->swap_file_number < 0)
2720 continue;
2721 if (e->swap_status != SWAP_OK)
2722 continue;
2723 if (e->object_len <= 0)
2724 continue;
2725 storeSwapFullPath(e->swap_file_number, swapfilename);
f5469bce 2726 x = fprintf(fp, "%08x %08x %08x %08x %9d %s\n",
2727 (int) e->swap_file_number,
2728 (int) e->timestamp,
2729 (int) e->expires,
2730 (int) e->lastmod,
2731 e->object_len,
2732 e->url);
30a4f2a8 2733 if (x < 0) {
a8f7d3ee 2734 debug(20, 0, "storeWriteCleanLog: %s: %s\n", tmp_filename, xstrerror());
30a4f2a8 2735 debug(20, 0, "storeWriteCleanLog: Current swap logfile not replaced.\n");
2736 fclose(fp);
2737 safeunlink(tmp_filename, 0);
2738 return 0;
2739 }
090089c4 2740 if ((++n & 0xFFF) == 0) {
b37a37f9 2741 getCurrentTime();
c943f331 2742 debug(20, 1, " %7d lines written so far.\n", n);
090089c4 2743 }
2744 }
30a4f2a8 2745 if (fclose(fp) < 0) {
a8f7d3ee 2746 debug(20, 0, "storeWriteCleanLog: %s: %s\n", tmp_filename, xstrerror());
30a4f2a8 2747 debug(20, 0, "storeWriteCleanLog: Current swap logfile not replaced.\n");
2748 safeunlink(tmp_filename, 0);
2749 return 0;
2750 }
090089c4 2751 if (file_write_unlock(swaplog_fd, swaplog_lock) != DISK_OK) {
c943f331 2752 debug(20, 0, "storeWriteCleanLog: Failed to unlock swaplog!\n");
2753 debug(20, 0, "storeWriteCleanLog: Current swap logfile not replaced.\n");
090089c4 2754 return 0;
2755 }
3facc31e 2756 if (rename(tmp_filename, swaplog_file) < 0) {
c943f331 2757 debug(20, 0, "storeWriteCleanLog: rename failed: %s\n",
090089c4 2758 xstrerror());
2759 return 0;
2760 }
2761 file_close(swaplog_fd);
e47afcdb 2762 swaplog_fd = file_open(swaplog_file, NULL, O_WRONLY | O_CREAT);
090089c4 2763 if (swaplog_fd < 0) {
c943f331 2764 sprintf(tmp_error_buf, "Cannot open swap logfile: %s", swaplog_file);
2765 fatal(tmp_error_buf);
090089c4 2766 }
090089c4 2767 swaplog_lock = file_write_lock(swaplog_fd);
2768
b37a37f9 2769 stop = getCurrentTime();
090089c4 2770 r = stop - start;
c943f331 2771 debug(20, 1, " Finished. Wrote %d lines.\n", n);
2772 debug(20, 1, " Took %d seconds (%6.1lf lines/sec).\n",
090089c4 2773 r > 0 ? r : 0, (double) n / (r > 0 ? r : 1));
2774
2775 /* touch a timestamp file */
3facc31e 2776 sprintf(tmp_filename, "%s/log-last-clean", swappath(0));
2777 file_close(file_open(tmp_filename, NULL, O_WRONLY | O_CREAT | O_TRUNC));
090089c4 2778 return n;
2779}
2780
2781int swapInError(fd_unused, entry)
2782 int fd_unused;
2783 StoreEntry *entry;
2784{
b8de7ebe 2785 squid_error_entry(entry, ERR_DISK_IO, xstrerror());
090089c4 2786 return 0;
2787}
2788
2789int storePendingNClients(e)
2790 StoreEntry *e;
2791{
2792 int npend = 0;
2793 int i;
2794
2795 if (!e->mem_obj)
2796 return 0;
22e4fa85 2797 for (npend = i = 0; i < (int) e->mem_obj->pending_list_size; i++) {
2798 if (e->mem_obj->pending[i])
090089c4 2799 npend++;
2800 }
2801 return npend;
2802}
d8b45066 2803
2804void storeRotateLog()
2805{
2806 char *fname = NULL;
2807 int i;
95d659f0 2808 LOCAL_ARRAY(char, from, MAXPATHLEN);
2809 LOCAL_ARRAY(char, to, MAXPATHLEN);
d8b45066 2810
234967c9 2811 if (storelog_fd > -1) {
2812 file_close(storelog_fd);
2813 storelog_fd = -1;
2814 }
b6f794d6 2815 if ((fname = Config.Log.store) == NULL)
2d25dc1a 2816 return;
d8b45066 2817
234967c9 2818 if (strcmp(fname, "none") == 0)
2819 return;
2820
d8b45066 2821 debug(20, 1, "storeRotateLog: Rotating.\n");
2822
2823 /* Rotate numbers 0 through N up one */
b6f794d6 2824 for (i = Config.Log.rotateNumber; i > 1;) {
2d25dc1a 2825 i--;
2826 sprintf(from, "%s.%d", fname, i - 1);
2827 sprintf(to, "%s.%d", fname, i);
2828 rename(from, to);
d8b45066 2829 }
2830 /* Rotate the current log to .0 */
b6f794d6 2831 if (Config.Log.rotateNumber > 0) {
2d25dc1a 2832 sprintf(to, "%s.%d", fname, 0);
2833 rename(fname, to);
d8b45066 2834 }
e47afcdb 2835 storelog_fd = file_open(fname, NULL, O_WRONLY | O_CREAT);
234967c9 2836 if (storelog_fd < 0) {
2837 debug(20, 0, "storeRotateLog: %s: %s\n", fname, xstrerror());
2838 debug(20, 1, "Store logging disabled\n");
2839 }
d8b45066 2840}
30a4f2a8 2841
2842/*
2843 * Check if its okay to remove the memory data for this object, but
2844 * leave the StoreEntry around. Designed to be called from
2845 * storeUnlockObject() and storeSwapOutHandle().
2846 */
2847static int storeCheckPurgeMem(e)
2848 StoreEntry *e;
2849{
2850 if (storeEntryLocked(e))
2851 return 0;
2852 if (e->store_status != STORE_OK)
2853 return 0;
3e98df20 2854 if (e->swap_status != SWAP_OK)
2855 return 0;
30a4f2a8 2856 if (store_hotobj_high)
2857 return 0;
2858 return 1;
2859}
a1e47288 2860
620da955 2861static int storeCheckExpired(e)
2862 StoreEntry *e;
2863{
2864 if (storeEntryLocked(e))
2865 return 0;
2866 if (squid_curtime < e->expires)
2867 return 0;
2868 return 1;
2869}
2870
a1e47288 2871static char *storeDescribeStatus(e)
2872 StoreEntry *e;
2873{
2874 static char buf[MAX_URL << 1];
2875 sprintf(buf, "mem:%13s ping:%12s store:%13s swap:%12s locks:%d %s\n",
2876 memStatusStr[e->mem_status],
2877 pingStatusStr[e->ping_status],
2878 storeStatusStr[e->store_status],
2879 swapStatusStr[e->swap_status],
2880 (int) e->lock_count,
2881 e->url);
2882 return buf;
2883}
457bedbd 2884
2885void storeCloseLog()
2886{
2887 file_close(swaplog_fd);
2888 file_close(storelog_fd);
2889}
Mirror of https://github.com/squid-cache/squid.git